From a8bdc7b07ee59c40812cbbf72f00e4198d6c1a92 Mon Sep 17 00:00:00 2001 From: Kirk True Date: Tue, 19 Sep 2023 10:24:14 -0700 Subject: [PATCH 01/72] KAFKA-14274 [6, 7]: Introduction of fetch request manager Changes: 1. Introduces FetchRequestManager that implements the RequestManager API for fetching messages from brokers. Unlike Fetcher, record decompression and deserialization is performed on the application thread inside CompletedFetch. 2. Restructured the code so that objects owned by the background thread are not instantiated until the background thread runs (via Supplier) to ensure that there are no references available to the application thread. 3. Ensuring resources are properly using Closeable and using IdempotentCloser to ensure they're only closed once. 4. Introduces ConsumerTestBuilder to reduce a lot of inconsistency in the way the objects were built up for tests. --- checkstyle/import-control.xml | 4 + checkstyle/suppressions.xml | 12 +- .../kafka/clients/consumer/KafkaConsumer.java | 6 +- .../consumer/internals/AbstractFetch.java | 83 +- .../consumer/internals/CachedSupplier.java | 43 + .../consumer/internals/CompletedFetch.java | 25 +- .../consumer/internals/ConsumerUtils.java | 5 +- .../internals/DefaultBackgroundThread.java | 247 +- .../internals/DefaultEventHandler.java | 134 +- .../consumer/internals/Deserializers.java | 8 + .../consumer/internals/ErrorEventHandler.java | 7 +- .../consumer/internals/FetchCollector.java | 14 +- .../consumer/internals/FetchConfig.java | 19 +- .../internals/FetchRequestManager.java | 136 + .../clients/consumer/internals/Fetcher.java | 85 +- .../internals/NetworkClientDelegate.java | 93 +- .../internals/PrototypeAsyncConsumer.java | 701 +++- .../consumer/internals/RequestManager.java | 9 +- .../consumer/internals/RequestManagers.java | 116 +- .../internals/events/ApplicationEvent.java | 2 +- .../events/ApplicationEventProcessor.java | 47 +- .../events/BackgroundEventProcessor.java | 103 + .../events/ErrorBackgroundEvent.java | 6 +- .../internals/events/EventHandler.java | 19 +- .../consumer/internals/events/FetchEvent.java | 31 + .../clients/consumer/KafkaConsumerTest.java | 4 +- .../internals/CompletedFetchTest.java | 96 +- .../internals/ConsumerTestBuilder.java | 261 ++ .../DefaultBackgroundThreadTest.java | 289 +- .../internals/DefaultEventHandlerTest.java | 55 +- .../internals/ErrorEventHandlerTest.java | 165 + .../internals/FetchCollectorTest.java | 39 +- .../consumer/internals/FetchConfigTest.java | 48 +- .../internals/FetchRequestManagerTest.java | 3566 +++++++++++++++++ .../consumer/internals/FetcherTest.java | 38 +- .../consumer/internals/OffsetFetcherTest.java | 4 +- .../internals/PrototypeAsyncConsumerTest.java | 154 +- 37 files changed, 5616 insertions(+), 1058 deletions(-) create mode 100644 clients/src/main/java/org/apache/kafka/clients/consumer/internals/CachedSupplier.java create mode 100644 clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchRequestManager.java create mode 100644 clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEventProcessor.java create mode 100644 clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/FetchEvent.java create mode 100644 clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java create mode 100644 clients/src/test/java/org/apache/kafka/clients/consumer/internals/ErrorEventHandlerTest.java create mode 100644 clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java diff --git a/checkstyle/import-control.xml b/checkstyle/import-control.xml index a12ee2ea93e8f..cde1dceade826 100644 --- a/checkstyle/import-control.xml +++ b/checkstyle/import-control.xml @@ -205,6 +205,10 @@ + + + + diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml index f01281acb0ee9..aa19bd181f482 100644 --- a/checkstyle/suppressions.xml +++ b/checkstyle/suppressions.xml @@ -61,6 +61,8 @@ files="AbstractRequest.java"/> + @@ -68,7 +70,7 @@ + files="(KafkaConsumer|PrototypeAsyncConsumer|ConsumerCoordinator).java"/> + files="(KafkaConsumer|PrototypeAsyncConsumer|ConsumerCoordinator|AbstractFetch|KafkaProducer|AbstractRequest|AbstractResponse|TransactionManager|Admin|KafkaAdminClient|MockAdminClient|KafkaRaftClient|KafkaRaftClientTest).java"/> @@ -108,10 +110,10 @@ + files="(Sender|Fetcher|FetchRequestManager|OffsetFetcher|KafkaConsumer|PrototypeAsyncConsumer|Metrics|RequestResponse|TransactionManager|KafkaAdminClient|Message|KafkaProducer)Test.java"/> + files="(ConsumerCoordinator|KafkaConsumer|RequestResponse|Fetcher|FetchRequestManager|KafkaAdminClient|Message|KafkaProducer)Test.java"/> @@ -120,7 +122,7 @@ files="(OffsetFetcher|RequestResponse)Test.java"/> + files="RequestResponseTest.java|FetcherTest.java|FetchRequestManagerTest.java|KafkaAdminClientTest.java"/> diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java index aa4b913b49373..f9dfa29b0da12 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java @@ -752,7 +752,7 @@ public KafkaConsumer(Map configs, config.getBoolean(ConsumerConfig.THROW_ON_FETCH_STABLE_OFFSET_UNSUPPORTED), config.getString(ConsumerConfig.CLIENT_RACK_CONFIG)); } - FetchConfig fetchConfig = createFetchConfig(config, this.deserializers); + FetchConfig fetchConfig = createFetchConfig(config); this.fetcher = new Fetcher<>( logContext, this.client, @@ -1252,7 +1252,7 @@ private Fetch pollForFetches(Timer timer) { Math.min(coordinator.timeToNextPoll(timer.currentTimeMs()), timer.remainingMs()); // if data is available already, return it immediately - final Fetch fetch = fetcher.collectFetch(); + final Fetch fetch = fetcher.collectFetch(deserializers); if (!fetch.isEmpty()) { return fetch; } @@ -1279,7 +1279,7 @@ private Fetch pollForFetches(Timer timer) { }); timer.update(pollTimer.currentTimeMs()); - return fetcher.collectFetch(); + return fetcher.collectFetch(deserializers); } /** diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractFetch.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractFetch.java index 4f579bb0e0fed..108be1999f6a8 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractFetch.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractFetch.java @@ -18,10 +18,13 @@ import org.apache.kafka.clients.ClientResponse; import org.apache.kafka.clients.FetchSessionHandler; +import org.apache.kafka.clients.KafkaClient; +import org.apache.kafka.clients.NetworkClientUtils; import org.apache.kafka.common.Cluster; import org.apache.kafka.common.Node; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.errors.AuthenticationException; import org.apache.kafka.common.internals.IdempotentCloser; import org.apache.kafka.common.message.FetchResponseData; import org.apache.kafka.common.protocol.ApiKeys; @@ -37,11 +40,9 @@ import org.slf4j.helpers.MessageFormatter; import java.io.Closeable; -import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; -import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -51,17 +52,14 @@ /** * {@code AbstractFetch} represents the basic state and logic for record fetching processing. - * @param Type for the message key - * @param Type for the message value */ -public abstract class AbstractFetch implements Closeable { +public abstract class AbstractFetch implements Closeable { private final Logger log; protected final LogContext logContext; - protected final ConsumerNetworkClient client; protected final ConsumerMetadata metadata; protected final SubscriptionState subscriptions; - protected final FetchConfig fetchConfig; + protected final FetchConfig fetchConfig; protected final Time time; protected final FetchMetricsManager metricsManager; protected final FetchBuffer fetchBuffer; @@ -72,15 +70,13 @@ public abstract class AbstractFetch implements Closeable { private final Map sessionHandlers; public AbstractFetch(final LogContext logContext, - final ConsumerNetworkClient client, final ConsumerMetadata metadata, final SubscriptionState subscriptions, - final FetchConfig fetchConfig, + final FetchConfig fetchConfig, final FetchMetricsManager metricsManager, final Time time) { this.log = logContext.logger(AbstractFetch.class); this.logContext = logContext; - this.client = client; this.metadata = metadata; this.subscriptions = subscriptions; this.fetchConfig = fetchConfig; @@ -92,6 +88,23 @@ public AbstractFetch(final LogContext logContext, this.time = time; } + /** + * Check if the node is disconnected and unavailable for immediate reconnection (i.e. if it is in + * reconnect backoff window following the disconnect). + * + * @param node {@link Node} to check for availability + * @see NetworkClientUtils#isUnavailable(KafkaClient, Node, Time) + */ + protected abstract boolean isUnavailable(Node node); + + /** + * Checks for an authentication error on a given node and throws the exception if it exists. + * + * @param node {@link Node} to check for a previous {@link AuthenticationException}; if found it is thrown + * @see NetworkClientUtils#maybeThrowAuthFailure(KafkaClient, Node) + */ + protected abstract void maybeThrowAuthFailure(Node node); + /** * Return whether we have any completed fetches pending return to the user. This method is thread-safe. Has * visibility for testing. @@ -317,7 +330,7 @@ Node selectReadReplica(final TopicPartition partition, final Node leaderReplica, } } - private Map prepareCloseFetchSessionRequests() { + protected Map prepareCloseFetchSessionRequests() { final Cluster cluster = metadata.fetch(); Map fetchable = new LinkedHashMap<>(); @@ -330,7 +343,7 @@ private Map prepareCloseFetchSession // skip sending the close request. final Node fetchTarget = cluster.nodeById(fetchTargetNodeId); - if (fetchTarget == null || client.isUnavailable(fetchTarget)) { + if (fetchTarget == null || isUnavailable(fetchTarget)) { log.debug("Skip sending close session request to broker {} since it is not reachable", fetchTarget); return; } @@ -377,8 +390,8 @@ protected Map prepareFetchRequests() // Use the preferred read replica if set, otherwise the partition's leader Node node = selectReadReplica(partition, leaderOpt.get(), currentTimeMs); - if (client.isUnavailable(node)) { - client.maybeThrowAuthFailure(node); + if (isUnavailable(node)) { + maybeThrowAuthFailure(node); // If we try to send during the reconnect backoff window, then the request is just // going to be failed anyway before being sent, so skip sending the request for now @@ -412,46 +425,6 @@ protected Map prepareFetchRequests() return reqs; } - protected void maybeCloseFetchSessions(final Timer timer) { - final List> requestFutures = new ArrayList<>(); - Map fetchRequestMap = prepareCloseFetchSessionRequests(); - - for (Map.Entry entry : fetchRequestMap.entrySet()) { - final Node fetchTarget = entry.getKey(); - final FetchSessionHandler.FetchRequestData data = entry.getValue(); - final FetchRequest.Builder request = createFetchRequest(fetchTarget, data); - final RequestFuture responseFuture = client.send(fetchTarget, request); - - responseFuture.addListener(new RequestFutureListener() { - @Override - public void onSuccess(ClientResponse value) { - handleCloseFetchSessionResponse(fetchTarget, data); - } - - @Override - public void onFailure(RuntimeException e) { - handleCloseFetchSessionResponse(fetchTarget, data, e); - } - }); - - requestFutures.add(responseFuture); - } - - // Poll to ensure that request has been written to the socket. Wait until either the timer has expired or until - // all requests have received a response. - while (timer.notExpired() && !requestFutures.stream().allMatch(RequestFuture::isDone)) { - client.poll(timer, null, true); - } - - if (!requestFutures.stream().allMatch(RequestFuture::isDone)) { - // we ran out of time before completing all futures. It is ok since we don't want to block the shutdown - // here. - log.debug("All requests couldn't be sent in the specific timeout period {}ms. " + - "This may result in unnecessary fetch sessions at the broker. Consider increasing the timeout passed for " + - "KafkaConsumer.close(Duration timeout)", timer.timeoutMs()); - } - } - // Visible for testing protected FetchSessionHandler sessionHandler(int node) { return sessionHandlers.get(node); @@ -467,8 +440,6 @@ protected FetchSessionHandler sessionHandler(int node) { // Visible for testing protected void closeInternal(Timer timer) { // we do not need to re-enable wake-ups since we are closing already - client.disableWakeups(); - maybeCloseFetchSessions(timer); Utils.closeQuietly(fetchBuffer, "fetchBuffer"); Utils.closeQuietly(decompressionBufferSupplier, "decompressionBufferSupplier"); } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CachedSupplier.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CachedSupplier.java new file mode 100644 index 0000000000000..dc891a2539b6d --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CachedSupplier.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.clients.consumer.internals; + +import java.util.function.Supplier; + +/** + * Simple {@link Supplier} that caches the initial creation of the object and stores it for later calls + * to {@link #get()}. + * + *

+ * + * Note: this class is not thread safe! Use only in contexts which are designed/guaranteed to be + * single-threaded. + */ +public abstract class CachedSupplier implements Supplier { + + private T result; + + protected abstract T create(); + + @Override + public T get() { + if (result == null) + result = create(); + + return result; + } +} diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CompletedFetch.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CompletedFetch.java index 7a8ee10515741..cd8e296db17c3 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CompletedFetch.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CompletedFetch.java @@ -53,7 +53,7 @@ /** * {@link CompletedFetch} represents a {@link RecordBatch batch} of {@link Record records} that was returned from the * broker via a {@link FetchRequest}. It contains logic to maintain state between calls to - * {@link #fetchRecords(FetchConfig, int)}. + * {@link #fetchRecords(FetchConfig, Deserializers, int)}. */ public class CompletedFetch { @@ -135,7 +135,8 @@ void recordAggregatedMetrics(int bytes, int records) { /** * Draining a {@link CompletedFetch} will signal that the data has been consumed and the underlying resources * are closed. This is somewhat analogous to {@link Closeable#close() closing}, though no error will result if a - * caller invokes {@link #fetchRecords(FetchConfig, int)}; an empty {@link List list} will be returned instead. + * caller invokes {@link #fetchRecords(FetchConfig, Deserializers, int)}; an empty {@link List list} will be + * returned instead. */ void drain() { if (!isConsumed) { @@ -151,7 +152,7 @@ void drain() { } } - private void maybeEnsureValid(FetchConfig fetchConfig, RecordBatch batch) { + private void maybeEnsureValid(FetchConfig fetchConfig, RecordBatch batch) { if (fetchConfig.checkCrcs && batch.magic() >= RecordBatch.MAGIC_VALUE_V2) { try { batch.ensureValid(); @@ -162,7 +163,7 @@ private void maybeEnsureValid(FetchConfig fetchConfig, RecordBatch } } - private void maybeEnsureValid(FetchConfig fetchConfig, Record record) { + private void maybeEnsureValid(FetchConfig fetchConfig, Record record) { if (fetchConfig.checkCrcs) { try { record.ensureValid(); @@ -180,7 +181,7 @@ private void maybeCloseRecordStream() { } } - private Record nextFetchedRecord(FetchConfig fetchConfig) { + private Record nextFetchedRecord(FetchConfig fetchConfig) { while (true) { if (records == null || !records.hasNext()) { maybeCloseRecordStream(); @@ -249,7 +250,9 @@ private Record nextFetchedRecord(FetchConfig fetchConfig) { * @param maxRecords The number of records to return; the number returned may be {@code 0 <= maxRecords} * @return {@link ConsumerRecord Consumer records} */ - List> fetchRecords(FetchConfig fetchConfig, int maxRecords) { + List> fetchRecords(FetchConfig fetchConfig, + Deserializers deserializers, + int maxRecords) { // Error when fetching the next record before deserialization. if (corruptLastRecord) throw new KafkaException("Received exception when fetching the next record from " + partition @@ -276,7 +279,7 @@ List> fetchRecords(FetchConfig fetchConfig, in Optional leaderEpoch = maybeLeaderEpoch(currentBatch.partitionLeaderEpoch()); TimestampType timestampType = currentBatch.timestampType(); - ConsumerRecord record = parseRecord(fetchConfig, partition, leaderEpoch, timestampType, lastRecord); + ConsumerRecord record = parseRecord(deserializers, partition, leaderEpoch, timestampType, lastRecord); records.add(record); recordsRead++; bytesRead += lastRecord.sizeInBytes(); @@ -302,7 +305,7 @@ List> fetchRecords(FetchConfig fetchConfig, in /** * Parse the record entry, deserializing the key / value fields if necessary */ - ConsumerRecord parseRecord(FetchConfig fetchConfig, + ConsumerRecord parseRecord(Deserializers deserializers, TopicPartition partition, Optional leaderEpoch, TimestampType timestampType, @@ -312,16 +315,16 @@ ConsumerRecord parseRecord(FetchConfig fetchConfig, long timestamp = record.timestamp(); Headers headers = new RecordHeaders(record.headers()); ByteBuffer keyBytes = record.key(); - K key = keyBytes == null ? null : fetchConfig.deserializers.keyDeserializer.deserialize(partition.topic(), headers, keyBytes); + K key = keyBytes == null ? null : deserializers.keyDeserializer.deserialize(partition.topic(), headers, keyBytes); ByteBuffer valueBytes = record.value(); - V value = valueBytes == null ? null : fetchConfig.deserializers.valueDeserializer.deserialize(partition.topic(), headers, valueBytes); + V value = valueBytes == null ? null : deserializers.valueDeserializer.deserialize(partition.topic(), headers, valueBytes); return new ConsumerRecord<>(partition.topic(), partition.partition(), offset, timestamp, timestampType, keyBytes == null ? ConsumerRecord.NULL_SIZE : keyBytes.remaining(), valueBytes == null ? ConsumerRecord.NULL_SIZE : valueBytes.remaining(), key, value, headers, leaderEpoch); } catch (RuntimeException e) { - log.error("Deserializers with error: {}", fetchConfig.deserializers); + log.error("Deserializers with error: {}", deserializers); throw new RecordDeserializationException(partition, record.offset(), "Error deserializing key/value for partition " + partition + " at offset " + record.offset() + ". If needed, please seek past the record to continue consumption.", e); diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerUtils.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerUtils.java index 68d2458e28d5e..06dcf207b58ac 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerUtils.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerUtils.java @@ -142,10 +142,9 @@ public static FetchMetricsManager createFetchMetricsManager(Metrics metrics) { return new FetchMetricsManager(metrics, metricsRegistry); } - public static FetchConfig createFetchConfig(ConsumerConfig config, - Deserializers deserializers) { + public static FetchConfig createFetchConfig(ConsumerConfig config) { IsolationLevel isolationLevel = configuredIsolationLevel(config); - return new FetchConfig<>(config, deserializers, isolationLevel); + return new FetchConfig(config, isolationLevel); } @SuppressWarnings("unchecked") diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThread.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThread.java index 9b4cd89361fdf..7d7c200e7f60d 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThread.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThread.java @@ -16,197 +16,78 @@ */ package org.apache.kafka.clients.consumer.internals; -import org.apache.kafka.clients.ApiVersions; -import org.apache.kafka.clients.ClientUtils; -import org.apache.kafka.clients.GroupRebalanceConfig; -import org.apache.kafka.clients.NetworkClient; -import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.internals.events.ApplicationEvent; import org.apache.kafka.clients.consumer.internals.events.ApplicationEventProcessor; -import org.apache.kafka.clients.consumer.internals.events.BackgroundEvent; +import org.apache.kafka.clients.consumer.internals.events.CompletableApplicationEvent; import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.errors.WakeupException; -import org.apache.kafka.common.metrics.Metrics; -import org.apache.kafka.common.metrics.Sensor; +import org.apache.kafka.common.internals.IdempotentCloser; import org.apache.kafka.common.utils.KafkaThread; import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.Time; import org.apache.kafka.common.utils.Utils; import org.slf4j.Logger; +import java.io.Closeable; +import java.util.ArrayList; import java.util.LinkedList; +import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.concurrent.BlockingQueue; - -import static java.util.Objects.requireNonNull; -import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_MAX_INFLIGHT_REQUESTS_PER_CONNECTION; -import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRIC_GROUP_PREFIX; -import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.configuredIsolationLevel; +import java.util.function.Supplier; /** * Background thread runnable that consumes {@code ApplicationEvent} and * produces {@code BackgroundEvent}. It uses an event loop to consume and * produce events, and poll the network client to handle network IO. - *

+ *

* It holds a reference to the {@link SubscriptionState}, which is * initialized by the polling thread. - *

- * For processing application events that have been submitted to the - * {@link #applicationEventQueue}, this relies on an {@link ApplicationEventProcessor}. Processing includes generating requests and - * handling responses with the appropriate {@link RequestManager}. The network operations for - * actually sending the requests is delegated to the {@link NetworkClientDelegate} - * */ -public class DefaultBackgroundThread extends KafkaThread { +public class DefaultBackgroundThread extends KafkaThread implements Closeable { + private static final long MAX_POLL_TIMEOUT_MS = 5000; private static final String BACKGROUND_THREAD_NAME = "consumer_background_thread"; private final Time time; private final Logger log; private final BlockingQueue applicationEventQueue; - private final BlockingQueue backgroundEventQueue; - private final ConsumerMetadata metadata; - private final ConsumerConfig config; + private final Supplier applicationEventProcessorSupplier; + private final Supplier networkClientDelegateSupplier; + private final Supplier requestManagersSupplier; // empty if groupId is null - private final ApplicationEventProcessor applicationEventProcessor; - private final NetworkClientDelegate networkClientDelegate; - private final ErrorEventHandler errorEventHandler; - private final GroupState groupState; - private boolean running; - - private final RequestManagers requestManagers; - - // Visible for testing - DefaultBackgroundThread(final Time time, - final ConsumerConfig config, - final LogContext logContext, - final BlockingQueue applicationEventQueue, - final BlockingQueue backgroundEventQueue, - final ErrorEventHandler errorEventHandler, - final ApplicationEventProcessor processor, - final ConsumerMetadata metadata, - final NetworkClientDelegate networkClient, - final GroupState groupState, - final CoordinatorRequestManager coordinatorManager, - final CommitRequestManager commitRequestManager, - final OffsetsRequestManager offsetsRequestManager) { + private ApplicationEventProcessor applicationEventProcessor; + private NetworkClientDelegate networkClientDelegate; + private RequestManagers requestManagers; + private volatile boolean running; + private final IdempotentCloser closer = new IdempotentCloser(); + + public DefaultBackgroundThread(Time time, + LogContext logContext, + BlockingQueue applicationEventQueue, + Supplier applicationEventProcessorSupplier, + Supplier networkClientDelegateSupplier, + Supplier requestManagersSupplier) { super(BACKGROUND_THREAD_NAME, true); this.time = time; - this.running = true; this.log = logContext.logger(getClass()); this.applicationEventQueue = applicationEventQueue; - this.backgroundEventQueue = backgroundEventQueue; - this.applicationEventProcessor = processor; - this.config = config; - this.metadata = metadata; - this.networkClientDelegate = networkClient; - this.errorEventHandler = errorEventHandler; - this.groupState = groupState; - - this.requestManagers = new RequestManagers( - offsetsRequestManager, - Optional.ofNullable(coordinatorManager), - Optional.ofNullable(commitRequestManager)); - } - - public DefaultBackgroundThread(final Time time, - final ConsumerConfig config, - final GroupRebalanceConfig rebalanceConfig, - final LogContext logContext, - final BlockingQueue applicationEventQueue, - final BlockingQueue backgroundEventQueue, - final ConsumerMetadata metadata, - final SubscriptionState subscriptionState, - final ApiVersions apiVersions, - final Metrics metrics, - final Sensor fetcherThrottleTimeSensor) { - super(BACKGROUND_THREAD_NAME, true); - requireNonNull(config); - requireNonNull(rebalanceConfig); - requireNonNull(logContext); - requireNonNull(applicationEventQueue); - requireNonNull(backgroundEventQueue); - requireNonNull(metadata); - requireNonNull(subscriptionState); - try { - this.time = time; - this.log = logContext.logger(getClass()); - this.applicationEventQueue = applicationEventQueue; - this.backgroundEventQueue = backgroundEventQueue; - this.config = config; - this.metadata = metadata; - final NetworkClient networkClient = ClientUtils.createNetworkClient(config, - metrics, - CONSUMER_METRIC_GROUP_PREFIX, - logContext, - apiVersions, - time, - CONSUMER_MAX_INFLIGHT_REQUESTS_PER_CONNECTION, - metadata, - fetcherThrottleTimeSensor); - this.networkClientDelegate = new NetworkClientDelegate( - this.time, - this.config, - logContext, - networkClient); - this.running = true; - this.errorEventHandler = new ErrorEventHandler(this.backgroundEventQueue); - this.groupState = new GroupState(rebalanceConfig); - long retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG); - long retryBackoffMaxMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MAX_MS_CONFIG); - final int requestTimeoutMs = config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG); - - OffsetsRequestManager offsetsRequestManager = - new OffsetsRequestManager( - subscriptionState, - metadata, - configuredIsolationLevel(config), - time, - retryBackoffMs, - requestTimeoutMs, - apiVersions, - networkClientDelegate, - logContext); - CoordinatorRequestManager coordinatorRequestManager = null; - CommitRequestManager commitRequestManager = null; - - if (groupState.groupId != null) { - coordinatorRequestManager = new CoordinatorRequestManager( - this.time, - logContext, - retryBackoffMs, - retryBackoffMaxMs, - this.errorEventHandler, - groupState.groupId); - commitRequestManager = new CommitRequestManager( - this.time, - logContext, - subscriptionState, - config, - coordinatorRequestManager, - groupState); - } - - this.requestManagers = new RequestManagers( - offsetsRequestManager, - Optional.ofNullable(coordinatorRequestManager), - Optional.ofNullable(commitRequestManager)); - - this.applicationEventProcessor = new ApplicationEventProcessor( - backgroundEventQueue, - requestManagers, - metadata); - - } catch (final Exception e) { - close(); - throw new KafkaException("Failed to construct background processor", e.getCause()); - } + this.applicationEventProcessorSupplier = applicationEventProcessorSupplier; + this.networkClientDelegateSupplier = networkClientDelegateSupplier; + this.requestManagersSupplier = requestManagersSupplier; } @Override public void run() { + closer.assertOpen("Consumer background thread is already closed"); + running = true; + try { log.debug("Background thread started"); + + // Wait until we're securely in the background thread to initialize these objects... + initializeResources(); + while (running) { try { runOnce(); @@ -217,13 +98,19 @@ public void run() { } } catch (final Throwable t) { log.error("The background thread failed due to unexpected error", t); - throw new RuntimeException(t); + throw new KafkaException(t); } finally { close(); - log.debug("{} closed", getClass()); + log.debug("Background thread closed"); } } + void initializeResources() { + applicationEventProcessor = applicationEventProcessorSupplier.get(); + networkClientDelegate = networkClientDelegateSupplier.get(); + requestManagers = requestManagersSupplier.get(); + } + /** * Poll and process an {@link ApplicationEvent}. It performs the following tasks: * 1. Drains and try to process all the requests in the queue. @@ -231,15 +118,13 @@ public void run() { * 3. Poll the networkClient to send and retrieve the response. */ void runOnce() { - if (!applicationEventQueue.isEmpty()) { - LinkedList res = new LinkedList<>(); - this.applicationEventQueue.drainTo(res); + LinkedList events = new LinkedList<>(); + applicationEventQueue.drainTo(events); - for (ApplicationEvent event : res) { - log.debug("Consuming application event: {}", event); - Objects.requireNonNull(event); - applicationEventProcessor.process(event); - } + for (ApplicationEvent event : events) { + log.trace("Dequeued event: {}", event); + Objects.requireNonNull(event); + applicationEventProcessor.process(event); } final long currentTimeMs = time.milliseconds(); @@ -259,17 +144,41 @@ long handlePollResult(NetworkClientDelegate.PollResult res) { } public boolean isRunning() { - return this.running; + return running; } public void wakeup() { - networkClientDelegate.wakeup(); + if (networkClientDelegate != null) + networkClientDelegate.wakeup(); } + @Override public void close() { - this.running = false; - this.wakeup(); - Utils.closeQuietly(networkClientDelegate, "network client utils"); - Utils.closeQuietly(metadata, "consumer metadata client"); + closer.close(() -> { + log.debug("Closing the consumer background thread"); + running = false; + wakeup(); + Utils.closeQuietly(requestManagers, "Request managers client"); + Utils.closeQuietly(networkClientDelegate, "network client utils"); + log.debug("Closed the consumer background thread"); + drainAndComplete(); + }, () -> log.warn("The consumer background thread was previously closed")); + } + + + /** + * It is possible for the background thread to close before complete processing all the events in the queue. In + * this case, we need throw an exception to notify the user the consumer is closed. + */ + private void drainAndComplete() { + List incompletedEvents = new ArrayList<>(); + applicationEventQueue.drainTo(incompletedEvents); + incompletedEvents.forEach(event -> { + if (event instanceof CompletableApplicationEvent) { + ((CompletableApplicationEvent) event).future().completeExceptionally( + new KafkaException("The consumer is closed")); + } + }); + log.debug("Discarding {} events because the consumer is closed", incompletedEvents.size()); } } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/DefaultEventHandler.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/DefaultEventHandler.java index 4d620a4ffac10..342c13c4b51a6 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/DefaultEventHandler.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/DefaultEventHandler.java @@ -16,119 +16,55 @@ */ package org.apache.kafka.clients.consumer.internals; -import org.apache.kafka.clients.ApiVersions; -import org.apache.kafka.clients.ClientUtils; -import org.apache.kafka.clients.GroupRebalanceConfig; -import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.internals.events.ApplicationEvent; +import org.apache.kafka.clients.consumer.internals.events.ApplicationEventProcessor; import org.apache.kafka.clients.consumer.internals.events.BackgroundEvent; import org.apache.kafka.clients.consumer.internals.events.CompletableApplicationEvent; import org.apache.kafka.clients.consumer.internals.events.EventHandler; -import org.apache.kafka.common.internals.ClusterResourceListeners; -import org.apache.kafka.common.metrics.Metrics; -import org.apache.kafka.common.metrics.Sensor; +import org.apache.kafka.common.KafkaException; +import org.apache.kafka.common.internals.IdempotentCloser; import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.Time; import org.apache.kafka.common.utils.Timer; +import org.slf4j.Logger; -import java.net.InetSocketAddress; -import java.util.List; +import java.time.Duration; import java.util.Objects; -import java.util.Optional; import java.util.concurrent.BlockingQueue; -import java.util.concurrent.LinkedBlockingQueue; +import java.util.function.Supplier; /** - * An {@code EventHandler} that uses a single background thread to consume {@code ApplicationEvent} and produce - * {@code BackgroundEvent} from the {@link DefaultBackgroundThread}. + * An {@link EventHandler} that uses a single background thread to consume {@link ApplicationEvent} and produce + * {@link BackgroundEvent} from the {@link DefaultBackgroundThread}. */ public class DefaultEventHandler implements EventHandler { + private final Logger log; private final BlockingQueue applicationEventQueue; - private final BlockingQueue backgroundEventQueue; private final DefaultBackgroundThread backgroundThread; - - public DefaultEventHandler(final ConsumerConfig config, - final GroupRebalanceConfig groupRebalanceConfig, - final LogContext logContext, - final SubscriptionState subscriptionState, - final ApiVersions apiVersions, - final Metrics metrics, - final ClusterResourceListeners clusterResourceListeners, - final Sensor fetcherThrottleTimeSensor) { - this(Time.SYSTEM, - config, - groupRebalanceConfig, - logContext, - new LinkedBlockingQueue<>(), - new LinkedBlockingQueue<>(), - subscriptionState, - apiVersions, - metrics, - clusterResourceListeners, - fetcherThrottleTimeSensor); - } + private final IdempotentCloser closer = new IdempotentCloser(); public DefaultEventHandler(final Time time, - final ConsumerConfig config, - final GroupRebalanceConfig groupRebalanceConfig, final LogContext logContext, final BlockingQueue applicationEventQueue, - final BlockingQueue backgroundEventQueue, - final SubscriptionState subscriptionState, - final ApiVersions apiVersions, - final Metrics metrics, - final ClusterResourceListeners clusterResourceListeners, - final Sensor fetcherThrottleTimeSensor) { + final Supplier applicationEventProcessorSupplier, + final Supplier networkClientDelegateSupplier, + final Supplier requestManagersSupplier) { + this.log = logContext.logger(DefaultEventHandler.class); this.applicationEventQueue = applicationEventQueue; - this.backgroundEventQueue = backgroundEventQueue; - - // Bootstrap a metadata object with the bootstrap server IP address, which will be used once for the - // subsequent metadata refresh once the background thread has started up. - final ConsumerMetadata metadata = new ConsumerMetadata(config, - subscriptionState, + this.backgroundThread = new DefaultBackgroundThread(time, logContext, - clusterResourceListeners); - final List addresses = ClientUtils.parseAndValidateAddresses(config); - metadata.bootstrap(addresses); - - this.backgroundThread = new DefaultBackgroundThread( - time, - config, - groupRebalanceConfig, - logContext, - this.applicationEventQueue, - this.backgroundEventQueue, - metadata, - subscriptionState, - apiVersions, - metrics, - fetcherThrottleTimeSensor); - backgroundThread.start(); - } - - // VisibleForTesting - DefaultEventHandler(final DefaultBackgroundThread backgroundThread, - final BlockingQueue applicationEventQueue, - final BlockingQueue backgroundEventQueue) { - this.backgroundThread = backgroundThread; - this.applicationEventQueue = applicationEventQueue; - this.backgroundEventQueue = backgroundEventQueue; - backgroundThread.start(); - } - - @Override - public Optional poll() { - return Optional.ofNullable(backgroundEventQueue.poll()); - } - - @Override - public boolean isEmpty() { - return backgroundEventQueue.isEmpty(); + applicationEventQueue, + applicationEventProcessorSupplier, + networkClientDelegateSupplier, + requestManagersSupplier); + this.backgroundThread.start(); } @Override public boolean add(final ApplicationEvent event) { + Objects.requireNonNull(event, "ApplicationEvent provided to add must be non-null"); + log.trace("Enqueued event: {}", event); backgroundThread.wakeup(); return applicationEventQueue.add(event); } @@ -141,11 +77,25 @@ public T addAndGet(final CompletableApplicationEvent event, final Timer t return event.get(timer); } - public void close() { - try { - backgroundThread.close(); - } catch (final Exception e) { - throw new RuntimeException(e); - } + public void close(final Duration timeout) { + Objects.requireNonNull(timeout, "Duration provided to close must be non-null"); + + closer.close( + () -> { + log.info("Closing the default consumer event handler"); + + try { + long timeoutMs = timeout.toMillis(); + + if (timeoutMs < 0) + throw new IllegalArgumentException("The timeout cannot be negative."); + + backgroundThread.close(); + log.info("The default consumer event handler was closed"); + } catch (final Exception e) { + throw new KafkaException(e); + } + }, + () -> log.info("The default consumer event handler was already closed")); } } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Deserializers.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Deserializers.java index 80ba5cb7d9e15..5de2a888775af 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Deserializers.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Deserializers.java @@ -75,4 +75,12 @@ public void close() { throw new KafkaException("Failed to close deserializers", exception); } } + + @Override + public String toString() { + return "Deserializers{" + + "keyDeserializer=" + keyDeserializer + + ", valueDeserializer=" + valueDeserializer + + '}'; + } } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ErrorEventHandler.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ErrorEventHandler.java index 31c1873121961..60bfc7476aafd 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ErrorEventHandler.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ErrorEventHandler.java @@ -22,13 +22,18 @@ import java.util.Queue; public class ErrorEventHandler { + private final Queue backgroundEventQueue; public ErrorEventHandler(Queue backgroundEventQueue) { this.backgroundEventQueue = backgroundEventQueue; } - public void handle(Throwable e) { + public void handle(RuntimeException e) { backgroundEventQueue.add(new ErrorBackgroundEvent(e)); } + + public void handle(Throwable e) { + handle(new RuntimeException(e)); + } } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchCollector.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchCollector.java index 0e1f2f18a3f97..3d2d636c53de4 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchCollector.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchCollector.java @@ -54,14 +54,14 @@ public class FetchCollector { private final Logger log; private final ConsumerMetadata metadata; private final SubscriptionState subscriptions; - private final FetchConfig fetchConfig; + private final FetchConfig fetchConfig; private final FetchMetricsManager metricsManager; private final Time time; public FetchCollector(final LogContext logContext, final ConsumerMetadata metadata, final SubscriptionState subscriptions, - final FetchConfig fetchConfig, + final FetchConfig fetchConfig, final FetchMetricsManager metricsManager, final Time time) { this.log = logContext.logger(FetchCollector.class); @@ -87,7 +87,7 @@ public FetchCollector(final LogContext logContext, * the defaultResetPolicy is NONE * @throws TopicAuthorizationException If there is TopicAuthorization error in fetchResponse. */ - public Fetch collectFetch(final FetchBuffer fetchBuffer) { + public Fetch collectFetch(final FetchBuffer fetchBuffer, final Deserializers deserializers) { final Fetch fetch = Fetch.empty(); final Queue pausedCompletedFetches = new ArrayDeque<>(); int recordsRemaining = fetchConfig.maxPollRecords; @@ -128,7 +128,7 @@ public Fetch collectFetch(final FetchBuffer fetchBuffer) { pausedCompletedFetches.add(nextInLineFetch); fetchBuffer.setNextInLineFetch(null); } else { - final Fetch nextFetch = fetchRecords(nextInLineFetch); + final Fetch nextFetch = fetchRecords(nextInLineFetch, deserializers); recordsRemaining -= nextFetch.numRecords(); fetch.add(nextFetch); } @@ -145,7 +145,7 @@ public Fetch collectFetch(final FetchBuffer fetchBuffer) { return fetch; } - private Fetch fetchRecords(final CompletedFetch nextInLineFetch) { + private Fetch fetchRecords(final CompletedFetch nextInLineFetch, Deserializers deserializers) { final TopicPartition tp = nextInLineFetch.partition; if (!subscriptions.isAssigned(tp)) { @@ -162,7 +162,9 @@ private Fetch fetchRecords(final CompletedFetch nextInLineFetch) { throw new IllegalStateException("Missing position for fetchable partition " + tp); if (nextInLineFetch.nextFetchOffset() == position.offset) { - List> partRecords = nextInLineFetch.fetchRecords(fetchConfig, fetchConfig.maxPollRecords); + List> partRecords = nextInLineFetch.fetchRecords(fetchConfig, + deserializers, + fetchConfig.maxPollRecords); log.trace("Returning {} fetched records at offset {} for assigned partition {}", partRecords.size(), position, tp); diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchConfig.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchConfig.java index 66670b54472c8..c0ce15120b912 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchConfig.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchConfig.java @@ -21,8 +21,6 @@ import org.apache.kafka.common.IsolationLevel; import org.apache.kafka.common.serialization.Deserializer; -import java.util.Objects; - /** * {@link FetchConfig} represents the static configuration for fetching records from Kafka. It is simply a way * to bundle the immutable settings that were presented at the time the {@link Consumer} was created for later use by @@ -41,9 +39,6 @@ *

  • {@link #maxPollRecords}: {@link ConsumerConfig#MAX_POLL_RECORDS_CONFIG}
    • *
  • {@link #checkCrcs}: {@link ConsumerConfig#CHECK_CRCS_CONFIG}
    • *
  • {@link #clientRackId}: {@link ConsumerConfig#CLIENT_RACK_CONFIG}
    • - *
  • {@link #deserializers}: - * {@link ConsumerConfig#KEY_DESERIALIZER_CLASS_CONFIG}/{@link ConsumerConfig#VALUE_DESERIALIZER_CLASS_CONFIG} - *
    • *
  • {@link #isolationLevel}: {@link ConsumerConfig#ISOLATION_LEVEL_CONFIG}
    • * * @@ -54,11 +49,8 @@ * * Note: the {@link Deserializer deserializers} used for the key and value are not closed by this class. They should be * closed by the creator of the {@link FetchConfig}. - * - * @param Type used to {@link Deserializer deserialize} the message/record key - * @param Type used to {@link Deserializer deserialize} the message/record value */ -public class FetchConfig { +public class FetchConfig { final int minBytes; final int maxBytes; @@ -67,7 +59,6 @@ public class FetchConfig { final int maxPollRecords; final boolean checkCrcs; final String clientRackId; - final Deserializers deserializers; final IsolationLevel isolationLevel; public FetchConfig(int minBytes, @@ -77,7 +68,6 @@ public FetchConfig(int minBytes, int maxPollRecords, boolean checkCrcs, String clientRackId, - Deserializers deserializers, IsolationLevel isolationLevel) { this.minBytes = minBytes; this.maxBytes = maxBytes; @@ -86,13 +76,10 @@ public FetchConfig(int minBytes, this.maxPollRecords = maxPollRecords; this.checkCrcs = checkCrcs; this.clientRackId = clientRackId; - this.deserializers = Objects.requireNonNull(deserializers, "Message deserializers provided to FetchConfig should not be null"); this.isolationLevel = isolationLevel; } - public FetchConfig(ConsumerConfig config, - Deserializers deserializers, - IsolationLevel isolationLevel) { + public FetchConfig(ConsumerConfig config, IsolationLevel isolationLevel) { this.minBytes = config.getInt(ConsumerConfig.FETCH_MIN_BYTES_CONFIG); this.maxBytes = config.getInt(ConsumerConfig.FETCH_MAX_BYTES_CONFIG); this.maxWaitMs = config.getInt(ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG); @@ -100,7 +87,6 @@ public FetchConfig(ConsumerConfig config, this.maxPollRecords = config.getInt(ConsumerConfig.MAX_POLL_RECORDS_CONFIG); this.checkCrcs = config.getBoolean(ConsumerConfig.CHECK_CRCS_CONFIG); this.clientRackId = config.getString(ConsumerConfig.CLIENT_RACK_CONFIG); - this.deserializers = Objects.requireNonNull(deserializers, "Message deserializers provided to FetchConfig should not be null"); this.isolationLevel = isolationLevel; } @@ -114,7 +100,6 @@ public String toString() { ", maxPollRecords=" + maxPollRecords + ", checkCrcs=" + checkCrcs + ", clientRackId='" + clientRackId + '\'' + - ", deserializers=" + deserializers + ", isolationLevel=" + isolationLevel + '}'; } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchRequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchRequestManager.java new file mode 100644 index 0000000000000..9c010d722f0b4 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchRequestManager.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.clients.consumer.internals; + +import java.util.LinkedList; +import java.util.List; +import java.util.Queue; +import java.util.function.BiConsumer; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +import org.apache.kafka.clients.ClientResponse; +import org.apache.kafka.clients.FetchSessionHandler; +import org.apache.kafka.clients.consumer.internals.NetworkClientDelegate.PollResult; +import org.apache.kafka.clients.consumer.internals.NetworkClientDelegate.UnsentRequest; +import org.apache.kafka.common.Node; +import org.apache.kafka.common.requests.FetchRequest; +import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.common.utils.Time; +import org.slf4j.Logger; + +/** + * {@code FetchRequestManager} is responsible for generating {@link FetchRequest} that represent the + * {@link SubscriptionState#fetchablePartitions(Predicate)} based on the user's topic subscription/partition + * assignment. + */ +public class FetchRequestManager extends AbstractFetch implements RequestManager { + + private final Logger log; + private final ErrorEventHandler errorEventHandler; + private final NetworkClientDelegate networkClientDelegate; + + FetchRequestManager(final LogContext logContext, + final Time time, + final ErrorEventHandler errorEventHandler, + final ConsumerMetadata metadata, + final SubscriptionState subscriptions, + final FetchConfig fetchConfig, + final FetchMetricsManager metricsManager, + final NetworkClientDelegate networkClientDelegate) { + super(logContext, metadata, subscriptions, fetchConfig, metricsManager, time); + this.log = logContext.logger(FetchRequestManager.class); + this.errorEventHandler = errorEventHandler; + this.networkClientDelegate = networkClientDelegate; + } + + @Override + protected boolean isUnavailable(Node node) { + return networkClientDelegate.isUnavailable(node); + } + + @Override + protected void maybeThrowAuthFailure(Node node) { + networkClientDelegate.maybeThrowAuthFailure(node); + } + + @Override + public PollResult poll(long currentTimeMs) { + List requests; + + if (!idempotentCloser.isClosed()) { + // If the fetcher is open (i.e. not closed), we will issue the normal fetch requests + requests = prepareFetchRequests().entrySet().stream().map(entry -> { + final Node fetchTarget = entry.getKey(); + final FetchSessionHandler.FetchRequestData data = entry.getValue(); + final FetchRequest.Builder request = createFetchRequest(fetchTarget, data); + final BiConsumer responseHandler = (clientResponse, t) -> { + if (t != null) { + handleFetchResponse(fetchTarget, t); + log.warn("Attempt to fetch data from node {} failed due to fatal exception", fetchTarget, t); + errorEventHandler.handle(t); + } else { + handleFetchResponse(fetchTarget, data, clientResponse); + } + }; + + return new UnsentRequest(request, fetchTarget, responseHandler); + }).collect(Collectors.toList()); + } else { + requests = prepareCloseFetchSessionRequests().entrySet().stream().map(entry -> { + final Node fetchTarget = entry.getKey(); + final FetchSessionHandler.FetchRequestData data = entry.getValue(); + final FetchRequest.Builder request = createFetchRequest(fetchTarget, data); + final BiConsumer responseHandler = (clientResponse, t) -> { + if (t != null) { + handleCloseFetchSessionResponse(fetchTarget, data, t); + log.warn("Attempt to close fetch session on node {} failed due to fatal exception", fetchTarget, t); + errorEventHandler.handle(t); + } else { + handleCloseFetchSessionResponse(fetchTarget, data); + } + }; + + return new UnsentRequest(request, fetchTarget, responseHandler); + }).collect(Collectors.toList()); + } + + return new PollResult(Long.MAX_VALUE, requests); + } + + /** + * Drains any of the {@link CompletedFetch} objects from the internal buffer to the returned {@link Queue}. + * + *

    + * + * This is used by the {@link org.apache.kafka.clients.consumer.internals.events.ApplicationEventProcessor} to + * pull off any fetch results that are stored in the background thread to provide them to the application thread. + * + * @return {@link Queue} containing zero or more {@link CompletedFetch} + */ + public Queue drain() { + Queue q = new LinkedList<>(); + CompletedFetch completedFetch = fetchBuffer.poll(); + + while (completedFetch != null) { + q.add(completedFetch); + completedFetch = fetchBuffer.poll(); + } + + return q; + } +} diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java index dec02c6b90f6c..a8d7dd0a07052 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java @@ -20,12 +20,17 @@ import org.apache.kafka.clients.FetchSessionHandler; import org.apache.kafka.common.Node; import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.internals.IdempotentCloser; import org.apache.kafka.common.requests.FetchRequest; import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.Time; +import org.apache.kafka.common.utils.Timer; +import org.slf4j.Logger; +import java.util.ArrayList; import java.util.Collection; import java.util.HashSet; +import java.util.List; import java.util.Map; /** @@ -47,18 +52,22 @@ * on a different thread. * */ -public class Fetcher extends AbstractFetch { +public class Fetcher extends AbstractFetch { + private final Logger log; + private final ConsumerNetworkClient client; private final FetchCollector fetchCollector; public Fetcher(LogContext logContext, ConsumerNetworkClient client, ConsumerMetadata metadata, SubscriptionState subscriptions, - FetchConfig fetchConfig, + FetchConfig fetchConfig, FetchMetricsManager metricsManager, Time time) { - super(logContext, client, metadata, subscriptions, fetchConfig, metricsManager, time); + super(logContext, metadata, subscriptions, fetchConfig, metricsManager, time); + this.log = logContext.logger(Fetcher.class); + this.client = client; this.fetchCollector = new FetchCollector<>(logContext, metadata, subscriptions, @@ -67,6 +76,16 @@ public Fetcher(LogContext logContext, time); } + @Override + protected boolean isUnavailable(Node node) { + return client.isUnavailable(node); + } + + @Override + protected void maybeThrowAuthFailure(Node node) { + client.maybeThrowAuthFailure(node); + } + public void clearBufferedDataForUnassignedPartitions(Collection assignedPartitions) { fetchBuffer.retainAll(new HashSet<>(assignedPartitions)); } @@ -79,6 +98,7 @@ public void clearBufferedDataForUnassignedPartitions(Collection public synchronized int sendFetches() { Map fetchRequestMap = prepareFetchRequests(); + for (Map.Entry entry : fetchRequestMap.entrySet()) { final Node fetchTarget = entry.getKey(); final FetchSessionHandler.FetchRequestData data = entry.getValue(); @@ -106,7 +126,62 @@ public void onFailure(RuntimeException e) { return fetchRequestMap.size(); } - public Fetch collectFetch() { - return fetchCollector.collectFetch(fetchBuffer); + protected void maybeCloseFetchSessions(final Timer timer) { + final List> requestFutures = new ArrayList<>(); + Map fetchRequestMap = prepareCloseFetchSessionRequests(); + + for (Map.Entry entry : fetchRequestMap.entrySet()) { + final Node fetchTarget = entry.getKey(); + final FetchSessionHandler.FetchRequestData data = entry.getValue(); + final FetchRequest.Builder request = createFetchRequest(fetchTarget, data); + final RequestFuture responseFuture = client.send(fetchTarget, request); + + responseFuture.addListener(new RequestFutureListener() { + @Override + public void onSuccess(ClientResponse value) { + handleCloseFetchSessionResponse(fetchTarget, data); + } + + @Override + public void onFailure(RuntimeException e) { + handleCloseFetchSessionResponse(fetchTarget, data, e); + } + }); + + requestFutures.add(responseFuture); + } + + // Poll to ensure that request has been written to the socket. Wait until either the timer has expired or until + // all requests have received a response. + while (timer.notExpired() && !requestFutures.stream().allMatch(RequestFuture::isDone)) { + client.poll(timer, null, true); + } + + if (!requestFutures.stream().allMatch(RequestFuture::isDone)) { + // we ran out of time before completing all futures. It is ok since we don't want to block the shutdown + // here. + log.debug("All requests couldn't be sent in the specific timeout period {}ms. " + + "This may result in unnecessary fetch sessions at the broker. Consider increasing the timeout passed for " + + "KafkaConsumer.close(Duration timeout)", timer.timeoutMs()); + } + } + + public Fetch collectFetch(Deserializers deserializers) { + return fetchCollector.collectFetch(fetchBuffer, deserializers); + } + + /** + * This method is called by {@link #close(Timer)} which is guarded by the {@link IdempotentCloser}) such as to only + * be executed once the first time that any of the {@link #close()} methods are called. Subclasses can override + * this method without the need for extra synchronization at the instance level. + * + * @param timer Timer to enforce time limit + */ + // Visible for testing + protected void closeInternal(Timer timer) { + // we do not need to re-enable wake-ups since we are closing already + client.disableWakeups(); + maybeCloseFetchSessions(timer); + super.closeInternal(timer); } } \ No newline at end of file diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/NetworkClientDelegate.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/NetworkClientDelegate.java index 445005bf54552..d66e8a42809e7 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/NetworkClientDelegate.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/NetworkClientDelegate.java @@ -16,8 +16,10 @@ */ package org.apache.kafka.clients.consumer.internals; +import org.apache.kafka.clients.ApiVersions; import org.apache.kafka.clients.ClientRequest; import org.apache.kafka.clients.ClientResponse; +import org.apache.kafka.clients.ClientUtils; import org.apache.kafka.clients.KafkaClient; import org.apache.kafka.clients.NetworkClientUtils; import org.apache.kafka.clients.RequestCompletionHandler; @@ -26,6 +28,7 @@ import org.apache.kafka.common.errors.AuthenticationException; import org.apache.kafka.common.errors.DisconnectException; import org.apache.kafka.common.errors.TimeoutException; +import org.apache.kafka.common.metrics.Metrics; import org.apache.kafka.common.requests.AbstractRequest; import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.Time; @@ -35,26 +38,29 @@ import java.io.IOException; import java.util.ArrayDeque; import java.util.Collections; -import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.Queue; -import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.function.BiConsumer; +import java.util.function.Supplier; + +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_MAX_INFLIGHT_REQUESTS_PER_CONNECTION; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRIC_GROUP_PREFIX; /** * A wrapper around the {@link org.apache.kafka.clients.NetworkClient} to handle network poll and send operations. */ public class NetworkClientDelegate implements AutoCloseable { + private final KafkaClient client; private final Time time; private final Logger log; private final int requestTimeoutMs; private final Queue unsentRequests; private final long retryBackoffMs; - private final Set tryConnectNodes; public NetworkClientDelegate( final Time time, @@ -67,9 +73,40 @@ public NetworkClientDelegate( this.unsentRequests = new ArrayDeque<>(); this.requestTimeoutMs = config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG); this.retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG); - this.tryConnectNodes = new HashSet<>(); } + // Visible for testing + Queue unsentRequests() { + return unsentRequests; + } + + /** + * Check if the node is disconnected and unavailable for immediate reconnection (i.e. if it is in + * reconnect backoff window following the disconnect). + * + * @param node {@link Node} to check for availability + * @see NetworkClientUtils#isUnavailable(KafkaClient, Node, Time) + */ + public boolean isUnavailable(Node node) { + return NetworkClientUtils.isUnavailable(client, node, time); + } + + /** + * Checks for an authentication error on a given node and throws the exception if it exists. + * + * @param node {@link Node} to check for a previous {@link AuthenticationException}; if found it is thrown + * @see NetworkClientUtils#maybeThrowAuthFailure(KafkaClient, Node) + */ + public void maybeThrowAuthFailure(Node node) { + NetworkClientUtils.maybeThrowAuthFailure(client, node); + } + + /** + * Initiate a connection if currently possible. This is only really useful for resetting + * the failed status of a socket. + * + * @param node The node to connect to + */ public void tryConnect(Node node) { NetworkClientUtils.tryConnect(client, node, time); } @@ -80,7 +117,6 @@ public void tryConnect(Node node) { * * @param timeoutMs timeout time * @param currentTimeMs current time - * @return a list of client response */ public void poll(final long timeoutMs, final long currentTimeMs) { trySend(currentTimeMs); @@ -118,8 +154,7 @@ private void trySend(final long currentTimeMs) { } } - private boolean doSend(final UnsentRequest r, - final long currentTimeMs) { + boolean doSend(final UnsentRequest r, final long currentTimeMs) { Node node = r.node.orElse(client.leastLoadedNode(currentTimeMs)); if (node == null || nodeUnavailable(node)) { log.debug("No broker available to send the request: {}. Retrying.", r); @@ -136,7 +171,7 @@ private boolean doSend(final UnsentRequest r, return true; } - private void checkDisconnects() { + protected void checkDisconnects() { // Check the connection of the unsent request. Disconnect the disconnected node if it is unable to be connected. Iterator iter = unsentRequests.iterator(); while (iter.hasNext()) { @@ -208,7 +243,7 @@ public PollResult(final long timeMsTillNextPoll, final List unsen public static class UnsentRequest { private final AbstractRequest.Builder requestBuilder; private final FutureCompletionHandler handler; - private Optional node; // empty if random node can be chosen + private final Optional node; // empty if random node can be chosen private Timer timer; public UnsentRequest(final AbstractRequest.Builder requestBuilder, final Optional node) { @@ -224,6 +259,13 @@ public UnsentRequest(final AbstractRequest.Builder requestBuilder, this.handler = handler; } + public UnsentRequest(final AbstractRequest.Builder requestBuilder, + final Node node, + final BiConsumer responseHandler) { + this(requestBuilder, Optional.of(node)); + future().whenComplete(responseHandler); + } + public void setTimer(final Time time, final long requestTimeoutMs) { this.timer = time.timer(requestTimeoutMs); } @@ -232,7 +274,7 @@ CompletableFuture future() { return handler.future; } - RequestCompletionHandler callback() { + FutureCompletionHandler callback() { return handler; } @@ -240,6 +282,10 @@ AbstractRequest.Builder requestBuilder() { return requestBuilder; } + Optional node() { + return node; + } + @Override public String toString() { return "UnsentRequest{" + @@ -281,4 +327,31 @@ public void onComplete(final ClientResponse response) { } } + /** + * Creates a {@link Supplier} for deferred creation during invocation by + * {@link org.apache.kafka.clients.consumer.internals.DefaultBackgroundThread}. + */ + public static Supplier supplier(final Time time, + final LogContext logContext, + final ConsumerMetadata metadata, + final ConsumerConfig config, + final ApiVersions apiVersions, + final Metrics metrics, + final FetchMetricsManager fetchMetricsManager) { + return new CachedSupplier() { + @Override + protected NetworkClientDelegate create() { + KafkaClient client = ClientUtils.createNetworkClient(config, + metrics, + CONSUMER_METRIC_GROUP_PREFIX, + logContext, + apiVersions, + time, + CONSUMER_MAX_INFLIGHT_REQUESTS_PER_CONNECTION, + metadata, + fetchMetricsManager.throttleTimeSensor()); + return new NetworkClientDelegate(time, config, logContext, client); + } + }; + } } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java index 40b411b6dd576..8cacac04d17a4 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java @@ -20,25 +20,34 @@ import org.apache.kafka.clients.ClientUtils; import org.apache.kafka.clients.CommonClientConfigs; import org.apache.kafka.clients.GroupRebalanceConfig; +import org.apache.kafka.clients.Metadata; import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.ConsumerGroupMetadata; import org.apache.kafka.clients.consumer.ConsumerInterceptor; +import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor; import org.apache.kafka.clients.consumer.ConsumerRebalanceListener; import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.consumer.NoOffsetForPartitionException; import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.clients.consumer.OffsetAndTimestamp; import org.apache.kafka.clients.consumer.OffsetCommitCallback; +import org.apache.kafka.clients.consumer.OffsetResetStrategy; +import org.apache.kafka.clients.consumer.internals.events.ApplicationEvent; +import org.apache.kafka.clients.consumer.internals.events.ApplicationEventProcessor; import org.apache.kafka.clients.consumer.internals.events.AssignmentChangeApplicationEvent; import org.apache.kafka.clients.consumer.internals.events.BackgroundEvent; +import org.apache.kafka.clients.consumer.internals.events.BackgroundEventProcessor; import org.apache.kafka.clients.consumer.internals.events.CommitApplicationEvent; import org.apache.kafka.clients.consumer.internals.events.EventHandler; +import org.apache.kafka.clients.consumer.internals.events.FetchEvent; import org.apache.kafka.clients.consumer.internals.events.ListOffsetsApplicationEvent; import org.apache.kafka.clients.consumer.internals.events.NewTopicsMetadataUpdateRequestEvent; import org.apache.kafka.clients.consumer.internals.events.OffsetFetchApplicationEvent; import org.apache.kafka.clients.consumer.internals.events.ResetPositionsApplicationEvent; import org.apache.kafka.clients.consumer.internals.events.ValidatePositionsApplicationEvent; +import org.apache.kafka.common.Cluster; +import org.apache.kafka.common.IsolationLevel; import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.Metric; import org.apache.kafka.common.MetricName; @@ -47,15 +56,17 @@ import org.apache.kafka.common.config.ConfigException; import org.apache.kafka.common.errors.InterruptException; import org.apache.kafka.common.errors.InvalidGroupIdException; -import org.apache.kafka.common.errors.WakeupException; +import org.apache.kafka.common.errors.TimeoutException; import org.apache.kafka.common.internals.ClusterResourceListeners; import org.apache.kafka.common.metrics.Metrics; import org.apache.kafka.common.requests.ListOffsetsRequest; import org.apache.kafka.common.serialization.Deserializer; +import org.apache.kafka.common.utils.AppInfoParser; import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.Time; import org.apache.kafka.common.utils.Timer; import org.slf4j.Logger; +import org.slf4j.event.Level; import java.net.InetSocketAddress; import java.time.Duration; @@ -66,30 +77,38 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.OptionalLong; import java.util.Properties; +import java.util.Queue; import java.util.Set; +import java.util.concurrent.BlockingQueue; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; +import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; +import java.util.function.Supplier; import java.util.regex.Pattern; import java.util.stream.Collectors; import static java.util.Objects.requireNonNull; import static org.apache.kafka.clients.consumer.ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG; import static org.apache.kafka.clients.consumer.ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG; -import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.configuredConsumerInterceptors; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_JMX_PREFIX; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRIC_GROUP_PREFIX; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createFetchConfig; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createFetchMetricsManager; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createLogContext; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createMetrics; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createSubscriptionState; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.configuredConsumerInterceptors; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.configuredIsolationLevel; import static org.apache.kafka.common.utils.Utils.closeQuietly; import static org.apache.kafka.common.utils.Utils.isBlank; import static org.apache.kafka.common.utils.Utils.join; import static org.apache.kafka.common.utils.Utils.propsToMap; +import static org.apache.kafka.common.utils.Utils.swallow; /** * This prototype consumer uses the EventHandler to process application @@ -100,21 +119,35 @@ public class PrototypeAsyncConsumer implements Consumer { static final long DEFAULT_CLOSE_TIMEOUT_MS = 30 * 1000; - private final LogContext logContext; private final EventHandler eventHandler; private final Time time; private final Optional groupId; - private final Logger log; + private final KafkaConsumerMetrics kafkaConsumerMetrics; + private Logger log; + private final String clientId; + private final BackgroundEventProcessor backgroundEventProcessor; private final Deserializers deserializers; + private final FetchBuffer fetchBuffer; + private final FetchCollector fetchCollector; + private final ConsumerInterceptors interceptors; + private final IsolationLevel isolationLevel; + private final SubscriptionState subscriptions; private final ConsumerMetadata metadata; private final Metrics metrics; + private final long retryBackoffMs; + private final long requestTimeoutMs; private final long defaultApiTimeoutMs; + private volatile boolean closed = false; + private final List assignors; - private WakeupTrigger wakeupTrigger = new WakeupTrigger(); - public PrototypeAsyncConsumer(Properties properties, - Deserializer keyDeserializer, - Deserializer valueDeserializer) { + // to keep from repeatedly scanning subscriptions in poll(), cache the result during metadata updates + private boolean cachedSubscriptionHasAllFetchPositions; + private final WakeupTrigger wakeupTrigger = new WakeupTrigger(); + + public PrototypeAsyncConsumer(final Properties properties, + final Deserializer keyDeserializer, + final Deserializer valueDeserializer) { this(propsToMap(properties), keyDeserializer, valueDeserializer); } @@ -127,56 +160,151 @@ public PrototypeAsyncConsumer(final Map configs, public PrototypeAsyncConsumer(final ConsumerConfig config, final Deserializer keyDeserializer, final Deserializer valueDeserializer) { - this.time = Time.SYSTEM; - GroupRebalanceConfig groupRebalanceConfig = new GroupRebalanceConfig(config, - GroupRebalanceConfig.ProtocolType.CONSUMER); - this.groupId = Optional.ofNullable(groupRebalanceConfig.groupId); - this.defaultApiTimeoutMs = config.getInt(ConsumerConfig.DEFAULT_API_TIMEOUT_MS_CONFIG); - this.logContext = createLogContext(config, groupRebalanceConfig); - this.log = logContext.logger(getClass()); - this.deserializers = new Deserializers<>(config, keyDeserializer, valueDeserializer); - this.subscriptions = createSubscriptionState(config, logContext); - this.metrics = createMetrics(config, time); - List> interceptorList = configuredConsumerInterceptors(config); - ClusterResourceListeners clusterResourceListeners = ClientUtils.configureClusterResourceListeners( - metrics.reporters(), - interceptorList, - Arrays.asList(deserializers.keyDeserializer, deserializers.valueDeserializer)); - this.metadata = new ConsumerMetadata(config, subscriptions, logContext, clusterResourceListeners); - final List addresses = ClientUtils.parseAndValidateAddresses(config); - metadata.bootstrap(addresses); - this.eventHandler = new DefaultEventHandler( - config, - groupRebalanceConfig, - logContext, - subscriptions, - new ApiVersions(), - this.metrics, - clusterResourceListeners, - null // this is coming from the fetcher, but we don't have one - ); + this(Time.SYSTEM, config, keyDeserializer, valueDeserializer); } - // Visible for testing - PrototypeAsyncConsumer(Time time, - LogContext logContext, - ConsumerConfig config, - SubscriptionState subscriptions, - ConsumerMetadata metadata, - EventHandler eventHandler, - Metrics metrics, - Optional groupId, - int defaultApiTimeoutMs) { - this.time = time; - this.logContext = logContext; + public PrototypeAsyncConsumer(final Time time, + final ConsumerConfig config, + final Deserializer keyDeserializer, + final Deserializer valueDeserializer) { + try { + GroupRebalanceConfig groupRebalanceConfig = new GroupRebalanceConfig(config, + GroupRebalanceConfig.ProtocolType.CONSUMER); + + this.groupId = Optional.ofNullable(groupRebalanceConfig.groupId); + this.clientId = config.getString(CommonClientConfigs.CLIENT_ID_CONFIG); + LogContext logContext = createLogContext(config, groupRebalanceConfig); + this.log = logContext.logger(getClass()); + groupId.ifPresent(groupIdStr -> { + if (groupIdStr.isEmpty()) { + log.warn("Support for using the empty group id by consumers is deprecated and will be removed in the next major release."); + } + }); + + log.debug("Initializing the Kafka consumer"); + this.requestTimeoutMs = config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG); + this.defaultApiTimeoutMs = config.getInt(ConsumerConfig.DEFAULT_API_TIMEOUT_MS_CONFIG); + this.time = time; + this.metrics = createMetrics(config, time); + this.retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG); + + List> interceptorList = configuredConsumerInterceptors(config); + this.interceptors = new ConsumerInterceptors<>(interceptorList); + this.deserializers = new Deserializers<>(config, keyDeserializer, valueDeserializer); + this.subscriptions = createSubscriptionState(config, logContext); + ClusterResourceListeners clusterResourceListeners = ClientUtils.configureClusterResourceListeners(metrics.reporters(), + interceptorList, + Arrays.asList(deserializers.keyDeserializer, deserializers.valueDeserializer)); + this.metadata = new ConsumerMetadata(config, subscriptions, logContext, clusterResourceListeners); + final List addresses = ClientUtils.parseAndValidateAddresses(config); + metadata.bootstrap(addresses); + + FetchMetricsManager fetchMetricsManager = createFetchMetricsManager(metrics); + this.isolationLevel = configuredIsolationLevel(config); + + ApiVersions apiVersions = new ApiVersions(); + final BlockingQueue applicationEventQueue = new LinkedBlockingQueue<>(); + final BlockingQueue backgroundEventQueue = new LinkedBlockingQueue<>(); + final Supplier networkClientDelegateSupplier = NetworkClientDelegate.supplier(time, + logContext, + metadata, + config, + apiVersions, + metrics, + fetchMetricsManager); + final Supplier requestManagersSupplier = RequestManagers.supplier(time, + logContext, + backgroundEventQueue, + metadata, + subscriptions, + config, + groupRebalanceConfig, + apiVersions, + fetchMetricsManager, + networkClientDelegateSupplier); + final Supplier applicationEventProcessorSupplier = ApplicationEventProcessor.supplier(logContext, + metadata, + backgroundEventQueue, + requestManagersSupplier); + this.eventHandler = new DefaultEventHandler(time, + logContext, + applicationEventQueue, + applicationEventProcessorSupplier, + networkClientDelegateSupplier, + requestManagersSupplier); + this.backgroundEventProcessor = new BackgroundEventProcessor(logContext, backgroundEventQueue); + this.assignors = ConsumerPartitionAssignor.getAssignorInstances( + config.getList(ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG), + config.originals(Collections.singletonMap(ConsumerConfig.CLIENT_ID_CONFIG, clientId)) + ); + + // no coordinator will be constructed for the default (null) group id + if (!groupId.isPresent()) { + config.ignore(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG); + //config.ignore(ConsumerConfig.THROW_ON_FETCH_STABLE_OFFSET_UNSUPPORTED); + } + // These are specific to the foreground thread + FetchConfig fetchConfig = createFetchConfig(config); + this.fetchBuffer = new FetchBuffer(logContext); + this.fetchCollector = new FetchCollector<>(logContext, + metadata, + subscriptions, + fetchConfig, + fetchMetricsManager, + time); + + this.kafkaConsumerMetrics = new KafkaConsumerMetrics(metrics, CONSUMER_METRIC_GROUP_PREFIX); + + config.logUnused(); + AppInfoParser.registerAppInfo(CONSUMER_JMX_PREFIX, clientId, metrics, time.milliseconds()); + log.debug("Kafka consumer initialized"); + } catch (Throwable t) { + // call close methods if internal objects are already constructed; this is to prevent resource leak. see KAFKA-2121 + // we do not need to call `close` at all when `log` is null, which means no internal objects were initialized. + if (this.log != null) { + close(Duration.ZERO, true); + } + // now propagate the exception + throw new KafkaException("Failed to construct kafka consumer", t); + } + } + + public PrototypeAsyncConsumer(LogContext logContext, + String clientId, + Deserializers deserializers, + FetchBuffer fetchBuffer, + FetchCollector fetchCollector, + ConsumerInterceptors interceptors, + Time time, + EventHandler eventHandler, + BlockingQueue backgroundEventQueue, + Metrics metrics, + SubscriptionState subscriptions, + ConsumerMetadata metadata, + long retryBackoffMs, + long requestTimeoutMs, + int defaultApiTimeoutMs, + List assignors, + String groupId) { this.log = logContext.logger(getClass()); this.subscriptions = subscriptions; - this.metadata = metadata; + this.clientId = clientId; + this.fetchBuffer = fetchBuffer; + this.fetchCollector = fetchCollector; + this.isolationLevel = IsolationLevel.READ_UNCOMMITTED; + this.interceptors = Objects.requireNonNull(interceptors); + this.time = time; + this.backgroundEventProcessor = new BackgroundEventProcessor(logContext, backgroundEventQueue); this.metrics = metrics; - this.groupId = groupId; + this.groupId = Optional.ofNullable(groupId); + this.metadata = metadata; + this.retryBackoffMs = retryBackoffMs; + this.requestTimeoutMs = requestTimeoutMs; this.defaultApiTimeoutMs = defaultApiTimeoutMs; - this.deserializers = new Deserializers<>(config); + this.deserializers = deserializers; this.eventHandler = eventHandler; + this.assignors = assignors; + this.kafkaConsumerMetrics = new KafkaConsumerMetrics(metrics, "consumer"); } /** @@ -192,71 +320,38 @@ public PrototypeAsyncConsumer(final ConsumerConfig config, @Override public ConsumerRecords poll(final Duration timeout) { Timer timer = time.timer(timeout); + try { - do { - if (!eventHandler.isEmpty()) { - final Optional backgroundEvent = eventHandler.poll(); - // processEvent() may process 3 types of event: - // 1. Errors - // 2. Callback Invocation - // 3. Fetch responses - // Errors will be handled or rethrown. - // Callback invocation will trigger callback function execution, which is blocking until completion. - // Successful fetch responses will be added to the completedFetches in the fetcher, which will then - // be processed in the collectFetches(). - backgroundEvent.ifPresent(event -> processEvent(event, timeout)); - } + backgroundEventProcessor.process(); - updateFetchPositionsIfNeeded(timer); + this.kafkaConsumerMetrics.recordPollStart(timer.currentTimeMs()); - // The idea here is to have the background thread sending fetches autonomously, and the fetcher - // uses the poll loop to retrieve successful fetchResponse and process them on the polling thread. - final Fetch fetch = collectFetches(); - if (!fetch.isEmpty()) { - return processFetchResults(fetch); - } - // We will wait for retryBackoffMs - } while (time.timer(timeout).notExpired()); - } catch (final Exception e) { - throw new RuntimeException(e); - } - // TODO: Once we implement poll(), clear wakeupTrigger in a finally block: wakeupTrigger.clearActiveTask(); + if (this.subscriptions.hasNoSubscriptionOrUserAssignment()) { + throw new IllegalStateException("Consumer is not subscribed to any topics or assigned any partitions"); + } - return ConsumerRecords.empty(); - } + do { + updateAssignmentMetadataIfNeeded(timer); + final Fetch fetch = pollForFetches(timer); - /** - * Set the fetch position to the committed position (if there is one) or reset it using the - * offset reset policy the user has configured (if partitions require reset) - * - * @return true if the operation completed without timing out - * @throws org.apache.kafka.common.errors.AuthenticationException if authentication fails. See the exception for more details - * @throws NoOffsetForPartitionException If no offset is stored for a given partition and no offset reset policy is - * defined - */ - private boolean updateFetchPositionsIfNeeded(final Timer timer) { - // If any partitions have been truncated due to a leader change, we need to validate the offsets - ValidatePositionsApplicationEvent validatePositionsEvent = new ValidatePositionsApplicationEvent(); - eventHandler.add(validatePositionsEvent); + if (!fetch.isEmpty()) { + sendFetches(); - // If there are any partitions which do not have a valid position and are not - // awaiting reset, then we need to fetch committed offsets. We will only do a - // coordinator lookup if there are partitions which have missing positions, so - // a consumer with manually assigned partitions can avoid a coordinator dependence - // by always ensuring that assigned partitions have an initial position. - if (isCommittedOffsetsManagementEnabled() && !refreshCommittedOffsetsIfNeeded(timer)) - return false; + if (fetch.records().isEmpty()) { + log.trace("Returning empty records from `poll()` " + + "since the consumer's position has advanced for at least one topic partition"); + } - // If there are partitions still needing a position and a reset policy is defined, - // request reset using the default policy. If no reset strategy is defined and there - // are partitions with a missing position, then we will raise a NoOffsetForPartitionException exception. - subscriptions.resetInitializingPositions(); + return this.interceptors.onConsume(new ConsumerRecords<>(fetch.records())); + } + // We will wait for retryBackoffMs + } while (timer.notExpired()); - // Finally send an asynchronous request to look up and update the positions of any - // partitions which are awaiting reset. - ResetPositionsApplicationEvent resetPositionsEvent = new ResetPositionsApplicationEvent(); - eventHandler.add(resetPositionsEvent); - return true; + return ConsumerRecords.empty(); + } finally { + this.kafkaConsumerMetrics.recordPollEnd(timer.currentTimeMs()); + } + // TODO: Once we implement poll(), clear wakeupTrigger in a finally block: wakeupTrigger.clearActiveTask(); } /** @@ -268,20 +363,6 @@ public void commitSync() { commitSync(Duration.ofMillis(defaultApiTimeoutMs)); } - private void processEvent(final BackgroundEvent backgroundEvent, final Duration timeout) { - // stubbed class - } - - private ConsumerRecords processFetchResults(final Fetch fetch) { - // stubbed class - return ConsumerRecords.empty(); - } - - private Fetch collectFetches() { - // stubbed class - return Fetch.empty(); - } - /** * This method sends a commit event to the EventHandler and return. */ @@ -306,7 +387,6 @@ public void commitAsync(Map offsets, OffsetCo commitCallback.onComplete(offsets, null); } }).exceptionally(e -> { - System.out.println(e); throw new KafkaException(e); }); } @@ -325,44 +405,90 @@ CompletableFuture commit(Map offsets, f @Override public void seek(TopicPartition partition, long offset) { - throw new KafkaException("method not implemented"); + if (offset < 0) + throw new IllegalArgumentException("seek offset must not be a negative number"); + + log.info("Seeking to offset {} for partition {}", offset, partition); + SubscriptionState.FetchPosition newPosition = new SubscriptionState.FetchPosition( + offset, + Optional.empty(), // This will ensure we skip validation + this.metadata.currentLeader(partition)); + this.subscriptions.seekUnvalidated(partition, newPosition); } @Override public void seek(TopicPartition partition, OffsetAndMetadata offsetAndMetadata) { - throw new KafkaException("method not implemented"); + long offset = offsetAndMetadata.offset(); + if (offset < 0) { + throw new IllegalArgumentException("seek offset must not be a negative number"); + } + + if (offsetAndMetadata.leaderEpoch().isPresent()) { + log.info("Seeking to offset {} for partition {} with epoch {}", + offset, partition, offsetAndMetadata.leaderEpoch().get()); + } else { + log.info("Seeking to offset {} for partition {}", offset, partition); + } + Metadata.LeaderAndEpoch currentLeaderAndEpoch = this.metadata.currentLeader(partition); + SubscriptionState.FetchPosition newPosition = new SubscriptionState.FetchPosition( + offsetAndMetadata.offset(), + offsetAndMetadata.leaderEpoch(), + currentLeaderAndEpoch); + this.updateLastSeenEpochIfNewer(partition, offsetAndMetadata); + this.subscriptions.seekUnvalidated(partition, newPosition); } @Override public void seekToBeginning(Collection partitions) { - throw new KafkaException("method not implemented"); + if (partitions == null) + throw new IllegalArgumentException("Partitions collection cannot be null"); + + Collection parts = partitions.size() == 0 ? this.subscriptions.assignedPartitions() : partitions; + subscriptions.requestOffsetReset(parts, OffsetResetStrategy.EARLIEST); } @Override public void seekToEnd(Collection partitions) { - throw new KafkaException("method not implemented"); + if (partitions == null) + throw new IllegalArgumentException("Partitions collection cannot be null"); + + Collection parts = partitions.size() == 0 ? this.subscriptions.assignedPartitions() : partitions; + subscriptions.requestOffsetReset(parts, OffsetResetStrategy.LATEST); } @Override public long position(TopicPartition partition) { - throw new KafkaException("method not implemented"); + return position(partition, Duration.ofMillis(defaultApiTimeoutMs)); } @Override public long position(TopicPartition partition, Duration timeout) { - throw new KafkaException("method not implemented"); + if (!this.subscriptions.isAssigned(partition)) + throw new IllegalStateException("You can only check the position for partitions assigned to this consumer."); + + Timer timer = time.timer(timeout); + do { + SubscriptionState.FetchPosition position = this.subscriptions.validPosition(partition); + if (position != null) + return position.offset; + + updateFetchPositions(timer); + } while (timer.notExpired()); + + throw new TimeoutException("Timeout of " + timeout.toMillis() + "ms expired before the position " + + "for partition " + partition + " could be determined"); } @Override @Deprecated public OffsetAndMetadata committed(TopicPartition partition) { - throw new KafkaException("method not implemented"); + return committed(partition, Duration.ofMillis(defaultApiTimeoutMs)); } @Override @Deprecated public OffsetAndMetadata committed(TopicPartition partition, Duration timeout) { - throw new KafkaException("method not implemented"); + return committed(Collections.singleton(partition), timeout).get(partition); } @Override @@ -396,12 +522,12 @@ private void maybeThrowInvalidGroupIdException() { @Override public Map metrics() { - throw new KafkaException("method not implemented"); + return Collections.unmodifiableMap(this.metrics.metrics()); } @Override public List partitionsFor(String topic) { - throw new KafkaException("method not implemented"); + return partitionsFor(topic, Duration.ofMillis(defaultApiTimeoutMs)); } @Override @@ -411,7 +537,7 @@ public List partitionsFor(String topic, Duration timeout) { @Override public Map> listTopics() { - throw new KafkaException("method not implemented"); + return listTopics(Duration.ofMillis(defaultApiTimeoutMs)); } @Override @@ -421,17 +547,23 @@ public Map> listTopics(Duration timeout) { @Override public Set paused() { - throw new KafkaException("method not implemented"); + return Collections.unmodifiableSet(subscriptions.pausedPartitions()); } @Override public void pause(Collection partitions) { - throw new KafkaException("method not implemented"); + log.debug("Pausing partitions {}", partitions); + for (TopicPartition partition: partitions) { + subscriptions.pause(partition); + } } @Override public void resume(Collection partitions) { - throw new KafkaException("method not implemented"); + log.debug("Resuming partitions {}", partitions); + for (TopicPartition partition: partitions) { + subscriptions.resume(partition); + } } @Override @@ -503,7 +635,25 @@ private Map beginningOrEndOffset(Collection firstException = new AtomicReference<>(); + + final Timer closeTimer = createTimerForRequest(timeout); + if (fetchBuffer != null) { + // the timeout for the session close is at-most the requestTimeoutMs + long remainingDurationInTimeout = Math.max(0, timeout.toMillis() - closeTimer.elapsedMs()); + if (remainingDurationInTimeout > 0) { + remainingDurationInTimeout = Math.min(requestTimeoutMs, remainingDurationInTimeout); + } + + closeTimer.reset(remainingDurationInTimeout); + + // This is a blocking call bound by the time remaining in closeTimer + swallow(log, Level.ERROR, "Failed to close fetcher", fetchBuffer::close, firstException); + } + + + closeQuietly(interceptors, "consumer interceptors", firstException); + closeQuietly(kafkaConsumerMetrics, "kafka consumer metrics", firstException); + closeQuietly(metrics, "consumer metrics", firstException); closeQuietly(this.eventHandler, "event handler", firstException); + closeQuietly(deserializers, "consumer deserializers", firstException); + + AppInfoParser.unregisterAppInfo(CONSUMER_JMX_PREFIX, clientId, metrics); log.debug("Kafka consumer has been closed"); Throwable exception = firstException.get(); - if (exception != null) { + if (exception != null && !swallowException) { if (exception instanceof InterruptException) { throw (InterruptException) exception; } @@ -563,19 +757,14 @@ public void commitSync(Map offsets) { @Override public void commitSync(Map offsets, Duration timeout) { - CompletableFuture commitFuture = commit(offsets, true); + long commitStart = time.nanoseconds(); try { - commitFuture.get(timeout.toMillis(), TimeUnit.MILLISECONDS); - } catch (final TimeoutException e) { - throw new org.apache.kafka.common.errors.TimeoutException(e); - } catch (final InterruptedException e) { - throw new InterruptException(e); - } catch (final ExecutionException e) { - if (e.getCause() instanceof WakeupException) - throw new WakeupException(); - throw new KafkaException(e); + CompletableFuture commitFuture = commit(offsets, true); + offsets.forEach(this::updateLastSeenEpochIfNewer); + ConsumerUtils.getResult(commitFuture, time.timer(timeout)); } finally { wakeupTrigger.clearActiveTask(); + kafkaConsumerMetrics.recordCommitSync(time.nanoseconds() - commitStart); } } @@ -596,12 +785,38 @@ public Set subscription() { @Override public void subscribe(Collection topics) { - throw new KafkaException("method not implemented"); + subscribe(topics, new NoOpConsumerRebalanceListener()); } @Override public void subscribe(Collection topics, ConsumerRebalanceListener callback) { - throw new KafkaException("method not implemented"); + maybeThrowInvalidGroupIdException(); + if (topics == null) + throw new IllegalArgumentException("Topic collection to subscribe to cannot be null"); + if (topics.isEmpty()) { + // treat subscribing to empty topic list as the same as unsubscribing + this.unsubscribe(); + } else { + for (String topic : topics) { + if (isBlank(topic)) + throw new IllegalArgumentException("Topic collection to subscribe to cannot contain null or empty topic"); + } + + throwIfNoAssignorsConfigured(); + + // Clear the buffered data which are not a part of newly assigned topics + final Set currentTopicPartitions = new HashSet<>(); + + for (TopicPartition tp : subscriptions.assignedPartitions()) { + if (topics.contains(tp.topic())) + currentTopicPartitions.add(tp); + } + + fetchBuffer.retainAll(currentTopicPartitions); + log.info("Subscribed to topic(s): {}", join(topics, ", ")); + if (this.subscriptions.subscribe(new HashSet<>(topics), callback)) + metadata.requestUpdateForNewTopics(); + } } @Override @@ -611,8 +826,7 @@ public void assign(Collection partitions) { } if (partitions.isEmpty()) { - // TODO: implementation of unsubscribe() will be included in forthcoming commits. - // this.unsubscribe(); + this.unsubscribe(); return; } @@ -622,12 +836,18 @@ public void assign(Collection partitions) { throw new IllegalArgumentException("Topic partitions to assign to cannot have null or empty topic"); } - // TODO: implementation of refactored Fetcher will be included in forthcoming commits. - // fetcher.clearBufferedDataForUnassignedPartitions(partitions); + // Clear the buffered data which are not a part of newly assigned topics + final Set currentTopicPartitions = new HashSet<>(); + + for (TopicPartition tp : subscriptions.assignedPartitions()) { + if (partitions.contains(tp)) + currentTopicPartitions.add(tp); + } + + fetchBuffer.retainAll(currentTopicPartitions); - // assignment change event will trigger autocommit if it is configured and the group id is specified. This is - // to make sure offsets of topic partitions the consumer is unsubscribing from are committed since there will - // be no following rebalance + // make sure the offsets of topic partitions the consumer is unsubscribing from + // are committed since there will be no following rebalance eventHandler.add(new AssignmentChangeApplicationEvent(this.subscriptions.allConsumed(), time.milliseconds())); log.info("Assigned to partition(s): {}", join(partitions, ", ")); @@ -636,24 +856,52 @@ public void assign(Collection partitions) { } @Override - public void subscribe(Pattern pattern, ConsumerRebalanceListener callback) { - throw new KafkaException("method not implemented"); + public void subscribe(Pattern pattern, ConsumerRebalanceListener listener) { + maybeThrowInvalidGroupIdException(); + if (pattern == null || pattern.toString().equals("")) + throw new IllegalArgumentException("Topic pattern to subscribe to cannot be " + (pattern == null ? + "null" : "empty")); + + throwIfNoAssignorsConfigured(); + log.info("Subscribed to pattern: '{}'", pattern); + this.subscriptions.subscribe(pattern, listener); + this.updatePatternSubscription(metadata.fetch()); + this.metadata.requestUpdateForNewTopics(); + } + + /** + * TODO: remove this when we implement the KIP-848 protocol. + * + *

    + * The contents of this method are shamelessly stolen from + * {@link ConsumerCoordinator#updatePatternSubscription(Cluster)} and are used here because we won't have access + * to a {@link ConsumerCoordinator} in this code. Perhaps it could be moved to a ConsumerUtils class? + * + * @param cluster Cluster from which we get the topics + */ + private void updatePatternSubscription(Cluster cluster) { + final Set topicsToSubscribe = cluster.topics().stream() + .filter(subscriptions::matchesSubscribedPattern) + .collect(Collectors.toSet()); + if (subscriptions.subscribeFromPattern(topicsToSubscribe)) + metadata.requestUpdateForNewTopics(); } @Override public void subscribe(Pattern pattern) { - throw new KafkaException("method not implemented"); + subscribe(pattern, new NoOpConsumerRebalanceListener()); } @Override public void unsubscribe() { - throw new KafkaException("method not implemented"); + fetchBuffer.retainAll(Collections.emptySet()); + this.subscriptions.unsubscribe(); } @Override @Deprecated - public ConsumerRecords poll(long timeout) { - throw new KafkaException("method not implemented"); + public ConsumerRecords poll(final long timeoutMs) { + return poll(Duration.ofMillis(timeoutMs)); } // Visible for testing @@ -661,17 +909,95 @@ WakeupTrigger wakeupTrigger() { return wakeupTrigger; } - private static ClusterResourceListeners configureClusterResourceListeners( - final Deserializer keyDeserializer, - final Deserializer valueDeserializer, - final List... candidateLists) { - ClusterResourceListeners clusterResourceListeners = new ClusterResourceListeners(); - for (List candidateList: candidateLists) - clusterResourceListeners.maybeAddAll(candidateList); + private void sendFetches() { + FetchEvent event = new FetchEvent(); + eventHandler.add(event); + + event.future().whenComplete((completedFetches, error) -> { + if (completedFetches != null && !completedFetches.isEmpty()) { + fetchBuffer.addAll(completedFetches); + } + }); + } + + /** + * @throws KafkaException if the rebalance callback throws exception + */ + private Fetch pollForFetches(Timer timer) { + long pollTimeout = timer.remainingMs(); + + // if data is available already, return it immediately + final Fetch fetch = fetchCollector.collectFetch(fetchBuffer, deserializers); + if (!fetch.isEmpty()) { + return fetch; + } + + // send any new fetches (won't resend pending fetches) + sendFetches(); + + // We do not want to be stuck blocking in poll if we are missing some positions + // since the offset lookup may be backing off after a failure + + // NOTE: the use of cachedSubscriptionHasAllFetchPositions means we MUST call + // updateAssignmentMetadataIfNeeded before this method. + if (!cachedSubscriptionHasAllFetchPositions && pollTimeout > retryBackoffMs) { + pollTimeout = retryBackoffMs; + } + + log.trace("Polling for fetches with timeout {}", pollTimeout); + + Timer pollTimer = time.timer(pollTimeout); + + // Attempt to fetch any data. It's OK if we time out here; it's a best case effort. The + // data may not be immediately available, but the calling method (poll) will correctly + // handle the overall timeout. + try { + Queue completedFetches = eventHandler.addAndGet(new FetchEvent(), pollTimer); + if (completedFetches != null && !completedFetches.isEmpty()) { + fetchBuffer.addAll(completedFetches); + } + } catch (TimeoutException e) { + log.trace("Timeout during fetch", e); + } finally { + timer.update(pollTimer.currentTimeMs()); + } - clusterResourceListeners.maybeAdd(keyDeserializer); - clusterResourceListeners.maybeAdd(valueDeserializer); - return clusterResourceListeners; + return fetchCollector.collectFetch(fetchBuffer, deserializers); + } + + /** + * Set the fetch position to the committed position (if there is one) + * or reset it using the offset reset policy the user has configured. + * + * @throws org.apache.kafka.common.errors.AuthenticationException if authentication fails. See the exception for more details + * @throws NoOffsetForPartitionException If no offset is stored for a given partition and no offset reset policy is + * defined + * @return true iff the operation completed without timing out + */ + private boolean updateFetchPositions(final Timer timer) { + // If any partitions have been truncated due to a leader change, we need to validate the offsets + eventHandler.add(new ValidatePositionsApplicationEvent()); + + cachedSubscriptionHasAllFetchPositions = subscriptions.hasAllFetchPositions(); + if (cachedSubscriptionHasAllFetchPositions) return true; + + // If there are any partitions which do not have a valid position and are not + // awaiting reset, then we need to fetch committed offsets. We will only do a + // coordinator lookup if there are partitions which have missing positions, so + // a consumer with manually assigned partitions can avoid a coordinator dependence + // by always ensuring that assigned partitions have an initial position. + if (isCommittedOffsetsManagementEnabled() && !refreshCommittedOffsetsIfNeeded(timer)) + return false; + + // If there are partitions still needing a position and a reset policy is defined, + // request reset using the default policy. If no reset strategy is defined and there + // are partitions with a missing position, then we will raise a NoOffsetForPartitionException exception. + subscriptions.resetInitializingPositions(); + + // Finally send an asynchronous request to look up and update the positions of any + // partitions which are awaiting reset. + eventHandler.add(new ResetPositionsApplicationEvent()); + return true; } /** @@ -684,7 +1010,7 @@ private boolean isCommittedOffsetsManagementEnabled() { } /** - * Refresh the committed offsets for partitions that require initialization. + * Refresh the committed offsets for provided partitions. * * @param timer Timer bounding how long this method can block * @return true iff the operation completed within the timeout @@ -702,7 +1028,6 @@ private boolean refreshCommittedOffsetsIfNeeded(Timer timer) { } } - // This is here temporary as we don't have public access to the ConsumerConfig in this module. public static Map appendDeserializerToConfig(Map configs, Deserializer keyDeserializer, @@ -720,6 +1045,17 @@ else if (newConfigs.get(VALUE_DESERIALIZER_CLASS_CONFIG) == null) return newConfigs; } + private void throwIfNoAssignorsConfigured() { + if (assignors.isEmpty()) + throw new IllegalStateException("Must configure at least one partition assigner class name to " + + ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG + " configuration property"); + } + + private void updateLastSeenEpochIfNewer(TopicPartition topicPartition, OffsetAndMetadata offsetAndMetadata) { + if (offsetAndMetadata != null) + offsetAndMetadata.leaderEpoch().ifPresent(epoch -> metadata.updateLastSeenEpochIfNewer(topicPartition, epoch)); + } + private class DefaultOffsetCommitCallback implements OffsetCommitCallback { @Override public void onComplete(Map offsets, Exception exception) { @@ -727,4 +1063,11 @@ public void onComplete(Map offsets, Exception log.error("Offset commit with offsets {} failed", offsets, exception); } } + + boolean updateAssignmentMetadataIfNeeded(Timer timer) { + // Keeping this updateAssignmentMetadataIfNeeded wrapping up the updateFetchPositions as + // in the previous implementation, because it will eventually involve group coordination + // logic + return updateFetchPositions(timer); + } } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManager.java index 4655f5ef7042e..5bd6e01c2bb15 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManager.java @@ -18,11 +18,18 @@ import org.apache.kafka.clients.consumer.internals.NetworkClientDelegate.PollResult; +import java.io.Closeable; + /** * {@code PollResult} consist of {@code UnsentRequest} if there are requests to send; otherwise, return the time till * the next poll event. */ -public interface RequestManager { +public interface RequestManager extends Closeable { + PollResult poll(long currentTimeMs); + @Override + default void close() { + // Do nothing... + } } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java index 0df55a8618428..7e15cb1477585 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java @@ -16,41 +16,149 @@ */ package org.apache.kafka.clients.consumer.internals; +import org.apache.kafka.clients.ApiVersions; +import org.apache.kafka.clients.GroupRebalanceConfig; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.internals.events.BackgroundEvent; +import org.apache.kafka.common.IsolationLevel; +import org.apache.kafka.common.internals.IdempotentCloser; +import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.common.utils.Time; +import org.slf4j.Logger; + +import java.io.Closeable; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.concurrent.BlockingQueue; +import java.util.function.Supplier; import static java.util.Objects.requireNonNull; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createFetchConfig; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.configuredIsolationLevel; /** * {@code RequestManagers} provides a means to pass around the set of {@link RequestManager} instances in the system. * This allows callers to both use the specific {@link RequestManager} instance, or to iterate over the list via * the {@link #entries()} method. */ -public class RequestManagers { +public class RequestManagers implements Closeable { + private final Logger log; public final Optional coordinatorRequestManager; public final Optional commitRequestManager; public final OffsetsRequestManager offsetsRequestManager; + public final FetchRequestManager fetchRequestManager; + private final List> entries; + private final IdempotentCloser closer = new IdempotentCloser(); - public RequestManagers(OffsetsRequestManager offsetsRequestManager, + public RequestManagers(LogContext logContext, + OffsetsRequestManager offsetsRequestManager, + FetchRequestManager fetchRequestManager, Optional coordinatorRequestManager, Optional commitRequestManager) { - this.offsetsRequestManager = requireNonNull(offsetsRequestManager, - "OffsetsRequestManager cannot be null"); + this.log = logContext.logger(RequestManagers.class); + this.offsetsRequestManager = requireNonNull(offsetsRequestManager, "OffsetsRequestManager cannot be null"); this.coordinatorRequestManager = coordinatorRequestManager; this.commitRequestManager = commitRequestManager; + this.fetchRequestManager = fetchRequestManager; List> list = new ArrayList<>(); list.add(coordinatorRequestManager); list.add(commitRequestManager); list.add(Optional.of(offsetsRequestManager)); + list.add(Optional.of(fetchRequestManager)); entries = Collections.unmodifiableList(list); } public List> entries() { return entries; } + + @Override + public void close() { + closer.close( + () -> { + log.debug("Closing RequestManagers"); + + entries.forEach(rm -> { + rm.ifPresent(requestManager -> { + try { + requestManager.close(); + } catch (Throwable t) { + log.debug("Error closing request manager {}", requestManager.getClass().getSimpleName(), t); + } + }); + }); + log.debug("RequestManagers has been closed"); + }, + () -> log.debug("RequestManagers was already closed")); + + } + + /** + * Creates a {@link Supplier} for deferred creation during invocation by + * {@link org.apache.kafka.clients.consumer.internals.DefaultBackgroundThread}. + */ + public static Supplier supplier(final Time time, + final LogContext logContext, + final BlockingQueue backgroundEventQueue, + final ConsumerMetadata metadata, + final SubscriptionState subscriptions, + final ConsumerConfig config, + final GroupRebalanceConfig groupRebalanceConfig, + final ApiVersions apiVersions, + final FetchMetricsManager fetchMetricsManager, + final Supplier networkClientDelegateSupplier) { + return new CachedSupplier() { + @Override + protected RequestManagers create() { + final NetworkClientDelegate networkClientDelegate = networkClientDelegateSupplier.get(); + final ErrorEventHandler errorEventHandler = new ErrorEventHandler(backgroundEventQueue); + final IsolationLevel isolationLevel = configuredIsolationLevel(config); + final FetchConfig fetchConfig = createFetchConfig(config); + long retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG); + long retryBackoffMaxMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MAX_MS_CONFIG); + final int requestTimeoutMs = config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG); + final OffsetsRequestManager listOffsets = new OffsetsRequestManager(subscriptions, + metadata, + isolationLevel, + time, + retryBackoffMs, + requestTimeoutMs, + apiVersions, + networkClientDelegate, + logContext); + final FetchRequestManager fetch = new FetchRequestManager(logContext, + time, + errorEventHandler, + metadata, + subscriptions, + fetchConfig, + fetchMetricsManager, + networkClientDelegate); + CoordinatorRequestManager coordinator = null; + CommitRequestManager commit = null; + + if (groupRebalanceConfig != null && groupRebalanceConfig.groupId != null) { + final GroupState groupState = new GroupState(groupRebalanceConfig); + coordinator = new CoordinatorRequestManager(time, + logContext, + retryBackoffMs, + retryBackoffMaxMs, + errorEventHandler, + groupState.groupId); + commit = new CommitRequestManager(time, logContext, subscriptions, config, coordinator, groupState); + } + + return new RequestManagers(logContext, + listOffsets, + fetch, + Optional.ofNullable(coordinator), + Optional.ofNullable(commit)); + } + }; + } } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEvent.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEvent.java index 65ba01959cd57..d0bbe6e399f9a 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEvent.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEvent.java @@ -25,7 +25,7 @@ public abstract class ApplicationEvent { public enum Type { NOOP, COMMIT, POLL, FETCH_COMMITTED_OFFSET, METADATA_UPDATE, ASSIGNMENT_CHANGE, - LIST_OFFSETS, RESET_POSITIONS, VALIDATE_POSITIONS, + LIST_OFFSETS, RESET_POSITIONS, VALIDATE_POSITIONS, FETCH } private final Type type; diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventProcessor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventProcessor.java index 234a228ba4f7b..370583abafedb 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventProcessor.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventProcessor.java @@ -17,35 +17,51 @@ package org.apache.kafka.clients.consumer.internals.events; import org.apache.kafka.clients.consumer.OffsetAndTimestamp; +import org.apache.kafka.clients.consumer.internals.CachedSupplier; import org.apache.kafka.clients.consumer.internals.CommitRequestManager; +import org.apache.kafka.clients.consumer.internals.CompletedFetch; import org.apache.kafka.clients.consumer.internals.ConsumerMetadata; import org.apache.kafka.clients.consumer.internals.RequestManagers; import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.utils.LogContext; +import org.slf4j.Logger; import java.util.Map; import java.util.Objects; +import java.util.Queue; import java.util.concurrent.BlockingQueue; import java.util.concurrent.CompletableFuture; +import java.util.function.Supplier; public class ApplicationEventProcessor { private final BlockingQueue backgroundEventQueue; + private final Logger log; + private final ConsumerMetadata metadata; private final RequestManagers requestManagers; public ApplicationEventProcessor(final BlockingQueue backgroundEventQueue, final RequestManagers requestManagers, - final ConsumerMetadata metadata) { + final ConsumerMetadata metadata, + final LogContext logContext) { + this.log = logContext.logger(ApplicationEventProcessor.class); this.backgroundEventQueue = backgroundEventQueue; this.requestManagers = requestManagers; this.metadata = metadata; } public boolean process(final ApplicationEvent event) { - Objects.requireNonNull(event); + Objects.requireNonNull(event, "Attempt to process null ApplicationEvent"); + Objects.requireNonNull(event.type(), "Attempt to process ApplicationEvent with null type: " + event); + + log.trace("Processing event: {}", event); + + // Make sure to use the event's type() method, not the type variable directly. This causes problems when + // unit tests mock the EventType. switch (event.type()) { case NOOP: return process((NoopApplicationEvent) event); @@ -61,6 +77,8 @@ public boolean process(final ApplicationEvent event) { return process((AssignmentChangeApplicationEvent) event); case LIST_OFFSETS: return process((ListOffsetsApplicationEvent) event); + case FETCH: + return process((FetchEvent) event); case RESET_POSITIONS: return processResetPositionsEvent(); case VALIDATE_POSITIONS: @@ -153,4 +171,29 @@ private boolean processValidatePositionsEvent() { requestManagers.offsetsRequestManager.validatePositionsIfNeeded(); return true; } + + private boolean process(final FetchEvent event) { + // The request manager keeps track of the completed fetches, so we pull any that are ready off, and return + // them to the application. + Queue completedFetches = requestManagers.fetchRequestManager.drain(); + event.future().complete(completedFetches); + return true; + } + + /** + * Creates a {@link Supplier} for deferred creation during invocation by + * {@link org.apache.kafka.clients.consumer.internals.DefaultBackgroundThread}. + */ + public static Supplier supplier(final LogContext logContext, + final ConsumerMetadata metadata, + final BlockingQueue backgroundEventQueue, + final Supplier requestManagersSupplier) { + return new CachedSupplier() { + @Override + protected ApplicationEventProcessor create() { + RequestManagers requestManagers = requestManagersSupplier.get(); + return new ApplicationEventProcessor(backgroundEventQueue, requestManagers, metadata, logContext); + } + }; + } } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEventProcessor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEventProcessor.java new file mode 100644 index 0000000000000..cdf7fa20d37df --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEventProcessor.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.clients.consumer.internals.events; + +import org.apache.kafka.common.utils.LogContext; +import org.slf4j.Logger; + +import java.util.LinkedList; +import java.util.Objects; +import java.util.concurrent.BlockingQueue; + +public class BackgroundEventProcessor { + + private final Logger log; + private final BlockingQueue backgroundEventQueue; + + public BackgroundEventProcessor(final LogContext logContext, + final BlockingQueue backgroundEventQueue) { + this.log = logContext.logger(BackgroundEventProcessor.class); + this.backgroundEventQueue = backgroundEventQueue; + } + + /** + * Drains all available {@link BackgroundEvent}s, and then processes them in order. If any + * errors are thrown as a result of a {@link ErrorBackgroundEvent} or an error occurs while processing + * another type of {@link BackgroundEvent}, only the first exception will be thrown, all + * subsequent errors will simply be logged at WARN level. + * + * @throws RuntimeException or subclass + */ + public void process() { + LinkedList events = new LinkedList<>(); + backgroundEventQueue.drainTo(events); + + RuntimeException first = null; + int errorCount = 0; + + for (BackgroundEvent event : events) { + log.debug("Consuming background event: {}", event); + + try { + process(event); + } catch (RuntimeException e) { + errorCount++; + + if (first == null) { + first = e; + log.warn("Error #{} from background thread (will be logged and thrown): {}", errorCount, e.getMessage(), e); + } else { + log.warn("Error #{} from background thread (will be logged only): {}", errorCount, e.getMessage(), e); + } + } + } + + if (first != null) { + throw first; + } + } + + private void process(final BackgroundEvent event) { + Objects.requireNonNull(event, "Attempt to process null BackgroundEvent"); + Objects.requireNonNull(event.type(), "Attempt to process BackgroundEvent with null type: " + event); + + log.debug("Processing event {}", event); + + // Make sure to use the event's type() method, not the type variable directly. This causes problems when + // unit tests mock the EventType. + switch (event.type()) { + case NOOP: + process((NoopBackgroundEvent) event); + return; + + case ERROR: + process((ErrorBackgroundEvent) event); + return; + + default: + throw new IllegalArgumentException("Background event type " + event.type() + " was not expected"); + } + } + + private void process(final NoopBackgroundEvent event) { + log.debug("Received no-op background event with message: {}", event.message()); + } + + private void process(final ErrorBackgroundEvent event) { + throw event.error(); + } +} diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ErrorBackgroundEvent.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ErrorBackgroundEvent.java index 4fc08290b715c..7f3d4e22429e3 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ErrorBackgroundEvent.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ErrorBackgroundEvent.java @@ -18,14 +18,14 @@ public class ErrorBackgroundEvent extends BackgroundEvent { - private final Throwable error; + private final RuntimeException error; - public ErrorBackgroundEvent(Throwable error) { + public ErrorBackgroundEvent(RuntimeException error) { super(Type.ERROR); this.error = error; } - public Throwable error() { + public RuntimeException error() { return error; } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/EventHandler.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/EventHandler.java index 75075fc2779fc..4c76b5f7bf104 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/EventHandler.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/EventHandler.java @@ -19,7 +19,7 @@ import org.apache.kafka.common.utils.Timer; import java.io.Closeable; -import java.util.Optional; +import java.time.Duration; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; @@ -28,17 +28,6 @@ * the {@code add()} method and to retrieve events via the {@code poll()} method. */ public interface EventHandler extends Closeable { - /** - * Retrieves and removes a {@link BackgroundEvent}. Returns an empty Optional instance if there is nothing. - * @return an Optional of {@link BackgroundEvent} if the value is present. Otherwise, an empty Optional. - */ - Optional poll(); - - /** - * Check whether there are pending {@code BackgroundEvent} await to be consumed. - * @return true if there are no pending event - */ - boolean isEmpty(); /** * Add an {@link ApplicationEvent} to the handler. The method returns true upon successful add; otherwise returns @@ -62,4 +51,10 @@ public interface EventHandler extends Closeable { * @param Type of return value of the event */ T addAndGet(final CompletableApplicationEvent event, final Timer timer); + + default void close() { + close(Duration.ofMillis(Long.MAX_VALUE)); + } + + void close(Duration timeout); } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/FetchEvent.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/FetchEvent.java new file mode 100644 index 0000000000000..ec68a3e4cda44 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/FetchEvent.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.clients.consumer.internals.events; + +import org.apache.kafka.clients.consumer.internals.CompletedFetch; + +import java.util.Queue; + +/** + * This event signals the background thread to submit a fetch request. + */ +public class FetchEvent extends CompletableApplicationEvent> { + + public FetchEvent() { + super(Type.FETCH); + } +} diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java index f61d4547d8b78..ebef3ecaca030 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java @@ -29,7 +29,6 @@ import org.apache.kafka.clients.consumer.internals.ConsumerMetrics; import org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient; import org.apache.kafka.clients.consumer.internals.ConsumerProtocol; -import org.apache.kafka.clients.consumer.internals.Deserializers; import org.apache.kafka.clients.consumer.internals.FetchConfig; import org.apache.kafka.clients.consumer.internals.FetchMetricsManager; import org.apache.kafka.clients.consumer.internals.Fetcher; @@ -2674,7 +2673,7 @@ private KafkaConsumer newConsumer(Time time, } IsolationLevel isolationLevel = IsolationLevel.READ_UNCOMMITTED; FetchMetricsManager metricsManager = new FetchMetricsManager(metrics, metricsRegistry.fetcherMetrics); - FetchConfig fetchConfig = new FetchConfig<>( + FetchConfig fetchConfig = new FetchConfig( minBytes, maxBytes, maxWaitMs, @@ -2682,7 +2681,6 @@ private KafkaConsumer newConsumer(Time time, maxPollRecords, checkCrcs, CommonClientConfigs.DEFAULT_CLIENT_RACK, - new Deserializers<>(keyDeserializer, deserializer), isolationLevel); Fetcher fetcher = new Fetcher<>( loggerFactory, diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/CompletedFetchTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/CompletedFetchTest.java index de97ee40de2df..2c39f4411febe 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/CompletedFetchTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/CompletedFetchTest.java @@ -34,7 +34,6 @@ import org.apache.kafka.common.record.Records; import org.apache.kafka.common.record.SimpleRecord; import org.apache.kafka.common.record.TimestampType; -import org.apache.kafka.common.serialization.Deserializer; import org.apache.kafka.common.serialization.StringDeserializer; import org.apache.kafka.common.serialization.UUIDDeserializer; import org.apache.kafka.common.serialization.UUIDSerializer; @@ -67,23 +66,22 @@ public void testSimple() { FetchResponseData.PartitionData partitionData = new FetchResponseData.PartitionData() .setRecords(newRecords(startingOffset, numRecords, fetchOffset)); - FetchConfig fetchConfig = newFetchConfig(new StringDeserializer(), - new StringDeserializer(), - IsolationLevel.READ_UNCOMMITTED, - true); + Deserializers deserializers = newStringDeserializers(); + FetchConfig fetchConfig = newFetchConfig(IsolationLevel.READ_UNCOMMITTED, true); + CompletedFetch completedFetch = newCompletedFetch(fetchOffset, partitionData); - List> records = completedFetch.fetchRecords(fetchConfig, 10); + List> records = completedFetch.fetchRecords(fetchConfig, deserializers, 10); assertEquals(10, records.size()); ConsumerRecord record = records.get(0); assertEquals(10, record.offset()); - records = completedFetch.fetchRecords(fetchConfig, 10); + records = completedFetch.fetchRecords(fetchConfig, deserializers, 10); assertEquals(1, records.size()); record = records.get(0); assertEquals(20, record.offset()); - records = completedFetch.fetchRecords(fetchConfig, 10); + records = completedFetch.fetchRecords(fetchConfig, deserializers, 10); assertEquals(0, records.size()); } @@ -96,21 +94,15 @@ public void testAbortedTransactionRecordsRemoved() { .setRecords(rawRecords) .setAbortedTransactions(newAbortedTransactions()); - try (final StringDeserializer deserializer = new StringDeserializer()) { - FetchConfig fetchConfig = newFetchConfig(deserializer, - deserializer, - IsolationLevel.READ_COMMITTED, - true); + try (final Deserializers deserializers = newStringDeserializers()) { + FetchConfig fetchConfig = newFetchConfig(IsolationLevel.READ_COMMITTED, true); CompletedFetch completedFetch = newCompletedFetch(0, partitionData); - List> records = completedFetch.fetchRecords(fetchConfig, 10); + List> records = completedFetch.fetchRecords(fetchConfig, deserializers, 10); assertEquals(0, records.size()); - fetchConfig = newFetchConfig(deserializer, - deserializer, - IsolationLevel.READ_UNCOMMITTED, - true); + fetchConfig = newFetchConfig(IsolationLevel.READ_UNCOMMITTED, true); completedFetch = newCompletedFetch(0, partitionData); - records = completedFetch.fetchRecords(fetchConfig, 10); + records = completedFetch.fetchRecords(fetchConfig, deserializers, 10); assertEquals(numRecords, records.size()); } } @@ -122,12 +114,9 @@ public void testCommittedTransactionRecordsIncluded() { FetchResponseData.PartitionData partitionData = new FetchResponseData.PartitionData() .setRecords(rawRecords); CompletedFetch completedFetch = newCompletedFetch(0, partitionData); - try (final StringDeserializer deserializer = new StringDeserializer()) { - FetchConfig fetchConfig = newFetchConfig(deserializer, - deserializer, - IsolationLevel.READ_COMMITTED, - true); - List> records = completedFetch.fetchRecords(fetchConfig, 10); + try (final Deserializers deserializers = newStringDeserializers()) { + FetchConfig fetchConfig = newFetchConfig(IsolationLevel.READ_COMMITTED, true); + List> records = completedFetch.fetchRecords(fetchConfig, deserializers, 10); assertEquals(10, records.size()); } } @@ -140,14 +129,13 @@ public void testNegativeFetchCount() { FetchResponseData.PartitionData partitionData = new FetchResponseData.PartitionData() .setRecords(newRecords(startingOffset, numRecords, fetchOffset)); - CompletedFetch completedFetch = newCompletedFetch(fetchOffset, partitionData); - FetchConfig fetchConfig = newFetchConfig(new StringDeserializer(), - new StringDeserializer(), - IsolationLevel.READ_UNCOMMITTED, - true); + try (final Deserializers deserializers = newStringDeserializers()) { + CompletedFetch completedFetch = newCompletedFetch(fetchOffset, partitionData); + FetchConfig fetchConfig = newFetchConfig(IsolationLevel.READ_UNCOMMITTED, true); - List> records = completedFetch.fetchRecords(fetchConfig, -10); - assertEquals(0, records.size()); + List> records = completedFetch.fetchRecords(fetchConfig, deserializers, -10); + assertEquals(0, records.size()); + } } @Test @@ -159,13 +147,9 @@ public void testNoRecordsInFetch() { .setLogStartOffset(0); CompletedFetch completedFetch = newCompletedFetch(1, partitionData); - try (final StringDeserializer deserializer = new StringDeserializer()) { - FetchConfig fetchConfig = newFetchConfig(deserializer, - deserializer, - IsolationLevel.READ_UNCOMMITTED, - true); - - List> records = completedFetch.fetchRecords(fetchConfig, 10); + try (final Deserializers deserializers = newStringDeserializers()) { + FetchConfig fetchConfig = newFetchConfig(IsolationLevel.READ_UNCOMMITTED, true); + List> records = completedFetch.fetchRecords(fetchConfig, deserializers, 10); assertEquals(0, records.size()); } } @@ -174,8 +158,7 @@ public void testNoRecordsInFetch() { public void testCorruptedMessage() { // Create one good record and then one "corrupted" record. try (final MemoryRecordsBuilder builder = MemoryRecords.builder(ByteBuffer.allocate(1024), CompressionType.NONE, TimestampType.CREATE_TIME, 0); - final UUIDSerializer serializer = new UUIDSerializer(); - final UUIDDeserializer deserializer = new UUIDDeserializer()) { + final UUIDSerializer serializer = new UUIDSerializer()) { builder.append(new SimpleRecord(serializer.serialize(TOPIC_NAME, UUID.randomUUID()))); builder.append(0L, "key".getBytes(), "value".getBytes()); Records records = builder.build(); @@ -187,16 +170,15 @@ public void testCorruptedMessage() { .setLogStartOffset(0) .setRecords(records); - FetchConfig fetchConfig = newFetchConfig(deserializer, - deserializer, - IsolationLevel.READ_COMMITTED, - false); - CompletedFetch completedFetch = newCompletedFetch(0, partitionData); + try (final Deserializers deserializers = newUuidDeserializers()) { + FetchConfig fetchConfig = newFetchConfig(IsolationLevel.READ_COMMITTED, false); + CompletedFetch completedFetch = newCompletedFetch(0, partitionData); - completedFetch.fetchRecords(fetchConfig, 10); + completedFetch.fetchRecords(fetchConfig, deserializers, 10); - assertThrows(RecordDeserializationException.class, - () -> completedFetch.fetchRecords(fetchConfig, 10)); + assertThrows(RecordDeserializationException.class, + () -> completedFetch.fetchRecords(fetchConfig, deserializers, 10)); + } } } @@ -219,11 +201,16 @@ private CompletedFetch newCompletedFetch(long fetchOffset, ApiKeys.FETCH.latestVersion()); } - private static FetchConfig newFetchConfig(Deserializer keyDeserializer, - Deserializer valueDeserializer, - IsolationLevel isolationLevel, - boolean checkCrcs) { - return new FetchConfig<>( + private static Deserializers newUuidDeserializers() { + return new Deserializers<>(new UUIDDeserializer(), new UUIDDeserializer()); + } + + private static Deserializers newStringDeserializers() { + return new Deserializers<>(new StringDeserializer(), new StringDeserializer()); + } + + private static FetchConfig newFetchConfig(IsolationLevel isolationLevel, boolean checkCrcs) { + return new FetchConfig( ConsumerConfig.DEFAULT_FETCH_MIN_BYTES, ConsumerConfig.DEFAULT_FETCH_MAX_BYTES, ConsumerConfig.DEFAULT_FETCH_MAX_WAIT_MS, @@ -231,7 +218,6 @@ private static FetchConfig newFetchConfig(Deserializer keyDeseri ConsumerConfig.DEFAULT_MAX_POLL_RECORDS, checkCrcs, ConsumerConfig.DEFAULT_CLIENT_RACK, - new Deserializers<>(keyDeserializer, valueDeserializer), isolationLevel ); } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java new file mode 100644 index 0000000000000..c054f283d260d --- /dev/null +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java @@ -0,0 +1,261 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.clients.consumer.internals; + +import org.apache.kafka.clients.ApiVersions; +import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.clients.GroupRebalanceConfig; +import org.apache.kafka.clients.MockClient; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor; +import org.apache.kafka.clients.consumer.internals.events.ApplicationEvent; +import org.apache.kafka.clients.consumer.internals.events.ApplicationEventProcessor; +import org.apache.kafka.clients.consumer.internals.events.BackgroundEvent; +import org.apache.kafka.clients.consumer.internals.events.BackgroundEventProcessor; +import org.apache.kafka.clients.consumer.internals.events.EventHandler; +import org.apache.kafka.common.IsolationLevel; +import org.apache.kafka.common.internals.ClusterResourceListeners; +import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.requests.MetadataResponse; +import org.apache.kafka.common.requests.RequestTestUtils; +import org.apache.kafka.common.serialization.StringDeserializer; +import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.common.utils.MockTime; +import org.apache.kafka.common.utils.Time; + +import java.io.Closeable; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Optional; +import java.util.Properties; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; + +import static org.apache.kafka.clients.consumer.ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG; +import static org.apache.kafka.clients.consumer.ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createFetchConfig; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createFetchMetricsManager; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createMetrics; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createSubscriptionState; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.configuredIsolationLevel; +import static org.mockito.Mockito.spy; + +public class ConsumerTestBuilder implements Closeable { + + static final long RETRY_BACKOFF_MS = 80; + static final long RETRY_BACKOFF_MAX_MS = 1000; + static final int REQUEST_TIMEOUT_MS = 500; + + final LogContext logContext = new LogContext(); + final Time time = new MockTime(0); + final BlockingQueue applicationEventQueue; + final LinkedBlockingQueue backgroundEventQueue; + final ConsumerConfig config; + final long retryBackoffMs; + final SubscriptionState subscriptions; + final ConsumerMetadata metadata; + final FetchConfig fetchConfig; + final Metrics metrics; + final FetchMetricsManager metricsManager; + final NetworkClientDelegate networkClientDelegate; + final OffsetsRequestManager offsetsRequestManager; + final CoordinatorRequestManager coordinatorRequestManager; + final CommitRequestManager commitRequestManager; + final FetchRequestManager fetchRequestManager; + final RequestManagers requestManagers; + final ApplicationEventProcessor applicationEventProcessor; + final BackgroundEventProcessor backgroundEventProcessor; + final MockClient client; + + public ConsumerTestBuilder() { + this.applicationEventQueue = new LinkedBlockingQueue<>(); + this.backgroundEventQueue = new LinkedBlockingQueue<>(); + ErrorEventHandler errorEventHandler = new ErrorEventHandler(backgroundEventQueue); + GroupRebalanceConfig groupRebalanceConfig = new GroupRebalanceConfig( + 100, + 100, + 100, + "group_id", + Optional.empty(), + RETRY_BACKOFF_MS, + RETRY_BACKOFF_MAX_MS, + true); + GroupState groupState = new GroupState(groupRebalanceConfig); + ApiVersions apiVersions = new ApiVersions(); + + Properties properties = new Properties(); + properties.put(KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); + properties.put(VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); + properties.put(CommonClientConfigs.RETRY_BACKOFF_MS_CONFIG, RETRY_BACKOFF_MS); + properties.put(CommonClientConfigs.REQUEST_TIMEOUT_MS_CONFIG, REQUEST_TIMEOUT_MS); + + this.config = new ConsumerConfig(properties); + IsolationLevel isolationLevel = configuredIsolationLevel(config); + this.retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG); + final long requestTimeoutMs = config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG); + this.metrics = createMetrics(config, time); + + this.subscriptions = spy(createSubscriptionState(config, logContext)); + this.metadata = spy(new ConsumerMetadata(config, subscriptions, logContext, new ClusterResourceListeners())); + this.fetchConfig = createFetchConfig(config); + this.metricsManager = createFetchMetricsManager(metrics); + + this.client = new MockClient(time, metadata); + MetadataResponse metadataResponse = RequestTestUtils.metadataUpdateWith(1, new HashMap() { + { + String topic1 = "test1"; + put(topic1, 1); + String topic2 = "test2"; + put(topic2, 1); + } + }); + this.client.updateMetadata(metadataResponse); + + this.networkClientDelegate = spy(new NetworkClientDelegate(time, + config, + logContext, + client)); + this.offsetsRequestManager = spy(new OffsetsRequestManager(subscriptions, + metadata, + isolationLevel, + time, + retryBackoffMs, + requestTimeoutMs, + apiVersions, + networkClientDelegate, + logContext)); + this.coordinatorRequestManager = spy(new CoordinatorRequestManager(time, + logContext, + RETRY_BACKOFF_MS, + RETRY_BACKOFF_MAX_MS, + errorEventHandler, + "group_id")); + this.commitRequestManager = spy(new CommitRequestManager(time, + logContext, + subscriptions, + config, + coordinatorRequestManager, + groupState)); + this.fetchRequestManager = spy(new FetchRequestManager(logContext, + time, + errorEventHandler, + metadata, + subscriptions, + fetchConfig, + metricsManager, + networkClientDelegate)); + this.requestManagers = new RequestManagers(logContext, + offsetsRequestManager, + fetchRequestManager, + Optional.of(coordinatorRequestManager), + Optional.of(commitRequestManager)); + this.applicationEventProcessor = spy(new ApplicationEventProcessor( + backgroundEventQueue, + requestManagers, + metadata, + logContext)); + this.backgroundEventProcessor = spy(new BackgroundEventProcessor(logContext, backgroundEventQueue)); + } + + @Override + public void close() { + requestManagers.close(); + } + + public static class DefaultBackgroundThreadTestBuilder extends ConsumerTestBuilder { + + final DefaultBackgroundThread backgroundThread; + + public DefaultBackgroundThreadTestBuilder() { + this.backgroundThread = new DefaultBackgroundThread( + time, + logContext, + applicationEventQueue, + () -> applicationEventProcessor, + () -> networkClientDelegate, + () -> requestManagers); + } + + @Override + public void close() { + backgroundThread.close(); + } + } + + public static class DefaultEventHandlerTestBuilder extends ConsumerTestBuilder { + + final EventHandler eventHandler; + + public DefaultEventHandlerTestBuilder() { + this.eventHandler = spy(new DefaultEventHandler( + time, + logContext, + applicationEventQueue, + () -> applicationEventProcessor, + () -> networkClientDelegate, + () -> requestManagers)); + } + + @Override + public void close() { + eventHandler.close(); + } + } + + public static class PrototypeAsyncConsumerTestBuilder extends DefaultEventHandlerTestBuilder { + + final PrototypeAsyncConsumer consumer; + + public PrototypeAsyncConsumerTestBuilder(Optional groupIdOpt) { + String clientId = config.getString(CommonClientConfigs.CLIENT_ID_CONFIG); + List assignors = ConsumerPartitionAssignor.getAssignorInstances( + config.getList(ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG), + config.originals(Collections.singletonMap(ConsumerConfig.CLIENT_ID_CONFIG, clientId)) + ); + FetchCollector fetchCollector = new FetchCollector<>(logContext, + metadata, + subscriptions, + fetchConfig, + metricsManager, + time); + this.consumer = spy(new PrototypeAsyncConsumer<>( + logContext, + clientId, + new Deserializers<>(new StringDeserializer(), new StringDeserializer()), + new FetchBuffer(logContext), + fetchCollector, + new ConsumerInterceptors<>(Collections.emptyList()), + time, + eventHandler, + backgroundEventQueue, + metrics, + subscriptions, + metadata, + retryBackoffMs, + REQUEST_TIMEOUT_MS, + 60000, + assignors, + groupIdOpt.orElse(null))); + } + + @Override + public void close() { + consumer.close(); + } + } +} diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThreadTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThreadTest.java index b5a1ced617ab1..825874d991d20 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThreadTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThreadTest.java @@ -16,29 +16,28 @@ */ package org.apache.kafka.clients.consumer.internals; -import org.apache.kafka.clients.GroupRebalanceConfig; -import org.apache.kafka.clients.consumer.ConsumerConfig; -import org.apache.kafka.clients.consumer.LogTruncationException; +import org.apache.kafka.clients.MockClient; import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.clients.consumer.internals.events.ApplicationEvent; import org.apache.kafka.clients.consumer.internals.events.ApplicationEventProcessor; import org.apache.kafka.clients.consumer.internals.events.AssignmentChangeApplicationEvent; -import org.apache.kafka.clients.consumer.internals.events.BackgroundEvent; import org.apache.kafka.clients.consumer.internals.events.CommitApplicationEvent; +import org.apache.kafka.clients.consumer.internals.events.CompletableApplicationEvent; import org.apache.kafka.clients.consumer.internals.events.ListOffsetsApplicationEvent; import org.apache.kafka.clients.consumer.internals.events.NewTopicsMetadataUpdateRequestEvent; import org.apache.kafka.clients.consumer.internals.events.NoopApplicationEvent; import org.apache.kafka.clients.consumer.internals.events.ResetPositionsApplicationEvent; -import org.apache.kafka.clients.consumer.internals.events.ValidatePositionsApplicationEvent; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.errors.TopicAuthorizationException; import org.apache.kafka.common.message.FindCoordinatorRequestData; +import org.apache.kafka.common.message.OffsetCommitRequestData; import org.apache.kafka.common.requests.FindCoordinatorRequest; -import org.apache.kafka.common.serialization.StringDeserializer; -import org.apache.kafka.common.utils.LogContext; -import org.apache.kafka.common.utils.MockTime; +import org.apache.kafka.common.requests.MetadataResponse; +import org.apache.kafka.common.requests.OffsetCommitRequest; +import org.apache.kafka.common.requests.RequestTestUtils; import org.apache.kafka.common.utils.Time; import org.apache.kafka.test.TestUtils; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mockito; @@ -48,87 +47,79 @@ import java.util.HashMap; import java.util.Map; import java.util.Optional; -import java.util.Properties; import java.util.concurrent.BlockingQueue; -import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.CompletableFuture; -import static org.apache.kafka.clients.consumer.ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG; -import static org.apache.kafka.clients.consumer.ConsumerConfig.RETRY_BACKOFF_MS_CONFIG; -import static org.apache.kafka.clients.consumer.ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; public class DefaultBackgroundThreadTest { - private static final long RETRY_BACKOFF_MS = 100; - private final Properties properties = new Properties(); - private MockTime time; + + private ConsumerTestBuilder.DefaultBackgroundThreadTestBuilder testBuilder; + private Time time; private ConsumerMetadata metadata; private NetworkClientDelegate networkClient; - private BlockingQueue backgroundEventsQueue; private BlockingQueue applicationEventsQueue; private ApplicationEventProcessor applicationEventProcessor; private CoordinatorRequestManager coordinatorManager; private OffsetsRequestManager offsetsRequestManager; - private ErrorEventHandler errorEventHandler; - private final int requestTimeoutMs = 500; - private GroupState groupState; private CommitRequestManager commitManager; + private DefaultBackgroundThread backgroundThread; + private MockClient client; @BeforeEach - @SuppressWarnings("unchecked") public void setup() { - this.time = new MockTime(0); - this.metadata = mock(ConsumerMetadata.class); - this.networkClient = mock(NetworkClientDelegate.class); - this.applicationEventsQueue = (BlockingQueue) mock(BlockingQueue.class); - this.backgroundEventsQueue = (BlockingQueue) mock(BlockingQueue.class); - this.applicationEventProcessor = mock(ApplicationEventProcessor.class); - this.coordinatorManager = mock(CoordinatorRequestManager.class); - this.offsetsRequestManager = mock(OffsetsRequestManager.class); - this.errorEventHandler = mock(ErrorEventHandler.class); - GroupRebalanceConfig rebalanceConfig = new GroupRebalanceConfig( - 100, - 100, - 100, - "group_id", - Optional.empty(), - 100, - 1000, - true); - this.groupState = new GroupState(rebalanceConfig); - this.commitManager = mock(CommitRequestManager.class); + this.testBuilder = new ConsumerTestBuilder.DefaultBackgroundThreadTestBuilder(); + this.time = testBuilder.time; + this.metadata = testBuilder.metadata; + this.networkClient = testBuilder.networkClientDelegate; + this.client = testBuilder.client; + this.applicationEventsQueue = testBuilder.applicationEventQueue; + this.applicationEventProcessor = testBuilder.applicationEventProcessor; + this.coordinatorManager = testBuilder.coordinatorRequestManager; + this.commitManager = testBuilder.commitRequestManager; + this.offsetsRequestManager = testBuilder.offsetsRequestManager; + this.backgroundThread = testBuilder.backgroundThread; + this.backgroundThread.initializeResources(); + } + + @AfterEach + public void tearDown() { + if (testBuilder != null) + testBuilder.close(); } @Test public void testStartupAndTearDown() throws InterruptedException { - when(coordinatorManager.poll(anyLong())).thenReturn(mockPollCoordinatorResult()); - when(commitManager.poll(anyLong())).thenReturn(mockPollCommitResult()); - when(offsetsRequestManager.poll(anyLong())).thenReturn(emptyPollOffsetsRequestResult()); - DefaultBackgroundThread backgroundThread = mockBackgroundThread(); + assertFalse(backgroundThread.isRunning()); backgroundThread.start(); - TestUtils.waitForCondition(backgroundThread::isRunning, "Failed awaiting for the background thread to be running"); + + // There's a nonzero amount of time between starting the thread and having it + // begin to execute our code. Wait for a bit before checking... + int maxWaitMs = 1000; + TestUtils.waitForCondition(backgroundThread::isRunning, + maxWaitMs, + "Thread did not start within " + maxWaitMs + " ms"); backgroundThread.close(); - assertFalse(backgroundThread.isRunning()); + TestUtils.waitForCondition(() -> !backgroundThread.isRunning(), + maxWaitMs, + "Thread did not stop within " + maxWaitMs + " ms"); } @Test public void testApplicationEvent() { - this.applicationEventsQueue = new LinkedBlockingQueue<>(); - this.backgroundEventsQueue = new LinkedBlockingQueue<>(); when(coordinatorManager.poll(anyLong())).thenReturn(mockPollCoordinatorResult()); when(commitManager.poll(anyLong())).thenReturn(mockPollCommitResult()); - when(offsetsRequestManager.poll(anyLong())).thenReturn(emptyPollOffsetsRequestResult()); - DefaultBackgroundThread backgroundThread = mockBackgroundThread(); ApplicationEvent e = new NoopApplicationEvent("noop event"); this.applicationEventsQueue.add(e); backgroundThread.runOnce(); @@ -138,16 +129,8 @@ public void testApplicationEvent() { @Test public void testMetadataUpdateEvent() { - this.applicationEventsQueue = new LinkedBlockingQueue<>(); - this.backgroundEventsQueue = new LinkedBlockingQueue<>(); - this.applicationEventProcessor = new ApplicationEventProcessor( - this.backgroundEventsQueue, - mockRequestManagers(), - metadata); when(coordinatorManager.poll(anyLong())).thenReturn(mockPollCoordinatorResult()); when(commitManager.poll(anyLong())).thenReturn(mockPollCommitResult()); - when(offsetsRequestManager.poll(anyLong())).thenReturn(emptyPollOffsetsRequestResult()); - DefaultBackgroundThread backgroundThread = mockBackgroundThread(); ApplicationEvent e = new NewTopicsMetadataUpdateRequestEvent(); this.applicationEventsQueue.add(e); backgroundThread.runOnce(); @@ -157,12 +140,8 @@ public void testMetadataUpdateEvent() { @Test public void testCommitEvent() { - this.applicationEventsQueue = new LinkedBlockingQueue<>(); - this.backgroundEventsQueue = new LinkedBlockingQueue<>(); when(coordinatorManager.poll(anyLong())).thenReturn(mockPollCoordinatorResult()); when(commitManager.poll(anyLong())).thenReturn(mockPollCommitResult()); - when(offsetsRequestManager.poll(anyLong())).thenReturn(emptyPollOffsetsRequestResult()); - DefaultBackgroundThread backgroundThread = mockBackgroundThread(); ApplicationEvent e = new CommitApplicationEvent(new HashMap<>()); this.applicationEventsQueue.add(e); backgroundThread.runOnce(); @@ -170,15 +149,8 @@ public void testCommitEvent() { backgroundThread.close(); } - @Test public void testListOffsetsEventIsProcessed() { - when(coordinatorManager.poll(anyLong())).thenReturn(mockPollCoordinatorResult()); - when(commitManager.poll(anyLong())).thenReturn(mockPollCommitResult()); - when(offsetsRequestManager.poll(anyLong())).thenReturn(emptyPollOffsetsRequestResult()); - this.applicationEventsQueue = new LinkedBlockingQueue<>(); - this.backgroundEventsQueue = new LinkedBlockingQueue<>(); - DefaultBackgroundThread backgroundThread = mockBackgroundThread(); Map timestamps = Collections.singletonMap(new TopicPartition("topic1", 1), 5L); ApplicationEvent e = new ListOffsetsApplicationEvent(timestamps, true); this.applicationEventsQueue.add(e); @@ -190,14 +162,8 @@ public void testListOffsetsEventIsProcessed() { @Test public void testResetPositionsEventIsProcessed() { - when(coordinatorManager.poll(anyLong())).thenReturn(mockPollCoordinatorResult()); - when(commitManager.poll(anyLong())).thenReturn(mockPollCommitResult()); - when(offsetsRequestManager.poll(anyLong())).thenReturn(emptyPollOffsetsRequestResult()); - this.applicationEventsQueue = new LinkedBlockingQueue<>(); - this.backgroundEventsQueue = new LinkedBlockingQueue<>(); - DefaultBackgroundThread backgroundThread = mockBackgroundThread(); - ResetPositionsApplicationEvent e = new ResetPositionsApplicationEvent(); - this.applicationEventsQueue.add(e); + ResetPositionsApplicationEvent event = new ResetPositionsApplicationEvent(); + this.applicationEventsQueue.add(event); backgroundThread.runOnce(); verify(applicationEventProcessor).process(any(ResetPositionsApplicationEvent.class)); assertTrue(applicationEventsQueue.isEmpty()); @@ -205,79 +171,20 @@ public void testResetPositionsEventIsProcessed() { } @Test - public void testResetPositionsProcessFailure() { - when(coordinatorManager.poll(anyLong())).thenReturn(mockPollCoordinatorResult()); - when(commitManager.poll(anyLong())).thenReturn(mockPollCommitResult()); - when(offsetsRequestManager.poll(anyLong())).thenReturn(emptyPollOffsetsRequestResult()); - this.applicationEventsQueue = new LinkedBlockingQueue<>(); - this.backgroundEventsQueue = new LinkedBlockingQueue<>(); - applicationEventProcessor = spy(new ApplicationEventProcessor( - this.backgroundEventsQueue, - mockRequestManagers(), - metadata)); - DefaultBackgroundThread backgroundThread = mockBackgroundThread(); - + public void testResetPositionsProcessFailureInterruptsBackgroundThread() { TopicAuthorizationException authException = new TopicAuthorizationException("Topic authorization failed"); doThrow(authException).when(offsetsRequestManager).resetPositionsIfNeeded(); ResetPositionsApplicationEvent event = new ResetPositionsApplicationEvent(); this.applicationEventsQueue.add(event); - assertThrows(TopicAuthorizationException.class, backgroundThread::runOnce); + assertThrows(TopicAuthorizationException.class, () -> backgroundThread.runOnce()); verify(applicationEventProcessor).process(any(ResetPositionsApplicationEvent.class)); backgroundThread.close(); } - @Test - public void testValidatePositionsEventIsProcessed() { - when(coordinatorManager.poll(anyLong())).thenReturn(mockPollCoordinatorResult()); - when(commitManager.poll(anyLong())).thenReturn(mockPollCommitResult()); - when(offsetsRequestManager.poll(anyLong())).thenReturn(emptyPollOffsetsRequestResult()); - this.applicationEventsQueue = new LinkedBlockingQueue<>(); - this.backgroundEventsQueue = new LinkedBlockingQueue<>(); - DefaultBackgroundThread backgroundThread = mockBackgroundThread(); - ValidatePositionsApplicationEvent e = new ValidatePositionsApplicationEvent(); - this.applicationEventsQueue.add(e); - backgroundThread.runOnce(); - verify(applicationEventProcessor).process(any(ValidatePositionsApplicationEvent.class)); - assertTrue(applicationEventsQueue.isEmpty()); - backgroundThread.close(); - } - - @Test - public void testValidatePositionsProcessFailure() { - when(coordinatorManager.poll(anyLong())).thenReturn(mockPollCoordinatorResult()); - when(commitManager.poll(anyLong())).thenReturn(mockPollCommitResult()); - when(offsetsRequestManager.poll(anyLong())).thenReturn(emptyPollOffsetsRequestResult()); - this.applicationEventsQueue = new LinkedBlockingQueue<>(); - this.backgroundEventsQueue = new LinkedBlockingQueue<>(); - applicationEventProcessor = spy(new ApplicationEventProcessor( - this.backgroundEventsQueue, - mockRequestManagers(), - metadata)); - DefaultBackgroundThread backgroundThread = mockBackgroundThread(); - - LogTruncationException logTruncationException = new LogTruncationException(Collections.emptyMap(), Collections.emptyMap()); - doThrow(logTruncationException).when(offsetsRequestManager).validatePositionsIfNeeded(); - - ValidatePositionsApplicationEvent event = new ValidatePositionsApplicationEvent(); - this.applicationEventsQueue.add(event); - assertThrows(LogTruncationException.class, backgroundThread::runOnce); - - verify(applicationEventProcessor).process(any(ValidatePositionsApplicationEvent.class)); - backgroundThread.close(); - } - @Test public void testAssignmentChangeEvent() { - this.applicationEventsQueue = new LinkedBlockingQueue<>(); - this.backgroundEventsQueue = new LinkedBlockingQueue<>(); - this.applicationEventProcessor = spy(new ApplicationEventProcessor( - this.backgroundEventsQueue, - mockRequestManagers(), - metadata)); - - DefaultBackgroundThread backgroundThread = mockBackgroundThread(); HashMap offset = mockTopicPartitionOffset(); final long currentTimeMs = time.milliseconds(); @@ -286,7 +193,6 @@ public void testAssignmentChangeEvent() { when(this.coordinatorManager.poll(anyLong())).thenReturn(mockPollCoordinatorResult()); when(this.commitManager.poll(anyLong())).thenReturn(mockPollCommitResult()); - when(this.offsetsRequestManager.poll(anyLong())).thenReturn(emptyPollOffsetsRequestResult()); backgroundThread.runOnce(); verify(applicationEventProcessor).process(any(AssignmentChangeApplicationEvent.class)); @@ -299,10 +205,8 @@ public void testAssignmentChangeEvent() { @Test void testFindCoordinator() { - DefaultBackgroundThread backgroundThread = mockBackgroundThread(); - when(this.coordinatorManager.poll(anyLong())).thenReturn(mockPollCoordinatorResult()); - when(this.commitManager.poll(anyLong())).thenReturn(mockPollCommitResult()); - when(this.offsetsRequestManager.poll(anyLong())).thenReturn(emptyPollOffsetsRequestResult()); + when(coordinatorManager.poll(anyLong())).thenReturn(mockPollCoordinatorResult()); + when(commitManager.poll(anyLong())).thenReturn(mockPollCommitResult()); backgroundThread.runOnce(); Mockito.verify(coordinatorManager, times(1)).poll(anyLong()); Mockito.verify(networkClient, times(1)).poll(anyLong(), anyLong()); @@ -311,11 +215,10 @@ void testFindCoordinator() { @Test void testPollResultTimer() { - DefaultBackgroundThread backgroundThread = mockBackgroundThread(); // purposely setting a non MAX time to ensure it is returning Long.MAX_VALUE upon success NetworkClientDelegate.PollResult success = new NetworkClientDelegate.PollResult( 10, - Collections.singletonList(findCoordinatorUnsentRequest(time, requestTimeoutMs))); + Collections.singletonList(findCoordinatorUnsentRequest())); assertEquals(10, backgroundThread.handlePollResult(success)); NetworkClientDelegate.PollResult failure = new NetworkClientDelegate.PollResult( @@ -324,6 +227,41 @@ void testPollResultTimer() { assertEquals(10, backgroundThread.handlePollResult(failure)); } + @Test + void testRequestManagersArePolledOnce() { + backgroundThread.runOnce(); + testBuilder.requestManagers.entries().forEach(requestManager -> + verify(requestManager.get(), times(1)).poll(anyLong())); + verify(networkClient, times(1)).poll(anyLong(), anyLong()); + backgroundThread.close(); + } + + @Test + void testEnsureMetadataUpdateOnPoll() { + MetadataResponse metadataResponse = RequestTestUtils.metadataUpdateWith(2, Collections.emptyMap()); + client.prepareMetadataUpdate(metadataResponse); + metadata.requestUpdate(false); + backgroundThread.runOnce(); + verify(this.metadata, times(1)).updateWithCurrentRequestVersion(eq(metadataResponse), eq(false), anyLong()); + backgroundThread.close(); + } + + @Test + void testEnsureEventsAreCompleted() { + CompletableApplicationEvent event = mock(CompletableApplicationEvent.class); + ApplicationEvent e = mock(ApplicationEvent.class); + CompletableFuture future = new CompletableFuture<>(); + when(event.future()).thenReturn(future); + applicationEventsQueue.add(event); + applicationEventsQueue.add(e); + assertFalse(future.isDone()); + assertFalse(applicationEventsQueue.isEmpty()); + + backgroundThread.close(); + assertTrue(future.isCompletedExceptionally()); + assertTrue(applicationEventsQueue.isEmpty()); + } + private HashMap mockTopicPartitionOffset() { final TopicPartition t0 = new TopicPartition("t0", 2); final TopicPartition t1 = new TopicPartition("t0", 3); @@ -333,61 +271,38 @@ private HashMap mockTopicPartitionOffset() { return topicPartitionOffsets; } - private RequestManagers mockRequestManagers() { - return new RequestManagers( - offsetsRequestManager, - Optional.of(coordinatorManager), - Optional.of(commitManager)); - } - - private static NetworkClientDelegate.UnsentRequest findCoordinatorUnsentRequest( - final Time time, - final long timeout - ) { + private NetworkClientDelegate.UnsentRequest findCoordinatorUnsentRequest() { NetworkClientDelegate.UnsentRequest req = new NetworkClientDelegate.UnsentRequest( new FindCoordinatorRequest.Builder( new FindCoordinatorRequestData() .setKeyType(FindCoordinatorRequest.CoordinatorType.TRANSACTION.id()) .setKey("foobar")), Optional.empty()); - req.setTimer(time, timeout); + req.setTimer(time, ConsumerTestBuilder.REQUEST_TIMEOUT_MS); return req; } - private DefaultBackgroundThread mockBackgroundThread() { - properties.put(KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); - properties.put(VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); - properties.put(RETRY_BACKOFF_MS_CONFIG, RETRY_BACKOFF_MS); - - return new DefaultBackgroundThread( - this.time, - new ConsumerConfig(properties), - new LogContext(), - applicationEventsQueue, - backgroundEventsQueue, - this.errorEventHandler, - applicationEventProcessor, - this.metadata, - this.networkClient, - this.groupState, - this.coordinatorManager, - this.commitManager, - this.offsetsRequestManager); - } - private NetworkClientDelegate.PollResult mockPollCoordinatorResult() { return new NetworkClientDelegate.PollResult( - RETRY_BACKOFF_MS, - Collections.singletonList(findCoordinatorUnsentRequest(time, requestTimeoutMs))); + ConsumerTestBuilder.RETRY_BACKOFF_MS, + Collections.singletonList(findCoordinatorUnsentRequest())); } private NetworkClientDelegate.PollResult mockPollCommitResult() { return new NetworkClientDelegate.PollResult( - RETRY_BACKOFF_MS, - Collections.singletonList(findCoordinatorUnsentRequest(time, requestTimeoutMs))); + ConsumerTestBuilder.RETRY_BACKOFF_MS, + Collections.singletonList(offsetCommitUnsentRequest())); } - private NetworkClientDelegate.PollResult emptyPollOffsetsRequestResult() { - return new NetworkClientDelegate.PollResult(Long.MAX_VALUE, Collections.emptyList()); + private NetworkClientDelegate.UnsentRequest offsetCommitUnsentRequest() { + NetworkClientDelegate.UnsentRequest req = new NetworkClientDelegate.UnsentRequest( + new OffsetCommitRequest.Builder( + new OffsetCommitRequestData() + .setGroupId("groupId") + .setMemberId("m1") + .setGroupInstanceId("i1") + .setTopics(new ArrayList<>())), Optional.empty()); + req.setTimer(time, ConsumerTestBuilder.REQUEST_TIMEOUT_MS); + return req; } -} \ No newline at end of file +} diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/DefaultEventHandlerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/DefaultEventHandlerTest.java index 7e4de791da66a..1bbd1d1bab083 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/DefaultEventHandlerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/DefaultEventHandlerTest.java @@ -16,66 +16,39 @@ */ package org.apache.kafka.clients.consumer.internals; -import org.apache.kafka.clients.GroupRebalanceConfig; import org.apache.kafka.clients.consumer.internals.events.ApplicationEvent; -import org.apache.kafka.clients.consumer.internals.events.BackgroundEvent; +import org.apache.kafka.clients.consumer.internals.events.EventHandler; import org.apache.kafka.clients.consumer.internals.events.NoopApplicationEvent; -import org.apache.kafka.common.serialization.StringDeserializer; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import java.util.Optional; -import java.util.Properties; import java.util.concurrent.BlockingQueue; -import java.util.concurrent.LinkedBlockingQueue; -import static org.apache.kafka.clients.consumer.ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG; -import static org.apache.kafka.clients.consumer.ConsumerConfig.RETRY_BACKOFF_MS_CONFIG; -import static org.apache.kafka.clients.consumer.ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; public class DefaultEventHandlerTest { - private int sessionTimeoutMs = 1000; - private int rebalanceTimeoutMs = 1000; - private int heartbeatIntervalMs = 1000; - private String groupId = "g-1"; - private Optional groupInstanceId = Optional.of("g-1"); - private long retryBackoffMs = 1000; - private final Properties properties = new Properties(); - private GroupRebalanceConfig rebalanceConfig; + + private ConsumerTestBuilder.DefaultEventHandlerTestBuilder testBuilder; + private EventHandler handler; + private BlockingQueue aq; @BeforeEach public void setup() { - properties.put(KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); - properties.put(VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); - properties.put(RETRY_BACKOFF_MS_CONFIG, "100"); + testBuilder = new ConsumerTestBuilder.DefaultEventHandlerTestBuilder(); + handler = testBuilder.eventHandler; + aq = testBuilder.applicationEventQueue; + } - this.rebalanceConfig = new GroupRebalanceConfig(sessionTimeoutMs, - rebalanceTimeoutMs, - heartbeatIntervalMs, - groupId, - groupInstanceId, - retryBackoffMs, - retryBackoffMs, - true); + @AfterEach + public void tearDown() { + if (testBuilder != null) + testBuilder.close(); } @Test public void testBasicHandlerOps() { - final DefaultBackgroundThread bt = mock(DefaultBackgroundThread.class); - final BlockingQueue aq = new LinkedBlockingQueue<>(); - final BlockingQueue bq = new LinkedBlockingQueue<>(); - final DefaultEventHandler handler = new DefaultEventHandler(bt, aq, bq); - assertTrue(handler.isEmpty()); - assertFalse(handler.poll().isPresent()); handler.add(new NoopApplicationEvent("test")); assertEquals(1, aq.size()); - handler.close(); - verify(bt, times(1)).close(); } } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ErrorEventHandlerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ErrorEventHandlerTest.java new file mode 100644 index 0000000000000..61e2e95a15aef --- /dev/null +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ErrorEventHandlerTest.java @@ -0,0 +1,165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.clients.consumer.internals; + +import org.apache.kafka.clients.consumer.internals.events.BackgroundEvent; +import org.apache.kafka.clients.consumer.internals.events.BackgroundEventProcessor; +import org.apache.kafka.clients.consumer.internals.events.ErrorBackgroundEvent; +import org.apache.kafka.clients.consumer.internals.events.NoopBackgroundEvent; +import org.apache.kafka.common.KafkaException; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.concurrent.BlockingQueue; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +public class ErrorEventHandlerTest { + + private ConsumerTestBuilder.DefaultEventHandlerTestBuilder testBuilder; + private BlockingQueue backgroundEventQueue; + private ErrorEventHandler errorEventHandler; + private BackgroundEventProcessor backgroundEventProcessor; + + @BeforeEach + public void setup() { + testBuilder = new ConsumerTestBuilder.DefaultEventHandlerTestBuilder(); + backgroundEventQueue = testBuilder.backgroundEventQueue; + errorEventHandler = new ErrorEventHandler(backgroundEventQueue); + backgroundEventProcessor = new BackgroundEventProcessor(testBuilder.logContext, backgroundEventQueue); + } + + @AfterEach + public void tearDown() { + if (testBuilder != null) + testBuilder.close(); + } + + @Test + public void testNoEvents() { + assertTrue(backgroundEventQueue.isEmpty()); + backgroundEventProcessor.process(); + assertTrue(backgroundEventQueue.isEmpty()); + } + + @Test + public void testSingleEvent() { + BackgroundEvent event = new NoopBackgroundEvent("A"); + backgroundEventQueue.add(event); + assertPeeked(event); + backgroundEventProcessor.process(); + assertTrue(backgroundEventQueue.isEmpty()); + } + + @Test + public void testSingleErrorEvent() { + KafkaException error = new KafkaException("error"); + BackgroundEvent event = new ErrorBackgroundEvent(error); + errorEventHandler.handle(error); + assertPeeked(event); + assertThrows(error); + } + + @Test + public void testInvalidEvent() { + String message = "I'm a naughty error!"; + BackgroundEvent event = new BackgroundEvent(BackgroundEvent.Type.NOOP) { + @Override + public Type type() { + return null; + } + + @Override + public String toString() { + return message; + } + }; + + backgroundEventQueue.add(event); + assertPeeked(event); + + Exception error = new NullPointerException(String.format("Attempt to process BackgroundEvent with null type: %s", message)); + assertThrows(error); + } + + @Test + public void testMultipleEvents() { + BackgroundEvent event1 = new NoopBackgroundEvent("A"); + backgroundEventQueue.add(event1); + backgroundEventQueue.add(new NoopBackgroundEvent("B")); + backgroundEventQueue.add(new NoopBackgroundEvent("C")); + + assertPeeked(event1); + backgroundEventProcessor.process(); + assertTrue(backgroundEventQueue.isEmpty()); + } + + @Test + public void testMultipleErrorEvents() { + Throwable error1 = new Throwable("error1"); + KafkaException error2 = new KafkaException("error2"); + KafkaException error3 = new KafkaException("error3"); + + errorEventHandler.handle(error1); + errorEventHandler.handle(error2); + errorEventHandler.handle(error3); + + assertThrows(new RuntimeException(error1)); + } + + @Test + public void testMixedEventsWithErrorEvents() { + Throwable error1 = new Throwable("error1"); + KafkaException error2 = new KafkaException("error2"); + KafkaException error3 = new KafkaException("error3"); + + backgroundEventQueue.add(new NoopBackgroundEvent("A")); + errorEventHandler.handle(error1); + backgroundEventQueue.add(new NoopBackgroundEvent("B")); + errorEventHandler.handle(error2); + backgroundEventQueue.add(new NoopBackgroundEvent("C")); + errorEventHandler.handle(error3); + backgroundEventQueue.add(new NoopBackgroundEvent("D")); + + assertThrows(new RuntimeException(error1)); + } + + private void assertPeeked(BackgroundEvent event) { + BackgroundEvent peekEvent = backgroundEventQueue.peek(); + assertNotNull(peekEvent); + assertEquals(event, peekEvent); + } + + private void assertThrows(Throwable error) { + assertFalse(backgroundEventQueue.isEmpty()); + + try { + backgroundEventProcessor.process(); + fail("Should have thrown error: " + error); + } catch (Throwable t) { + assertEquals(error.getClass(), t.getClass()); + assertEquals(error.getMessage(), t.getMessage()); + } + + assertTrue(backgroundEventQueue.isEmpty()); + } +} diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchCollectorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchCollectorTest.java index 1f157b84189da..66ed1d952bcd6 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchCollectorTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchCollectorTest.java @@ -79,10 +79,11 @@ public class FetchCollectorTest { private LogContext logContext; private SubscriptionState subscriptions; - private FetchConfig fetchConfig; + private FetchConfig fetchConfig; private FetchMetricsManager metricsManager; private ConsumerMetadata metadata; private FetchBuffer fetchBuffer; + private Deserializers deserializers; private FetchCollector fetchCollector; private CompletedFetchBuilder completedFetchBuilder; @@ -105,7 +106,7 @@ public void testFetchNormal() { assertFalse(completedFetch.isInitialized()); // Fetch the data and validate that we get all the records we want back. - Fetch fetch = fetchCollector.collectFetch(fetchBuffer); + Fetch fetch = fetchCollector.collectFetch(fetchBuffer, deserializers); assertFalse(fetch.isEmpty()); assertEquals(recordCount, fetch.numRecords()); @@ -129,7 +130,7 @@ public void testFetchNormal() { assertEquals(recordCount, position.offset); // Now attempt to collect more records from the fetch buffer. - fetch = fetchCollector.collectFetch(fetchBuffer); + fetch = fetchCollector.collectFetch(fetchBuffer, deserializers); // The Fetch object is non-null, but it's empty. assertEquals(0, fetch.numRecords()); @@ -153,7 +154,7 @@ public void testFetchWithReadReplica() { CompletedFetch completedFetch = completedFetchBuilder.build(); fetchBuffer.add(completedFetch); - Fetch fetch = fetchCollector.collectFetch(fetchBuffer); + Fetch fetch = fetchCollector.collectFetch(fetchBuffer, deserializers); // The Fetch and read replica settings should be empty. assertEquals(DEFAULT_RECORD_COUNT, fetch.numRecords()); @@ -176,7 +177,7 @@ public void testNoResultsIfInitializing() { // Add some valid CompletedFetch records to the FetchBuffer queue and collect them into the Fetch. CompletedFetch completedFetch = completedFetchBuilder.build(); fetchBuffer.add(completedFetch); - Fetch fetch = fetchCollector.collectFetch(fetchBuffer); + Fetch fetch = fetchCollector.collectFetch(fetchBuffer, deserializers); // Verify that no records are fetched for the partition as it did not have a valid position set. assertEquals(0, fetch.numRecords()); @@ -212,7 +213,7 @@ protected CompletedFetch initialize(final CompletedFetch completedFetch) { assertFalse(fetchBuffer.isEmpty()); // Now run our ill-fated collectFetch. - assertThrows(expectedException.getClass(), () -> fetchCollector.collectFetch(fetchBuffer)); + assertThrows(expectedException.getClass(), () -> fetchCollector.collectFetch(fetchBuffer, deserializers)); // If the number of records in the CompletedFetch was 0, the call to FetchCollector.collectFetch() will // remove it from the queue. If there are records in the CompletedFetch, FetchCollector.collectFetch will @@ -245,7 +246,7 @@ public void testFetchingPausedPartitionsYieldsNoRecords() { // Ensure that the partition for the next-in-line CompletedFetch is still 'paused'. assertTrue(subscriptions.isPaused(completedFetch.partition)); - Fetch fetch = fetchCollector.collectFetch(fetchBuffer); + Fetch fetch = fetchCollector.collectFetch(fetchBuffer, deserializers); // There should be no records in the Fetch as the partition being fetched is 'paused'. assertEquals(0, fetch.numRecords()); @@ -277,7 +278,7 @@ public void testFetchWithMetadataRefreshErrors(final Errors error) { assertEquals(Optional.of(preferredReadReplicaId), subscriptions.preferredReadReplica(topicAPartition0, time.milliseconds())); // Fetch the data and validate that we get all the records we want back. - Fetch fetch = fetchCollector.collectFetch(fetchBuffer); + Fetch fetch = fetchCollector.collectFetch(fetchBuffer, deserializers); assertTrue(fetch.isEmpty()); assertTrue(metadata.updateRequested()); assertEquals(Optional.empty(), subscriptions.preferredReadReplica(topicAPartition0, time.milliseconds())); @@ -292,7 +293,7 @@ public void testFetchWithOffsetOutOfRange() { fetchBuffer.add(completedFetch); // Fetch the data and validate that we get our first batch of records back. - Fetch fetch = fetchCollector.collectFetch(fetchBuffer); + Fetch fetch = fetchCollector.collectFetch(fetchBuffer, deserializers); assertFalse(fetch.isEmpty()); assertEquals(DEFAULT_RECORD_COUNT, fetch.numRecords()); @@ -302,7 +303,7 @@ public void testFetchWithOffsetOutOfRange() { .error(Errors.OFFSET_OUT_OF_RANGE) .build(); fetchBuffer.add(completedFetch); - fetch = fetchCollector.collectFetch(fetchBuffer); + fetch = fetchCollector.collectFetch(fetchBuffer, deserializers); assertTrue(fetch.isEmpty()); // Try to fetch more data and validate that we get an empty Fetch back. @@ -311,7 +312,7 @@ public void testFetchWithOffsetOutOfRange() { .error(Errors.OFFSET_OUT_OF_RANGE) .build(); fetchBuffer.add(completedFetch); - fetch = fetchCollector.collectFetch(fetchBuffer); + fetch = fetchCollector.collectFetch(fetchBuffer, deserializers); assertTrue(fetch.isEmpty()); } @@ -331,7 +332,7 @@ public void testFetchWithOffsetOutOfRangeWithPreferredReadReplica() { .error(Errors.OFFSET_OUT_OF_RANGE) .build(); fetchBuffer.add(completedFetch); - Fetch fetch = fetchCollector.collectFetch(fetchBuffer); + Fetch fetch = fetchCollector.collectFetch(fetchBuffer, deserializers); // The Fetch and read replica settings should be empty. assertTrue(fetch.isEmpty()); @@ -348,7 +349,7 @@ public void testFetchWithTopicAuthorizationFailed() { .error(Errors.TOPIC_AUTHORIZATION_FAILED) .build(); fetchBuffer.add(completedFetch); - assertThrows(TopicAuthorizationException.class, () -> fetchCollector.collectFetch(fetchBuffer)); + assertThrows(TopicAuthorizationException.class, () -> fetchCollector.collectFetch(fetchBuffer, deserializers)); } @Test @@ -361,7 +362,7 @@ public void testFetchWithUnknownLeaderEpoch() { .error(Errors.UNKNOWN_LEADER_EPOCH) .build(); fetchBuffer.add(completedFetch); - Fetch fetch = fetchCollector.collectFetch(fetchBuffer); + Fetch fetch = fetchCollector.collectFetch(fetchBuffer, deserializers); assertTrue(fetch.isEmpty()); } @@ -375,7 +376,7 @@ public void testFetchWithUnknownServerError() { .error(Errors.UNKNOWN_SERVER_ERROR) .build(); fetchBuffer.add(completedFetch); - Fetch fetch = fetchCollector.collectFetch(fetchBuffer); + Fetch fetch = fetchCollector.collectFetch(fetchBuffer, deserializers); assertTrue(fetch.isEmpty()); } @@ -389,7 +390,7 @@ public void testFetchWithCorruptMessage() { .error(Errors.CORRUPT_MESSAGE) .build(); fetchBuffer.add(completedFetch); - assertThrows(KafkaException.class, () -> fetchCollector.collectFetch(fetchBuffer)); + assertThrows(KafkaException.class, () -> fetchCollector.collectFetch(fetchBuffer, deserializers)); } @ParameterizedTest @@ -402,7 +403,7 @@ public void testFetchWithOtherErrors(final Errors error) { .error(error) .build(); fetchBuffer.add(completedFetch); - assertThrows(IllegalStateException.class, () -> fetchCollector.collectFetch(fetchBuffer)); + assertThrows(IllegalStateException.class, () -> fetchCollector.collectFetch(fetchBuffer, deserializers)); } /** @@ -427,10 +428,10 @@ private void buildDependencies(int maxPollRecords) { ConsumerConfig config = new ConsumerConfig(p); - Deserializers deserializers = new Deserializers<>(new StringDeserializer(), new StringDeserializer()); + deserializers = new Deserializers<>(new StringDeserializer(), new StringDeserializer()); subscriptions = createSubscriptionState(config, logContext); - fetchConfig = createFetchConfig(config, deserializers); + fetchConfig = createFetchConfig(config); Metrics metrics = createMetrics(config, time); metricsManager = createFetchMetricsManager(metrics); diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchConfigTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchConfigTest.java index 1f89822fdb253..c5d0755d2a107 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchConfigTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchConfigTest.java @@ -19,14 +19,11 @@ import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.common.IsolationLevel; import org.apache.kafka.common.serialization.Deserializer; -import org.apache.kafka.common.serialization.StringDeserializer; import org.apache.kafka.common.serialization.StringSerializer; import org.junit.jupiter.api.Test; import java.util.Properties; -import static org.junit.jupiter.api.Assertions.assertThrows; - public class FetchConfigTest { /** @@ -35,60 +32,27 @@ public class FetchConfigTest { */ @Test public void testBasicFromConsumerConfig() { - try (StringDeserializer keyDeserializer = new StringDeserializer(); StringDeserializer valueDeserializer = new StringDeserializer()) { - newFetchConfigFromConsumerConfig(keyDeserializer, valueDeserializer); - newFetchConfigFromValues(keyDeserializer, valueDeserializer); - } - } - - /** - * Verify an exception is thrown if the key {@link Deserializer deserializer} provided to the - * {@link FetchConfig} constructors is {@code null}. - */ - @Test - public void testPreventNullKeyDeserializer() { - try (StringDeserializer valueDeserializer = new StringDeserializer()) { - assertThrows(NullPointerException.class, () -> newFetchConfigFromConsumerConfig(null, valueDeserializer)); - assertThrows(NullPointerException.class, () -> newFetchConfigFromValues(null, valueDeserializer)); - } + newFetchConfigFromConsumerConfig(); + newFetchConfigFromValues(); } - /** - * Verify an exception is thrown if the value {@link Deserializer deserializer} provided to the - * {@link FetchConfig} constructors is {@code null}. - */ - @Test - @SuppressWarnings("resources") - public void testPreventNullValueDeserializer() { - try (StringDeserializer keyDeserializer = new StringDeserializer()) { - assertThrows(NullPointerException.class, () -> newFetchConfigFromConsumerConfig(keyDeserializer, null)); - assertThrows(NullPointerException.class, () -> newFetchConfigFromValues(keyDeserializer, null)); - } - } - - private void newFetchConfigFromConsumerConfig(Deserializer keyDeserializer, - Deserializer valueDeserializer) { + private void newFetchConfigFromConsumerConfig() { Properties p = new Properties(); p.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092"); p.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); p.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); ConsumerConfig config = new ConsumerConfig(p); - new FetchConfig<>( - config, - new Deserializers<>(keyDeserializer, valueDeserializer), - IsolationLevel.READ_UNCOMMITTED); + new FetchConfig(config, IsolationLevel.READ_UNCOMMITTED); } - private void newFetchConfigFromValues(Deserializer keyDeserializer, - Deserializer valueDeserializer) { - new FetchConfig<>(ConsumerConfig.DEFAULT_FETCH_MIN_BYTES, + private void newFetchConfigFromValues() { + new FetchConfig(ConsumerConfig.DEFAULT_FETCH_MIN_BYTES, ConsumerConfig.DEFAULT_FETCH_MAX_BYTES, ConsumerConfig.DEFAULT_FETCH_MAX_WAIT_MS, ConsumerConfig.DEFAULT_MAX_PARTITION_FETCH_BYTES, ConsumerConfig.DEFAULT_MAX_POLL_RECORDS, true, ConsumerConfig.DEFAULT_CLIENT_RACK, - new Deserializers<>(keyDeserializer, valueDeserializer), IsolationLevel.READ_UNCOMMITTED); } } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java new file mode 100644 index 0000000000000..2a74f9441f9dc --- /dev/null +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java @@ -0,0 +1,3566 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.clients.consumer.internals; + +import org.apache.kafka.clients.ApiVersions; +import org.apache.kafka.clients.ClientRequest; +import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.clients.KafkaClient; +import org.apache.kafka.clients.Metadata; +import org.apache.kafka.clients.MockClient; +import org.apache.kafka.clients.NetworkClient; +import org.apache.kafka.clients.NodeApiVersions; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRebalanceListener; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.OffsetOutOfRangeException; +import org.apache.kafka.clients.consumer.OffsetResetStrategy; +import org.apache.kafka.common.Cluster; +import org.apache.kafka.common.IsolationLevel; +import org.apache.kafka.common.KafkaException; +import org.apache.kafka.common.MetricName; +import org.apache.kafka.common.MetricNameTemplate; +import org.apache.kafka.common.Node; +import org.apache.kafka.common.TopicIdPartition; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.errors.DisconnectException; +import org.apache.kafka.common.errors.RecordTooLargeException; +import org.apache.kafka.common.errors.SerializationException; +import org.apache.kafka.common.errors.TopicAuthorizationException; +import org.apache.kafka.common.header.Header; +import org.apache.kafka.common.header.internals.RecordHeader; +import org.apache.kafka.common.internals.ClusterResourceListeners; +import org.apache.kafka.common.message.ApiMessageType; +import org.apache.kafka.common.message.FetchResponseData; +import org.apache.kafka.common.message.OffsetForLeaderEpochResponseData; +import org.apache.kafka.common.message.OffsetForLeaderEpochResponseData.EpochEndOffset; +import org.apache.kafka.common.message.OffsetForLeaderEpochResponseData.OffsetForLeaderTopicResult; +import org.apache.kafka.common.metrics.KafkaMetric; +import org.apache.kafka.common.metrics.MetricConfig; +import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.network.NetworkReceive; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.record.CompressionType; +import org.apache.kafka.common.record.ControlRecordType; +import org.apache.kafka.common.record.DefaultRecordBatch; +import org.apache.kafka.common.record.EndTransactionMarker; +import org.apache.kafka.common.record.LegacyRecord; +import org.apache.kafka.common.record.MemoryRecords; +import org.apache.kafka.common.record.MemoryRecordsBuilder; +import org.apache.kafka.common.record.Record; +import org.apache.kafka.common.record.RecordBatch; +import org.apache.kafka.common.record.Records; +import org.apache.kafka.common.record.SimpleRecord; +import org.apache.kafka.common.record.TimestampType; +import org.apache.kafka.common.requests.ApiVersionsResponse; +import org.apache.kafka.common.requests.FetchMetadata; +import org.apache.kafka.common.requests.FetchRequest; +import org.apache.kafka.common.requests.FetchRequest.PartitionData; +import org.apache.kafka.common.requests.FetchResponse; +import org.apache.kafka.common.requests.MetadataResponse; +import org.apache.kafka.common.requests.OffsetsForLeaderEpochResponse; +import org.apache.kafka.common.requests.RequestTestUtils; +import org.apache.kafka.common.serialization.ByteArrayDeserializer; +import org.apache.kafka.common.serialization.BytesDeserializer; +import org.apache.kafka.common.serialization.Deserializer; +import org.apache.kafka.common.serialization.StringDeserializer; +import org.apache.kafka.common.utils.BufferSupplier; +import org.apache.kafka.common.utils.ByteBufferOutputStream; +import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.common.utils.MockTime; +import org.apache.kafka.common.utils.Time; +import org.apache.kafka.common.utils.Timer; +import org.apache.kafka.common.utils.Utils; +import org.apache.kafka.test.DelayedReceive; +import org.apache.kafka.test.MockSelector; +import org.apache.kafka.test.TestUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.DataOutputStream; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; + +import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; +import static java.util.Collections.emptySet; +import static java.util.Collections.singleton; +import static java.util.Collections.singletonList; +import static java.util.Collections.singletonMap; +import static org.apache.kafka.clients.consumer.ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG; +import static org.apache.kafka.clients.consumer.ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG; +import static org.apache.kafka.common.requests.FetchMetadata.INVALID_SESSION_ID; +import static org.apache.kafka.common.utils.Utils.mkSet; +import static org.apache.kafka.test.TestUtils.assertOptional; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +public class FetchRequestManagerTest { + + private static final double EPSILON = 0.0001; + + private ConsumerRebalanceListener listener = new NoOpConsumerRebalanceListener(); + private String topicName = "test"; + private String groupId = "test-group"; + private Uuid topicId = Uuid.randomUuid(); + private Map topicIds = new HashMap() { + { + put(topicName, topicId); + } + }; + private Map topicNames = singletonMap(topicId, topicName); + private final String metricGroup = "consumer" + groupId + "-fetch-manager-metrics"; + private TopicPartition tp0 = new TopicPartition(topicName, 0); + private TopicPartition tp1 = new TopicPartition(topicName, 1); + private TopicPartition tp2 = new TopicPartition(topicName, 2); + private TopicPartition tp3 = new TopicPartition(topicName, 3); + private TopicIdPartition tidp0 = new TopicIdPartition(topicId, tp0); + private TopicIdPartition tidp1 = new TopicIdPartition(topicId, tp1); + private TopicIdPartition tidp2 = new TopicIdPartition(topicId, tp2); + private TopicIdPartition tidp3 = new TopicIdPartition(topicId, tp3); + private int validLeaderEpoch = 0; + private MetadataResponse initialUpdateResponse = + RequestTestUtils.metadataUpdateWithIds(1, singletonMap(topicName, 4), topicIds); + + private int minBytes = 1; + private int maxBytes = Integer.MAX_VALUE; + private int maxWaitMs = 0; + private int fetchSize = 1000; + private long retryBackoffMs = 100; + private long requestTimeoutMs = 30000; + private MockTime time = new MockTime(1); + private SubscriptionState subscriptions; + private ConsumerMetadata metadata; + private FetchMetricsRegistry metricsRegistry; + private FetchMetricsManager metricsManager; + private MockClient client; + private Metrics metrics; + private ApiVersions apiVersions = new ApiVersions(); + private ConsumerNetworkClient oldConsumerClient; + private Deserializers deserializers; + private TestableFetchRequestManager fetcher; + private TestableNetworkClientDelegate consumerClient; + private OffsetFetcher offsetFetcher; + + private MemoryRecords records; + private MemoryRecords nextRecords; + private MemoryRecords emptyRecords; + private MemoryRecords partialRecords; + + @BeforeEach + public void setup() { + records = buildRecords(1L, 3, 1); + nextRecords = buildRecords(4L, 2, 4); + emptyRecords = buildRecords(0L, 0, 0); + partialRecords = buildRecords(4L, 1, 0); + partialRecords.buffer().putInt(Records.SIZE_OFFSET, 10000); + } + + private void assignFromUser(Set partitions) { + subscriptions.assignFromUser(partitions); + client.updateMetadata(initialUpdateResponse); + + // A dummy metadata update to ensure valid leader epoch. + metadata.updateWithCurrentRequestVersion(RequestTestUtils.metadataUpdateWithIds("dummy", 1, + Collections.emptyMap(), singletonMap(topicName, 4), + tp -> validLeaderEpoch, topicIds), false, 0L); + } + + private void assignFromUserNoId(Set partitions) { + subscriptions.assignFromUser(partitions); + client.updateMetadata(RequestTestUtils.metadataUpdateWithIds(1, singletonMap("noId", 1), Collections.emptyMap())); + + // A dummy metadata update to ensure valid leader epoch. + metadata.update(9, RequestTestUtils.metadataUpdateWithIds("dummy", 1, + Collections.emptyMap(), singletonMap("noId", 1), + tp -> validLeaderEpoch, topicIds), false, 0L); + } + + @AfterEach + public void teardown() throws Exception { + if (metrics != null) + this.metrics.close(); + if (fetcher != null) + this.fetcher.close(); + } + + private int sendFetches() { + offsetFetcher.validatePositionsOnMetadataChange(); + return fetcher.sendFetches(); + } + + @Test + public void testFetchNormal() { + buildFetcher(); + + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + // normal fetch + assertEquals(1, sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + + Map>> partitionRecords = fetchedRecords(); + assertTrue(partitionRecords.containsKey(tp0)); + + List> records = partitionRecords.get(tp0); + assertEquals(3, records.size()); + assertEquals(4L, subscriptions.position(tp0).offset); // this is the next fetching position + long offset = 1; + for (ConsumerRecord record : records) { + assertEquals(offset, record.offset()); + offset += 1; + } + } + + @Test + public void testInflightFetchOnPendingPartitions() { + buildFetcher(); + + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + assertEquals(1, sendFetches()); + subscriptions.markPendingRevocation(singleton(tp0)); + + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + consumerClient.poll(time.timer(0)); + assertNull(fetchedRecords().get(tp0)); + } + + @Test + public void testCloseShouldBeIdempotent() { + buildFetcher(); + + fetcher.close(); + fetcher.close(); + fetcher.close(); + + verify(fetcher, times(1)).closeInternal(any(Timer.class)); + } + + @Test + public void testFetcherCloseClosesFetchSessionsInBroker() { + buildFetcher(); + + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + // normal fetch + assertEquals(1, sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + + final FetchResponse fetchResponse = fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0); + client.prepareResponse(fetchResponse); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + assertEquals(0, consumerClient.pendingRequestCount()); + + final ArgumentCaptor argument = ArgumentCaptor.forClass(NetworkClientDelegate.UnsentRequest.class); + + // send request to close the fetcher + Timer timer = time.timer(Duration.ofSeconds(10)); + this.fetcher.close(timer); + NetworkClientDelegate.PollResult pollResult = fetcher.poll(time.milliseconds()); + consumerClient.addAll(pollResult.unsentRequests); + consumerClient.poll(timer); + + // validate that Fetcher.close() has sent a request with final epoch. 2 requests are sent, one for the normal + // fetch earlier and another for the finish fetch here. + verify(consumerClient, times(2)).doSend(argument.capture(), any(Long.class)); + NetworkClientDelegate.UnsentRequest unsentRequest = argument.getValue(); + FetchRequest.Builder builder = (FetchRequest.Builder) unsentRequest.requestBuilder(); + // session Id is the same + assertEquals(fetchResponse.sessionId(), builder.metadata().sessionId()); + // contains final epoch + assertEquals(FetchMetadata.FINAL_EPOCH, builder.metadata().epoch()); // final epoch indicates we want to close the session + assertTrue(builder.fetchData().isEmpty()); // partition data should be empty + } + + @Test + public void testFetchingPendingPartitions() { + buildFetcher(); + + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + // normal fetch + assertEquals(1, sendFetches()); + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + fetchedRecords(); + assertEquals(4L, subscriptions.position(tp0).offset); // this is the next fetching position + + // mark partition unfetchable + subscriptions.markPendingRevocation(singleton(tp0)); + assertEquals(0, sendFetches()); + consumerClient.poll(time.timer(0)); + assertFalse(fetcher.hasCompletedFetches()); + fetchedRecords(); + assertEquals(4L, subscriptions.position(tp0).offset); + } + + @Test + public void testFetchWithNoTopicId() { + // Should work and default to using old request type. + buildFetcher(); + + TopicIdPartition noId = new TopicIdPartition(Uuid.ZERO_UUID, new TopicPartition("noId", 0)); + assignFromUserNoId(singleton(noId.topicPartition())); + subscriptions.seek(noId.topicPartition(), 0); + + // Fetch should use request version 12 + assertEquals(1, sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + + client.prepareResponse( + fetchRequestMatcher((short) 12, noId, 0, Optional.of(validLeaderEpoch)), + fullFetchResponse(noId, this.records, Errors.NONE, 100L, 0) + ); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + + Map>> partitionRecords = fetchedRecords(); + assertTrue(partitionRecords.containsKey(noId.topicPartition())); + + List> records = partitionRecords.get(noId.topicPartition()); + assertEquals(3, records.size()); + assertEquals(4L, subscriptions.position(noId.topicPartition()).offset); // this is the next fetching position + long offset = 1; + for (ConsumerRecord record : records) { + assertEquals(offset, record.offset()); + offset += 1; + } + } + + @Test + public void testFetchWithTopicId() { + buildFetcher(); + + TopicIdPartition tp = new TopicIdPartition(topicId, new TopicPartition(topicName, 0)); + assignFromUser(singleton(tp.topicPartition())); + subscriptions.seek(tp.topicPartition(), 0); + + // Fetch should use latest version + assertEquals(1, sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + + client.prepareResponse( + fetchRequestMatcher(ApiKeys.FETCH.latestVersion(), tp, 0, Optional.of(validLeaderEpoch)), + fullFetchResponse(tp, this.records, Errors.NONE, 100L, 0) + ); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + + Map>> partitionRecords = fetchedRecords(); + assertTrue(partitionRecords.containsKey(tp.topicPartition())); + + List> records = partitionRecords.get(tp.topicPartition()); + assertEquals(3, records.size()); + assertEquals(4L, subscriptions.position(tp.topicPartition()).offset); // this is the next fetching position + long offset = 1; + for (ConsumerRecord record : records) { + assertEquals(offset, record.offset()); + offset += 1; + } + } + + @Test + public void testFetchForgetTopicIdWhenUnassigned() { + buildFetcher(); + + TopicIdPartition foo = new TopicIdPartition(Uuid.randomUuid(), new TopicPartition("foo", 0)); + TopicIdPartition bar = new TopicIdPartition(Uuid.randomUuid(), new TopicPartition("bar", 0)); + + // Assign foo and bar. + subscriptions.assignFromUser(singleton(foo.topicPartition())); + client.updateMetadata(RequestTestUtils.metadataUpdateWithIds(1, singleton(foo), tp -> validLeaderEpoch)); + subscriptions.seek(foo.topicPartition(), 0); + + // Fetch should use latest version. + assertEquals(1, sendFetches()); + + client.prepareResponse( + fetchRequestMatcher(ApiKeys.FETCH.latestVersion(), + singletonMap(foo, new PartitionData( + foo.topicId(), + 0, + FetchRequest.INVALID_LOG_START_OFFSET, + fetchSize, + Optional.of(validLeaderEpoch)) + ), + emptyList() + ), + fullFetchResponse(1, foo, this.records, Errors.NONE, 100L, 0) + ); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + fetchedRecords(); + + // Assign bar and unassign foo. + subscriptions.assignFromUser(singleton(bar.topicPartition())); + client.updateMetadata(RequestTestUtils.metadataUpdateWithIds(1, singleton(bar), tp -> validLeaderEpoch)); + subscriptions.seek(bar.topicPartition(), 0); + + // Fetch should use latest version. + assertEquals(1, sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + + client.prepareResponse( + fetchRequestMatcher(ApiKeys.FETCH.latestVersion(), + singletonMap(bar, new PartitionData( + bar.topicId(), + 0, + FetchRequest.INVALID_LOG_START_OFFSET, + fetchSize, + Optional.of(validLeaderEpoch)) + ), + singletonList(foo) + ), + fullFetchResponse(1, bar, this.records, Errors.NONE, 100L, 0) + ); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + fetchedRecords(); + } + + @Test + public void testFetchForgetTopicIdWhenReplaced() { + buildFetcher(); + + TopicIdPartition fooWithOldTopicId = new TopicIdPartition(Uuid.randomUuid(), new TopicPartition("foo", 0)); + TopicIdPartition fooWithNewTopicId = new TopicIdPartition(Uuid.randomUuid(), new TopicPartition("foo", 0)); + + // Assign foo with old topic id. + subscriptions.assignFromUser(singleton(fooWithOldTopicId.topicPartition())); + client.updateMetadata(RequestTestUtils.metadataUpdateWithIds(1, singleton(fooWithOldTopicId), tp -> validLeaderEpoch)); + subscriptions.seek(fooWithOldTopicId.topicPartition(), 0); + + // Fetch should use latest version. + assertEquals(1, sendFetches()); + + client.prepareResponse( + fetchRequestMatcher(ApiKeys.FETCH.latestVersion(), + singletonMap(fooWithOldTopicId, new PartitionData( + fooWithOldTopicId.topicId(), + 0, + FetchRequest.INVALID_LOG_START_OFFSET, + fetchSize, + Optional.of(validLeaderEpoch)) + ), + emptyList() + ), + fullFetchResponse(1, fooWithOldTopicId, this.records, Errors.NONE, 100L, 0) + ); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + fetchedRecords(); + + // Replace foo with old topic id with foo with new topic id. + subscriptions.assignFromUser(singleton(fooWithNewTopicId.topicPartition())); + client.updateMetadata(RequestTestUtils.metadataUpdateWithIds(1, singleton(fooWithNewTopicId), tp -> validLeaderEpoch)); + subscriptions.seek(fooWithNewTopicId.topicPartition(), 0); + + // Fetch should use latest version. + assertEquals(1, sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + + // foo with old topic id should be removed from the session. + client.prepareResponse( + fetchRequestMatcher(ApiKeys.FETCH.latestVersion(), + singletonMap(fooWithNewTopicId, new PartitionData( + fooWithNewTopicId.topicId(), + 0, + FetchRequest.INVALID_LOG_START_OFFSET, + fetchSize, + Optional.of(validLeaderEpoch)) + ), + singletonList(fooWithOldTopicId) + ), + fullFetchResponse(1, fooWithNewTopicId, this.records, Errors.NONE, 100L, 0) + ); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + fetchedRecords(); + } + + @Test + public void testFetchTopicIdUpgradeDowngrade() { + buildFetcher(); + + TopicIdPartition fooWithoutId = new TopicIdPartition(Uuid.ZERO_UUID, new TopicPartition("foo", 0)); + TopicIdPartition fooWithId = new TopicIdPartition(Uuid.randomUuid(), new TopicPartition("foo", 0)); + + // Assign foo without a topic id. + subscriptions.assignFromUser(singleton(fooWithoutId.topicPartition())); + client.updateMetadata(RequestTestUtils.metadataUpdateWithIds(1, singleton(fooWithoutId), tp -> validLeaderEpoch)); + subscriptions.seek(fooWithoutId.topicPartition(), 0); + + // Fetch should use version 12. + assertEquals(1, sendFetches()); + + client.prepareResponse( + fetchRequestMatcher((short) 12, + singletonMap(fooWithoutId, new PartitionData( + fooWithoutId.topicId(), + 0, + FetchRequest.INVALID_LOG_START_OFFSET, + fetchSize, + Optional.of(validLeaderEpoch)) + ), + emptyList() + ), + fullFetchResponse(1, fooWithoutId, this.records, Errors.NONE, 100L, 0) + ); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + fetchedRecords(); + + // Upgrade. + subscriptions.assignFromUser(singleton(fooWithId.topicPartition())); + client.updateMetadata(RequestTestUtils.metadataUpdateWithIds(1, singleton(fooWithId), tp -> validLeaderEpoch)); + subscriptions.seek(fooWithId.topicPartition(), 0); + + // Fetch should use latest version. + assertEquals(1, sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + + // foo with old topic id should be removed from the session. + client.prepareResponse( + fetchRequestMatcher(ApiKeys.FETCH.latestVersion(), + singletonMap(fooWithId, new PartitionData( + fooWithId.topicId(), + 0, + FetchRequest.INVALID_LOG_START_OFFSET, + fetchSize, + Optional.of(validLeaderEpoch)) + ), + emptyList() + ), + fullFetchResponse(1, fooWithId, this.records, Errors.NONE, 100L, 0) + ); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + fetchedRecords(); + + // Downgrade. + subscriptions.assignFromUser(singleton(fooWithoutId.topicPartition())); + client.updateMetadata(RequestTestUtils.metadataUpdateWithIds(1, singleton(fooWithoutId), tp -> validLeaderEpoch)); + subscriptions.seek(fooWithoutId.topicPartition(), 0); + + // Fetch should use version 12. + assertEquals(1, sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + + // foo with old topic id should be removed from the session. + client.prepareResponse( + fetchRequestMatcher((short) 12, + singletonMap(fooWithoutId, new PartitionData( + fooWithoutId.topicId(), + 0, + FetchRequest.INVALID_LOG_START_OFFSET, + fetchSize, + Optional.of(validLeaderEpoch)) + ), + emptyList() + ), + fullFetchResponse(1, fooWithoutId, this.records, Errors.NONE, 100L, 0) + ); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + fetchedRecords(); + } + + private MockClient.RequestMatcher fetchRequestMatcher( + short expectedVersion, + TopicIdPartition tp, + long expectedFetchOffset, + Optional expectedCurrentLeaderEpoch + ) { + return fetchRequestMatcher( + expectedVersion, + singletonMap(tp, new PartitionData( + tp.topicId(), + expectedFetchOffset, + FetchRequest.INVALID_LOG_START_OFFSET, + fetchSize, + expectedCurrentLeaderEpoch + )), + emptyList() + ); + } + + private MockClient.RequestMatcher fetchRequestMatcher( + short expectedVersion, + Map fetch, + List forgotten + ) { + return body -> { + if (body instanceof FetchRequest) { + FetchRequest fetchRequest = (FetchRequest) body; + assertEquals(expectedVersion, fetchRequest.version()); + assertEquals(fetch, fetchRequest.fetchData(topicNames(new ArrayList<>(fetch.keySet())))); + assertEquals(forgotten, fetchRequest.forgottenTopics(topicNames(forgotten))); + return true; + } else { + fail("Should have seen FetchRequest"); + return false; + } + }; + } + + private Map topicNames(List partitions) { + Map topicNames = new HashMap<>(); + partitions.forEach(partition -> topicNames.putIfAbsent(partition.topicId(), partition.topic())); + return topicNames; + } + + @Test + public void testMissingLeaderEpochInRecords() { + buildFetcher(); + + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + ByteBuffer buffer = ByteBuffer.allocate(1024); + MemoryRecordsBuilder builder = MemoryRecords.builder(buffer, RecordBatch.MAGIC_VALUE_V0, + CompressionType.NONE, TimestampType.CREATE_TIME, 0L, System.currentTimeMillis(), + RecordBatch.NO_PARTITION_LEADER_EPOCH); + builder.append(0L, "key".getBytes(), "1".getBytes()); + builder.append(0L, "key".getBytes(), "2".getBytes()); + MemoryRecords records = builder.build(); + + assertEquals(1, sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + + Map>> partitionRecords = fetchedRecords(); + assertTrue(partitionRecords.containsKey(tp0)); + assertEquals(2, partitionRecords.get(tp0).size()); + + for (ConsumerRecord record : partitionRecords.get(tp0)) { + assertEquals(Optional.empty(), record.leaderEpoch()); + } + } + + @Test + public void testLeaderEpochInConsumerRecord() { + buildFetcher(); + + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + Integer partitionLeaderEpoch = 1; + + ByteBuffer buffer = ByteBuffer.allocate(1024); + MemoryRecordsBuilder builder = MemoryRecords.builder(buffer, RecordBatch.CURRENT_MAGIC_VALUE, + CompressionType.NONE, TimestampType.CREATE_TIME, 0L, System.currentTimeMillis(), + partitionLeaderEpoch); + builder.append(0L, "key".getBytes(), partitionLeaderEpoch.toString().getBytes()); + builder.append(0L, "key".getBytes(), partitionLeaderEpoch.toString().getBytes()); + builder.close(); + + partitionLeaderEpoch += 7; + + builder = MemoryRecords.builder(buffer, RecordBatch.CURRENT_MAGIC_VALUE, CompressionType.NONE, + TimestampType.CREATE_TIME, 2L, System.currentTimeMillis(), partitionLeaderEpoch); + builder.append(0L, "key".getBytes(), partitionLeaderEpoch.toString().getBytes()); + builder.close(); + + partitionLeaderEpoch += 5; + builder = MemoryRecords.builder(buffer, RecordBatch.CURRENT_MAGIC_VALUE, CompressionType.NONE, + TimestampType.CREATE_TIME, 3L, System.currentTimeMillis(), partitionLeaderEpoch); + builder.append(0L, "key".getBytes(), partitionLeaderEpoch.toString().getBytes()); + builder.append(0L, "key".getBytes(), partitionLeaderEpoch.toString().getBytes()); + builder.append(0L, "key".getBytes(), partitionLeaderEpoch.toString().getBytes()); + builder.close(); + + buffer.flip(); + MemoryRecords records = MemoryRecords.readableRecords(buffer); + + assertEquals(1, sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + + Map>> partitionRecords = fetchedRecords(); + assertTrue(partitionRecords.containsKey(tp0)); + assertEquals(6, partitionRecords.get(tp0).size()); + + for (ConsumerRecord record : partitionRecords.get(tp0)) { + int expectedLeaderEpoch = Integer.parseInt(Utils.utf8(record.value())); + assertEquals(Optional.of(expectedLeaderEpoch), record.leaderEpoch()); + } + } + + @Test + public void testClearBufferedDataForTopicPartitions() { + buildFetcher(); + + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + // normal fetch + assertEquals(1, sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + Set newAssignedTopicPartitions = new HashSet<>(); + newAssignedTopicPartitions.add(tp1); + + fetcher.clearBufferedDataForUnassignedPartitions(newAssignedTopicPartitions); + assertFalse(fetcher.hasCompletedFetches()); + } + + @Test + public void testFetchSkipsBlackedOutNodes() { + buildFetcher(); + + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + Node node = initialUpdateResponse.brokers().iterator().next(); + + client.backoff(node, 500); + assertEquals(0, sendFetches()); + + time.sleep(500); + assertEquals(1, sendFetches()); + } + + @Test + public void testFetcherIgnoresControlRecords() { + buildFetcher(); + + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + // normal fetch + assertEquals(1, sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + + long producerId = 1; + short producerEpoch = 0; + int baseSequence = 0; + int partitionLeaderEpoch = 0; + + ByteBuffer buffer = ByteBuffer.allocate(1024); + MemoryRecordsBuilder builder = MemoryRecords.idempotentBuilder(buffer, CompressionType.NONE, 0L, producerId, + producerEpoch, baseSequence); + builder.append(0L, "key".getBytes(), null); + builder.close(); + + MemoryRecords.writeEndTransactionalMarker(buffer, 1L, time.milliseconds(), partitionLeaderEpoch, producerId, producerEpoch, + new EndTransactionMarker(ControlRecordType.ABORT, 0)); + + buffer.flip(); + + client.prepareResponse(fullFetchResponse(tidp0, MemoryRecords.readableRecords(buffer), Errors.NONE, 100L, 0)); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + + Map>> partitionRecords = fetchedRecords(); + assertTrue(partitionRecords.containsKey(tp0)); + + List> records = partitionRecords.get(tp0); + assertEquals(1, records.size()); + assertEquals(2L, subscriptions.position(tp0).offset); + + ConsumerRecord record = records.get(0); + assertArrayEquals("key".getBytes(), record.key()); + } + + @Test + public void testFetchError() { + buildFetcher(); + + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + assertEquals(1, sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NOT_LEADER_OR_FOLLOWER, 100L, 0)); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + + Map>> partitionRecords = fetchedRecords(); + assertFalse(partitionRecords.containsKey(tp0)); + } + + private MockClient.RequestMatcher matchesOffset(final TopicIdPartition tp, final long offset) { + return body -> { + FetchRequest fetch = (FetchRequest) body; + Map fetchData = fetch.fetchData(topicNames); + return fetchData.containsKey(tp) && + fetchData.get(tp).fetchOffset == offset; + }; + } + + @Test + public void testFetchedRecordsRaisesOnSerializationErrors() { + // raise an exception from somewhere in the middle of the fetch response + // so that we can verify that our position does not advance after raising + ByteArrayDeserializer deserializer = new ByteArrayDeserializer() { + int i = 0; + @Override + public byte[] deserialize(String topic, byte[] data) { + if (i++ % 2 == 1) { + // Should be blocked on the value deserialization of the first record. + assertEquals("value-1", new String(data, StandardCharsets.UTF_8)); + throw new SerializationException(); + } + return data; + } + }; + + buildFetcher(deserializer, deserializer); + + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 1); + + client.prepareResponse(matchesOffset(tidp0, 1), fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + + assertEquals(1, sendFetches()); + consumerClient.poll(time.timer(0)); + // The fetcher should block on Deserialization error + for (int i = 0; i < 2; i++) { + try { + fetcher.collectFetch(); + fail("fetchedRecords should have raised"); + } catch (SerializationException e) { + // the position should not advance since no data has been returned + assertEquals(1, subscriptions.position(tp0).offset); + } + } + } + + @Test + public void testParseCorruptedRecord() throws Exception { + buildFetcher(); + assignFromUser(singleton(tp0)); + + ByteBuffer buffer = ByteBuffer.allocate(1024); + DataOutputStream out = new DataOutputStream(new ByteBufferOutputStream(buffer)); + + byte magic = RecordBatch.MAGIC_VALUE_V1; + byte[] key = "foo".getBytes(); + byte[] value = "baz".getBytes(); + long offset = 0; + long timestamp = 500L; + + int size = LegacyRecord.recordSize(magic, key.length, value.length); + byte attributes = LegacyRecord.computeAttributes(magic, CompressionType.NONE, TimestampType.CREATE_TIME); + long crc = LegacyRecord.computeChecksum(magic, attributes, timestamp, key, value); + + // write one valid record + out.writeLong(offset); + out.writeInt(size); + LegacyRecord.write(out, magic, crc, LegacyRecord.computeAttributes(magic, CompressionType.NONE, TimestampType.CREATE_TIME), timestamp, key, value); + + // and one invalid record (note the crc) + out.writeLong(offset + 1); + out.writeInt(size); + LegacyRecord.write(out, magic, crc + 1, LegacyRecord.computeAttributes(magic, CompressionType.NONE, TimestampType.CREATE_TIME), timestamp, key, value); + + // write one valid record + out.writeLong(offset + 2); + out.writeInt(size); + LegacyRecord.write(out, magic, crc, LegacyRecord.computeAttributes(magic, CompressionType.NONE, TimestampType.CREATE_TIME), timestamp, key, value); + + // Write a record whose size field is invalid. + out.writeLong(offset + 3); + out.writeInt(1); + + // write one valid record + out.writeLong(offset + 4); + out.writeInt(size); + LegacyRecord.write(out, magic, crc, LegacyRecord.computeAttributes(magic, CompressionType.NONE, TimestampType.CREATE_TIME), timestamp, key, value); + + buffer.flip(); + + subscriptions.seekUnvalidated(tp0, new SubscriptionState.FetchPosition(0, Optional.empty(), metadata.currentLeader(tp0))); + + // normal fetch + assertEquals(1, sendFetches()); + client.prepareResponse(fullFetchResponse(tidp0, MemoryRecords.readableRecords(buffer), Errors.NONE, 100L, 0)); + consumerClient.poll(time.timer(0)); + + // the first fetchedRecords() should return the first valid message + assertEquals(1, fetchedRecords().get(tp0).size()); + assertEquals(1, subscriptions.position(tp0).offset); + + ensureBlockOnRecord(1L); + seekAndConsumeRecord(buffer, 2L); + ensureBlockOnRecord(3L); + try { + // For a record that cannot be retrieved from the iterator, we cannot seek over it within the batch. + seekAndConsumeRecord(buffer, 4L); + fail("Should have thrown exception when fail to retrieve a record from iterator."); + } catch (KafkaException ke) { + // let it go + } + ensureBlockOnRecord(4L); + } + + private void ensureBlockOnRecord(long blockedOffset) { + // the fetchedRecords() should always throw exception due to the invalid message at the starting offset. + for (int i = 0; i < 2; i++) { + try { + fetcher.collectFetch(); + fail("fetchedRecords should have raised KafkaException"); + } catch (KafkaException e) { + assertEquals(blockedOffset, subscriptions.position(tp0).offset); + } + } + } + + private void seekAndConsumeRecord(ByteBuffer responseBuffer, long toOffset) { + // Seek to skip the bad record and fetch again. + subscriptions.seekUnvalidated(tp0, new SubscriptionState.FetchPosition(toOffset, Optional.empty(), metadata.currentLeader(tp0))); + // Should not throw exception after the seek. + fetcher.collectFetch(); + assertEquals(1, sendFetches()); + client.prepareResponse(fullFetchResponse(tidp0, MemoryRecords.readableRecords(responseBuffer), Errors.NONE, 100L, 0)); + consumerClient.poll(time.timer(0)); + + Map>> recordsByPartition = fetchedRecords(); + List> records = recordsByPartition.get(tp0); + assertEquals(1, records.size()); + assertEquals(toOffset, records.get(0).offset()); + assertEquals(toOffset + 1, subscriptions.position(tp0).offset); + } + + @Test + public void testInvalidDefaultRecordBatch() { + buildFetcher(); + + ByteBuffer buffer = ByteBuffer.allocate(1024); + ByteBufferOutputStream out = new ByteBufferOutputStream(buffer); + + MemoryRecordsBuilder builder = new MemoryRecordsBuilder(out, + DefaultRecordBatch.CURRENT_MAGIC_VALUE, + CompressionType.NONE, + TimestampType.CREATE_TIME, + 0L, 10L, 0L, (short) 0, 0, false, false, 0, 1024); + builder.append(10L, "key".getBytes(), "value".getBytes()); + builder.close(); + buffer.flip(); + + // Garble the CRC + buffer.position(17); + buffer.put("beef".getBytes()); + buffer.position(0); + + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + // normal fetch + assertEquals(1, sendFetches()); + client.prepareResponse(fullFetchResponse(tidp0, MemoryRecords.readableRecords(buffer), Errors.NONE, 100L, 0)); + consumerClient.poll(time.timer(0)); + + // the fetchedRecords() should always throw exception due to the bad batch. + for (int i = 0; i < 2; i++) { + try { + fetcher.collectFetch(); + fail("fetchedRecords should have raised KafkaException"); + } catch (KafkaException e) { + assertEquals(0, subscriptions.position(tp0).offset); + } + } + } + + @Test + public void testParseInvalidRecordBatch() { + buildFetcher(); + MemoryRecords records = MemoryRecords.withRecords(RecordBatch.MAGIC_VALUE_V2, 0L, + CompressionType.NONE, TimestampType.CREATE_TIME, + new SimpleRecord(1L, "a".getBytes(), "1".getBytes()), + new SimpleRecord(2L, "b".getBytes(), "2".getBytes()), + new SimpleRecord(3L, "c".getBytes(), "3".getBytes())); + ByteBuffer buffer = records.buffer(); + + // flip some bits to fail the crc + buffer.putInt(32, buffer.get(32) ^ 87238423); + + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + // normal fetch + assertEquals(1, sendFetches()); + client.prepareResponse(fullFetchResponse(tidp0, MemoryRecords.readableRecords(buffer), Errors.NONE, 100L, 0)); + consumerClient.poll(time.timer(0)); + try { + fetcher.collectFetch(); + fail("fetchedRecords should have raised"); + } catch (KafkaException e) { + // the position should not advance since no data has been returned + assertEquals(0, subscriptions.position(tp0).offset); + } + } + + @Test + public void testHeaders() { + buildFetcher(); + + MemoryRecordsBuilder builder = MemoryRecords.builder(ByteBuffer.allocate(1024), CompressionType.NONE, TimestampType.CREATE_TIME, 1L); + builder.append(0L, "key".getBytes(), "value-1".getBytes()); + + Header[] headersArray = new Header[1]; + headersArray[0] = new RecordHeader("headerKey", "headerValue".getBytes(StandardCharsets.UTF_8)); + builder.append(0L, "key".getBytes(), "value-2".getBytes(), headersArray); + + Header[] headersArray2 = new Header[2]; + headersArray2[0] = new RecordHeader("headerKey", "headerValue".getBytes(StandardCharsets.UTF_8)); + headersArray2[1] = new RecordHeader("headerKey", "headerValue2".getBytes(StandardCharsets.UTF_8)); + builder.append(0L, "key".getBytes(), "value-3".getBytes(), headersArray2); + + MemoryRecords memoryRecords = builder.build(); + + List> records; + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 1); + + client.prepareResponse(matchesOffset(tidp0, 1), fullFetchResponse(tidp0, memoryRecords, Errors.NONE, 100L, 0)); + + assertEquals(1, sendFetches()); + consumerClient.poll(time.timer(0)); + Map>> recordsByPartition = fetchedRecords(); + records = recordsByPartition.get(tp0); + + assertEquals(3, records.size()); + + Iterator> recordIterator = records.iterator(); + + ConsumerRecord record = recordIterator.next(); + assertNull(record.headers().lastHeader("headerKey")); + + record = recordIterator.next(); + assertEquals("headerValue", new String(record.headers().lastHeader("headerKey").value(), StandardCharsets.UTF_8)); + assertEquals("headerKey", record.headers().lastHeader("headerKey").key()); + + record = recordIterator.next(); + assertEquals("headerValue2", new String(record.headers().lastHeader("headerKey").value(), StandardCharsets.UTF_8)); + assertEquals("headerKey", record.headers().lastHeader("headerKey").key()); + } + + @Test + public void testFetchMaxPollRecords() { + buildFetcher(2); + + List> records; + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 1); + + client.prepareResponse(matchesOffset(tidp0, 1), fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + client.prepareResponse(matchesOffset(tidp0, 4), fullFetchResponse(tidp0, this.nextRecords, Errors.NONE, 100L, 0)); + + assertEquals(1, sendFetches()); + consumerClient.poll(time.timer(0)); + Map>> recordsByPartition = fetchedRecords(); + records = recordsByPartition.get(tp0); + assertEquals(2, records.size()); + assertEquals(3L, subscriptions.position(tp0).offset); + assertEquals(1, records.get(0).offset()); + assertEquals(2, records.get(1).offset()); + + assertEquals(0, sendFetches()); + consumerClient.poll(time.timer(0)); + recordsByPartition = fetchedRecords(); + records = recordsByPartition.get(tp0); + assertEquals(1, records.size()); + assertEquals(4L, subscriptions.position(tp0).offset); + assertEquals(3, records.get(0).offset()); + + assertTrue(sendFetches() > 0); + consumerClient.poll(time.timer(0)); + recordsByPartition = fetchedRecords(); + records = recordsByPartition.get(tp0); + assertEquals(2, records.size()); + assertEquals(6L, subscriptions.position(tp0).offset); + assertEquals(4, records.get(0).offset()); + assertEquals(5, records.get(1).offset()); + } + + /** + * Test the scenario where a partition with fetched but not consumed records (i.e. max.poll.records is + * less than the number of fetched records) is unassigned and a different partition is assigned. This is a + * pattern used by Streams state restoration and KAFKA-5097 would have been caught by this test. + */ + @Test + public void testFetchAfterPartitionWithFetchedRecordsIsUnassigned() { + buildFetcher(2); + + List> records; + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 1); + + // Returns 3 records while `max.poll.records` is configured to 2 + client.prepareResponse(matchesOffset(tidp0, 1), fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + + assertEquals(1, sendFetches()); + consumerClient.poll(time.timer(0)); + Map>> recordsByPartition = fetchedRecords(); + records = recordsByPartition.get(tp0); + assertEquals(2, records.size()); + assertEquals(3L, subscriptions.position(tp0).offset); + assertEquals(1, records.get(0).offset()); + assertEquals(2, records.get(1).offset()); + + assignFromUser(singleton(tp1)); + client.prepareResponse(matchesOffset(tidp1, 4), fullFetchResponse(tidp1, this.nextRecords, Errors.NONE, 100L, 0)); + subscriptions.seek(tp1, 4); + + assertEquals(1, sendFetches()); + consumerClient.poll(time.timer(0)); + Map>> fetchedRecords = fetchedRecords(); + assertNull(fetchedRecords.get(tp0)); + records = fetchedRecords.get(tp1); + assertEquals(2, records.size()); + assertEquals(6L, subscriptions.position(tp1).offset); + assertEquals(4, records.get(0).offset()); + assertEquals(5, records.get(1).offset()); + } + + @Test + public void testFetchNonContinuousRecords() { + // if we are fetching from a compacted topic, there may be gaps in the returned records + // this test verifies the fetcher updates the current fetched/consumed positions correctly for this case + buildFetcher(); + + MemoryRecordsBuilder builder = MemoryRecords.builder(ByteBuffer.allocate(1024), CompressionType.NONE, + TimestampType.CREATE_TIME, 0L); + builder.appendWithOffset(15L, 0L, "key".getBytes(), "value-1".getBytes()); + builder.appendWithOffset(20L, 0L, "key".getBytes(), "value-2".getBytes()); + builder.appendWithOffset(30L, 0L, "key".getBytes(), "value-3".getBytes()); + MemoryRecords records = builder.build(); + + List> consumerRecords; + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + // normal fetch + assertEquals(1, sendFetches()); + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); + consumerClient.poll(time.timer(0)); + Map>> recordsByPartition = fetchedRecords(); + consumerRecords = recordsByPartition.get(tp0); + assertEquals(3, consumerRecords.size()); + assertEquals(31L, subscriptions.position(tp0).offset); // this is the next fetching position + + assertEquals(15L, consumerRecords.get(0).offset()); + assertEquals(20L, consumerRecords.get(1).offset()); + assertEquals(30L, consumerRecords.get(2).offset()); + } + + /** + * Test the case where the client makes a pre-v3 FetchRequest, but the server replies with only a partial + * request. This happens when a single message is larger than the per-partition limit. + */ + @Test + public void testFetchRequestWhenRecordTooLarge() { + try { + buildFetcher(); + + client.setNodeApiVersions(NodeApiVersions.create(ApiKeys.FETCH.id, (short) 2, (short) 2)); + makeFetchRequestWithIncompleteRecord(); + try { + fetcher.collectFetch(); + fail("RecordTooLargeException should have been raised"); + } catch (RecordTooLargeException e) { + assertTrue(e.getMessage().startsWith("There are some messages at [Partition=Offset]: ")); + // the position should not advance since no data has been returned + assertEquals(0, subscriptions.position(tp0).offset); + } + } finally { + client.setNodeApiVersions(NodeApiVersions.create()); + } + } + + /** + * Test the case where the client makes a post KIP-74 FetchRequest, but the server replies with only a + * partial request. For v3 and later FetchRequests, the implementation of KIP-74 changed the behavior + * so that at least one message is always returned. Therefore, this case should not happen, and it indicates + * that an internal error has taken place. + */ + @Test + public void testFetchRequestInternalError() { + buildFetcher(); + makeFetchRequestWithIncompleteRecord(); + try { + fetcher.collectFetch(); + fail("RecordTooLargeException should have been raised"); + } catch (KafkaException e) { + assertTrue(e.getMessage().startsWith("Failed to make progress reading messages")); + // the position should not advance since no data has been returned + assertEquals(0, subscriptions.position(tp0).offset); + } + } + + private void makeFetchRequestWithIncompleteRecord() { + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + assertEquals(1, sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + MemoryRecords partialRecord = MemoryRecords.readableRecords( + ByteBuffer.wrap(new byte[]{0, 0, 0, 0, 0, 0, 0, 0})); + client.prepareResponse(fullFetchResponse(tidp0, partialRecord, Errors.NONE, 100L, 0)); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + } + + @Test + public void testUnauthorizedTopic() { + buildFetcher(); + + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + // resize the limit of the buffer to pretend it is only fetch-size large + assertEquals(1, sendFetches()); + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.TOPIC_AUTHORIZATION_FAILED, 100L, 0)); + consumerClient.poll(time.timer(0)); + try { + fetcher.collectFetch(); + fail("fetchedRecords should have thrown"); + } catch (TopicAuthorizationException e) { + assertEquals(singleton(topicName), e.unauthorizedTopics()); + } + } + + @Test + public void testFetchDuringEagerRebalance() { + buildFetcher(); + + subscriptions.subscribe(singleton(topicName), listener); + subscriptions.assignFromSubscribed(singleton(tp0)); + subscriptions.seek(tp0, 0); + + client.updateMetadata(RequestTestUtils.metadataUpdateWithIds( + 1, singletonMap(topicName, 4), tp -> validLeaderEpoch, topicIds)); + + assertEquals(1, sendFetches()); + + // Now the eager rebalance happens and fetch positions are cleared + subscriptions.assignFromSubscribed(Collections.emptyList()); + + subscriptions.assignFromSubscribed(singleton(tp0)); + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + consumerClient.poll(time.timer(0)); + + // The active fetch should be ignored since its position is no longer valid + assertTrue(fetchedRecords().isEmpty()); + } + + @Test + public void testFetchDuringCooperativeRebalance() { + buildFetcher(); + + subscriptions.subscribe(singleton(topicName), listener); + subscriptions.assignFromSubscribed(singleton(tp0)); + subscriptions.seek(tp0, 0); + + client.updateMetadata(RequestTestUtils.metadataUpdateWithIds( + 1, singletonMap(topicName, 4), tp -> validLeaderEpoch, topicIds)); + + assertEquals(1, sendFetches()); + + // Now the cooperative rebalance happens and fetch positions are NOT cleared for unrevoked partitions + subscriptions.assignFromSubscribed(singleton(tp0)); + + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + consumerClient.poll(time.timer(0)); + + Map>> fetchedRecords = fetchedRecords(); + + // The active fetch should NOT be ignored since the position for tp0 is still valid + assertEquals(1, fetchedRecords.size()); + assertEquals(3, fetchedRecords.get(tp0).size()); + } + + @Test + public void testInFlightFetchOnPausedPartition() { + buildFetcher(); + + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + assertEquals(1, sendFetches()); + subscriptions.pause(tp0); + + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + consumerClient.poll(time.timer(0)); + assertNull(fetchedRecords().get(tp0)); + } + + @Test + public void testFetchOnPausedPartition() { + buildFetcher(); + + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + subscriptions.pause(tp0); + assertFalse(sendFetches() > 0); + assertTrue(client.requests().isEmpty()); + } + + @Test + public void testFetchOnCompletedFetchesForPausedAndResumedPartitions() { + buildFetcher(); + + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + assertEquals(1, sendFetches()); + + subscriptions.pause(tp0); + + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + consumerClient.poll(time.timer(0)); + + assertEmptyFetch("Should not return any records or advance position when partition is paused"); + + assertTrue(fetcher.hasCompletedFetches(), "Should still contain completed fetches"); + assertFalse(fetcher.hasAvailableFetches(), "Should not have any available (non-paused) completed fetches"); + assertEquals(0, sendFetches()); + + subscriptions.resume(tp0); + + assertTrue(fetcher.hasAvailableFetches(), "Should have available (non-paused) completed fetches"); + + consumerClient.poll(time.timer(0)); + Map>> fetchedRecords = fetchedRecords(); + assertEquals(1, fetchedRecords.size(), "Should return records when partition is resumed"); + assertNotNull(fetchedRecords.get(tp0)); + assertEquals(3, fetchedRecords.get(tp0).size()); + + consumerClient.poll(time.timer(0)); + assertEmptyFetch("Should not return records or advance position after previously paused partitions are fetched"); + assertFalse(fetcher.hasCompletedFetches(), "Should no longer contain completed fetches"); + } + + @Test + public void testFetchOnCompletedFetchesForSomePausedPartitions() { + buildFetcher(); + + Map>> fetchedRecords; + + assignFromUser(mkSet(tp0, tp1)); + + // seek to tp0 and tp1 in two polls to generate 2 complete requests and responses + + // #1 seek, request, poll, response + subscriptions.seekUnvalidated(tp0, new SubscriptionState.FetchPosition(1, Optional.empty(), metadata.currentLeader(tp0))); + assertEquals(1, sendFetches()); + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + consumerClient.poll(time.timer(0)); + + // #2 seek, request, poll, response + subscriptions.seekUnvalidated(tp1, new SubscriptionState.FetchPosition(1, Optional.empty(), metadata.currentLeader(tp1))); + assertEquals(1, sendFetches()); + client.prepareResponse(fullFetchResponse(tidp1, this.nextRecords, Errors.NONE, 100L, 0)); + + subscriptions.pause(tp0); + consumerClient.poll(time.timer(0)); + + fetchedRecords = fetchedRecords(); + assertEquals(1, fetchedRecords.size(), "Should return completed fetch for unpaused partitions"); + assertTrue(fetcher.hasCompletedFetches(), "Should still contain completed fetches"); + assertNotNull(fetchedRecords.get(tp1)); + assertNull(fetchedRecords.get(tp0)); + + assertEmptyFetch("Should not return records or advance position for remaining paused partition"); + assertTrue(fetcher.hasCompletedFetches(), "Should still contain completed fetches"); + } + + @Test + public void testFetchOnCompletedFetchesForAllPausedPartitions() { + buildFetcher(); + + assignFromUser(mkSet(tp0, tp1)); + + // seek to tp0 and tp1 in two polls to generate 2 complete requests and responses + + // #1 seek, request, poll, response + subscriptions.seekUnvalidated(tp0, new SubscriptionState.FetchPosition(1, Optional.empty(), metadata.currentLeader(tp0))); + assertEquals(1, sendFetches()); + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + consumerClient.poll(time.timer(0)); + + // #2 seek, request, poll, response + subscriptions.seekUnvalidated(tp1, new SubscriptionState.FetchPosition(1, Optional.empty(), metadata.currentLeader(tp1))); + assertEquals(1, sendFetches()); + client.prepareResponse(fullFetchResponse(tidp1, this.nextRecords, Errors.NONE, 100L, 0)); + + subscriptions.pause(tp0); + subscriptions.pause(tp1); + + consumerClient.poll(time.timer(0)); + + assertEmptyFetch("Should not return records or advance position for all paused partitions"); + assertTrue(fetcher.hasCompletedFetches(), "Should still contain completed fetches"); + assertFalse(fetcher.hasAvailableFetches(), "Should not have any available (non-paused) completed fetches"); + } + + @Test + public void testPartialFetchWithPausedPartitions() { + // this test sends creates a completed fetch with 3 records and a max poll of 2 records to assert + // that a fetch that must be returned over at least 2 polls can be cached successfully when its partition is + // paused, then returned successfully after its been resumed again later + buildFetcher(2); + + Map>> fetchedRecords; + + assignFromUser(mkSet(tp0, tp1)); + + subscriptions.seek(tp0, 1); + assertEquals(1, sendFetches()); + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + consumerClient.poll(time.timer(0)); + + fetchedRecords = fetchedRecords(); + + assertEquals(2, fetchedRecords.get(tp0).size(), "Should return 2 records from fetch with 3 records"); + assertFalse(fetcher.hasCompletedFetches(), "Should have no completed fetches"); + + subscriptions.pause(tp0); + consumerClient.poll(time.timer(0)); + + fetchedRecords = fetchedRecords(); + + assertEmptyFetch("Should not return records or advance position for paused partitions"); + assertTrue(fetcher.hasCompletedFetches(), "Should have 1 entry in completed fetches"); + assertFalse(fetcher.hasAvailableFetches(), "Should not have any available (non-paused) completed fetches"); + + subscriptions.resume(tp0); + + consumerClient.poll(time.timer(0)); + + fetchedRecords = fetchedRecords(); + + assertEquals(1, fetchedRecords.get(tp0).size(), "Should return last remaining record"); + assertFalse(fetcher.hasCompletedFetches(), "Should have no completed fetches"); + } + + @Test + public void testFetchDiscardedAfterPausedPartitionResumedAndSeekedToNewOffset() { + buildFetcher(); + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + assertEquals(1, sendFetches()); + subscriptions.pause(tp0); + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + + subscriptions.seek(tp0, 3); + subscriptions.resume(tp0); + consumerClient.poll(time.timer(0)); + + assertTrue(fetcher.hasCompletedFetches(), "Should have 1 entry in completed fetches"); + Fetch fetch = collectFetch(); + assertEquals(emptyMap(), fetch.records(), "Should not return any records because we seeked to a new offset"); + assertFalse(fetch.positionAdvanced()); + assertFalse(fetcher.hasCompletedFetches(), "Should have no completed fetches"); + } + + @Test + public void testFetchNotLeaderOrFollower() { + buildFetcher(); + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + assertEquals(1, sendFetches()); + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NOT_LEADER_OR_FOLLOWER, 100L, 0)); + consumerClient.poll(time.timer(0)); + assertEmptyFetch("Should not return records or advance position on fetch error"); + assertEquals(0L, metadata.timeToNextUpdate(time.milliseconds())); + } + + @Test + public void testFetchUnknownTopicOrPartition() { + buildFetcher(); + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + assertEquals(1, sendFetches()); + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.UNKNOWN_TOPIC_OR_PARTITION, 100L, 0)); + consumerClient.poll(time.timer(0)); + assertEmptyFetch("Should not return records or advance position on fetch error"); + assertEquals(0L, metadata.timeToNextUpdate(time.milliseconds())); + } + + @Test + public void testFetchUnknownTopicId() { + buildFetcher(); + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + assertEquals(1, sendFetches()); + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.UNKNOWN_TOPIC_ID, -1L, 0)); + consumerClient.poll(time.timer(0)); + assertEmptyFetch("Should not return records or advance position on fetch error"); + assertEquals(0L, metadata.timeToNextUpdate(time.milliseconds())); + } + + @Test + public void testFetchSessionIdError() { + buildFetcher(); + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + assertEquals(1, sendFetches()); + client.prepareResponse(fetchResponseWithTopLevelError(tidp0, Errors.FETCH_SESSION_TOPIC_ID_ERROR, 0)); + consumerClient.poll(time.timer(0)); + assertEmptyFetch("Should not return records or advance position on fetch error"); + assertEquals(0L, metadata.timeToNextUpdate(time.milliseconds())); + } + + @Test + public void testFetchInconsistentTopicId() { + buildFetcher(); + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + assertEquals(1, sendFetches()); + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.INCONSISTENT_TOPIC_ID, -1L, 0)); + consumerClient.poll(time.timer(0)); + assertEmptyFetch("Should not return records or advance position on fetch error"); + assertEquals(0L, metadata.timeToNextUpdate(time.milliseconds())); + } + + @Test + public void testFetchFencedLeaderEpoch() { + buildFetcher(); + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + assertEquals(1, sendFetches()); + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.FENCED_LEADER_EPOCH, 100L, 0)); + consumerClient.poll(time.timer(0)); + + assertEmptyFetch("Should not return records or advance position on fetch error"); + assertEquals(0L, metadata.timeToNextUpdate(time.milliseconds()), "Should have requested metadata update"); + } + + @Test + public void testFetchUnknownLeaderEpoch() { + buildFetcher(); + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + assertEquals(1, sendFetches()); + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.UNKNOWN_LEADER_EPOCH, 100L, 0)); + consumerClient.poll(time.timer(0)); + + assertEmptyFetch("Should not return records or advance position on fetch error"); + assertNotEquals(0L, metadata.timeToNextUpdate(time.milliseconds()), "Should not have requested metadata update"); + } + + @Test + public void testEpochSetInFetchRequest() { + buildFetcher(); + subscriptions.assignFromUser(singleton(tp0)); + MetadataResponse metadataResponse = RequestTestUtils.metadataUpdateWithIds("dummy", 1, + Collections.emptyMap(), Collections.singletonMap(topicName, 4), tp -> 99, topicIds); + client.updateMetadata(metadataResponse); + + subscriptions.seek(tp0, 10); + assertEquals(1, sendFetches()); + + // Check for epoch in outgoing request + MockClient.RequestMatcher matcher = body -> { + if (body instanceof FetchRequest) { + FetchRequest fetchRequest = (FetchRequest) body; + fetchRequest.fetchData(topicNames).values().forEach(partitionData -> { + assertTrue(partitionData.currentLeaderEpoch.isPresent(), "Expected Fetcher to set leader epoch in request"); + assertEquals(99, partitionData.currentLeaderEpoch.get().longValue(), "Expected leader epoch to match epoch from metadata update"); + }); + return true; + } else { + fail("Should have seen FetchRequest"); + return false; + } + }; + client.prepareResponse(matcher, fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + consumerClient.pollNoWakeup(); + } + + @Test + public void testFetchOffsetOutOfRange() { + buildFetcher(); + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + assertEquals(1, sendFetches()); + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.OFFSET_OUT_OF_RANGE, 100L, 0)); + consumerClient.poll(time.timer(0)); + assertEmptyFetch("Should not return records or advance position on fetch error"); + assertTrue(subscriptions.isOffsetResetNeeded(tp0)); + assertNull(subscriptions.validPosition(tp0)); + assertNull(subscriptions.position(tp0)); + } + + @Test + public void testStaleOutOfRangeError() { + // verify that an out of range error which arrives after a seek + // does not cause us to reset our position or throw an exception + buildFetcher(); + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + assertEquals(1, sendFetches()); + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.OFFSET_OUT_OF_RANGE, 100L, 0)); + subscriptions.seek(tp0, 1); + consumerClient.poll(time.timer(0)); + assertEmptyFetch("Should not return records or advance position on fetch error"); + assertFalse(subscriptions.isOffsetResetNeeded(tp0)); + assertEquals(1, subscriptions.position(tp0).offset); + } + + @Test + public void testFetchedRecordsAfterSeek() { + buildFetcher(OffsetResetStrategy.NONE, new ByteArrayDeserializer(), + new ByteArrayDeserializer(), 2, IsolationLevel.READ_UNCOMMITTED); + + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + assertTrue(sendFetches() > 0); + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.OFFSET_OUT_OF_RANGE, 100L, 0)); + consumerClient.poll(time.timer(0)); + assertFalse(subscriptions.isOffsetResetNeeded(tp0)); + subscriptions.seek(tp0, 2); + assertEmptyFetch("Should not return records or advance position after seeking to end of topic partition"); + } + + @Test + public void testFetchOffsetOutOfRangeException() { + buildFetcher(OffsetResetStrategy.NONE, new ByteArrayDeserializer(), + new ByteArrayDeserializer(), 2, IsolationLevel.READ_UNCOMMITTED); + + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + sendFetches(); + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.OFFSET_OUT_OF_RANGE, 100L, 0)); + consumerClient.poll(time.timer(0)); + + assertFalse(subscriptions.isOffsetResetNeeded(tp0)); + for (int i = 0; i < 2; i++) { + OffsetOutOfRangeException e = assertThrows(OffsetOutOfRangeException.class, () -> + fetcher.collectFetch()); + assertEquals(singleton(tp0), e.offsetOutOfRangePartitions().keySet()); + assertEquals(0L, e.offsetOutOfRangePartitions().get(tp0).longValue()); + } + } + + @Test + public void testFetchPositionAfterException() { + // verify the advancement in the next fetch offset equals to the number of fetched records when + // some fetched partitions cause Exception. This ensures that consumer won't lose record upon exception + buildFetcher(OffsetResetStrategy.NONE, new ByteArrayDeserializer(), + new ByteArrayDeserializer(), Integer.MAX_VALUE, IsolationLevel.READ_UNCOMMITTED); + assignFromUser(mkSet(tp0, tp1)); + subscriptions.seek(tp0, 1); + subscriptions.seek(tp1, 1); + + assertEquals(1, sendFetches()); + + Map partitions = new LinkedHashMap<>(); + partitions.put(tidp1, new FetchResponseData.PartitionData() + .setPartitionIndex(tp1.partition()) + .setHighWatermark(100) + .setRecords(records)); + partitions.put(tidp0, new FetchResponseData.PartitionData() + .setPartitionIndex(tp0.partition()) + .setErrorCode(Errors.OFFSET_OUT_OF_RANGE.code()) + .setHighWatermark(100)); + client.prepareResponse(FetchResponse.of(Errors.NONE, 0, INVALID_SESSION_ID, new LinkedHashMap<>(partitions))); + consumerClient.poll(time.timer(0)); + + List> allFetchedRecords = new ArrayList<>(); + fetchRecordsInto(allFetchedRecords); + + assertEquals(1, subscriptions.position(tp0).offset); + assertEquals(4, subscriptions.position(tp1).offset); + assertEquals(3, allFetchedRecords.size()); + + OffsetOutOfRangeException e = assertThrows(OffsetOutOfRangeException.class, () -> + fetchRecordsInto(allFetchedRecords)); + + assertEquals(singleton(tp0), e.offsetOutOfRangePartitions().keySet()); + assertEquals(1L, e.offsetOutOfRangePartitions().get(tp0).longValue()); + + assertEquals(1, subscriptions.position(tp0).offset); + assertEquals(4, subscriptions.position(tp1).offset); + assertEquals(3, allFetchedRecords.size()); + } + + private void fetchRecordsInto(List> allFetchedRecords) { + Map>> fetchedRecords = fetchedRecords(); + fetchedRecords.values().forEach(allFetchedRecords::addAll); + } + + @Test + public void testCompletedFetchRemoval() { + // Ensure the removal of completed fetches that cause an Exception if and only if they contain empty records. + buildFetcher(OffsetResetStrategy.NONE, new ByteArrayDeserializer(), + new ByteArrayDeserializer(), Integer.MAX_VALUE, IsolationLevel.READ_UNCOMMITTED); + assignFromUser(mkSet(tp0, tp1, tp2, tp3)); + + subscriptions.seek(tp0, 1); + subscriptions.seek(tp1, 1); + subscriptions.seek(tp2, 1); + subscriptions.seek(tp3, 1); + + assertEquals(1, sendFetches()); + + Map partitions = new LinkedHashMap<>(); + partitions.put(tidp1, new FetchResponseData.PartitionData() + .setPartitionIndex(tp1.partition()) + .setHighWatermark(100) + .setRecords(records)); + partitions.put(tidp0, new FetchResponseData.PartitionData() + .setPartitionIndex(tp0.partition()) + .setErrorCode(Errors.OFFSET_OUT_OF_RANGE.code()) + .setHighWatermark(100)); + partitions.put(tidp2, new FetchResponseData.PartitionData() + .setPartitionIndex(tp2.partition()) + .setHighWatermark(100) + .setLastStableOffset(4) + .setLogStartOffset(0) + .setRecords(nextRecords)); + partitions.put(tidp3, new FetchResponseData.PartitionData() + .setPartitionIndex(tp3.partition()) + .setHighWatermark(100) + .setLastStableOffset(4) + .setLogStartOffset(0) + .setRecords(partialRecords)); + client.prepareResponse(FetchResponse.of(Errors.NONE, 0, INVALID_SESSION_ID, new LinkedHashMap<>(partitions))); + consumerClient.poll(time.timer(0)); + + List> fetchedRecords = new ArrayList<>(); + Map>> recordsByPartition = fetchedRecords(); + for (List> records : recordsByPartition.values()) + fetchedRecords.addAll(records); + + assertEquals(fetchedRecords.size(), subscriptions.position(tp1).offset - 1); + assertEquals(4, subscriptions.position(tp1).offset); + assertEquals(3, fetchedRecords.size()); + + List oorExceptions = new ArrayList<>(); + try { + recordsByPartition = fetchedRecords(); + for (List> records : recordsByPartition.values()) + fetchedRecords.addAll(records); + } catch (OffsetOutOfRangeException oor) { + oorExceptions.add(oor); + } + + // Should have received one OffsetOutOfRangeException for partition tp1 + assertEquals(1, oorExceptions.size()); + OffsetOutOfRangeException oor = oorExceptions.get(0); + assertTrue(oor.offsetOutOfRangePartitions().containsKey(tp0)); + assertEquals(oor.offsetOutOfRangePartitions().size(), 1); + + recordsByPartition = fetchedRecords(); + for (List> records : recordsByPartition.values()) + fetchedRecords.addAll(records); + + // Should not have received an Exception for tp2. + assertEquals(6, subscriptions.position(tp2).offset); + assertEquals(5, fetchedRecords.size()); + + int numExceptionsExpected = 3; + List kafkaExceptions = new ArrayList<>(); + for (int i = 1; i <= numExceptionsExpected; i++) { + try { + recordsByPartition = fetchedRecords(); + for (List> records : recordsByPartition.values()) + fetchedRecords.addAll(records); + } catch (KafkaException e) { + kafkaExceptions.add(e); + } + } + // Should have received as much as numExceptionsExpected Kafka exceptions for tp3. + assertEquals(numExceptionsExpected, kafkaExceptions.size()); + } + + @Test + public void testSeekBeforeException() { + buildFetcher(OffsetResetStrategy.NONE, new ByteArrayDeserializer(), + new ByteArrayDeserializer(), 2, IsolationLevel.READ_UNCOMMITTED); + + assignFromUser(mkSet(tp0)); + subscriptions.seek(tp0, 1); + assertEquals(1, sendFetches()); + Map partitions = new HashMap<>(); + partitions.put(tidp0, new FetchResponseData.PartitionData() + .setPartitionIndex(tp0.partition()) + .setHighWatermark(100) + .setRecords(records)); + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + consumerClient.poll(time.timer(0)); + + assertEquals(2, fetchedRecords().get(tp0).size()); + + subscriptions.assignFromUser(mkSet(tp0, tp1)); + subscriptions.seekUnvalidated(tp1, new SubscriptionState.FetchPosition(1, Optional.empty(), metadata.currentLeader(tp1))); + + assertEquals(1, sendFetches()); + partitions = new HashMap<>(); + partitions.put(tidp1, new FetchResponseData.PartitionData() + .setPartitionIndex(tp1.partition()) + .setErrorCode(Errors.OFFSET_OUT_OF_RANGE.code()) + .setHighWatermark(100)); + client.prepareResponse(FetchResponse.of(Errors.NONE, 0, INVALID_SESSION_ID, new LinkedHashMap<>(partitions))); + consumerClient.poll(time.timer(0)); + assertEquals(1, fetchedRecords().get(tp0).size()); + + subscriptions.seek(tp1, 10); + // Should not throw OffsetOutOfRangeException after the seek + assertEmptyFetch("Should not return records or advance position after seeking to end of topic partitions"); + } + + @Test + public void testFetchDisconnected() { + buildFetcher(); + + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + assertEquals(1, sendFetches()); + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0), true); + consumerClient.poll(time.timer(0)); + assertEmptyFetch("Should not return records or advance position on disconnect"); + + // disconnects should have no affect on subscription state + assertFalse(subscriptions.isOffsetResetNeeded(tp0)); + assertTrue(subscriptions.isFetchable(tp0)); + assertEquals(0, subscriptions.position(tp0).offset); + } + + /* + * Send multiple requests. Verify that the client side quota metrics have the right values + */ + @Test + public void testQuotaMetrics() { + buildFetcher(); + + MockSelector selector = new MockSelector(time); + Cluster cluster = TestUtils.singletonCluster("test", 1); + Node node = cluster.nodes().get(0); + NetworkClient client = new NetworkClient(selector, metadata, "mock", Integer.MAX_VALUE, + 1000, 1000, 64 * 1024, 64 * 1024, 1000, 10 * 1000, 127 * 1000, + time, true, new ApiVersions(), metricsManager.throttleTimeSensor(), new LogContext()); + + ApiVersionsResponse apiVersionsResponse = TestUtils.defaultApiVersionsResponse( + 400, ApiMessageType.ListenerType.ZK_BROKER); + ByteBuffer buffer = RequestTestUtils.serializeResponseWithHeader(apiVersionsResponse, ApiKeys.API_VERSIONS.latestVersion(), 0); + + selector.delayedReceive(new DelayedReceive(node.idString(), new NetworkReceive(node.idString(), buffer))); + while (!client.ready(node, time.milliseconds())) { + client.poll(1, time.milliseconds()); + // If a throttled response is received, advance the time to ensure progress. + time.sleep(client.throttleDelayMs(node, time.milliseconds())); + } + selector.clear(); + + for (int i = 1; i <= 3; i++) { + int throttleTimeMs = 100 * i; + FetchRequest.Builder builder = FetchRequest.Builder.forConsumer(ApiKeys.FETCH.latestVersion(), 100, 100, new LinkedHashMap<>()); + builder.rackId(""); + ClientRequest request = client.newClientRequest(node.idString(), builder, time.milliseconds(), true); + client.send(request, time.milliseconds()); + client.poll(1, time.milliseconds()); + FetchResponse response = fullFetchResponse(tidp0, nextRecords, Errors.NONE, i, throttleTimeMs); + buffer = RequestTestUtils.serializeResponseWithHeader(response, ApiKeys.FETCH.latestVersion(), request.correlationId()); + selector.completeReceive(new NetworkReceive(node.idString(), buffer)); + client.poll(1, time.milliseconds()); + // If a throttled response is received, advance the time to ensure progress. + time.sleep(client.throttleDelayMs(node, time.milliseconds())); + selector.clear(); + } + Map allMetrics = metrics.metrics(); + KafkaMetric avgMetric = allMetrics.get(metrics.metricInstance(metricsRegistry.fetchThrottleTimeAvg)); + KafkaMetric maxMetric = allMetrics.get(metrics.metricInstance(metricsRegistry.fetchThrottleTimeMax)); + // Throttle times are ApiVersions=400, Fetch=(100, 200, 300) + assertEquals(250, (Double) avgMetric.metricValue(), EPSILON); + assertEquals(400, (Double) maxMetric.metricValue(), EPSILON); + client.close(); + } + + /* + * Send multiple requests. Verify that the client side quota metrics have the right values + */ + @Test + public void testFetcherMetrics() { + buildFetcher(); + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + MetricName maxLagMetric = metrics.metricInstance(metricsRegistry.recordsLagMax); + Map tags = new HashMap<>(); + tags.put("topic", tp0.topic()); + tags.put("partition", String.valueOf(tp0.partition())); + MetricName partitionLagMetric = metrics.metricName("records-lag", metricGroup, tags); + + Map allMetrics = metrics.metrics(); + KafkaMetric recordsFetchLagMax = allMetrics.get(maxLagMetric); + + // recordsFetchLagMax should be initialized to NaN + assertEquals(Double.NaN, (Double) recordsFetchLagMax.metricValue(), EPSILON); + + // recordsFetchLagMax should be hw - fetchOffset after receiving an empty FetchResponse + fetchRecords(tidp0, MemoryRecords.EMPTY, Errors.NONE, 100L, 0); + assertEquals(100, (Double) recordsFetchLagMax.metricValue(), EPSILON); + + KafkaMetric partitionLag = allMetrics.get(partitionLagMetric); + assertEquals(100, (Double) partitionLag.metricValue(), EPSILON); + + // recordsFetchLagMax should be hw - offset of the last message after receiving a non-empty FetchResponse + MemoryRecordsBuilder builder = MemoryRecords.builder(ByteBuffer.allocate(1024), CompressionType.NONE, + TimestampType.CREATE_TIME, 0L); + for (int v = 0; v < 3; v++) + builder.appendWithOffset(v, RecordBatch.NO_TIMESTAMP, "key".getBytes(), ("value-" + v).getBytes()); + fetchRecords(tidp0, builder.build(), Errors.NONE, 200L, 0); + assertEquals(197, (Double) recordsFetchLagMax.metricValue(), EPSILON); + assertEquals(197, (Double) partitionLag.metricValue(), EPSILON); + + // verify de-registration of partition lag + subscriptions.unsubscribe(); + sendFetches(); + assertFalse(allMetrics.containsKey(partitionLagMetric)); + } + + @Test + public void testFetcherLeadMetric() { + buildFetcher(); + + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + MetricName minLeadMetric = metrics.metricInstance(metricsRegistry.recordsLeadMin); + Map tags = new HashMap<>(2); + tags.put("topic", tp0.topic()); + tags.put("partition", String.valueOf(tp0.partition())); + MetricName partitionLeadMetric = metrics.metricName("records-lead", metricGroup, "", tags); + + Map allMetrics = metrics.metrics(); + KafkaMetric recordsFetchLeadMin = allMetrics.get(minLeadMetric); + + // recordsFetchLeadMin should be initialized to NaN + assertEquals(Double.NaN, (Double) recordsFetchLeadMin.metricValue(), EPSILON); + + // recordsFetchLeadMin should be position - logStartOffset after receiving an empty FetchResponse + fetchRecords(tidp0, MemoryRecords.EMPTY, Errors.NONE, 100L, -1L, 0L, 0); + assertEquals(0L, (Double) recordsFetchLeadMin.metricValue(), EPSILON); + + KafkaMetric partitionLead = allMetrics.get(partitionLeadMetric); + assertEquals(0L, (Double) partitionLead.metricValue(), EPSILON); + + // recordsFetchLeadMin should be position - logStartOffset after receiving a non-empty FetchResponse + MemoryRecordsBuilder builder = MemoryRecords.builder(ByteBuffer.allocate(1024), CompressionType.NONE, + TimestampType.CREATE_TIME, 0L); + for (int v = 0; v < 3; v++) { + builder.appendWithOffset(v, RecordBatch.NO_TIMESTAMP, "key".getBytes(), ("value-" + v).getBytes()); + } + fetchRecords(tidp0, builder.build(), Errors.NONE, 200L, -1L, 0L, 0); + assertEquals(0L, (Double) recordsFetchLeadMin.metricValue(), EPSILON); + assertEquals(3L, (Double) partitionLead.metricValue(), EPSILON); + + // verify de-registration of partition lag + subscriptions.unsubscribe(); + sendFetches(); + assertFalse(allMetrics.containsKey(partitionLeadMetric)); + } + + @Test + public void testReadCommittedLagMetric() { + buildFetcher(OffsetResetStrategy.EARLIEST, new ByteArrayDeserializer(), + new ByteArrayDeserializer(), Integer.MAX_VALUE, IsolationLevel.READ_COMMITTED); + + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + MetricName maxLagMetric = metrics.metricInstance(metricsRegistry.recordsLagMax); + + Map tags = new HashMap<>(); + tags.put("topic", tp0.topic()); + tags.put("partition", String.valueOf(tp0.partition())); + MetricName partitionLagMetric = metrics.metricName("records-lag", metricGroup, tags); + + Map allMetrics = metrics.metrics(); + KafkaMetric recordsFetchLagMax = allMetrics.get(maxLagMetric); + + // recordsFetchLagMax should be initialized to NaN + assertEquals(Double.NaN, (Double) recordsFetchLagMax.metricValue(), EPSILON); + + // recordsFetchLagMax should be lso - fetchOffset after receiving an empty FetchResponse + fetchRecords(tidp0, MemoryRecords.EMPTY, Errors.NONE, 100L, 50L, 0); + assertEquals(50, (Double) recordsFetchLagMax.metricValue(), EPSILON); + + KafkaMetric partitionLag = allMetrics.get(partitionLagMetric); + assertEquals(50, (Double) partitionLag.metricValue(), EPSILON); + + // recordsFetchLagMax should be lso - offset of the last message after receiving a non-empty FetchResponse + MemoryRecordsBuilder builder = MemoryRecords.builder(ByteBuffer.allocate(1024), CompressionType.NONE, + TimestampType.CREATE_TIME, 0L); + for (int v = 0; v < 3; v++) + builder.appendWithOffset(v, RecordBatch.NO_TIMESTAMP, "key".getBytes(), ("value-" + v).getBytes()); + fetchRecords(tidp0, builder.build(), Errors.NONE, 200L, 150L, 0); + assertEquals(147, (Double) recordsFetchLagMax.metricValue(), EPSILON); + assertEquals(147, (Double) partitionLag.metricValue(), EPSILON); + + // verify de-registration of partition lag + subscriptions.unsubscribe(); + sendFetches(); + assertFalse(allMetrics.containsKey(partitionLagMetric)); + } + + @Test + public void testFetchResponseMetrics() { + buildFetcher(); + + String topic1 = "foo"; + String topic2 = "bar"; + TopicPartition tp1 = new TopicPartition(topic1, 0); + TopicPartition tp2 = new TopicPartition(topic2, 0); + + subscriptions.assignFromUser(mkSet(tp1, tp2)); + + Map partitionCounts = new HashMap<>(); + partitionCounts.put(topic1, 1); + partitionCounts.put(topic2, 1); + topicIds.put(topic1, Uuid.randomUuid()); + topicIds.put(topic2, Uuid.randomUuid()); + TopicIdPartition tidp1 = new TopicIdPartition(topicIds.get(topic1), tp1); + TopicIdPartition tidp2 = new TopicIdPartition(topicIds.get(topic2), tp2); + client.updateMetadata(RequestTestUtils.metadataUpdateWithIds(1, partitionCounts, tp -> validLeaderEpoch, topicIds)); + + int expectedBytes = 0; + LinkedHashMap fetchPartitionData = new LinkedHashMap<>(); + + for (TopicIdPartition tp : mkSet(tidp1, tidp2)) { + subscriptions.seek(tp.topicPartition(), 0); + + MemoryRecordsBuilder builder = MemoryRecords.builder(ByteBuffer.allocate(1024), CompressionType.NONE, + TimestampType.CREATE_TIME, 0L); + for (int v = 0; v < 3; v++) + builder.appendWithOffset(v, RecordBatch.NO_TIMESTAMP, "key".getBytes(), ("value-" + v).getBytes()); + MemoryRecords records = builder.build(); + for (Record record : records.records()) + expectedBytes += record.sizeInBytes(); + + fetchPartitionData.put(tp, new FetchResponseData.PartitionData() + .setPartitionIndex(tp.topicPartition().partition()) + .setHighWatermark(15) + .setLogStartOffset(0) + .setRecords(records)); + } + + assertEquals(1, sendFetches()); + client.prepareResponse(FetchResponse.of(Errors.NONE, 0, INVALID_SESSION_ID, fetchPartitionData)); + consumerClient.poll(time.timer(0)); + + Map>> fetchedRecords = fetchedRecords(); + assertEquals(3, fetchedRecords.get(tp1).size()); + assertEquals(3, fetchedRecords.get(tp2).size()); + + Map allMetrics = metrics.metrics(); + KafkaMetric fetchSizeAverage = allMetrics.get(metrics.metricInstance(metricsRegistry.fetchSizeAvg)); + KafkaMetric recordsCountAverage = allMetrics.get(metrics.metricInstance(metricsRegistry.recordsPerRequestAvg)); + assertEquals(expectedBytes, (Double) fetchSizeAverage.metricValue(), EPSILON); + assertEquals(6, (Double) recordsCountAverage.metricValue(), EPSILON); + } + + @Test + public void testFetchResponseMetricsPartialResponse() { + buildFetcher(); + + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 1); + + Map allMetrics = metrics.metrics(); + KafkaMetric fetchSizeAverage = allMetrics.get(metrics.metricInstance(metricsRegistry.fetchSizeAvg)); + KafkaMetric recordsCountAverage = allMetrics.get(metrics.metricInstance(metricsRegistry.recordsPerRequestAvg)); + + MemoryRecordsBuilder builder = MemoryRecords.builder(ByteBuffer.allocate(1024), CompressionType.NONE, + TimestampType.CREATE_TIME, 0L); + for (int v = 0; v < 3; v++) + builder.appendWithOffset(v, RecordBatch.NO_TIMESTAMP, "key".getBytes(), ("value-" + v).getBytes()); + MemoryRecords records = builder.build(); + + int expectedBytes = 0; + for (Record record : records.records()) { + if (record.offset() >= 1) + expectedBytes += record.sizeInBytes(); + } + + fetchRecords(tidp0, records, Errors.NONE, 100L, 0); + assertEquals(expectedBytes, (Double) fetchSizeAverage.metricValue(), EPSILON); + assertEquals(2, (Double) recordsCountAverage.metricValue(), EPSILON); + } + + @Test + public void testFetchResponseMetricsWithOnePartitionError() { + buildFetcher(); + assignFromUser(mkSet(tp0, tp1)); + subscriptions.seek(tp0, 0); + subscriptions.seek(tp1, 0); + + Map allMetrics = metrics.metrics(); + KafkaMetric fetchSizeAverage = allMetrics.get(metrics.metricInstance(metricsRegistry.fetchSizeAvg)); + KafkaMetric recordsCountAverage = allMetrics.get(metrics.metricInstance(metricsRegistry.recordsPerRequestAvg)); + + MemoryRecordsBuilder builder = MemoryRecords.builder(ByteBuffer.allocate(1024), CompressionType.NONE, + TimestampType.CREATE_TIME, 0L); + for (int v = 0; v < 3; v++) + builder.appendWithOffset(v, RecordBatch.NO_TIMESTAMP, "key".getBytes(), ("value-" + v).getBytes()); + MemoryRecords records = builder.build(); + + Map partitions = new HashMap<>(); + partitions.put(tidp0, new FetchResponseData.PartitionData() + .setPartitionIndex(tp0.partition()) + .setHighWatermark(100) + .setLogStartOffset(0) + .setRecords(records)); + partitions.put(tidp1, new FetchResponseData.PartitionData() + .setPartitionIndex(tp1.partition()) + .setErrorCode(Errors.OFFSET_OUT_OF_RANGE.code()) + .setHighWatermark(100) + .setLogStartOffset(0)); + + assertEquals(1, sendFetches()); + client.prepareResponse(FetchResponse.of(Errors.NONE, 0, INVALID_SESSION_ID, new LinkedHashMap<>(partitions))); + consumerClient.poll(time.timer(0)); + fetcher.collectFetch(); + + int expectedBytes = 0; + for (Record record : records.records()) + expectedBytes += record.sizeInBytes(); + + assertEquals(expectedBytes, (Double) fetchSizeAverage.metricValue(), EPSILON); + assertEquals(3, (Double) recordsCountAverage.metricValue(), EPSILON); + } + + @Test + public void testFetchResponseMetricsWithOnePartitionAtTheWrongOffset() { + buildFetcher(); + + assignFromUser(mkSet(tp0, tp1)); + subscriptions.seek(tp0, 0); + subscriptions.seek(tp1, 0); + + Map allMetrics = metrics.metrics(); + KafkaMetric fetchSizeAverage = allMetrics.get(metrics.metricInstance(metricsRegistry.fetchSizeAvg)); + KafkaMetric recordsCountAverage = allMetrics.get(metrics.metricInstance(metricsRegistry.recordsPerRequestAvg)); + + // send the fetch and then seek to a new offset + assertEquals(1, sendFetches()); + subscriptions.seek(tp1, 5); + + MemoryRecordsBuilder builder = MemoryRecords.builder(ByteBuffer.allocate(1024), CompressionType.NONE, + TimestampType.CREATE_TIME, 0L); + for (int v = 0; v < 3; v++) + builder.appendWithOffset(v, RecordBatch.NO_TIMESTAMP, "key".getBytes(), ("value-" + v).getBytes()); + MemoryRecords records = builder.build(); + + Map partitions = new HashMap<>(); + partitions.put(tidp0, new FetchResponseData.PartitionData() + .setPartitionIndex(tp0.partition()) + .setHighWatermark(100) + .setLogStartOffset(0) + .setRecords(records)); + partitions.put(tidp1, new FetchResponseData.PartitionData() + .setPartitionIndex(tp1.partition()) + .setHighWatermark(100) + .setLogStartOffset(0) + .setRecords(MemoryRecords.withRecords(CompressionType.NONE, new SimpleRecord("val".getBytes())))); + + client.prepareResponse(FetchResponse.of(Errors.NONE, 0, INVALID_SESSION_ID, new LinkedHashMap<>(partitions))); + consumerClient.poll(time.timer(0)); + fetcher.collectFetch(); + + // we should have ignored the record at the wrong offset + int expectedBytes = 0; + for (Record record : records.records()) + expectedBytes += record.sizeInBytes(); + + assertEquals(expectedBytes, (Double) fetchSizeAverage.metricValue(), EPSILON); + assertEquals(3, (Double) recordsCountAverage.metricValue(), EPSILON); + } + + @Test + public void testFetcherMetricsTemplates() { + Map clientTags = Collections.singletonMap("client-id", "clientA"); + buildFetcher(new MetricConfig().tags(clientTags), OffsetResetStrategy.EARLIEST, new ByteArrayDeserializer(), + new ByteArrayDeserializer(), Integer.MAX_VALUE, IsolationLevel.READ_UNCOMMITTED); + + // Fetch from topic to generate topic metrics + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + assertEquals(1, sendFetches()); + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + Map>> partitionRecords = fetchedRecords(); + assertTrue(partitionRecords.containsKey(tp0)); + + // Verify that all metrics except metrics-count have registered templates + Set allMetrics = new HashSet<>(); + for (MetricName n : metrics.metrics().keySet()) { + String name = n.name().replaceAll(tp0.toString(), "{topic}-{partition}"); + if (!n.group().equals("kafka-metrics-count")) + allMetrics.add(new MetricNameTemplate(name, n.group(), "", n.tags().keySet())); + } + TestUtils.checkEquals(allMetrics, new HashSet<>(metricsRegistry.getAllTemplates()), "metrics", "templates"); + } + + private Map>> fetchRecords( + TopicIdPartition tp, MemoryRecords records, Errors error, long hw, int throttleTime) { + return fetchRecords(tp, records, error, hw, FetchResponse.INVALID_LAST_STABLE_OFFSET, throttleTime); + } + + private Map>> fetchRecords( + TopicIdPartition tp, MemoryRecords records, Errors error, long hw, long lastStableOffset, int throttleTime) { + assertEquals(1, sendFetches()); + client.prepareResponse(fullFetchResponse(tp, records, error, hw, lastStableOffset, throttleTime)); + consumerClient.poll(time.timer(0)); + return fetchedRecords(); + } + + private Map>> fetchRecords( + TopicIdPartition tp, MemoryRecords records, Errors error, long hw, long lastStableOffset, long logStartOffset, int throttleTime) { + assertEquals(1, sendFetches()); + client.prepareResponse(fetchResponse(tp, records, error, hw, lastStableOffset, logStartOffset, throttleTime)); + consumerClient.poll(time.timer(0)); + return fetchedRecords(); + } + + @Test + public void testSkippingAbortedTransactions() { + buildFetcher(OffsetResetStrategy.EARLIEST, new ByteArrayDeserializer(), + new ByteArrayDeserializer(), Integer.MAX_VALUE, IsolationLevel.READ_COMMITTED); + ByteBuffer buffer = ByteBuffer.allocate(1024); + int currentOffset = 0; + + currentOffset += appendTransactionalRecords(buffer, 1L, currentOffset, + new SimpleRecord(time.milliseconds(), "key".getBytes(), "value".getBytes()), + new SimpleRecord(time.milliseconds(), "key".getBytes(), "value".getBytes())); + + abortTransaction(buffer, 1L, currentOffset); + + buffer.flip(); + + List abortedTransactions = Collections.singletonList( + new FetchResponseData.AbortedTransaction().setProducerId(1).setFirstOffset(0)); + MemoryRecords records = MemoryRecords.readableRecords(buffer); + assignFromUser(singleton(tp0)); + + subscriptions.seek(tp0, 0); + + // normal fetch + assertEquals(1, sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + + client.prepareResponse(fullFetchResponseWithAbortedTransactions(records, abortedTransactions, Errors.NONE, 100L, 100L, 0)); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + + Fetch fetch = collectFetch(); + assertEquals(emptyMap(), fetch.records()); + assertTrue(fetch.positionAdvanced()); + } + + @Test + public void testReturnCommittedTransactions() { + buildFetcher(OffsetResetStrategy.EARLIEST, new ByteArrayDeserializer(), + new ByteArrayDeserializer(), Integer.MAX_VALUE, IsolationLevel.READ_COMMITTED); + ByteBuffer buffer = ByteBuffer.allocate(1024); + int currentOffset = 0; + + currentOffset += appendTransactionalRecords(buffer, 1L, currentOffset, + new SimpleRecord(time.milliseconds(), "key".getBytes(), "value".getBytes()), + new SimpleRecord(time.milliseconds(), "key".getBytes(), "value".getBytes())); + + commitTransaction(buffer, 1L, currentOffset); + buffer.flip(); + + MemoryRecords records = MemoryRecords.readableRecords(buffer); + assignFromUser(singleton(tp0)); + + subscriptions.seek(tp0, 0); + + // normal fetch + assertEquals(1, sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + client.prepareResponse(body -> { + FetchRequest request = (FetchRequest) body; + assertEquals(IsolationLevel.READ_COMMITTED, request.isolationLevel()); + return true; + }, fullFetchResponseWithAbortedTransactions(records, Collections.emptyList(), Errors.NONE, 100L, 100L, 0)); + + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + + Map>> fetchedRecords = fetchedRecords(); + assertTrue(fetchedRecords.containsKey(tp0)); + assertEquals(fetchedRecords.get(tp0).size(), 2); + } + + @Test + public void testReadCommittedWithCommittedAndAbortedTransactions() { + buildFetcher(OffsetResetStrategy.EARLIEST, new ByteArrayDeserializer(), + new ByteArrayDeserializer(), Integer.MAX_VALUE, IsolationLevel.READ_COMMITTED); + ByteBuffer buffer = ByteBuffer.allocate(1024); + + List abortedTransactions = new ArrayList<>(); + + long pid1 = 1L; + long pid2 = 2L; + + // Appends for producer 1 (eventually committed) + appendTransactionalRecords(buffer, pid1, 0L, + new SimpleRecord("commit1-1".getBytes(), "value".getBytes()), + new SimpleRecord("commit1-2".getBytes(), "value".getBytes())); + + // Appends for producer 2 (eventually aborted) + appendTransactionalRecords(buffer, pid2, 2L, + new SimpleRecord("abort2-1".getBytes(), "value".getBytes())); + + // commit producer 1 + commitTransaction(buffer, pid1, 3L); + + // append more for producer 2 (eventually aborted) + appendTransactionalRecords(buffer, pid2, 4L, + new SimpleRecord("abort2-2".getBytes(), "value".getBytes())); + + // abort producer 2 + abortTransaction(buffer, pid2, 5L); + abortedTransactions.add(new FetchResponseData.AbortedTransaction().setProducerId(pid2).setFirstOffset(2L)); + + // New transaction for producer 1 (eventually aborted) + appendTransactionalRecords(buffer, pid1, 6L, + new SimpleRecord("abort1-1".getBytes(), "value".getBytes())); + + // New transaction for producer 2 (eventually committed) + appendTransactionalRecords(buffer, pid2, 7L, + new SimpleRecord("commit2-1".getBytes(), "value".getBytes())); + + // Add messages for producer 1 (eventually aborted) + appendTransactionalRecords(buffer, pid1, 8L, + new SimpleRecord("abort1-2".getBytes(), "value".getBytes())); + + // abort producer 1 + abortTransaction(buffer, pid1, 9L); + abortedTransactions.add(new FetchResponseData.AbortedTransaction().setProducerId(1).setFirstOffset(6)); + + // commit producer 2 + commitTransaction(buffer, pid2, 10L); + + buffer.flip(); + + MemoryRecords records = MemoryRecords.readableRecords(buffer); + assignFromUser(singleton(tp0)); + + subscriptions.seek(tp0, 0); + + // normal fetch + assertEquals(1, sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + + client.prepareResponse(fullFetchResponseWithAbortedTransactions(records, abortedTransactions, Errors.NONE, 100L, 100L, 0)); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + + Map>> fetchedRecords = fetchedRecords(); + assertTrue(fetchedRecords.containsKey(tp0)); + // There are only 3 committed records + List> fetchedConsumerRecords = fetchedRecords.get(tp0); + Set fetchedKeys = new HashSet<>(); + for (ConsumerRecord consumerRecord : fetchedConsumerRecords) { + fetchedKeys.add(new String(consumerRecord.key(), StandardCharsets.UTF_8)); + } + assertEquals(mkSet("commit1-1", "commit1-2", "commit2-1"), fetchedKeys); + } + + @Test + public void testMultipleAbortMarkers() { + buildFetcher(OffsetResetStrategy.EARLIEST, new ByteArrayDeserializer(), + new ByteArrayDeserializer(), Integer.MAX_VALUE, IsolationLevel.READ_COMMITTED); + ByteBuffer buffer = ByteBuffer.allocate(1024); + int currentOffset = 0; + + currentOffset += appendTransactionalRecords(buffer, 1L, currentOffset, + new SimpleRecord(time.milliseconds(), "abort1-1".getBytes(), "value".getBytes()), + new SimpleRecord(time.milliseconds(), "abort1-2".getBytes(), "value".getBytes())); + + currentOffset += abortTransaction(buffer, 1L, currentOffset); + // Duplicate abort -- should be ignored. + currentOffset += abortTransaction(buffer, 1L, currentOffset); + // Now commit a transaction. + currentOffset += appendTransactionalRecords(buffer, 1L, currentOffset, + new SimpleRecord(time.milliseconds(), "commit1-1".getBytes(), "value".getBytes()), + new SimpleRecord(time.milliseconds(), "commit1-2".getBytes(), "value".getBytes())); + commitTransaction(buffer, 1L, currentOffset); + buffer.flip(); + + List abortedTransactions = Collections.singletonList( + new FetchResponseData.AbortedTransaction().setProducerId(1).setFirstOffset(0) + ); + MemoryRecords records = MemoryRecords.readableRecords(buffer); + assignFromUser(singleton(tp0)); + + subscriptions.seek(tp0, 0); + + // normal fetch + assertEquals(1, sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + + client.prepareResponse(fullFetchResponseWithAbortedTransactions(records, abortedTransactions, Errors.NONE, 100L, 100L, 0)); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + + Map>> fetchedRecords = fetchedRecords(); + assertTrue(fetchedRecords.containsKey(tp0)); + assertEquals(fetchedRecords.get(tp0).size(), 2); + List> fetchedConsumerRecords = fetchedRecords.get(tp0); + Set committedKeys = new HashSet<>(Arrays.asList("commit1-1", "commit1-2")); + Set actuallyCommittedKeys = new HashSet<>(); + for (ConsumerRecord consumerRecord : fetchedConsumerRecords) { + actuallyCommittedKeys.add(new String(consumerRecord.key(), StandardCharsets.UTF_8)); + } + assertEquals(actuallyCommittedKeys, committedKeys); + } + + @Test + public void testReadCommittedAbortMarkerWithNoData() { + buildFetcher(OffsetResetStrategy.EARLIEST, new StringDeserializer(), + new StringDeserializer(), Integer.MAX_VALUE, IsolationLevel.READ_COMMITTED); + ByteBuffer buffer = ByteBuffer.allocate(1024); + + long producerId = 1L; + + abortTransaction(buffer, producerId, 5L); + + appendTransactionalRecords(buffer, producerId, 6L, + new SimpleRecord("6".getBytes(), null), + new SimpleRecord("7".getBytes(), null), + new SimpleRecord("8".getBytes(), null)); + + commitTransaction(buffer, producerId, 9L); + + buffer.flip(); + + // send the fetch + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + assertEquals(1, sendFetches()); + + // prepare the response. the aborted transactions begin at offsets which are no longer in the log + List abortedTransactions = Collections.singletonList( + new FetchResponseData.AbortedTransaction().setProducerId(producerId).setFirstOffset(0L)); + + client.prepareResponse(fullFetchResponseWithAbortedTransactions(MemoryRecords.readableRecords(buffer), + abortedTransactions, Errors.NONE, 100L, 100L, 0)); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + + Map>> allFetchedRecords = fetchedRecords(); + assertTrue(allFetchedRecords.containsKey(tp0)); + List> fetchedRecords = allFetchedRecords.get(tp0); + assertEquals(3, fetchedRecords.size()); + assertEquals(Arrays.asList(6L, 7L, 8L), collectRecordOffsets(fetchedRecords)); + } + + @Test + public void testUpdatePositionWithLastRecordMissingFromBatch() { + buildFetcher(); + + MemoryRecords records = MemoryRecords.withRecords(CompressionType.NONE, + new SimpleRecord("0".getBytes(), "v".getBytes()), + new SimpleRecord("1".getBytes(), "v".getBytes()), + new SimpleRecord("2".getBytes(), "v".getBytes()), + new SimpleRecord(null, "value".getBytes())); + + // Remove the last record to simulate compaction + MemoryRecords.FilterResult result = records.filterTo(tp0, new MemoryRecords.RecordFilter(0, 0) { + @Override + protected BatchRetentionResult checkBatchRetention(RecordBatch batch) { + return new BatchRetentionResult(BatchRetention.DELETE_EMPTY, false); + } + + @Override + protected boolean shouldRetainRecord(RecordBatch recordBatch, Record record) { + return record.key() != null; + } + }, ByteBuffer.allocate(1024), Integer.MAX_VALUE, BufferSupplier.NO_CACHING); + result.outputBuffer().flip(); + MemoryRecords compactedRecords = MemoryRecords.readableRecords(result.outputBuffer()); + + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + assertEquals(1, sendFetches()); + client.prepareResponse(fullFetchResponse(tidp0, compactedRecords, Errors.NONE, 100L, 0)); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + + Map>> allFetchedRecords = fetchedRecords(); + assertTrue(allFetchedRecords.containsKey(tp0)); + List> fetchedRecords = allFetchedRecords.get(tp0); + assertEquals(3, fetchedRecords.size()); + + for (int i = 0; i < 3; i++) { + assertEquals(Integer.toString(i), new String(fetchedRecords.get(i).key())); + } + + // The next offset should point to the next batch + assertEquals(4L, subscriptions.position(tp0).offset); + } + + @Test + public void testUpdatePositionOnEmptyBatch() { + buildFetcher(); + + long producerId = 1; + short producerEpoch = 0; + int sequence = 1; + long baseOffset = 37; + long lastOffset = 54; + int partitionLeaderEpoch = 7; + ByteBuffer buffer = ByteBuffer.allocate(DefaultRecordBatch.RECORD_BATCH_OVERHEAD); + DefaultRecordBatch.writeEmptyHeader(buffer, RecordBatch.CURRENT_MAGIC_VALUE, producerId, producerEpoch, + sequence, baseOffset, lastOffset, partitionLeaderEpoch, TimestampType.CREATE_TIME, + System.currentTimeMillis(), false, false); + buffer.flip(); + MemoryRecords recordsWithEmptyBatch = MemoryRecords.readableRecords(buffer); + + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + assertEquals(1, sendFetches()); + client.prepareResponse(fullFetchResponse(tidp0, recordsWithEmptyBatch, Errors.NONE, 100L, 0)); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + + Fetch fetch = collectFetch(); + assertEquals(emptyMap(), fetch.records()); + assertTrue(fetch.positionAdvanced()); + + // The next offset should point to the next batch + assertEquals(lastOffset + 1, subscriptions.position(tp0).offset); + } + + @Test + public void testReadCommittedWithCompactedTopic() { + buildFetcher(OffsetResetStrategy.EARLIEST, new StringDeserializer(), + new StringDeserializer(), Integer.MAX_VALUE, IsolationLevel.READ_COMMITTED); + ByteBuffer buffer = ByteBuffer.allocate(1024); + + long pid1 = 1L; + long pid2 = 2L; + long pid3 = 3L; + + appendTransactionalRecords(buffer, pid3, 3L, + new SimpleRecord("3".getBytes(), "value".getBytes()), + new SimpleRecord("4".getBytes(), "value".getBytes())); + + appendTransactionalRecords(buffer, pid2, 15L, + new SimpleRecord("15".getBytes(), "value".getBytes()), + new SimpleRecord("16".getBytes(), "value".getBytes()), + new SimpleRecord("17".getBytes(), "value".getBytes())); + + appendTransactionalRecords(buffer, pid1, 22L, + new SimpleRecord("22".getBytes(), "value".getBytes()), + new SimpleRecord("23".getBytes(), "value".getBytes())); + + abortTransaction(buffer, pid2, 28L); + + appendTransactionalRecords(buffer, pid3, 30L, + new SimpleRecord("30".getBytes(), "value".getBytes()), + new SimpleRecord("31".getBytes(), "value".getBytes()), + new SimpleRecord("32".getBytes(), "value".getBytes())); + + commitTransaction(buffer, pid3, 35L); + + appendTransactionalRecords(buffer, pid1, 39L, + new SimpleRecord("39".getBytes(), "value".getBytes()), + new SimpleRecord("40".getBytes(), "value".getBytes())); + + // transaction from pid1 is aborted, but the marker is not included in the fetch + + buffer.flip(); + + // send the fetch + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + assertEquals(1, sendFetches()); + + // prepare the response. the aborted transactions begin at offsets which are no longer in the log + List abortedTransactions = Arrays.asList( + new FetchResponseData.AbortedTransaction().setProducerId(pid2).setFirstOffset(6), + new FetchResponseData.AbortedTransaction().setProducerId(pid1).setFirstOffset(0) + ); + + client.prepareResponse(fullFetchResponseWithAbortedTransactions(MemoryRecords.readableRecords(buffer), + abortedTransactions, Errors.NONE, 100L, 100L, 0)); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + + Map>> allFetchedRecords = fetchedRecords(); + assertTrue(allFetchedRecords.containsKey(tp0)); + List> fetchedRecords = allFetchedRecords.get(tp0); + assertEquals(5, fetchedRecords.size()); + assertEquals(Arrays.asList(3L, 4L, 30L, 31L, 32L), collectRecordOffsets(fetchedRecords)); + } + + @Test + public void testReturnAbortedTransactionsinUncommittedMode() { + buildFetcher(OffsetResetStrategy.EARLIEST, new ByteArrayDeserializer(), + new ByteArrayDeserializer(), Integer.MAX_VALUE, IsolationLevel.READ_UNCOMMITTED); + ByteBuffer buffer = ByteBuffer.allocate(1024); + int currentOffset = 0; + + currentOffset += appendTransactionalRecords(buffer, 1L, currentOffset, + new SimpleRecord(time.milliseconds(), "key".getBytes(), "value".getBytes()), + new SimpleRecord(time.milliseconds(), "key".getBytes(), "value".getBytes())); + + abortTransaction(buffer, 1L, currentOffset); + + buffer.flip(); + + List abortedTransactions = Collections.singletonList( + new FetchResponseData.AbortedTransaction().setProducerId(1).setFirstOffset(0)); + MemoryRecords records = MemoryRecords.readableRecords(buffer); + assignFromUser(singleton(tp0)); + + subscriptions.seek(tp0, 0); + + // normal fetch + assertEquals(1, sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + + client.prepareResponse(fullFetchResponseWithAbortedTransactions(records, abortedTransactions, Errors.NONE, 100L, 100L, 0)); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + + Map>> fetchedRecords = fetchedRecords(); + assertTrue(fetchedRecords.containsKey(tp0)); + } + + @Test + public void testConsumerPositionUpdatedWhenSkippingAbortedTransactions() { + buildFetcher(OffsetResetStrategy.EARLIEST, new ByteArrayDeserializer(), + new ByteArrayDeserializer(), Integer.MAX_VALUE, IsolationLevel.READ_COMMITTED); + ByteBuffer buffer = ByteBuffer.allocate(1024); + long currentOffset = 0; + + currentOffset += appendTransactionalRecords(buffer, 1L, currentOffset, + new SimpleRecord(time.milliseconds(), "abort1-1".getBytes(), "value".getBytes()), + new SimpleRecord(time.milliseconds(), "abort1-2".getBytes(), "value".getBytes())); + + currentOffset += abortTransaction(buffer, 1L, currentOffset); + buffer.flip(); + + List abortedTransactions = Collections.singletonList( + new FetchResponseData.AbortedTransaction().setProducerId(1).setFirstOffset(0)); + MemoryRecords records = MemoryRecords.readableRecords(buffer); + assignFromUser(singleton(tp0)); + + subscriptions.seek(tp0, 0); + + // normal fetch + assertEquals(1, sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + + client.prepareResponse(fullFetchResponseWithAbortedTransactions(records, abortedTransactions, Errors.NONE, 100L, 100L, 0)); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + + Map>> fetchedRecords = fetchedRecords(); + + // Ensure that we don't return any of the aborted records, but yet advance the consumer position. + assertFalse(fetchedRecords.containsKey(tp0)); + assertEquals(currentOffset, subscriptions.position(tp0).offset); + } + + @Test + public void testConsumingViaIncrementalFetchRequests() { + buildFetcher(2); + + List> records; + assignFromUser(new HashSet<>(Arrays.asList(tp0, tp1))); + subscriptions.seekValidated(tp0, new SubscriptionState.FetchPosition(0, Optional.empty(), metadata.currentLeader(tp0))); + subscriptions.seekValidated(tp1, new SubscriptionState.FetchPosition(1, Optional.empty(), metadata.currentLeader(tp1))); + + // Fetch some records and establish an incremental fetch session. + LinkedHashMap partitions1 = new LinkedHashMap<>(); + partitions1.put(tidp0, new FetchResponseData.PartitionData() + .setPartitionIndex(tp0.partition()) + .setHighWatermark(2) + .setLastStableOffset(2) + .setLogStartOffset(0) + .setRecords(this.records)); + partitions1.put(tidp1, new FetchResponseData.PartitionData() + .setPartitionIndex(tp1.partition()) + .setHighWatermark(100) + .setLogStartOffset(0) + .setRecords(emptyRecords)); + FetchResponse resp1 = FetchResponse.of(Errors.NONE, 0, 123, partitions1); + client.prepareResponse(resp1); + assertEquals(1, sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + Map>> fetchedRecords = fetchedRecords(); + assertFalse(fetchedRecords.containsKey(tp1)); + records = fetchedRecords.get(tp0); + assertEquals(2, records.size()); + assertEquals(3L, subscriptions.position(tp0).offset); + assertEquals(1L, subscriptions.position(tp1).offset); + assertEquals(1, records.get(0).offset()); + assertEquals(2, records.get(1).offset()); + + // There is still a buffered record. + assertEquals(0, sendFetches()); + fetchedRecords = fetchedRecords(); + assertFalse(fetchedRecords.containsKey(tp1)); + records = fetchedRecords.get(tp0); + assertEquals(1, records.size()); + assertEquals(3, records.get(0).offset()); + assertEquals(4L, subscriptions.position(tp0).offset); + + // The second response contains no new records. + LinkedHashMap partitions2 = new LinkedHashMap<>(); + FetchResponse resp2 = FetchResponse.of(Errors.NONE, 0, 123, partitions2); + client.prepareResponse(resp2); + assertEquals(1, sendFetches()); + consumerClient.poll(time.timer(0)); + fetchedRecords = fetchedRecords(); + assertTrue(fetchedRecords.isEmpty()); + assertEquals(4L, subscriptions.position(tp0).offset); + assertEquals(1L, subscriptions.position(tp1).offset); + + // The third response contains some new records for tp0. + LinkedHashMap partitions3 = new LinkedHashMap<>(); + partitions3.put(tidp0, new FetchResponseData.PartitionData() + .setPartitionIndex(tp0.partition()) + .setHighWatermark(100) + .setLastStableOffset(4) + .setLogStartOffset(0) + .setRecords(this.nextRecords)); + FetchResponse resp3 = FetchResponse.of(Errors.NONE, 0, 123, partitions3); + client.prepareResponse(resp3); + assertEquals(1, sendFetches()); + consumerClient.poll(time.timer(0)); + fetchedRecords = fetchedRecords(); + assertFalse(fetchedRecords.containsKey(tp1)); + records = fetchedRecords.get(tp0); + assertEquals(2, records.size()); + assertEquals(6L, subscriptions.position(tp0).offset); + assertEquals(1L, subscriptions.position(tp1).offset); + assertEquals(4, records.get(0).offset()); + assertEquals(5, records.get(1).offset()); + } + + @Test + public void testEmptyControlBatch() { + buildFetcher(OffsetResetStrategy.EARLIEST, new ByteArrayDeserializer(), + new ByteArrayDeserializer(), Integer.MAX_VALUE, IsolationLevel.READ_COMMITTED); + ByteBuffer buffer = ByteBuffer.allocate(1024); + int currentOffset = 1; + + // Empty control batch should not cause an exception + DefaultRecordBatch.writeEmptyHeader(buffer, RecordBatch.MAGIC_VALUE_V2, 1L, + (short) 0, -1, 0, 0, + RecordBatch.NO_PARTITION_LEADER_EPOCH, TimestampType.CREATE_TIME, time.milliseconds(), + true, true); + + currentOffset += appendTransactionalRecords(buffer, 1L, currentOffset, + new SimpleRecord(time.milliseconds(), "key".getBytes(), "value".getBytes()), + new SimpleRecord(time.milliseconds(), "key".getBytes(), "value".getBytes())); + + commitTransaction(buffer, 1L, currentOffset); + buffer.flip(); + + MemoryRecords records = MemoryRecords.readableRecords(buffer); + assignFromUser(singleton(tp0)); + + subscriptions.seek(tp0, 0); + + // normal fetch + assertEquals(1, sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + client.prepareResponse(body -> { + FetchRequest request = (FetchRequest) body; + assertEquals(IsolationLevel.READ_COMMITTED, request.isolationLevel()); + return true; + }, fullFetchResponseWithAbortedTransactions(records, Collections.emptyList(), Errors.NONE, 100L, 100L, 0)); + + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + + Map>> fetchedRecords = fetchedRecords(); + assertTrue(fetchedRecords.containsKey(tp0)); + assertEquals(fetchedRecords.get(tp0).size(), 2); + } + + private MemoryRecords buildRecords(long baseOffset, int count, long firstMessageId) { + MemoryRecordsBuilder builder = MemoryRecords.builder(ByteBuffer.allocate(1024), CompressionType.NONE, TimestampType.CREATE_TIME, baseOffset); + for (int i = 0; i < count; i++) + builder.append(0L, "key".getBytes(), ("value-" + (firstMessageId + i)).getBytes()); + return builder.build(); + } + + private int appendTransactionalRecords(ByteBuffer buffer, long pid, long baseOffset, int baseSequence, SimpleRecord... records) { + MemoryRecordsBuilder builder = MemoryRecords.builder(buffer, RecordBatch.CURRENT_MAGIC_VALUE, CompressionType.NONE, + TimestampType.CREATE_TIME, baseOffset, time.milliseconds(), pid, (short) 0, baseSequence, true, + RecordBatch.NO_PARTITION_LEADER_EPOCH); + + for (SimpleRecord record : records) { + builder.append(record); + } + builder.build(); + return records.length; + } + + private int appendTransactionalRecords(ByteBuffer buffer, long pid, long baseOffset, SimpleRecord... records) { + return appendTransactionalRecords(buffer, pid, baseOffset, (int) baseOffset, records); + } + + private void commitTransaction(ByteBuffer buffer, long producerId, long baseOffset) { + short producerEpoch = 0; + int partitionLeaderEpoch = 0; + MemoryRecords.writeEndTransactionalMarker(buffer, baseOffset, time.milliseconds(), partitionLeaderEpoch, producerId, producerEpoch, + new EndTransactionMarker(ControlRecordType.COMMIT, 0)); + } + + private int abortTransaction(ByteBuffer buffer, long producerId, long baseOffset) { + short producerEpoch = 0; + int partitionLeaderEpoch = 0; + MemoryRecords.writeEndTransactionalMarker(buffer, baseOffset, time.milliseconds(), partitionLeaderEpoch, producerId, producerEpoch, + new EndTransactionMarker(ControlRecordType.ABORT, 0)); + return 1; + } + + @Test + public void testSubscriptionPositionUpdatedWithEpoch() { + // Create some records that include a leader epoch (1) + MemoryRecordsBuilder builder = MemoryRecords.builder( + ByteBuffer.allocate(1024), + RecordBatch.CURRENT_MAGIC_VALUE, + CompressionType.NONE, + TimestampType.CREATE_TIME, + 0L, + RecordBatch.NO_TIMESTAMP, + RecordBatch.NO_PRODUCER_ID, + RecordBatch.NO_PRODUCER_EPOCH, + RecordBatch.NO_SEQUENCE, + false, + 1 + ); + builder.appendWithOffset(0L, 0L, "key".getBytes(), "value-1".getBytes()); + builder.appendWithOffset(1L, 0L, "key".getBytes(), "value-2".getBytes()); + builder.appendWithOffset(2L, 0L, "key".getBytes(), "value-3".getBytes()); + MemoryRecords records = builder.build(); + + buildFetcher(); + assignFromUser(singleton(tp0)); + + // Initialize the epoch=1 + Map partitionCounts = new HashMap<>(); + partitionCounts.put(tp0.topic(), 4); + MetadataResponse metadataResponse = RequestTestUtils.metadataUpdateWithIds("dummy", 1, Collections.emptyMap(), partitionCounts, tp -> 1, topicIds); + metadata.updateWithCurrentRequestVersion(metadataResponse, false, 0L); + + // Seek + subscriptions.seek(tp0, 0); + + // Do a normal fetch + assertEquals(1, sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); + consumerClient.pollNoWakeup(); + assertTrue(fetcher.hasCompletedFetches()); + + Map>> partitionRecords = fetchedRecords(); + assertTrue(partitionRecords.containsKey(tp0)); + + assertEquals(subscriptions.position(tp0).offset, 3L); + assertOptional(subscriptions.position(tp0).offsetEpoch, value -> assertEquals(value.intValue(), 1)); + } + + @Test + public void testPreferredReadReplica() { + buildFetcher(new MetricConfig(), OffsetResetStrategy.EARLIEST, new BytesDeserializer(), new BytesDeserializer(), + Integer.MAX_VALUE, IsolationLevel.READ_COMMITTED, Duration.ofMinutes(5).toMillis()); + + subscriptions.assignFromUser(singleton(tp0)); + client.updateMetadata(RequestTestUtils.metadataUpdateWithIds(2, singletonMap(topicName, 4), tp -> validLeaderEpoch, topicIds, false)); + subscriptions.seek(tp0, 0); + + // Node preferred replica before first fetch response + Node selected = fetcher.selectReadReplica(tp0, Node.noNode(), time.milliseconds()); + assertEquals(-1, selected.id()); + + assertEquals(1, sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + + // Set preferred read replica to node=1 + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, + FetchResponse.INVALID_LAST_STABLE_OFFSET, 0, Optional.of(1))); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + + Map>> partitionRecords = fetchedRecords(); + assertTrue(partitionRecords.containsKey(tp0)); + + // Verify + selected = fetcher.selectReadReplica(tp0, Node.noNode(), time.milliseconds()); + assertEquals(1, selected.id()); + + + assertEquals(1, sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + + // Set preferred read replica to node=2, which isn't in our metadata, should revert to leader + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, + FetchResponse.INVALID_LAST_STABLE_OFFSET, 0, Optional.of(2))); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + fetchedRecords(); + selected = fetcher.selectReadReplica(tp0, Node.noNode(), time.milliseconds()); + assertEquals(-1, selected.id()); + } + + @Test + public void testFetchDisconnectedShouldClearPreferredReadReplica() { + buildFetcher(new MetricConfig(), OffsetResetStrategy.EARLIEST, new BytesDeserializer(), new BytesDeserializer(), + Integer.MAX_VALUE, IsolationLevel.READ_COMMITTED, Duration.ofMinutes(5).toMillis()); + + subscriptions.assignFromUser(singleton(tp0)); + client.updateMetadata(RequestTestUtils.metadataUpdateWithIds(2, singletonMap(topicName, 4), tp -> validLeaderEpoch, topicIds, false)); + subscriptions.seek(tp0, 0); + assertEquals(1, sendFetches()); + + // Set preferred read replica to node=1 + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, + FetchResponse.INVALID_LAST_STABLE_OFFSET, 0, Optional.of(1))); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + fetchedRecords(); + + // Verify + Node selected = fetcher.selectReadReplica(tp0, Node.noNode(), time.milliseconds()); + assertEquals(1, selected.id()); + assertEquals(1, sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + + // Disconnect - preferred read replica should be cleared. + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0), true); + + consumerClient.poll(time.timer(0)); + assertFalse(fetcher.hasCompletedFetches()); + fetchedRecords(); + selected = fetcher.selectReadReplica(tp0, Node.noNode(), time.milliseconds()); + assertEquals(-1, selected.id()); + } + + @Test + public void testFetchDisconnectedShouldNotClearPreferredReadReplicaIfUnassigned() { + buildFetcher(new MetricConfig(), OffsetResetStrategy.EARLIEST, new BytesDeserializer(), new BytesDeserializer(), + Integer.MAX_VALUE, IsolationLevel.READ_COMMITTED, Duration.ofMinutes(5).toMillis()); + + subscriptions.assignFromUser(singleton(tp0)); + client.updateMetadata(RequestTestUtils.metadataUpdateWithIds(2, singletonMap(topicName, 4), tp -> validLeaderEpoch, topicIds, false)); + subscriptions.seek(tp0, 0); + assertEquals(1, sendFetches()); + + // Set preferred read replica to node=1 + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, + FetchResponse.INVALID_LAST_STABLE_OFFSET, 0, Optional.of(1))); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + fetchedRecords(); + + // Verify + Node selected = fetcher.selectReadReplica(tp0, Node.noNode(), time.milliseconds()); + assertEquals(1, selected.id()); + assertEquals(1, sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + + // Disconnect and remove tp0 from assignment + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0), true); + subscriptions.assignFromUser(emptySet()); + + // Preferred read replica should not be cleared + consumerClient.poll(time.timer(0)); + assertFalse(fetcher.hasCompletedFetches()); + fetchedRecords(); + selected = fetcher.selectReadReplica(tp0, Node.noNode(), time.milliseconds()); + assertEquals(-1, selected.id()); + } + + @Test + public void testFetchErrorShouldClearPreferredReadReplica() { + buildFetcher(new MetricConfig(), OffsetResetStrategy.EARLIEST, new BytesDeserializer(), new BytesDeserializer(), + Integer.MAX_VALUE, IsolationLevel.READ_COMMITTED, Duration.ofMinutes(5).toMillis()); + + subscriptions.assignFromUser(singleton(tp0)); + client.updateMetadata(RequestTestUtils.metadataUpdateWithIds(2, singletonMap(topicName, 4), tp -> validLeaderEpoch, topicIds, false)); + subscriptions.seek(tp0, 0); + assertEquals(1, sendFetches()); + + // Set preferred read replica to node=1 + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, + FetchResponse.INVALID_LAST_STABLE_OFFSET, 0, Optional.of(1))); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + fetchedRecords(); + + // Verify + Node selected = fetcher.selectReadReplica(tp0, Node.noNode(), time.milliseconds()); + assertEquals(1, selected.id()); + assertEquals(1, sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + + // Error - preferred read replica should be cleared. An actual error response will contain -1 as the + // preferred read replica. In the test we want to ensure that we are handling the error. + client.prepareResponse(fullFetchResponse(tidp0, MemoryRecords.EMPTY, Errors.NOT_LEADER_OR_FOLLOWER, -1L, + FetchResponse.INVALID_LAST_STABLE_OFFSET, 0, Optional.of(1))); + + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + fetchedRecords(); + selected = fetcher.selectReadReplica(tp0, Node.noNode(), time.milliseconds()); + assertEquals(-1, selected.id()); + } + + @Test + public void testPreferredReadReplicaOffsetError() { + buildFetcher(new MetricConfig(), OffsetResetStrategy.EARLIEST, new BytesDeserializer(), new BytesDeserializer(), + Integer.MAX_VALUE, IsolationLevel.READ_COMMITTED, Duration.ofMinutes(5).toMillis()); + + subscriptions.assignFromUser(singleton(tp0)); + client.updateMetadata(RequestTestUtils.metadataUpdateWithIds(2, singletonMap(topicName, 4), tp -> validLeaderEpoch, topicIds, false)); + + subscriptions.seek(tp0, 0); + + assertEquals(1, sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, + FetchResponse.INVALID_LAST_STABLE_OFFSET, 0, Optional.of(1))); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + + fetchedRecords(); + + Node selected = fetcher.selectReadReplica(tp0, Node.noNode(), time.milliseconds()); + assertEquals(selected.id(), 1); + + // Return an error, should unset the preferred read replica + assertEquals(1, sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.OFFSET_OUT_OF_RANGE, 100L, + FetchResponse.INVALID_LAST_STABLE_OFFSET, 0, Optional.empty())); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + + fetchedRecords(); + + selected = fetcher.selectReadReplica(tp0, Node.noNode(), time.milliseconds()); + assertEquals(selected.id(), -1); + } + + @Test + public void testFetchCompletedBeforeHandlerAdded() { + buildFetcher(); + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + sendFetches(); + client.prepareResponse(fullFetchResponse(tidp0, buildRecords(1L, 1, 1), Errors.NONE, 100L, 0)); + consumerClient.poll(time.timer(0)); + fetchedRecords(); + + Metadata.LeaderAndEpoch leaderAndEpoch = subscriptions.position(tp0).currentLeader; + assertTrue(leaderAndEpoch.leader.isPresent()); + Node readReplica = fetcher.selectReadReplica(tp0, leaderAndEpoch.leader.get(), time.milliseconds()); + + AtomicBoolean wokenUp = new AtomicBoolean(false); + client.setWakeupHook(() -> { + if (!wokenUp.getAndSet(true)) { + consumerClient.disconnectAsync(readReplica); + consumerClient.poll(time.timer(0)); + } + }); + + assertEquals(1, sendFetches()); + + consumerClient.disconnectAsync(readReplica); + consumerClient.poll(time.timer(0)); + + assertEquals(1, sendFetches()); + } + + @Test + public void testCorruptMessageError() { + buildFetcher(); + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + assertEquals(1, sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + + // Prepare a response with the CORRUPT_MESSAGE error. + client.prepareResponse(fullFetchResponse( + tidp0, + buildRecords(1L, 1, 1), + Errors.CORRUPT_MESSAGE, + 100L, 0)); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + + // Trigger the exception. + assertThrows(KafkaException.class, this::fetchedRecords); + } + + private OffsetsForLeaderEpochResponse prepareOffsetsForLeaderEpochResponse( + TopicPartition topicPartition, + Errors error, + int leaderEpoch, + long endOffset + ) { + OffsetForLeaderEpochResponseData data = new OffsetForLeaderEpochResponseData(); + data.topics().add(new OffsetForLeaderTopicResult() + .setTopic(topicPartition.topic()) + .setPartitions(Collections.singletonList(new EpochEndOffset() + .setPartition(topicPartition.partition()) + .setErrorCode(error.code()) + .setLeaderEpoch(leaderEpoch) + .setEndOffset(endOffset)))); + return new OffsetsForLeaderEpochResponse(data); + } + + private FetchResponse fetchResponseWithTopLevelError(TopicIdPartition tp, Errors error, int throttleTime) { + Map partitions = Collections.singletonMap(tp, + new FetchResponseData.PartitionData() + .setPartitionIndex(tp.topicPartition().partition()) + .setErrorCode(error.code()) + .setHighWatermark(FetchResponse.INVALID_HIGH_WATERMARK)); + return FetchResponse.of(error, throttleTime, INVALID_SESSION_ID, new LinkedHashMap<>(partitions)); + } + + private FetchResponse fullFetchResponseWithAbortedTransactions(MemoryRecords records, + List abortedTransactions, + Errors error, + long lastStableOffset, + long hw, + int throttleTime) { + Map partitions = Collections.singletonMap(tidp0, + new FetchResponseData.PartitionData() + .setPartitionIndex(tp0.partition()) + .setErrorCode(error.code()) + .setHighWatermark(hw) + .setLastStableOffset(lastStableOffset) + .setLogStartOffset(0) + .setAbortedTransactions(abortedTransactions) + .setRecords(records)); + return FetchResponse.of(Errors.NONE, throttleTime, INVALID_SESSION_ID, new LinkedHashMap<>(partitions)); + } + + private FetchResponse fullFetchResponse(int sessionId, TopicIdPartition tp, MemoryRecords records, Errors error, long hw, int throttleTime) { + return fullFetchResponse(sessionId, tp, records, error, hw, FetchResponse.INVALID_LAST_STABLE_OFFSET, throttleTime); + } + + private FetchResponse fullFetchResponse(TopicIdPartition tp, MemoryRecords records, Errors error, long hw, int throttleTime) { + return fullFetchResponse(tp, records, error, hw, FetchResponse.INVALID_LAST_STABLE_OFFSET, throttleTime); + } + + private FetchResponse fullFetchResponse(TopicIdPartition tp, MemoryRecords records, Errors error, long hw, + long lastStableOffset, int throttleTime) { + return fullFetchResponse(INVALID_SESSION_ID, tp, records, error, hw, lastStableOffset, throttleTime); + } + + private FetchResponse fullFetchResponse(int sessionId, TopicIdPartition tp, MemoryRecords records, Errors error, long hw, + long lastStableOffset, int throttleTime) { + Map partitions = Collections.singletonMap(tp, + new FetchResponseData.PartitionData() + .setPartitionIndex(tp.topicPartition().partition()) + .setErrorCode(error.code()) + .setHighWatermark(hw) + .setLastStableOffset(lastStableOffset) + .setLogStartOffset(0) + .setRecords(records)); + return FetchResponse.of(Errors.NONE, throttleTime, sessionId, new LinkedHashMap<>(partitions)); + } + + private FetchResponse fullFetchResponse(TopicIdPartition tp, MemoryRecords records, Errors error, long hw, + long lastStableOffset, int throttleTime, Optional preferredReplicaId) { + Map partitions = Collections.singletonMap(tp, + new FetchResponseData.PartitionData() + .setPartitionIndex(tp.topicPartition().partition()) + .setErrorCode(error.code()) + .setHighWatermark(hw) + .setLastStableOffset(lastStableOffset) + .setLogStartOffset(0) + .setRecords(records) + .setPreferredReadReplica(preferredReplicaId.orElse(FetchResponse.INVALID_PREFERRED_REPLICA_ID))); + return FetchResponse.of(Errors.NONE, throttleTime, INVALID_SESSION_ID, new LinkedHashMap<>(partitions)); + } + + private FetchResponse fetchResponse(TopicIdPartition tp, MemoryRecords records, Errors error, long hw, + long lastStableOffset, long logStartOffset, int throttleTime) { + Map partitions = Collections.singletonMap(tp, + new FetchResponseData.PartitionData() + .setPartitionIndex(tp.topicPartition().partition()) + .setErrorCode(error.code()) + .setHighWatermark(hw) + .setLastStableOffset(lastStableOffset) + .setLogStartOffset(logStartOffset) + .setRecords(records)); + return FetchResponse.of(Errors.NONE, throttleTime, INVALID_SESSION_ID, new LinkedHashMap<>(partitions)); + } + + /** + * Assert that the {@link Fetcher#collectFetch(Deserializers)} latest fetch} does not contain any + * {@link Fetch#records() user-visible records}, did not + * {@link Fetch#positionAdvanced() advance the consumer's position}, + * and is {@link Fetch#isEmpty() empty}. + * @param reason the reason to include for assertion methods such as {@link org.junit.jupiter.api.Assertions#assertTrue(boolean, String)} + */ + private void assertEmptyFetch(String reason) { + Fetch fetch = collectFetch(); + assertEquals(Collections.emptyMap(), fetch.records(), reason); + assertFalse(fetch.positionAdvanced(), reason); + assertTrue(fetch.isEmpty(), reason); + } + + private Map>> fetchedRecords() { + Fetch fetch = collectFetch(); + return fetch.records(); + } + + @SuppressWarnings("unchecked") + private Fetch collectFetch() { + return (Fetch) fetcher.collectFetch(); + } + + private void buildFetcher(int maxPollRecords) { + buildFetcher(OffsetResetStrategy.EARLIEST, new ByteArrayDeserializer(), new ByteArrayDeserializer(), + maxPollRecords, IsolationLevel.READ_UNCOMMITTED); + } + + private void buildFetcher() { + buildFetcher(Integer.MAX_VALUE); + } + + private void buildFetcher(Deserializer keyDeserializer, + Deserializer valueDeserializer) { + buildFetcher(OffsetResetStrategy.EARLIEST, keyDeserializer, valueDeserializer, + Integer.MAX_VALUE, IsolationLevel.READ_UNCOMMITTED); + } + + private void buildFetcher(OffsetResetStrategy offsetResetStrategy, + Deserializer keyDeserializer, + Deserializer valueDeserializer, + int maxPollRecords, + IsolationLevel isolationLevel) { + buildFetcher(new MetricConfig(), offsetResetStrategy, keyDeserializer, valueDeserializer, + maxPollRecords, isolationLevel); + } + + private void buildFetcher(MetricConfig metricConfig, + OffsetResetStrategy offsetResetStrategy, + Deserializer keyDeserializer, + Deserializer valueDeserializer, + int maxPollRecords, + IsolationLevel isolationLevel) { + buildFetcher(metricConfig, offsetResetStrategy, keyDeserializer, valueDeserializer, maxPollRecords, isolationLevel, Long.MAX_VALUE); + } + + private void buildFetcher(MetricConfig metricConfig, + OffsetResetStrategy offsetResetStrategy, + Deserializer keyDeserializer, + Deserializer valueDeserializer, + int maxPollRecords, + IsolationLevel isolationLevel, + long metadataExpireMs) { + LogContext logContext = new LogContext(); + SubscriptionState subscriptionState = new SubscriptionState(logContext, offsetResetStrategy); + buildFetcher(metricConfig, keyDeserializer, valueDeserializer, maxPollRecords, isolationLevel, metadataExpireMs, + subscriptionState, logContext); + } + + private void buildFetcher(MetricConfig metricConfig, + Deserializer keyDeserializer, + Deserializer valueDeserializer, + int maxPollRecords, + IsolationLevel isolationLevel, + long metadataExpireMs, + SubscriptionState subscriptionState, + LogContext logContext) { + buildDependencies(metricConfig, metadataExpireMs, subscriptionState, logContext); + deserializers = new Deserializers<>(keyDeserializer, valueDeserializer); + FetchConfig fetchConfig = new FetchConfig( + minBytes, + maxBytes, + maxWaitMs, + fetchSize, + maxPollRecords, + true, // check crc + CommonClientConfigs.DEFAULT_CLIENT_RACK, + isolationLevel); + FetchCollector fetchCollector = new FetchCollector<>(logContext, + metadata, + subscriptions, + fetchConfig, + metricsManager, + time); + fetcher = spy(new TestableFetchRequestManager<>( + logContext, + time, + new ErrorEventHandler(new LinkedBlockingQueue<>()), + metadata, + subscriptionState, + fetchConfig, + metricsManager, + consumerClient, + fetchCollector)); + offsetFetcher = new OffsetFetcher(logContext, + oldConsumerClient, + metadata, + subscriptions, + time, + retryBackoffMs, + requestTimeoutMs, + isolationLevel, + apiVersions); + } + + private void buildDependencies(MetricConfig metricConfig, + long metadataExpireMs, + SubscriptionState subscriptionState, + LogContext logContext) { + time = new MockTime(1, 0, 0); + subscriptions = subscriptionState; + metadata = new ConsumerMetadata(0, 0, metadataExpireMs, false, false, + subscriptions, logContext, new ClusterResourceListeners()); + client = new MockClient(time, metadata); + metrics = new Metrics(metricConfig, time); + oldConsumerClient = spy(new ConsumerNetworkClient(logContext, client, metadata, time, + retryBackoffMs, (int) requestTimeoutMs, Integer.MAX_VALUE)); + metricsRegistry = new FetchMetricsRegistry(metricConfig.tags().keySet(), "consumer" + groupId); + metricsManager = new FetchMetricsManager(metrics, metricsRegistry); + + Properties properties = new Properties(); + properties.put(KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); + properties.put(VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); + properties.setProperty(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG, String.valueOf(requestTimeoutMs)); + properties.setProperty(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG, String.valueOf(retryBackoffMs)); + ConsumerConfig config = new ConsumerConfig(properties); + consumerClient = spy(new TestableNetworkClientDelegate(time, config, logContext, client)); + } + + private List collectRecordOffsets(List> records) { + return records.stream().map(ConsumerRecord::offset).collect(Collectors.toList()); + } + + private class TestableFetchRequestManager extends FetchRequestManager { + + private final FetchCollector fetchCollector; + + public TestableFetchRequestManager(LogContext logContext, + Time time, + ErrorEventHandler errorEventHandler, + ConsumerMetadata metadata, + SubscriptionState subscriptions, + FetchConfig fetchConfig, + FetchMetricsManager metricsManager, + NetworkClientDelegate networkClientDelegate, + FetchCollector fetchCollector) { + super(logContext, time, errorEventHandler, metadata, subscriptions, fetchConfig, metricsManager, networkClientDelegate); + this.fetchCollector = fetchCollector; + } + + @SuppressWarnings("unchecked") + private Fetch collectFetch() { + return fetchCollector.collectFetch(fetchBuffer, (Deserializers) deserializers); + } + + private int sendFetches() { + NetworkClientDelegate.PollResult pollResult = fetcher.poll(time.milliseconds()); + consumerClient.addAll(pollResult.unsentRequests); + return pollResult.unsentRequests.size(); + } + + private void clearBufferedDataForUnassignedPartitions(Set partitions) { + fetchBuffer.retainAll(partitions); + } + } + + private class TestableNetworkClientDelegate extends NetworkClientDelegate { + + private final Logger log = LoggerFactory.getLogger(NetworkClientDelegate.class); + private final ConcurrentLinkedQueue pendingDisconnects = new ConcurrentLinkedQueue<>(); + + public TestableNetworkClientDelegate(Time time, + ConsumerConfig config, + LogContext logContext, + KafkaClient client) { + super(time, config, logContext, client); + } + + @Override + public void poll(final long timeoutMs, final long currentTimeMs) { + handlePendingDisconnects(); + super.poll(timeoutMs, currentTimeMs); + } + + public void pollNoWakeup() { + poll(time.timer(0)); + } + + public int pendingRequestCount() { + return unsentRequests().size() + client.inFlightRequestCount(); + } + + public void poll(final Timer timer) { + long pollTimeout = Math.min(timer.remainingMs(), requestTimeoutMs); + if (client.inFlightRequestCount() == 0) + pollTimeout = Math.min(pollTimeout, retryBackoffMs); + poll(pollTimeout, timer.currentTimeMs()); + } + + private Set unsentRequestNodes() { + Set set = new HashSet<>(); + + for (UnsentRequest u : unsentRequests()) + u.node().ifPresent(set::add); + + return set; + } + + private List removeUnsentRequestByNode(Node node) { + List list = new ArrayList<>(); + + Iterator iter = unsentRequests().iterator(); + + while (iter.hasNext()) { + UnsentRequest u = iter.next(); + + if (node.equals(u.node().orElse(null))) { + iter.remove(); + list.add(u); + } + } + + return list; + } + + @Override + protected void checkDisconnects() { + // any disconnects affecting requests that have already been transmitted will be handled + // by NetworkClient, so we just need to check whether connections for any of the unsent + // requests have been disconnected; if they have, then we complete the corresponding future + // and set the disconnect flag in the ClientResponse + for (Node node : unsentRequestNodes()) { + if (client.connectionFailed(node)) { + // Remove entry before invoking request callback to avoid callbacks handling + // coordinator failures traversing the unsent list again. + for (UnsentRequest unsentRequest : removeUnsentRequestByNode(node)) { + // TODO: this should likely emulate what's done in ConsumerNetworkClient + log.error("checkDisconnects - please update! unsentRequest: {}", unsentRequest); + } + } + } + } + + private void handlePendingDisconnects() { + while (true) { + Node node = pendingDisconnects.poll(); + if (node == null) + break; + + failUnsentRequests(node, DisconnectException.INSTANCE); + client.disconnect(node.idString()); + } + } + + public void disconnectAsync(Node node) { + pendingDisconnects.offer(node); + client.wakeup(); + } + + private void failUnsentRequests(Node node, RuntimeException e) { + // clear unsent requests to node and fail their corresponding futures + for (UnsentRequest unsentRequest : removeUnsentRequestByNode(node)) { + FutureCompletionHandler handler = unsentRequest.callback(); + handler.onFailure(e); + } + } + } +} diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java index 6336c11219e78..6a5fcf741c9ed 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java @@ -181,6 +181,7 @@ public class FetcherTest { private Metrics metrics; private ApiVersions apiVersions = new ApiVersions(); private ConsumerNetworkClient consumerClient; + private Deserializers deserializers; private Fetcher fetcher; private OffsetFetcher offsetFetcher; @@ -879,7 +880,7 @@ public byte[] deserialize(String topic, byte[] data) { // The fetcher should block on Deserialization error for (int i = 0; i < 2; i++) { try { - fetcher.collectFetch(); + collectFetch(); fail("fetchedRecords should have raised"); } catch (SerializationException e) { // the position should not advance since no data has been returned @@ -960,7 +961,7 @@ private void ensureBlockOnRecord(long blockedOffset) { // the fetchedRecords() should always throw exception due to the invalid message at the starting offset. for (int i = 0; i < 2; i++) { try { - fetcher.collectFetch(); + collectFetch(); fail("fetchedRecords should have raised KafkaException"); } catch (KafkaException e) { assertEquals(blockedOffset, subscriptions.position(tp0).offset); @@ -972,7 +973,7 @@ private void seekAndConsumeRecord(ByteBuffer responseBuffer, long toOffset) { // Seek to skip the bad record and fetch again. subscriptions.seekUnvalidated(tp0, new SubscriptionState.FetchPosition(toOffset, Optional.empty(), metadata.currentLeader(tp0))); // Should not throw exception after the seek. - fetcher.collectFetch(); + collectFetch(); assertEquals(1, sendFetches()); client.prepareResponse(fullFetchResponse(tidp0, MemoryRecords.readableRecords(responseBuffer), Errors.NONE, 100L, 0)); consumerClient.poll(time.timer(0)); @@ -1016,7 +1017,7 @@ public void testInvalidDefaultRecordBatch() { // the fetchedRecords() should always throw exception due to the bad batch. for (int i = 0; i < 2; i++) { try { - fetcher.collectFetch(); + collectFetch(); fail("fetchedRecords should have raised KafkaException"); } catch (KafkaException e) { assertEquals(0, subscriptions.position(tp0).offset); @@ -1045,7 +1046,7 @@ public void testParseInvalidRecordBatch() { client.prepareResponse(fullFetchResponse(tidp0, MemoryRecords.readableRecords(buffer), Errors.NONE, 100L, 0)); consumerClient.poll(time.timer(0)); try { - fetcher.collectFetch(); + collectFetch(); fail("fetchedRecords should have raised"); } catch (KafkaException e) { // the position should not advance since no data has been returned @@ -1219,7 +1220,7 @@ public void testFetchRequestWhenRecordTooLarge() { client.setNodeApiVersions(NodeApiVersions.create(ApiKeys.FETCH.id, (short) 2, (short) 2)); makeFetchRequestWithIncompleteRecord(); try { - fetcher.collectFetch(); + collectFetch(); fail("RecordTooLargeException should have been raised"); } catch (RecordTooLargeException e) { assertTrue(e.getMessage().startsWith("There are some messages at [Partition=Offset]: ")); @@ -1242,7 +1243,7 @@ public void testFetchRequestInternalError() { buildFetcher(); makeFetchRequestWithIncompleteRecord(); try { - fetcher.collectFetch(); + collectFetch(); fail("RecordTooLargeException should have been raised"); } catch (KafkaException e) { assertTrue(e.getMessage().startsWith("Failed to make progress reading messages")); @@ -1275,7 +1276,7 @@ public void testUnauthorizedTopic() { client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.TOPIC_AUTHORIZATION_FAILED, 100L, 0)); consumerClient.poll(time.timer(0)); try { - fetcher.collectFetch(); + collectFetch(); fail("fetchedRecords should have thrown"); } catch (TopicAuthorizationException e) { assertEquals(singleton(topicName), e.unauthorizedTopics()); @@ -1703,7 +1704,7 @@ public void testFetchOffsetOutOfRangeException() { assertFalse(subscriptions.isOffsetResetNeeded(tp0)); for (int i = 0; i < 2; i++) { OffsetOutOfRangeException e = assertThrows(OffsetOutOfRangeException.class, () -> - fetcher.collectFetch()); + collectFetch()); assertEquals(singleton(tp0), e.offsetOutOfRangePartitions().keySet()); assertEquals(0L, e.offsetOutOfRangePartitions().get(tp0).longValue()); } @@ -2189,7 +2190,7 @@ public void testFetchResponseMetricsWithOnePartitionError() { assertEquals(1, sendFetches()); client.prepareResponse(FetchResponse.of(Errors.NONE, 0, INVALID_SESSION_ID, new LinkedHashMap<>(partitions))); consumerClient.poll(time.timer(0)); - fetcher.collectFetch(); + collectFetch(); int expectedBytes = 0; for (Record record : records.records()) @@ -2235,7 +2236,7 @@ public void testFetchResponseMetricsWithOnePartitionAtTheWrongOffset() { client.prepareResponse(FetchResponse.of(Errors.NONE, 0, INVALID_SESSION_ID, new LinkedHashMap<>(partitions))); consumerClient.poll(time.timer(0)); - fetcher.collectFetch(); + collectFetch(); // we should have ignored the record at the wrong offset int expectedBytes = 0; @@ -2840,7 +2841,8 @@ public void testFetcherConcurrency() throws Exception { isolationLevel, apiVersions); - FetchConfig fetchConfig = new FetchConfig<>( + deserializers = new Deserializers<>(new ByteArrayDeserializer(), new ByteArrayDeserializer()); + FetchConfig fetchConfig = new FetchConfig( minBytes, maxBytes, maxWaitMs, @@ -2848,7 +2850,6 @@ public void testFetcherConcurrency() throws Exception { 2 * numPartitions, true, // check crcs CommonClientConfigs.DEFAULT_CLIENT_RACK, - new Deserializers<>(new ByteArrayDeserializer(), new ByteArrayDeserializer()), isolationLevel); fetcher = new Fetcher( logContext, @@ -3564,7 +3565,7 @@ private FetchResponse fetchResponse(TopicIdPartition tp, MemoryRecords records, } /** - * Assert that the {@link Fetcher#collectFetch() latest fetch} does not contain any + * Assert that the {@link Fetcher#collectFetch(Deserializers) latest fetch} does not contain any * {@link Fetch#records() user-visible records}, did not * {@link Fetch#positionAdvanced() advance the consumer's position}, * and is {@link Fetch#isEmpty() empty}. @@ -3584,7 +3585,10 @@ private Map>> fetchedRecords() @SuppressWarnings("unchecked") private Fetch collectFetch() { - return (Fetch) fetcher.collectFetch(); + // Ugh. What am I doing wrong here? + Fetcher f = (Fetcher) fetcher; + Deserializers d = (Deserializers) deserializers; + return f.collectFetch(d); } private void buildFetcher(int maxPollRecords) { @@ -3642,7 +3646,7 @@ private void buildFetcher(MetricConfig metricConfig, SubscriptionState subscriptionState, LogContext logContext) { buildDependencies(metricConfig, metadataExpireMs, subscriptionState, logContext); - FetchConfig fetchConfig = new FetchConfig<>( + FetchConfig fetchConfig = new FetchConfig( minBytes, maxBytes, maxWaitMs, @@ -3650,8 +3654,8 @@ private void buildFetcher(MetricConfig metricConfig, maxPollRecords, true, // check crc CommonClientConfigs.DEFAULT_CLIENT_RACK, - new Deserializers<>(keyDeserializer, valueDeserializer), isolationLevel); + deserializers = new Deserializers<>(keyDeserializer, valueDeserializer); fetcher = spy(new Fetcher<>( logContext, consumerClient, diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/OffsetFetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/OffsetFetcherTest.java index 4959a1c7b77bb..d3ae0402274e8 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/OffsetFetcherTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/OffsetFetcherTest.java @@ -56,7 +56,6 @@ import org.apache.kafka.common.requests.OffsetsForLeaderEpochRequest; import org.apache.kafka.common.requests.OffsetsForLeaderEpochResponse; import org.apache.kafka.common.requests.RequestTestUtils; -import org.apache.kafka.common.serialization.ByteArrayDeserializer; import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.MockTime; import org.apache.kafka.common.utils.Utils; @@ -1246,7 +1245,7 @@ public void testOffsetValidationSkippedForOldBroker() { buildFetcher(metricConfig, isolationLevel, metadataExpireMs, subscriptionState, logContext); FetchMetricsRegistry metricsRegistry = new FetchMetricsRegistry(metricConfig.tags().keySet(), "consumertest-group"); - FetchConfig fetchConfig = new FetchConfig<>( + FetchConfig fetchConfig = new FetchConfig( minBytes, maxBytes, maxWaitMs, @@ -1254,7 +1253,6 @@ public void testOffsetValidationSkippedForOldBroker() { maxPollRecords, true, // check crc CommonClientConfigs.DEFAULT_CLIENT_RACK, - new Deserializers<>(new ByteArrayDeserializer(), new ByteArrayDeserializer()), isolationLevel); Fetcher fetcher = new Fetcher<>( logContext, diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumerTest.java index 46892215bfa5a..3d0177085d9e1 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumerTest.java @@ -16,14 +16,11 @@ */ package org.apache.kafka.clients.consumer.internals; -import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.clients.consumer.OffsetAndTimestamp; import org.apache.kafka.clients.consumer.OffsetCommitCallback; -import org.apache.kafka.clients.consumer.OffsetResetStrategy; import org.apache.kafka.clients.consumer.internals.events.ApplicationEvent; import org.apache.kafka.clients.consumer.internals.events.AssignmentChangeApplicationEvent; -import org.apache.kafka.clients.consumer.internals.events.BackgroundEvent; import org.apache.kafka.clients.consumer.internals.events.EventHandler; import org.apache.kafka.clients.consumer.internals.events.ListOffsetsApplicationEvent; import org.apache.kafka.clients.consumer.internals.events.NewTopicsMetadataUpdateRequestEvent; @@ -35,12 +32,6 @@ import org.apache.kafka.common.errors.InvalidGroupIdException; import org.apache.kafka.common.errors.TimeoutException; import org.apache.kafka.common.errors.WakeupException; -import org.apache.kafka.common.metrics.Metrics; -import org.apache.kafka.common.serialization.Deserializer; -import org.apache.kafka.common.serialization.StringDeserializer; -import org.apache.kafka.common.utils.LogContext; -import org.apache.kafka.common.utils.MockTime; -import org.apache.kafka.common.utils.Time; import org.apache.kafka.common.utils.Timer; import org.apache.kafka.test.TestUtils; import org.junit.jupiter.api.AfterEach; @@ -57,20 +48,15 @@ import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.concurrent.BlockingQueue; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Future; -import java.util.concurrent.LinkedBlockingQueue; import java.util.stream.Collectors; import static java.util.Collections.singleton; -import static org.apache.kafka.clients.consumer.ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG; -import static org.apache.kafka.clients.consumer.ConsumerConfig.DEFAULT_API_TIMEOUT_MS_CONFIG; -import static org.apache.kafka.clients.consumer.ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG; -import static org.apache.kafka.clients.consumer.ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; @@ -79,56 +65,44 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mockConstruction; import static org.mockito.Mockito.never; -import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; public class PrototypeAsyncConsumerTest { private PrototypeAsyncConsumer consumer; - private final Map consumerProps = new HashMap<>(); + private static final Optional DEFAULT_GROUP_ID = Optional.of("group.id"); - private final Time time = new MockTime(); - private LogContext logContext; - private SubscriptionState subscriptions; - private ConsumerMetadata metadata; + private ConsumerTestBuilder.PrototypeAsyncConsumerTestBuilder testBuilder; private EventHandler eventHandler; - private Metrics metrics; - - private String groupId = "group.id"; - private ConsumerConfig config; @BeforeEach public void setup() { - injectConsumerConfigs(); - this.config = new ConsumerConfig(consumerProps); - this.logContext = new LogContext(); - this.subscriptions = mock(SubscriptionState.class); - this.metadata = mock(ConsumerMetadata.class); - final DefaultBackgroundThread bt = mock(DefaultBackgroundThread.class); - final BlockingQueue aq = new LinkedBlockingQueue<>(); - final BlockingQueue bq = new LinkedBlockingQueue<>(); - this.eventHandler = spy(new DefaultEventHandler(bt, aq, bq)); - this.metrics = new Metrics(time); + setup(DEFAULT_GROUP_ID); } @AfterEach public void cleanup() { - if (consumer != null) { - consumer.close(Duration.ZERO); + if (testBuilder != null) { + testBuilder.close(); } } + private void setup(Optional groupIdOpt) { + testBuilder = new ConsumerTestBuilder.PrototypeAsyncConsumerTestBuilder(groupIdOpt); + eventHandler = testBuilder.eventHandler; + consumer = testBuilder.consumer; + } + @Test public void testSuccessfulStartupShutdown() { - consumer = newConsumer(time, new StringDeserializer(), new StringDeserializer()); assertDoesNotThrow(() -> consumer.close()); } @Test public void testInvalidGroupId() { - this.groupId = null; - consumer = newConsumer(time, new StringDeserializer(), new StringDeserializer()); + cleanup(); + setup(Optional.empty()); assertThrows(InvalidGroupIdException.class, () -> consumer.committed(new HashSet<>())); } @@ -139,11 +113,10 @@ public void testCommitAsync_NullCallback() throws InterruptedException { offsets.put(new TopicPartition("my-topic", 0), new OffsetAndMetadata(100L)); offsets.put(new TopicPartition("my-topic", 1), new OffsetAndMetadata(200L)); - PrototypeAsyncConsumer mockedConsumer = spy(newConsumer(time, new StringDeserializer(), new StringDeserializer())); - doReturn(future).when(mockedConsumer).commit(offsets, false); - mockedConsumer.commitAsync(offsets, null); + doReturn(future).when(consumer).commit(offsets, false); + consumer.commitAsync(offsets, null); future.complete(null); - TestUtils.waitForCondition(() -> future.isDone(), + TestUtils.waitForCondition(future::isDone, 2000, "commit future should complete"); @@ -158,11 +131,9 @@ public void testCommitAsync_UserSuppliedCallback() { offsets.put(new TopicPartition("my-topic", 0), new OffsetAndMetadata(100L)); offsets.put(new TopicPartition("my-topic", 1), new OffsetAndMetadata(200L)); - PrototypeAsyncConsumer consumer = newConsumer(time, new StringDeserializer(), new StringDeserializer()); - PrototypeAsyncConsumer mockedConsumer = spy(consumer); - doReturn(future).when(mockedConsumer).commit(offsets, false); + doReturn(future).when(consumer).commit(offsets, false); OffsetCommitCallback customCallback = mock(OffsetCommitCallback.class); - mockedConsumer.commitAsync(offsets, customCallback); + consumer.commitAsync(offsets, customCallback); future.complete(null); verify(customCallback).onComplete(offsets, null); } @@ -174,7 +145,6 @@ public void testCommitted() { committedFuture.complete(offsets); try (MockedConstruction ignored = offsetFetchEventMocker(committedFuture)) { - this.consumer = newConsumer(time, new StringDeserializer(), new StringDeserializer()); assertDoesNotThrow(() -> consumer.committed(offsets.keySet(), Duration.ofMillis(1000))); verify(eventHandler).add(ArgumentMatchers.isA(OffsetFetchApplicationEvent.class)); } @@ -187,7 +157,6 @@ public void testCommitted_ExceptionThrown() { committedFuture.completeExceptionally(new KafkaException("Test exception")); try (MockedConstruction ignored = offsetFetchEventMocker(committedFuture)) { - this.consumer = newConsumer(time, new StringDeserializer(), new StringDeserializer()); assertThrows(KafkaException.class, () -> consumer.committed(offsets.keySet(), Duration.ofMillis(1000))); verify(eventHandler).add(ArgumentMatchers.isA(OffsetFetchApplicationEvent.class)); } @@ -229,8 +198,6 @@ private static MockedConstruction offsetFetchEventM @Test public void testAssign() { - this.subscriptions = new SubscriptionState(logContext, OffsetResetStrategy.EARLIEST); - this.consumer = newConsumer(time, new StringDeserializer(), new StringDeserializer()); final TopicPartition tp = new TopicPartition("foo", 3); consumer.assign(singleton(tp)); assertTrue(consumer.subscription().isEmpty()); @@ -241,13 +208,11 @@ public void testAssign() { @Test public void testAssignOnNullTopicPartition() { - consumer = newConsumer(time, new StringDeserializer(), new StringDeserializer()); assertThrows(IllegalArgumentException.class, () -> consumer.assign(null)); } @Test public void testAssignOnEmptyTopicPartition() { - consumer = spy(newConsumer(time, new StringDeserializer(), new StringDeserializer())); consumer.assign(Collections.emptyList()); assertTrue(consumer.subscription().isEmpty()); assertTrue(consumer.assignment().isEmpty()); @@ -255,26 +220,22 @@ public void testAssignOnEmptyTopicPartition() { @Test public void testAssignOnNullTopicInPartition() { - consumer = newConsumer(time, new StringDeserializer(), new StringDeserializer()); assertThrows(IllegalArgumentException.class, () -> consumer.assign(singleton(new TopicPartition(null, 0)))); } @Test public void testAssignOnEmptyTopicInPartition() { - consumer = newConsumer(time, new StringDeserializer(), new StringDeserializer()); assertThrows(IllegalArgumentException.class, () -> consumer.assign(singleton(new TopicPartition(" ", 0)))); } @Test public void testBeginningOffsetsFailsIfNullPartitions() { - consumer = newConsumer(time, new StringDeserializer(), new StringDeserializer()); assertThrows(NullPointerException.class, () -> consumer.beginningOffsets(null, Duration.ofMillis(1))); } @Test public void testBeginningOffsets() { - PrototypeAsyncConsumer consumer = newConsumer(time, new StringDeserializer(), new StringDeserializer()); Map expectedOffsetsAndTimestamp = mockOffsetAndTimestamp(); Set partitions = expectedOffsetsAndTimestamp.keySet(); @@ -283,7 +244,7 @@ public void testBeginningOffsets() { assertDoesNotThrow(() -> consumer.beginningOffsets(partitions, Duration.ofMillis(1))); Map expectedOffsets = expectedOffsetsAndTimestamp.entrySet().stream() - .collect(Collectors.toMap(e -> e.getKey(), e -> e.getValue().offset())); + .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().offset())); assertEquals(expectedOffsets, result); verify(eventHandler).addAndGet(ArgumentMatchers.isA(ListOffsetsApplicationEvent.class), ArgumentMatchers.isA(Timer.class)); @@ -291,7 +252,6 @@ public void testBeginningOffsets() { @Test public void testBeginningOffsetsThrowsKafkaExceptionForUnderlyingExecutionFailure() { - PrototypeAsyncConsumer consumer = newConsumer(time, new StringDeserializer(), new StringDeserializer()); Set partitions = mockTopicPartitionOffset().keySet(); Throwable eventProcessingFailure = new KafkaException("Unexpected failure " + "processing List Offsets event"); @@ -305,7 +265,6 @@ public void testBeginningOffsetsThrowsKafkaExceptionForUnderlyingExecutionFailur @Test public void testBeginningOffsetsTimeoutOnEventProcessingTimeout() { - PrototypeAsyncConsumer consumer = newConsumer(time, new StringDeserializer(), new StringDeserializer()); doThrow(new TimeoutException()).when(eventHandler).addAndGet(any(), any()); assertThrows(TimeoutException.class, () -> consumer.beginningOffsets( @@ -317,14 +276,12 @@ public void testBeginningOffsetsTimeoutOnEventProcessingTimeout() { @Test public void testOffsetsForTimesOnNullPartitions() { - PrototypeAsyncConsumer consumer = newConsumer(time, new StringDeserializer(), new StringDeserializer()); assertThrows(NullPointerException.class, () -> consumer.offsetsForTimes(null, Duration.ofMillis(1))); } @Test public void testOffsetsForTimes() { - PrototypeAsyncConsumer consumer = newConsumer(time, new StringDeserializer(), new StringDeserializer()); Map expectedResult = mockOffsetAndTimestamp(); Map timestampToSearch = mockTimestampToSearch(); @@ -341,7 +298,6 @@ public void testOffsetsForTimes() { // OffsetAndTimestamp as value. @Test public void testOffsetsForTimesWithZeroTimeout() { - PrototypeAsyncConsumer consumer = newConsumer(time, new StringDeserializer(), new StringDeserializer()); TopicPartition tp = new TopicPartition("topic1", 0); Map expectedResult = Collections.singletonMap(tp, null); @@ -357,8 +313,6 @@ public void testOffsetsForTimesWithZeroTimeout() { @Test public void testWakeup_committed() { - consumer = newConsumer(time, new StringDeserializer(), - new StringDeserializer()); consumer.wakeup(); assertThrows(WakeupException.class, () -> consumer.committed(mockTopicPartitionOffset().keySet())); assertNoPendingWakeup(consumer.wakeupTrigger()); @@ -366,20 +320,25 @@ public void testWakeup_committed() { @Test public void testRefreshCommittedOffsetsSuccess() { - Map committedOffsets = - Collections.singletonMap(new TopicPartition("t1", 1), new OffsetAndMetadata(10L)); - testRefreshCommittedOffsetsSuccess(committedOffsets); + TopicPartition partition = new TopicPartition("t1", 1); + Set partitions = Collections.singleton(partition); + Map committedOffsets = Collections.singletonMap(partition, new OffsetAndMetadata(10L)); + testRefreshCommittedOffsetsSuccess(partitions, committedOffsets); } @Test public void testRefreshCommittedOffsetsSuccessButNoCommittedOffsetsFound() { - testRefreshCommittedOffsetsSuccess(Collections.emptyMap()); + TopicPartition partition = new TopicPartition("t1", 1); + Set partitions = Collections.singleton(partition); + Map committedOffsets = Collections.emptyMap(); + testRefreshCommittedOffsetsSuccess(partitions, committedOffsets); } @Test public void testRefreshCommittedOffsetsShouldNotResetIfFailedWithTimeout() { // Create consumer with group id to enable committed offset usage - this.groupId = "consumer-group-1"; + cleanup(); + setup(Optional.of("consumer-group-1")); testUpdateFetchPositionsWithFetchCommittedOffsetsTimeout(true); } @@ -387,22 +346,20 @@ public void testRefreshCommittedOffsetsShouldNotResetIfFailedWithTimeout() { @Test public void testRefreshCommittedOffsetsNotCalledIfNoGroupId() { // Create consumer without group id so committed offsets are not used for updating positions - this.groupId = null; - consumer = newConsumer(time, new StringDeserializer(), new StringDeserializer()); + cleanup(); + setup(Optional.empty()); testUpdateFetchPositionsWithFetchCommittedOffsetsTimeout(false); } private void testUpdateFetchPositionsWithFetchCommittedOffsetsTimeout(boolean committedOffsetsEnabled) { - consumer = newConsumer(time, new StringDeserializer(), new StringDeserializer()); - // Uncompleted future that will time out if used CompletableFuture> committedFuture = new CompletableFuture<>(); - when(subscriptions.initializingPartitions()).thenReturn(Collections.singleton(new TopicPartition("t1", 1))); - try (MockedConstruction ignored = offsetFetchEventMocker(committedFuture)) { + consumer.assign(singleton(new TopicPartition("t1", 1))); + try (MockedConstruction ignored = offsetFetchEventMocker(committedFuture)) { // Poll with 0 timeout to run a single iteration of the poll loop consumer.poll(Duration.ofMillis(0)); @@ -422,17 +379,17 @@ private void testUpdateFetchPositionsWithFetchCommittedOffsetsTimeout(boolean co } } - private void testRefreshCommittedOffsetsSuccess(Map committedOffsets) { + private void testRefreshCommittedOffsetsSuccess(Set partitions, + Map committedOffsets) { CompletableFuture> committedFuture = new CompletableFuture<>(); committedFuture.complete(committedOffsets); // Create consumer with group id to enable committed offset usage - this.groupId = "consumer-group-1"; - consumer = newConsumer(time, new StringDeserializer(), new StringDeserializer()); + cleanup(); + setup(Optional.of("consumer-group-id")); + consumer.assign(partitions); try (MockedConstruction ignored = offsetFetchEventMocker(committedFuture)) { - when(subscriptions.initializingPartitions()).thenReturn(committedOffsets.keySet()); - // Poll with 0 timeout to run a single iteration of the poll loop consumer.poll(Duration.ofMillis(0)); @@ -443,10 +400,10 @@ private void testRefreshCommittedOffsetsSuccess(Map mockTopicPartitionOffset() { + private HashMap mockTopicPartitionOffset() { final TopicPartition t0 = new TopicPartition("t0", 2); final TopicPartition t1 = new TopicPartition("t0", 3); HashMap topicPartitionOffsets = new HashMap<>(); @@ -455,7 +412,7 @@ private Map mockTopicPartitionOffset() { return topicPartitionOffsets; } - private Map mockOffsetAndTimestamp() { + private HashMap mockOffsetAndTimestamp() { final TopicPartition t0 = new TopicPartition("t0", 2); final TopicPartition t1 = new TopicPartition("t0", 3); HashMap offsetAndTimestamp = new HashMap<>(); @@ -464,7 +421,7 @@ private Map mockOffsetAndTimestamp() { return offsetAndTimestamp; } - private Map mockTimestampToSearch() { + private HashMap mockTimestampToSearch() { final TopicPartition t0 = new TopicPartition("t0", 2); final TopicPartition t1 = new TopicPartition("t0", 3); HashMap timestampToSearch = new HashMap<>(); @@ -472,30 +429,5 @@ private Map mockTimestampToSearch() { timestampToSearch.put(t1, 2L); return timestampToSearch; } - - private void injectConsumerConfigs() { - consumerProps.put(BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); - consumerProps.put(DEFAULT_API_TIMEOUT_MS_CONFIG, "60000"); - consumerProps.put(KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); - consumerProps.put(VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); - } - - private PrototypeAsyncConsumer newConsumer(final Time time, - final Deserializer keyDeserializer, - final Deserializer valueDeserializer) { - consumerProps.put(KEY_DESERIALIZER_CLASS_CONFIG, keyDeserializer.getClass()); - consumerProps.put(VALUE_DESERIALIZER_CLASS_CONFIG, valueDeserializer.getClass()); - - return new PrototypeAsyncConsumer<>( - time, - logContext, - config, - subscriptions, - metadata, - eventHandler, - metrics, - Optional.ofNullable(this.groupId), - config.getInt(DEFAULT_API_TIMEOUT_MS_CONFIG)); - } } From 7ab56f21ddbdedf136b3247df67b867e957ae949 Mon Sep 17 00:00:00 2001 From: Kirk True Date: Tue, 19 Sep 2023 10:55:21 -0700 Subject: [PATCH 02/72] Reverted change to make FetchCollector.collectFetch() not take a Deserializers --- .../kafka/clients/consumer/KafkaConsumer.java | 5 +-- .../consumer/internals/CompletedFetch.java | 6 ++-- .../consumer/internals/FetchCollector.java | 9 +++-- .../clients/consumer/internals/Fetcher.java | 6 ++-- .../internals/PrototypeAsyncConsumer.java | 5 +-- .../clients/consumer/KafkaConsumerTest.java | 2 ++ .../internals/ConsumerTestBuilder.java | 4 ++- .../internals/FetchCollectorTest.java | 34 ++++++++++--------- .../internals/FetchRequestManagerTest.java | 9 +++-- .../consumer/internals/FetcherTest.java | 13 +++---- .../consumer/internals/OffsetFetcherTest.java | 2 ++ 11 files changed, 53 insertions(+), 42 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java index f9dfa29b0da12..3c4867d134ee3 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java @@ -759,6 +759,7 @@ public KafkaConsumer(Map configs, this.metadata, this.subscriptions, fetchConfig, + this.deserializers, fetchMetricsManager, this.time); this.offsetFetcher = new OffsetFetcher(logContext, @@ -1252,7 +1253,7 @@ private Fetch pollForFetches(Timer timer) { Math.min(coordinator.timeToNextPoll(timer.currentTimeMs()), timer.remainingMs()); // if data is available already, return it immediately - final Fetch fetch = fetcher.collectFetch(deserializers); + final Fetch fetch = fetcher.collectFetch(); if (!fetch.isEmpty()) { return fetch; } @@ -1279,7 +1280,7 @@ private Fetch pollForFetches(Timer timer) { }); timer.update(pollTimer.currentTimeMs()); - return fetcher.collectFetch(deserializers); + return fetcher.collectFetch(); } /** diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CompletedFetch.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CompletedFetch.java index cd8e296db17c3..5e5141d4a19d8 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CompletedFetch.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CompletedFetch.java @@ -152,7 +152,7 @@ void drain() { } } - private void maybeEnsureValid(FetchConfig fetchConfig, RecordBatch batch) { + private void maybeEnsureValid(FetchConfig fetchConfig, RecordBatch batch) { if (fetchConfig.checkCrcs && batch.magic() >= RecordBatch.MAGIC_VALUE_V2) { try { batch.ensureValid(); @@ -163,7 +163,7 @@ private void maybeEnsureValid(FetchConfig fetchConfig, RecordBatch batch) } } - private void maybeEnsureValid(FetchConfig fetchConfig, Record record) { + private void maybeEnsureValid(FetchConfig fetchConfig, Record record) { if (fetchConfig.checkCrcs) { try { record.ensureValid(); @@ -181,7 +181,7 @@ private void maybeCloseRecordStream() { } } - private Record nextFetchedRecord(FetchConfig fetchConfig) { + private Record nextFetchedRecord(FetchConfig fetchConfig) { while (true) { if (records == null || !records.hasNext()) { maybeCloseRecordStream(); diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchCollector.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchCollector.java index 3d2d636c53de4..e98441321d399 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchCollector.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchCollector.java @@ -55,6 +55,7 @@ public class FetchCollector { private final ConsumerMetadata metadata; private final SubscriptionState subscriptions; private final FetchConfig fetchConfig; + private final Deserializers deserializers; private final FetchMetricsManager metricsManager; private final Time time; @@ -62,12 +63,14 @@ public FetchCollector(final LogContext logContext, final ConsumerMetadata metadata, final SubscriptionState subscriptions, final FetchConfig fetchConfig, + final Deserializers deserializers, final FetchMetricsManager metricsManager, final Time time) { this.log = logContext.logger(FetchCollector.class); this.metadata = metadata; this.subscriptions = subscriptions; this.fetchConfig = fetchConfig; + this.deserializers = deserializers; this.metricsManager = metricsManager; this.time = time; } @@ -87,7 +90,7 @@ public FetchCollector(final LogContext logContext, * the defaultResetPolicy is NONE * @throws TopicAuthorizationException If there is TopicAuthorization error in fetchResponse. */ - public Fetch collectFetch(final FetchBuffer fetchBuffer, final Deserializers deserializers) { + public Fetch collectFetch(final FetchBuffer fetchBuffer) { final Fetch fetch = Fetch.empty(); final Queue pausedCompletedFetches = new ArrayDeque<>(); int recordsRemaining = fetchConfig.maxPollRecords; @@ -128,7 +131,7 @@ public Fetch collectFetch(final FetchBuffer fetchBuffer, final Deserialize pausedCompletedFetches.add(nextInLineFetch); fetchBuffer.setNextInLineFetch(null); } else { - final Fetch nextFetch = fetchRecords(nextInLineFetch, deserializers); + final Fetch nextFetch = fetchRecords(nextInLineFetch); recordsRemaining -= nextFetch.numRecords(); fetch.add(nextFetch); } @@ -145,7 +148,7 @@ public Fetch collectFetch(final FetchBuffer fetchBuffer, final Deserialize return fetch; } - private Fetch fetchRecords(final CompletedFetch nextInLineFetch, Deserializers deserializers) { + private Fetch fetchRecords(final CompletedFetch nextInLineFetch) { final TopicPartition tp = nextInLineFetch.partition; if (!subscriptions.isAssigned(tp)) { diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java index a8d7dd0a07052..4059b2e6aef9c 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java @@ -63,6 +63,7 @@ public Fetcher(LogContext logContext, ConsumerMetadata metadata, SubscriptionState subscriptions, FetchConfig fetchConfig, + Deserializers deserializers, FetchMetricsManager metricsManager, Time time) { super(logContext, metadata, subscriptions, fetchConfig, metricsManager, time); @@ -72,6 +73,7 @@ public Fetcher(LogContext logContext, metadata, subscriptions, fetchConfig, + deserializers, metricsManager, time); } @@ -166,8 +168,8 @@ public void onFailure(RuntimeException e) { } } - public Fetch collectFetch(Deserializers deserializers) { - return fetchCollector.collectFetch(fetchBuffer, deserializers); + public Fetch collectFetch() { + return fetchCollector.collectFetch(fetchBuffer); } /** diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java index 8cacac04d17a4..ac09ddf9c10b7 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java @@ -250,6 +250,7 @@ public PrototypeAsyncConsumer(final Time time, metadata, subscriptions, fetchConfig, + deserializers, fetchMetricsManager, time); @@ -927,7 +928,7 @@ private Fetch pollForFetches(Timer timer) { long pollTimeout = timer.remainingMs(); // if data is available already, return it immediately - final Fetch fetch = fetchCollector.collectFetch(fetchBuffer, deserializers); + final Fetch fetch = fetchCollector.collectFetch(fetchBuffer); if (!fetch.isEmpty()) { return fetch; } @@ -962,7 +963,7 @@ private Fetch pollForFetches(Timer timer) { timer.update(pollTimer.currentTimeMs()); } - return fetchCollector.collectFetch(fetchBuffer, deserializers); + return fetchCollector.collectFetch(fetchBuffer); } /** diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java index ebef3ecaca030..6a46b45319124 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java @@ -29,6 +29,7 @@ import org.apache.kafka.clients.consumer.internals.ConsumerMetrics; import org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient; import org.apache.kafka.clients.consumer.internals.ConsumerProtocol; +import org.apache.kafka.clients.consumer.internals.Deserializers; import org.apache.kafka.clients.consumer.internals.FetchConfig; import org.apache.kafka.clients.consumer.internals.FetchMetricsManager; import org.apache.kafka.clients.consumer.internals.Fetcher; @@ -2688,6 +2689,7 @@ private KafkaConsumer newConsumer(Time time, metadata, subscription, fetchConfig, + new Deserializers<>(keyDeserializer, deserializer), metricsManager, time); OffsetFetcher offsetFetcher = new OffsetFetcher(loggerFactory, diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java index c054f283d260d..60422d58f8484 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java @@ -227,16 +227,18 @@ public PrototypeAsyncConsumerTestBuilder(Optional groupIdOpt) { config.getList(ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG), config.originals(Collections.singletonMap(ConsumerConfig.CLIENT_ID_CONFIG, clientId)) ); + Deserializers deserializers = new Deserializers<>(new StringDeserializer(), new StringDeserializer()); FetchCollector fetchCollector = new FetchCollector<>(logContext, metadata, subscriptions, fetchConfig, + deserializers, metricsManager, time); this.consumer = spy(new PrototypeAsyncConsumer<>( logContext, clientId, - new Deserializers<>(new StringDeserializer(), new StringDeserializer()), + deserializers, new FetchBuffer(logContext), fetchCollector, new ConsumerInterceptors<>(Collections.emptyList()), diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchCollectorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchCollectorTest.java index 66ed1d952bcd6..4a8126143ea2f 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchCollectorTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchCollectorTest.java @@ -106,7 +106,7 @@ public void testFetchNormal() { assertFalse(completedFetch.isInitialized()); // Fetch the data and validate that we get all the records we want back. - Fetch fetch = fetchCollector.collectFetch(fetchBuffer, deserializers); + Fetch fetch = fetchCollector.collectFetch(fetchBuffer); assertFalse(fetch.isEmpty()); assertEquals(recordCount, fetch.numRecords()); @@ -130,7 +130,7 @@ public void testFetchNormal() { assertEquals(recordCount, position.offset); // Now attempt to collect more records from the fetch buffer. - fetch = fetchCollector.collectFetch(fetchBuffer, deserializers); + fetch = fetchCollector.collectFetch(fetchBuffer); // The Fetch object is non-null, but it's empty. assertEquals(0, fetch.numRecords()); @@ -154,7 +154,7 @@ public void testFetchWithReadReplica() { CompletedFetch completedFetch = completedFetchBuilder.build(); fetchBuffer.add(completedFetch); - Fetch fetch = fetchCollector.collectFetch(fetchBuffer, deserializers); + Fetch fetch = fetchCollector.collectFetch(fetchBuffer); // The Fetch and read replica settings should be empty. assertEquals(DEFAULT_RECORD_COUNT, fetch.numRecords()); @@ -177,7 +177,7 @@ public void testNoResultsIfInitializing() { // Add some valid CompletedFetch records to the FetchBuffer queue and collect them into the Fetch. CompletedFetch completedFetch = completedFetchBuilder.build(); fetchBuffer.add(completedFetch); - Fetch fetch = fetchCollector.collectFetch(fetchBuffer, deserializers); + Fetch fetch = fetchCollector.collectFetch(fetchBuffer); // Verify that no records are fetched for the partition as it did not have a valid position set. assertEquals(0, fetch.numRecords()); @@ -194,6 +194,7 @@ public void testErrorInInitialize(int recordCount, RuntimeException expectedExce metadata, subscriptions, fetchConfig, + deserializers, metricsManager, time) { @@ -213,7 +214,7 @@ protected CompletedFetch initialize(final CompletedFetch completedFetch) { assertFalse(fetchBuffer.isEmpty()); // Now run our ill-fated collectFetch. - assertThrows(expectedException.getClass(), () -> fetchCollector.collectFetch(fetchBuffer, deserializers)); + assertThrows(expectedException.getClass(), () -> fetchCollector.collectFetch(fetchBuffer)); // If the number of records in the CompletedFetch was 0, the call to FetchCollector.collectFetch() will // remove it from the queue. If there are records in the CompletedFetch, FetchCollector.collectFetch will @@ -246,7 +247,7 @@ public void testFetchingPausedPartitionsYieldsNoRecords() { // Ensure that the partition for the next-in-line CompletedFetch is still 'paused'. assertTrue(subscriptions.isPaused(completedFetch.partition)); - Fetch fetch = fetchCollector.collectFetch(fetchBuffer, deserializers); + Fetch fetch = fetchCollector.collectFetch(fetchBuffer); // There should be no records in the Fetch as the partition being fetched is 'paused'. assertEquals(0, fetch.numRecords()); @@ -278,7 +279,7 @@ public void testFetchWithMetadataRefreshErrors(final Errors error) { assertEquals(Optional.of(preferredReadReplicaId), subscriptions.preferredReadReplica(topicAPartition0, time.milliseconds())); // Fetch the data and validate that we get all the records we want back. - Fetch fetch = fetchCollector.collectFetch(fetchBuffer, deserializers); + Fetch fetch = fetchCollector.collectFetch(fetchBuffer); assertTrue(fetch.isEmpty()); assertTrue(metadata.updateRequested()); assertEquals(Optional.empty(), subscriptions.preferredReadReplica(topicAPartition0, time.milliseconds())); @@ -293,7 +294,7 @@ public void testFetchWithOffsetOutOfRange() { fetchBuffer.add(completedFetch); // Fetch the data and validate that we get our first batch of records back. - Fetch fetch = fetchCollector.collectFetch(fetchBuffer, deserializers); + Fetch fetch = fetchCollector.collectFetch(fetchBuffer); assertFalse(fetch.isEmpty()); assertEquals(DEFAULT_RECORD_COUNT, fetch.numRecords()); @@ -303,7 +304,7 @@ public void testFetchWithOffsetOutOfRange() { .error(Errors.OFFSET_OUT_OF_RANGE) .build(); fetchBuffer.add(completedFetch); - fetch = fetchCollector.collectFetch(fetchBuffer, deserializers); + fetch = fetchCollector.collectFetch(fetchBuffer); assertTrue(fetch.isEmpty()); // Try to fetch more data and validate that we get an empty Fetch back. @@ -312,7 +313,7 @@ public void testFetchWithOffsetOutOfRange() { .error(Errors.OFFSET_OUT_OF_RANGE) .build(); fetchBuffer.add(completedFetch); - fetch = fetchCollector.collectFetch(fetchBuffer, deserializers); + fetch = fetchCollector.collectFetch(fetchBuffer); assertTrue(fetch.isEmpty()); } @@ -332,7 +333,7 @@ public void testFetchWithOffsetOutOfRangeWithPreferredReadReplica() { .error(Errors.OFFSET_OUT_OF_RANGE) .build(); fetchBuffer.add(completedFetch); - Fetch fetch = fetchCollector.collectFetch(fetchBuffer, deserializers); + Fetch fetch = fetchCollector.collectFetch(fetchBuffer); // The Fetch and read replica settings should be empty. assertTrue(fetch.isEmpty()); @@ -349,7 +350,7 @@ public void testFetchWithTopicAuthorizationFailed() { .error(Errors.TOPIC_AUTHORIZATION_FAILED) .build(); fetchBuffer.add(completedFetch); - assertThrows(TopicAuthorizationException.class, () -> fetchCollector.collectFetch(fetchBuffer, deserializers)); + assertThrows(TopicAuthorizationException.class, () -> fetchCollector.collectFetch(fetchBuffer)); } @Test @@ -362,7 +363,7 @@ public void testFetchWithUnknownLeaderEpoch() { .error(Errors.UNKNOWN_LEADER_EPOCH) .build(); fetchBuffer.add(completedFetch); - Fetch fetch = fetchCollector.collectFetch(fetchBuffer, deserializers); + Fetch fetch = fetchCollector.collectFetch(fetchBuffer); assertTrue(fetch.isEmpty()); } @@ -376,7 +377,7 @@ public void testFetchWithUnknownServerError() { .error(Errors.UNKNOWN_SERVER_ERROR) .build(); fetchBuffer.add(completedFetch); - Fetch fetch = fetchCollector.collectFetch(fetchBuffer, deserializers); + Fetch fetch = fetchCollector.collectFetch(fetchBuffer); assertTrue(fetch.isEmpty()); } @@ -390,7 +391,7 @@ public void testFetchWithCorruptMessage() { .error(Errors.CORRUPT_MESSAGE) .build(); fetchBuffer.add(completedFetch); - assertThrows(KafkaException.class, () -> fetchCollector.collectFetch(fetchBuffer, deserializers)); + assertThrows(KafkaException.class, () -> fetchCollector.collectFetch(fetchBuffer)); } @ParameterizedTest @@ -403,7 +404,7 @@ public void testFetchWithOtherErrors(final Errors error) { .error(error) .build(); fetchBuffer.add(completedFetch); - assertThrows(IllegalStateException.class, () -> fetchCollector.collectFetch(fetchBuffer, deserializers)); + assertThrows(IllegalStateException.class, () -> fetchCollector.collectFetch(fetchBuffer)); } /** @@ -449,6 +450,7 @@ private void buildDependencies(int maxPollRecords) { metadata, subscriptions, fetchConfig, + deserializers, metricsManager, time); fetchBuffer = new FetchBuffer(logContext); diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java index 2a74f9441f9dc..fd4f327aa964f 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java @@ -185,7 +185,6 @@ public class FetchRequestManagerTest { private Metrics metrics; private ApiVersions apiVersions = new ApiVersions(); private ConsumerNetworkClient oldConsumerClient; - private Deserializers deserializers; private TestableFetchRequestManager fetcher; private TestableNetworkClientDelegate consumerClient; private OffsetFetcher offsetFetcher; @@ -3286,7 +3285,7 @@ private FetchResponse fetchResponse(TopicIdPartition tp, MemoryRecords records, } /** - * Assert that the {@link Fetcher#collectFetch(Deserializers)} latest fetch} does not contain any + * Assert that the {@link Fetcher#collectFetch()} latest fetch} does not contain any * {@link Fetch#records() user-visible records}, did not * {@link Fetch#positionAdvanced() advance the consumer's position}, * and is {@link Fetch#isEmpty() empty}. @@ -3364,7 +3363,7 @@ private void buildFetcher(MetricConfig metricConfig, SubscriptionState subscriptionState, LogContext logContext) { buildDependencies(metricConfig, metadataExpireMs, subscriptionState, logContext); - deserializers = new Deserializers<>(keyDeserializer, valueDeserializer); + Deserializers deserializers = new Deserializers<>(keyDeserializer, valueDeserializer); FetchConfig fetchConfig = new FetchConfig( minBytes, maxBytes, @@ -3378,6 +3377,7 @@ private void buildFetcher(MetricConfig metricConfig, metadata, subscriptions, fetchConfig, + deserializers, metricsManager, time); fetcher = spy(new TestableFetchRequestManager<>( @@ -3446,9 +3446,8 @@ public TestableFetchRequestManager(LogContext logContext, this.fetchCollector = fetchCollector; } - @SuppressWarnings("unchecked") private Fetch collectFetch() { - return fetchCollector.collectFetch(fetchBuffer, (Deserializers) deserializers); + return fetchCollector.collectFetch(fetchBuffer); } private int sendFetches() { diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java index 6a5fcf741c9ed..1fa4a6c2e5c8d 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java @@ -181,7 +181,6 @@ public class FetcherTest { private Metrics metrics; private ApiVersions apiVersions = new ApiVersions(); private ConsumerNetworkClient consumerClient; - private Deserializers deserializers; private Fetcher fetcher; private OffsetFetcher offsetFetcher; @@ -2841,7 +2840,7 @@ public void testFetcherConcurrency() throws Exception { isolationLevel, apiVersions); - deserializers = new Deserializers<>(new ByteArrayDeserializer(), new ByteArrayDeserializer()); + Deserializers deserializers = new Deserializers<>(new ByteArrayDeserializer(), new ByteArrayDeserializer()); FetchConfig fetchConfig = new FetchConfig( minBytes, maxBytes, @@ -2857,6 +2856,7 @@ public void testFetcherConcurrency() throws Exception { metadata, subscriptions, fetchConfig, + deserializers, metricsManager, time) { @Override @@ -3565,7 +3565,7 @@ private FetchResponse fetchResponse(TopicIdPartition tp, MemoryRecords records, } /** - * Assert that the {@link Fetcher#collectFetch(Deserializers) latest fetch} does not contain any + * Assert that the {@link Fetcher#collectFetch() latest fetch} does not contain any * {@link Fetch#records() user-visible records}, did not * {@link Fetch#positionAdvanced() advance the consumer's position}, * and is {@link Fetch#isEmpty() empty}. @@ -3585,10 +3585,7 @@ private Map>> fetchedRecords() @SuppressWarnings("unchecked") private Fetch collectFetch() { - // Ugh. What am I doing wrong here? - Fetcher f = (Fetcher) fetcher; - Deserializers d = (Deserializers) deserializers; - return f.collectFetch(d); + return (Fetch) fetcher.collectFetch(); } private void buildFetcher(int maxPollRecords) { @@ -3655,13 +3652,13 @@ private void buildFetcher(MetricConfig metricConfig, true, // check crc CommonClientConfigs.DEFAULT_CLIENT_RACK, isolationLevel); - deserializers = new Deserializers<>(keyDeserializer, valueDeserializer); fetcher = spy(new Fetcher<>( logContext, consumerClient, metadata, subscriptionState, fetchConfig, + new Deserializers<>(keyDeserializer, valueDeserializer), metricsManager, time)); offsetFetcher = new OffsetFetcher(logContext, diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/OffsetFetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/OffsetFetcherTest.java index d3ae0402274e8..fe3535eda64b4 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/OffsetFetcherTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/OffsetFetcherTest.java @@ -56,6 +56,7 @@ import org.apache.kafka.common.requests.OffsetsForLeaderEpochRequest; import org.apache.kafka.common.requests.OffsetsForLeaderEpochResponse; import org.apache.kafka.common.requests.RequestTestUtils; +import org.apache.kafka.common.serialization.ByteArrayDeserializer; import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.MockTime; import org.apache.kafka.common.utils.Utils; @@ -1260,6 +1261,7 @@ public void testOffsetValidationSkippedForOldBroker() { metadata, subscriptions, fetchConfig, + new Deserializers<>(new ByteArrayDeserializer(), new ByteArrayDeserializer()), new FetchMetricsManager(metrics, metricsRegistry), time); From 772ff2eaaae8d15d337d572ca64d2748a7d4fefc Mon Sep 17 00:00:00 2001 From: Kirk True Date: Tue, 19 Sep 2023 10:24:14 -0700 Subject: [PATCH 03/72] KAFKA-14274 [6, 7]: Introduction of fetch request manager Changes: 1. Introduces FetchRequestManager that implements the RequestManager API for fetching messages from brokers. Unlike Fetcher, record decompression and deserialization is performed on the application thread inside CompletedFetch. 2. Restructured the code so that objects owned by the background thread are not instantiated until the background thread runs (via Supplier) to ensure that there are no references available to the application thread. 3. Ensuring resources are properly using Closeable and using IdempotentCloser to ensure they're only closed once. 4. Introduces ConsumerTestBuilder to reduce a lot of inconsistency in the way the objects were built up for tests. --- checkstyle/import-control.xml | 4 + checkstyle/suppressions.xml | 12 +- .../kafka/clients/consumer/KafkaConsumer.java | 6 +- .../consumer/internals/AbstractFetch.java | 83 +- .../consumer/internals/CachedSupplier.java | 43 + .../consumer/internals/CompletedFetch.java | 25 +- .../consumer/internals/ConsumerUtils.java | 5 +- .../internals/DefaultBackgroundThread.java | 246 +- .../internals/DefaultEventHandler.java | 134 +- .../consumer/internals/Deserializers.java | 8 + .../consumer/internals/ErrorEventHandler.java | 7 +- .../consumer/internals/FetchCollector.java | 14 +- .../consumer/internals/FetchConfig.java | 19 +- .../internals/FetchRequestManager.java | 136 + .../clients/consumer/internals/Fetcher.java | 85 +- .../internals/NetworkClientDelegate.java | 93 +- .../internals/PrototypeAsyncConsumer.java | 700 +++- .../consumer/internals/RequestManager.java | 9 +- .../consumer/internals/RequestManagers.java | 119 +- .../internals/events/ApplicationEvent.java | 2 +- .../events/ApplicationEventProcessor.java | 47 +- .../events/BackgroundEventProcessor.java | 103 + .../events/ErrorBackgroundEvent.java | 6 +- .../internals/events/EventHandler.java | 19 +- .../consumer/internals/events/FetchEvent.java | 31 + .../clients/consumer/KafkaConsumerTest.java | 4 +- .../internals/CompletedFetchTest.java | 96 +- .../internals/ConsumerTestBuilder.java | 265 ++ .../DefaultBackgroundThreadTest.java | 242 +- .../internals/DefaultEventHandlerTest.java | 55 +- .../internals/ErrorEventHandlerTest.java | 165 + .../internals/FetchCollectorTest.java | 39 +- .../consumer/internals/FetchConfigTest.java | 48 +- .../internals/FetchRequestManagerTest.java | 3566 +++++++++++++++++ .../consumer/internals/FetcherTest.java | 38 +- .../consumer/internals/OffsetFetcherTest.java | 4 +- .../internals/PrototypeAsyncConsumerTest.java | 154 +- 37 files changed, 5625 insertions(+), 1007 deletions(-) create mode 100644 clients/src/main/java/org/apache/kafka/clients/consumer/internals/CachedSupplier.java create mode 100644 clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchRequestManager.java create mode 100644 clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEventProcessor.java create mode 100644 clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/FetchEvent.java create mode 100644 clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java create mode 100644 clients/src/test/java/org/apache/kafka/clients/consumer/internals/ErrorEventHandlerTest.java create mode 100644 clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java diff --git a/checkstyle/import-control.xml b/checkstyle/import-control.xml index a12ee2ea93e8f..cde1dceade826 100644 --- a/checkstyle/import-control.xml +++ b/checkstyle/import-control.xml @@ -205,6 +205,10 @@ + + + + diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml index f01281acb0ee9..aa19bd181f482 100644 --- a/checkstyle/suppressions.xml +++ b/checkstyle/suppressions.xml @@ -61,6 +61,8 @@ files="AbstractRequest.java"/> + @@ -68,7 +70,7 @@ + files="(KafkaConsumer|PrototypeAsyncConsumer|ConsumerCoordinator).java"/> + files="(KafkaConsumer|PrototypeAsyncConsumer|ConsumerCoordinator|AbstractFetch|KafkaProducer|AbstractRequest|AbstractResponse|TransactionManager|Admin|KafkaAdminClient|MockAdminClient|KafkaRaftClient|KafkaRaftClientTest).java"/> @@ -108,10 +110,10 @@ + files="(Sender|Fetcher|FetchRequestManager|OffsetFetcher|KafkaConsumer|PrototypeAsyncConsumer|Metrics|RequestResponse|TransactionManager|KafkaAdminClient|Message|KafkaProducer)Test.java"/> + files="(ConsumerCoordinator|KafkaConsumer|RequestResponse|Fetcher|FetchRequestManager|KafkaAdminClient|Message|KafkaProducer)Test.java"/> @@ -120,7 +122,7 @@ files="(OffsetFetcher|RequestResponse)Test.java"/> + files="RequestResponseTest.java|FetcherTest.java|FetchRequestManagerTest.java|KafkaAdminClientTest.java"/> diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java index aa4b913b49373..f9dfa29b0da12 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java @@ -752,7 +752,7 @@ public KafkaConsumer(Map configs, config.getBoolean(ConsumerConfig.THROW_ON_FETCH_STABLE_OFFSET_UNSUPPORTED), config.getString(ConsumerConfig.CLIENT_RACK_CONFIG)); } - FetchConfig fetchConfig = createFetchConfig(config, this.deserializers); + FetchConfig fetchConfig = createFetchConfig(config); this.fetcher = new Fetcher<>( logContext, this.client, @@ -1252,7 +1252,7 @@ private Fetch pollForFetches(Timer timer) { Math.min(coordinator.timeToNextPoll(timer.currentTimeMs()), timer.remainingMs()); // if data is available already, return it immediately - final Fetch fetch = fetcher.collectFetch(); + final Fetch fetch = fetcher.collectFetch(deserializers); if (!fetch.isEmpty()) { return fetch; } @@ -1279,7 +1279,7 @@ private Fetch pollForFetches(Timer timer) { }); timer.update(pollTimer.currentTimeMs()); - return fetcher.collectFetch(); + return fetcher.collectFetch(deserializers); } /** diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractFetch.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractFetch.java index 4f579bb0e0fed..108be1999f6a8 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractFetch.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractFetch.java @@ -18,10 +18,13 @@ import org.apache.kafka.clients.ClientResponse; import org.apache.kafka.clients.FetchSessionHandler; +import org.apache.kafka.clients.KafkaClient; +import org.apache.kafka.clients.NetworkClientUtils; import org.apache.kafka.common.Cluster; import org.apache.kafka.common.Node; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.errors.AuthenticationException; import org.apache.kafka.common.internals.IdempotentCloser; import org.apache.kafka.common.message.FetchResponseData; import org.apache.kafka.common.protocol.ApiKeys; @@ -37,11 +40,9 @@ import org.slf4j.helpers.MessageFormatter; import java.io.Closeable; -import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; -import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -51,17 +52,14 @@ /** * {@code AbstractFetch} represents the basic state and logic for record fetching processing. - * @param Type for the message key - * @param Type for the message value */ -public abstract class AbstractFetch implements Closeable { +public abstract class AbstractFetch implements Closeable { private final Logger log; protected final LogContext logContext; - protected final ConsumerNetworkClient client; protected final ConsumerMetadata metadata; protected final SubscriptionState subscriptions; - protected final FetchConfig fetchConfig; + protected final FetchConfig fetchConfig; protected final Time time; protected final FetchMetricsManager metricsManager; protected final FetchBuffer fetchBuffer; @@ -72,15 +70,13 @@ public abstract class AbstractFetch implements Closeable { private final Map sessionHandlers; public AbstractFetch(final LogContext logContext, - final ConsumerNetworkClient client, final ConsumerMetadata metadata, final SubscriptionState subscriptions, - final FetchConfig fetchConfig, + final FetchConfig fetchConfig, final FetchMetricsManager metricsManager, final Time time) { this.log = logContext.logger(AbstractFetch.class); this.logContext = logContext; - this.client = client; this.metadata = metadata; this.subscriptions = subscriptions; this.fetchConfig = fetchConfig; @@ -92,6 +88,23 @@ public AbstractFetch(final LogContext logContext, this.time = time; } + /** + * Check if the node is disconnected and unavailable for immediate reconnection (i.e. if it is in + * reconnect backoff window following the disconnect). + * + * @param node {@link Node} to check for availability + * @see NetworkClientUtils#isUnavailable(KafkaClient, Node, Time) + */ + protected abstract boolean isUnavailable(Node node); + + /** + * Checks for an authentication error on a given node and throws the exception if it exists. + * + * @param node {@link Node} to check for a previous {@link AuthenticationException}; if found it is thrown + * @see NetworkClientUtils#maybeThrowAuthFailure(KafkaClient, Node) + */ + protected abstract void maybeThrowAuthFailure(Node node); + /** * Return whether we have any completed fetches pending return to the user. This method is thread-safe. Has * visibility for testing. @@ -317,7 +330,7 @@ Node selectReadReplica(final TopicPartition partition, final Node leaderReplica, } } - private Map prepareCloseFetchSessionRequests() { + protected Map prepareCloseFetchSessionRequests() { final Cluster cluster = metadata.fetch(); Map fetchable = new LinkedHashMap<>(); @@ -330,7 +343,7 @@ private Map prepareCloseFetchSession // skip sending the close request. final Node fetchTarget = cluster.nodeById(fetchTargetNodeId); - if (fetchTarget == null || client.isUnavailable(fetchTarget)) { + if (fetchTarget == null || isUnavailable(fetchTarget)) { log.debug("Skip sending close session request to broker {} since it is not reachable", fetchTarget); return; } @@ -377,8 +390,8 @@ protected Map prepareFetchRequests() // Use the preferred read replica if set, otherwise the partition's leader Node node = selectReadReplica(partition, leaderOpt.get(), currentTimeMs); - if (client.isUnavailable(node)) { - client.maybeThrowAuthFailure(node); + if (isUnavailable(node)) { + maybeThrowAuthFailure(node); // If we try to send during the reconnect backoff window, then the request is just // going to be failed anyway before being sent, so skip sending the request for now @@ -412,46 +425,6 @@ protected Map prepareFetchRequests() return reqs; } - protected void maybeCloseFetchSessions(final Timer timer) { - final List> requestFutures = new ArrayList<>(); - Map fetchRequestMap = prepareCloseFetchSessionRequests(); - - for (Map.Entry entry : fetchRequestMap.entrySet()) { - final Node fetchTarget = entry.getKey(); - final FetchSessionHandler.FetchRequestData data = entry.getValue(); - final FetchRequest.Builder request = createFetchRequest(fetchTarget, data); - final RequestFuture responseFuture = client.send(fetchTarget, request); - - responseFuture.addListener(new RequestFutureListener() { - @Override - public void onSuccess(ClientResponse value) { - handleCloseFetchSessionResponse(fetchTarget, data); - } - - @Override - public void onFailure(RuntimeException e) { - handleCloseFetchSessionResponse(fetchTarget, data, e); - } - }); - - requestFutures.add(responseFuture); - } - - // Poll to ensure that request has been written to the socket. Wait until either the timer has expired or until - // all requests have received a response. - while (timer.notExpired() && !requestFutures.stream().allMatch(RequestFuture::isDone)) { - client.poll(timer, null, true); - } - - if (!requestFutures.stream().allMatch(RequestFuture::isDone)) { - // we ran out of time before completing all futures. It is ok since we don't want to block the shutdown - // here. - log.debug("All requests couldn't be sent in the specific timeout period {}ms. " + - "This may result in unnecessary fetch sessions at the broker. Consider increasing the timeout passed for " + - "KafkaConsumer.close(Duration timeout)", timer.timeoutMs()); - } - } - // Visible for testing protected FetchSessionHandler sessionHandler(int node) { return sessionHandlers.get(node); @@ -467,8 +440,6 @@ protected FetchSessionHandler sessionHandler(int node) { // Visible for testing protected void closeInternal(Timer timer) { // we do not need to re-enable wake-ups since we are closing already - client.disableWakeups(); - maybeCloseFetchSessions(timer); Utils.closeQuietly(fetchBuffer, "fetchBuffer"); Utils.closeQuietly(decompressionBufferSupplier, "decompressionBufferSupplier"); } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CachedSupplier.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CachedSupplier.java new file mode 100644 index 0000000000000..dc891a2539b6d --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CachedSupplier.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.clients.consumer.internals; + +import java.util.function.Supplier; + +/** + * Simple {@link Supplier} that caches the initial creation of the object and stores it for later calls + * to {@link #get()}. + * + *

    + * + * Note: this class is not thread safe! Use only in contexts which are designed/guaranteed to be + * single-threaded. + */ +public abstract class CachedSupplier implements Supplier { + + private T result; + + protected abstract T create(); + + @Override + public T get() { + if (result == null) + result = create(); + + return result; + } +} diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CompletedFetch.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CompletedFetch.java index 7a8ee10515741..cd8e296db17c3 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CompletedFetch.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CompletedFetch.java @@ -53,7 +53,7 @@ /** * {@link CompletedFetch} represents a {@link RecordBatch batch} of {@link Record records} that was returned from the * broker via a {@link FetchRequest}. It contains logic to maintain state between calls to - * {@link #fetchRecords(FetchConfig, int)}. + * {@link #fetchRecords(FetchConfig, Deserializers, int)}. */ public class CompletedFetch { @@ -135,7 +135,8 @@ void recordAggregatedMetrics(int bytes, int records) { /** * Draining a {@link CompletedFetch} will signal that the data has been consumed and the underlying resources * are closed. This is somewhat analogous to {@link Closeable#close() closing}, though no error will result if a - * caller invokes {@link #fetchRecords(FetchConfig, int)}; an empty {@link List list} will be returned instead. + * caller invokes {@link #fetchRecords(FetchConfig, Deserializers, int)}; an empty {@link List list} will be + * returned instead. */ void drain() { if (!isConsumed) { @@ -151,7 +152,7 @@ void drain() { } } - private void maybeEnsureValid(FetchConfig fetchConfig, RecordBatch batch) { + private void maybeEnsureValid(FetchConfig fetchConfig, RecordBatch batch) { if (fetchConfig.checkCrcs && batch.magic() >= RecordBatch.MAGIC_VALUE_V2) { try { batch.ensureValid(); @@ -162,7 +163,7 @@ private void maybeEnsureValid(FetchConfig fetchConfig, RecordBatch } } - private void maybeEnsureValid(FetchConfig fetchConfig, Record record) { + private void maybeEnsureValid(FetchConfig fetchConfig, Record record) { if (fetchConfig.checkCrcs) { try { record.ensureValid(); @@ -180,7 +181,7 @@ private void maybeCloseRecordStream() { } } - private Record nextFetchedRecord(FetchConfig fetchConfig) { + private Record nextFetchedRecord(FetchConfig fetchConfig) { while (true) { if (records == null || !records.hasNext()) { maybeCloseRecordStream(); @@ -249,7 +250,9 @@ private Record nextFetchedRecord(FetchConfig fetchConfig) { * @param maxRecords The number of records to return; the number returned may be {@code 0 <= maxRecords} * @return {@link ConsumerRecord Consumer records} */ - List> fetchRecords(FetchConfig fetchConfig, int maxRecords) { + List> fetchRecords(FetchConfig fetchConfig, + Deserializers deserializers, + int maxRecords) { // Error when fetching the next record before deserialization. if (corruptLastRecord) throw new KafkaException("Received exception when fetching the next record from " + partition @@ -276,7 +279,7 @@ List> fetchRecords(FetchConfig fetchConfig, in Optional leaderEpoch = maybeLeaderEpoch(currentBatch.partitionLeaderEpoch()); TimestampType timestampType = currentBatch.timestampType(); - ConsumerRecord record = parseRecord(fetchConfig, partition, leaderEpoch, timestampType, lastRecord); + ConsumerRecord record = parseRecord(deserializers, partition, leaderEpoch, timestampType, lastRecord); records.add(record); recordsRead++; bytesRead += lastRecord.sizeInBytes(); @@ -302,7 +305,7 @@ List> fetchRecords(FetchConfig fetchConfig, in /** * Parse the record entry, deserializing the key / value fields if necessary */ - ConsumerRecord parseRecord(FetchConfig fetchConfig, + ConsumerRecord parseRecord(Deserializers deserializers, TopicPartition partition, Optional leaderEpoch, TimestampType timestampType, @@ -312,16 +315,16 @@ ConsumerRecord parseRecord(FetchConfig fetchConfig, long timestamp = record.timestamp(); Headers headers = new RecordHeaders(record.headers()); ByteBuffer keyBytes = record.key(); - K key = keyBytes == null ? null : fetchConfig.deserializers.keyDeserializer.deserialize(partition.topic(), headers, keyBytes); + K key = keyBytes == null ? null : deserializers.keyDeserializer.deserialize(partition.topic(), headers, keyBytes); ByteBuffer valueBytes = record.value(); - V value = valueBytes == null ? null : fetchConfig.deserializers.valueDeserializer.deserialize(partition.topic(), headers, valueBytes); + V value = valueBytes == null ? null : deserializers.valueDeserializer.deserialize(partition.topic(), headers, valueBytes); return new ConsumerRecord<>(partition.topic(), partition.partition(), offset, timestamp, timestampType, keyBytes == null ? ConsumerRecord.NULL_SIZE : keyBytes.remaining(), valueBytes == null ? ConsumerRecord.NULL_SIZE : valueBytes.remaining(), key, value, headers, leaderEpoch); } catch (RuntimeException e) { - log.error("Deserializers with error: {}", fetchConfig.deserializers); + log.error("Deserializers with error: {}", deserializers); throw new RecordDeserializationException(partition, record.offset(), "Error deserializing key/value for partition " + partition + " at offset " + record.offset() + ". If needed, please seek past the record to continue consumption.", e); diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerUtils.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerUtils.java index 68d2458e28d5e..06dcf207b58ac 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerUtils.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerUtils.java @@ -142,10 +142,9 @@ public static FetchMetricsManager createFetchMetricsManager(Metrics metrics) { return new FetchMetricsManager(metrics, metricsRegistry); } - public static FetchConfig createFetchConfig(ConsumerConfig config, - Deserializers deserializers) { + public static FetchConfig createFetchConfig(ConsumerConfig config) { IsolationLevel isolationLevel = configuredIsolationLevel(config); - return new FetchConfig<>(config, deserializers, isolationLevel); + return new FetchConfig(config, isolationLevel); } @SuppressWarnings("unchecked") diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThread.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThread.java index 8150f56debbe4..330fedaa45991 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThread.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThread.java @@ -16,201 +16,79 @@ */ package org.apache.kafka.clients.consumer.internals; -import org.apache.kafka.clients.ApiVersions; -import org.apache.kafka.clients.ClientUtils; -import org.apache.kafka.clients.GroupRebalanceConfig; -import org.apache.kafka.clients.NetworkClient; -import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.internals.events.ApplicationEvent; import org.apache.kafka.clients.consumer.internals.events.ApplicationEventProcessor; -import org.apache.kafka.clients.consumer.internals.events.BackgroundEvent; +import org.apache.kafka.clients.consumer.internals.events.CompletableApplicationEvent; import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.errors.WakeupException; -import org.apache.kafka.common.metrics.Metrics; -import org.apache.kafka.common.metrics.Sensor; +import org.apache.kafka.common.internals.IdempotentCloser; import org.apache.kafka.common.utils.KafkaThread; import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.Time; import org.apache.kafka.common.utils.Utils; import org.slf4j.Logger; +import java.io.Closeable; +import java.util.ArrayList; import java.util.LinkedList; +import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.concurrent.BlockingQueue; - -import static java.util.Objects.requireNonNull; -import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_MAX_INFLIGHT_REQUESTS_PER_CONNECTION; -import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRIC_GROUP_PREFIX; -import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.configuredIsolationLevel; +import java.util.function.Supplier; /** * Background thread runnable that consumes {@code ApplicationEvent} and * produces {@code BackgroundEvent}. It uses an event loop to consume and * produce events, and poll the network client to handle network IO. - *

    + *

    * It holds a reference to the {@link SubscriptionState}, which is * initialized by the polling thread. - *

    - * For processing application events that have been submitted to the - * {@link #applicationEventQueue}, this relies on an {@link ApplicationEventProcessor}. Processing includes generating requests and - * handling responses with the appropriate {@link RequestManager}. The network operations for - * actually sending the requests is delegated to the {@link NetworkClientDelegate} - * */ -public class DefaultBackgroundThread extends KafkaThread { +public class DefaultBackgroundThread extends KafkaThread implements Closeable { + private static final long MAX_POLL_TIMEOUT_MS = 5000; private static final String BACKGROUND_THREAD_NAME = "consumer_background_thread"; private final Time time; private final Logger log; private final BlockingQueue applicationEventQueue; - private final BlockingQueue backgroundEventQueue; - private final ConsumerMetadata metadata; - private final ConsumerConfig config; + private final Supplier applicationEventProcessorSupplier; + private final Supplier networkClientDelegateSupplier; + private final Supplier requestManagersSupplier; // empty if groupId is null - private final ApplicationEventProcessor applicationEventProcessor; - private final NetworkClientDelegate networkClientDelegate; - private final ErrorEventHandler errorEventHandler; - private final GroupState groupState; - private boolean running; - - private final RequestManagers requestManagers; + private ApplicationEventProcessor applicationEventProcessor; + private NetworkClientDelegate networkClientDelegate; + private RequestManagers requestManagers; + private volatile boolean running; + private final IdempotentCloser closer = new IdempotentCloser(); // Visible for testing - @SuppressWarnings("ParameterNumber") - DefaultBackgroundThread(final Time time, - final ConsumerConfig config, - final LogContext logContext, - final BlockingQueue applicationEventQueue, - final BlockingQueue backgroundEventQueue, - final ErrorEventHandler errorEventHandler, - final ApplicationEventProcessor processor, - final ConsumerMetadata metadata, - final NetworkClientDelegate networkClient, - final GroupState groupState, - final CoordinatorRequestManager coordinatorManager, - final CommitRequestManager commitRequestManager, - final OffsetsRequestManager offsetsRequestManager, - final TopicMetadataRequestManager topicMetadataRequestManager) { + public DefaultBackgroundThread(Time time, + LogContext logContext, + BlockingQueue applicationEventQueue, + Supplier applicationEventProcessorSupplier, + Supplier networkClientDelegateSupplier, + Supplier requestManagersSupplier) { super(BACKGROUND_THREAD_NAME, true); this.time = time; - this.running = true; this.log = logContext.logger(getClass()); this.applicationEventQueue = applicationEventQueue; - this.backgroundEventQueue = backgroundEventQueue; - this.applicationEventProcessor = processor; - this.config = config; - this.metadata = metadata; - this.networkClientDelegate = networkClient; - this.errorEventHandler = errorEventHandler; - this.groupState = groupState; - this.requestManagers = new RequestManagers( - offsetsRequestManager, - topicMetadataRequestManager, - Optional.ofNullable(coordinatorManager), - Optional.ofNullable(commitRequestManager)); - } - - public DefaultBackgroundThread(final Time time, - final ConsumerConfig config, - final GroupRebalanceConfig rebalanceConfig, - final LogContext logContext, - final BlockingQueue applicationEventQueue, - final BlockingQueue backgroundEventQueue, - final ConsumerMetadata metadata, - final SubscriptionState subscriptionState, - final ApiVersions apiVersions, - final Metrics metrics, - final Sensor fetcherThrottleTimeSensor) { - super(BACKGROUND_THREAD_NAME, true); - requireNonNull(config); - requireNonNull(rebalanceConfig); - requireNonNull(logContext); - requireNonNull(applicationEventQueue); - requireNonNull(backgroundEventQueue); - requireNonNull(metadata); - requireNonNull(subscriptionState); - try { - this.time = time; - this.log = logContext.logger(getClass()); - this.applicationEventQueue = applicationEventQueue; - this.backgroundEventQueue = backgroundEventQueue; - this.config = config; - this.metadata = metadata; - final NetworkClient networkClient = ClientUtils.createNetworkClient(config, - metrics, - CONSUMER_METRIC_GROUP_PREFIX, - logContext, - apiVersions, - time, - CONSUMER_MAX_INFLIGHT_REQUESTS_PER_CONNECTION, - metadata, - fetcherThrottleTimeSensor); - this.networkClientDelegate = new NetworkClientDelegate( - this.time, - this.config, - logContext, - networkClient); - this.running = true; - this.errorEventHandler = new ErrorEventHandler(this.backgroundEventQueue); - this.groupState = new GroupState(rebalanceConfig); - long retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG); - long retryBackoffMaxMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MAX_MS_CONFIG); - final int requestTimeoutMs = config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG); - - OffsetsRequestManager offsetsRequestManager = - new OffsetsRequestManager( - subscriptionState, - metadata, - configuredIsolationLevel(config), - time, - retryBackoffMs, - requestTimeoutMs, - apiVersions, - networkClientDelegate, - logContext); - CoordinatorRequestManager coordinatorRequestManager = null; - CommitRequestManager commitRequestManager = null; - TopicMetadataRequestManager topicMetadataRequestManger = new TopicMetadataRequestManager( - logContext, - config); - - if (groupState.groupId != null) { - coordinatorRequestManager = new CoordinatorRequestManager( - this.time, - logContext, - retryBackoffMs, - retryBackoffMaxMs, - this.errorEventHandler, - groupState.groupId); - commitRequestManager = new CommitRequestManager( - this.time, - logContext, - subscriptionState, - config, - coordinatorRequestManager, - groupState); - } - - this.requestManagers = new RequestManagers( - offsetsRequestManager, - topicMetadataRequestManger, - Optional.ofNullable(coordinatorRequestManager), - Optional.ofNullable(commitRequestManager)); - this.applicationEventProcessor = new ApplicationEventProcessor( - backgroundEventQueue, - requestManagers, - metadata); - } catch (final Exception e) { - close(); - throw new KafkaException("Failed to construct background processor", e.getCause()); - } + this.applicationEventProcessorSupplier = applicationEventProcessorSupplier; + this.networkClientDelegateSupplier = networkClientDelegateSupplier; + this.requestManagersSupplier = requestManagersSupplier; } @Override public void run() { + closer.assertOpen("Consumer background thread is already closed"); + running = true; + try { log.debug("Background thread started"); + + // Wait until we're securely in the background thread to initialize these objects... + initializeResources(); + while (running) { try { runOnce(); @@ -224,10 +102,16 @@ public void run() { throw new KafkaException(t); } finally { close(); - log.debug("{} closed", getClass()); + log.debug("Background thread closed"); } } + void initializeResources() { + applicationEventProcessor = applicationEventProcessorSupplier.get(); + networkClientDelegate = networkClientDelegateSupplier.get(); + requestManagers = requestManagersSupplier.get(); + } + /** * Poll and process an {@link ApplicationEvent}. It performs the following tasks: * 1. Drains and try to process all the requests in the queue. @@ -235,15 +119,13 @@ public void run() { * 3. Poll the networkClient to send and retrieve the response. */ void runOnce() { - if (!applicationEventQueue.isEmpty()) { - LinkedList res = new LinkedList<>(); - this.applicationEventQueue.drainTo(res); + LinkedList events = new LinkedList<>(); + applicationEventQueue.drainTo(events); - for (ApplicationEvent event : res) { - log.debug("Consuming application event: {}", event); - Objects.requireNonNull(event); - applicationEventProcessor.process(event); - } + for (ApplicationEvent event : events) { + log.trace("Dequeued event: {}", event); + Objects.requireNonNull(event); + applicationEventProcessor.process(event); } final long currentTimeMs = time.milliseconds(); @@ -263,17 +145,41 @@ long handlePollResult(NetworkClientDelegate.PollResult res) { } public boolean isRunning() { - return this.running; + return running; } public void wakeup() { - networkClientDelegate.wakeup(); + if (networkClientDelegate != null) + networkClientDelegate.wakeup(); } + @Override public void close() { - this.running = false; - this.wakeup(); - Utils.closeQuietly(networkClientDelegate, "network client utils"); - Utils.closeQuietly(metadata, "consumer metadata client"); + closer.close(() -> { + log.debug("Closing the consumer background thread"); + running = false; + wakeup(); + Utils.closeQuietly(requestManagers, "Request managers client"); + Utils.closeQuietly(networkClientDelegate, "network client utils"); + log.debug("Closed the consumer background thread"); + drainAndComplete(); + }, () -> log.warn("The consumer background thread was previously closed")); + } + + + /** + * It is possible for the background thread to close before complete processing all the events in the queue. In + * this case, we need throw an exception to notify the user the consumer is closed. + */ + private void drainAndComplete() { + List incompletedEvents = new ArrayList<>(); + applicationEventQueue.drainTo(incompletedEvents); + incompletedEvents.forEach(event -> { + if (event instanceof CompletableApplicationEvent) { + ((CompletableApplicationEvent) event).future().completeExceptionally( + new KafkaException("The consumer is closed")); + } + }); + log.debug("Discarding {} events because the consumer is closed", incompletedEvents.size()); } } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/DefaultEventHandler.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/DefaultEventHandler.java index 4d620a4ffac10..342c13c4b51a6 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/DefaultEventHandler.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/DefaultEventHandler.java @@ -16,119 +16,55 @@ */ package org.apache.kafka.clients.consumer.internals; -import org.apache.kafka.clients.ApiVersions; -import org.apache.kafka.clients.ClientUtils; -import org.apache.kafka.clients.GroupRebalanceConfig; -import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.internals.events.ApplicationEvent; +import org.apache.kafka.clients.consumer.internals.events.ApplicationEventProcessor; import org.apache.kafka.clients.consumer.internals.events.BackgroundEvent; import org.apache.kafka.clients.consumer.internals.events.CompletableApplicationEvent; import org.apache.kafka.clients.consumer.internals.events.EventHandler; -import org.apache.kafka.common.internals.ClusterResourceListeners; -import org.apache.kafka.common.metrics.Metrics; -import org.apache.kafka.common.metrics.Sensor; +import org.apache.kafka.common.KafkaException; +import org.apache.kafka.common.internals.IdempotentCloser; import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.Time; import org.apache.kafka.common.utils.Timer; +import org.slf4j.Logger; -import java.net.InetSocketAddress; -import java.util.List; +import java.time.Duration; import java.util.Objects; -import java.util.Optional; import java.util.concurrent.BlockingQueue; -import java.util.concurrent.LinkedBlockingQueue; +import java.util.function.Supplier; /** - * An {@code EventHandler} that uses a single background thread to consume {@code ApplicationEvent} and produce - * {@code BackgroundEvent} from the {@link DefaultBackgroundThread}. + * An {@link EventHandler} that uses a single background thread to consume {@link ApplicationEvent} and produce + * {@link BackgroundEvent} from the {@link DefaultBackgroundThread}. */ public class DefaultEventHandler implements EventHandler { + private final Logger log; private final BlockingQueue applicationEventQueue; - private final BlockingQueue backgroundEventQueue; private final DefaultBackgroundThread backgroundThread; - - public DefaultEventHandler(final ConsumerConfig config, - final GroupRebalanceConfig groupRebalanceConfig, - final LogContext logContext, - final SubscriptionState subscriptionState, - final ApiVersions apiVersions, - final Metrics metrics, - final ClusterResourceListeners clusterResourceListeners, - final Sensor fetcherThrottleTimeSensor) { - this(Time.SYSTEM, - config, - groupRebalanceConfig, - logContext, - new LinkedBlockingQueue<>(), - new LinkedBlockingQueue<>(), - subscriptionState, - apiVersions, - metrics, - clusterResourceListeners, - fetcherThrottleTimeSensor); - } + private final IdempotentCloser closer = new IdempotentCloser(); public DefaultEventHandler(final Time time, - final ConsumerConfig config, - final GroupRebalanceConfig groupRebalanceConfig, final LogContext logContext, final BlockingQueue applicationEventQueue, - final BlockingQueue backgroundEventQueue, - final SubscriptionState subscriptionState, - final ApiVersions apiVersions, - final Metrics metrics, - final ClusterResourceListeners clusterResourceListeners, - final Sensor fetcherThrottleTimeSensor) { + final Supplier applicationEventProcessorSupplier, + final Supplier networkClientDelegateSupplier, + final Supplier requestManagersSupplier) { + this.log = logContext.logger(DefaultEventHandler.class); this.applicationEventQueue = applicationEventQueue; - this.backgroundEventQueue = backgroundEventQueue; - - // Bootstrap a metadata object with the bootstrap server IP address, which will be used once for the - // subsequent metadata refresh once the background thread has started up. - final ConsumerMetadata metadata = new ConsumerMetadata(config, - subscriptionState, + this.backgroundThread = new DefaultBackgroundThread(time, logContext, - clusterResourceListeners); - final List addresses = ClientUtils.parseAndValidateAddresses(config); - metadata.bootstrap(addresses); - - this.backgroundThread = new DefaultBackgroundThread( - time, - config, - groupRebalanceConfig, - logContext, - this.applicationEventQueue, - this.backgroundEventQueue, - metadata, - subscriptionState, - apiVersions, - metrics, - fetcherThrottleTimeSensor); - backgroundThread.start(); - } - - // VisibleForTesting - DefaultEventHandler(final DefaultBackgroundThread backgroundThread, - final BlockingQueue applicationEventQueue, - final BlockingQueue backgroundEventQueue) { - this.backgroundThread = backgroundThread; - this.applicationEventQueue = applicationEventQueue; - this.backgroundEventQueue = backgroundEventQueue; - backgroundThread.start(); - } - - @Override - public Optional poll() { - return Optional.ofNullable(backgroundEventQueue.poll()); - } - - @Override - public boolean isEmpty() { - return backgroundEventQueue.isEmpty(); + applicationEventQueue, + applicationEventProcessorSupplier, + networkClientDelegateSupplier, + requestManagersSupplier); + this.backgroundThread.start(); } @Override public boolean add(final ApplicationEvent event) { + Objects.requireNonNull(event, "ApplicationEvent provided to add must be non-null"); + log.trace("Enqueued event: {}", event); backgroundThread.wakeup(); return applicationEventQueue.add(event); } @@ -141,11 +77,25 @@ public T addAndGet(final CompletableApplicationEvent event, final Timer t return event.get(timer); } - public void close() { - try { - backgroundThread.close(); - } catch (final Exception e) { - throw new RuntimeException(e); - } + public void close(final Duration timeout) { + Objects.requireNonNull(timeout, "Duration provided to close must be non-null"); + + closer.close( + () -> { + log.info("Closing the default consumer event handler"); + + try { + long timeoutMs = timeout.toMillis(); + + if (timeoutMs < 0) + throw new IllegalArgumentException("The timeout cannot be negative."); + + backgroundThread.close(); + log.info("The default consumer event handler was closed"); + } catch (final Exception e) { + throw new KafkaException(e); + } + }, + () -> log.info("The default consumer event handler was already closed")); } } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Deserializers.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Deserializers.java index 80ba5cb7d9e15..5de2a888775af 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Deserializers.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Deserializers.java @@ -75,4 +75,12 @@ public void close() { throw new KafkaException("Failed to close deserializers", exception); } } + + @Override + public String toString() { + return "Deserializers{" + + "keyDeserializer=" + keyDeserializer + + ", valueDeserializer=" + valueDeserializer + + '}'; + } } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ErrorEventHandler.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ErrorEventHandler.java index 31c1873121961..60bfc7476aafd 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ErrorEventHandler.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ErrorEventHandler.java @@ -22,13 +22,18 @@ import java.util.Queue; public class ErrorEventHandler { + private final Queue backgroundEventQueue; public ErrorEventHandler(Queue backgroundEventQueue) { this.backgroundEventQueue = backgroundEventQueue; } - public void handle(Throwable e) { + public void handle(RuntimeException e) { backgroundEventQueue.add(new ErrorBackgroundEvent(e)); } + + public void handle(Throwable e) { + handle(new RuntimeException(e)); + } } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchCollector.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchCollector.java index 0e1f2f18a3f97..3d2d636c53de4 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchCollector.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchCollector.java @@ -54,14 +54,14 @@ public class FetchCollector { private final Logger log; private final ConsumerMetadata metadata; private final SubscriptionState subscriptions; - private final FetchConfig fetchConfig; + private final FetchConfig fetchConfig; private final FetchMetricsManager metricsManager; private final Time time; public FetchCollector(final LogContext logContext, final ConsumerMetadata metadata, final SubscriptionState subscriptions, - final FetchConfig fetchConfig, + final FetchConfig fetchConfig, final FetchMetricsManager metricsManager, final Time time) { this.log = logContext.logger(FetchCollector.class); @@ -87,7 +87,7 @@ public FetchCollector(final LogContext logContext, * the defaultResetPolicy is NONE * @throws TopicAuthorizationException If there is TopicAuthorization error in fetchResponse. */ - public Fetch collectFetch(final FetchBuffer fetchBuffer) { + public Fetch collectFetch(final FetchBuffer fetchBuffer, final Deserializers deserializers) { final Fetch fetch = Fetch.empty(); final Queue pausedCompletedFetches = new ArrayDeque<>(); int recordsRemaining = fetchConfig.maxPollRecords; @@ -128,7 +128,7 @@ public Fetch collectFetch(final FetchBuffer fetchBuffer) { pausedCompletedFetches.add(nextInLineFetch); fetchBuffer.setNextInLineFetch(null); } else { - final Fetch nextFetch = fetchRecords(nextInLineFetch); + final Fetch nextFetch = fetchRecords(nextInLineFetch, deserializers); recordsRemaining -= nextFetch.numRecords(); fetch.add(nextFetch); } @@ -145,7 +145,7 @@ public Fetch collectFetch(final FetchBuffer fetchBuffer) { return fetch; } - private Fetch fetchRecords(final CompletedFetch nextInLineFetch) { + private Fetch fetchRecords(final CompletedFetch nextInLineFetch, Deserializers deserializers) { final TopicPartition tp = nextInLineFetch.partition; if (!subscriptions.isAssigned(tp)) { @@ -162,7 +162,9 @@ private Fetch fetchRecords(final CompletedFetch nextInLineFetch) { throw new IllegalStateException("Missing position for fetchable partition " + tp); if (nextInLineFetch.nextFetchOffset() == position.offset) { - List> partRecords = nextInLineFetch.fetchRecords(fetchConfig, fetchConfig.maxPollRecords); + List> partRecords = nextInLineFetch.fetchRecords(fetchConfig, + deserializers, + fetchConfig.maxPollRecords); log.trace("Returning {} fetched records at offset {} for assigned partition {}", partRecords.size(), position, tp); diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchConfig.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchConfig.java index 66670b54472c8..c0ce15120b912 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchConfig.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchConfig.java @@ -21,8 +21,6 @@ import org.apache.kafka.common.IsolationLevel; import org.apache.kafka.common.serialization.Deserializer; -import java.util.Objects; - /** * {@link FetchConfig} represents the static configuration for fetching records from Kafka. It is simply a way * to bundle the immutable settings that were presented at the time the {@link Consumer} was created for later use by @@ -41,9 +39,6 @@ *

  • {@link #maxPollRecords}: {@link ConsumerConfig#MAX_POLL_RECORDS_CONFIG}
    • *
  • {@link #checkCrcs}: {@link ConsumerConfig#CHECK_CRCS_CONFIG}
    • *
  • {@link #clientRackId}: {@link ConsumerConfig#CLIENT_RACK_CONFIG}
    • - *
  • {@link #deserializers}: - * {@link ConsumerConfig#KEY_DESERIALIZER_CLASS_CONFIG}/{@link ConsumerConfig#VALUE_DESERIALIZER_CLASS_CONFIG} - *
    • *
  • {@link #isolationLevel}: {@link ConsumerConfig#ISOLATION_LEVEL_CONFIG}
    • * * @@ -54,11 +49,8 @@ * * Note: the {@link Deserializer deserializers} used for the key and value are not closed by this class. They should be * closed by the creator of the {@link FetchConfig}. - * - * @param Type used to {@link Deserializer deserialize} the message/record key - * @param Type used to {@link Deserializer deserialize} the message/record value */ -public class FetchConfig { +public class FetchConfig { final int minBytes; final int maxBytes; @@ -67,7 +59,6 @@ public class FetchConfig { final int maxPollRecords; final boolean checkCrcs; final String clientRackId; - final Deserializers deserializers; final IsolationLevel isolationLevel; public FetchConfig(int minBytes, @@ -77,7 +68,6 @@ public FetchConfig(int minBytes, int maxPollRecords, boolean checkCrcs, String clientRackId, - Deserializers deserializers, IsolationLevel isolationLevel) { this.minBytes = minBytes; this.maxBytes = maxBytes; @@ -86,13 +76,10 @@ public FetchConfig(int minBytes, this.maxPollRecords = maxPollRecords; this.checkCrcs = checkCrcs; this.clientRackId = clientRackId; - this.deserializers = Objects.requireNonNull(deserializers, "Message deserializers provided to FetchConfig should not be null"); this.isolationLevel = isolationLevel; } - public FetchConfig(ConsumerConfig config, - Deserializers deserializers, - IsolationLevel isolationLevel) { + public FetchConfig(ConsumerConfig config, IsolationLevel isolationLevel) { this.minBytes = config.getInt(ConsumerConfig.FETCH_MIN_BYTES_CONFIG); this.maxBytes = config.getInt(ConsumerConfig.FETCH_MAX_BYTES_CONFIG); this.maxWaitMs = config.getInt(ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG); @@ -100,7 +87,6 @@ public FetchConfig(ConsumerConfig config, this.maxPollRecords = config.getInt(ConsumerConfig.MAX_POLL_RECORDS_CONFIG); this.checkCrcs = config.getBoolean(ConsumerConfig.CHECK_CRCS_CONFIG); this.clientRackId = config.getString(ConsumerConfig.CLIENT_RACK_CONFIG); - this.deserializers = Objects.requireNonNull(deserializers, "Message deserializers provided to FetchConfig should not be null"); this.isolationLevel = isolationLevel; } @@ -114,7 +100,6 @@ public String toString() { ", maxPollRecords=" + maxPollRecords + ", checkCrcs=" + checkCrcs + ", clientRackId='" + clientRackId + '\'' + - ", deserializers=" + deserializers + ", isolationLevel=" + isolationLevel + '}'; } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchRequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchRequestManager.java new file mode 100644 index 0000000000000..9c010d722f0b4 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchRequestManager.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.clients.consumer.internals; + +import java.util.LinkedList; +import java.util.List; +import java.util.Queue; +import java.util.function.BiConsumer; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +import org.apache.kafka.clients.ClientResponse; +import org.apache.kafka.clients.FetchSessionHandler; +import org.apache.kafka.clients.consumer.internals.NetworkClientDelegate.PollResult; +import org.apache.kafka.clients.consumer.internals.NetworkClientDelegate.UnsentRequest; +import org.apache.kafka.common.Node; +import org.apache.kafka.common.requests.FetchRequest; +import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.common.utils.Time; +import org.slf4j.Logger; + +/** + * {@code FetchRequestManager} is responsible for generating {@link FetchRequest} that represent the + * {@link SubscriptionState#fetchablePartitions(Predicate)} based on the user's topic subscription/partition + * assignment. + */ +public class FetchRequestManager extends AbstractFetch implements RequestManager { + + private final Logger log; + private final ErrorEventHandler errorEventHandler; + private final NetworkClientDelegate networkClientDelegate; + + FetchRequestManager(final LogContext logContext, + final Time time, + final ErrorEventHandler errorEventHandler, + final ConsumerMetadata metadata, + final SubscriptionState subscriptions, + final FetchConfig fetchConfig, + final FetchMetricsManager metricsManager, + final NetworkClientDelegate networkClientDelegate) { + super(logContext, metadata, subscriptions, fetchConfig, metricsManager, time); + this.log = logContext.logger(FetchRequestManager.class); + this.errorEventHandler = errorEventHandler; + this.networkClientDelegate = networkClientDelegate; + } + + @Override + protected boolean isUnavailable(Node node) { + return networkClientDelegate.isUnavailable(node); + } + + @Override + protected void maybeThrowAuthFailure(Node node) { + networkClientDelegate.maybeThrowAuthFailure(node); + } + + @Override + public PollResult poll(long currentTimeMs) { + List requests; + + if (!idempotentCloser.isClosed()) { + // If the fetcher is open (i.e. not closed), we will issue the normal fetch requests + requests = prepareFetchRequests().entrySet().stream().map(entry -> { + final Node fetchTarget = entry.getKey(); + final FetchSessionHandler.FetchRequestData data = entry.getValue(); + final FetchRequest.Builder request = createFetchRequest(fetchTarget, data); + final BiConsumer responseHandler = (clientResponse, t) -> { + if (t != null) { + handleFetchResponse(fetchTarget, t); + log.warn("Attempt to fetch data from node {} failed due to fatal exception", fetchTarget, t); + errorEventHandler.handle(t); + } else { + handleFetchResponse(fetchTarget, data, clientResponse); + } + }; + + return new UnsentRequest(request, fetchTarget, responseHandler); + }).collect(Collectors.toList()); + } else { + requests = prepareCloseFetchSessionRequests().entrySet().stream().map(entry -> { + final Node fetchTarget = entry.getKey(); + final FetchSessionHandler.FetchRequestData data = entry.getValue(); + final FetchRequest.Builder request = createFetchRequest(fetchTarget, data); + final BiConsumer responseHandler = (clientResponse, t) -> { + if (t != null) { + handleCloseFetchSessionResponse(fetchTarget, data, t); + log.warn("Attempt to close fetch session on node {} failed due to fatal exception", fetchTarget, t); + errorEventHandler.handle(t); + } else { + handleCloseFetchSessionResponse(fetchTarget, data); + } + }; + + return new UnsentRequest(request, fetchTarget, responseHandler); + }).collect(Collectors.toList()); + } + + return new PollResult(Long.MAX_VALUE, requests); + } + + /** + * Drains any of the {@link CompletedFetch} objects from the internal buffer to the returned {@link Queue}. + * + *

    + * + * This is used by the {@link org.apache.kafka.clients.consumer.internals.events.ApplicationEventProcessor} to + * pull off any fetch results that are stored in the background thread to provide them to the application thread. + * + * @return {@link Queue} containing zero or more {@link CompletedFetch} + */ + public Queue drain() { + Queue q = new LinkedList<>(); + CompletedFetch completedFetch = fetchBuffer.poll(); + + while (completedFetch != null) { + q.add(completedFetch); + completedFetch = fetchBuffer.poll(); + } + + return q; + } +} diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java index dec02c6b90f6c..a8d7dd0a07052 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java @@ -20,12 +20,17 @@ import org.apache.kafka.clients.FetchSessionHandler; import org.apache.kafka.common.Node; import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.internals.IdempotentCloser; import org.apache.kafka.common.requests.FetchRequest; import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.Time; +import org.apache.kafka.common.utils.Timer; +import org.slf4j.Logger; +import java.util.ArrayList; import java.util.Collection; import java.util.HashSet; +import java.util.List; import java.util.Map; /** @@ -47,18 +52,22 @@ * on a different thread. * */ -public class Fetcher extends AbstractFetch { +public class Fetcher extends AbstractFetch { + private final Logger log; + private final ConsumerNetworkClient client; private final FetchCollector fetchCollector; public Fetcher(LogContext logContext, ConsumerNetworkClient client, ConsumerMetadata metadata, SubscriptionState subscriptions, - FetchConfig fetchConfig, + FetchConfig fetchConfig, FetchMetricsManager metricsManager, Time time) { - super(logContext, client, metadata, subscriptions, fetchConfig, metricsManager, time); + super(logContext, metadata, subscriptions, fetchConfig, metricsManager, time); + this.log = logContext.logger(Fetcher.class); + this.client = client; this.fetchCollector = new FetchCollector<>(logContext, metadata, subscriptions, @@ -67,6 +76,16 @@ public Fetcher(LogContext logContext, time); } + @Override + protected boolean isUnavailable(Node node) { + return client.isUnavailable(node); + } + + @Override + protected void maybeThrowAuthFailure(Node node) { + client.maybeThrowAuthFailure(node); + } + public void clearBufferedDataForUnassignedPartitions(Collection assignedPartitions) { fetchBuffer.retainAll(new HashSet<>(assignedPartitions)); } @@ -79,6 +98,7 @@ public void clearBufferedDataForUnassignedPartitions(Collection public synchronized int sendFetches() { Map fetchRequestMap = prepareFetchRequests(); + for (Map.Entry entry : fetchRequestMap.entrySet()) { final Node fetchTarget = entry.getKey(); final FetchSessionHandler.FetchRequestData data = entry.getValue(); @@ -106,7 +126,62 @@ public void onFailure(RuntimeException e) { return fetchRequestMap.size(); } - public Fetch collectFetch() { - return fetchCollector.collectFetch(fetchBuffer); + protected void maybeCloseFetchSessions(final Timer timer) { + final List> requestFutures = new ArrayList<>(); + Map fetchRequestMap = prepareCloseFetchSessionRequests(); + + for (Map.Entry entry : fetchRequestMap.entrySet()) { + final Node fetchTarget = entry.getKey(); + final FetchSessionHandler.FetchRequestData data = entry.getValue(); + final FetchRequest.Builder request = createFetchRequest(fetchTarget, data); + final RequestFuture responseFuture = client.send(fetchTarget, request); + + responseFuture.addListener(new RequestFutureListener() { + @Override + public void onSuccess(ClientResponse value) { + handleCloseFetchSessionResponse(fetchTarget, data); + } + + @Override + public void onFailure(RuntimeException e) { + handleCloseFetchSessionResponse(fetchTarget, data, e); + } + }); + + requestFutures.add(responseFuture); + } + + // Poll to ensure that request has been written to the socket. Wait until either the timer has expired or until + // all requests have received a response. + while (timer.notExpired() && !requestFutures.stream().allMatch(RequestFuture::isDone)) { + client.poll(timer, null, true); + } + + if (!requestFutures.stream().allMatch(RequestFuture::isDone)) { + // we ran out of time before completing all futures. It is ok since we don't want to block the shutdown + // here. + log.debug("All requests couldn't be sent in the specific timeout period {}ms. " + + "This may result in unnecessary fetch sessions at the broker. Consider increasing the timeout passed for " + + "KafkaConsumer.close(Duration timeout)", timer.timeoutMs()); + } + } + + public Fetch collectFetch(Deserializers deserializers) { + return fetchCollector.collectFetch(fetchBuffer, deserializers); + } + + /** + * This method is called by {@link #close(Timer)} which is guarded by the {@link IdempotentCloser}) such as to only + * be executed once the first time that any of the {@link #close()} methods are called. Subclasses can override + * this method without the need for extra synchronization at the instance level. + * + * @param timer Timer to enforce time limit + */ + // Visible for testing + protected void closeInternal(Timer timer) { + // we do not need to re-enable wake-ups since we are closing already + client.disableWakeups(); + maybeCloseFetchSessions(timer); + super.closeInternal(timer); } } \ No newline at end of file diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/NetworkClientDelegate.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/NetworkClientDelegate.java index 24b6b942481fe..c26541b67479e 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/NetworkClientDelegate.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/NetworkClientDelegate.java @@ -16,8 +16,10 @@ */ package org.apache.kafka.clients.consumer.internals; +import org.apache.kafka.clients.ApiVersions; import org.apache.kafka.clients.ClientRequest; import org.apache.kafka.clients.ClientResponse; +import org.apache.kafka.clients.ClientUtils; import org.apache.kafka.clients.KafkaClient; import org.apache.kafka.clients.NetworkClientUtils; import org.apache.kafka.clients.RequestCompletionHandler; @@ -26,6 +28,7 @@ import org.apache.kafka.common.errors.AuthenticationException; import org.apache.kafka.common.errors.DisconnectException; import org.apache.kafka.common.errors.TimeoutException; +import org.apache.kafka.common.metrics.Metrics; import org.apache.kafka.common.requests.AbstractRequest; import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.Time; @@ -35,27 +38,29 @@ import java.io.IOException; import java.util.ArrayDeque; import java.util.Collections; -import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.Queue; -import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.function.BiConsumer; +import java.util.function.Supplier; + +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_MAX_INFLIGHT_REQUESTS_PER_CONNECTION; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRIC_GROUP_PREFIX; /** * A wrapper around the {@link org.apache.kafka.clients.NetworkClient} to handle network poll and send operations. */ public class NetworkClientDelegate implements AutoCloseable { + private final KafkaClient client; private final Time time; private final Logger log; private final int requestTimeoutMs; private final Queue unsentRequests; private final long retryBackoffMs; - private final Set tryConnectNodes; public NetworkClientDelegate( final Time time, @@ -68,9 +73,40 @@ public NetworkClientDelegate( this.unsentRequests = new ArrayDeque<>(); this.requestTimeoutMs = config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG); this.retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG); - this.tryConnectNodes = new HashSet<>(); } + // Visible for testing + Queue unsentRequests() { + return unsentRequests; + } + + /** + * Check if the node is disconnected and unavailable for immediate reconnection (i.e. if it is in + * reconnect backoff window following the disconnect). + * + * @param node {@link Node} to check for availability + * @see NetworkClientUtils#isUnavailable(KafkaClient, Node, Time) + */ + public boolean isUnavailable(Node node) { + return NetworkClientUtils.isUnavailable(client, node, time); + } + + /** + * Checks for an authentication error on a given node and throws the exception if it exists. + * + * @param node {@link Node} to check for a previous {@link AuthenticationException}; if found it is thrown + * @see NetworkClientUtils#maybeThrowAuthFailure(KafkaClient, Node) + */ + public void maybeThrowAuthFailure(Node node) { + NetworkClientUtils.maybeThrowAuthFailure(client, node); + } + + /** + * Initiate a connection if currently possible. This is only really useful for resetting + * the failed status of a socket. + * + * @param node The node to connect to + */ public void tryConnect(Node node) { NetworkClientUtils.tryConnect(client, node, time); } @@ -81,7 +117,6 @@ public void tryConnect(Node node) { * * @param timeoutMs timeout time * @param currentTimeMs current time - * @return a list of client response */ public void poll(final long timeoutMs, final long currentTimeMs) { trySend(currentTimeMs); @@ -119,8 +154,7 @@ private void trySend(final long currentTimeMs) { } } - private boolean doSend(final UnsentRequest r, - final long currentTimeMs) { + boolean doSend(final UnsentRequest r, final long currentTimeMs) { Node node = r.node.orElse(client.leastLoadedNode(currentTimeMs)); if (node == null || nodeUnavailable(node)) { log.debug("No broker available to send the request: {}. Retrying.", r); @@ -137,7 +171,7 @@ private boolean doSend(final UnsentRequest r, return true; } - private void checkDisconnects() { + protected void checkDisconnects() { // Check the connection of the unsent request. Disconnect the disconnected node if it is unable to be connected. Iterator iter = unsentRequests.iterator(); while (iter.hasNext()) { @@ -209,7 +243,7 @@ public PollResult(final long timeMsTillNextPoll, final List unsen public static class UnsentRequest { private final AbstractRequest.Builder requestBuilder; private final FutureCompletionHandler handler; - private Optional node; // empty if random node can be chosen + private final Optional node; // empty if random node can be chosen private Timer timer; public UnsentRequest(final AbstractRequest.Builder requestBuilder, @@ -227,6 +261,13 @@ public UnsentRequest(final AbstractRequest.Builder requestBuilder, this.handler.future.whenComplete(callback); } + public UnsentRequest(final AbstractRequest.Builder requestBuilder, + final Node node, + final BiConsumer responseHandler) { + this(requestBuilder, Optional.of(node)); + future().whenComplete(responseHandler); + } + public void setTimer(final Time time, final long requestTimeoutMs) { this.timer = time.timer(requestTimeoutMs); } @@ -235,7 +276,7 @@ CompletableFuture future() { return handler.future; } - RequestCompletionHandler callback() { + FutureCompletionHandler callback() { return handler; } @@ -243,6 +284,10 @@ AbstractRequest.Builder requestBuilder() { return requestBuilder; } + Optional node() { + return node; + } + @Override public String toString() { return "UnsentRequest{" + @@ -279,4 +324,32 @@ public void onComplete(final ClientResponse response) { } } } + + /** + * Creates a {@link Supplier} for deferred creation during invocation by + * {@link org.apache.kafka.clients.consumer.internals.DefaultBackgroundThread}. + */ + public static Supplier supplier(final Time time, + final LogContext logContext, + final ConsumerMetadata metadata, + final ConsumerConfig config, + final ApiVersions apiVersions, + final Metrics metrics, + final FetchMetricsManager fetchMetricsManager) { + return new CachedSupplier() { + @Override + protected NetworkClientDelegate create() { + KafkaClient client = ClientUtils.createNetworkClient(config, + metrics, + CONSUMER_METRIC_GROUP_PREFIX, + logContext, + apiVersions, + time, + CONSUMER_MAX_INFLIGHT_REQUESTS_PER_CONNECTION, + metadata, + fetchMetricsManager.throttleTimeSensor()); + return new NetworkClientDelegate(time, config, logContext, client); + } + }; + } } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java index 4fe8fe80c44c3..8cacac04d17a4 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java @@ -20,25 +20,34 @@ import org.apache.kafka.clients.ClientUtils; import org.apache.kafka.clients.CommonClientConfigs; import org.apache.kafka.clients.GroupRebalanceConfig; +import org.apache.kafka.clients.Metadata; import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.ConsumerGroupMetadata; import org.apache.kafka.clients.consumer.ConsumerInterceptor; +import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor; import org.apache.kafka.clients.consumer.ConsumerRebalanceListener; import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.consumer.NoOffsetForPartitionException; import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.clients.consumer.OffsetAndTimestamp; import org.apache.kafka.clients.consumer.OffsetCommitCallback; +import org.apache.kafka.clients.consumer.OffsetResetStrategy; +import org.apache.kafka.clients.consumer.internals.events.ApplicationEvent; +import org.apache.kafka.clients.consumer.internals.events.ApplicationEventProcessor; import org.apache.kafka.clients.consumer.internals.events.AssignmentChangeApplicationEvent; import org.apache.kafka.clients.consumer.internals.events.BackgroundEvent; +import org.apache.kafka.clients.consumer.internals.events.BackgroundEventProcessor; import org.apache.kafka.clients.consumer.internals.events.CommitApplicationEvent; import org.apache.kafka.clients.consumer.internals.events.EventHandler; +import org.apache.kafka.clients.consumer.internals.events.FetchEvent; import org.apache.kafka.clients.consumer.internals.events.ListOffsetsApplicationEvent; import org.apache.kafka.clients.consumer.internals.events.NewTopicsMetadataUpdateRequestEvent; import org.apache.kafka.clients.consumer.internals.events.OffsetFetchApplicationEvent; import org.apache.kafka.clients.consumer.internals.events.ResetPositionsApplicationEvent; import org.apache.kafka.clients.consumer.internals.events.ValidatePositionsApplicationEvent; +import org.apache.kafka.common.Cluster; +import org.apache.kafka.common.IsolationLevel; import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.Metric; import org.apache.kafka.common.MetricName; @@ -47,15 +56,17 @@ import org.apache.kafka.common.config.ConfigException; import org.apache.kafka.common.errors.InterruptException; import org.apache.kafka.common.errors.InvalidGroupIdException; -import org.apache.kafka.common.errors.WakeupException; +import org.apache.kafka.common.errors.TimeoutException; import org.apache.kafka.common.internals.ClusterResourceListeners; import org.apache.kafka.common.metrics.Metrics; import org.apache.kafka.common.requests.ListOffsetsRequest; import org.apache.kafka.common.serialization.Deserializer; +import org.apache.kafka.common.utils.AppInfoParser; import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.Time; import org.apache.kafka.common.utils.Timer; import org.slf4j.Logger; +import org.slf4j.event.Level; import java.net.InetSocketAddress; import java.time.Duration; @@ -66,30 +77,38 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.OptionalLong; import java.util.Properties; +import java.util.Queue; import java.util.Set; +import java.util.concurrent.BlockingQueue; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; +import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; +import java.util.function.Supplier; import java.util.regex.Pattern; import java.util.stream.Collectors; import static java.util.Objects.requireNonNull; import static org.apache.kafka.clients.consumer.ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG; import static org.apache.kafka.clients.consumer.ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG; -import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.configuredConsumerInterceptors; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_JMX_PREFIX; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRIC_GROUP_PREFIX; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createFetchConfig; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createFetchMetricsManager; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createLogContext; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createMetrics; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createSubscriptionState; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.configuredConsumerInterceptors; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.configuredIsolationLevel; import static org.apache.kafka.common.utils.Utils.closeQuietly; import static org.apache.kafka.common.utils.Utils.isBlank; import static org.apache.kafka.common.utils.Utils.join; import static org.apache.kafka.common.utils.Utils.propsToMap; +import static org.apache.kafka.common.utils.Utils.swallow; /** * This prototype consumer uses the EventHandler to process application @@ -100,21 +119,35 @@ public class PrototypeAsyncConsumer implements Consumer { static final long DEFAULT_CLOSE_TIMEOUT_MS = 30 * 1000; - private final LogContext logContext; private final EventHandler eventHandler; private final Time time; private final Optional groupId; - private final Logger log; + private final KafkaConsumerMetrics kafkaConsumerMetrics; + private Logger log; + private final String clientId; + private final BackgroundEventProcessor backgroundEventProcessor; private final Deserializers deserializers; + private final FetchBuffer fetchBuffer; + private final FetchCollector fetchCollector; + private final ConsumerInterceptors interceptors; + private final IsolationLevel isolationLevel; + private final SubscriptionState subscriptions; private final ConsumerMetadata metadata; private final Metrics metrics; + private final long retryBackoffMs; + private final long requestTimeoutMs; private final long defaultApiTimeoutMs; + private volatile boolean closed = false; + private final List assignors; - private WakeupTrigger wakeupTrigger = new WakeupTrigger(); - public PrototypeAsyncConsumer(Properties properties, - Deserializer keyDeserializer, - Deserializer valueDeserializer) { + // to keep from repeatedly scanning subscriptions in poll(), cache the result during metadata updates + private boolean cachedSubscriptionHasAllFetchPositions; + private final WakeupTrigger wakeupTrigger = new WakeupTrigger(); + + public PrototypeAsyncConsumer(final Properties properties, + final Deserializer keyDeserializer, + final Deserializer valueDeserializer) { this(propsToMap(properties), keyDeserializer, valueDeserializer); } @@ -127,56 +160,151 @@ public PrototypeAsyncConsumer(final Map configs, public PrototypeAsyncConsumer(final ConsumerConfig config, final Deserializer keyDeserializer, final Deserializer valueDeserializer) { - this.time = Time.SYSTEM; - GroupRebalanceConfig groupRebalanceConfig = new GroupRebalanceConfig(config, - GroupRebalanceConfig.ProtocolType.CONSUMER); - this.groupId = Optional.ofNullable(groupRebalanceConfig.groupId); - this.defaultApiTimeoutMs = config.getInt(ConsumerConfig.DEFAULT_API_TIMEOUT_MS_CONFIG); - this.logContext = createLogContext(config, groupRebalanceConfig); - this.log = logContext.logger(getClass()); - this.deserializers = new Deserializers<>(config, keyDeserializer, valueDeserializer); - this.subscriptions = createSubscriptionState(config, logContext); - this.metrics = createMetrics(config, time); - List> interceptorList = configuredConsumerInterceptors(config); - ClusterResourceListeners clusterResourceListeners = ClientUtils.configureClusterResourceListeners( - metrics.reporters(), - interceptorList, - Arrays.asList(deserializers.keyDeserializer, deserializers.valueDeserializer)); - this.metadata = new ConsumerMetadata(config, subscriptions, logContext, clusterResourceListeners); - final List addresses = ClientUtils.parseAndValidateAddresses(config); - metadata.bootstrap(addresses); - this.eventHandler = new DefaultEventHandler( - config, - groupRebalanceConfig, - logContext, - subscriptions, - new ApiVersions(), - this.metrics, - clusterResourceListeners, - null // this is coming from the fetcher, but we don't have one - ); + this(Time.SYSTEM, config, keyDeserializer, valueDeserializer); } - // Visible for testing - PrototypeAsyncConsumer(Time time, - LogContext logContext, - ConsumerConfig config, - SubscriptionState subscriptions, - ConsumerMetadata metadata, - EventHandler eventHandler, - Metrics metrics, - Optional groupId, - int defaultApiTimeoutMs) { - this.time = time; - this.logContext = logContext; + public PrototypeAsyncConsumer(final Time time, + final ConsumerConfig config, + final Deserializer keyDeserializer, + final Deserializer valueDeserializer) { + try { + GroupRebalanceConfig groupRebalanceConfig = new GroupRebalanceConfig(config, + GroupRebalanceConfig.ProtocolType.CONSUMER); + + this.groupId = Optional.ofNullable(groupRebalanceConfig.groupId); + this.clientId = config.getString(CommonClientConfigs.CLIENT_ID_CONFIG); + LogContext logContext = createLogContext(config, groupRebalanceConfig); + this.log = logContext.logger(getClass()); + groupId.ifPresent(groupIdStr -> { + if (groupIdStr.isEmpty()) { + log.warn("Support for using the empty group id by consumers is deprecated and will be removed in the next major release."); + } + }); + + log.debug("Initializing the Kafka consumer"); + this.requestTimeoutMs = config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG); + this.defaultApiTimeoutMs = config.getInt(ConsumerConfig.DEFAULT_API_TIMEOUT_MS_CONFIG); + this.time = time; + this.metrics = createMetrics(config, time); + this.retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG); + + List> interceptorList = configuredConsumerInterceptors(config); + this.interceptors = new ConsumerInterceptors<>(interceptorList); + this.deserializers = new Deserializers<>(config, keyDeserializer, valueDeserializer); + this.subscriptions = createSubscriptionState(config, logContext); + ClusterResourceListeners clusterResourceListeners = ClientUtils.configureClusterResourceListeners(metrics.reporters(), + interceptorList, + Arrays.asList(deserializers.keyDeserializer, deserializers.valueDeserializer)); + this.metadata = new ConsumerMetadata(config, subscriptions, logContext, clusterResourceListeners); + final List addresses = ClientUtils.parseAndValidateAddresses(config); + metadata.bootstrap(addresses); + + FetchMetricsManager fetchMetricsManager = createFetchMetricsManager(metrics); + this.isolationLevel = configuredIsolationLevel(config); + + ApiVersions apiVersions = new ApiVersions(); + final BlockingQueue applicationEventQueue = new LinkedBlockingQueue<>(); + final BlockingQueue backgroundEventQueue = new LinkedBlockingQueue<>(); + final Supplier networkClientDelegateSupplier = NetworkClientDelegate.supplier(time, + logContext, + metadata, + config, + apiVersions, + metrics, + fetchMetricsManager); + final Supplier requestManagersSupplier = RequestManagers.supplier(time, + logContext, + backgroundEventQueue, + metadata, + subscriptions, + config, + groupRebalanceConfig, + apiVersions, + fetchMetricsManager, + networkClientDelegateSupplier); + final Supplier applicationEventProcessorSupplier = ApplicationEventProcessor.supplier(logContext, + metadata, + backgroundEventQueue, + requestManagersSupplier); + this.eventHandler = new DefaultEventHandler(time, + logContext, + applicationEventQueue, + applicationEventProcessorSupplier, + networkClientDelegateSupplier, + requestManagersSupplier); + this.backgroundEventProcessor = new BackgroundEventProcessor(logContext, backgroundEventQueue); + this.assignors = ConsumerPartitionAssignor.getAssignorInstances( + config.getList(ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG), + config.originals(Collections.singletonMap(ConsumerConfig.CLIENT_ID_CONFIG, clientId)) + ); + + // no coordinator will be constructed for the default (null) group id + if (!groupId.isPresent()) { + config.ignore(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG); + //config.ignore(ConsumerConfig.THROW_ON_FETCH_STABLE_OFFSET_UNSUPPORTED); + } + // These are specific to the foreground thread + FetchConfig fetchConfig = createFetchConfig(config); + this.fetchBuffer = new FetchBuffer(logContext); + this.fetchCollector = new FetchCollector<>(logContext, + metadata, + subscriptions, + fetchConfig, + fetchMetricsManager, + time); + + this.kafkaConsumerMetrics = new KafkaConsumerMetrics(metrics, CONSUMER_METRIC_GROUP_PREFIX); + + config.logUnused(); + AppInfoParser.registerAppInfo(CONSUMER_JMX_PREFIX, clientId, metrics, time.milliseconds()); + log.debug("Kafka consumer initialized"); + } catch (Throwable t) { + // call close methods if internal objects are already constructed; this is to prevent resource leak. see KAFKA-2121 + // we do not need to call `close` at all when `log` is null, which means no internal objects were initialized. + if (this.log != null) { + close(Duration.ZERO, true); + } + // now propagate the exception + throw new KafkaException("Failed to construct kafka consumer", t); + } + } + + public PrototypeAsyncConsumer(LogContext logContext, + String clientId, + Deserializers deserializers, + FetchBuffer fetchBuffer, + FetchCollector fetchCollector, + ConsumerInterceptors interceptors, + Time time, + EventHandler eventHandler, + BlockingQueue backgroundEventQueue, + Metrics metrics, + SubscriptionState subscriptions, + ConsumerMetadata metadata, + long retryBackoffMs, + long requestTimeoutMs, + int defaultApiTimeoutMs, + List assignors, + String groupId) { this.log = logContext.logger(getClass()); this.subscriptions = subscriptions; - this.metadata = metadata; + this.clientId = clientId; + this.fetchBuffer = fetchBuffer; + this.fetchCollector = fetchCollector; + this.isolationLevel = IsolationLevel.READ_UNCOMMITTED; + this.interceptors = Objects.requireNonNull(interceptors); + this.time = time; + this.backgroundEventProcessor = new BackgroundEventProcessor(logContext, backgroundEventQueue); this.metrics = metrics; - this.groupId = groupId; + this.groupId = Optional.ofNullable(groupId); + this.metadata = metadata; + this.retryBackoffMs = retryBackoffMs; + this.requestTimeoutMs = requestTimeoutMs; this.defaultApiTimeoutMs = defaultApiTimeoutMs; - this.deserializers = new Deserializers<>(config); + this.deserializers = deserializers; this.eventHandler = eventHandler; + this.assignors = assignors; + this.kafkaConsumerMetrics = new KafkaConsumerMetrics(metrics, "consumer"); } /** @@ -192,71 +320,38 @@ public PrototypeAsyncConsumer(final ConsumerConfig config, @Override public ConsumerRecords poll(final Duration timeout) { Timer timer = time.timer(timeout); + try { - do { - if (!eventHandler.isEmpty()) { - final Optional backgroundEvent = eventHandler.poll(); - // processEvent() may process 3 types of event: - // 1. Errors - // 2. Callback Invocation - // 3. Fetch responses - // Errors will be handled or rethrown. - // Callback invocation will trigger callback function execution, which is blocking until completion. - // Successful fetch responses will be added to the completedFetches in the fetcher, which will then - // be processed in the collectFetches(). - backgroundEvent.ifPresent(event -> processEvent(event, timeout)); - } + backgroundEventProcessor.process(); - updateFetchPositionsIfNeeded(timer); + this.kafkaConsumerMetrics.recordPollStart(timer.currentTimeMs()); - // The idea here is to have the background thread sending fetches autonomously, and the fetcher - // uses the poll loop to retrieve successful fetchResponse and process them on the polling thread. - final Fetch fetch = collectFetches(); - if (!fetch.isEmpty()) { - return processFetchResults(fetch); - } - // We will wait for retryBackoffMs - } while (time.timer(timeout).notExpired()); - } catch (final Exception e) { - throw new RuntimeException(e); - } - // TODO: Once we implement poll(), clear wakeupTrigger in a finally block: wakeupTrigger.clearActiveTask(); + if (this.subscriptions.hasNoSubscriptionOrUserAssignment()) { + throw new IllegalStateException("Consumer is not subscribed to any topics or assigned any partitions"); + } - return ConsumerRecords.empty(); - } + do { + updateAssignmentMetadataIfNeeded(timer); + final Fetch fetch = pollForFetches(timer); - /** - * Set the fetch position to the committed position (if there is one) or reset it using the - * offset reset policy the user has configured (if partitions require reset) - * - * @return true if the operation completed without timing out - * @throws org.apache.kafka.common.errors.AuthenticationException if authentication fails. See the exception for more details - * @throws NoOffsetForPartitionException If no offset is stored for a given partition and no offset reset policy is - * defined - */ - private boolean updateFetchPositionsIfNeeded(final Timer timer) { - // If any partitions have been truncated due to a leader change, we need to validate the offsets - ValidatePositionsApplicationEvent validatePositionsEvent = new ValidatePositionsApplicationEvent(); - eventHandler.add(validatePositionsEvent); + if (!fetch.isEmpty()) { + sendFetches(); - // If there are any partitions which do not have a valid position and are not - // awaiting reset, then we need to fetch committed offsets. We will only do a - // coordinator lookup if there are partitions which have missing positions, so - // a consumer with manually assigned partitions can avoid a coordinator dependence - // by always ensuring that assigned partitions have an initial position. - if (isCommittedOffsetsManagementEnabled() && !refreshCommittedOffsetsIfNeeded(timer)) - return false; + if (fetch.records().isEmpty()) { + log.trace("Returning empty records from `poll()` " + + "since the consumer's position has advanced for at least one topic partition"); + } - // If there are partitions still needing a position and a reset policy is defined, - // request reset using the default policy. If no reset strategy is defined and there - // are partitions with a missing position, then we will raise a NoOffsetForPartitionException exception. - subscriptions.resetInitializingPositions(); + return this.interceptors.onConsume(new ConsumerRecords<>(fetch.records())); + } + // We will wait for retryBackoffMs + } while (timer.notExpired()); - // Finally send an asynchronous request to look up and update the positions of any - // partitions which are awaiting reset. - ResetPositionsApplicationEvent resetPositionsEvent = new ResetPositionsApplicationEvent(); - eventHandler.add(resetPositionsEvent); - return true; + return ConsumerRecords.empty(); + } finally { + this.kafkaConsumerMetrics.recordPollEnd(timer.currentTimeMs()); + } + // TODO: Once we implement poll(), clear wakeupTrigger in a finally block: wakeupTrigger.clearActiveTask(); } /** @@ -268,20 +363,6 @@ public void commitSync() { commitSync(Duration.ofMillis(defaultApiTimeoutMs)); } - private void processEvent(final BackgroundEvent backgroundEvent, final Duration timeout) { - // stubbed class - } - - private ConsumerRecords processFetchResults(final Fetch fetch) { - // stubbed class - return ConsumerRecords.empty(); - } - - private Fetch collectFetches() { - // stubbed class - return Fetch.empty(); - } - /** * This method sends a commit event to the EventHandler and return. */ @@ -324,44 +405,90 @@ CompletableFuture commit(Map offsets, f @Override public void seek(TopicPartition partition, long offset) { - throw new KafkaException("method not implemented"); + if (offset < 0) + throw new IllegalArgumentException("seek offset must not be a negative number"); + + log.info("Seeking to offset {} for partition {}", offset, partition); + SubscriptionState.FetchPosition newPosition = new SubscriptionState.FetchPosition( + offset, + Optional.empty(), // This will ensure we skip validation + this.metadata.currentLeader(partition)); + this.subscriptions.seekUnvalidated(partition, newPosition); } @Override public void seek(TopicPartition partition, OffsetAndMetadata offsetAndMetadata) { - throw new KafkaException("method not implemented"); + long offset = offsetAndMetadata.offset(); + if (offset < 0) { + throw new IllegalArgumentException("seek offset must not be a negative number"); + } + + if (offsetAndMetadata.leaderEpoch().isPresent()) { + log.info("Seeking to offset {} for partition {} with epoch {}", + offset, partition, offsetAndMetadata.leaderEpoch().get()); + } else { + log.info("Seeking to offset {} for partition {}", offset, partition); + } + Metadata.LeaderAndEpoch currentLeaderAndEpoch = this.metadata.currentLeader(partition); + SubscriptionState.FetchPosition newPosition = new SubscriptionState.FetchPosition( + offsetAndMetadata.offset(), + offsetAndMetadata.leaderEpoch(), + currentLeaderAndEpoch); + this.updateLastSeenEpochIfNewer(partition, offsetAndMetadata); + this.subscriptions.seekUnvalidated(partition, newPosition); } @Override public void seekToBeginning(Collection partitions) { - throw new KafkaException("method not implemented"); + if (partitions == null) + throw new IllegalArgumentException("Partitions collection cannot be null"); + + Collection parts = partitions.size() == 0 ? this.subscriptions.assignedPartitions() : partitions; + subscriptions.requestOffsetReset(parts, OffsetResetStrategy.EARLIEST); } @Override public void seekToEnd(Collection partitions) { - throw new KafkaException("method not implemented"); + if (partitions == null) + throw new IllegalArgumentException("Partitions collection cannot be null"); + + Collection parts = partitions.size() == 0 ? this.subscriptions.assignedPartitions() : partitions; + subscriptions.requestOffsetReset(parts, OffsetResetStrategy.LATEST); } @Override public long position(TopicPartition partition) { - throw new KafkaException("method not implemented"); + return position(partition, Duration.ofMillis(defaultApiTimeoutMs)); } @Override public long position(TopicPartition partition, Duration timeout) { - throw new KafkaException("method not implemented"); + if (!this.subscriptions.isAssigned(partition)) + throw new IllegalStateException("You can only check the position for partitions assigned to this consumer."); + + Timer timer = time.timer(timeout); + do { + SubscriptionState.FetchPosition position = this.subscriptions.validPosition(partition); + if (position != null) + return position.offset; + + updateFetchPositions(timer); + } while (timer.notExpired()); + + throw new TimeoutException("Timeout of " + timeout.toMillis() + "ms expired before the position " + + "for partition " + partition + " could be determined"); } @Override @Deprecated public OffsetAndMetadata committed(TopicPartition partition) { - throw new KafkaException("method not implemented"); + return committed(partition, Duration.ofMillis(defaultApiTimeoutMs)); } @Override @Deprecated public OffsetAndMetadata committed(TopicPartition partition, Duration timeout) { - throw new KafkaException("method not implemented"); + return committed(Collections.singleton(partition), timeout).get(partition); } @Override @@ -395,12 +522,12 @@ private void maybeThrowInvalidGroupIdException() { @Override public Map metrics() { - throw new KafkaException("method not implemented"); + return Collections.unmodifiableMap(this.metrics.metrics()); } @Override public List partitionsFor(String topic) { - throw new KafkaException("method not implemented"); + return partitionsFor(topic, Duration.ofMillis(defaultApiTimeoutMs)); } @Override @@ -410,7 +537,7 @@ public List partitionsFor(String topic, Duration timeout) { @Override public Map> listTopics() { - throw new KafkaException("method not implemented"); + return listTopics(Duration.ofMillis(defaultApiTimeoutMs)); } @Override @@ -420,17 +547,23 @@ public Map> listTopics(Duration timeout) { @Override public Set paused() { - throw new KafkaException("method not implemented"); + return Collections.unmodifiableSet(subscriptions.pausedPartitions()); } @Override public void pause(Collection partitions) { - throw new KafkaException("method not implemented"); + log.debug("Pausing partitions {}", partitions); + for (TopicPartition partition: partitions) { + subscriptions.pause(partition); + } } @Override public void resume(Collection partitions) { - throw new KafkaException("method not implemented"); + log.debug("Resuming partitions {}", partitions); + for (TopicPartition partition: partitions) { + subscriptions.resume(partition); + } } @Override @@ -502,7 +635,25 @@ private Map beginningOrEndOffset(Collection firstException = new AtomicReference<>(); + + final Timer closeTimer = createTimerForRequest(timeout); + if (fetchBuffer != null) { + // the timeout for the session close is at-most the requestTimeoutMs + long remainingDurationInTimeout = Math.max(0, timeout.toMillis() - closeTimer.elapsedMs()); + if (remainingDurationInTimeout > 0) { + remainingDurationInTimeout = Math.min(requestTimeoutMs, remainingDurationInTimeout); + } + + closeTimer.reset(remainingDurationInTimeout); + + // This is a blocking call bound by the time remaining in closeTimer + swallow(log, Level.ERROR, "Failed to close fetcher", fetchBuffer::close, firstException); + } + + + closeQuietly(interceptors, "consumer interceptors", firstException); + closeQuietly(kafkaConsumerMetrics, "kafka consumer metrics", firstException); + closeQuietly(metrics, "consumer metrics", firstException); closeQuietly(this.eventHandler, "event handler", firstException); + closeQuietly(deserializers, "consumer deserializers", firstException); + + AppInfoParser.unregisterAppInfo(CONSUMER_JMX_PREFIX, clientId, metrics); log.debug("Kafka consumer has been closed"); Throwable exception = firstException.get(); - if (exception != null) { + if (exception != null && !swallowException) { if (exception instanceof InterruptException) { throw (InterruptException) exception; } @@ -562,19 +757,14 @@ public void commitSync(Map offsets) { @Override public void commitSync(Map offsets, Duration timeout) { - CompletableFuture commitFuture = commit(offsets, true); + long commitStart = time.nanoseconds(); try { - commitFuture.get(timeout.toMillis(), TimeUnit.MILLISECONDS); - } catch (final TimeoutException e) { - throw new org.apache.kafka.common.errors.TimeoutException(e); - } catch (final InterruptedException e) { - throw new InterruptException(e); - } catch (final ExecutionException e) { - if (e.getCause() instanceof WakeupException) - throw new WakeupException(); - throw new KafkaException(e); + CompletableFuture commitFuture = commit(offsets, true); + offsets.forEach(this::updateLastSeenEpochIfNewer); + ConsumerUtils.getResult(commitFuture, time.timer(timeout)); } finally { wakeupTrigger.clearActiveTask(); + kafkaConsumerMetrics.recordCommitSync(time.nanoseconds() - commitStart); } } @@ -595,12 +785,38 @@ public Set subscription() { @Override public void subscribe(Collection topics) { - throw new KafkaException("method not implemented"); + subscribe(topics, new NoOpConsumerRebalanceListener()); } @Override public void subscribe(Collection topics, ConsumerRebalanceListener callback) { - throw new KafkaException("method not implemented"); + maybeThrowInvalidGroupIdException(); + if (topics == null) + throw new IllegalArgumentException("Topic collection to subscribe to cannot be null"); + if (topics.isEmpty()) { + // treat subscribing to empty topic list as the same as unsubscribing + this.unsubscribe(); + } else { + for (String topic : topics) { + if (isBlank(topic)) + throw new IllegalArgumentException("Topic collection to subscribe to cannot contain null or empty topic"); + } + + throwIfNoAssignorsConfigured(); + + // Clear the buffered data which are not a part of newly assigned topics + final Set currentTopicPartitions = new HashSet<>(); + + for (TopicPartition tp : subscriptions.assignedPartitions()) { + if (topics.contains(tp.topic())) + currentTopicPartitions.add(tp); + } + + fetchBuffer.retainAll(currentTopicPartitions); + log.info("Subscribed to topic(s): {}", join(topics, ", ")); + if (this.subscriptions.subscribe(new HashSet<>(topics), callback)) + metadata.requestUpdateForNewTopics(); + } } @Override @@ -610,8 +826,7 @@ public void assign(Collection partitions) { } if (partitions.isEmpty()) { - // TODO: implementation of unsubscribe() will be included in forthcoming commits. - // this.unsubscribe(); + this.unsubscribe(); return; } @@ -621,12 +836,18 @@ public void assign(Collection partitions) { throw new IllegalArgumentException("Topic partitions to assign to cannot have null or empty topic"); } - // TODO: implementation of refactored Fetcher will be included in forthcoming commits. - // fetcher.clearBufferedDataForUnassignedPartitions(partitions); + // Clear the buffered data which are not a part of newly assigned topics + final Set currentTopicPartitions = new HashSet<>(); + + for (TopicPartition tp : subscriptions.assignedPartitions()) { + if (partitions.contains(tp)) + currentTopicPartitions.add(tp); + } + + fetchBuffer.retainAll(currentTopicPartitions); - // assignment change event will trigger autocommit if it is configured and the group id is specified. This is - // to make sure offsets of topic partitions the consumer is unsubscribing from are committed since there will - // be no following rebalance + // make sure the offsets of topic partitions the consumer is unsubscribing from + // are committed since there will be no following rebalance eventHandler.add(new AssignmentChangeApplicationEvent(this.subscriptions.allConsumed(), time.milliseconds())); log.info("Assigned to partition(s): {}", join(partitions, ", ")); @@ -635,24 +856,52 @@ public void assign(Collection partitions) { } @Override - public void subscribe(Pattern pattern, ConsumerRebalanceListener callback) { - throw new KafkaException("method not implemented"); + public void subscribe(Pattern pattern, ConsumerRebalanceListener listener) { + maybeThrowInvalidGroupIdException(); + if (pattern == null || pattern.toString().equals("")) + throw new IllegalArgumentException("Topic pattern to subscribe to cannot be " + (pattern == null ? + "null" : "empty")); + + throwIfNoAssignorsConfigured(); + log.info("Subscribed to pattern: '{}'", pattern); + this.subscriptions.subscribe(pattern, listener); + this.updatePatternSubscription(metadata.fetch()); + this.metadata.requestUpdateForNewTopics(); + } + + /** + * TODO: remove this when we implement the KIP-848 protocol. + * + *

    + * The contents of this method are shamelessly stolen from + * {@link ConsumerCoordinator#updatePatternSubscription(Cluster)} and are used here because we won't have access + * to a {@link ConsumerCoordinator} in this code. Perhaps it could be moved to a ConsumerUtils class? + * + * @param cluster Cluster from which we get the topics + */ + private void updatePatternSubscription(Cluster cluster) { + final Set topicsToSubscribe = cluster.topics().stream() + .filter(subscriptions::matchesSubscribedPattern) + .collect(Collectors.toSet()); + if (subscriptions.subscribeFromPattern(topicsToSubscribe)) + metadata.requestUpdateForNewTopics(); } @Override public void subscribe(Pattern pattern) { - throw new KafkaException("method not implemented"); + subscribe(pattern, new NoOpConsumerRebalanceListener()); } @Override public void unsubscribe() { - throw new KafkaException("method not implemented"); + fetchBuffer.retainAll(Collections.emptySet()); + this.subscriptions.unsubscribe(); } @Override @Deprecated - public ConsumerRecords poll(long timeout) { - throw new KafkaException("method not implemented"); + public ConsumerRecords poll(final long timeoutMs) { + return poll(Duration.ofMillis(timeoutMs)); } // Visible for testing @@ -660,17 +909,95 @@ WakeupTrigger wakeupTrigger() { return wakeupTrigger; } - private static ClusterResourceListeners configureClusterResourceListeners( - final Deserializer keyDeserializer, - final Deserializer valueDeserializer, - final List... candidateLists) { - ClusterResourceListeners clusterResourceListeners = new ClusterResourceListeners(); - for (List candidateList: candidateLists) - clusterResourceListeners.maybeAddAll(candidateList); + private void sendFetches() { + FetchEvent event = new FetchEvent(); + eventHandler.add(event); + + event.future().whenComplete((completedFetches, error) -> { + if (completedFetches != null && !completedFetches.isEmpty()) { + fetchBuffer.addAll(completedFetches); + } + }); + } + + /** + * @throws KafkaException if the rebalance callback throws exception + */ + private Fetch pollForFetches(Timer timer) { + long pollTimeout = timer.remainingMs(); + + // if data is available already, return it immediately + final Fetch fetch = fetchCollector.collectFetch(fetchBuffer, deserializers); + if (!fetch.isEmpty()) { + return fetch; + } + + // send any new fetches (won't resend pending fetches) + sendFetches(); + + // We do not want to be stuck blocking in poll if we are missing some positions + // since the offset lookup may be backing off after a failure + + // NOTE: the use of cachedSubscriptionHasAllFetchPositions means we MUST call + // updateAssignmentMetadataIfNeeded before this method. + if (!cachedSubscriptionHasAllFetchPositions && pollTimeout > retryBackoffMs) { + pollTimeout = retryBackoffMs; + } + + log.trace("Polling for fetches with timeout {}", pollTimeout); + + Timer pollTimer = time.timer(pollTimeout); + + // Attempt to fetch any data. It's OK if we time out here; it's a best case effort. The + // data may not be immediately available, but the calling method (poll) will correctly + // handle the overall timeout. + try { + Queue completedFetches = eventHandler.addAndGet(new FetchEvent(), pollTimer); + if (completedFetches != null && !completedFetches.isEmpty()) { + fetchBuffer.addAll(completedFetches); + } + } catch (TimeoutException e) { + log.trace("Timeout during fetch", e); + } finally { + timer.update(pollTimer.currentTimeMs()); + } - clusterResourceListeners.maybeAdd(keyDeserializer); - clusterResourceListeners.maybeAdd(valueDeserializer); - return clusterResourceListeners; + return fetchCollector.collectFetch(fetchBuffer, deserializers); + } + + /** + * Set the fetch position to the committed position (if there is one) + * or reset it using the offset reset policy the user has configured. + * + * @throws org.apache.kafka.common.errors.AuthenticationException if authentication fails. See the exception for more details + * @throws NoOffsetForPartitionException If no offset is stored for a given partition and no offset reset policy is + * defined + * @return true iff the operation completed without timing out + */ + private boolean updateFetchPositions(final Timer timer) { + // If any partitions have been truncated due to a leader change, we need to validate the offsets + eventHandler.add(new ValidatePositionsApplicationEvent()); + + cachedSubscriptionHasAllFetchPositions = subscriptions.hasAllFetchPositions(); + if (cachedSubscriptionHasAllFetchPositions) return true; + + // If there are any partitions which do not have a valid position and are not + // awaiting reset, then we need to fetch committed offsets. We will only do a + // coordinator lookup if there are partitions which have missing positions, so + // a consumer with manually assigned partitions can avoid a coordinator dependence + // by always ensuring that assigned partitions have an initial position. + if (isCommittedOffsetsManagementEnabled() && !refreshCommittedOffsetsIfNeeded(timer)) + return false; + + // If there are partitions still needing a position and a reset policy is defined, + // request reset using the default policy. If no reset strategy is defined and there + // are partitions with a missing position, then we will raise a NoOffsetForPartitionException exception. + subscriptions.resetInitializingPositions(); + + // Finally send an asynchronous request to look up and update the positions of any + // partitions which are awaiting reset. + eventHandler.add(new ResetPositionsApplicationEvent()); + return true; } /** @@ -683,7 +1010,7 @@ private boolean isCommittedOffsetsManagementEnabled() { } /** - * Refresh the committed offsets for partitions that require initialization. + * Refresh the committed offsets for provided partitions. * * @param timer Timer bounding how long this method can block * @return true iff the operation completed within the timeout @@ -701,7 +1028,6 @@ private boolean refreshCommittedOffsetsIfNeeded(Timer timer) { } } - // This is here temporary as we don't have public access to the ConsumerConfig in this module. public static Map appendDeserializerToConfig(Map configs, Deserializer keyDeserializer, @@ -719,6 +1045,17 @@ else if (newConfigs.get(VALUE_DESERIALIZER_CLASS_CONFIG) == null) return newConfigs; } + private void throwIfNoAssignorsConfigured() { + if (assignors.isEmpty()) + throw new IllegalStateException("Must configure at least one partition assigner class name to " + + ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG + " configuration property"); + } + + private void updateLastSeenEpochIfNewer(TopicPartition topicPartition, OffsetAndMetadata offsetAndMetadata) { + if (offsetAndMetadata != null) + offsetAndMetadata.leaderEpoch().ifPresent(epoch -> metadata.updateLastSeenEpochIfNewer(topicPartition, epoch)); + } + private class DefaultOffsetCommitCallback implements OffsetCommitCallback { @Override public void onComplete(Map offsets, Exception exception) { @@ -726,4 +1063,11 @@ public void onComplete(Map offsets, Exception log.error("Offset commit with offsets {} failed", offsets, exception); } } + + boolean updateAssignmentMetadataIfNeeded(Timer timer) { + // Keeping this updateAssignmentMetadataIfNeeded wrapping up the updateFetchPositions as + // in the previous implementation, because it will eventually involve group coordination + // logic + return updateFetchPositions(timer); + } } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManager.java index 4655f5ef7042e..5bd6e01c2bb15 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManager.java @@ -18,11 +18,18 @@ import org.apache.kafka.clients.consumer.internals.NetworkClientDelegate.PollResult; +import java.io.Closeable; + /** * {@code PollResult} consist of {@code UnsentRequest} if there are requests to send; otherwise, return the time till * the next poll event. */ -public interface RequestManager { +public interface RequestManager extends Closeable { + PollResult poll(long currentTimeMs); + @Override + default void close() { + // Do nothing... + } } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java index 3864b1fcaa6f9..d1f520dddcef0 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java @@ -16,45 +16,156 @@ */ package org.apache.kafka.clients.consumer.internals; +import org.apache.kafka.clients.ApiVersions; +import org.apache.kafka.clients.GroupRebalanceConfig; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.internals.events.BackgroundEvent; +import org.apache.kafka.common.IsolationLevel; +import org.apache.kafka.common.internals.IdempotentCloser; +import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.common.utils.Time; +import org.slf4j.Logger; + +import java.io.Closeable; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.concurrent.BlockingQueue; +import java.util.function.Supplier; import static java.util.Objects.requireNonNull; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createFetchConfig; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.configuredIsolationLevel; /** * {@code RequestManagers} provides a means to pass around the set of {@link RequestManager} instances in the system. * This allows callers to both use the specific {@link RequestManager} instance, or to iterate over the list via * the {@link #entries()} method. */ -public class RequestManagers { +public class RequestManagers implements Closeable { + private final Logger log; public final Optional coordinatorRequestManager; public final Optional commitRequestManager; public final OffsetsRequestManager offsetsRequestManager; public final TopicMetadataRequestManager topicMetadataRequestManager; + public final FetchRequestManager fetchRequestManager; private final List> entries; + private final IdempotentCloser closer = new IdempotentCloser(); - public RequestManagers(OffsetsRequestManager offsetsRequestManager, + public RequestManagers(LogContext logContext, + OffsetsRequestManager offsetsRequestManager, TopicMetadataRequestManager topicMetadataRequestManager, + FetchRequestManager fetchRequestManager, Optional coordinatorRequestManager, Optional commitRequestManager) { - this.offsetsRequestManager = requireNonNull(offsetsRequestManager, - "OffsetsRequestManager cannot be null"); + this.log = logContext.logger(RequestManagers.class); + this.offsetsRequestManager = requireNonNull(offsetsRequestManager, "OffsetsRequestManager cannot be null"); this.coordinatorRequestManager = coordinatorRequestManager; this.commitRequestManager = commitRequestManager; this.topicMetadataRequestManager = topicMetadataRequestManager; + this.fetchRequestManager = fetchRequestManager; List> list = new ArrayList<>(); list.add(coordinatorRequestManager); list.add(commitRequestManager); list.add(Optional.of(offsetsRequestManager)); list.add(Optional.of(topicMetadataRequestManager)); + list.add(Optional.of(fetchRequestManager)); entries = Collections.unmodifiableList(list); } public List> entries() { return entries; } + + @Override + public void close() { + closer.close( + () -> { + log.debug("Closing RequestManagers"); + + entries.forEach(rm -> { + rm.ifPresent(requestManager -> { + try { + requestManager.close(); + } catch (Throwable t) { + log.debug("Error closing request manager {}", requestManager.getClass().getSimpleName(), t); + } + }); + }); + log.debug("RequestManagers has been closed"); + }, + () -> log.debug("RequestManagers was already closed")); + + } + + /** + * Creates a {@link Supplier} for deferred creation during invocation by + * {@link org.apache.kafka.clients.consumer.internals.DefaultBackgroundThread}. + */ + public static Supplier supplier(final Time time, + final LogContext logContext, + final BlockingQueue backgroundEventQueue, + final ConsumerMetadata metadata, + final SubscriptionState subscriptions, + final ConsumerConfig config, + final GroupRebalanceConfig groupRebalanceConfig, + final ApiVersions apiVersions, + final FetchMetricsManager fetchMetricsManager, + final Supplier networkClientDelegateSupplier) { + return new CachedSupplier() { + @Override + protected RequestManagers create() { + final NetworkClientDelegate networkClientDelegate = networkClientDelegateSupplier.get(); + final ErrorEventHandler errorEventHandler = new ErrorEventHandler(backgroundEventQueue); + final IsolationLevel isolationLevel = configuredIsolationLevel(config); + final FetchConfig fetchConfig = createFetchConfig(config); + long retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG); + long retryBackoffMaxMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MAX_MS_CONFIG); + final int requestTimeoutMs = config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG); + final OffsetsRequestManager listOffsets = new OffsetsRequestManager(subscriptions, + metadata, + isolationLevel, + time, + retryBackoffMs, + requestTimeoutMs, + apiVersions, + networkClientDelegate, + logContext); + final FetchRequestManager fetch = new FetchRequestManager(logContext, + time, + errorEventHandler, + metadata, + subscriptions, + fetchConfig, + fetchMetricsManager, + networkClientDelegate); + final TopicMetadataRequestManager topic = new TopicMetadataRequestManager( + logContext, + config); + CoordinatorRequestManager coordinator = null; + CommitRequestManager commit = null; + + if (groupRebalanceConfig != null && groupRebalanceConfig.groupId != null) { + final GroupState groupState = new GroupState(groupRebalanceConfig); + coordinator = new CoordinatorRequestManager(time, + logContext, + retryBackoffMs, + retryBackoffMaxMs, + errorEventHandler, + groupState.groupId); + commit = new CommitRequestManager(time, logContext, subscriptions, config, coordinator, groupState); + } + + return new RequestManagers(logContext, + listOffsets, + topic, + fetch, + Optional.ofNullable(coordinator), + Optional.ofNullable(commit)); + } + }; + } } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEvent.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEvent.java index bf9bb1d496217..9badce91cfed0 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEvent.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEvent.java @@ -25,7 +25,7 @@ public abstract class ApplicationEvent { public enum Type { NOOP, COMMIT, POLL, FETCH_COMMITTED_OFFSET, METADATA_UPDATE, ASSIGNMENT_CHANGE, - LIST_OFFSETS, RESET_POSITIONS, VALIDATE_POSITIONS, TOPIC_METADATA + LIST_OFFSETS, RESET_POSITIONS, VALIDATE_POSITIONS, TOPIC_METADATA, FETCH } private final Type type; diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventProcessor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventProcessor.java index f2f558600044a..19462de520cfc 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventProcessor.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventProcessor.java @@ -17,38 +17,54 @@ package org.apache.kafka.clients.consumer.internals.events; import org.apache.kafka.clients.consumer.OffsetAndTimestamp; +import org.apache.kafka.clients.consumer.internals.CachedSupplier; import org.apache.kafka.clients.consumer.internals.CommitRequestManager; +import org.apache.kafka.clients.consumer.internals.CompletedFetch; import org.apache.kafka.clients.consumer.internals.ConsumerMetadata; import org.apache.kafka.clients.consumer.internals.RequestManagers; import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.PartitionInfo; import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.utils.LogContext; +import org.slf4j.Logger; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.Queue; import java.util.concurrent.BlockingQueue; import java.util.concurrent.CompletableFuture; +import java.util.function.Supplier; public class ApplicationEventProcessor { private final BlockingQueue backgroundEventQueue; + private final Logger log; + private final ConsumerMetadata metadata; private final RequestManagers requestManagers; public ApplicationEventProcessor(final BlockingQueue backgroundEventQueue, final RequestManagers requestManagers, - final ConsumerMetadata metadata) { + final ConsumerMetadata metadata, + final LogContext logContext) { + this.log = logContext.logger(ApplicationEventProcessor.class); this.backgroundEventQueue = backgroundEventQueue; this.requestManagers = requestManagers; this.metadata = metadata; } public boolean process(final ApplicationEvent event) { - Objects.requireNonNull(event); + Objects.requireNonNull(event, "Attempt to process null ApplicationEvent"); + Objects.requireNonNull(event.type(), "Attempt to process ApplicationEvent with null type: " + event); + + log.trace("Processing event: {}", event); + + // Make sure to use the event's type() method, not the type variable directly. This causes problems when + // unit tests mock the EventType. switch (event.type()) { case NOOP: return process((NoopApplicationEvent) event); @@ -66,6 +82,8 @@ public boolean process(final ApplicationEvent event) { return process((TopicMetadataApplicationEvent) event); case LIST_OFFSETS: return process((ListOffsetsApplicationEvent) event); + case FETCH: + return process((FetchEvent) event); case RESET_POSITIONS: return processResetPositionsEvent(); case VALIDATE_POSITIONS: @@ -159,4 +177,29 @@ private boolean process(final TopicMetadataApplicationEvent event) { event.chain(future); return true; } + + private boolean process(final FetchEvent event) { + // The request manager keeps track of the completed fetches, so we pull any that are ready off, and return + // them to the application. + Queue completedFetches = requestManagers.fetchRequestManager.drain(); + event.future().complete(completedFetches); + return true; + } + + /** + * Creates a {@link Supplier} for deferred creation during invocation by + * {@link org.apache.kafka.clients.consumer.internals.DefaultBackgroundThread}. + */ + public static Supplier supplier(final LogContext logContext, + final ConsumerMetadata metadata, + final BlockingQueue backgroundEventQueue, + final Supplier requestManagersSupplier) { + return new CachedSupplier() { + @Override + protected ApplicationEventProcessor create() { + RequestManagers requestManagers = requestManagersSupplier.get(); + return new ApplicationEventProcessor(backgroundEventQueue, requestManagers, metadata, logContext); + } + }; + } } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEventProcessor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEventProcessor.java new file mode 100644 index 0000000000000..cdf7fa20d37df --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEventProcessor.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.clients.consumer.internals.events; + +import org.apache.kafka.common.utils.LogContext; +import org.slf4j.Logger; + +import java.util.LinkedList; +import java.util.Objects; +import java.util.concurrent.BlockingQueue; + +public class BackgroundEventProcessor { + + private final Logger log; + private final BlockingQueue backgroundEventQueue; + + public BackgroundEventProcessor(final LogContext logContext, + final BlockingQueue backgroundEventQueue) { + this.log = logContext.logger(BackgroundEventProcessor.class); + this.backgroundEventQueue = backgroundEventQueue; + } + + /** + * Drains all available {@link BackgroundEvent}s, and then processes them in order. If any + * errors are thrown as a result of a {@link ErrorBackgroundEvent} or an error occurs while processing + * another type of {@link BackgroundEvent}, only the first exception will be thrown, all + * subsequent errors will simply be logged at WARN level. + * + * @throws RuntimeException or subclass + */ + public void process() { + LinkedList events = new LinkedList<>(); + backgroundEventQueue.drainTo(events); + + RuntimeException first = null; + int errorCount = 0; + + for (BackgroundEvent event : events) { + log.debug("Consuming background event: {}", event); + + try { + process(event); + } catch (RuntimeException e) { + errorCount++; + + if (first == null) { + first = e; + log.warn("Error #{} from background thread (will be logged and thrown): {}", errorCount, e.getMessage(), e); + } else { + log.warn("Error #{} from background thread (will be logged only): {}", errorCount, e.getMessage(), e); + } + } + } + + if (first != null) { + throw first; + } + } + + private void process(final BackgroundEvent event) { + Objects.requireNonNull(event, "Attempt to process null BackgroundEvent"); + Objects.requireNonNull(event.type(), "Attempt to process BackgroundEvent with null type: " + event); + + log.debug("Processing event {}", event); + + // Make sure to use the event's type() method, not the type variable directly. This causes problems when + // unit tests mock the EventType. + switch (event.type()) { + case NOOP: + process((NoopBackgroundEvent) event); + return; + + case ERROR: + process((ErrorBackgroundEvent) event); + return; + + default: + throw new IllegalArgumentException("Background event type " + event.type() + " was not expected"); + } + } + + private void process(final NoopBackgroundEvent event) { + log.debug("Received no-op background event with message: {}", event.message()); + } + + private void process(final ErrorBackgroundEvent event) { + throw event.error(); + } +} diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ErrorBackgroundEvent.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ErrorBackgroundEvent.java index 4fc08290b715c..7f3d4e22429e3 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ErrorBackgroundEvent.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ErrorBackgroundEvent.java @@ -18,14 +18,14 @@ public class ErrorBackgroundEvent extends BackgroundEvent { - private final Throwable error; + private final RuntimeException error; - public ErrorBackgroundEvent(Throwable error) { + public ErrorBackgroundEvent(RuntimeException error) { super(Type.ERROR); this.error = error; } - public Throwable error() { + public RuntimeException error() { return error; } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/EventHandler.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/EventHandler.java index 75075fc2779fc..4c76b5f7bf104 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/EventHandler.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/EventHandler.java @@ -19,7 +19,7 @@ import org.apache.kafka.common.utils.Timer; import java.io.Closeable; -import java.util.Optional; +import java.time.Duration; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; @@ -28,17 +28,6 @@ * the {@code add()} method and to retrieve events via the {@code poll()} method. */ public interface EventHandler extends Closeable { - /** - * Retrieves and removes a {@link BackgroundEvent}. Returns an empty Optional instance if there is nothing. - * @return an Optional of {@link BackgroundEvent} if the value is present. Otherwise, an empty Optional. - */ - Optional poll(); - - /** - * Check whether there are pending {@code BackgroundEvent} await to be consumed. - * @return true if there are no pending event - */ - boolean isEmpty(); /** * Add an {@link ApplicationEvent} to the handler. The method returns true upon successful add; otherwise returns @@ -62,4 +51,10 @@ public interface EventHandler extends Closeable { * @param Type of return value of the event */ T addAndGet(final CompletableApplicationEvent event, final Timer timer); + + default void close() { + close(Duration.ofMillis(Long.MAX_VALUE)); + } + + void close(Duration timeout); } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/FetchEvent.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/FetchEvent.java new file mode 100644 index 0000000000000..ec68a3e4cda44 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/FetchEvent.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.clients.consumer.internals.events; + +import org.apache.kafka.clients.consumer.internals.CompletedFetch; + +import java.util.Queue; + +/** + * This event signals the background thread to submit a fetch request. + */ +public class FetchEvent extends CompletableApplicationEvent> { + + public FetchEvent() { + super(Type.FETCH); + } +} diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java index f61d4547d8b78..ebef3ecaca030 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java @@ -29,7 +29,6 @@ import org.apache.kafka.clients.consumer.internals.ConsumerMetrics; import org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient; import org.apache.kafka.clients.consumer.internals.ConsumerProtocol; -import org.apache.kafka.clients.consumer.internals.Deserializers; import org.apache.kafka.clients.consumer.internals.FetchConfig; import org.apache.kafka.clients.consumer.internals.FetchMetricsManager; import org.apache.kafka.clients.consumer.internals.Fetcher; @@ -2674,7 +2673,7 @@ private KafkaConsumer newConsumer(Time time, } IsolationLevel isolationLevel = IsolationLevel.READ_UNCOMMITTED; FetchMetricsManager metricsManager = new FetchMetricsManager(metrics, metricsRegistry.fetcherMetrics); - FetchConfig fetchConfig = new FetchConfig<>( + FetchConfig fetchConfig = new FetchConfig( minBytes, maxBytes, maxWaitMs, @@ -2682,7 +2681,6 @@ private KafkaConsumer newConsumer(Time time, maxPollRecords, checkCrcs, CommonClientConfigs.DEFAULT_CLIENT_RACK, - new Deserializers<>(keyDeserializer, deserializer), isolationLevel); Fetcher fetcher = new Fetcher<>( loggerFactory, diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/CompletedFetchTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/CompletedFetchTest.java index de97ee40de2df..2c39f4411febe 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/CompletedFetchTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/CompletedFetchTest.java @@ -34,7 +34,6 @@ import org.apache.kafka.common.record.Records; import org.apache.kafka.common.record.SimpleRecord; import org.apache.kafka.common.record.TimestampType; -import org.apache.kafka.common.serialization.Deserializer; import org.apache.kafka.common.serialization.StringDeserializer; import org.apache.kafka.common.serialization.UUIDDeserializer; import org.apache.kafka.common.serialization.UUIDSerializer; @@ -67,23 +66,22 @@ public void testSimple() { FetchResponseData.PartitionData partitionData = new FetchResponseData.PartitionData() .setRecords(newRecords(startingOffset, numRecords, fetchOffset)); - FetchConfig fetchConfig = newFetchConfig(new StringDeserializer(), - new StringDeserializer(), - IsolationLevel.READ_UNCOMMITTED, - true); + Deserializers deserializers = newStringDeserializers(); + FetchConfig fetchConfig = newFetchConfig(IsolationLevel.READ_UNCOMMITTED, true); + CompletedFetch completedFetch = newCompletedFetch(fetchOffset, partitionData); - List> records = completedFetch.fetchRecords(fetchConfig, 10); + List> records = completedFetch.fetchRecords(fetchConfig, deserializers, 10); assertEquals(10, records.size()); ConsumerRecord record = records.get(0); assertEquals(10, record.offset()); - records = completedFetch.fetchRecords(fetchConfig, 10); + records = completedFetch.fetchRecords(fetchConfig, deserializers, 10); assertEquals(1, records.size()); record = records.get(0); assertEquals(20, record.offset()); - records = completedFetch.fetchRecords(fetchConfig, 10); + records = completedFetch.fetchRecords(fetchConfig, deserializers, 10); assertEquals(0, records.size()); } @@ -96,21 +94,15 @@ public void testAbortedTransactionRecordsRemoved() { .setRecords(rawRecords) .setAbortedTransactions(newAbortedTransactions()); - try (final StringDeserializer deserializer = new StringDeserializer()) { - FetchConfig fetchConfig = newFetchConfig(deserializer, - deserializer, - IsolationLevel.READ_COMMITTED, - true); + try (final Deserializers deserializers = newStringDeserializers()) { + FetchConfig fetchConfig = newFetchConfig(IsolationLevel.READ_COMMITTED, true); CompletedFetch completedFetch = newCompletedFetch(0, partitionData); - List> records = completedFetch.fetchRecords(fetchConfig, 10); + List> records = completedFetch.fetchRecords(fetchConfig, deserializers, 10); assertEquals(0, records.size()); - fetchConfig = newFetchConfig(deserializer, - deserializer, - IsolationLevel.READ_UNCOMMITTED, - true); + fetchConfig = newFetchConfig(IsolationLevel.READ_UNCOMMITTED, true); completedFetch = newCompletedFetch(0, partitionData); - records = completedFetch.fetchRecords(fetchConfig, 10); + records = completedFetch.fetchRecords(fetchConfig, deserializers, 10); assertEquals(numRecords, records.size()); } } @@ -122,12 +114,9 @@ public void testCommittedTransactionRecordsIncluded() { FetchResponseData.PartitionData partitionData = new FetchResponseData.PartitionData() .setRecords(rawRecords); CompletedFetch completedFetch = newCompletedFetch(0, partitionData); - try (final StringDeserializer deserializer = new StringDeserializer()) { - FetchConfig fetchConfig = newFetchConfig(deserializer, - deserializer, - IsolationLevel.READ_COMMITTED, - true); - List> records = completedFetch.fetchRecords(fetchConfig, 10); + try (final Deserializers deserializers = newStringDeserializers()) { + FetchConfig fetchConfig = newFetchConfig(IsolationLevel.READ_COMMITTED, true); + List> records = completedFetch.fetchRecords(fetchConfig, deserializers, 10); assertEquals(10, records.size()); } } @@ -140,14 +129,13 @@ public void testNegativeFetchCount() { FetchResponseData.PartitionData partitionData = new FetchResponseData.PartitionData() .setRecords(newRecords(startingOffset, numRecords, fetchOffset)); - CompletedFetch completedFetch = newCompletedFetch(fetchOffset, partitionData); - FetchConfig fetchConfig = newFetchConfig(new StringDeserializer(), - new StringDeserializer(), - IsolationLevel.READ_UNCOMMITTED, - true); + try (final Deserializers deserializers = newStringDeserializers()) { + CompletedFetch completedFetch = newCompletedFetch(fetchOffset, partitionData); + FetchConfig fetchConfig = newFetchConfig(IsolationLevel.READ_UNCOMMITTED, true); - List> records = completedFetch.fetchRecords(fetchConfig, -10); - assertEquals(0, records.size()); + List> records = completedFetch.fetchRecords(fetchConfig, deserializers, -10); + assertEquals(0, records.size()); + } } @Test @@ -159,13 +147,9 @@ public void testNoRecordsInFetch() { .setLogStartOffset(0); CompletedFetch completedFetch = newCompletedFetch(1, partitionData); - try (final StringDeserializer deserializer = new StringDeserializer()) { - FetchConfig fetchConfig = newFetchConfig(deserializer, - deserializer, - IsolationLevel.READ_UNCOMMITTED, - true); - - List> records = completedFetch.fetchRecords(fetchConfig, 10); + try (final Deserializers deserializers = newStringDeserializers()) { + FetchConfig fetchConfig = newFetchConfig(IsolationLevel.READ_UNCOMMITTED, true); + List> records = completedFetch.fetchRecords(fetchConfig, deserializers, 10); assertEquals(0, records.size()); } } @@ -174,8 +158,7 @@ public void testNoRecordsInFetch() { public void testCorruptedMessage() { // Create one good record and then one "corrupted" record. try (final MemoryRecordsBuilder builder = MemoryRecords.builder(ByteBuffer.allocate(1024), CompressionType.NONE, TimestampType.CREATE_TIME, 0); - final UUIDSerializer serializer = new UUIDSerializer(); - final UUIDDeserializer deserializer = new UUIDDeserializer()) { + final UUIDSerializer serializer = new UUIDSerializer()) { builder.append(new SimpleRecord(serializer.serialize(TOPIC_NAME, UUID.randomUUID()))); builder.append(0L, "key".getBytes(), "value".getBytes()); Records records = builder.build(); @@ -187,16 +170,15 @@ public void testCorruptedMessage() { .setLogStartOffset(0) .setRecords(records); - FetchConfig fetchConfig = newFetchConfig(deserializer, - deserializer, - IsolationLevel.READ_COMMITTED, - false); - CompletedFetch completedFetch = newCompletedFetch(0, partitionData); + try (final Deserializers deserializers = newUuidDeserializers()) { + FetchConfig fetchConfig = newFetchConfig(IsolationLevel.READ_COMMITTED, false); + CompletedFetch completedFetch = newCompletedFetch(0, partitionData); - completedFetch.fetchRecords(fetchConfig, 10); + completedFetch.fetchRecords(fetchConfig, deserializers, 10); - assertThrows(RecordDeserializationException.class, - () -> completedFetch.fetchRecords(fetchConfig, 10)); + assertThrows(RecordDeserializationException.class, + () -> completedFetch.fetchRecords(fetchConfig, deserializers, 10)); + } } } @@ -219,11 +201,16 @@ private CompletedFetch newCompletedFetch(long fetchOffset, ApiKeys.FETCH.latestVersion()); } - private static FetchConfig newFetchConfig(Deserializer keyDeserializer, - Deserializer valueDeserializer, - IsolationLevel isolationLevel, - boolean checkCrcs) { - return new FetchConfig<>( + private static Deserializers newUuidDeserializers() { + return new Deserializers<>(new UUIDDeserializer(), new UUIDDeserializer()); + } + + private static Deserializers newStringDeserializers() { + return new Deserializers<>(new StringDeserializer(), new StringDeserializer()); + } + + private static FetchConfig newFetchConfig(IsolationLevel isolationLevel, boolean checkCrcs) { + return new FetchConfig( ConsumerConfig.DEFAULT_FETCH_MIN_BYTES, ConsumerConfig.DEFAULT_FETCH_MAX_BYTES, ConsumerConfig.DEFAULT_FETCH_MAX_WAIT_MS, @@ -231,7 +218,6 @@ private static FetchConfig newFetchConfig(Deserializer keyDeseri ConsumerConfig.DEFAULT_MAX_POLL_RECORDS, checkCrcs, ConsumerConfig.DEFAULT_CLIENT_RACK, - new Deserializers<>(keyDeserializer, valueDeserializer), isolationLevel ); } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java new file mode 100644 index 0000000000000..202e442fbcf91 --- /dev/null +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java @@ -0,0 +1,265 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.clients.consumer.internals; + +import org.apache.kafka.clients.ApiVersions; +import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.clients.GroupRebalanceConfig; +import org.apache.kafka.clients.MockClient; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor; +import org.apache.kafka.clients.consumer.internals.events.ApplicationEvent; +import org.apache.kafka.clients.consumer.internals.events.ApplicationEventProcessor; +import org.apache.kafka.clients.consumer.internals.events.BackgroundEvent; +import org.apache.kafka.clients.consumer.internals.events.BackgroundEventProcessor; +import org.apache.kafka.clients.consumer.internals.events.EventHandler; +import org.apache.kafka.common.IsolationLevel; +import org.apache.kafka.common.internals.ClusterResourceListeners; +import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.requests.MetadataResponse; +import org.apache.kafka.common.requests.RequestTestUtils; +import org.apache.kafka.common.serialization.StringDeserializer; +import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.common.utils.MockTime; +import org.apache.kafka.common.utils.Time; + +import java.io.Closeable; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Optional; +import java.util.Properties; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; + +import static org.apache.kafka.clients.consumer.ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG; +import static org.apache.kafka.clients.consumer.ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createFetchConfig; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createFetchMetricsManager; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createMetrics; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createSubscriptionState; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.configuredIsolationLevel; +import static org.mockito.Mockito.spy; + +public class ConsumerTestBuilder implements Closeable { + + static final long RETRY_BACKOFF_MS = 80; + static final long RETRY_BACKOFF_MAX_MS = 1000; + static final int REQUEST_TIMEOUT_MS = 500; + + final LogContext logContext = new LogContext(); + final Time time = new MockTime(0); + final BlockingQueue applicationEventQueue; + final LinkedBlockingQueue backgroundEventQueue; + final ConsumerConfig config; + final long retryBackoffMs; + final SubscriptionState subscriptions; + final ConsumerMetadata metadata; + final FetchConfig fetchConfig; + final Metrics metrics; + final FetchMetricsManager metricsManager; + final NetworkClientDelegate networkClientDelegate; + final OffsetsRequestManager offsetsRequestManager; + final CoordinatorRequestManager coordinatorRequestManager; + final CommitRequestManager commitRequestManager; + final TopicMetadataRequestManager topicMetadataRequestManager; + final FetchRequestManager fetchRequestManager; + final RequestManagers requestManagers; + final ApplicationEventProcessor applicationEventProcessor; + final BackgroundEventProcessor backgroundEventProcessor; + final MockClient client; + + public ConsumerTestBuilder() { + this.applicationEventQueue = new LinkedBlockingQueue<>(); + this.backgroundEventQueue = new LinkedBlockingQueue<>(); + ErrorEventHandler errorEventHandler = new ErrorEventHandler(backgroundEventQueue); + GroupRebalanceConfig groupRebalanceConfig = new GroupRebalanceConfig( + 100, + 100, + 100, + "group_id", + Optional.empty(), + RETRY_BACKOFF_MS, + RETRY_BACKOFF_MAX_MS, + true); + GroupState groupState = new GroupState(groupRebalanceConfig); + ApiVersions apiVersions = new ApiVersions(); + + Properties properties = new Properties(); + properties.put(KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); + properties.put(VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); + properties.put(CommonClientConfigs.RETRY_BACKOFF_MS_CONFIG, RETRY_BACKOFF_MS); + properties.put(CommonClientConfigs.REQUEST_TIMEOUT_MS_CONFIG, REQUEST_TIMEOUT_MS); + + this.config = new ConsumerConfig(properties); + IsolationLevel isolationLevel = configuredIsolationLevel(config); + this.retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG); + final long requestTimeoutMs = config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG); + this.metrics = createMetrics(config, time); + + this.subscriptions = spy(createSubscriptionState(config, logContext)); + this.metadata = spy(new ConsumerMetadata(config, subscriptions, logContext, new ClusterResourceListeners())); + this.fetchConfig = createFetchConfig(config); + this.metricsManager = createFetchMetricsManager(metrics); + + this.client = new MockClient(time, metadata); + MetadataResponse metadataResponse = RequestTestUtils.metadataUpdateWith(1, new HashMap() { + { + String topic1 = "test1"; + put(topic1, 1); + String topic2 = "test2"; + put(topic2, 1); + } + }); + this.client.updateMetadata(metadataResponse); + + this.networkClientDelegate = spy(new NetworkClientDelegate(time, + config, + logContext, + client)); + this.offsetsRequestManager = spy(new OffsetsRequestManager(subscriptions, + metadata, + isolationLevel, + time, + retryBackoffMs, + requestTimeoutMs, + apiVersions, + networkClientDelegate, + logContext)); + this.coordinatorRequestManager = spy(new CoordinatorRequestManager(time, + logContext, + RETRY_BACKOFF_MS, + RETRY_BACKOFF_MAX_MS, + errorEventHandler, + "group_id")); + this.commitRequestManager = spy(new CommitRequestManager(time, + logContext, + subscriptions, + config, + coordinatorRequestManager, + groupState)); + this.fetchRequestManager = spy(new FetchRequestManager(logContext, + time, + errorEventHandler, + metadata, + subscriptions, + fetchConfig, + metricsManager, + networkClientDelegate)); + this.topicMetadataRequestManager = spy(new TopicMetadataRequestManager(logContext, + config)); + this.requestManagers = new RequestManagers(logContext, + offsetsRequestManager, + topicMetadataRequestManager, + fetchRequestManager, + Optional.of(coordinatorRequestManager), + Optional.of(commitRequestManager)); + this.applicationEventProcessor = spy(new ApplicationEventProcessor( + backgroundEventQueue, + requestManagers, + metadata, + logContext)); + this.backgroundEventProcessor = spy(new BackgroundEventProcessor(logContext, backgroundEventQueue)); + } + + @Override + public void close() { + requestManagers.close(); + } + + public static class DefaultBackgroundThreadTestBuilder extends ConsumerTestBuilder { + + final DefaultBackgroundThread backgroundThread; + + public DefaultBackgroundThreadTestBuilder() { + this.backgroundThread = new DefaultBackgroundThread( + time, + logContext, + applicationEventQueue, + () -> applicationEventProcessor, + () -> networkClientDelegate, + () -> requestManagers); + } + + @Override + public void close() { + backgroundThread.close(); + } + } + + public static class DefaultEventHandlerTestBuilder extends ConsumerTestBuilder { + + final EventHandler eventHandler; + + public DefaultEventHandlerTestBuilder() { + this.eventHandler = spy(new DefaultEventHandler( + time, + logContext, + applicationEventQueue, + () -> applicationEventProcessor, + () -> networkClientDelegate, + () -> requestManagers)); + } + + @Override + public void close() { + eventHandler.close(); + } + } + + public static class PrototypeAsyncConsumerTestBuilder extends DefaultEventHandlerTestBuilder { + + final PrototypeAsyncConsumer consumer; + + public PrototypeAsyncConsumerTestBuilder(Optional groupIdOpt) { + String clientId = config.getString(CommonClientConfigs.CLIENT_ID_CONFIG); + List assignors = ConsumerPartitionAssignor.getAssignorInstances( + config.getList(ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG), + config.originals(Collections.singletonMap(ConsumerConfig.CLIENT_ID_CONFIG, clientId)) + ); + FetchCollector fetchCollector = new FetchCollector<>(logContext, + metadata, + subscriptions, + fetchConfig, + metricsManager, + time); + this.consumer = spy(new PrototypeAsyncConsumer<>( + logContext, + clientId, + new Deserializers<>(new StringDeserializer(), new StringDeserializer()), + new FetchBuffer(logContext), + fetchCollector, + new ConsumerInterceptors<>(Collections.emptyList()), + time, + eventHandler, + backgroundEventQueue, + metrics, + subscriptions, + metadata, + retryBackoffMs, + REQUEST_TIMEOUT_MS, + 60000, + assignors, + groupIdOpt.orElse(null))); + } + + @Override + public void close() { + consumer.close(); + } + } +} diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThreadTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThreadTest.java index 137bd106d6ece..83155b192adb2 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThreadTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThreadTest.java @@ -16,15 +16,14 @@ */ package org.apache.kafka.clients.consumer.internals; -import org.apache.kafka.clients.GroupRebalanceConfig; -import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.MockClient; import org.apache.kafka.clients.consumer.LogTruncationException; import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.clients.consumer.internals.events.ApplicationEvent; import org.apache.kafka.clients.consumer.internals.events.ApplicationEventProcessor; import org.apache.kafka.clients.consumer.internals.events.AssignmentChangeApplicationEvent; -import org.apache.kafka.clients.consumer.internals.events.BackgroundEvent; import org.apache.kafka.clients.consumer.internals.events.CommitApplicationEvent; +import org.apache.kafka.clients.consumer.internals.events.CompletableApplicationEvent; import org.apache.kafka.clients.consumer.internals.events.ListOffsetsApplicationEvent; import org.apache.kafka.clients.consumer.internals.events.NewTopicsMetadataUpdateRequestEvent; import org.apache.kafka.clients.consumer.internals.events.NoopApplicationEvent; @@ -34,12 +33,14 @@ import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.errors.TopicAuthorizationException; import org.apache.kafka.common.message.FindCoordinatorRequestData; +import org.apache.kafka.common.message.OffsetCommitRequestData; import org.apache.kafka.common.requests.FindCoordinatorRequest; -import org.apache.kafka.common.serialization.StringDeserializer; -import org.apache.kafka.common.utils.LogContext; -import org.apache.kafka.common.utils.MockTime; +import org.apache.kafka.common.requests.MetadataResponse; +import org.apache.kafka.common.requests.OffsetCommitRequest; +import org.apache.kafka.common.requests.RequestTestUtils; import org.apache.kafka.common.utils.Time; import org.apache.kafka.test.TestUtils; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mockito; @@ -49,68 +50,61 @@ import java.util.HashMap; import java.util.Map; import java.util.Optional; -import java.util.Properties; import java.util.concurrent.BlockingQueue; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.LinkedBlockingQueue; -import static org.apache.kafka.clients.consumer.ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG; -import static org.apache.kafka.clients.consumer.ConsumerConfig.RETRY_BACKOFF_MS_CONFIG; -import static org.apache.kafka.clients.consumer.ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @SuppressWarnings("ClassDataAbstractionCoupling") public class DefaultBackgroundThreadTest { - private static final long RETRY_BACKOFF_MS = 100; - private final Properties properties = new Properties(); - private MockTime time; + + private ConsumerTestBuilder.DefaultBackgroundThreadTestBuilder testBuilder; + private Time time; private ConsumerMetadata metadata; private NetworkClientDelegate networkClient; - private BlockingQueue backgroundEventsQueue; private BlockingQueue applicationEventsQueue; private ApplicationEventProcessor applicationEventProcessor; private CoordinatorRequestManager coordinatorManager; private OffsetsRequestManager offsetsRequestManager; - private ErrorEventHandler errorEventHandler; - private final int requestTimeoutMs = 500; - private GroupState groupState; private CommitRequestManager commitManager; private TopicMetadataRequestManager topicMetadataRequestManager; + private DefaultBackgroundThread backgroundThread; + private MockClient client; @BeforeEach - @SuppressWarnings("unchecked") public void setup() { - this.time = new MockTime(0); - this.metadata = mock(ConsumerMetadata.class); - this.networkClient = mock(NetworkClientDelegate.class); - this.applicationEventsQueue = (BlockingQueue) mock(BlockingQueue.class); - this.backgroundEventsQueue = (BlockingQueue) mock(BlockingQueue.class); - this.applicationEventProcessor = mock(ApplicationEventProcessor.class); - this.coordinatorManager = mock(CoordinatorRequestManager.class); - this.offsetsRequestManager = mock(OffsetsRequestManager.class); - this.errorEventHandler = mock(ErrorEventHandler.class); - GroupRebalanceConfig rebalanceConfig = new GroupRebalanceConfig( - 100, - 100, - 100, - "group_id", - Optional.empty(), - 100, - 1000, - true); - this.groupState = new GroupState(rebalanceConfig); this.commitManager = mock(CommitRequestManager.class); this.topicMetadataRequestManager = mock(TopicMetadataRequestManager.class); + this.testBuilder = new ConsumerTestBuilder.DefaultBackgroundThreadTestBuilder(); + this.time = testBuilder.time; + this.metadata = testBuilder.metadata; + this.networkClient = testBuilder.networkClientDelegate; + this.client = testBuilder.client; + this.applicationEventsQueue = testBuilder.applicationEventQueue; + this.applicationEventProcessor = testBuilder.applicationEventProcessor; + this.coordinatorManager = testBuilder.coordinatorRequestManager; + this.commitManager = testBuilder.commitRequestManager; + this.offsetsRequestManager = testBuilder.offsetsRequestManager; + this.backgroundThread = testBuilder.backgroundThread; + this.backgroundThread.initializeResources(); + } + + @AfterEach + public void tearDown() { + if (testBuilder != null) + testBuilder.close(); } @Test @@ -119,22 +113,30 @@ public void testStartupAndTearDown() throws InterruptedException { when(commitManager.poll(anyLong())).thenReturn(mockPollCommitResult()); when(offsetsRequestManager.poll(anyLong())).thenReturn(emptyPollOffsetsRequestResult()); when(topicMetadataRequestManager.poll(anyLong())).thenReturn(emptyPollOffsetsRequestResult()); - DefaultBackgroundThread backgroundThread = mockBackgroundThread(); backgroundThread.start(); TestUtils.waitForCondition(backgroundThread::isRunning, "Failed awaiting for the background thread to be running"); backgroundThread.close(); assertFalse(backgroundThread.isRunning()); + backgroundThread.start(); + + // There's a nonzero amount of time between starting the thread and having it + // begin to execute our code. Wait for a bit before checking... + int maxWaitMs = 1000; + TestUtils.waitForCondition(backgroundThread::isRunning, + maxWaitMs, + "Thread did not start within " + maxWaitMs + " ms"); + backgroundThread.close(); + TestUtils.waitForCondition(() -> !backgroundThread.isRunning(), + maxWaitMs, + "Thread did not stop within " + maxWaitMs + " ms"); } @Test public void testApplicationEvent() { - this.applicationEventsQueue = new LinkedBlockingQueue<>(); - this.backgroundEventsQueue = new LinkedBlockingQueue<>(); when(this.coordinatorManager.poll(anyLong())).thenReturn(mockPollCoordinatorResult()); when(this.commitManager.poll(anyLong())).thenReturn(mockPollCommitResult()); when(this.offsetsRequestManager.poll(anyLong())).thenReturn(emptyPollOffsetsRequestResult()); when(this.topicMetadataRequestManager.poll(anyLong())).thenReturn(emptyPollOffsetsRequestResult()); - DefaultBackgroundThread backgroundThread = mockBackgroundThread(); ApplicationEvent e = new NoopApplicationEvent("noop event"); this.applicationEventsQueue.add(e); backgroundThread.runOnce(); @@ -144,17 +146,12 @@ public void testApplicationEvent() { @Test public void testMetadataUpdateEvent() { - this.applicationEventsQueue = new LinkedBlockingQueue<>(); - this.backgroundEventsQueue = new LinkedBlockingQueue<>(); - this.applicationEventProcessor = new ApplicationEventProcessor( - this.backgroundEventsQueue, - mockRequestManagers(), - metadata); when(this.coordinatorManager.poll(anyLong())).thenReturn(mockPollCoordinatorResult()); when(this.commitManager.poll(anyLong())).thenReturn(mockPollCommitResult()); when(this.offsetsRequestManager.poll(anyLong())).thenReturn(emptyPollOffsetsRequestResult()); when(this.topicMetadataRequestManager.poll(anyLong())).thenReturn(emptyPollOffsetsRequestResult()); - DefaultBackgroundThread backgroundThread = mockBackgroundThread(); + when(coordinatorManager.poll(anyLong())).thenReturn(mockPollCoordinatorResult()); + when(commitManager.poll(anyLong())).thenReturn(mockPollCommitResult()); ApplicationEvent e = new NewTopicsMetadataUpdateRequestEvent(); this.applicationEventsQueue.add(e); backgroundThread.runOnce(); @@ -164,13 +161,12 @@ public void testMetadataUpdateEvent() { @Test public void testCommitEvent() { - this.applicationEventsQueue = new LinkedBlockingQueue<>(); - this.backgroundEventsQueue = new LinkedBlockingQueue<>(); when(this.coordinatorManager.poll(anyLong())).thenReturn(mockPollCoordinatorResult()); when(this.commitManager.poll(anyLong())).thenReturn(mockPollCommitResult()); when(this.offsetsRequestManager.poll(anyLong())).thenReturn(emptyPollOffsetsRequestResult()); when(this.topicMetadataRequestManager.poll(anyLong())).thenReturn(emptyPollOffsetsRequestResult()); - DefaultBackgroundThread backgroundThread = mockBackgroundThread(); + when(coordinatorManager.poll(anyLong())).thenReturn(mockPollCoordinatorResult()); + when(commitManager.poll(anyLong())).thenReturn(mockPollCommitResult()); ApplicationEvent e = new CommitApplicationEvent(new HashMap<>()); this.applicationEventsQueue.add(e); backgroundThread.runOnce(); @@ -178,16 +174,12 @@ public void testCommitEvent() { backgroundThread.close(); } - @Test public void testListOffsetsEventIsProcessed() { when(coordinatorManager.poll(anyLong())).thenReturn(mockPollCoordinatorResult()); when(commitManager.poll(anyLong())).thenReturn(mockPollCommitResult()); when(offsetsRequestManager.poll(anyLong())).thenReturn(emptyPollOffsetsRequestResult()); when(this.topicMetadataRequestManager.poll(anyLong())).thenReturn(emptyPollOffsetsRequestResult()); - this.applicationEventsQueue = new LinkedBlockingQueue<>(); - this.backgroundEventsQueue = new LinkedBlockingQueue<>(); - DefaultBackgroundThread backgroundThread = mockBackgroundThread(); Map timestamps = Collections.singletonMap(new TopicPartition("topic1", 1), 5L); ApplicationEvent e = new ListOffsetsApplicationEvent(timestamps, true); this.applicationEventsQueue.add(e); @@ -203,9 +195,6 @@ public void testResetPositionsEventIsProcessed() { when(commitManager.poll(anyLong())).thenReturn(mockPollCommitResult()); when(offsetsRequestManager.poll(anyLong())).thenReturn(emptyPollOffsetsRequestResult()); when(this.topicMetadataRequestManager.poll(anyLong())).thenReturn(emptyPollOffsetsRequestResult()); - this.applicationEventsQueue = new LinkedBlockingQueue<>(); - this.backgroundEventsQueue = new LinkedBlockingQueue<>(); - DefaultBackgroundThread backgroundThread = mockBackgroundThread(); ResetPositionsApplicationEvent e = new ResetPositionsApplicationEvent(); this.applicationEventsQueue.add(e); backgroundThread.runOnce(); @@ -215,24 +204,13 @@ public void testResetPositionsEventIsProcessed() { } @Test - public void testResetPositionsProcessFailure() { - when(coordinatorManager.poll(anyLong())).thenReturn(mockPollCoordinatorResult()); - when(commitManager.poll(anyLong())).thenReturn(mockPollCommitResult()); - when(offsetsRequestManager.poll(anyLong())).thenReturn(emptyPollOffsetsRequestResult()); - this.applicationEventsQueue = new LinkedBlockingQueue<>(); - this.backgroundEventsQueue = new LinkedBlockingQueue<>(); - applicationEventProcessor = spy(new ApplicationEventProcessor( - this.backgroundEventsQueue, - mockRequestManagers(), - metadata)); - DefaultBackgroundThread backgroundThread = mockBackgroundThread(); - + public void testResetPositionsProcessFailureInterruptsBackgroundThread() { TopicAuthorizationException authException = new TopicAuthorizationException("Topic authorization failed"); doThrow(authException).when(offsetsRequestManager).resetPositionsIfNeeded(); ResetPositionsApplicationEvent event = new ResetPositionsApplicationEvent(); this.applicationEventsQueue.add(event); - assertThrows(TopicAuthorizationException.class, backgroundThread::runOnce); + assertThrows(TopicAuthorizationException.class, () -> backgroundThread.runOnce()); verify(applicationEventProcessor).process(any(ResetPositionsApplicationEvent.class)); backgroundThread.close(); @@ -244,9 +222,6 @@ public void testValidatePositionsEventIsProcessed() { when(commitManager.poll(anyLong())).thenReturn(mockPollCommitResult()); when(offsetsRequestManager.poll(anyLong())).thenReturn(emptyPollOffsetsRequestResult()); when(topicMetadataRequestManager.poll(anyLong())).thenReturn(emptyPollOffsetsRequestResult()); - this.applicationEventsQueue = new LinkedBlockingQueue<>(); - this.backgroundEventsQueue = new LinkedBlockingQueue<>(); - DefaultBackgroundThread backgroundThread = mockBackgroundThread(); ValidatePositionsApplicationEvent e = new ValidatePositionsApplicationEvent(); this.applicationEventsQueue.add(e); backgroundThread.runOnce(); @@ -260,14 +235,6 @@ public void testValidatePositionsProcessFailure() { when(coordinatorManager.poll(anyLong())).thenReturn(mockPollCoordinatorResult()); when(commitManager.poll(anyLong())).thenReturn(mockPollCommitResult()); when(offsetsRequestManager.poll(anyLong())).thenReturn(emptyPollOffsetsRequestResult()); - this.applicationEventsQueue = new LinkedBlockingQueue<>(); - this.backgroundEventsQueue = new LinkedBlockingQueue<>(); - applicationEventProcessor = spy(new ApplicationEventProcessor( - this.backgroundEventsQueue, - mockRequestManagers(), - metadata)); - DefaultBackgroundThread backgroundThread = mockBackgroundThread(); - LogTruncationException logTruncationException = new LogTruncationException(Collections.emptyMap(), Collections.emptyMap()); doThrow(logTruncationException).when(offsetsRequestManager).validatePositionsIfNeeded(); @@ -281,14 +248,6 @@ public void testValidatePositionsProcessFailure() { @Test public void testAssignmentChangeEvent() { - this.applicationEventsQueue = new LinkedBlockingQueue<>(); - this.backgroundEventsQueue = new LinkedBlockingQueue<>(); - this.applicationEventProcessor = spy(new ApplicationEventProcessor( - this.backgroundEventsQueue, - mockRequestManagers(), - metadata)); - - DefaultBackgroundThread backgroundThread = mockBackgroundThread(); HashMap offset = mockTopicPartitionOffset(); final long currentTimeMs = time.milliseconds(); @@ -311,7 +270,6 @@ public void testAssignmentChangeEvent() { @Test void testFindCoordinator() { - DefaultBackgroundThread backgroundThread = mockBackgroundThread(); when(this.coordinatorManager.poll(anyLong())).thenReturn(mockPollCoordinatorResult()); when(this.commitManager.poll(anyLong())).thenReturn(mockPollCommitResult()); when(this.offsetsRequestManager.poll(anyLong())).thenReturn(emptyPollOffsetsRequestResult()); @@ -325,7 +283,6 @@ void testFindCoordinator() { @Test void testFetchTopicMetadata() { this.applicationEventsQueue = new LinkedBlockingQueue<>(); - DefaultBackgroundThread backgroundThread = mockBackgroundThread(); when(this.coordinatorManager.poll(anyLong())).thenReturn(mockPollCoordinatorResult()); when(this.commitManager.poll(anyLong())).thenReturn(mockPollCommitResult()); when(this.offsetsRequestManager.poll(anyLong())).thenReturn(emptyPollOffsetsRequestResult()); @@ -338,11 +295,10 @@ void testFetchTopicMetadata() { @Test void testPollResultTimer() { - DefaultBackgroundThread backgroundThread = mockBackgroundThread(); // purposely setting a non MAX time to ensure it is returning Long.MAX_VALUE upon success NetworkClientDelegate.PollResult success = new NetworkClientDelegate.PollResult( 10, - Collections.singletonList(findCoordinatorUnsentRequest(time, requestTimeoutMs))); + Collections.singletonList(findCoordinatorUnsentRequest())); assertEquals(10, backgroundThread.handlePollResult(success)); NetworkClientDelegate.PollResult failure = new NetworkClientDelegate.PollResult( @@ -351,6 +307,42 @@ void testPollResultTimer() { assertEquals(10, backgroundThread.handlePollResult(failure)); } + @Test + void testRequestManagersArePolledOnce() { + backgroundThread.runOnce(); + testBuilder.requestManagers.entries().forEach(requestManager -> + verify(requestManager.get(), times(1)).poll(anyLong())); + verify(networkClient, times(1)).poll(anyLong(), anyLong()); + backgroundThread.close(); + } + + @Test + void testEnsureMetadataUpdateOnPoll() { + MetadataResponse metadataResponse = RequestTestUtils.metadataUpdateWith(2, Collections.emptyMap()); + client.prepareMetadataUpdate(metadataResponse); + metadata.requestUpdate(false); + backgroundThread.runOnce(); + verify(this.metadata, times(1)).updateWithCurrentRequestVersion(eq(metadataResponse), eq(false), anyLong()); + backgroundThread.close(); + } + + @Test + @SuppressWarnings("unchecked") + void testEnsureEventsAreCompleted() { + CompletableApplicationEvent event = (CompletableApplicationEvent) mock(CompletableApplicationEvent.class); + ApplicationEvent e = mock(ApplicationEvent.class); + CompletableFuture future = new CompletableFuture<>(); + when(event.future()).thenReturn(future); + applicationEventsQueue.add(event); + applicationEventsQueue.add(e); + assertFalse(future.isDone()); + assertFalse(applicationEventsQueue.isEmpty()); + + backgroundThread.close(); + assertTrue(future.isCompletedExceptionally()); + assertTrue(applicationEventsQueue.isEmpty()); + } + private HashMap mockTopicPartitionOffset() { final TopicPartition t0 = new TopicPartition("t0", 2); final TopicPartition t1 = new TopicPartition("t0", 3); @@ -359,64 +351,42 @@ private HashMap mockTopicPartitionOffset() { topicPartitionOffsets.put(t1, new OffsetAndMetadata(20L)); return topicPartitionOffsets; } - - private RequestManagers mockRequestManagers() { - return new RequestManagers( - offsetsRequestManager, - topicMetadataRequestManager, - Optional.of(coordinatorManager), - Optional.of(commitManager)); - } - - private static NetworkClientDelegate.UnsentRequest findCoordinatorUnsentRequest( - final Time time, - final long timeout - ) { + private NetworkClientDelegate.UnsentRequest findCoordinatorUnsentRequest() { NetworkClientDelegate.UnsentRequest req = new NetworkClientDelegate.UnsentRequest( new FindCoordinatorRequest.Builder( new FindCoordinatorRequestData() .setKeyType(FindCoordinatorRequest.CoordinatorType.TRANSACTION.id()) .setKey("foobar")), Optional.empty()); - req.setTimer(time, timeout); + req.setTimer(time, ConsumerTestBuilder.REQUEST_TIMEOUT_MS); return req; } - private DefaultBackgroundThread mockBackgroundThread() { - properties.put(KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); - properties.put(VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); - properties.put(RETRY_BACKOFF_MS_CONFIG, RETRY_BACKOFF_MS); - - return new DefaultBackgroundThread( - this.time, - new ConsumerConfig(properties), - new LogContext(), - applicationEventsQueue, - backgroundEventsQueue, - this.errorEventHandler, - applicationEventProcessor, - this.metadata, - this.networkClient, - this.groupState, - this.coordinatorManager, - this.commitManager, - this.offsetsRequestManager, - this.topicMetadataRequestManager); - } - private NetworkClientDelegate.PollResult mockPollCoordinatorResult() { return new NetworkClientDelegate.PollResult( - RETRY_BACKOFF_MS, - Collections.singletonList(findCoordinatorUnsentRequest(time, requestTimeoutMs))); + ConsumerTestBuilder.RETRY_BACKOFF_MS, + Collections.singletonList(findCoordinatorUnsentRequest())); } private NetworkClientDelegate.PollResult mockPollCommitResult() { return new NetworkClientDelegate.PollResult( - RETRY_BACKOFF_MS, - Collections.singletonList(findCoordinatorUnsentRequest(time, requestTimeoutMs))); + ConsumerTestBuilder.RETRY_BACKOFF_MS, + Collections.singletonList(offsetCommitUnsentRequest())); } private NetworkClientDelegate.PollResult emptyPollOffsetsRequestResult() { return new NetworkClientDelegate.PollResult(Long.MAX_VALUE, Collections.emptyList()); } -} \ No newline at end of file + + private NetworkClientDelegate.UnsentRequest offsetCommitUnsentRequest() { + NetworkClientDelegate.UnsentRequest req = new NetworkClientDelegate.UnsentRequest( + new OffsetCommitRequest.Builder( + new OffsetCommitRequestData() + .setGroupId("groupId") + .setMemberId("m1") + .setGroupInstanceId("i1") + .setTopics(new ArrayList<>())), Optional.empty()); + req.setTimer(time, ConsumerTestBuilder.REQUEST_TIMEOUT_MS); + return req; + } +} diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/DefaultEventHandlerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/DefaultEventHandlerTest.java index 7e4de791da66a..1bbd1d1bab083 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/DefaultEventHandlerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/DefaultEventHandlerTest.java @@ -16,66 +16,39 @@ */ package org.apache.kafka.clients.consumer.internals; -import org.apache.kafka.clients.GroupRebalanceConfig; import org.apache.kafka.clients.consumer.internals.events.ApplicationEvent; -import org.apache.kafka.clients.consumer.internals.events.BackgroundEvent; +import org.apache.kafka.clients.consumer.internals.events.EventHandler; import org.apache.kafka.clients.consumer.internals.events.NoopApplicationEvent; -import org.apache.kafka.common.serialization.StringDeserializer; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import java.util.Optional; -import java.util.Properties; import java.util.concurrent.BlockingQueue; -import java.util.concurrent.LinkedBlockingQueue; -import static org.apache.kafka.clients.consumer.ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG; -import static org.apache.kafka.clients.consumer.ConsumerConfig.RETRY_BACKOFF_MS_CONFIG; -import static org.apache.kafka.clients.consumer.ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; public class DefaultEventHandlerTest { - private int sessionTimeoutMs = 1000; - private int rebalanceTimeoutMs = 1000; - private int heartbeatIntervalMs = 1000; - private String groupId = "g-1"; - private Optional groupInstanceId = Optional.of("g-1"); - private long retryBackoffMs = 1000; - private final Properties properties = new Properties(); - private GroupRebalanceConfig rebalanceConfig; + + private ConsumerTestBuilder.DefaultEventHandlerTestBuilder testBuilder; + private EventHandler handler; + private BlockingQueue aq; @BeforeEach public void setup() { - properties.put(KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); - properties.put(VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); - properties.put(RETRY_BACKOFF_MS_CONFIG, "100"); + testBuilder = new ConsumerTestBuilder.DefaultEventHandlerTestBuilder(); + handler = testBuilder.eventHandler; + aq = testBuilder.applicationEventQueue; + } - this.rebalanceConfig = new GroupRebalanceConfig(sessionTimeoutMs, - rebalanceTimeoutMs, - heartbeatIntervalMs, - groupId, - groupInstanceId, - retryBackoffMs, - retryBackoffMs, - true); + @AfterEach + public void tearDown() { + if (testBuilder != null) + testBuilder.close(); } @Test public void testBasicHandlerOps() { - final DefaultBackgroundThread bt = mock(DefaultBackgroundThread.class); - final BlockingQueue aq = new LinkedBlockingQueue<>(); - final BlockingQueue bq = new LinkedBlockingQueue<>(); - final DefaultEventHandler handler = new DefaultEventHandler(bt, aq, bq); - assertTrue(handler.isEmpty()); - assertFalse(handler.poll().isPresent()); handler.add(new NoopApplicationEvent("test")); assertEquals(1, aq.size()); - handler.close(); - verify(bt, times(1)).close(); } } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ErrorEventHandlerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ErrorEventHandlerTest.java new file mode 100644 index 0000000000000..61e2e95a15aef --- /dev/null +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ErrorEventHandlerTest.java @@ -0,0 +1,165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.clients.consumer.internals; + +import org.apache.kafka.clients.consumer.internals.events.BackgroundEvent; +import org.apache.kafka.clients.consumer.internals.events.BackgroundEventProcessor; +import org.apache.kafka.clients.consumer.internals.events.ErrorBackgroundEvent; +import org.apache.kafka.clients.consumer.internals.events.NoopBackgroundEvent; +import org.apache.kafka.common.KafkaException; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.concurrent.BlockingQueue; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +public class ErrorEventHandlerTest { + + private ConsumerTestBuilder.DefaultEventHandlerTestBuilder testBuilder; + private BlockingQueue backgroundEventQueue; + private ErrorEventHandler errorEventHandler; + private BackgroundEventProcessor backgroundEventProcessor; + + @BeforeEach + public void setup() { + testBuilder = new ConsumerTestBuilder.DefaultEventHandlerTestBuilder(); + backgroundEventQueue = testBuilder.backgroundEventQueue; + errorEventHandler = new ErrorEventHandler(backgroundEventQueue); + backgroundEventProcessor = new BackgroundEventProcessor(testBuilder.logContext, backgroundEventQueue); + } + + @AfterEach + public void tearDown() { + if (testBuilder != null) + testBuilder.close(); + } + + @Test + public void testNoEvents() { + assertTrue(backgroundEventQueue.isEmpty()); + backgroundEventProcessor.process(); + assertTrue(backgroundEventQueue.isEmpty()); + } + + @Test + public void testSingleEvent() { + BackgroundEvent event = new NoopBackgroundEvent("A"); + backgroundEventQueue.add(event); + assertPeeked(event); + backgroundEventProcessor.process(); + assertTrue(backgroundEventQueue.isEmpty()); + } + + @Test + public void testSingleErrorEvent() { + KafkaException error = new KafkaException("error"); + BackgroundEvent event = new ErrorBackgroundEvent(error); + errorEventHandler.handle(error); + assertPeeked(event); + assertThrows(error); + } + + @Test + public void testInvalidEvent() { + String message = "I'm a naughty error!"; + BackgroundEvent event = new BackgroundEvent(BackgroundEvent.Type.NOOP) { + @Override + public Type type() { + return null; + } + + @Override + public String toString() { + return message; + } + }; + + backgroundEventQueue.add(event); + assertPeeked(event); + + Exception error = new NullPointerException(String.format("Attempt to process BackgroundEvent with null type: %s", message)); + assertThrows(error); + } + + @Test + public void testMultipleEvents() { + BackgroundEvent event1 = new NoopBackgroundEvent("A"); + backgroundEventQueue.add(event1); + backgroundEventQueue.add(new NoopBackgroundEvent("B")); + backgroundEventQueue.add(new NoopBackgroundEvent("C")); + + assertPeeked(event1); + backgroundEventProcessor.process(); + assertTrue(backgroundEventQueue.isEmpty()); + } + + @Test + public void testMultipleErrorEvents() { + Throwable error1 = new Throwable("error1"); + KafkaException error2 = new KafkaException("error2"); + KafkaException error3 = new KafkaException("error3"); + + errorEventHandler.handle(error1); + errorEventHandler.handle(error2); + errorEventHandler.handle(error3); + + assertThrows(new RuntimeException(error1)); + } + + @Test + public void testMixedEventsWithErrorEvents() { + Throwable error1 = new Throwable("error1"); + KafkaException error2 = new KafkaException("error2"); + KafkaException error3 = new KafkaException("error3"); + + backgroundEventQueue.add(new NoopBackgroundEvent("A")); + errorEventHandler.handle(error1); + backgroundEventQueue.add(new NoopBackgroundEvent("B")); + errorEventHandler.handle(error2); + backgroundEventQueue.add(new NoopBackgroundEvent("C")); + errorEventHandler.handle(error3); + backgroundEventQueue.add(new NoopBackgroundEvent("D")); + + assertThrows(new RuntimeException(error1)); + } + + private void assertPeeked(BackgroundEvent event) { + BackgroundEvent peekEvent = backgroundEventQueue.peek(); + assertNotNull(peekEvent); + assertEquals(event, peekEvent); + } + + private void assertThrows(Throwable error) { + assertFalse(backgroundEventQueue.isEmpty()); + + try { + backgroundEventProcessor.process(); + fail("Should have thrown error: " + error); + } catch (Throwable t) { + assertEquals(error.getClass(), t.getClass()); + assertEquals(error.getMessage(), t.getMessage()); + } + + assertTrue(backgroundEventQueue.isEmpty()); + } +} diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchCollectorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchCollectorTest.java index 1f157b84189da..66ed1d952bcd6 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchCollectorTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchCollectorTest.java @@ -79,10 +79,11 @@ public class FetchCollectorTest { private LogContext logContext; private SubscriptionState subscriptions; - private FetchConfig fetchConfig; + private FetchConfig fetchConfig; private FetchMetricsManager metricsManager; private ConsumerMetadata metadata; private FetchBuffer fetchBuffer; + private Deserializers deserializers; private FetchCollector fetchCollector; private CompletedFetchBuilder completedFetchBuilder; @@ -105,7 +106,7 @@ public void testFetchNormal() { assertFalse(completedFetch.isInitialized()); // Fetch the data and validate that we get all the records we want back. - Fetch fetch = fetchCollector.collectFetch(fetchBuffer); + Fetch fetch = fetchCollector.collectFetch(fetchBuffer, deserializers); assertFalse(fetch.isEmpty()); assertEquals(recordCount, fetch.numRecords()); @@ -129,7 +130,7 @@ public void testFetchNormal() { assertEquals(recordCount, position.offset); // Now attempt to collect more records from the fetch buffer. - fetch = fetchCollector.collectFetch(fetchBuffer); + fetch = fetchCollector.collectFetch(fetchBuffer, deserializers); // The Fetch object is non-null, but it's empty. assertEquals(0, fetch.numRecords()); @@ -153,7 +154,7 @@ public void testFetchWithReadReplica() { CompletedFetch completedFetch = completedFetchBuilder.build(); fetchBuffer.add(completedFetch); - Fetch fetch = fetchCollector.collectFetch(fetchBuffer); + Fetch fetch = fetchCollector.collectFetch(fetchBuffer, deserializers); // The Fetch and read replica settings should be empty. assertEquals(DEFAULT_RECORD_COUNT, fetch.numRecords()); @@ -176,7 +177,7 @@ public void testNoResultsIfInitializing() { // Add some valid CompletedFetch records to the FetchBuffer queue and collect them into the Fetch. CompletedFetch completedFetch = completedFetchBuilder.build(); fetchBuffer.add(completedFetch); - Fetch fetch = fetchCollector.collectFetch(fetchBuffer); + Fetch fetch = fetchCollector.collectFetch(fetchBuffer, deserializers); // Verify that no records are fetched for the partition as it did not have a valid position set. assertEquals(0, fetch.numRecords()); @@ -212,7 +213,7 @@ protected CompletedFetch initialize(final CompletedFetch completedFetch) { assertFalse(fetchBuffer.isEmpty()); // Now run our ill-fated collectFetch. - assertThrows(expectedException.getClass(), () -> fetchCollector.collectFetch(fetchBuffer)); + assertThrows(expectedException.getClass(), () -> fetchCollector.collectFetch(fetchBuffer, deserializers)); // If the number of records in the CompletedFetch was 0, the call to FetchCollector.collectFetch() will // remove it from the queue. If there are records in the CompletedFetch, FetchCollector.collectFetch will @@ -245,7 +246,7 @@ public void testFetchingPausedPartitionsYieldsNoRecords() { // Ensure that the partition for the next-in-line CompletedFetch is still 'paused'. assertTrue(subscriptions.isPaused(completedFetch.partition)); - Fetch fetch = fetchCollector.collectFetch(fetchBuffer); + Fetch fetch = fetchCollector.collectFetch(fetchBuffer, deserializers); // There should be no records in the Fetch as the partition being fetched is 'paused'. assertEquals(0, fetch.numRecords()); @@ -277,7 +278,7 @@ public void testFetchWithMetadataRefreshErrors(final Errors error) { assertEquals(Optional.of(preferredReadReplicaId), subscriptions.preferredReadReplica(topicAPartition0, time.milliseconds())); // Fetch the data and validate that we get all the records we want back. - Fetch fetch = fetchCollector.collectFetch(fetchBuffer); + Fetch fetch = fetchCollector.collectFetch(fetchBuffer, deserializers); assertTrue(fetch.isEmpty()); assertTrue(metadata.updateRequested()); assertEquals(Optional.empty(), subscriptions.preferredReadReplica(topicAPartition0, time.milliseconds())); @@ -292,7 +293,7 @@ public void testFetchWithOffsetOutOfRange() { fetchBuffer.add(completedFetch); // Fetch the data and validate that we get our first batch of records back. - Fetch fetch = fetchCollector.collectFetch(fetchBuffer); + Fetch fetch = fetchCollector.collectFetch(fetchBuffer, deserializers); assertFalse(fetch.isEmpty()); assertEquals(DEFAULT_RECORD_COUNT, fetch.numRecords()); @@ -302,7 +303,7 @@ public void testFetchWithOffsetOutOfRange() { .error(Errors.OFFSET_OUT_OF_RANGE) .build(); fetchBuffer.add(completedFetch); - fetch = fetchCollector.collectFetch(fetchBuffer); + fetch = fetchCollector.collectFetch(fetchBuffer, deserializers); assertTrue(fetch.isEmpty()); // Try to fetch more data and validate that we get an empty Fetch back. @@ -311,7 +312,7 @@ public void testFetchWithOffsetOutOfRange() { .error(Errors.OFFSET_OUT_OF_RANGE) .build(); fetchBuffer.add(completedFetch); - fetch = fetchCollector.collectFetch(fetchBuffer); + fetch = fetchCollector.collectFetch(fetchBuffer, deserializers); assertTrue(fetch.isEmpty()); } @@ -331,7 +332,7 @@ public void testFetchWithOffsetOutOfRangeWithPreferredReadReplica() { .error(Errors.OFFSET_OUT_OF_RANGE) .build(); fetchBuffer.add(completedFetch); - Fetch fetch = fetchCollector.collectFetch(fetchBuffer); + Fetch fetch = fetchCollector.collectFetch(fetchBuffer, deserializers); // The Fetch and read replica settings should be empty. assertTrue(fetch.isEmpty()); @@ -348,7 +349,7 @@ public void testFetchWithTopicAuthorizationFailed() { .error(Errors.TOPIC_AUTHORIZATION_FAILED) .build(); fetchBuffer.add(completedFetch); - assertThrows(TopicAuthorizationException.class, () -> fetchCollector.collectFetch(fetchBuffer)); + assertThrows(TopicAuthorizationException.class, () -> fetchCollector.collectFetch(fetchBuffer, deserializers)); } @Test @@ -361,7 +362,7 @@ public void testFetchWithUnknownLeaderEpoch() { .error(Errors.UNKNOWN_LEADER_EPOCH) .build(); fetchBuffer.add(completedFetch); - Fetch fetch = fetchCollector.collectFetch(fetchBuffer); + Fetch fetch = fetchCollector.collectFetch(fetchBuffer, deserializers); assertTrue(fetch.isEmpty()); } @@ -375,7 +376,7 @@ public void testFetchWithUnknownServerError() { .error(Errors.UNKNOWN_SERVER_ERROR) .build(); fetchBuffer.add(completedFetch); - Fetch fetch = fetchCollector.collectFetch(fetchBuffer); + Fetch fetch = fetchCollector.collectFetch(fetchBuffer, deserializers); assertTrue(fetch.isEmpty()); } @@ -389,7 +390,7 @@ public void testFetchWithCorruptMessage() { .error(Errors.CORRUPT_MESSAGE) .build(); fetchBuffer.add(completedFetch); - assertThrows(KafkaException.class, () -> fetchCollector.collectFetch(fetchBuffer)); + assertThrows(KafkaException.class, () -> fetchCollector.collectFetch(fetchBuffer, deserializers)); } @ParameterizedTest @@ -402,7 +403,7 @@ public void testFetchWithOtherErrors(final Errors error) { .error(error) .build(); fetchBuffer.add(completedFetch); - assertThrows(IllegalStateException.class, () -> fetchCollector.collectFetch(fetchBuffer)); + assertThrows(IllegalStateException.class, () -> fetchCollector.collectFetch(fetchBuffer, deserializers)); } /** @@ -427,10 +428,10 @@ private void buildDependencies(int maxPollRecords) { ConsumerConfig config = new ConsumerConfig(p); - Deserializers deserializers = new Deserializers<>(new StringDeserializer(), new StringDeserializer()); + deserializers = new Deserializers<>(new StringDeserializer(), new StringDeserializer()); subscriptions = createSubscriptionState(config, logContext); - fetchConfig = createFetchConfig(config, deserializers); + fetchConfig = createFetchConfig(config); Metrics metrics = createMetrics(config, time); metricsManager = createFetchMetricsManager(metrics); diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchConfigTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchConfigTest.java index 1f89822fdb253..c5d0755d2a107 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchConfigTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchConfigTest.java @@ -19,14 +19,11 @@ import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.common.IsolationLevel; import org.apache.kafka.common.serialization.Deserializer; -import org.apache.kafka.common.serialization.StringDeserializer; import org.apache.kafka.common.serialization.StringSerializer; import org.junit.jupiter.api.Test; import java.util.Properties; -import static org.junit.jupiter.api.Assertions.assertThrows; - public class FetchConfigTest { /** @@ -35,60 +32,27 @@ public class FetchConfigTest { */ @Test public void testBasicFromConsumerConfig() { - try (StringDeserializer keyDeserializer = new StringDeserializer(); StringDeserializer valueDeserializer = new StringDeserializer()) { - newFetchConfigFromConsumerConfig(keyDeserializer, valueDeserializer); - newFetchConfigFromValues(keyDeserializer, valueDeserializer); - } - } - - /** - * Verify an exception is thrown if the key {@link Deserializer deserializer} provided to the - * {@link FetchConfig} constructors is {@code null}. - */ - @Test - public void testPreventNullKeyDeserializer() { - try (StringDeserializer valueDeserializer = new StringDeserializer()) { - assertThrows(NullPointerException.class, () -> newFetchConfigFromConsumerConfig(null, valueDeserializer)); - assertThrows(NullPointerException.class, () -> newFetchConfigFromValues(null, valueDeserializer)); - } + newFetchConfigFromConsumerConfig(); + newFetchConfigFromValues(); } - /** - * Verify an exception is thrown if the value {@link Deserializer deserializer} provided to the - * {@link FetchConfig} constructors is {@code null}. - */ - @Test - @SuppressWarnings("resources") - public void testPreventNullValueDeserializer() { - try (StringDeserializer keyDeserializer = new StringDeserializer()) { - assertThrows(NullPointerException.class, () -> newFetchConfigFromConsumerConfig(keyDeserializer, null)); - assertThrows(NullPointerException.class, () -> newFetchConfigFromValues(keyDeserializer, null)); - } - } - - private void newFetchConfigFromConsumerConfig(Deserializer keyDeserializer, - Deserializer valueDeserializer) { + private void newFetchConfigFromConsumerConfig() { Properties p = new Properties(); p.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092"); p.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); p.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); ConsumerConfig config = new ConsumerConfig(p); - new FetchConfig<>( - config, - new Deserializers<>(keyDeserializer, valueDeserializer), - IsolationLevel.READ_UNCOMMITTED); + new FetchConfig(config, IsolationLevel.READ_UNCOMMITTED); } - private void newFetchConfigFromValues(Deserializer keyDeserializer, - Deserializer valueDeserializer) { - new FetchConfig<>(ConsumerConfig.DEFAULT_FETCH_MIN_BYTES, + private void newFetchConfigFromValues() { + new FetchConfig(ConsumerConfig.DEFAULT_FETCH_MIN_BYTES, ConsumerConfig.DEFAULT_FETCH_MAX_BYTES, ConsumerConfig.DEFAULT_FETCH_MAX_WAIT_MS, ConsumerConfig.DEFAULT_MAX_PARTITION_FETCH_BYTES, ConsumerConfig.DEFAULT_MAX_POLL_RECORDS, true, ConsumerConfig.DEFAULT_CLIENT_RACK, - new Deserializers<>(keyDeserializer, valueDeserializer), IsolationLevel.READ_UNCOMMITTED); } } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java new file mode 100644 index 0000000000000..2a74f9441f9dc --- /dev/null +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java @@ -0,0 +1,3566 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.clients.consumer.internals; + +import org.apache.kafka.clients.ApiVersions; +import org.apache.kafka.clients.ClientRequest; +import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.clients.KafkaClient; +import org.apache.kafka.clients.Metadata; +import org.apache.kafka.clients.MockClient; +import org.apache.kafka.clients.NetworkClient; +import org.apache.kafka.clients.NodeApiVersions; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRebalanceListener; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.OffsetOutOfRangeException; +import org.apache.kafka.clients.consumer.OffsetResetStrategy; +import org.apache.kafka.common.Cluster; +import org.apache.kafka.common.IsolationLevel; +import org.apache.kafka.common.KafkaException; +import org.apache.kafka.common.MetricName; +import org.apache.kafka.common.MetricNameTemplate; +import org.apache.kafka.common.Node; +import org.apache.kafka.common.TopicIdPartition; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.errors.DisconnectException; +import org.apache.kafka.common.errors.RecordTooLargeException; +import org.apache.kafka.common.errors.SerializationException; +import org.apache.kafka.common.errors.TopicAuthorizationException; +import org.apache.kafka.common.header.Header; +import org.apache.kafka.common.header.internals.RecordHeader; +import org.apache.kafka.common.internals.ClusterResourceListeners; +import org.apache.kafka.common.message.ApiMessageType; +import org.apache.kafka.common.message.FetchResponseData; +import org.apache.kafka.common.message.OffsetForLeaderEpochResponseData; +import org.apache.kafka.common.message.OffsetForLeaderEpochResponseData.EpochEndOffset; +import org.apache.kafka.common.message.OffsetForLeaderEpochResponseData.OffsetForLeaderTopicResult; +import org.apache.kafka.common.metrics.KafkaMetric; +import org.apache.kafka.common.metrics.MetricConfig; +import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.network.NetworkReceive; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.record.CompressionType; +import org.apache.kafka.common.record.ControlRecordType; +import org.apache.kafka.common.record.DefaultRecordBatch; +import org.apache.kafka.common.record.EndTransactionMarker; +import org.apache.kafka.common.record.LegacyRecord; +import org.apache.kafka.common.record.MemoryRecords; +import org.apache.kafka.common.record.MemoryRecordsBuilder; +import org.apache.kafka.common.record.Record; +import org.apache.kafka.common.record.RecordBatch; +import org.apache.kafka.common.record.Records; +import org.apache.kafka.common.record.SimpleRecord; +import org.apache.kafka.common.record.TimestampType; +import org.apache.kafka.common.requests.ApiVersionsResponse; +import org.apache.kafka.common.requests.FetchMetadata; +import org.apache.kafka.common.requests.FetchRequest; +import org.apache.kafka.common.requests.FetchRequest.PartitionData; +import org.apache.kafka.common.requests.FetchResponse; +import org.apache.kafka.common.requests.MetadataResponse; +import org.apache.kafka.common.requests.OffsetsForLeaderEpochResponse; +import org.apache.kafka.common.requests.RequestTestUtils; +import org.apache.kafka.common.serialization.ByteArrayDeserializer; +import org.apache.kafka.common.serialization.BytesDeserializer; +import org.apache.kafka.common.serialization.Deserializer; +import org.apache.kafka.common.serialization.StringDeserializer; +import org.apache.kafka.common.utils.BufferSupplier; +import org.apache.kafka.common.utils.ByteBufferOutputStream; +import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.common.utils.MockTime; +import org.apache.kafka.common.utils.Time; +import org.apache.kafka.common.utils.Timer; +import org.apache.kafka.common.utils.Utils; +import org.apache.kafka.test.DelayedReceive; +import org.apache.kafka.test.MockSelector; +import org.apache.kafka.test.TestUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.DataOutputStream; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; + +import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; +import static java.util.Collections.emptySet; +import static java.util.Collections.singleton; +import static java.util.Collections.singletonList; +import static java.util.Collections.singletonMap; +import static org.apache.kafka.clients.consumer.ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG; +import static org.apache.kafka.clients.consumer.ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG; +import static org.apache.kafka.common.requests.FetchMetadata.INVALID_SESSION_ID; +import static org.apache.kafka.common.utils.Utils.mkSet; +import static org.apache.kafka.test.TestUtils.assertOptional; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +public class FetchRequestManagerTest { + + private static final double EPSILON = 0.0001; + + private ConsumerRebalanceListener listener = new NoOpConsumerRebalanceListener(); + private String topicName = "test"; + private String groupId = "test-group"; + private Uuid topicId = Uuid.randomUuid(); + private Map topicIds = new HashMap() { + { + put(topicName, topicId); + } + }; + private Map topicNames = singletonMap(topicId, topicName); + private final String metricGroup = "consumer" + groupId + "-fetch-manager-metrics"; + private TopicPartition tp0 = new TopicPartition(topicName, 0); + private TopicPartition tp1 = new TopicPartition(topicName, 1); + private TopicPartition tp2 = new TopicPartition(topicName, 2); + private TopicPartition tp3 = new TopicPartition(topicName, 3); + private TopicIdPartition tidp0 = new TopicIdPartition(topicId, tp0); + private TopicIdPartition tidp1 = new TopicIdPartition(topicId, tp1); + private TopicIdPartition tidp2 = new TopicIdPartition(topicId, tp2); + private TopicIdPartition tidp3 = new TopicIdPartition(topicId, tp3); + private int validLeaderEpoch = 0; + private MetadataResponse initialUpdateResponse = + RequestTestUtils.metadataUpdateWithIds(1, singletonMap(topicName, 4), topicIds); + + private int minBytes = 1; + private int maxBytes = Integer.MAX_VALUE; + private int maxWaitMs = 0; + private int fetchSize = 1000; + private long retryBackoffMs = 100; + private long requestTimeoutMs = 30000; + private MockTime time = new MockTime(1); + private SubscriptionState subscriptions; + private ConsumerMetadata metadata; + private FetchMetricsRegistry metricsRegistry; + private FetchMetricsManager metricsManager; + private MockClient client; + private Metrics metrics; + private ApiVersions apiVersions = new ApiVersions(); + private ConsumerNetworkClient oldConsumerClient; + private Deserializers deserializers; + private TestableFetchRequestManager fetcher; + private TestableNetworkClientDelegate consumerClient; + private OffsetFetcher offsetFetcher; + + private MemoryRecords records; + private MemoryRecords nextRecords; + private MemoryRecords emptyRecords; + private MemoryRecords partialRecords; + + @BeforeEach + public void setup() { + records = buildRecords(1L, 3, 1); + nextRecords = buildRecords(4L, 2, 4); + emptyRecords = buildRecords(0L, 0, 0); + partialRecords = buildRecords(4L, 1, 0); + partialRecords.buffer().putInt(Records.SIZE_OFFSET, 10000); + } + + private void assignFromUser(Set partitions) { + subscriptions.assignFromUser(partitions); + client.updateMetadata(initialUpdateResponse); + + // A dummy metadata update to ensure valid leader epoch. + metadata.updateWithCurrentRequestVersion(RequestTestUtils.metadataUpdateWithIds("dummy", 1, + Collections.emptyMap(), singletonMap(topicName, 4), + tp -> validLeaderEpoch, topicIds), false, 0L); + } + + private void assignFromUserNoId(Set partitions) { + subscriptions.assignFromUser(partitions); + client.updateMetadata(RequestTestUtils.metadataUpdateWithIds(1, singletonMap("noId", 1), Collections.emptyMap())); + + // A dummy metadata update to ensure valid leader epoch. + metadata.update(9, RequestTestUtils.metadataUpdateWithIds("dummy", 1, + Collections.emptyMap(), singletonMap("noId", 1), + tp -> validLeaderEpoch, topicIds), false, 0L); + } + + @AfterEach + public void teardown() throws Exception { + if (metrics != null) + this.metrics.close(); + if (fetcher != null) + this.fetcher.close(); + } + + private int sendFetches() { + offsetFetcher.validatePositionsOnMetadataChange(); + return fetcher.sendFetches(); + } + + @Test + public void testFetchNormal() { + buildFetcher(); + + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + // normal fetch + assertEquals(1, sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + + Map>> partitionRecords = fetchedRecords(); + assertTrue(partitionRecords.containsKey(tp0)); + + List> records = partitionRecords.get(tp0); + assertEquals(3, records.size()); + assertEquals(4L, subscriptions.position(tp0).offset); // this is the next fetching position + long offset = 1; + for (ConsumerRecord record : records) { + assertEquals(offset, record.offset()); + offset += 1; + } + } + + @Test + public void testInflightFetchOnPendingPartitions() { + buildFetcher(); + + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + assertEquals(1, sendFetches()); + subscriptions.markPendingRevocation(singleton(tp0)); + + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + consumerClient.poll(time.timer(0)); + assertNull(fetchedRecords().get(tp0)); + } + + @Test + public void testCloseShouldBeIdempotent() { + buildFetcher(); + + fetcher.close(); + fetcher.close(); + fetcher.close(); + + verify(fetcher, times(1)).closeInternal(any(Timer.class)); + } + + @Test + public void testFetcherCloseClosesFetchSessionsInBroker() { + buildFetcher(); + + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + // normal fetch + assertEquals(1, sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + + final FetchResponse fetchResponse = fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0); + client.prepareResponse(fetchResponse); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + assertEquals(0, consumerClient.pendingRequestCount()); + + final ArgumentCaptor argument = ArgumentCaptor.forClass(NetworkClientDelegate.UnsentRequest.class); + + // send request to close the fetcher + Timer timer = time.timer(Duration.ofSeconds(10)); + this.fetcher.close(timer); + NetworkClientDelegate.PollResult pollResult = fetcher.poll(time.milliseconds()); + consumerClient.addAll(pollResult.unsentRequests); + consumerClient.poll(timer); + + // validate that Fetcher.close() has sent a request with final epoch. 2 requests are sent, one for the normal + // fetch earlier and another for the finish fetch here. + verify(consumerClient, times(2)).doSend(argument.capture(), any(Long.class)); + NetworkClientDelegate.UnsentRequest unsentRequest = argument.getValue(); + FetchRequest.Builder builder = (FetchRequest.Builder) unsentRequest.requestBuilder(); + // session Id is the same + assertEquals(fetchResponse.sessionId(), builder.metadata().sessionId()); + // contains final epoch + assertEquals(FetchMetadata.FINAL_EPOCH, builder.metadata().epoch()); // final epoch indicates we want to close the session + assertTrue(builder.fetchData().isEmpty()); // partition data should be empty + } + + @Test + public void testFetchingPendingPartitions() { + buildFetcher(); + + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + // normal fetch + assertEquals(1, sendFetches()); + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + fetchedRecords(); + assertEquals(4L, subscriptions.position(tp0).offset); // this is the next fetching position + + // mark partition unfetchable + subscriptions.markPendingRevocation(singleton(tp0)); + assertEquals(0, sendFetches()); + consumerClient.poll(time.timer(0)); + assertFalse(fetcher.hasCompletedFetches()); + fetchedRecords(); + assertEquals(4L, subscriptions.position(tp0).offset); + } + + @Test + public void testFetchWithNoTopicId() { + // Should work and default to using old request type. + buildFetcher(); + + TopicIdPartition noId = new TopicIdPartition(Uuid.ZERO_UUID, new TopicPartition("noId", 0)); + assignFromUserNoId(singleton(noId.topicPartition())); + subscriptions.seek(noId.topicPartition(), 0); + + // Fetch should use request version 12 + assertEquals(1, sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + + client.prepareResponse( + fetchRequestMatcher((short) 12, noId, 0, Optional.of(validLeaderEpoch)), + fullFetchResponse(noId, this.records, Errors.NONE, 100L, 0) + ); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + + Map>> partitionRecords = fetchedRecords(); + assertTrue(partitionRecords.containsKey(noId.topicPartition())); + + List> records = partitionRecords.get(noId.topicPartition()); + assertEquals(3, records.size()); + assertEquals(4L, subscriptions.position(noId.topicPartition()).offset); // this is the next fetching position + long offset = 1; + for (ConsumerRecord record : records) { + assertEquals(offset, record.offset()); + offset += 1; + } + } + + @Test + public void testFetchWithTopicId() { + buildFetcher(); + + TopicIdPartition tp = new TopicIdPartition(topicId, new TopicPartition(topicName, 0)); + assignFromUser(singleton(tp.topicPartition())); + subscriptions.seek(tp.topicPartition(), 0); + + // Fetch should use latest version + assertEquals(1, sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + + client.prepareResponse( + fetchRequestMatcher(ApiKeys.FETCH.latestVersion(), tp, 0, Optional.of(validLeaderEpoch)), + fullFetchResponse(tp, this.records, Errors.NONE, 100L, 0) + ); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + + Map>> partitionRecords = fetchedRecords(); + assertTrue(partitionRecords.containsKey(tp.topicPartition())); + + List> records = partitionRecords.get(tp.topicPartition()); + assertEquals(3, records.size()); + assertEquals(4L, subscriptions.position(tp.topicPartition()).offset); // this is the next fetching position + long offset = 1; + for (ConsumerRecord record : records) { + assertEquals(offset, record.offset()); + offset += 1; + } + } + + @Test + public void testFetchForgetTopicIdWhenUnassigned() { + buildFetcher(); + + TopicIdPartition foo = new TopicIdPartition(Uuid.randomUuid(), new TopicPartition("foo", 0)); + TopicIdPartition bar = new TopicIdPartition(Uuid.randomUuid(), new TopicPartition("bar", 0)); + + // Assign foo and bar. + subscriptions.assignFromUser(singleton(foo.topicPartition())); + client.updateMetadata(RequestTestUtils.metadataUpdateWithIds(1, singleton(foo), tp -> validLeaderEpoch)); + subscriptions.seek(foo.topicPartition(), 0); + + // Fetch should use latest version. + assertEquals(1, sendFetches()); + + client.prepareResponse( + fetchRequestMatcher(ApiKeys.FETCH.latestVersion(), + singletonMap(foo, new PartitionData( + foo.topicId(), + 0, + FetchRequest.INVALID_LOG_START_OFFSET, + fetchSize, + Optional.of(validLeaderEpoch)) + ), + emptyList() + ), + fullFetchResponse(1, foo, this.records, Errors.NONE, 100L, 0) + ); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + fetchedRecords(); + + // Assign bar and unassign foo. + subscriptions.assignFromUser(singleton(bar.topicPartition())); + client.updateMetadata(RequestTestUtils.metadataUpdateWithIds(1, singleton(bar), tp -> validLeaderEpoch)); + subscriptions.seek(bar.topicPartition(), 0); + + // Fetch should use latest version. + assertEquals(1, sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + + client.prepareResponse( + fetchRequestMatcher(ApiKeys.FETCH.latestVersion(), + singletonMap(bar, new PartitionData( + bar.topicId(), + 0, + FetchRequest.INVALID_LOG_START_OFFSET, + fetchSize, + Optional.of(validLeaderEpoch)) + ), + singletonList(foo) + ), + fullFetchResponse(1, bar, this.records, Errors.NONE, 100L, 0) + ); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + fetchedRecords(); + } + + @Test + public void testFetchForgetTopicIdWhenReplaced() { + buildFetcher(); + + TopicIdPartition fooWithOldTopicId = new TopicIdPartition(Uuid.randomUuid(), new TopicPartition("foo", 0)); + TopicIdPartition fooWithNewTopicId = new TopicIdPartition(Uuid.randomUuid(), new TopicPartition("foo", 0)); + + // Assign foo with old topic id. + subscriptions.assignFromUser(singleton(fooWithOldTopicId.topicPartition())); + client.updateMetadata(RequestTestUtils.metadataUpdateWithIds(1, singleton(fooWithOldTopicId), tp -> validLeaderEpoch)); + subscriptions.seek(fooWithOldTopicId.topicPartition(), 0); + + // Fetch should use latest version. + assertEquals(1, sendFetches()); + + client.prepareResponse( + fetchRequestMatcher(ApiKeys.FETCH.latestVersion(), + singletonMap(fooWithOldTopicId, new PartitionData( + fooWithOldTopicId.topicId(), + 0, + FetchRequest.INVALID_LOG_START_OFFSET, + fetchSize, + Optional.of(validLeaderEpoch)) + ), + emptyList() + ), + fullFetchResponse(1, fooWithOldTopicId, this.records, Errors.NONE, 100L, 0) + ); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + fetchedRecords(); + + // Replace foo with old topic id with foo with new topic id. + subscriptions.assignFromUser(singleton(fooWithNewTopicId.topicPartition())); + client.updateMetadata(RequestTestUtils.metadataUpdateWithIds(1, singleton(fooWithNewTopicId), tp -> validLeaderEpoch)); + subscriptions.seek(fooWithNewTopicId.topicPartition(), 0); + + // Fetch should use latest version. + assertEquals(1, sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + + // foo with old topic id should be removed from the session. + client.prepareResponse( + fetchRequestMatcher(ApiKeys.FETCH.latestVersion(), + singletonMap(fooWithNewTopicId, new PartitionData( + fooWithNewTopicId.topicId(), + 0, + FetchRequest.INVALID_LOG_START_OFFSET, + fetchSize, + Optional.of(validLeaderEpoch)) + ), + singletonList(fooWithOldTopicId) + ), + fullFetchResponse(1, fooWithNewTopicId, this.records, Errors.NONE, 100L, 0) + ); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + fetchedRecords(); + } + + @Test + public void testFetchTopicIdUpgradeDowngrade() { + buildFetcher(); + + TopicIdPartition fooWithoutId = new TopicIdPartition(Uuid.ZERO_UUID, new TopicPartition("foo", 0)); + TopicIdPartition fooWithId = new TopicIdPartition(Uuid.randomUuid(), new TopicPartition("foo", 0)); + + // Assign foo without a topic id. + subscriptions.assignFromUser(singleton(fooWithoutId.topicPartition())); + client.updateMetadata(RequestTestUtils.metadataUpdateWithIds(1, singleton(fooWithoutId), tp -> validLeaderEpoch)); + subscriptions.seek(fooWithoutId.topicPartition(), 0); + + // Fetch should use version 12. + assertEquals(1, sendFetches()); + + client.prepareResponse( + fetchRequestMatcher((short) 12, + singletonMap(fooWithoutId, new PartitionData( + fooWithoutId.topicId(), + 0, + FetchRequest.INVALID_LOG_START_OFFSET, + fetchSize, + Optional.of(validLeaderEpoch)) + ), + emptyList() + ), + fullFetchResponse(1, fooWithoutId, this.records, Errors.NONE, 100L, 0) + ); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + fetchedRecords(); + + // Upgrade. + subscriptions.assignFromUser(singleton(fooWithId.topicPartition())); + client.updateMetadata(RequestTestUtils.metadataUpdateWithIds(1, singleton(fooWithId), tp -> validLeaderEpoch)); + subscriptions.seek(fooWithId.topicPartition(), 0); + + // Fetch should use latest version. + assertEquals(1, sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + + // foo with old topic id should be removed from the session. + client.prepareResponse( + fetchRequestMatcher(ApiKeys.FETCH.latestVersion(), + singletonMap(fooWithId, new PartitionData( + fooWithId.topicId(), + 0, + FetchRequest.INVALID_LOG_START_OFFSET, + fetchSize, + Optional.of(validLeaderEpoch)) + ), + emptyList() + ), + fullFetchResponse(1, fooWithId, this.records, Errors.NONE, 100L, 0) + ); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + fetchedRecords(); + + // Downgrade. + subscriptions.assignFromUser(singleton(fooWithoutId.topicPartition())); + client.updateMetadata(RequestTestUtils.metadataUpdateWithIds(1, singleton(fooWithoutId), tp -> validLeaderEpoch)); + subscriptions.seek(fooWithoutId.topicPartition(), 0); + + // Fetch should use version 12. + assertEquals(1, sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + + // foo with old topic id should be removed from the session. + client.prepareResponse( + fetchRequestMatcher((short) 12, + singletonMap(fooWithoutId, new PartitionData( + fooWithoutId.topicId(), + 0, + FetchRequest.INVALID_LOG_START_OFFSET, + fetchSize, + Optional.of(validLeaderEpoch)) + ), + emptyList() + ), + fullFetchResponse(1, fooWithoutId, this.records, Errors.NONE, 100L, 0) + ); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + fetchedRecords(); + } + + private MockClient.RequestMatcher fetchRequestMatcher( + short expectedVersion, + TopicIdPartition tp, + long expectedFetchOffset, + Optional expectedCurrentLeaderEpoch + ) { + return fetchRequestMatcher( + expectedVersion, + singletonMap(tp, new PartitionData( + tp.topicId(), + expectedFetchOffset, + FetchRequest.INVALID_LOG_START_OFFSET, + fetchSize, + expectedCurrentLeaderEpoch + )), + emptyList() + ); + } + + private MockClient.RequestMatcher fetchRequestMatcher( + short expectedVersion, + Map fetch, + List forgotten + ) { + return body -> { + if (body instanceof FetchRequest) { + FetchRequest fetchRequest = (FetchRequest) body; + assertEquals(expectedVersion, fetchRequest.version()); + assertEquals(fetch, fetchRequest.fetchData(topicNames(new ArrayList<>(fetch.keySet())))); + assertEquals(forgotten, fetchRequest.forgottenTopics(topicNames(forgotten))); + return true; + } else { + fail("Should have seen FetchRequest"); + return false; + } + }; + } + + private Map topicNames(List partitions) { + Map topicNames = new HashMap<>(); + partitions.forEach(partition -> topicNames.putIfAbsent(partition.topicId(), partition.topic())); + return topicNames; + } + + @Test + public void testMissingLeaderEpochInRecords() { + buildFetcher(); + + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + ByteBuffer buffer = ByteBuffer.allocate(1024); + MemoryRecordsBuilder builder = MemoryRecords.builder(buffer, RecordBatch.MAGIC_VALUE_V0, + CompressionType.NONE, TimestampType.CREATE_TIME, 0L, System.currentTimeMillis(), + RecordBatch.NO_PARTITION_LEADER_EPOCH); + builder.append(0L, "key".getBytes(), "1".getBytes()); + builder.append(0L, "key".getBytes(), "2".getBytes()); + MemoryRecords records = builder.build(); + + assertEquals(1, sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + + Map>> partitionRecords = fetchedRecords(); + assertTrue(partitionRecords.containsKey(tp0)); + assertEquals(2, partitionRecords.get(tp0).size()); + + for (ConsumerRecord record : partitionRecords.get(tp0)) { + assertEquals(Optional.empty(), record.leaderEpoch()); + } + } + + @Test + public void testLeaderEpochInConsumerRecord() { + buildFetcher(); + + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + Integer partitionLeaderEpoch = 1; + + ByteBuffer buffer = ByteBuffer.allocate(1024); + MemoryRecordsBuilder builder = MemoryRecords.builder(buffer, RecordBatch.CURRENT_MAGIC_VALUE, + CompressionType.NONE, TimestampType.CREATE_TIME, 0L, System.currentTimeMillis(), + partitionLeaderEpoch); + builder.append(0L, "key".getBytes(), partitionLeaderEpoch.toString().getBytes()); + builder.append(0L, "key".getBytes(), partitionLeaderEpoch.toString().getBytes()); + builder.close(); + + partitionLeaderEpoch += 7; + + builder = MemoryRecords.builder(buffer, RecordBatch.CURRENT_MAGIC_VALUE, CompressionType.NONE, + TimestampType.CREATE_TIME, 2L, System.currentTimeMillis(), partitionLeaderEpoch); + builder.append(0L, "key".getBytes(), partitionLeaderEpoch.toString().getBytes()); + builder.close(); + + partitionLeaderEpoch += 5; + builder = MemoryRecords.builder(buffer, RecordBatch.CURRENT_MAGIC_VALUE, CompressionType.NONE, + TimestampType.CREATE_TIME, 3L, System.currentTimeMillis(), partitionLeaderEpoch); + builder.append(0L, "key".getBytes(), partitionLeaderEpoch.toString().getBytes()); + builder.append(0L, "key".getBytes(), partitionLeaderEpoch.toString().getBytes()); + builder.append(0L, "key".getBytes(), partitionLeaderEpoch.toString().getBytes()); + builder.close(); + + buffer.flip(); + MemoryRecords records = MemoryRecords.readableRecords(buffer); + + assertEquals(1, sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + + Map>> partitionRecords = fetchedRecords(); + assertTrue(partitionRecords.containsKey(tp0)); + assertEquals(6, partitionRecords.get(tp0).size()); + + for (ConsumerRecord record : partitionRecords.get(tp0)) { + int expectedLeaderEpoch = Integer.parseInt(Utils.utf8(record.value())); + assertEquals(Optional.of(expectedLeaderEpoch), record.leaderEpoch()); + } + } + + @Test + public void testClearBufferedDataForTopicPartitions() { + buildFetcher(); + + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + // normal fetch + assertEquals(1, sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + Set newAssignedTopicPartitions = new HashSet<>(); + newAssignedTopicPartitions.add(tp1); + + fetcher.clearBufferedDataForUnassignedPartitions(newAssignedTopicPartitions); + assertFalse(fetcher.hasCompletedFetches()); + } + + @Test + public void testFetchSkipsBlackedOutNodes() { + buildFetcher(); + + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + Node node = initialUpdateResponse.brokers().iterator().next(); + + client.backoff(node, 500); + assertEquals(0, sendFetches()); + + time.sleep(500); + assertEquals(1, sendFetches()); + } + + @Test + public void testFetcherIgnoresControlRecords() { + buildFetcher(); + + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + // normal fetch + assertEquals(1, sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + + long producerId = 1; + short producerEpoch = 0; + int baseSequence = 0; + int partitionLeaderEpoch = 0; + + ByteBuffer buffer = ByteBuffer.allocate(1024); + MemoryRecordsBuilder builder = MemoryRecords.idempotentBuilder(buffer, CompressionType.NONE, 0L, producerId, + producerEpoch, baseSequence); + builder.append(0L, "key".getBytes(), null); + builder.close(); + + MemoryRecords.writeEndTransactionalMarker(buffer, 1L, time.milliseconds(), partitionLeaderEpoch, producerId, producerEpoch, + new EndTransactionMarker(ControlRecordType.ABORT, 0)); + + buffer.flip(); + + client.prepareResponse(fullFetchResponse(tidp0, MemoryRecords.readableRecords(buffer), Errors.NONE, 100L, 0)); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + + Map>> partitionRecords = fetchedRecords(); + assertTrue(partitionRecords.containsKey(tp0)); + + List> records = partitionRecords.get(tp0); + assertEquals(1, records.size()); + assertEquals(2L, subscriptions.position(tp0).offset); + + ConsumerRecord record = records.get(0); + assertArrayEquals("key".getBytes(), record.key()); + } + + @Test + public void testFetchError() { + buildFetcher(); + + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + assertEquals(1, sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NOT_LEADER_OR_FOLLOWER, 100L, 0)); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + + Map>> partitionRecords = fetchedRecords(); + assertFalse(partitionRecords.containsKey(tp0)); + } + + private MockClient.RequestMatcher matchesOffset(final TopicIdPartition tp, final long offset) { + return body -> { + FetchRequest fetch = (FetchRequest) body; + Map fetchData = fetch.fetchData(topicNames); + return fetchData.containsKey(tp) && + fetchData.get(tp).fetchOffset == offset; + }; + } + + @Test + public void testFetchedRecordsRaisesOnSerializationErrors() { + // raise an exception from somewhere in the middle of the fetch response + // so that we can verify that our position does not advance after raising + ByteArrayDeserializer deserializer = new ByteArrayDeserializer() { + int i = 0; + @Override + public byte[] deserialize(String topic, byte[] data) { + if (i++ % 2 == 1) { + // Should be blocked on the value deserialization of the first record. + assertEquals("value-1", new String(data, StandardCharsets.UTF_8)); + throw new SerializationException(); + } + return data; + } + }; + + buildFetcher(deserializer, deserializer); + + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 1); + + client.prepareResponse(matchesOffset(tidp0, 1), fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + + assertEquals(1, sendFetches()); + consumerClient.poll(time.timer(0)); + // The fetcher should block on Deserialization error + for (int i = 0; i < 2; i++) { + try { + fetcher.collectFetch(); + fail("fetchedRecords should have raised"); + } catch (SerializationException e) { + // the position should not advance since no data has been returned + assertEquals(1, subscriptions.position(tp0).offset); + } + } + } + + @Test + public void testParseCorruptedRecord() throws Exception { + buildFetcher(); + assignFromUser(singleton(tp0)); + + ByteBuffer buffer = ByteBuffer.allocate(1024); + DataOutputStream out = new DataOutputStream(new ByteBufferOutputStream(buffer)); + + byte magic = RecordBatch.MAGIC_VALUE_V1; + byte[] key = "foo".getBytes(); + byte[] value = "baz".getBytes(); + long offset = 0; + long timestamp = 500L; + + int size = LegacyRecord.recordSize(magic, key.length, value.length); + byte attributes = LegacyRecord.computeAttributes(magic, CompressionType.NONE, TimestampType.CREATE_TIME); + long crc = LegacyRecord.computeChecksum(magic, attributes, timestamp, key, value); + + // write one valid record + out.writeLong(offset); + out.writeInt(size); + LegacyRecord.write(out, magic, crc, LegacyRecord.computeAttributes(magic, CompressionType.NONE, TimestampType.CREATE_TIME), timestamp, key, value); + + // and one invalid record (note the crc) + out.writeLong(offset + 1); + out.writeInt(size); + LegacyRecord.write(out, magic, crc + 1, LegacyRecord.computeAttributes(magic, CompressionType.NONE, TimestampType.CREATE_TIME), timestamp, key, value); + + // write one valid record + out.writeLong(offset + 2); + out.writeInt(size); + LegacyRecord.write(out, magic, crc, LegacyRecord.computeAttributes(magic, CompressionType.NONE, TimestampType.CREATE_TIME), timestamp, key, value); + + // Write a record whose size field is invalid. + out.writeLong(offset + 3); + out.writeInt(1); + + // write one valid record + out.writeLong(offset + 4); + out.writeInt(size); + LegacyRecord.write(out, magic, crc, LegacyRecord.computeAttributes(magic, CompressionType.NONE, TimestampType.CREATE_TIME), timestamp, key, value); + + buffer.flip(); + + subscriptions.seekUnvalidated(tp0, new SubscriptionState.FetchPosition(0, Optional.empty(), metadata.currentLeader(tp0))); + + // normal fetch + assertEquals(1, sendFetches()); + client.prepareResponse(fullFetchResponse(tidp0, MemoryRecords.readableRecords(buffer), Errors.NONE, 100L, 0)); + consumerClient.poll(time.timer(0)); + + // the first fetchedRecords() should return the first valid message + assertEquals(1, fetchedRecords().get(tp0).size()); + assertEquals(1, subscriptions.position(tp0).offset); + + ensureBlockOnRecord(1L); + seekAndConsumeRecord(buffer, 2L); + ensureBlockOnRecord(3L); + try { + // For a record that cannot be retrieved from the iterator, we cannot seek over it within the batch. + seekAndConsumeRecord(buffer, 4L); + fail("Should have thrown exception when fail to retrieve a record from iterator."); + } catch (KafkaException ke) { + // let it go + } + ensureBlockOnRecord(4L); + } + + private void ensureBlockOnRecord(long blockedOffset) { + // the fetchedRecords() should always throw exception due to the invalid message at the starting offset. + for (int i = 0; i < 2; i++) { + try { + fetcher.collectFetch(); + fail("fetchedRecords should have raised KafkaException"); + } catch (KafkaException e) { + assertEquals(blockedOffset, subscriptions.position(tp0).offset); + } + } + } + + private void seekAndConsumeRecord(ByteBuffer responseBuffer, long toOffset) { + // Seek to skip the bad record and fetch again. + subscriptions.seekUnvalidated(tp0, new SubscriptionState.FetchPosition(toOffset, Optional.empty(), metadata.currentLeader(tp0))); + // Should not throw exception after the seek. + fetcher.collectFetch(); + assertEquals(1, sendFetches()); + client.prepareResponse(fullFetchResponse(tidp0, MemoryRecords.readableRecords(responseBuffer), Errors.NONE, 100L, 0)); + consumerClient.poll(time.timer(0)); + + Map>> recordsByPartition = fetchedRecords(); + List> records = recordsByPartition.get(tp0); + assertEquals(1, records.size()); + assertEquals(toOffset, records.get(0).offset()); + assertEquals(toOffset + 1, subscriptions.position(tp0).offset); + } + + @Test + public void testInvalidDefaultRecordBatch() { + buildFetcher(); + + ByteBuffer buffer = ByteBuffer.allocate(1024); + ByteBufferOutputStream out = new ByteBufferOutputStream(buffer); + + MemoryRecordsBuilder builder = new MemoryRecordsBuilder(out, + DefaultRecordBatch.CURRENT_MAGIC_VALUE, + CompressionType.NONE, + TimestampType.CREATE_TIME, + 0L, 10L, 0L, (short) 0, 0, false, false, 0, 1024); + builder.append(10L, "key".getBytes(), "value".getBytes()); + builder.close(); + buffer.flip(); + + // Garble the CRC + buffer.position(17); + buffer.put("beef".getBytes()); + buffer.position(0); + + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + // normal fetch + assertEquals(1, sendFetches()); + client.prepareResponse(fullFetchResponse(tidp0, MemoryRecords.readableRecords(buffer), Errors.NONE, 100L, 0)); + consumerClient.poll(time.timer(0)); + + // the fetchedRecords() should always throw exception due to the bad batch. + for (int i = 0; i < 2; i++) { + try { + fetcher.collectFetch(); + fail("fetchedRecords should have raised KafkaException"); + } catch (KafkaException e) { + assertEquals(0, subscriptions.position(tp0).offset); + } + } + } + + @Test + public void testParseInvalidRecordBatch() { + buildFetcher(); + MemoryRecords records = MemoryRecords.withRecords(RecordBatch.MAGIC_VALUE_V2, 0L, + CompressionType.NONE, TimestampType.CREATE_TIME, + new SimpleRecord(1L, "a".getBytes(), "1".getBytes()), + new SimpleRecord(2L, "b".getBytes(), "2".getBytes()), + new SimpleRecord(3L, "c".getBytes(), "3".getBytes())); + ByteBuffer buffer = records.buffer(); + + // flip some bits to fail the crc + buffer.putInt(32, buffer.get(32) ^ 87238423); + + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + // normal fetch + assertEquals(1, sendFetches()); + client.prepareResponse(fullFetchResponse(tidp0, MemoryRecords.readableRecords(buffer), Errors.NONE, 100L, 0)); + consumerClient.poll(time.timer(0)); + try { + fetcher.collectFetch(); + fail("fetchedRecords should have raised"); + } catch (KafkaException e) { + // the position should not advance since no data has been returned + assertEquals(0, subscriptions.position(tp0).offset); + } + } + + @Test + public void testHeaders() { + buildFetcher(); + + MemoryRecordsBuilder builder = MemoryRecords.builder(ByteBuffer.allocate(1024), CompressionType.NONE, TimestampType.CREATE_TIME, 1L); + builder.append(0L, "key".getBytes(), "value-1".getBytes()); + + Header[] headersArray = new Header[1]; + headersArray[0] = new RecordHeader("headerKey", "headerValue".getBytes(StandardCharsets.UTF_8)); + builder.append(0L, "key".getBytes(), "value-2".getBytes(), headersArray); + + Header[] headersArray2 = new Header[2]; + headersArray2[0] = new RecordHeader("headerKey", "headerValue".getBytes(StandardCharsets.UTF_8)); + headersArray2[1] = new RecordHeader("headerKey", "headerValue2".getBytes(StandardCharsets.UTF_8)); + builder.append(0L, "key".getBytes(), "value-3".getBytes(), headersArray2); + + MemoryRecords memoryRecords = builder.build(); + + List> records; + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 1); + + client.prepareResponse(matchesOffset(tidp0, 1), fullFetchResponse(tidp0, memoryRecords, Errors.NONE, 100L, 0)); + + assertEquals(1, sendFetches()); + consumerClient.poll(time.timer(0)); + Map>> recordsByPartition = fetchedRecords(); + records = recordsByPartition.get(tp0); + + assertEquals(3, records.size()); + + Iterator> recordIterator = records.iterator(); + + ConsumerRecord record = recordIterator.next(); + assertNull(record.headers().lastHeader("headerKey")); + + record = recordIterator.next(); + assertEquals("headerValue", new String(record.headers().lastHeader("headerKey").value(), StandardCharsets.UTF_8)); + assertEquals("headerKey", record.headers().lastHeader("headerKey").key()); + + record = recordIterator.next(); + assertEquals("headerValue2", new String(record.headers().lastHeader("headerKey").value(), StandardCharsets.UTF_8)); + assertEquals("headerKey", record.headers().lastHeader("headerKey").key()); + } + + @Test + public void testFetchMaxPollRecords() { + buildFetcher(2); + + List> records; + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 1); + + client.prepareResponse(matchesOffset(tidp0, 1), fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + client.prepareResponse(matchesOffset(tidp0, 4), fullFetchResponse(tidp0, this.nextRecords, Errors.NONE, 100L, 0)); + + assertEquals(1, sendFetches()); + consumerClient.poll(time.timer(0)); + Map>> recordsByPartition = fetchedRecords(); + records = recordsByPartition.get(tp0); + assertEquals(2, records.size()); + assertEquals(3L, subscriptions.position(tp0).offset); + assertEquals(1, records.get(0).offset()); + assertEquals(2, records.get(1).offset()); + + assertEquals(0, sendFetches()); + consumerClient.poll(time.timer(0)); + recordsByPartition = fetchedRecords(); + records = recordsByPartition.get(tp0); + assertEquals(1, records.size()); + assertEquals(4L, subscriptions.position(tp0).offset); + assertEquals(3, records.get(0).offset()); + + assertTrue(sendFetches() > 0); + consumerClient.poll(time.timer(0)); + recordsByPartition = fetchedRecords(); + records = recordsByPartition.get(tp0); + assertEquals(2, records.size()); + assertEquals(6L, subscriptions.position(tp0).offset); + assertEquals(4, records.get(0).offset()); + assertEquals(5, records.get(1).offset()); + } + + /** + * Test the scenario where a partition with fetched but not consumed records (i.e. max.poll.records is + * less than the number of fetched records) is unassigned and a different partition is assigned. This is a + * pattern used by Streams state restoration and KAFKA-5097 would have been caught by this test. + */ + @Test + public void testFetchAfterPartitionWithFetchedRecordsIsUnassigned() { + buildFetcher(2); + + List> records; + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 1); + + // Returns 3 records while `max.poll.records` is configured to 2 + client.prepareResponse(matchesOffset(tidp0, 1), fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + + assertEquals(1, sendFetches()); + consumerClient.poll(time.timer(0)); + Map>> recordsByPartition = fetchedRecords(); + records = recordsByPartition.get(tp0); + assertEquals(2, records.size()); + assertEquals(3L, subscriptions.position(tp0).offset); + assertEquals(1, records.get(0).offset()); + assertEquals(2, records.get(1).offset()); + + assignFromUser(singleton(tp1)); + client.prepareResponse(matchesOffset(tidp1, 4), fullFetchResponse(tidp1, this.nextRecords, Errors.NONE, 100L, 0)); + subscriptions.seek(tp1, 4); + + assertEquals(1, sendFetches()); + consumerClient.poll(time.timer(0)); + Map>> fetchedRecords = fetchedRecords(); + assertNull(fetchedRecords.get(tp0)); + records = fetchedRecords.get(tp1); + assertEquals(2, records.size()); + assertEquals(6L, subscriptions.position(tp1).offset); + assertEquals(4, records.get(0).offset()); + assertEquals(5, records.get(1).offset()); + } + + @Test + public void testFetchNonContinuousRecords() { + // if we are fetching from a compacted topic, there may be gaps in the returned records + // this test verifies the fetcher updates the current fetched/consumed positions correctly for this case + buildFetcher(); + + MemoryRecordsBuilder builder = MemoryRecords.builder(ByteBuffer.allocate(1024), CompressionType.NONE, + TimestampType.CREATE_TIME, 0L); + builder.appendWithOffset(15L, 0L, "key".getBytes(), "value-1".getBytes()); + builder.appendWithOffset(20L, 0L, "key".getBytes(), "value-2".getBytes()); + builder.appendWithOffset(30L, 0L, "key".getBytes(), "value-3".getBytes()); + MemoryRecords records = builder.build(); + + List> consumerRecords; + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + // normal fetch + assertEquals(1, sendFetches()); + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); + consumerClient.poll(time.timer(0)); + Map>> recordsByPartition = fetchedRecords(); + consumerRecords = recordsByPartition.get(tp0); + assertEquals(3, consumerRecords.size()); + assertEquals(31L, subscriptions.position(tp0).offset); // this is the next fetching position + + assertEquals(15L, consumerRecords.get(0).offset()); + assertEquals(20L, consumerRecords.get(1).offset()); + assertEquals(30L, consumerRecords.get(2).offset()); + } + + /** + * Test the case where the client makes a pre-v3 FetchRequest, but the server replies with only a partial + * request. This happens when a single message is larger than the per-partition limit. + */ + @Test + public void testFetchRequestWhenRecordTooLarge() { + try { + buildFetcher(); + + client.setNodeApiVersions(NodeApiVersions.create(ApiKeys.FETCH.id, (short) 2, (short) 2)); + makeFetchRequestWithIncompleteRecord(); + try { + fetcher.collectFetch(); + fail("RecordTooLargeException should have been raised"); + } catch (RecordTooLargeException e) { + assertTrue(e.getMessage().startsWith("There are some messages at [Partition=Offset]: ")); + // the position should not advance since no data has been returned + assertEquals(0, subscriptions.position(tp0).offset); + } + } finally { + client.setNodeApiVersions(NodeApiVersions.create()); + } + } + + /** + * Test the case where the client makes a post KIP-74 FetchRequest, but the server replies with only a + * partial request. For v3 and later FetchRequests, the implementation of KIP-74 changed the behavior + * so that at least one message is always returned. Therefore, this case should not happen, and it indicates + * that an internal error has taken place. + */ + @Test + public void testFetchRequestInternalError() { + buildFetcher(); + makeFetchRequestWithIncompleteRecord(); + try { + fetcher.collectFetch(); + fail("RecordTooLargeException should have been raised"); + } catch (KafkaException e) { + assertTrue(e.getMessage().startsWith("Failed to make progress reading messages")); + // the position should not advance since no data has been returned + assertEquals(0, subscriptions.position(tp0).offset); + } + } + + private void makeFetchRequestWithIncompleteRecord() { + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + assertEquals(1, sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + MemoryRecords partialRecord = MemoryRecords.readableRecords( + ByteBuffer.wrap(new byte[]{0, 0, 0, 0, 0, 0, 0, 0})); + client.prepareResponse(fullFetchResponse(tidp0, partialRecord, Errors.NONE, 100L, 0)); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + } + + @Test + public void testUnauthorizedTopic() { + buildFetcher(); + + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + // resize the limit of the buffer to pretend it is only fetch-size large + assertEquals(1, sendFetches()); + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.TOPIC_AUTHORIZATION_FAILED, 100L, 0)); + consumerClient.poll(time.timer(0)); + try { + fetcher.collectFetch(); + fail("fetchedRecords should have thrown"); + } catch (TopicAuthorizationException e) { + assertEquals(singleton(topicName), e.unauthorizedTopics()); + } + } + + @Test + public void testFetchDuringEagerRebalance() { + buildFetcher(); + + subscriptions.subscribe(singleton(topicName), listener); + subscriptions.assignFromSubscribed(singleton(tp0)); + subscriptions.seek(tp0, 0); + + client.updateMetadata(RequestTestUtils.metadataUpdateWithIds( + 1, singletonMap(topicName, 4), tp -> validLeaderEpoch, topicIds)); + + assertEquals(1, sendFetches()); + + // Now the eager rebalance happens and fetch positions are cleared + subscriptions.assignFromSubscribed(Collections.emptyList()); + + subscriptions.assignFromSubscribed(singleton(tp0)); + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + consumerClient.poll(time.timer(0)); + + // The active fetch should be ignored since its position is no longer valid + assertTrue(fetchedRecords().isEmpty()); + } + + @Test + public void testFetchDuringCooperativeRebalance() { + buildFetcher(); + + subscriptions.subscribe(singleton(topicName), listener); + subscriptions.assignFromSubscribed(singleton(tp0)); + subscriptions.seek(tp0, 0); + + client.updateMetadata(RequestTestUtils.metadataUpdateWithIds( + 1, singletonMap(topicName, 4), tp -> validLeaderEpoch, topicIds)); + + assertEquals(1, sendFetches()); + + // Now the cooperative rebalance happens and fetch positions are NOT cleared for unrevoked partitions + subscriptions.assignFromSubscribed(singleton(tp0)); + + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + consumerClient.poll(time.timer(0)); + + Map>> fetchedRecords = fetchedRecords(); + + // The active fetch should NOT be ignored since the position for tp0 is still valid + assertEquals(1, fetchedRecords.size()); + assertEquals(3, fetchedRecords.get(tp0).size()); + } + + @Test + public void testInFlightFetchOnPausedPartition() { + buildFetcher(); + + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + assertEquals(1, sendFetches()); + subscriptions.pause(tp0); + + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + consumerClient.poll(time.timer(0)); + assertNull(fetchedRecords().get(tp0)); + } + + @Test + public void testFetchOnPausedPartition() { + buildFetcher(); + + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + subscriptions.pause(tp0); + assertFalse(sendFetches() > 0); + assertTrue(client.requests().isEmpty()); + } + + @Test + public void testFetchOnCompletedFetchesForPausedAndResumedPartitions() { + buildFetcher(); + + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + assertEquals(1, sendFetches()); + + subscriptions.pause(tp0); + + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + consumerClient.poll(time.timer(0)); + + assertEmptyFetch("Should not return any records or advance position when partition is paused"); + + assertTrue(fetcher.hasCompletedFetches(), "Should still contain completed fetches"); + assertFalse(fetcher.hasAvailableFetches(), "Should not have any available (non-paused) completed fetches"); + assertEquals(0, sendFetches()); + + subscriptions.resume(tp0); + + assertTrue(fetcher.hasAvailableFetches(), "Should have available (non-paused) completed fetches"); + + consumerClient.poll(time.timer(0)); + Map>> fetchedRecords = fetchedRecords(); + assertEquals(1, fetchedRecords.size(), "Should return records when partition is resumed"); + assertNotNull(fetchedRecords.get(tp0)); + assertEquals(3, fetchedRecords.get(tp0).size()); + + consumerClient.poll(time.timer(0)); + assertEmptyFetch("Should not return records or advance position after previously paused partitions are fetched"); + assertFalse(fetcher.hasCompletedFetches(), "Should no longer contain completed fetches"); + } + + @Test + public void testFetchOnCompletedFetchesForSomePausedPartitions() { + buildFetcher(); + + Map>> fetchedRecords; + + assignFromUser(mkSet(tp0, tp1)); + + // seek to tp0 and tp1 in two polls to generate 2 complete requests and responses + + // #1 seek, request, poll, response + subscriptions.seekUnvalidated(tp0, new SubscriptionState.FetchPosition(1, Optional.empty(), metadata.currentLeader(tp0))); + assertEquals(1, sendFetches()); + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + consumerClient.poll(time.timer(0)); + + // #2 seek, request, poll, response + subscriptions.seekUnvalidated(tp1, new SubscriptionState.FetchPosition(1, Optional.empty(), metadata.currentLeader(tp1))); + assertEquals(1, sendFetches()); + client.prepareResponse(fullFetchResponse(tidp1, this.nextRecords, Errors.NONE, 100L, 0)); + + subscriptions.pause(tp0); + consumerClient.poll(time.timer(0)); + + fetchedRecords = fetchedRecords(); + assertEquals(1, fetchedRecords.size(), "Should return completed fetch for unpaused partitions"); + assertTrue(fetcher.hasCompletedFetches(), "Should still contain completed fetches"); + assertNotNull(fetchedRecords.get(tp1)); + assertNull(fetchedRecords.get(tp0)); + + assertEmptyFetch("Should not return records or advance position for remaining paused partition"); + assertTrue(fetcher.hasCompletedFetches(), "Should still contain completed fetches"); + } + + @Test + public void testFetchOnCompletedFetchesForAllPausedPartitions() { + buildFetcher(); + + assignFromUser(mkSet(tp0, tp1)); + + // seek to tp0 and tp1 in two polls to generate 2 complete requests and responses + + // #1 seek, request, poll, response + subscriptions.seekUnvalidated(tp0, new SubscriptionState.FetchPosition(1, Optional.empty(), metadata.currentLeader(tp0))); + assertEquals(1, sendFetches()); + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + consumerClient.poll(time.timer(0)); + + // #2 seek, request, poll, response + subscriptions.seekUnvalidated(tp1, new SubscriptionState.FetchPosition(1, Optional.empty(), metadata.currentLeader(tp1))); + assertEquals(1, sendFetches()); + client.prepareResponse(fullFetchResponse(tidp1, this.nextRecords, Errors.NONE, 100L, 0)); + + subscriptions.pause(tp0); + subscriptions.pause(tp1); + + consumerClient.poll(time.timer(0)); + + assertEmptyFetch("Should not return records or advance position for all paused partitions"); + assertTrue(fetcher.hasCompletedFetches(), "Should still contain completed fetches"); + assertFalse(fetcher.hasAvailableFetches(), "Should not have any available (non-paused) completed fetches"); + } + + @Test + public void testPartialFetchWithPausedPartitions() { + // this test sends creates a completed fetch with 3 records and a max poll of 2 records to assert + // that a fetch that must be returned over at least 2 polls can be cached successfully when its partition is + // paused, then returned successfully after its been resumed again later + buildFetcher(2); + + Map>> fetchedRecords; + + assignFromUser(mkSet(tp0, tp1)); + + subscriptions.seek(tp0, 1); + assertEquals(1, sendFetches()); + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + consumerClient.poll(time.timer(0)); + + fetchedRecords = fetchedRecords(); + + assertEquals(2, fetchedRecords.get(tp0).size(), "Should return 2 records from fetch with 3 records"); + assertFalse(fetcher.hasCompletedFetches(), "Should have no completed fetches"); + + subscriptions.pause(tp0); + consumerClient.poll(time.timer(0)); + + fetchedRecords = fetchedRecords(); + + assertEmptyFetch("Should not return records or advance position for paused partitions"); + assertTrue(fetcher.hasCompletedFetches(), "Should have 1 entry in completed fetches"); + assertFalse(fetcher.hasAvailableFetches(), "Should not have any available (non-paused) completed fetches"); + + subscriptions.resume(tp0); + + consumerClient.poll(time.timer(0)); + + fetchedRecords = fetchedRecords(); + + assertEquals(1, fetchedRecords.get(tp0).size(), "Should return last remaining record"); + assertFalse(fetcher.hasCompletedFetches(), "Should have no completed fetches"); + } + + @Test + public void testFetchDiscardedAfterPausedPartitionResumedAndSeekedToNewOffset() { + buildFetcher(); + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + assertEquals(1, sendFetches()); + subscriptions.pause(tp0); + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + + subscriptions.seek(tp0, 3); + subscriptions.resume(tp0); + consumerClient.poll(time.timer(0)); + + assertTrue(fetcher.hasCompletedFetches(), "Should have 1 entry in completed fetches"); + Fetch fetch = collectFetch(); + assertEquals(emptyMap(), fetch.records(), "Should not return any records because we seeked to a new offset"); + assertFalse(fetch.positionAdvanced()); + assertFalse(fetcher.hasCompletedFetches(), "Should have no completed fetches"); + } + + @Test + public void testFetchNotLeaderOrFollower() { + buildFetcher(); + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + assertEquals(1, sendFetches()); + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NOT_LEADER_OR_FOLLOWER, 100L, 0)); + consumerClient.poll(time.timer(0)); + assertEmptyFetch("Should not return records or advance position on fetch error"); + assertEquals(0L, metadata.timeToNextUpdate(time.milliseconds())); + } + + @Test + public void testFetchUnknownTopicOrPartition() { + buildFetcher(); + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + assertEquals(1, sendFetches()); + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.UNKNOWN_TOPIC_OR_PARTITION, 100L, 0)); + consumerClient.poll(time.timer(0)); + assertEmptyFetch("Should not return records or advance position on fetch error"); + assertEquals(0L, metadata.timeToNextUpdate(time.milliseconds())); + } + + @Test + public void testFetchUnknownTopicId() { + buildFetcher(); + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + assertEquals(1, sendFetches()); + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.UNKNOWN_TOPIC_ID, -1L, 0)); + consumerClient.poll(time.timer(0)); + assertEmptyFetch("Should not return records or advance position on fetch error"); + assertEquals(0L, metadata.timeToNextUpdate(time.milliseconds())); + } + + @Test + public void testFetchSessionIdError() { + buildFetcher(); + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + assertEquals(1, sendFetches()); + client.prepareResponse(fetchResponseWithTopLevelError(tidp0, Errors.FETCH_SESSION_TOPIC_ID_ERROR, 0)); + consumerClient.poll(time.timer(0)); + assertEmptyFetch("Should not return records or advance position on fetch error"); + assertEquals(0L, metadata.timeToNextUpdate(time.milliseconds())); + } + + @Test + public void testFetchInconsistentTopicId() { + buildFetcher(); + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + assertEquals(1, sendFetches()); + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.INCONSISTENT_TOPIC_ID, -1L, 0)); + consumerClient.poll(time.timer(0)); + assertEmptyFetch("Should not return records or advance position on fetch error"); + assertEquals(0L, metadata.timeToNextUpdate(time.milliseconds())); + } + + @Test + public void testFetchFencedLeaderEpoch() { + buildFetcher(); + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + assertEquals(1, sendFetches()); + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.FENCED_LEADER_EPOCH, 100L, 0)); + consumerClient.poll(time.timer(0)); + + assertEmptyFetch("Should not return records or advance position on fetch error"); + assertEquals(0L, metadata.timeToNextUpdate(time.milliseconds()), "Should have requested metadata update"); + } + + @Test + public void testFetchUnknownLeaderEpoch() { + buildFetcher(); + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + assertEquals(1, sendFetches()); + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.UNKNOWN_LEADER_EPOCH, 100L, 0)); + consumerClient.poll(time.timer(0)); + + assertEmptyFetch("Should not return records or advance position on fetch error"); + assertNotEquals(0L, metadata.timeToNextUpdate(time.milliseconds()), "Should not have requested metadata update"); + } + + @Test + public void testEpochSetInFetchRequest() { + buildFetcher(); + subscriptions.assignFromUser(singleton(tp0)); + MetadataResponse metadataResponse = RequestTestUtils.metadataUpdateWithIds("dummy", 1, + Collections.emptyMap(), Collections.singletonMap(topicName, 4), tp -> 99, topicIds); + client.updateMetadata(metadataResponse); + + subscriptions.seek(tp0, 10); + assertEquals(1, sendFetches()); + + // Check for epoch in outgoing request + MockClient.RequestMatcher matcher = body -> { + if (body instanceof FetchRequest) { + FetchRequest fetchRequest = (FetchRequest) body; + fetchRequest.fetchData(topicNames).values().forEach(partitionData -> { + assertTrue(partitionData.currentLeaderEpoch.isPresent(), "Expected Fetcher to set leader epoch in request"); + assertEquals(99, partitionData.currentLeaderEpoch.get().longValue(), "Expected leader epoch to match epoch from metadata update"); + }); + return true; + } else { + fail("Should have seen FetchRequest"); + return false; + } + }; + client.prepareResponse(matcher, fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + consumerClient.pollNoWakeup(); + } + + @Test + public void testFetchOffsetOutOfRange() { + buildFetcher(); + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + assertEquals(1, sendFetches()); + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.OFFSET_OUT_OF_RANGE, 100L, 0)); + consumerClient.poll(time.timer(0)); + assertEmptyFetch("Should not return records or advance position on fetch error"); + assertTrue(subscriptions.isOffsetResetNeeded(tp0)); + assertNull(subscriptions.validPosition(tp0)); + assertNull(subscriptions.position(tp0)); + } + + @Test + public void testStaleOutOfRangeError() { + // verify that an out of range error which arrives after a seek + // does not cause us to reset our position or throw an exception + buildFetcher(); + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + assertEquals(1, sendFetches()); + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.OFFSET_OUT_OF_RANGE, 100L, 0)); + subscriptions.seek(tp0, 1); + consumerClient.poll(time.timer(0)); + assertEmptyFetch("Should not return records or advance position on fetch error"); + assertFalse(subscriptions.isOffsetResetNeeded(tp0)); + assertEquals(1, subscriptions.position(tp0).offset); + } + + @Test + public void testFetchedRecordsAfterSeek() { + buildFetcher(OffsetResetStrategy.NONE, new ByteArrayDeserializer(), + new ByteArrayDeserializer(), 2, IsolationLevel.READ_UNCOMMITTED); + + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + assertTrue(sendFetches() > 0); + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.OFFSET_OUT_OF_RANGE, 100L, 0)); + consumerClient.poll(time.timer(0)); + assertFalse(subscriptions.isOffsetResetNeeded(tp0)); + subscriptions.seek(tp0, 2); + assertEmptyFetch("Should not return records or advance position after seeking to end of topic partition"); + } + + @Test + public void testFetchOffsetOutOfRangeException() { + buildFetcher(OffsetResetStrategy.NONE, new ByteArrayDeserializer(), + new ByteArrayDeserializer(), 2, IsolationLevel.READ_UNCOMMITTED); + + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + sendFetches(); + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.OFFSET_OUT_OF_RANGE, 100L, 0)); + consumerClient.poll(time.timer(0)); + + assertFalse(subscriptions.isOffsetResetNeeded(tp0)); + for (int i = 0; i < 2; i++) { + OffsetOutOfRangeException e = assertThrows(OffsetOutOfRangeException.class, () -> + fetcher.collectFetch()); + assertEquals(singleton(tp0), e.offsetOutOfRangePartitions().keySet()); + assertEquals(0L, e.offsetOutOfRangePartitions().get(tp0).longValue()); + } + } + + @Test + public void testFetchPositionAfterException() { + // verify the advancement in the next fetch offset equals to the number of fetched records when + // some fetched partitions cause Exception. This ensures that consumer won't lose record upon exception + buildFetcher(OffsetResetStrategy.NONE, new ByteArrayDeserializer(), + new ByteArrayDeserializer(), Integer.MAX_VALUE, IsolationLevel.READ_UNCOMMITTED); + assignFromUser(mkSet(tp0, tp1)); + subscriptions.seek(tp0, 1); + subscriptions.seek(tp1, 1); + + assertEquals(1, sendFetches()); + + Map partitions = new LinkedHashMap<>(); + partitions.put(tidp1, new FetchResponseData.PartitionData() + .setPartitionIndex(tp1.partition()) + .setHighWatermark(100) + .setRecords(records)); + partitions.put(tidp0, new FetchResponseData.PartitionData() + .setPartitionIndex(tp0.partition()) + .setErrorCode(Errors.OFFSET_OUT_OF_RANGE.code()) + .setHighWatermark(100)); + client.prepareResponse(FetchResponse.of(Errors.NONE, 0, INVALID_SESSION_ID, new LinkedHashMap<>(partitions))); + consumerClient.poll(time.timer(0)); + + List> allFetchedRecords = new ArrayList<>(); + fetchRecordsInto(allFetchedRecords); + + assertEquals(1, subscriptions.position(tp0).offset); + assertEquals(4, subscriptions.position(tp1).offset); + assertEquals(3, allFetchedRecords.size()); + + OffsetOutOfRangeException e = assertThrows(OffsetOutOfRangeException.class, () -> + fetchRecordsInto(allFetchedRecords)); + + assertEquals(singleton(tp0), e.offsetOutOfRangePartitions().keySet()); + assertEquals(1L, e.offsetOutOfRangePartitions().get(tp0).longValue()); + + assertEquals(1, subscriptions.position(tp0).offset); + assertEquals(4, subscriptions.position(tp1).offset); + assertEquals(3, allFetchedRecords.size()); + } + + private void fetchRecordsInto(List> allFetchedRecords) { + Map>> fetchedRecords = fetchedRecords(); + fetchedRecords.values().forEach(allFetchedRecords::addAll); + } + + @Test + public void testCompletedFetchRemoval() { + // Ensure the removal of completed fetches that cause an Exception if and only if they contain empty records. + buildFetcher(OffsetResetStrategy.NONE, new ByteArrayDeserializer(), + new ByteArrayDeserializer(), Integer.MAX_VALUE, IsolationLevel.READ_UNCOMMITTED); + assignFromUser(mkSet(tp0, tp1, tp2, tp3)); + + subscriptions.seek(tp0, 1); + subscriptions.seek(tp1, 1); + subscriptions.seek(tp2, 1); + subscriptions.seek(tp3, 1); + + assertEquals(1, sendFetches()); + + Map partitions = new LinkedHashMap<>(); + partitions.put(tidp1, new FetchResponseData.PartitionData() + .setPartitionIndex(tp1.partition()) + .setHighWatermark(100) + .setRecords(records)); + partitions.put(tidp0, new FetchResponseData.PartitionData() + .setPartitionIndex(tp0.partition()) + .setErrorCode(Errors.OFFSET_OUT_OF_RANGE.code()) + .setHighWatermark(100)); + partitions.put(tidp2, new FetchResponseData.PartitionData() + .setPartitionIndex(tp2.partition()) + .setHighWatermark(100) + .setLastStableOffset(4) + .setLogStartOffset(0) + .setRecords(nextRecords)); + partitions.put(tidp3, new FetchResponseData.PartitionData() + .setPartitionIndex(tp3.partition()) + .setHighWatermark(100) + .setLastStableOffset(4) + .setLogStartOffset(0) + .setRecords(partialRecords)); + client.prepareResponse(FetchResponse.of(Errors.NONE, 0, INVALID_SESSION_ID, new LinkedHashMap<>(partitions))); + consumerClient.poll(time.timer(0)); + + List> fetchedRecords = new ArrayList<>(); + Map>> recordsByPartition = fetchedRecords(); + for (List> records : recordsByPartition.values()) + fetchedRecords.addAll(records); + + assertEquals(fetchedRecords.size(), subscriptions.position(tp1).offset - 1); + assertEquals(4, subscriptions.position(tp1).offset); + assertEquals(3, fetchedRecords.size()); + + List oorExceptions = new ArrayList<>(); + try { + recordsByPartition = fetchedRecords(); + for (List> records : recordsByPartition.values()) + fetchedRecords.addAll(records); + } catch (OffsetOutOfRangeException oor) { + oorExceptions.add(oor); + } + + // Should have received one OffsetOutOfRangeException for partition tp1 + assertEquals(1, oorExceptions.size()); + OffsetOutOfRangeException oor = oorExceptions.get(0); + assertTrue(oor.offsetOutOfRangePartitions().containsKey(tp0)); + assertEquals(oor.offsetOutOfRangePartitions().size(), 1); + + recordsByPartition = fetchedRecords(); + for (List> records : recordsByPartition.values()) + fetchedRecords.addAll(records); + + // Should not have received an Exception for tp2. + assertEquals(6, subscriptions.position(tp2).offset); + assertEquals(5, fetchedRecords.size()); + + int numExceptionsExpected = 3; + List kafkaExceptions = new ArrayList<>(); + for (int i = 1; i <= numExceptionsExpected; i++) { + try { + recordsByPartition = fetchedRecords(); + for (List> records : recordsByPartition.values()) + fetchedRecords.addAll(records); + } catch (KafkaException e) { + kafkaExceptions.add(e); + } + } + // Should have received as much as numExceptionsExpected Kafka exceptions for tp3. + assertEquals(numExceptionsExpected, kafkaExceptions.size()); + } + + @Test + public void testSeekBeforeException() { + buildFetcher(OffsetResetStrategy.NONE, new ByteArrayDeserializer(), + new ByteArrayDeserializer(), 2, IsolationLevel.READ_UNCOMMITTED); + + assignFromUser(mkSet(tp0)); + subscriptions.seek(tp0, 1); + assertEquals(1, sendFetches()); + Map partitions = new HashMap<>(); + partitions.put(tidp0, new FetchResponseData.PartitionData() + .setPartitionIndex(tp0.partition()) + .setHighWatermark(100) + .setRecords(records)); + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + consumerClient.poll(time.timer(0)); + + assertEquals(2, fetchedRecords().get(tp0).size()); + + subscriptions.assignFromUser(mkSet(tp0, tp1)); + subscriptions.seekUnvalidated(tp1, new SubscriptionState.FetchPosition(1, Optional.empty(), metadata.currentLeader(tp1))); + + assertEquals(1, sendFetches()); + partitions = new HashMap<>(); + partitions.put(tidp1, new FetchResponseData.PartitionData() + .setPartitionIndex(tp1.partition()) + .setErrorCode(Errors.OFFSET_OUT_OF_RANGE.code()) + .setHighWatermark(100)); + client.prepareResponse(FetchResponse.of(Errors.NONE, 0, INVALID_SESSION_ID, new LinkedHashMap<>(partitions))); + consumerClient.poll(time.timer(0)); + assertEquals(1, fetchedRecords().get(tp0).size()); + + subscriptions.seek(tp1, 10); + // Should not throw OffsetOutOfRangeException after the seek + assertEmptyFetch("Should not return records or advance position after seeking to end of topic partitions"); + } + + @Test + public void testFetchDisconnected() { + buildFetcher(); + + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + assertEquals(1, sendFetches()); + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0), true); + consumerClient.poll(time.timer(0)); + assertEmptyFetch("Should not return records or advance position on disconnect"); + + // disconnects should have no affect on subscription state + assertFalse(subscriptions.isOffsetResetNeeded(tp0)); + assertTrue(subscriptions.isFetchable(tp0)); + assertEquals(0, subscriptions.position(tp0).offset); + } + + /* + * Send multiple requests. Verify that the client side quota metrics have the right values + */ + @Test + public void testQuotaMetrics() { + buildFetcher(); + + MockSelector selector = new MockSelector(time); + Cluster cluster = TestUtils.singletonCluster("test", 1); + Node node = cluster.nodes().get(0); + NetworkClient client = new NetworkClient(selector, metadata, "mock", Integer.MAX_VALUE, + 1000, 1000, 64 * 1024, 64 * 1024, 1000, 10 * 1000, 127 * 1000, + time, true, new ApiVersions(), metricsManager.throttleTimeSensor(), new LogContext()); + + ApiVersionsResponse apiVersionsResponse = TestUtils.defaultApiVersionsResponse( + 400, ApiMessageType.ListenerType.ZK_BROKER); + ByteBuffer buffer = RequestTestUtils.serializeResponseWithHeader(apiVersionsResponse, ApiKeys.API_VERSIONS.latestVersion(), 0); + + selector.delayedReceive(new DelayedReceive(node.idString(), new NetworkReceive(node.idString(), buffer))); + while (!client.ready(node, time.milliseconds())) { + client.poll(1, time.milliseconds()); + // If a throttled response is received, advance the time to ensure progress. + time.sleep(client.throttleDelayMs(node, time.milliseconds())); + } + selector.clear(); + + for (int i = 1; i <= 3; i++) { + int throttleTimeMs = 100 * i; + FetchRequest.Builder builder = FetchRequest.Builder.forConsumer(ApiKeys.FETCH.latestVersion(), 100, 100, new LinkedHashMap<>()); + builder.rackId(""); + ClientRequest request = client.newClientRequest(node.idString(), builder, time.milliseconds(), true); + client.send(request, time.milliseconds()); + client.poll(1, time.milliseconds()); + FetchResponse response = fullFetchResponse(tidp0, nextRecords, Errors.NONE, i, throttleTimeMs); + buffer = RequestTestUtils.serializeResponseWithHeader(response, ApiKeys.FETCH.latestVersion(), request.correlationId()); + selector.completeReceive(new NetworkReceive(node.idString(), buffer)); + client.poll(1, time.milliseconds()); + // If a throttled response is received, advance the time to ensure progress. + time.sleep(client.throttleDelayMs(node, time.milliseconds())); + selector.clear(); + } + Map allMetrics = metrics.metrics(); + KafkaMetric avgMetric = allMetrics.get(metrics.metricInstance(metricsRegistry.fetchThrottleTimeAvg)); + KafkaMetric maxMetric = allMetrics.get(metrics.metricInstance(metricsRegistry.fetchThrottleTimeMax)); + // Throttle times are ApiVersions=400, Fetch=(100, 200, 300) + assertEquals(250, (Double) avgMetric.metricValue(), EPSILON); + assertEquals(400, (Double) maxMetric.metricValue(), EPSILON); + client.close(); + } + + /* + * Send multiple requests. Verify that the client side quota metrics have the right values + */ + @Test + public void testFetcherMetrics() { + buildFetcher(); + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + MetricName maxLagMetric = metrics.metricInstance(metricsRegistry.recordsLagMax); + Map tags = new HashMap<>(); + tags.put("topic", tp0.topic()); + tags.put("partition", String.valueOf(tp0.partition())); + MetricName partitionLagMetric = metrics.metricName("records-lag", metricGroup, tags); + + Map allMetrics = metrics.metrics(); + KafkaMetric recordsFetchLagMax = allMetrics.get(maxLagMetric); + + // recordsFetchLagMax should be initialized to NaN + assertEquals(Double.NaN, (Double) recordsFetchLagMax.metricValue(), EPSILON); + + // recordsFetchLagMax should be hw - fetchOffset after receiving an empty FetchResponse + fetchRecords(tidp0, MemoryRecords.EMPTY, Errors.NONE, 100L, 0); + assertEquals(100, (Double) recordsFetchLagMax.metricValue(), EPSILON); + + KafkaMetric partitionLag = allMetrics.get(partitionLagMetric); + assertEquals(100, (Double) partitionLag.metricValue(), EPSILON); + + // recordsFetchLagMax should be hw - offset of the last message after receiving a non-empty FetchResponse + MemoryRecordsBuilder builder = MemoryRecords.builder(ByteBuffer.allocate(1024), CompressionType.NONE, + TimestampType.CREATE_TIME, 0L); + for (int v = 0; v < 3; v++) + builder.appendWithOffset(v, RecordBatch.NO_TIMESTAMP, "key".getBytes(), ("value-" + v).getBytes()); + fetchRecords(tidp0, builder.build(), Errors.NONE, 200L, 0); + assertEquals(197, (Double) recordsFetchLagMax.metricValue(), EPSILON); + assertEquals(197, (Double) partitionLag.metricValue(), EPSILON); + + // verify de-registration of partition lag + subscriptions.unsubscribe(); + sendFetches(); + assertFalse(allMetrics.containsKey(partitionLagMetric)); + } + + @Test + public void testFetcherLeadMetric() { + buildFetcher(); + + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + MetricName minLeadMetric = metrics.metricInstance(metricsRegistry.recordsLeadMin); + Map tags = new HashMap<>(2); + tags.put("topic", tp0.topic()); + tags.put("partition", String.valueOf(tp0.partition())); + MetricName partitionLeadMetric = metrics.metricName("records-lead", metricGroup, "", tags); + + Map allMetrics = metrics.metrics(); + KafkaMetric recordsFetchLeadMin = allMetrics.get(minLeadMetric); + + // recordsFetchLeadMin should be initialized to NaN + assertEquals(Double.NaN, (Double) recordsFetchLeadMin.metricValue(), EPSILON); + + // recordsFetchLeadMin should be position - logStartOffset after receiving an empty FetchResponse + fetchRecords(tidp0, MemoryRecords.EMPTY, Errors.NONE, 100L, -1L, 0L, 0); + assertEquals(0L, (Double) recordsFetchLeadMin.metricValue(), EPSILON); + + KafkaMetric partitionLead = allMetrics.get(partitionLeadMetric); + assertEquals(0L, (Double) partitionLead.metricValue(), EPSILON); + + // recordsFetchLeadMin should be position - logStartOffset after receiving a non-empty FetchResponse + MemoryRecordsBuilder builder = MemoryRecords.builder(ByteBuffer.allocate(1024), CompressionType.NONE, + TimestampType.CREATE_TIME, 0L); + for (int v = 0; v < 3; v++) { + builder.appendWithOffset(v, RecordBatch.NO_TIMESTAMP, "key".getBytes(), ("value-" + v).getBytes()); + } + fetchRecords(tidp0, builder.build(), Errors.NONE, 200L, -1L, 0L, 0); + assertEquals(0L, (Double) recordsFetchLeadMin.metricValue(), EPSILON); + assertEquals(3L, (Double) partitionLead.metricValue(), EPSILON); + + // verify de-registration of partition lag + subscriptions.unsubscribe(); + sendFetches(); + assertFalse(allMetrics.containsKey(partitionLeadMetric)); + } + + @Test + public void testReadCommittedLagMetric() { + buildFetcher(OffsetResetStrategy.EARLIEST, new ByteArrayDeserializer(), + new ByteArrayDeserializer(), Integer.MAX_VALUE, IsolationLevel.READ_COMMITTED); + + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + MetricName maxLagMetric = metrics.metricInstance(metricsRegistry.recordsLagMax); + + Map tags = new HashMap<>(); + tags.put("topic", tp0.topic()); + tags.put("partition", String.valueOf(tp0.partition())); + MetricName partitionLagMetric = metrics.metricName("records-lag", metricGroup, tags); + + Map allMetrics = metrics.metrics(); + KafkaMetric recordsFetchLagMax = allMetrics.get(maxLagMetric); + + // recordsFetchLagMax should be initialized to NaN + assertEquals(Double.NaN, (Double) recordsFetchLagMax.metricValue(), EPSILON); + + // recordsFetchLagMax should be lso - fetchOffset after receiving an empty FetchResponse + fetchRecords(tidp0, MemoryRecords.EMPTY, Errors.NONE, 100L, 50L, 0); + assertEquals(50, (Double) recordsFetchLagMax.metricValue(), EPSILON); + + KafkaMetric partitionLag = allMetrics.get(partitionLagMetric); + assertEquals(50, (Double) partitionLag.metricValue(), EPSILON); + + // recordsFetchLagMax should be lso - offset of the last message after receiving a non-empty FetchResponse + MemoryRecordsBuilder builder = MemoryRecords.builder(ByteBuffer.allocate(1024), CompressionType.NONE, + TimestampType.CREATE_TIME, 0L); + for (int v = 0; v < 3; v++) + builder.appendWithOffset(v, RecordBatch.NO_TIMESTAMP, "key".getBytes(), ("value-" + v).getBytes()); + fetchRecords(tidp0, builder.build(), Errors.NONE, 200L, 150L, 0); + assertEquals(147, (Double) recordsFetchLagMax.metricValue(), EPSILON); + assertEquals(147, (Double) partitionLag.metricValue(), EPSILON); + + // verify de-registration of partition lag + subscriptions.unsubscribe(); + sendFetches(); + assertFalse(allMetrics.containsKey(partitionLagMetric)); + } + + @Test + public void testFetchResponseMetrics() { + buildFetcher(); + + String topic1 = "foo"; + String topic2 = "bar"; + TopicPartition tp1 = new TopicPartition(topic1, 0); + TopicPartition tp2 = new TopicPartition(topic2, 0); + + subscriptions.assignFromUser(mkSet(tp1, tp2)); + + Map partitionCounts = new HashMap<>(); + partitionCounts.put(topic1, 1); + partitionCounts.put(topic2, 1); + topicIds.put(topic1, Uuid.randomUuid()); + topicIds.put(topic2, Uuid.randomUuid()); + TopicIdPartition tidp1 = new TopicIdPartition(topicIds.get(topic1), tp1); + TopicIdPartition tidp2 = new TopicIdPartition(topicIds.get(topic2), tp2); + client.updateMetadata(RequestTestUtils.metadataUpdateWithIds(1, partitionCounts, tp -> validLeaderEpoch, topicIds)); + + int expectedBytes = 0; + LinkedHashMap fetchPartitionData = new LinkedHashMap<>(); + + for (TopicIdPartition tp : mkSet(tidp1, tidp2)) { + subscriptions.seek(tp.topicPartition(), 0); + + MemoryRecordsBuilder builder = MemoryRecords.builder(ByteBuffer.allocate(1024), CompressionType.NONE, + TimestampType.CREATE_TIME, 0L); + for (int v = 0; v < 3; v++) + builder.appendWithOffset(v, RecordBatch.NO_TIMESTAMP, "key".getBytes(), ("value-" + v).getBytes()); + MemoryRecords records = builder.build(); + for (Record record : records.records()) + expectedBytes += record.sizeInBytes(); + + fetchPartitionData.put(tp, new FetchResponseData.PartitionData() + .setPartitionIndex(tp.topicPartition().partition()) + .setHighWatermark(15) + .setLogStartOffset(0) + .setRecords(records)); + } + + assertEquals(1, sendFetches()); + client.prepareResponse(FetchResponse.of(Errors.NONE, 0, INVALID_SESSION_ID, fetchPartitionData)); + consumerClient.poll(time.timer(0)); + + Map>> fetchedRecords = fetchedRecords(); + assertEquals(3, fetchedRecords.get(tp1).size()); + assertEquals(3, fetchedRecords.get(tp2).size()); + + Map allMetrics = metrics.metrics(); + KafkaMetric fetchSizeAverage = allMetrics.get(metrics.metricInstance(metricsRegistry.fetchSizeAvg)); + KafkaMetric recordsCountAverage = allMetrics.get(metrics.metricInstance(metricsRegistry.recordsPerRequestAvg)); + assertEquals(expectedBytes, (Double) fetchSizeAverage.metricValue(), EPSILON); + assertEquals(6, (Double) recordsCountAverage.metricValue(), EPSILON); + } + + @Test + public void testFetchResponseMetricsPartialResponse() { + buildFetcher(); + + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 1); + + Map allMetrics = metrics.metrics(); + KafkaMetric fetchSizeAverage = allMetrics.get(metrics.metricInstance(metricsRegistry.fetchSizeAvg)); + KafkaMetric recordsCountAverage = allMetrics.get(metrics.metricInstance(metricsRegistry.recordsPerRequestAvg)); + + MemoryRecordsBuilder builder = MemoryRecords.builder(ByteBuffer.allocate(1024), CompressionType.NONE, + TimestampType.CREATE_TIME, 0L); + for (int v = 0; v < 3; v++) + builder.appendWithOffset(v, RecordBatch.NO_TIMESTAMP, "key".getBytes(), ("value-" + v).getBytes()); + MemoryRecords records = builder.build(); + + int expectedBytes = 0; + for (Record record : records.records()) { + if (record.offset() >= 1) + expectedBytes += record.sizeInBytes(); + } + + fetchRecords(tidp0, records, Errors.NONE, 100L, 0); + assertEquals(expectedBytes, (Double) fetchSizeAverage.metricValue(), EPSILON); + assertEquals(2, (Double) recordsCountAverage.metricValue(), EPSILON); + } + + @Test + public void testFetchResponseMetricsWithOnePartitionError() { + buildFetcher(); + assignFromUser(mkSet(tp0, tp1)); + subscriptions.seek(tp0, 0); + subscriptions.seek(tp1, 0); + + Map allMetrics = metrics.metrics(); + KafkaMetric fetchSizeAverage = allMetrics.get(metrics.metricInstance(metricsRegistry.fetchSizeAvg)); + KafkaMetric recordsCountAverage = allMetrics.get(metrics.metricInstance(metricsRegistry.recordsPerRequestAvg)); + + MemoryRecordsBuilder builder = MemoryRecords.builder(ByteBuffer.allocate(1024), CompressionType.NONE, + TimestampType.CREATE_TIME, 0L); + for (int v = 0; v < 3; v++) + builder.appendWithOffset(v, RecordBatch.NO_TIMESTAMP, "key".getBytes(), ("value-" + v).getBytes()); + MemoryRecords records = builder.build(); + + Map partitions = new HashMap<>(); + partitions.put(tidp0, new FetchResponseData.PartitionData() + .setPartitionIndex(tp0.partition()) + .setHighWatermark(100) + .setLogStartOffset(0) + .setRecords(records)); + partitions.put(tidp1, new FetchResponseData.PartitionData() + .setPartitionIndex(tp1.partition()) + .setErrorCode(Errors.OFFSET_OUT_OF_RANGE.code()) + .setHighWatermark(100) + .setLogStartOffset(0)); + + assertEquals(1, sendFetches()); + client.prepareResponse(FetchResponse.of(Errors.NONE, 0, INVALID_SESSION_ID, new LinkedHashMap<>(partitions))); + consumerClient.poll(time.timer(0)); + fetcher.collectFetch(); + + int expectedBytes = 0; + for (Record record : records.records()) + expectedBytes += record.sizeInBytes(); + + assertEquals(expectedBytes, (Double) fetchSizeAverage.metricValue(), EPSILON); + assertEquals(3, (Double) recordsCountAverage.metricValue(), EPSILON); + } + + @Test + public void testFetchResponseMetricsWithOnePartitionAtTheWrongOffset() { + buildFetcher(); + + assignFromUser(mkSet(tp0, tp1)); + subscriptions.seek(tp0, 0); + subscriptions.seek(tp1, 0); + + Map allMetrics = metrics.metrics(); + KafkaMetric fetchSizeAverage = allMetrics.get(metrics.metricInstance(metricsRegistry.fetchSizeAvg)); + KafkaMetric recordsCountAverage = allMetrics.get(metrics.metricInstance(metricsRegistry.recordsPerRequestAvg)); + + // send the fetch and then seek to a new offset + assertEquals(1, sendFetches()); + subscriptions.seek(tp1, 5); + + MemoryRecordsBuilder builder = MemoryRecords.builder(ByteBuffer.allocate(1024), CompressionType.NONE, + TimestampType.CREATE_TIME, 0L); + for (int v = 0; v < 3; v++) + builder.appendWithOffset(v, RecordBatch.NO_TIMESTAMP, "key".getBytes(), ("value-" + v).getBytes()); + MemoryRecords records = builder.build(); + + Map partitions = new HashMap<>(); + partitions.put(tidp0, new FetchResponseData.PartitionData() + .setPartitionIndex(tp0.partition()) + .setHighWatermark(100) + .setLogStartOffset(0) + .setRecords(records)); + partitions.put(tidp1, new FetchResponseData.PartitionData() + .setPartitionIndex(tp1.partition()) + .setHighWatermark(100) + .setLogStartOffset(0) + .setRecords(MemoryRecords.withRecords(CompressionType.NONE, new SimpleRecord("val".getBytes())))); + + client.prepareResponse(FetchResponse.of(Errors.NONE, 0, INVALID_SESSION_ID, new LinkedHashMap<>(partitions))); + consumerClient.poll(time.timer(0)); + fetcher.collectFetch(); + + // we should have ignored the record at the wrong offset + int expectedBytes = 0; + for (Record record : records.records()) + expectedBytes += record.sizeInBytes(); + + assertEquals(expectedBytes, (Double) fetchSizeAverage.metricValue(), EPSILON); + assertEquals(3, (Double) recordsCountAverage.metricValue(), EPSILON); + } + + @Test + public void testFetcherMetricsTemplates() { + Map clientTags = Collections.singletonMap("client-id", "clientA"); + buildFetcher(new MetricConfig().tags(clientTags), OffsetResetStrategy.EARLIEST, new ByteArrayDeserializer(), + new ByteArrayDeserializer(), Integer.MAX_VALUE, IsolationLevel.READ_UNCOMMITTED); + + // Fetch from topic to generate topic metrics + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + assertEquals(1, sendFetches()); + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + Map>> partitionRecords = fetchedRecords(); + assertTrue(partitionRecords.containsKey(tp0)); + + // Verify that all metrics except metrics-count have registered templates + Set allMetrics = new HashSet<>(); + for (MetricName n : metrics.metrics().keySet()) { + String name = n.name().replaceAll(tp0.toString(), "{topic}-{partition}"); + if (!n.group().equals("kafka-metrics-count")) + allMetrics.add(new MetricNameTemplate(name, n.group(), "", n.tags().keySet())); + } + TestUtils.checkEquals(allMetrics, new HashSet<>(metricsRegistry.getAllTemplates()), "metrics", "templates"); + } + + private Map>> fetchRecords( + TopicIdPartition tp, MemoryRecords records, Errors error, long hw, int throttleTime) { + return fetchRecords(tp, records, error, hw, FetchResponse.INVALID_LAST_STABLE_OFFSET, throttleTime); + } + + private Map>> fetchRecords( + TopicIdPartition tp, MemoryRecords records, Errors error, long hw, long lastStableOffset, int throttleTime) { + assertEquals(1, sendFetches()); + client.prepareResponse(fullFetchResponse(tp, records, error, hw, lastStableOffset, throttleTime)); + consumerClient.poll(time.timer(0)); + return fetchedRecords(); + } + + private Map>> fetchRecords( + TopicIdPartition tp, MemoryRecords records, Errors error, long hw, long lastStableOffset, long logStartOffset, int throttleTime) { + assertEquals(1, sendFetches()); + client.prepareResponse(fetchResponse(tp, records, error, hw, lastStableOffset, logStartOffset, throttleTime)); + consumerClient.poll(time.timer(0)); + return fetchedRecords(); + } + + @Test + public void testSkippingAbortedTransactions() { + buildFetcher(OffsetResetStrategy.EARLIEST, new ByteArrayDeserializer(), + new ByteArrayDeserializer(), Integer.MAX_VALUE, IsolationLevel.READ_COMMITTED); + ByteBuffer buffer = ByteBuffer.allocate(1024); + int currentOffset = 0; + + currentOffset += appendTransactionalRecords(buffer, 1L, currentOffset, + new SimpleRecord(time.milliseconds(), "key".getBytes(), "value".getBytes()), + new SimpleRecord(time.milliseconds(), "key".getBytes(), "value".getBytes())); + + abortTransaction(buffer, 1L, currentOffset); + + buffer.flip(); + + List abortedTransactions = Collections.singletonList( + new FetchResponseData.AbortedTransaction().setProducerId(1).setFirstOffset(0)); + MemoryRecords records = MemoryRecords.readableRecords(buffer); + assignFromUser(singleton(tp0)); + + subscriptions.seek(tp0, 0); + + // normal fetch + assertEquals(1, sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + + client.prepareResponse(fullFetchResponseWithAbortedTransactions(records, abortedTransactions, Errors.NONE, 100L, 100L, 0)); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + + Fetch fetch = collectFetch(); + assertEquals(emptyMap(), fetch.records()); + assertTrue(fetch.positionAdvanced()); + } + + @Test + public void testReturnCommittedTransactions() { + buildFetcher(OffsetResetStrategy.EARLIEST, new ByteArrayDeserializer(), + new ByteArrayDeserializer(), Integer.MAX_VALUE, IsolationLevel.READ_COMMITTED); + ByteBuffer buffer = ByteBuffer.allocate(1024); + int currentOffset = 0; + + currentOffset += appendTransactionalRecords(buffer, 1L, currentOffset, + new SimpleRecord(time.milliseconds(), "key".getBytes(), "value".getBytes()), + new SimpleRecord(time.milliseconds(), "key".getBytes(), "value".getBytes())); + + commitTransaction(buffer, 1L, currentOffset); + buffer.flip(); + + MemoryRecords records = MemoryRecords.readableRecords(buffer); + assignFromUser(singleton(tp0)); + + subscriptions.seek(tp0, 0); + + // normal fetch + assertEquals(1, sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + client.prepareResponse(body -> { + FetchRequest request = (FetchRequest) body; + assertEquals(IsolationLevel.READ_COMMITTED, request.isolationLevel()); + return true; + }, fullFetchResponseWithAbortedTransactions(records, Collections.emptyList(), Errors.NONE, 100L, 100L, 0)); + + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + + Map>> fetchedRecords = fetchedRecords(); + assertTrue(fetchedRecords.containsKey(tp0)); + assertEquals(fetchedRecords.get(tp0).size(), 2); + } + + @Test + public void testReadCommittedWithCommittedAndAbortedTransactions() { + buildFetcher(OffsetResetStrategy.EARLIEST, new ByteArrayDeserializer(), + new ByteArrayDeserializer(), Integer.MAX_VALUE, IsolationLevel.READ_COMMITTED); + ByteBuffer buffer = ByteBuffer.allocate(1024); + + List abortedTransactions = new ArrayList<>(); + + long pid1 = 1L; + long pid2 = 2L; + + // Appends for producer 1 (eventually committed) + appendTransactionalRecords(buffer, pid1, 0L, + new SimpleRecord("commit1-1".getBytes(), "value".getBytes()), + new SimpleRecord("commit1-2".getBytes(), "value".getBytes())); + + // Appends for producer 2 (eventually aborted) + appendTransactionalRecords(buffer, pid2, 2L, + new SimpleRecord("abort2-1".getBytes(), "value".getBytes())); + + // commit producer 1 + commitTransaction(buffer, pid1, 3L); + + // append more for producer 2 (eventually aborted) + appendTransactionalRecords(buffer, pid2, 4L, + new SimpleRecord("abort2-2".getBytes(), "value".getBytes())); + + // abort producer 2 + abortTransaction(buffer, pid2, 5L); + abortedTransactions.add(new FetchResponseData.AbortedTransaction().setProducerId(pid2).setFirstOffset(2L)); + + // New transaction for producer 1 (eventually aborted) + appendTransactionalRecords(buffer, pid1, 6L, + new SimpleRecord("abort1-1".getBytes(), "value".getBytes())); + + // New transaction for producer 2 (eventually committed) + appendTransactionalRecords(buffer, pid2, 7L, + new SimpleRecord("commit2-1".getBytes(), "value".getBytes())); + + // Add messages for producer 1 (eventually aborted) + appendTransactionalRecords(buffer, pid1, 8L, + new SimpleRecord("abort1-2".getBytes(), "value".getBytes())); + + // abort producer 1 + abortTransaction(buffer, pid1, 9L); + abortedTransactions.add(new FetchResponseData.AbortedTransaction().setProducerId(1).setFirstOffset(6)); + + // commit producer 2 + commitTransaction(buffer, pid2, 10L); + + buffer.flip(); + + MemoryRecords records = MemoryRecords.readableRecords(buffer); + assignFromUser(singleton(tp0)); + + subscriptions.seek(tp0, 0); + + // normal fetch + assertEquals(1, sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + + client.prepareResponse(fullFetchResponseWithAbortedTransactions(records, abortedTransactions, Errors.NONE, 100L, 100L, 0)); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + + Map>> fetchedRecords = fetchedRecords(); + assertTrue(fetchedRecords.containsKey(tp0)); + // There are only 3 committed records + List> fetchedConsumerRecords = fetchedRecords.get(tp0); + Set fetchedKeys = new HashSet<>(); + for (ConsumerRecord consumerRecord : fetchedConsumerRecords) { + fetchedKeys.add(new String(consumerRecord.key(), StandardCharsets.UTF_8)); + } + assertEquals(mkSet("commit1-1", "commit1-2", "commit2-1"), fetchedKeys); + } + + @Test + public void testMultipleAbortMarkers() { + buildFetcher(OffsetResetStrategy.EARLIEST, new ByteArrayDeserializer(), + new ByteArrayDeserializer(), Integer.MAX_VALUE, IsolationLevel.READ_COMMITTED); + ByteBuffer buffer = ByteBuffer.allocate(1024); + int currentOffset = 0; + + currentOffset += appendTransactionalRecords(buffer, 1L, currentOffset, + new SimpleRecord(time.milliseconds(), "abort1-1".getBytes(), "value".getBytes()), + new SimpleRecord(time.milliseconds(), "abort1-2".getBytes(), "value".getBytes())); + + currentOffset += abortTransaction(buffer, 1L, currentOffset); + // Duplicate abort -- should be ignored. + currentOffset += abortTransaction(buffer, 1L, currentOffset); + // Now commit a transaction. + currentOffset += appendTransactionalRecords(buffer, 1L, currentOffset, + new SimpleRecord(time.milliseconds(), "commit1-1".getBytes(), "value".getBytes()), + new SimpleRecord(time.milliseconds(), "commit1-2".getBytes(), "value".getBytes())); + commitTransaction(buffer, 1L, currentOffset); + buffer.flip(); + + List abortedTransactions = Collections.singletonList( + new FetchResponseData.AbortedTransaction().setProducerId(1).setFirstOffset(0) + ); + MemoryRecords records = MemoryRecords.readableRecords(buffer); + assignFromUser(singleton(tp0)); + + subscriptions.seek(tp0, 0); + + // normal fetch + assertEquals(1, sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + + client.prepareResponse(fullFetchResponseWithAbortedTransactions(records, abortedTransactions, Errors.NONE, 100L, 100L, 0)); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + + Map>> fetchedRecords = fetchedRecords(); + assertTrue(fetchedRecords.containsKey(tp0)); + assertEquals(fetchedRecords.get(tp0).size(), 2); + List> fetchedConsumerRecords = fetchedRecords.get(tp0); + Set committedKeys = new HashSet<>(Arrays.asList("commit1-1", "commit1-2")); + Set actuallyCommittedKeys = new HashSet<>(); + for (ConsumerRecord consumerRecord : fetchedConsumerRecords) { + actuallyCommittedKeys.add(new String(consumerRecord.key(), StandardCharsets.UTF_8)); + } + assertEquals(actuallyCommittedKeys, committedKeys); + } + + @Test + public void testReadCommittedAbortMarkerWithNoData() { + buildFetcher(OffsetResetStrategy.EARLIEST, new StringDeserializer(), + new StringDeserializer(), Integer.MAX_VALUE, IsolationLevel.READ_COMMITTED); + ByteBuffer buffer = ByteBuffer.allocate(1024); + + long producerId = 1L; + + abortTransaction(buffer, producerId, 5L); + + appendTransactionalRecords(buffer, producerId, 6L, + new SimpleRecord("6".getBytes(), null), + new SimpleRecord("7".getBytes(), null), + new SimpleRecord("8".getBytes(), null)); + + commitTransaction(buffer, producerId, 9L); + + buffer.flip(); + + // send the fetch + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + assertEquals(1, sendFetches()); + + // prepare the response. the aborted transactions begin at offsets which are no longer in the log + List abortedTransactions = Collections.singletonList( + new FetchResponseData.AbortedTransaction().setProducerId(producerId).setFirstOffset(0L)); + + client.prepareResponse(fullFetchResponseWithAbortedTransactions(MemoryRecords.readableRecords(buffer), + abortedTransactions, Errors.NONE, 100L, 100L, 0)); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + + Map>> allFetchedRecords = fetchedRecords(); + assertTrue(allFetchedRecords.containsKey(tp0)); + List> fetchedRecords = allFetchedRecords.get(tp0); + assertEquals(3, fetchedRecords.size()); + assertEquals(Arrays.asList(6L, 7L, 8L), collectRecordOffsets(fetchedRecords)); + } + + @Test + public void testUpdatePositionWithLastRecordMissingFromBatch() { + buildFetcher(); + + MemoryRecords records = MemoryRecords.withRecords(CompressionType.NONE, + new SimpleRecord("0".getBytes(), "v".getBytes()), + new SimpleRecord("1".getBytes(), "v".getBytes()), + new SimpleRecord("2".getBytes(), "v".getBytes()), + new SimpleRecord(null, "value".getBytes())); + + // Remove the last record to simulate compaction + MemoryRecords.FilterResult result = records.filterTo(tp0, new MemoryRecords.RecordFilter(0, 0) { + @Override + protected BatchRetentionResult checkBatchRetention(RecordBatch batch) { + return new BatchRetentionResult(BatchRetention.DELETE_EMPTY, false); + } + + @Override + protected boolean shouldRetainRecord(RecordBatch recordBatch, Record record) { + return record.key() != null; + } + }, ByteBuffer.allocate(1024), Integer.MAX_VALUE, BufferSupplier.NO_CACHING); + result.outputBuffer().flip(); + MemoryRecords compactedRecords = MemoryRecords.readableRecords(result.outputBuffer()); + + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + assertEquals(1, sendFetches()); + client.prepareResponse(fullFetchResponse(tidp0, compactedRecords, Errors.NONE, 100L, 0)); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + + Map>> allFetchedRecords = fetchedRecords(); + assertTrue(allFetchedRecords.containsKey(tp0)); + List> fetchedRecords = allFetchedRecords.get(tp0); + assertEquals(3, fetchedRecords.size()); + + for (int i = 0; i < 3; i++) { + assertEquals(Integer.toString(i), new String(fetchedRecords.get(i).key())); + } + + // The next offset should point to the next batch + assertEquals(4L, subscriptions.position(tp0).offset); + } + + @Test + public void testUpdatePositionOnEmptyBatch() { + buildFetcher(); + + long producerId = 1; + short producerEpoch = 0; + int sequence = 1; + long baseOffset = 37; + long lastOffset = 54; + int partitionLeaderEpoch = 7; + ByteBuffer buffer = ByteBuffer.allocate(DefaultRecordBatch.RECORD_BATCH_OVERHEAD); + DefaultRecordBatch.writeEmptyHeader(buffer, RecordBatch.CURRENT_MAGIC_VALUE, producerId, producerEpoch, + sequence, baseOffset, lastOffset, partitionLeaderEpoch, TimestampType.CREATE_TIME, + System.currentTimeMillis(), false, false); + buffer.flip(); + MemoryRecords recordsWithEmptyBatch = MemoryRecords.readableRecords(buffer); + + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + assertEquals(1, sendFetches()); + client.prepareResponse(fullFetchResponse(tidp0, recordsWithEmptyBatch, Errors.NONE, 100L, 0)); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + + Fetch fetch = collectFetch(); + assertEquals(emptyMap(), fetch.records()); + assertTrue(fetch.positionAdvanced()); + + // The next offset should point to the next batch + assertEquals(lastOffset + 1, subscriptions.position(tp0).offset); + } + + @Test + public void testReadCommittedWithCompactedTopic() { + buildFetcher(OffsetResetStrategy.EARLIEST, new StringDeserializer(), + new StringDeserializer(), Integer.MAX_VALUE, IsolationLevel.READ_COMMITTED); + ByteBuffer buffer = ByteBuffer.allocate(1024); + + long pid1 = 1L; + long pid2 = 2L; + long pid3 = 3L; + + appendTransactionalRecords(buffer, pid3, 3L, + new SimpleRecord("3".getBytes(), "value".getBytes()), + new SimpleRecord("4".getBytes(), "value".getBytes())); + + appendTransactionalRecords(buffer, pid2, 15L, + new SimpleRecord("15".getBytes(), "value".getBytes()), + new SimpleRecord("16".getBytes(), "value".getBytes()), + new SimpleRecord("17".getBytes(), "value".getBytes())); + + appendTransactionalRecords(buffer, pid1, 22L, + new SimpleRecord("22".getBytes(), "value".getBytes()), + new SimpleRecord("23".getBytes(), "value".getBytes())); + + abortTransaction(buffer, pid2, 28L); + + appendTransactionalRecords(buffer, pid3, 30L, + new SimpleRecord("30".getBytes(), "value".getBytes()), + new SimpleRecord("31".getBytes(), "value".getBytes()), + new SimpleRecord("32".getBytes(), "value".getBytes())); + + commitTransaction(buffer, pid3, 35L); + + appendTransactionalRecords(buffer, pid1, 39L, + new SimpleRecord("39".getBytes(), "value".getBytes()), + new SimpleRecord("40".getBytes(), "value".getBytes())); + + // transaction from pid1 is aborted, but the marker is not included in the fetch + + buffer.flip(); + + // send the fetch + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + assertEquals(1, sendFetches()); + + // prepare the response. the aborted transactions begin at offsets which are no longer in the log + List abortedTransactions = Arrays.asList( + new FetchResponseData.AbortedTransaction().setProducerId(pid2).setFirstOffset(6), + new FetchResponseData.AbortedTransaction().setProducerId(pid1).setFirstOffset(0) + ); + + client.prepareResponse(fullFetchResponseWithAbortedTransactions(MemoryRecords.readableRecords(buffer), + abortedTransactions, Errors.NONE, 100L, 100L, 0)); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + + Map>> allFetchedRecords = fetchedRecords(); + assertTrue(allFetchedRecords.containsKey(tp0)); + List> fetchedRecords = allFetchedRecords.get(tp0); + assertEquals(5, fetchedRecords.size()); + assertEquals(Arrays.asList(3L, 4L, 30L, 31L, 32L), collectRecordOffsets(fetchedRecords)); + } + + @Test + public void testReturnAbortedTransactionsinUncommittedMode() { + buildFetcher(OffsetResetStrategy.EARLIEST, new ByteArrayDeserializer(), + new ByteArrayDeserializer(), Integer.MAX_VALUE, IsolationLevel.READ_UNCOMMITTED); + ByteBuffer buffer = ByteBuffer.allocate(1024); + int currentOffset = 0; + + currentOffset += appendTransactionalRecords(buffer, 1L, currentOffset, + new SimpleRecord(time.milliseconds(), "key".getBytes(), "value".getBytes()), + new SimpleRecord(time.milliseconds(), "key".getBytes(), "value".getBytes())); + + abortTransaction(buffer, 1L, currentOffset); + + buffer.flip(); + + List abortedTransactions = Collections.singletonList( + new FetchResponseData.AbortedTransaction().setProducerId(1).setFirstOffset(0)); + MemoryRecords records = MemoryRecords.readableRecords(buffer); + assignFromUser(singleton(tp0)); + + subscriptions.seek(tp0, 0); + + // normal fetch + assertEquals(1, sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + + client.prepareResponse(fullFetchResponseWithAbortedTransactions(records, abortedTransactions, Errors.NONE, 100L, 100L, 0)); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + + Map>> fetchedRecords = fetchedRecords(); + assertTrue(fetchedRecords.containsKey(tp0)); + } + + @Test + public void testConsumerPositionUpdatedWhenSkippingAbortedTransactions() { + buildFetcher(OffsetResetStrategy.EARLIEST, new ByteArrayDeserializer(), + new ByteArrayDeserializer(), Integer.MAX_VALUE, IsolationLevel.READ_COMMITTED); + ByteBuffer buffer = ByteBuffer.allocate(1024); + long currentOffset = 0; + + currentOffset += appendTransactionalRecords(buffer, 1L, currentOffset, + new SimpleRecord(time.milliseconds(), "abort1-1".getBytes(), "value".getBytes()), + new SimpleRecord(time.milliseconds(), "abort1-2".getBytes(), "value".getBytes())); + + currentOffset += abortTransaction(buffer, 1L, currentOffset); + buffer.flip(); + + List abortedTransactions = Collections.singletonList( + new FetchResponseData.AbortedTransaction().setProducerId(1).setFirstOffset(0)); + MemoryRecords records = MemoryRecords.readableRecords(buffer); + assignFromUser(singleton(tp0)); + + subscriptions.seek(tp0, 0); + + // normal fetch + assertEquals(1, sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + + client.prepareResponse(fullFetchResponseWithAbortedTransactions(records, abortedTransactions, Errors.NONE, 100L, 100L, 0)); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + + Map>> fetchedRecords = fetchedRecords(); + + // Ensure that we don't return any of the aborted records, but yet advance the consumer position. + assertFalse(fetchedRecords.containsKey(tp0)); + assertEquals(currentOffset, subscriptions.position(tp0).offset); + } + + @Test + public void testConsumingViaIncrementalFetchRequests() { + buildFetcher(2); + + List> records; + assignFromUser(new HashSet<>(Arrays.asList(tp0, tp1))); + subscriptions.seekValidated(tp0, new SubscriptionState.FetchPosition(0, Optional.empty(), metadata.currentLeader(tp0))); + subscriptions.seekValidated(tp1, new SubscriptionState.FetchPosition(1, Optional.empty(), metadata.currentLeader(tp1))); + + // Fetch some records and establish an incremental fetch session. + LinkedHashMap partitions1 = new LinkedHashMap<>(); + partitions1.put(tidp0, new FetchResponseData.PartitionData() + .setPartitionIndex(tp0.partition()) + .setHighWatermark(2) + .setLastStableOffset(2) + .setLogStartOffset(0) + .setRecords(this.records)); + partitions1.put(tidp1, new FetchResponseData.PartitionData() + .setPartitionIndex(tp1.partition()) + .setHighWatermark(100) + .setLogStartOffset(0) + .setRecords(emptyRecords)); + FetchResponse resp1 = FetchResponse.of(Errors.NONE, 0, 123, partitions1); + client.prepareResponse(resp1); + assertEquals(1, sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + Map>> fetchedRecords = fetchedRecords(); + assertFalse(fetchedRecords.containsKey(tp1)); + records = fetchedRecords.get(tp0); + assertEquals(2, records.size()); + assertEquals(3L, subscriptions.position(tp0).offset); + assertEquals(1L, subscriptions.position(tp1).offset); + assertEquals(1, records.get(0).offset()); + assertEquals(2, records.get(1).offset()); + + // There is still a buffered record. + assertEquals(0, sendFetches()); + fetchedRecords = fetchedRecords(); + assertFalse(fetchedRecords.containsKey(tp1)); + records = fetchedRecords.get(tp0); + assertEquals(1, records.size()); + assertEquals(3, records.get(0).offset()); + assertEquals(4L, subscriptions.position(tp0).offset); + + // The second response contains no new records. + LinkedHashMap partitions2 = new LinkedHashMap<>(); + FetchResponse resp2 = FetchResponse.of(Errors.NONE, 0, 123, partitions2); + client.prepareResponse(resp2); + assertEquals(1, sendFetches()); + consumerClient.poll(time.timer(0)); + fetchedRecords = fetchedRecords(); + assertTrue(fetchedRecords.isEmpty()); + assertEquals(4L, subscriptions.position(tp0).offset); + assertEquals(1L, subscriptions.position(tp1).offset); + + // The third response contains some new records for tp0. + LinkedHashMap partitions3 = new LinkedHashMap<>(); + partitions3.put(tidp0, new FetchResponseData.PartitionData() + .setPartitionIndex(tp0.partition()) + .setHighWatermark(100) + .setLastStableOffset(4) + .setLogStartOffset(0) + .setRecords(this.nextRecords)); + FetchResponse resp3 = FetchResponse.of(Errors.NONE, 0, 123, partitions3); + client.prepareResponse(resp3); + assertEquals(1, sendFetches()); + consumerClient.poll(time.timer(0)); + fetchedRecords = fetchedRecords(); + assertFalse(fetchedRecords.containsKey(tp1)); + records = fetchedRecords.get(tp0); + assertEquals(2, records.size()); + assertEquals(6L, subscriptions.position(tp0).offset); + assertEquals(1L, subscriptions.position(tp1).offset); + assertEquals(4, records.get(0).offset()); + assertEquals(5, records.get(1).offset()); + } + + @Test + public void testEmptyControlBatch() { + buildFetcher(OffsetResetStrategy.EARLIEST, new ByteArrayDeserializer(), + new ByteArrayDeserializer(), Integer.MAX_VALUE, IsolationLevel.READ_COMMITTED); + ByteBuffer buffer = ByteBuffer.allocate(1024); + int currentOffset = 1; + + // Empty control batch should not cause an exception + DefaultRecordBatch.writeEmptyHeader(buffer, RecordBatch.MAGIC_VALUE_V2, 1L, + (short) 0, -1, 0, 0, + RecordBatch.NO_PARTITION_LEADER_EPOCH, TimestampType.CREATE_TIME, time.milliseconds(), + true, true); + + currentOffset += appendTransactionalRecords(buffer, 1L, currentOffset, + new SimpleRecord(time.milliseconds(), "key".getBytes(), "value".getBytes()), + new SimpleRecord(time.milliseconds(), "key".getBytes(), "value".getBytes())); + + commitTransaction(buffer, 1L, currentOffset); + buffer.flip(); + + MemoryRecords records = MemoryRecords.readableRecords(buffer); + assignFromUser(singleton(tp0)); + + subscriptions.seek(tp0, 0); + + // normal fetch + assertEquals(1, sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + client.prepareResponse(body -> { + FetchRequest request = (FetchRequest) body; + assertEquals(IsolationLevel.READ_COMMITTED, request.isolationLevel()); + return true; + }, fullFetchResponseWithAbortedTransactions(records, Collections.emptyList(), Errors.NONE, 100L, 100L, 0)); + + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + + Map>> fetchedRecords = fetchedRecords(); + assertTrue(fetchedRecords.containsKey(tp0)); + assertEquals(fetchedRecords.get(tp0).size(), 2); + } + + private MemoryRecords buildRecords(long baseOffset, int count, long firstMessageId) { + MemoryRecordsBuilder builder = MemoryRecords.builder(ByteBuffer.allocate(1024), CompressionType.NONE, TimestampType.CREATE_TIME, baseOffset); + for (int i = 0; i < count; i++) + builder.append(0L, "key".getBytes(), ("value-" + (firstMessageId + i)).getBytes()); + return builder.build(); + } + + private int appendTransactionalRecords(ByteBuffer buffer, long pid, long baseOffset, int baseSequence, SimpleRecord... records) { + MemoryRecordsBuilder builder = MemoryRecords.builder(buffer, RecordBatch.CURRENT_MAGIC_VALUE, CompressionType.NONE, + TimestampType.CREATE_TIME, baseOffset, time.milliseconds(), pid, (short) 0, baseSequence, true, + RecordBatch.NO_PARTITION_LEADER_EPOCH); + + for (SimpleRecord record : records) { + builder.append(record); + } + builder.build(); + return records.length; + } + + private int appendTransactionalRecords(ByteBuffer buffer, long pid, long baseOffset, SimpleRecord... records) { + return appendTransactionalRecords(buffer, pid, baseOffset, (int) baseOffset, records); + } + + private void commitTransaction(ByteBuffer buffer, long producerId, long baseOffset) { + short producerEpoch = 0; + int partitionLeaderEpoch = 0; + MemoryRecords.writeEndTransactionalMarker(buffer, baseOffset, time.milliseconds(), partitionLeaderEpoch, producerId, producerEpoch, + new EndTransactionMarker(ControlRecordType.COMMIT, 0)); + } + + private int abortTransaction(ByteBuffer buffer, long producerId, long baseOffset) { + short producerEpoch = 0; + int partitionLeaderEpoch = 0; + MemoryRecords.writeEndTransactionalMarker(buffer, baseOffset, time.milliseconds(), partitionLeaderEpoch, producerId, producerEpoch, + new EndTransactionMarker(ControlRecordType.ABORT, 0)); + return 1; + } + + @Test + public void testSubscriptionPositionUpdatedWithEpoch() { + // Create some records that include a leader epoch (1) + MemoryRecordsBuilder builder = MemoryRecords.builder( + ByteBuffer.allocate(1024), + RecordBatch.CURRENT_MAGIC_VALUE, + CompressionType.NONE, + TimestampType.CREATE_TIME, + 0L, + RecordBatch.NO_TIMESTAMP, + RecordBatch.NO_PRODUCER_ID, + RecordBatch.NO_PRODUCER_EPOCH, + RecordBatch.NO_SEQUENCE, + false, + 1 + ); + builder.appendWithOffset(0L, 0L, "key".getBytes(), "value-1".getBytes()); + builder.appendWithOffset(1L, 0L, "key".getBytes(), "value-2".getBytes()); + builder.appendWithOffset(2L, 0L, "key".getBytes(), "value-3".getBytes()); + MemoryRecords records = builder.build(); + + buildFetcher(); + assignFromUser(singleton(tp0)); + + // Initialize the epoch=1 + Map partitionCounts = new HashMap<>(); + partitionCounts.put(tp0.topic(), 4); + MetadataResponse metadataResponse = RequestTestUtils.metadataUpdateWithIds("dummy", 1, Collections.emptyMap(), partitionCounts, tp -> 1, topicIds); + metadata.updateWithCurrentRequestVersion(metadataResponse, false, 0L); + + // Seek + subscriptions.seek(tp0, 0); + + // Do a normal fetch + assertEquals(1, sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); + consumerClient.pollNoWakeup(); + assertTrue(fetcher.hasCompletedFetches()); + + Map>> partitionRecords = fetchedRecords(); + assertTrue(partitionRecords.containsKey(tp0)); + + assertEquals(subscriptions.position(tp0).offset, 3L); + assertOptional(subscriptions.position(tp0).offsetEpoch, value -> assertEquals(value.intValue(), 1)); + } + + @Test + public void testPreferredReadReplica() { + buildFetcher(new MetricConfig(), OffsetResetStrategy.EARLIEST, new BytesDeserializer(), new BytesDeserializer(), + Integer.MAX_VALUE, IsolationLevel.READ_COMMITTED, Duration.ofMinutes(5).toMillis()); + + subscriptions.assignFromUser(singleton(tp0)); + client.updateMetadata(RequestTestUtils.metadataUpdateWithIds(2, singletonMap(topicName, 4), tp -> validLeaderEpoch, topicIds, false)); + subscriptions.seek(tp0, 0); + + // Node preferred replica before first fetch response + Node selected = fetcher.selectReadReplica(tp0, Node.noNode(), time.milliseconds()); + assertEquals(-1, selected.id()); + + assertEquals(1, sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + + // Set preferred read replica to node=1 + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, + FetchResponse.INVALID_LAST_STABLE_OFFSET, 0, Optional.of(1))); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + + Map>> partitionRecords = fetchedRecords(); + assertTrue(partitionRecords.containsKey(tp0)); + + // Verify + selected = fetcher.selectReadReplica(tp0, Node.noNode(), time.milliseconds()); + assertEquals(1, selected.id()); + + + assertEquals(1, sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + + // Set preferred read replica to node=2, which isn't in our metadata, should revert to leader + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, + FetchResponse.INVALID_LAST_STABLE_OFFSET, 0, Optional.of(2))); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + fetchedRecords(); + selected = fetcher.selectReadReplica(tp0, Node.noNode(), time.milliseconds()); + assertEquals(-1, selected.id()); + } + + @Test + public void testFetchDisconnectedShouldClearPreferredReadReplica() { + buildFetcher(new MetricConfig(), OffsetResetStrategy.EARLIEST, new BytesDeserializer(), new BytesDeserializer(), + Integer.MAX_VALUE, IsolationLevel.READ_COMMITTED, Duration.ofMinutes(5).toMillis()); + + subscriptions.assignFromUser(singleton(tp0)); + client.updateMetadata(RequestTestUtils.metadataUpdateWithIds(2, singletonMap(topicName, 4), tp -> validLeaderEpoch, topicIds, false)); + subscriptions.seek(tp0, 0); + assertEquals(1, sendFetches()); + + // Set preferred read replica to node=1 + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, + FetchResponse.INVALID_LAST_STABLE_OFFSET, 0, Optional.of(1))); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + fetchedRecords(); + + // Verify + Node selected = fetcher.selectReadReplica(tp0, Node.noNode(), time.milliseconds()); + assertEquals(1, selected.id()); + assertEquals(1, sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + + // Disconnect - preferred read replica should be cleared. + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0), true); + + consumerClient.poll(time.timer(0)); + assertFalse(fetcher.hasCompletedFetches()); + fetchedRecords(); + selected = fetcher.selectReadReplica(tp0, Node.noNode(), time.milliseconds()); + assertEquals(-1, selected.id()); + } + + @Test + public void testFetchDisconnectedShouldNotClearPreferredReadReplicaIfUnassigned() { + buildFetcher(new MetricConfig(), OffsetResetStrategy.EARLIEST, new BytesDeserializer(), new BytesDeserializer(), + Integer.MAX_VALUE, IsolationLevel.READ_COMMITTED, Duration.ofMinutes(5).toMillis()); + + subscriptions.assignFromUser(singleton(tp0)); + client.updateMetadata(RequestTestUtils.metadataUpdateWithIds(2, singletonMap(topicName, 4), tp -> validLeaderEpoch, topicIds, false)); + subscriptions.seek(tp0, 0); + assertEquals(1, sendFetches()); + + // Set preferred read replica to node=1 + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, + FetchResponse.INVALID_LAST_STABLE_OFFSET, 0, Optional.of(1))); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + fetchedRecords(); + + // Verify + Node selected = fetcher.selectReadReplica(tp0, Node.noNode(), time.milliseconds()); + assertEquals(1, selected.id()); + assertEquals(1, sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + + // Disconnect and remove tp0 from assignment + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0), true); + subscriptions.assignFromUser(emptySet()); + + // Preferred read replica should not be cleared + consumerClient.poll(time.timer(0)); + assertFalse(fetcher.hasCompletedFetches()); + fetchedRecords(); + selected = fetcher.selectReadReplica(tp0, Node.noNode(), time.milliseconds()); + assertEquals(-1, selected.id()); + } + + @Test + public void testFetchErrorShouldClearPreferredReadReplica() { + buildFetcher(new MetricConfig(), OffsetResetStrategy.EARLIEST, new BytesDeserializer(), new BytesDeserializer(), + Integer.MAX_VALUE, IsolationLevel.READ_COMMITTED, Duration.ofMinutes(5).toMillis()); + + subscriptions.assignFromUser(singleton(tp0)); + client.updateMetadata(RequestTestUtils.metadataUpdateWithIds(2, singletonMap(topicName, 4), tp -> validLeaderEpoch, topicIds, false)); + subscriptions.seek(tp0, 0); + assertEquals(1, sendFetches()); + + // Set preferred read replica to node=1 + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, + FetchResponse.INVALID_LAST_STABLE_OFFSET, 0, Optional.of(1))); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + fetchedRecords(); + + // Verify + Node selected = fetcher.selectReadReplica(tp0, Node.noNode(), time.milliseconds()); + assertEquals(1, selected.id()); + assertEquals(1, sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + + // Error - preferred read replica should be cleared. An actual error response will contain -1 as the + // preferred read replica. In the test we want to ensure that we are handling the error. + client.prepareResponse(fullFetchResponse(tidp0, MemoryRecords.EMPTY, Errors.NOT_LEADER_OR_FOLLOWER, -1L, + FetchResponse.INVALID_LAST_STABLE_OFFSET, 0, Optional.of(1))); + + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + fetchedRecords(); + selected = fetcher.selectReadReplica(tp0, Node.noNode(), time.milliseconds()); + assertEquals(-1, selected.id()); + } + + @Test + public void testPreferredReadReplicaOffsetError() { + buildFetcher(new MetricConfig(), OffsetResetStrategy.EARLIEST, new BytesDeserializer(), new BytesDeserializer(), + Integer.MAX_VALUE, IsolationLevel.READ_COMMITTED, Duration.ofMinutes(5).toMillis()); + + subscriptions.assignFromUser(singleton(tp0)); + client.updateMetadata(RequestTestUtils.metadataUpdateWithIds(2, singletonMap(topicName, 4), tp -> validLeaderEpoch, topicIds, false)); + + subscriptions.seek(tp0, 0); + + assertEquals(1, sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, + FetchResponse.INVALID_LAST_STABLE_OFFSET, 0, Optional.of(1))); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + + fetchedRecords(); + + Node selected = fetcher.selectReadReplica(tp0, Node.noNode(), time.milliseconds()); + assertEquals(selected.id(), 1); + + // Return an error, should unset the preferred read replica + assertEquals(1, sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + + client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.OFFSET_OUT_OF_RANGE, 100L, + FetchResponse.INVALID_LAST_STABLE_OFFSET, 0, Optional.empty())); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + + fetchedRecords(); + + selected = fetcher.selectReadReplica(tp0, Node.noNode(), time.milliseconds()); + assertEquals(selected.id(), -1); + } + + @Test + public void testFetchCompletedBeforeHandlerAdded() { + buildFetcher(); + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + sendFetches(); + client.prepareResponse(fullFetchResponse(tidp0, buildRecords(1L, 1, 1), Errors.NONE, 100L, 0)); + consumerClient.poll(time.timer(0)); + fetchedRecords(); + + Metadata.LeaderAndEpoch leaderAndEpoch = subscriptions.position(tp0).currentLeader; + assertTrue(leaderAndEpoch.leader.isPresent()); + Node readReplica = fetcher.selectReadReplica(tp0, leaderAndEpoch.leader.get(), time.milliseconds()); + + AtomicBoolean wokenUp = new AtomicBoolean(false); + client.setWakeupHook(() -> { + if (!wokenUp.getAndSet(true)) { + consumerClient.disconnectAsync(readReplica); + consumerClient.poll(time.timer(0)); + } + }); + + assertEquals(1, sendFetches()); + + consumerClient.disconnectAsync(readReplica); + consumerClient.poll(time.timer(0)); + + assertEquals(1, sendFetches()); + } + + @Test + public void testCorruptMessageError() { + buildFetcher(); + assignFromUser(singleton(tp0)); + subscriptions.seek(tp0, 0); + + assertEquals(1, sendFetches()); + assertFalse(fetcher.hasCompletedFetches()); + + // Prepare a response with the CORRUPT_MESSAGE error. + client.prepareResponse(fullFetchResponse( + tidp0, + buildRecords(1L, 1, 1), + Errors.CORRUPT_MESSAGE, + 100L, 0)); + consumerClient.poll(time.timer(0)); + assertTrue(fetcher.hasCompletedFetches()); + + // Trigger the exception. + assertThrows(KafkaException.class, this::fetchedRecords); + } + + private OffsetsForLeaderEpochResponse prepareOffsetsForLeaderEpochResponse( + TopicPartition topicPartition, + Errors error, + int leaderEpoch, + long endOffset + ) { + OffsetForLeaderEpochResponseData data = new OffsetForLeaderEpochResponseData(); + data.topics().add(new OffsetForLeaderTopicResult() + .setTopic(topicPartition.topic()) + .setPartitions(Collections.singletonList(new EpochEndOffset() + .setPartition(topicPartition.partition()) + .setErrorCode(error.code()) + .setLeaderEpoch(leaderEpoch) + .setEndOffset(endOffset)))); + return new OffsetsForLeaderEpochResponse(data); + } + + private FetchResponse fetchResponseWithTopLevelError(TopicIdPartition tp, Errors error, int throttleTime) { + Map partitions = Collections.singletonMap(tp, + new FetchResponseData.PartitionData() + .setPartitionIndex(tp.topicPartition().partition()) + .setErrorCode(error.code()) + .setHighWatermark(FetchResponse.INVALID_HIGH_WATERMARK)); + return FetchResponse.of(error, throttleTime, INVALID_SESSION_ID, new LinkedHashMap<>(partitions)); + } + + private FetchResponse fullFetchResponseWithAbortedTransactions(MemoryRecords records, + List abortedTransactions, + Errors error, + long lastStableOffset, + long hw, + int throttleTime) { + Map partitions = Collections.singletonMap(tidp0, + new FetchResponseData.PartitionData() + .setPartitionIndex(tp0.partition()) + .setErrorCode(error.code()) + .setHighWatermark(hw) + .setLastStableOffset(lastStableOffset) + .setLogStartOffset(0) + .setAbortedTransactions(abortedTransactions) + .setRecords(records)); + return FetchResponse.of(Errors.NONE, throttleTime, INVALID_SESSION_ID, new LinkedHashMap<>(partitions)); + } + + private FetchResponse fullFetchResponse(int sessionId, TopicIdPartition tp, MemoryRecords records, Errors error, long hw, int throttleTime) { + return fullFetchResponse(sessionId, tp, records, error, hw, FetchResponse.INVALID_LAST_STABLE_OFFSET, throttleTime); + } + + private FetchResponse fullFetchResponse(TopicIdPartition tp, MemoryRecords records, Errors error, long hw, int throttleTime) { + return fullFetchResponse(tp, records, error, hw, FetchResponse.INVALID_LAST_STABLE_OFFSET, throttleTime); + } + + private FetchResponse fullFetchResponse(TopicIdPartition tp, MemoryRecords records, Errors error, long hw, + long lastStableOffset, int throttleTime) { + return fullFetchResponse(INVALID_SESSION_ID, tp, records, error, hw, lastStableOffset, throttleTime); + } + + private FetchResponse fullFetchResponse(int sessionId, TopicIdPartition tp, MemoryRecords records, Errors error, long hw, + long lastStableOffset, int throttleTime) { + Map partitions = Collections.singletonMap(tp, + new FetchResponseData.PartitionData() + .setPartitionIndex(tp.topicPartition().partition()) + .setErrorCode(error.code()) + .setHighWatermark(hw) + .setLastStableOffset(lastStableOffset) + .setLogStartOffset(0) + .setRecords(records)); + return FetchResponse.of(Errors.NONE, throttleTime, sessionId, new LinkedHashMap<>(partitions)); + } + + private FetchResponse fullFetchResponse(TopicIdPartition tp, MemoryRecords records, Errors error, long hw, + long lastStableOffset, int throttleTime, Optional preferredReplicaId) { + Map partitions = Collections.singletonMap(tp, + new FetchResponseData.PartitionData() + .setPartitionIndex(tp.topicPartition().partition()) + .setErrorCode(error.code()) + .setHighWatermark(hw) + .setLastStableOffset(lastStableOffset) + .setLogStartOffset(0) + .setRecords(records) + .setPreferredReadReplica(preferredReplicaId.orElse(FetchResponse.INVALID_PREFERRED_REPLICA_ID))); + return FetchResponse.of(Errors.NONE, throttleTime, INVALID_SESSION_ID, new LinkedHashMap<>(partitions)); + } + + private FetchResponse fetchResponse(TopicIdPartition tp, MemoryRecords records, Errors error, long hw, + long lastStableOffset, long logStartOffset, int throttleTime) { + Map partitions = Collections.singletonMap(tp, + new FetchResponseData.PartitionData() + .setPartitionIndex(tp.topicPartition().partition()) + .setErrorCode(error.code()) + .setHighWatermark(hw) + .setLastStableOffset(lastStableOffset) + .setLogStartOffset(logStartOffset) + .setRecords(records)); + return FetchResponse.of(Errors.NONE, throttleTime, INVALID_SESSION_ID, new LinkedHashMap<>(partitions)); + } + + /** + * Assert that the {@link Fetcher#collectFetch(Deserializers)} latest fetch} does not contain any + * {@link Fetch#records() user-visible records}, did not + * {@link Fetch#positionAdvanced() advance the consumer's position}, + * and is {@link Fetch#isEmpty() empty}. + * @param reason the reason to include for assertion methods such as {@link org.junit.jupiter.api.Assertions#assertTrue(boolean, String)} + */ + private void assertEmptyFetch(String reason) { + Fetch fetch = collectFetch(); + assertEquals(Collections.emptyMap(), fetch.records(), reason); + assertFalse(fetch.positionAdvanced(), reason); + assertTrue(fetch.isEmpty(), reason); + } + + private Map>> fetchedRecords() { + Fetch fetch = collectFetch(); + return fetch.records(); + } + + @SuppressWarnings("unchecked") + private Fetch collectFetch() { + return (Fetch) fetcher.collectFetch(); + } + + private void buildFetcher(int maxPollRecords) { + buildFetcher(OffsetResetStrategy.EARLIEST, new ByteArrayDeserializer(), new ByteArrayDeserializer(), + maxPollRecords, IsolationLevel.READ_UNCOMMITTED); + } + + private void buildFetcher() { + buildFetcher(Integer.MAX_VALUE); + } + + private void buildFetcher(Deserializer keyDeserializer, + Deserializer valueDeserializer) { + buildFetcher(OffsetResetStrategy.EARLIEST, keyDeserializer, valueDeserializer, + Integer.MAX_VALUE, IsolationLevel.READ_UNCOMMITTED); + } + + private void buildFetcher(OffsetResetStrategy offsetResetStrategy, + Deserializer keyDeserializer, + Deserializer valueDeserializer, + int maxPollRecords, + IsolationLevel isolationLevel) { + buildFetcher(new MetricConfig(), offsetResetStrategy, keyDeserializer, valueDeserializer, + maxPollRecords, isolationLevel); + } + + private void buildFetcher(MetricConfig metricConfig, + OffsetResetStrategy offsetResetStrategy, + Deserializer keyDeserializer, + Deserializer valueDeserializer, + int maxPollRecords, + IsolationLevel isolationLevel) { + buildFetcher(metricConfig, offsetResetStrategy, keyDeserializer, valueDeserializer, maxPollRecords, isolationLevel, Long.MAX_VALUE); + } + + private void buildFetcher(MetricConfig metricConfig, + OffsetResetStrategy offsetResetStrategy, + Deserializer keyDeserializer, + Deserializer valueDeserializer, + int maxPollRecords, + IsolationLevel isolationLevel, + long metadataExpireMs) { + LogContext logContext = new LogContext(); + SubscriptionState subscriptionState = new SubscriptionState(logContext, offsetResetStrategy); + buildFetcher(metricConfig, keyDeserializer, valueDeserializer, maxPollRecords, isolationLevel, metadataExpireMs, + subscriptionState, logContext); + } + + private void buildFetcher(MetricConfig metricConfig, + Deserializer keyDeserializer, + Deserializer valueDeserializer, + int maxPollRecords, + IsolationLevel isolationLevel, + long metadataExpireMs, + SubscriptionState subscriptionState, + LogContext logContext) { + buildDependencies(metricConfig, metadataExpireMs, subscriptionState, logContext); + deserializers = new Deserializers<>(keyDeserializer, valueDeserializer); + FetchConfig fetchConfig = new FetchConfig( + minBytes, + maxBytes, + maxWaitMs, + fetchSize, + maxPollRecords, + true, // check crc + CommonClientConfigs.DEFAULT_CLIENT_RACK, + isolationLevel); + FetchCollector fetchCollector = new FetchCollector<>(logContext, + metadata, + subscriptions, + fetchConfig, + metricsManager, + time); + fetcher = spy(new TestableFetchRequestManager<>( + logContext, + time, + new ErrorEventHandler(new LinkedBlockingQueue<>()), + metadata, + subscriptionState, + fetchConfig, + metricsManager, + consumerClient, + fetchCollector)); + offsetFetcher = new OffsetFetcher(logContext, + oldConsumerClient, + metadata, + subscriptions, + time, + retryBackoffMs, + requestTimeoutMs, + isolationLevel, + apiVersions); + } + + private void buildDependencies(MetricConfig metricConfig, + long metadataExpireMs, + SubscriptionState subscriptionState, + LogContext logContext) { + time = new MockTime(1, 0, 0); + subscriptions = subscriptionState; + metadata = new ConsumerMetadata(0, 0, metadataExpireMs, false, false, + subscriptions, logContext, new ClusterResourceListeners()); + client = new MockClient(time, metadata); + metrics = new Metrics(metricConfig, time); + oldConsumerClient = spy(new ConsumerNetworkClient(logContext, client, metadata, time, + retryBackoffMs, (int) requestTimeoutMs, Integer.MAX_VALUE)); + metricsRegistry = new FetchMetricsRegistry(metricConfig.tags().keySet(), "consumer" + groupId); + metricsManager = new FetchMetricsManager(metrics, metricsRegistry); + + Properties properties = new Properties(); + properties.put(KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); + properties.put(VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); + properties.setProperty(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG, String.valueOf(requestTimeoutMs)); + properties.setProperty(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG, String.valueOf(retryBackoffMs)); + ConsumerConfig config = new ConsumerConfig(properties); + consumerClient = spy(new TestableNetworkClientDelegate(time, config, logContext, client)); + } + + private List collectRecordOffsets(List> records) { + return records.stream().map(ConsumerRecord::offset).collect(Collectors.toList()); + } + + private class TestableFetchRequestManager extends FetchRequestManager { + + private final FetchCollector fetchCollector; + + public TestableFetchRequestManager(LogContext logContext, + Time time, + ErrorEventHandler errorEventHandler, + ConsumerMetadata metadata, + SubscriptionState subscriptions, + FetchConfig fetchConfig, + FetchMetricsManager metricsManager, + NetworkClientDelegate networkClientDelegate, + FetchCollector fetchCollector) { + super(logContext, time, errorEventHandler, metadata, subscriptions, fetchConfig, metricsManager, networkClientDelegate); + this.fetchCollector = fetchCollector; + } + + @SuppressWarnings("unchecked") + private Fetch collectFetch() { + return fetchCollector.collectFetch(fetchBuffer, (Deserializers) deserializers); + } + + private int sendFetches() { + NetworkClientDelegate.PollResult pollResult = fetcher.poll(time.milliseconds()); + consumerClient.addAll(pollResult.unsentRequests); + return pollResult.unsentRequests.size(); + } + + private void clearBufferedDataForUnassignedPartitions(Set partitions) { + fetchBuffer.retainAll(partitions); + } + } + + private class TestableNetworkClientDelegate extends NetworkClientDelegate { + + private final Logger log = LoggerFactory.getLogger(NetworkClientDelegate.class); + private final ConcurrentLinkedQueue pendingDisconnects = new ConcurrentLinkedQueue<>(); + + public TestableNetworkClientDelegate(Time time, + ConsumerConfig config, + LogContext logContext, + KafkaClient client) { + super(time, config, logContext, client); + } + + @Override + public void poll(final long timeoutMs, final long currentTimeMs) { + handlePendingDisconnects(); + super.poll(timeoutMs, currentTimeMs); + } + + public void pollNoWakeup() { + poll(time.timer(0)); + } + + public int pendingRequestCount() { + return unsentRequests().size() + client.inFlightRequestCount(); + } + + public void poll(final Timer timer) { + long pollTimeout = Math.min(timer.remainingMs(), requestTimeoutMs); + if (client.inFlightRequestCount() == 0) + pollTimeout = Math.min(pollTimeout, retryBackoffMs); + poll(pollTimeout, timer.currentTimeMs()); + } + + private Set unsentRequestNodes() { + Set set = new HashSet<>(); + + for (UnsentRequest u : unsentRequests()) + u.node().ifPresent(set::add); + + return set; + } + + private List removeUnsentRequestByNode(Node node) { + List list = new ArrayList<>(); + + Iterator iter = unsentRequests().iterator(); + + while (iter.hasNext()) { + UnsentRequest u = iter.next(); + + if (node.equals(u.node().orElse(null))) { + iter.remove(); + list.add(u); + } + } + + return list; + } + + @Override + protected void checkDisconnects() { + // any disconnects affecting requests that have already been transmitted will be handled + // by NetworkClient, so we just need to check whether connections for any of the unsent + // requests have been disconnected; if they have, then we complete the corresponding future + // and set the disconnect flag in the ClientResponse + for (Node node : unsentRequestNodes()) { + if (client.connectionFailed(node)) { + // Remove entry before invoking request callback to avoid callbacks handling + // coordinator failures traversing the unsent list again. + for (UnsentRequest unsentRequest : removeUnsentRequestByNode(node)) { + // TODO: this should likely emulate what's done in ConsumerNetworkClient + log.error("checkDisconnects - please update! unsentRequest: {}", unsentRequest); + } + } + } + } + + private void handlePendingDisconnects() { + while (true) { + Node node = pendingDisconnects.poll(); + if (node == null) + break; + + failUnsentRequests(node, DisconnectException.INSTANCE); + client.disconnect(node.idString()); + } + } + + public void disconnectAsync(Node node) { + pendingDisconnects.offer(node); + client.wakeup(); + } + + private void failUnsentRequests(Node node, RuntimeException e) { + // clear unsent requests to node and fail their corresponding futures + for (UnsentRequest unsentRequest : removeUnsentRequestByNode(node)) { + FutureCompletionHandler handler = unsentRequest.callback(); + handler.onFailure(e); + } + } + } +} diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java index 6336c11219e78..6a5fcf741c9ed 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java @@ -181,6 +181,7 @@ public class FetcherTest { private Metrics metrics; private ApiVersions apiVersions = new ApiVersions(); private ConsumerNetworkClient consumerClient; + private Deserializers deserializers; private Fetcher fetcher; private OffsetFetcher offsetFetcher; @@ -879,7 +880,7 @@ public byte[] deserialize(String topic, byte[] data) { // The fetcher should block on Deserialization error for (int i = 0; i < 2; i++) { try { - fetcher.collectFetch(); + collectFetch(); fail("fetchedRecords should have raised"); } catch (SerializationException e) { // the position should not advance since no data has been returned @@ -960,7 +961,7 @@ private void ensureBlockOnRecord(long blockedOffset) { // the fetchedRecords() should always throw exception due to the invalid message at the starting offset. for (int i = 0; i < 2; i++) { try { - fetcher.collectFetch(); + collectFetch(); fail("fetchedRecords should have raised KafkaException"); } catch (KafkaException e) { assertEquals(blockedOffset, subscriptions.position(tp0).offset); @@ -972,7 +973,7 @@ private void seekAndConsumeRecord(ByteBuffer responseBuffer, long toOffset) { // Seek to skip the bad record and fetch again. subscriptions.seekUnvalidated(tp0, new SubscriptionState.FetchPosition(toOffset, Optional.empty(), metadata.currentLeader(tp0))); // Should not throw exception after the seek. - fetcher.collectFetch(); + collectFetch(); assertEquals(1, sendFetches()); client.prepareResponse(fullFetchResponse(tidp0, MemoryRecords.readableRecords(responseBuffer), Errors.NONE, 100L, 0)); consumerClient.poll(time.timer(0)); @@ -1016,7 +1017,7 @@ public void testInvalidDefaultRecordBatch() { // the fetchedRecords() should always throw exception due to the bad batch. for (int i = 0; i < 2; i++) { try { - fetcher.collectFetch(); + collectFetch(); fail("fetchedRecords should have raised KafkaException"); } catch (KafkaException e) { assertEquals(0, subscriptions.position(tp0).offset); @@ -1045,7 +1046,7 @@ public void testParseInvalidRecordBatch() { client.prepareResponse(fullFetchResponse(tidp0, MemoryRecords.readableRecords(buffer), Errors.NONE, 100L, 0)); consumerClient.poll(time.timer(0)); try { - fetcher.collectFetch(); + collectFetch(); fail("fetchedRecords should have raised"); } catch (KafkaException e) { // the position should not advance since no data has been returned @@ -1219,7 +1220,7 @@ public void testFetchRequestWhenRecordTooLarge() { client.setNodeApiVersions(NodeApiVersions.create(ApiKeys.FETCH.id, (short) 2, (short) 2)); makeFetchRequestWithIncompleteRecord(); try { - fetcher.collectFetch(); + collectFetch(); fail("RecordTooLargeException should have been raised"); } catch (RecordTooLargeException e) { assertTrue(e.getMessage().startsWith("There are some messages at [Partition=Offset]: ")); @@ -1242,7 +1243,7 @@ public void testFetchRequestInternalError() { buildFetcher(); makeFetchRequestWithIncompleteRecord(); try { - fetcher.collectFetch(); + collectFetch(); fail("RecordTooLargeException should have been raised"); } catch (KafkaException e) { assertTrue(e.getMessage().startsWith("Failed to make progress reading messages")); @@ -1275,7 +1276,7 @@ public void testUnauthorizedTopic() { client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.TOPIC_AUTHORIZATION_FAILED, 100L, 0)); consumerClient.poll(time.timer(0)); try { - fetcher.collectFetch(); + collectFetch(); fail("fetchedRecords should have thrown"); } catch (TopicAuthorizationException e) { assertEquals(singleton(topicName), e.unauthorizedTopics()); @@ -1703,7 +1704,7 @@ public void testFetchOffsetOutOfRangeException() { assertFalse(subscriptions.isOffsetResetNeeded(tp0)); for (int i = 0; i < 2; i++) { OffsetOutOfRangeException e = assertThrows(OffsetOutOfRangeException.class, () -> - fetcher.collectFetch()); + collectFetch()); assertEquals(singleton(tp0), e.offsetOutOfRangePartitions().keySet()); assertEquals(0L, e.offsetOutOfRangePartitions().get(tp0).longValue()); } @@ -2189,7 +2190,7 @@ public void testFetchResponseMetricsWithOnePartitionError() { assertEquals(1, sendFetches()); client.prepareResponse(FetchResponse.of(Errors.NONE, 0, INVALID_SESSION_ID, new LinkedHashMap<>(partitions))); consumerClient.poll(time.timer(0)); - fetcher.collectFetch(); + collectFetch(); int expectedBytes = 0; for (Record record : records.records()) @@ -2235,7 +2236,7 @@ public void testFetchResponseMetricsWithOnePartitionAtTheWrongOffset() { client.prepareResponse(FetchResponse.of(Errors.NONE, 0, INVALID_SESSION_ID, new LinkedHashMap<>(partitions))); consumerClient.poll(time.timer(0)); - fetcher.collectFetch(); + collectFetch(); // we should have ignored the record at the wrong offset int expectedBytes = 0; @@ -2840,7 +2841,8 @@ public void testFetcherConcurrency() throws Exception { isolationLevel, apiVersions); - FetchConfig fetchConfig = new FetchConfig<>( + deserializers = new Deserializers<>(new ByteArrayDeserializer(), new ByteArrayDeserializer()); + FetchConfig fetchConfig = new FetchConfig( minBytes, maxBytes, maxWaitMs, @@ -2848,7 +2850,6 @@ public void testFetcherConcurrency() throws Exception { 2 * numPartitions, true, // check crcs CommonClientConfigs.DEFAULT_CLIENT_RACK, - new Deserializers<>(new ByteArrayDeserializer(), new ByteArrayDeserializer()), isolationLevel); fetcher = new Fetcher( logContext, @@ -3564,7 +3565,7 @@ private FetchResponse fetchResponse(TopicIdPartition tp, MemoryRecords records, } /** - * Assert that the {@link Fetcher#collectFetch() latest fetch} does not contain any + * Assert that the {@link Fetcher#collectFetch(Deserializers) latest fetch} does not contain any * {@link Fetch#records() user-visible records}, did not * {@link Fetch#positionAdvanced() advance the consumer's position}, * and is {@link Fetch#isEmpty() empty}. @@ -3584,7 +3585,10 @@ private Map>> fetchedRecords() @SuppressWarnings("unchecked") private Fetch collectFetch() { - return (Fetch) fetcher.collectFetch(); + // Ugh. What am I doing wrong here? + Fetcher f = (Fetcher) fetcher; + Deserializers d = (Deserializers) deserializers; + return f.collectFetch(d); } private void buildFetcher(int maxPollRecords) { @@ -3642,7 +3646,7 @@ private void buildFetcher(MetricConfig metricConfig, SubscriptionState subscriptionState, LogContext logContext) { buildDependencies(metricConfig, metadataExpireMs, subscriptionState, logContext); - FetchConfig fetchConfig = new FetchConfig<>( + FetchConfig fetchConfig = new FetchConfig( minBytes, maxBytes, maxWaitMs, @@ -3650,8 +3654,8 @@ private void buildFetcher(MetricConfig metricConfig, maxPollRecords, true, // check crc CommonClientConfigs.DEFAULT_CLIENT_RACK, - new Deserializers<>(keyDeserializer, valueDeserializer), isolationLevel); + deserializers = new Deserializers<>(keyDeserializer, valueDeserializer); fetcher = spy(new Fetcher<>( logContext, consumerClient, diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/OffsetFetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/OffsetFetcherTest.java index 4959a1c7b77bb..d3ae0402274e8 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/OffsetFetcherTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/OffsetFetcherTest.java @@ -56,7 +56,6 @@ import org.apache.kafka.common.requests.OffsetsForLeaderEpochRequest; import org.apache.kafka.common.requests.OffsetsForLeaderEpochResponse; import org.apache.kafka.common.requests.RequestTestUtils; -import org.apache.kafka.common.serialization.ByteArrayDeserializer; import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.MockTime; import org.apache.kafka.common.utils.Utils; @@ -1246,7 +1245,7 @@ public void testOffsetValidationSkippedForOldBroker() { buildFetcher(metricConfig, isolationLevel, metadataExpireMs, subscriptionState, logContext); FetchMetricsRegistry metricsRegistry = new FetchMetricsRegistry(metricConfig.tags().keySet(), "consumertest-group"); - FetchConfig fetchConfig = new FetchConfig<>( + FetchConfig fetchConfig = new FetchConfig( minBytes, maxBytes, maxWaitMs, @@ -1254,7 +1253,6 @@ public void testOffsetValidationSkippedForOldBroker() { maxPollRecords, true, // check crc CommonClientConfigs.DEFAULT_CLIENT_RACK, - new Deserializers<>(new ByteArrayDeserializer(), new ByteArrayDeserializer()), isolationLevel); Fetcher fetcher = new Fetcher<>( logContext, diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumerTest.java index 46892215bfa5a..3d0177085d9e1 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumerTest.java @@ -16,14 +16,11 @@ */ package org.apache.kafka.clients.consumer.internals; -import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.clients.consumer.OffsetAndTimestamp; import org.apache.kafka.clients.consumer.OffsetCommitCallback; -import org.apache.kafka.clients.consumer.OffsetResetStrategy; import org.apache.kafka.clients.consumer.internals.events.ApplicationEvent; import org.apache.kafka.clients.consumer.internals.events.AssignmentChangeApplicationEvent; -import org.apache.kafka.clients.consumer.internals.events.BackgroundEvent; import org.apache.kafka.clients.consumer.internals.events.EventHandler; import org.apache.kafka.clients.consumer.internals.events.ListOffsetsApplicationEvent; import org.apache.kafka.clients.consumer.internals.events.NewTopicsMetadataUpdateRequestEvent; @@ -35,12 +32,6 @@ import org.apache.kafka.common.errors.InvalidGroupIdException; import org.apache.kafka.common.errors.TimeoutException; import org.apache.kafka.common.errors.WakeupException; -import org.apache.kafka.common.metrics.Metrics; -import org.apache.kafka.common.serialization.Deserializer; -import org.apache.kafka.common.serialization.StringDeserializer; -import org.apache.kafka.common.utils.LogContext; -import org.apache.kafka.common.utils.MockTime; -import org.apache.kafka.common.utils.Time; import org.apache.kafka.common.utils.Timer; import org.apache.kafka.test.TestUtils; import org.junit.jupiter.api.AfterEach; @@ -57,20 +48,15 @@ import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.concurrent.BlockingQueue; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Future; -import java.util.concurrent.LinkedBlockingQueue; import java.util.stream.Collectors; import static java.util.Collections.singleton; -import static org.apache.kafka.clients.consumer.ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG; -import static org.apache.kafka.clients.consumer.ConsumerConfig.DEFAULT_API_TIMEOUT_MS_CONFIG; -import static org.apache.kafka.clients.consumer.ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG; -import static org.apache.kafka.clients.consumer.ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; @@ -79,56 +65,44 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mockConstruction; import static org.mockito.Mockito.never; -import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; public class PrototypeAsyncConsumerTest { private PrototypeAsyncConsumer consumer; - private final Map consumerProps = new HashMap<>(); + private static final Optional DEFAULT_GROUP_ID = Optional.of("group.id"); - private final Time time = new MockTime(); - private LogContext logContext; - private SubscriptionState subscriptions; - private ConsumerMetadata metadata; + private ConsumerTestBuilder.PrototypeAsyncConsumerTestBuilder testBuilder; private EventHandler eventHandler; - private Metrics metrics; - - private String groupId = "group.id"; - private ConsumerConfig config; @BeforeEach public void setup() { - injectConsumerConfigs(); - this.config = new ConsumerConfig(consumerProps); - this.logContext = new LogContext(); - this.subscriptions = mock(SubscriptionState.class); - this.metadata = mock(ConsumerMetadata.class); - final DefaultBackgroundThread bt = mock(DefaultBackgroundThread.class); - final BlockingQueue aq = new LinkedBlockingQueue<>(); - final BlockingQueue bq = new LinkedBlockingQueue<>(); - this.eventHandler = spy(new DefaultEventHandler(bt, aq, bq)); - this.metrics = new Metrics(time); + setup(DEFAULT_GROUP_ID); } @AfterEach public void cleanup() { - if (consumer != null) { - consumer.close(Duration.ZERO); + if (testBuilder != null) { + testBuilder.close(); } } + private void setup(Optional groupIdOpt) { + testBuilder = new ConsumerTestBuilder.PrototypeAsyncConsumerTestBuilder(groupIdOpt); + eventHandler = testBuilder.eventHandler; + consumer = testBuilder.consumer; + } + @Test public void testSuccessfulStartupShutdown() { - consumer = newConsumer(time, new StringDeserializer(), new StringDeserializer()); assertDoesNotThrow(() -> consumer.close()); } @Test public void testInvalidGroupId() { - this.groupId = null; - consumer = newConsumer(time, new StringDeserializer(), new StringDeserializer()); + cleanup(); + setup(Optional.empty()); assertThrows(InvalidGroupIdException.class, () -> consumer.committed(new HashSet<>())); } @@ -139,11 +113,10 @@ public void testCommitAsync_NullCallback() throws InterruptedException { offsets.put(new TopicPartition("my-topic", 0), new OffsetAndMetadata(100L)); offsets.put(new TopicPartition("my-topic", 1), new OffsetAndMetadata(200L)); - PrototypeAsyncConsumer mockedConsumer = spy(newConsumer(time, new StringDeserializer(), new StringDeserializer())); - doReturn(future).when(mockedConsumer).commit(offsets, false); - mockedConsumer.commitAsync(offsets, null); + doReturn(future).when(consumer).commit(offsets, false); + consumer.commitAsync(offsets, null); future.complete(null); - TestUtils.waitForCondition(() -> future.isDone(), + TestUtils.waitForCondition(future::isDone, 2000, "commit future should complete"); @@ -158,11 +131,9 @@ public void testCommitAsync_UserSuppliedCallback() { offsets.put(new TopicPartition("my-topic", 0), new OffsetAndMetadata(100L)); offsets.put(new TopicPartition("my-topic", 1), new OffsetAndMetadata(200L)); - PrototypeAsyncConsumer consumer = newConsumer(time, new StringDeserializer(), new StringDeserializer()); - PrototypeAsyncConsumer mockedConsumer = spy(consumer); - doReturn(future).when(mockedConsumer).commit(offsets, false); + doReturn(future).when(consumer).commit(offsets, false); OffsetCommitCallback customCallback = mock(OffsetCommitCallback.class); - mockedConsumer.commitAsync(offsets, customCallback); + consumer.commitAsync(offsets, customCallback); future.complete(null); verify(customCallback).onComplete(offsets, null); } @@ -174,7 +145,6 @@ public void testCommitted() { committedFuture.complete(offsets); try (MockedConstruction ignored = offsetFetchEventMocker(committedFuture)) { - this.consumer = newConsumer(time, new StringDeserializer(), new StringDeserializer()); assertDoesNotThrow(() -> consumer.committed(offsets.keySet(), Duration.ofMillis(1000))); verify(eventHandler).add(ArgumentMatchers.isA(OffsetFetchApplicationEvent.class)); } @@ -187,7 +157,6 @@ public void testCommitted_ExceptionThrown() { committedFuture.completeExceptionally(new KafkaException("Test exception")); try (MockedConstruction ignored = offsetFetchEventMocker(committedFuture)) { - this.consumer = newConsumer(time, new StringDeserializer(), new StringDeserializer()); assertThrows(KafkaException.class, () -> consumer.committed(offsets.keySet(), Duration.ofMillis(1000))); verify(eventHandler).add(ArgumentMatchers.isA(OffsetFetchApplicationEvent.class)); } @@ -229,8 +198,6 @@ private static MockedConstruction offsetFetchEventM @Test public void testAssign() { - this.subscriptions = new SubscriptionState(logContext, OffsetResetStrategy.EARLIEST); - this.consumer = newConsumer(time, new StringDeserializer(), new StringDeserializer()); final TopicPartition tp = new TopicPartition("foo", 3); consumer.assign(singleton(tp)); assertTrue(consumer.subscription().isEmpty()); @@ -241,13 +208,11 @@ public void testAssign() { @Test public void testAssignOnNullTopicPartition() { - consumer = newConsumer(time, new StringDeserializer(), new StringDeserializer()); assertThrows(IllegalArgumentException.class, () -> consumer.assign(null)); } @Test public void testAssignOnEmptyTopicPartition() { - consumer = spy(newConsumer(time, new StringDeserializer(), new StringDeserializer())); consumer.assign(Collections.emptyList()); assertTrue(consumer.subscription().isEmpty()); assertTrue(consumer.assignment().isEmpty()); @@ -255,26 +220,22 @@ public void testAssignOnEmptyTopicPartition() { @Test public void testAssignOnNullTopicInPartition() { - consumer = newConsumer(time, new StringDeserializer(), new StringDeserializer()); assertThrows(IllegalArgumentException.class, () -> consumer.assign(singleton(new TopicPartition(null, 0)))); } @Test public void testAssignOnEmptyTopicInPartition() { - consumer = newConsumer(time, new StringDeserializer(), new StringDeserializer()); assertThrows(IllegalArgumentException.class, () -> consumer.assign(singleton(new TopicPartition(" ", 0)))); } @Test public void testBeginningOffsetsFailsIfNullPartitions() { - consumer = newConsumer(time, new StringDeserializer(), new StringDeserializer()); assertThrows(NullPointerException.class, () -> consumer.beginningOffsets(null, Duration.ofMillis(1))); } @Test public void testBeginningOffsets() { - PrototypeAsyncConsumer consumer = newConsumer(time, new StringDeserializer(), new StringDeserializer()); Map expectedOffsetsAndTimestamp = mockOffsetAndTimestamp(); Set partitions = expectedOffsetsAndTimestamp.keySet(); @@ -283,7 +244,7 @@ public void testBeginningOffsets() { assertDoesNotThrow(() -> consumer.beginningOffsets(partitions, Duration.ofMillis(1))); Map expectedOffsets = expectedOffsetsAndTimestamp.entrySet().stream() - .collect(Collectors.toMap(e -> e.getKey(), e -> e.getValue().offset())); + .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().offset())); assertEquals(expectedOffsets, result); verify(eventHandler).addAndGet(ArgumentMatchers.isA(ListOffsetsApplicationEvent.class), ArgumentMatchers.isA(Timer.class)); @@ -291,7 +252,6 @@ public void testBeginningOffsets() { @Test public void testBeginningOffsetsThrowsKafkaExceptionForUnderlyingExecutionFailure() { - PrototypeAsyncConsumer consumer = newConsumer(time, new StringDeserializer(), new StringDeserializer()); Set partitions = mockTopicPartitionOffset().keySet(); Throwable eventProcessingFailure = new KafkaException("Unexpected failure " + "processing List Offsets event"); @@ -305,7 +265,6 @@ public void testBeginningOffsetsThrowsKafkaExceptionForUnderlyingExecutionFailur @Test public void testBeginningOffsetsTimeoutOnEventProcessingTimeout() { - PrototypeAsyncConsumer consumer = newConsumer(time, new StringDeserializer(), new StringDeserializer()); doThrow(new TimeoutException()).when(eventHandler).addAndGet(any(), any()); assertThrows(TimeoutException.class, () -> consumer.beginningOffsets( @@ -317,14 +276,12 @@ public void testBeginningOffsetsTimeoutOnEventProcessingTimeout() { @Test public void testOffsetsForTimesOnNullPartitions() { - PrototypeAsyncConsumer consumer = newConsumer(time, new StringDeserializer(), new StringDeserializer()); assertThrows(NullPointerException.class, () -> consumer.offsetsForTimes(null, Duration.ofMillis(1))); } @Test public void testOffsetsForTimes() { - PrototypeAsyncConsumer consumer = newConsumer(time, new StringDeserializer(), new StringDeserializer()); Map expectedResult = mockOffsetAndTimestamp(); Map timestampToSearch = mockTimestampToSearch(); @@ -341,7 +298,6 @@ public void testOffsetsForTimes() { // OffsetAndTimestamp as value. @Test public void testOffsetsForTimesWithZeroTimeout() { - PrototypeAsyncConsumer consumer = newConsumer(time, new StringDeserializer(), new StringDeserializer()); TopicPartition tp = new TopicPartition("topic1", 0); Map expectedResult = Collections.singletonMap(tp, null); @@ -357,8 +313,6 @@ public void testOffsetsForTimesWithZeroTimeout() { @Test public void testWakeup_committed() { - consumer = newConsumer(time, new StringDeserializer(), - new StringDeserializer()); consumer.wakeup(); assertThrows(WakeupException.class, () -> consumer.committed(mockTopicPartitionOffset().keySet())); assertNoPendingWakeup(consumer.wakeupTrigger()); @@ -366,20 +320,25 @@ public void testWakeup_committed() { @Test public void testRefreshCommittedOffsetsSuccess() { - Map committedOffsets = - Collections.singletonMap(new TopicPartition("t1", 1), new OffsetAndMetadata(10L)); - testRefreshCommittedOffsetsSuccess(committedOffsets); + TopicPartition partition = new TopicPartition("t1", 1); + Set partitions = Collections.singleton(partition); + Map committedOffsets = Collections.singletonMap(partition, new OffsetAndMetadata(10L)); + testRefreshCommittedOffsetsSuccess(partitions, committedOffsets); } @Test public void testRefreshCommittedOffsetsSuccessButNoCommittedOffsetsFound() { - testRefreshCommittedOffsetsSuccess(Collections.emptyMap()); + TopicPartition partition = new TopicPartition("t1", 1); + Set partitions = Collections.singleton(partition); + Map committedOffsets = Collections.emptyMap(); + testRefreshCommittedOffsetsSuccess(partitions, committedOffsets); } @Test public void testRefreshCommittedOffsetsShouldNotResetIfFailedWithTimeout() { // Create consumer with group id to enable committed offset usage - this.groupId = "consumer-group-1"; + cleanup(); + setup(Optional.of("consumer-group-1")); testUpdateFetchPositionsWithFetchCommittedOffsetsTimeout(true); } @@ -387,22 +346,20 @@ public void testRefreshCommittedOffsetsShouldNotResetIfFailedWithTimeout() { @Test public void testRefreshCommittedOffsetsNotCalledIfNoGroupId() { // Create consumer without group id so committed offsets are not used for updating positions - this.groupId = null; - consumer = newConsumer(time, new StringDeserializer(), new StringDeserializer()); + cleanup(); + setup(Optional.empty()); testUpdateFetchPositionsWithFetchCommittedOffsetsTimeout(false); } private void testUpdateFetchPositionsWithFetchCommittedOffsetsTimeout(boolean committedOffsetsEnabled) { - consumer = newConsumer(time, new StringDeserializer(), new StringDeserializer()); - // Uncompleted future that will time out if used CompletableFuture> committedFuture = new CompletableFuture<>(); - when(subscriptions.initializingPartitions()).thenReturn(Collections.singleton(new TopicPartition("t1", 1))); - try (MockedConstruction ignored = offsetFetchEventMocker(committedFuture)) { + consumer.assign(singleton(new TopicPartition("t1", 1))); + try (MockedConstruction ignored = offsetFetchEventMocker(committedFuture)) { // Poll with 0 timeout to run a single iteration of the poll loop consumer.poll(Duration.ofMillis(0)); @@ -422,17 +379,17 @@ private void testUpdateFetchPositionsWithFetchCommittedOffsetsTimeout(boolean co } } - private void testRefreshCommittedOffsetsSuccess(Map committedOffsets) { + private void testRefreshCommittedOffsetsSuccess(Set partitions, + Map committedOffsets) { CompletableFuture> committedFuture = new CompletableFuture<>(); committedFuture.complete(committedOffsets); // Create consumer with group id to enable committed offset usage - this.groupId = "consumer-group-1"; - consumer = newConsumer(time, new StringDeserializer(), new StringDeserializer()); + cleanup(); + setup(Optional.of("consumer-group-id")); + consumer.assign(partitions); try (MockedConstruction ignored = offsetFetchEventMocker(committedFuture)) { - when(subscriptions.initializingPartitions()).thenReturn(committedOffsets.keySet()); - // Poll with 0 timeout to run a single iteration of the poll loop consumer.poll(Duration.ofMillis(0)); @@ -443,10 +400,10 @@ private void testRefreshCommittedOffsetsSuccess(Map mockTopicPartitionOffset() { + private HashMap mockTopicPartitionOffset() { final TopicPartition t0 = new TopicPartition("t0", 2); final TopicPartition t1 = new TopicPartition("t0", 3); HashMap topicPartitionOffsets = new HashMap<>(); @@ -455,7 +412,7 @@ private Map mockTopicPartitionOffset() { return topicPartitionOffsets; } - private Map mockOffsetAndTimestamp() { + private HashMap mockOffsetAndTimestamp() { final TopicPartition t0 = new TopicPartition("t0", 2); final TopicPartition t1 = new TopicPartition("t0", 3); HashMap offsetAndTimestamp = new HashMap<>(); @@ -464,7 +421,7 @@ private Map mockOffsetAndTimestamp() { return offsetAndTimestamp; } - private Map mockTimestampToSearch() { + private HashMap mockTimestampToSearch() { final TopicPartition t0 = new TopicPartition("t0", 2); final TopicPartition t1 = new TopicPartition("t0", 3); HashMap timestampToSearch = new HashMap<>(); @@ -472,30 +429,5 @@ private Map mockTimestampToSearch() { timestampToSearch.put(t1, 2L); return timestampToSearch; } - - private void injectConsumerConfigs() { - consumerProps.put(BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); - consumerProps.put(DEFAULT_API_TIMEOUT_MS_CONFIG, "60000"); - consumerProps.put(KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); - consumerProps.put(VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); - } - - private PrototypeAsyncConsumer newConsumer(final Time time, - final Deserializer keyDeserializer, - final Deserializer valueDeserializer) { - consumerProps.put(KEY_DESERIALIZER_CLASS_CONFIG, keyDeserializer.getClass()); - consumerProps.put(VALUE_DESERIALIZER_CLASS_CONFIG, valueDeserializer.getClass()); - - return new PrototypeAsyncConsumer<>( - time, - logContext, - config, - subscriptions, - metadata, - eventHandler, - metrics, - Optional.ofNullable(this.groupId), - config.getInt(DEFAULT_API_TIMEOUT_MS_CONFIG)); - } } From a30ac4eae784e193ecc3efc653f390cc94ea3fef Mon Sep 17 00:00:00 2001 From: Kirk True Date: Tue, 19 Sep 2023 10:55:21 -0700 Subject: [PATCH 04/72] Reverted change to make FetchCollector.collectFetch() not take a Deserializers --- .../kafka/clients/consumer/KafkaConsumer.java | 5 +-- .../consumer/internals/CompletedFetch.java | 6 ++-- .../consumer/internals/FetchCollector.java | 9 +++-- .../clients/consumer/internals/Fetcher.java | 6 ++-- .../internals/PrototypeAsyncConsumer.java | 5 +-- .../clients/consumer/KafkaConsumerTest.java | 2 ++ .../internals/ConsumerTestBuilder.java | 4 ++- .../internals/FetchCollectorTest.java | 34 ++++++++++--------- .../internals/FetchRequestManagerTest.java | 9 +++-- .../consumer/internals/FetcherTest.java | 13 +++---- .../consumer/internals/OffsetFetcherTest.java | 2 ++ 11 files changed, 53 insertions(+), 42 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java index f9dfa29b0da12..3c4867d134ee3 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java @@ -759,6 +759,7 @@ public KafkaConsumer(Map configs, this.metadata, this.subscriptions, fetchConfig, + this.deserializers, fetchMetricsManager, this.time); this.offsetFetcher = new OffsetFetcher(logContext, @@ -1252,7 +1253,7 @@ private Fetch pollForFetches(Timer timer) { Math.min(coordinator.timeToNextPoll(timer.currentTimeMs()), timer.remainingMs()); // if data is available already, return it immediately - final Fetch fetch = fetcher.collectFetch(deserializers); + final Fetch fetch = fetcher.collectFetch(); if (!fetch.isEmpty()) { return fetch; } @@ -1279,7 +1280,7 @@ private Fetch pollForFetches(Timer timer) { }); timer.update(pollTimer.currentTimeMs()); - return fetcher.collectFetch(deserializers); + return fetcher.collectFetch(); } /** diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CompletedFetch.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CompletedFetch.java index cd8e296db17c3..5e5141d4a19d8 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CompletedFetch.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CompletedFetch.java @@ -152,7 +152,7 @@ void drain() { } } - private void maybeEnsureValid(FetchConfig fetchConfig, RecordBatch batch) { + private void maybeEnsureValid(FetchConfig fetchConfig, RecordBatch batch) { if (fetchConfig.checkCrcs && batch.magic() >= RecordBatch.MAGIC_VALUE_V2) { try { batch.ensureValid(); @@ -163,7 +163,7 @@ private void maybeEnsureValid(FetchConfig fetchConfig, RecordBatch batch) } } - private void maybeEnsureValid(FetchConfig fetchConfig, Record record) { + private void maybeEnsureValid(FetchConfig fetchConfig, Record record) { if (fetchConfig.checkCrcs) { try { record.ensureValid(); @@ -181,7 +181,7 @@ private void maybeCloseRecordStream() { } } - private Record nextFetchedRecord(FetchConfig fetchConfig) { + private Record nextFetchedRecord(FetchConfig fetchConfig) { while (true) { if (records == null || !records.hasNext()) { maybeCloseRecordStream(); diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchCollector.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchCollector.java index 3d2d636c53de4..e98441321d399 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchCollector.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchCollector.java @@ -55,6 +55,7 @@ public class FetchCollector { private final ConsumerMetadata metadata; private final SubscriptionState subscriptions; private final FetchConfig fetchConfig; + private final Deserializers deserializers; private final FetchMetricsManager metricsManager; private final Time time; @@ -62,12 +63,14 @@ public FetchCollector(final LogContext logContext, final ConsumerMetadata metadata, final SubscriptionState subscriptions, final FetchConfig fetchConfig, + final Deserializers deserializers, final FetchMetricsManager metricsManager, final Time time) { this.log = logContext.logger(FetchCollector.class); this.metadata = metadata; this.subscriptions = subscriptions; this.fetchConfig = fetchConfig; + this.deserializers = deserializers; this.metricsManager = metricsManager; this.time = time; } @@ -87,7 +90,7 @@ public FetchCollector(final LogContext logContext, * the defaultResetPolicy is NONE * @throws TopicAuthorizationException If there is TopicAuthorization error in fetchResponse. */ - public Fetch collectFetch(final FetchBuffer fetchBuffer, final Deserializers deserializers) { + public Fetch collectFetch(final FetchBuffer fetchBuffer) { final Fetch fetch = Fetch.empty(); final Queue pausedCompletedFetches = new ArrayDeque<>(); int recordsRemaining = fetchConfig.maxPollRecords; @@ -128,7 +131,7 @@ public Fetch collectFetch(final FetchBuffer fetchBuffer, final Deserialize pausedCompletedFetches.add(nextInLineFetch); fetchBuffer.setNextInLineFetch(null); } else { - final Fetch nextFetch = fetchRecords(nextInLineFetch, deserializers); + final Fetch nextFetch = fetchRecords(nextInLineFetch); recordsRemaining -= nextFetch.numRecords(); fetch.add(nextFetch); } @@ -145,7 +148,7 @@ public Fetch collectFetch(final FetchBuffer fetchBuffer, final Deserialize return fetch; } - private Fetch fetchRecords(final CompletedFetch nextInLineFetch, Deserializers deserializers) { + private Fetch fetchRecords(final CompletedFetch nextInLineFetch) { final TopicPartition tp = nextInLineFetch.partition; if (!subscriptions.isAssigned(tp)) { diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java index a8d7dd0a07052..4059b2e6aef9c 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java @@ -63,6 +63,7 @@ public Fetcher(LogContext logContext, ConsumerMetadata metadata, SubscriptionState subscriptions, FetchConfig fetchConfig, + Deserializers deserializers, FetchMetricsManager metricsManager, Time time) { super(logContext, metadata, subscriptions, fetchConfig, metricsManager, time); @@ -72,6 +73,7 @@ public Fetcher(LogContext logContext, metadata, subscriptions, fetchConfig, + deserializers, metricsManager, time); } @@ -166,8 +168,8 @@ public void onFailure(RuntimeException e) { } } - public Fetch collectFetch(Deserializers deserializers) { - return fetchCollector.collectFetch(fetchBuffer, deserializers); + public Fetch collectFetch() { + return fetchCollector.collectFetch(fetchBuffer); } /** diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java index 8cacac04d17a4..ac09ddf9c10b7 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java @@ -250,6 +250,7 @@ public PrototypeAsyncConsumer(final Time time, metadata, subscriptions, fetchConfig, + deserializers, fetchMetricsManager, time); @@ -927,7 +928,7 @@ private Fetch pollForFetches(Timer timer) { long pollTimeout = timer.remainingMs(); // if data is available already, return it immediately - final Fetch fetch = fetchCollector.collectFetch(fetchBuffer, deserializers); + final Fetch fetch = fetchCollector.collectFetch(fetchBuffer); if (!fetch.isEmpty()) { return fetch; } @@ -962,7 +963,7 @@ private Fetch pollForFetches(Timer timer) { timer.update(pollTimer.currentTimeMs()); } - return fetchCollector.collectFetch(fetchBuffer, deserializers); + return fetchCollector.collectFetch(fetchBuffer); } /** diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java index ebef3ecaca030..6a46b45319124 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java @@ -29,6 +29,7 @@ import org.apache.kafka.clients.consumer.internals.ConsumerMetrics; import org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient; import org.apache.kafka.clients.consumer.internals.ConsumerProtocol; +import org.apache.kafka.clients.consumer.internals.Deserializers; import org.apache.kafka.clients.consumer.internals.FetchConfig; import org.apache.kafka.clients.consumer.internals.FetchMetricsManager; import org.apache.kafka.clients.consumer.internals.Fetcher; @@ -2688,6 +2689,7 @@ private KafkaConsumer newConsumer(Time time, metadata, subscription, fetchConfig, + new Deserializers<>(keyDeserializer, deserializer), metricsManager, time); OffsetFetcher offsetFetcher = new OffsetFetcher(loggerFactory, diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java index 202e442fbcf91..02457b022162b 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java @@ -231,16 +231,18 @@ public PrototypeAsyncConsumerTestBuilder(Optional groupIdOpt) { config.getList(ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG), config.originals(Collections.singletonMap(ConsumerConfig.CLIENT_ID_CONFIG, clientId)) ); + Deserializers deserializers = new Deserializers<>(new StringDeserializer(), new StringDeserializer()); FetchCollector fetchCollector = new FetchCollector<>(logContext, metadata, subscriptions, fetchConfig, + deserializers, metricsManager, time); this.consumer = spy(new PrototypeAsyncConsumer<>( logContext, clientId, - new Deserializers<>(new StringDeserializer(), new StringDeserializer()), + deserializers, new FetchBuffer(logContext), fetchCollector, new ConsumerInterceptors<>(Collections.emptyList()), diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchCollectorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchCollectorTest.java index 66ed1d952bcd6..4a8126143ea2f 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchCollectorTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchCollectorTest.java @@ -106,7 +106,7 @@ public void testFetchNormal() { assertFalse(completedFetch.isInitialized()); // Fetch the data and validate that we get all the records we want back. - Fetch fetch = fetchCollector.collectFetch(fetchBuffer, deserializers); + Fetch fetch = fetchCollector.collectFetch(fetchBuffer); assertFalse(fetch.isEmpty()); assertEquals(recordCount, fetch.numRecords()); @@ -130,7 +130,7 @@ public void testFetchNormal() { assertEquals(recordCount, position.offset); // Now attempt to collect more records from the fetch buffer. - fetch = fetchCollector.collectFetch(fetchBuffer, deserializers); + fetch = fetchCollector.collectFetch(fetchBuffer); // The Fetch object is non-null, but it's empty. assertEquals(0, fetch.numRecords()); @@ -154,7 +154,7 @@ public void testFetchWithReadReplica() { CompletedFetch completedFetch = completedFetchBuilder.build(); fetchBuffer.add(completedFetch); - Fetch fetch = fetchCollector.collectFetch(fetchBuffer, deserializers); + Fetch fetch = fetchCollector.collectFetch(fetchBuffer); // The Fetch and read replica settings should be empty. assertEquals(DEFAULT_RECORD_COUNT, fetch.numRecords()); @@ -177,7 +177,7 @@ public void testNoResultsIfInitializing() { // Add some valid CompletedFetch records to the FetchBuffer queue and collect them into the Fetch. CompletedFetch completedFetch = completedFetchBuilder.build(); fetchBuffer.add(completedFetch); - Fetch fetch = fetchCollector.collectFetch(fetchBuffer, deserializers); + Fetch fetch = fetchCollector.collectFetch(fetchBuffer); // Verify that no records are fetched for the partition as it did not have a valid position set. assertEquals(0, fetch.numRecords()); @@ -194,6 +194,7 @@ public void testErrorInInitialize(int recordCount, RuntimeException expectedExce metadata, subscriptions, fetchConfig, + deserializers, metricsManager, time) { @@ -213,7 +214,7 @@ protected CompletedFetch initialize(final CompletedFetch completedFetch) { assertFalse(fetchBuffer.isEmpty()); // Now run our ill-fated collectFetch. - assertThrows(expectedException.getClass(), () -> fetchCollector.collectFetch(fetchBuffer, deserializers)); + assertThrows(expectedException.getClass(), () -> fetchCollector.collectFetch(fetchBuffer)); // If the number of records in the CompletedFetch was 0, the call to FetchCollector.collectFetch() will // remove it from the queue. If there are records in the CompletedFetch, FetchCollector.collectFetch will @@ -246,7 +247,7 @@ public void testFetchingPausedPartitionsYieldsNoRecords() { // Ensure that the partition for the next-in-line CompletedFetch is still 'paused'. assertTrue(subscriptions.isPaused(completedFetch.partition)); - Fetch fetch = fetchCollector.collectFetch(fetchBuffer, deserializers); + Fetch fetch = fetchCollector.collectFetch(fetchBuffer); // There should be no records in the Fetch as the partition being fetched is 'paused'. assertEquals(0, fetch.numRecords()); @@ -278,7 +279,7 @@ public void testFetchWithMetadataRefreshErrors(final Errors error) { assertEquals(Optional.of(preferredReadReplicaId), subscriptions.preferredReadReplica(topicAPartition0, time.milliseconds())); // Fetch the data and validate that we get all the records we want back. - Fetch fetch = fetchCollector.collectFetch(fetchBuffer, deserializers); + Fetch fetch = fetchCollector.collectFetch(fetchBuffer); assertTrue(fetch.isEmpty()); assertTrue(metadata.updateRequested()); assertEquals(Optional.empty(), subscriptions.preferredReadReplica(topicAPartition0, time.milliseconds())); @@ -293,7 +294,7 @@ public void testFetchWithOffsetOutOfRange() { fetchBuffer.add(completedFetch); // Fetch the data and validate that we get our first batch of records back. - Fetch fetch = fetchCollector.collectFetch(fetchBuffer, deserializers); + Fetch fetch = fetchCollector.collectFetch(fetchBuffer); assertFalse(fetch.isEmpty()); assertEquals(DEFAULT_RECORD_COUNT, fetch.numRecords()); @@ -303,7 +304,7 @@ public void testFetchWithOffsetOutOfRange() { .error(Errors.OFFSET_OUT_OF_RANGE) .build(); fetchBuffer.add(completedFetch); - fetch = fetchCollector.collectFetch(fetchBuffer, deserializers); + fetch = fetchCollector.collectFetch(fetchBuffer); assertTrue(fetch.isEmpty()); // Try to fetch more data and validate that we get an empty Fetch back. @@ -312,7 +313,7 @@ public void testFetchWithOffsetOutOfRange() { .error(Errors.OFFSET_OUT_OF_RANGE) .build(); fetchBuffer.add(completedFetch); - fetch = fetchCollector.collectFetch(fetchBuffer, deserializers); + fetch = fetchCollector.collectFetch(fetchBuffer); assertTrue(fetch.isEmpty()); } @@ -332,7 +333,7 @@ public void testFetchWithOffsetOutOfRangeWithPreferredReadReplica() { .error(Errors.OFFSET_OUT_OF_RANGE) .build(); fetchBuffer.add(completedFetch); - Fetch fetch = fetchCollector.collectFetch(fetchBuffer, deserializers); + Fetch fetch = fetchCollector.collectFetch(fetchBuffer); // The Fetch and read replica settings should be empty. assertTrue(fetch.isEmpty()); @@ -349,7 +350,7 @@ public void testFetchWithTopicAuthorizationFailed() { .error(Errors.TOPIC_AUTHORIZATION_FAILED) .build(); fetchBuffer.add(completedFetch); - assertThrows(TopicAuthorizationException.class, () -> fetchCollector.collectFetch(fetchBuffer, deserializers)); + assertThrows(TopicAuthorizationException.class, () -> fetchCollector.collectFetch(fetchBuffer)); } @Test @@ -362,7 +363,7 @@ public void testFetchWithUnknownLeaderEpoch() { .error(Errors.UNKNOWN_LEADER_EPOCH) .build(); fetchBuffer.add(completedFetch); - Fetch fetch = fetchCollector.collectFetch(fetchBuffer, deserializers); + Fetch fetch = fetchCollector.collectFetch(fetchBuffer); assertTrue(fetch.isEmpty()); } @@ -376,7 +377,7 @@ public void testFetchWithUnknownServerError() { .error(Errors.UNKNOWN_SERVER_ERROR) .build(); fetchBuffer.add(completedFetch); - Fetch fetch = fetchCollector.collectFetch(fetchBuffer, deserializers); + Fetch fetch = fetchCollector.collectFetch(fetchBuffer); assertTrue(fetch.isEmpty()); } @@ -390,7 +391,7 @@ public void testFetchWithCorruptMessage() { .error(Errors.CORRUPT_MESSAGE) .build(); fetchBuffer.add(completedFetch); - assertThrows(KafkaException.class, () -> fetchCollector.collectFetch(fetchBuffer, deserializers)); + assertThrows(KafkaException.class, () -> fetchCollector.collectFetch(fetchBuffer)); } @ParameterizedTest @@ -403,7 +404,7 @@ public void testFetchWithOtherErrors(final Errors error) { .error(error) .build(); fetchBuffer.add(completedFetch); - assertThrows(IllegalStateException.class, () -> fetchCollector.collectFetch(fetchBuffer, deserializers)); + assertThrows(IllegalStateException.class, () -> fetchCollector.collectFetch(fetchBuffer)); } /** @@ -449,6 +450,7 @@ private void buildDependencies(int maxPollRecords) { metadata, subscriptions, fetchConfig, + deserializers, metricsManager, time); fetchBuffer = new FetchBuffer(logContext); diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java index 2a74f9441f9dc..fd4f327aa964f 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java @@ -185,7 +185,6 @@ public class FetchRequestManagerTest { private Metrics metrics; private ApiVersions apiVersions = new ApiVersions(); private ConsumerNetworkClient oldConsumerClient; - private Deserializers deserializers; private TestableFetchRequestManager fetcher; private TestableNetworkClientDelegate consumerClient; private OffsetFetcher offsetFetcher; @@ -3286,7 +3285,7 @@ private FetchResponse fetchResponse(TopicIdPartition tp, MemoryRecords records, } /** - * Assert that the {@link Fetcher#collectFetch(Deserializers)} latest fetch} does not contain any + * Assert that the {@link Fetcher#collectFetch()} latest fetch} does not contain any * {@link Fetch#records() user-visible records}, did not * {@link Fetch#positionAdvanced() advance the consumer's position}, * and is {@link Fetch#isEmpty() empty}. @@ -3364,7 +3363,7 @@ private void buildFetcher(MetricConfig metricConfig, SubscriptionState subscriptionState, LogContext logContext) { buildDependencies(metricConfig, metadataExpireMs, subscriptionState, logContext); - deserializers = new Deserializers<>(keyDeserializer, valueDeserializer); + Deserializers deserializers = new Deserializers<>(keyDeserializer, valueDeserializer); FetchConfig fetchConfig = new FetchConfig( minBytes, maxBytes, @@ -3378,6 +3377,7 @@ private void buildFetcher(MetricConfig metricConfig, metadata, subscriptions, fetchConfig, + deserializers, metricsManager, time); fetcher = spy(new TestableFetchRequestManager<>( @@ -3446,9 +3446,8 @@ public TestableFetchRequestManager(LogContext logContext, this.fetchCollector = fetchCollector; } - @SuppressWarnings("unchecked") private Fetch collectFetch() { - return fetchCollector.collectFetch(fetchBuffer, (Deserializers) deserializers); + return fetchCollector.collectFetch(fetchBuffer); } private int sendFetches() { diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java index 6a5fcf741c9ed..1fa4a6c2e5c8d 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java @@ -181,7 +181,6 @@ public class FetcherTest { private Metrics metrics; private ApiVersions apiVersions = new ApiVersions(); private ConsumerNetworkClient consumerClient; - private Deserializers deserializers; private Fetcher fetcher; private OffsetFetcher offsetFetcher; @@ -2841,7 +2840,7 @@ public void testFetcherConcurrency() throws Exception { isolationLevel, apiVersions); - deserializers = new Deserializers<>(new ByteArrayDeserializer(), new ByteArrayDeserializer()); + Deserializers deserializers = new Deserializers<>(new ByteArrayDeserializer(), new ByteArrayDeserializer()); FetchConfig fetchConfig = new FetchConfig( minBytes, maxBytes, @@ -2857,6 +2856,7 @@ public void testFetcherConcurrency() throws Exception { metadata, subscriptions, fetchConfig, + deserializers, metricsManager, time) { @Override @@ -3565,7 +3565,7 @@ private FetchResponse fetchResponse(TopicIdPartition tp, MemoryRecords records, } /** - * Assert that the {@link Fetcher#collectFetch(Deserializers) latest fetch} does not contain any + * Assert that the {@link Fetcher#collectFetch() latest fetch} does not contain any * {@link Fetch#records() user-visible records}, did not * {@link Fetch#positionAdvanced() advance the consumer's position}, * and is {@link Fetch#isEmpty() empty}. @@ -3585,10 +3585,7 @@ private Map>> fetchedRecords() @SuppressWarnings("unchecked") private Fetch collectFetch() { - // Ugh. What am I doing wrong here? - Fetcher f = (Fetcher) fetcher; - Deserializers d = (Deserializers) deserializers; - return f.collectFetch(d); + return (Fetch) fetcher.collectFetch(); } private void buildFetcher(int maxPollRecords) { @@ -3655,13 +3652,13 @@ private void buildFetcher(MetricConfig metricConfig, true, // check crc CommonClientConfigs.DEFAULT_CLIENT_RACK, isolationLevel); - deserializers = new Deserializers<>(keyDeserializer, valueDeserializer); fetcher = spy(new Fetcher<>( logContext, consumerClient, metadata, subscriptionState, fetchConfig, + new Deserializers<>(keyDeserializer, valueDeserializer), metricsManager, time)); offsetFetcher = new OffsetFetcher(logContext, diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/OffsetFetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/OffsetFetcherTest.java index d3ae0402274e8..fe3535eda64b4 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/OffsetFetcherTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/OffsetFetcherTest.java @@ -56,6 +56,7 @@ import org.apache.kafka.common.requests.OffsetsForLeaderEpochRequest; import org.apache.kafka.common.requests.OffsetsForLeaderEpochResponse; import org.apache.kafka.common.requests.RequestTestUtils; +import org.apache.kafka.common.serialization.ByteArrayDeserializer; import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.MockTime; import org.apache.kafka.common.utils.Utils; @@ -1260,6 +1261,7 @@ public void testOffsetValidationSkippedForOldBroker() { metadata, subscriptions, fetchConfig, + new Deserializers<>(new ByteArrayDeserializer(), new ByteArrayDeserializer()), new FetchMetricsManager(metrics, metricsRegistry), time); From 72b6b27f7ec8b43a37e334a0af5c2145ebdcc7c6 Mon Sep 17 00:00:00 2001 From: Kirk True Date: Fri, 22 Sep 2023 15:02:34 -0700 Subject: [PATCH 05/72] Post-rebase clean up --- .../consumer/internals/DefaultBackgroundThreadTest.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThreadTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThreadTest.java index 83155b192adb2..023ded730d1be 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThreadTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThreadTest.java @@ -115,9 +115,6 @@ public void testStartupAndTearDown() throws InterruptedException { when(topicMetadataRequestManager.poll(anyLong())).thenReturn(emptyPollOffsetsRequestResult()); backgroundThread.start(); TestUtils.waitForCondition(backgroundThread::isRunning, "Failed awaiting for the background thread to be running"); - backgroundThread.close(); - assertFalse(backgroundThread.isRunning()); - backgroundThread.start(); // There's a nonzero amount of time between starting the thread and having it // begin to execute our code. Wait for a bit before checking... From 7c8b945532deec943649dc6147aaafc645f564a5 Mon Sep 17 00:00:00 2001 From: Kirk True Date: Fri, 22 Sep 2023 15:50:28 -0700 Subject: [PATCH 06/72] Update NetworkClientDelegate.java --- .../clients/consumer/internals/NetworkClientDelegate.java | 7 ------- 1 file changed, 7 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/NetworkClientDelegate.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/NetworkClientDelegate.java index f9dfdd17d66b0..c26541b67479e 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/NetworkClientDelegate.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/NetworkClientDelegate.java @@ -268,13 +268,6 @@ public UnsentRequest(final AbstractRequest.Builder requestBuilder, future().whenComplete(responseHandler); } - public UnsentRequest(final AbstractRequest.Builder requestBuilder, - final Node node, - final BiConsumer responseHandler) { - this(requestBuilder, Optional.of(node)); - future().whenComplete(responseHandler); - } - public void setTimer(final Time time, final long requestTimeoutMs) { this.timer = time.timer(requestTimeoutMs); } From 59d0daf247a066ad408f713dbfb19e92fc243200 Mon Sep 17 00:00:00 2001 From: Kirk True Date: Fri, 22 Sep 2023 17:48:25 -0700 Subject: [PATCH 07/72] Minor clean up and refactoring based on first pass of reviews --- .../kafka/clients/consumer/KafkaConsumer.java | 6 +- .../internals/CommitRequestManager.java | 2 +- .../consumer/internals/CompletedFetch.java | 3 +- .../consumer/internals/ConsumerUtils.java | 5 -- .../internals/DefaultBackgroundThread.java | 18 ++--- .../consumer/internals/ErrorEventHandler.java | 3 +- .../consumer/internals/FetchConfig.java | 69 +++++++++---------- .../clients/consumer/internals/Fetcher.java | 1 - .../internals/PrototypeAsyncConsumer.java | 47 +++++++------ .../consumer/internals/RequestManagers.java | 7 +- .../internals/ConsumerTestBuilder.java | 7 +- .../internals/ErrorEventHandlerTest.java | 4 +- .../internals/FetchCollectorTest.java | 3 +- .../consumer/internals/FetchConfigTest.java | 2 +- 14 files changed, 83 insertions(+), 94 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java index 3c4867d134ee3..7ce4c2d367a90 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java @@ -78,13 +78,11 @@ import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_JMX_PREFIX; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRIC_GROUP_PREFIX; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createConsumerNetworkClient; -import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createFetchConfig; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createFetchMetricsManager; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createLogContext; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createMetrics; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createSubscriptionState; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.configuredConsumerInterceptors; -import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.configuredIsolationLevel; import static org.apache.kafka.common.utils.Utils.closeQuietly; import static org.apache.kafka.common.utils.Utils.isBlank; import static org.apache.kafka.common.utils.Utils.join; @@ -714,7 +712,8 @@ public KafkaConsumer(Map configs, this.metadata.bootstrap(addresses); FetchMetricsManager fetchMetricsManager = createFetchMetricsManager(metrics); - this.isolationLevel = configuredIsolationLevel(config); + FetchConfig fetchConfig = new FetchConfig(config); + this.isolationLevel = fetchConfig.isolationLevel; ApiVersions apiVersions = new ApiVersions(); this.client = createConsumerNetworkClient(config, @@ -752,7 +751,6 @@ public KafkaConsumer(Map configs, config.getBoolean(ConsumerConfig.THROW_ON_FETCH_STABLE_OFFSET_UNSUPPORTED), config.getString(ConsumerConfig.CLIENT_RACK_CONFIG)); } - FetchConfig fetchConfig = createFetchConfig(config); this.fetcher = new Fetcher<>( logContext, this.client, diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CommitRequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CommitRequestManager.java index 64fc41d3f9e1a..0fdeaef8b842b 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CommitRequestManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CommitRequestManager.java @@ -49,9 +49,9 @@ import java.util.stream.Collectors; public class CommitRequestManager implements RequestManager { + // TODO: current in ConsumerConfig but inaccessible in the internal package. private static final String THROW_ON_FETCH_STABLE_OFFSET_UNSUPPORTED = "internal.throw.on.fetch.stable.offset.unsupported"; - // TODO: We will need to refactor the subscriptionState private final SubscriptionState subscriptions; private final LogContext logContext; private final Logger log; diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CompletedFetch.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CompletedFetch.java index 5e5141d4a19d8..8959345bffdcf 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CompletedFetch.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CompletedFetch.java @@ -246,7 +246,8 @@ private Record nextFetchedRecord(FetchConfig fetchConfig) { * {@link Deserializer deserialization} of the {@link Record record's} key and value are performed in * this step. * - * @param fetchConfig {@link FetchConfig Configuration} to use, including, but not limited to, {@link Deserializer}s + * @param fetchConfig {@link FetchConfig Configuration} to use + * @param deserializers {@link Deserializer}s to use to convert the raw bytes to the expected key and value types * @param maxRecords The number of records to return; the number returned may be {@code 0 <= maxRecords} * @return {@link ConsumerRecord Consumer records} */ diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerUtils.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerUtils.java index 06dcf207b58ac..01c4d908eebbc 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerUtils.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerUtils.java @@ -142,11 +142,6 @@ public static FetchMetricsManager createFetchMetricsManager(Metrics metrics) { return new FetchMetricsManager(metrics, metricsRegistry); } - public static FetchConfig createFetchConfig(ConsumerConfig config) { - IsolationLevel isolationLevel = configuredIsolationLevel(config); - return new FetchConfig(config, isolationLevel); - } - @SuppressWarnings("unchecked") public static List> configuredConsumerInterceptors(ConsumerConfig config) { return (List>) ClientUtils.configuredInterceptors(config, ConsumerConfig.INTERCEPTOR_CLASSES_CONFIG, ConsumerInterceptor.class); diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThread.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThread.java index 7d7c200e7f60d..df29e181a1813 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThread.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThread.java @@ -35,6 +35,7 @@ import java.util.Objects; import java.util.Optional; import java.util.concurrent.BlockingQueue; +import java.util.concurrent.CompletableFuture; import java.util.function.Supplier; /** @@ -55,7 +56,6 @@ public class DefaultBackgroundThread extends KafkaThread implements Closeable { private final Supplier applicationEventProcessorSupplier; private final Supplier networkClientDelegateSupplier; private final Supplier requestManagersSupplier; - // empty if groupId is null private ApplicationEventProcessor applicationEventProcessor; private NetworkClientDelegate networkClientDelegate; private RequestManagers requestManagers; @@ -155,7 +155,7 @@ public void wakeup() { @Override public void close() { closer.close(() -> { - log.debug("Closing the consumer background thread"); + log.trace("Closing the consumer background thread"); running = false; wakeup(); Utils.closeQuietly(requestManagers, "Request managers client"); @@ -168,17 +168,17 @@ public void close() { /** * It is possible for the background thread to close before complete processing all the events in the queue. In - * this case, we need throw an exception to notify the user the consumer is closed. + * this case, we need to throw an exception to notify the user the consumer is closed. */ private void drainAndComplete() { - List incompletedEvents = new ArrayList<>(); - applicationEventQueue.drainTo(incompletedEvents); - incompletedEvents.forEach(event -> { + List incompleteEvents = new ArrayList<>(); + applicationEventQueue.drainTo(incompleteEvents); + incompleteEvents.forEach(event -> { if (event instanceof CompletableApplicationEvent) { - ((CompletableApplicationEvent) event).future().completeExceptionally( - new KafkaException("The consumer is closed")); + CompletableFuture future = ((CompletableApplicationEvent) event).future(); + future.completeExceptionally(new KafkaException("The consumer is closed")); } }); - log.debug("Discarding {} events because the consumer is closed", incompletedEvents.size()); + log.debug("Discarding {} events because the consumer is closed", incompleteEvents.size()); } } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ErrorEventHandler.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ErrorEventHandler.java index 60bfc7476aafd..a6eaf9c3bc0c1 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ErrorEventHandler.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ErrorEventHandler.java @@ -18,6 +18,7 @@ import org.apache.kafka.clients.consumer.internals.events.BackgroundEvent; import org.apache.kafka.clients.consumer.internals.events.ErrorBackgroundEvent; +import org.apache.kafka.common.KafkaException; import java.util.Queue; @@ -34,6 +35,6 @@ public void handle(RuntimeException e) { } public void handle(Throwable e) { - handle(new RuntimeException(e)); + handle(new KafkaException(e)); } } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchConfig.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchConfig.java index c0ce15120b912..24de49fee1d9f 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchConfig.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchConfig.java @@ -19,48 +19,30 @@ import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.common.IsolationLevel; -import org.apache.kafka.common.serialization.Deserializer; + +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.configuredIsolationLevel; /** * {@link FetchConfig} represents the static configuration for fetching records from Kafka. It is simply a way * to bundle the immutable settings that were presented at the time the {@link Consumer} was created for later use by * classes like {@link Fetcher}, {@link CompletedFetch}, etc. - * - *

    - * - * In most cases, the values stored and returned by {@link FetchConfig} will be those stored in the following - * {@link ConsumerConfig consumer configuration} settings: - * - *

      - *
    • {@link #minBytes}: {@link ConsumerConfig#FETCH_MIN_BYTES_CONFIG}
      • - *
    • {@link #maxBytes}: {@link ConsumerConfig#FETCH_MAX_BYTES_CONFIG}
      • - *
    • {@link #maxWaitMs}: {@link ConsumerConfig#FETCH_MAX_WAIT_MS_CONFIG}
      • - *
    • {@link #fetchSize}: {@link ConsumerConfig#MAX_PARTITION_FETCH_BYTES_CONFIG}
      • - *
    • {@link #maxPollRecords}: {@link ConsumerConfig#MAX_POLL_RECORDS_CONFIG}
      • - *
    • {@link #checkCrcs}: {@link ConsumerConfig#CHECK_CRCS_CONFIG}
      • - *
    • {@link #clientRackId}: {@link ConsumerConfig#CLIENT_RACK_CONFIG}
      • - *
    • {@link #isolationLevel}: {@link ConsumerConfig#ISOLATION_LEVEL_CONFIG}
      • - *
    - * - * However, there are places in the code where additional logic is used to determine these fetch-related configuration - * values. In those cases, the values are calculated outside of this class and simply passed in when constructed. - * - *

    - * - * Note: the {@link Deserializer deserializers} used for the key and value are not closed by this class. They should be - * closed by the creator of the {@link FetchConfig}. */ public class FetchConfig { - final int minBytes; - final int maxBytes; - final int maxWaitMs; - final int fetchSize; - final int maxPollRecords; - final boolean checkCrcs; - final String clientRackId; - final IsolationLevel isolationLevel; + public final int minBytes; + public final int maxBytes; + public final int maxWaitMs; + public final int fetchSize; + public final int maxPollRecords; + public final boolean checkCrcs; + public final String clientRackId; + public final IsolationLevel isolationLevel; + /** + * Constructs a new {@link FetchConfig} using explicitly provided values. This is provided here for tests that + * want to exercise different scenarios can construct specific configuration values rather than going through + * the hassle of constructing a {@link ConsumerConfig}. + */ public FetchConfig(int minBytes, int maxBytes, int maxWaitMs, @@ -79,7 +61,24 @@ public FetchConfig(int minBytes, this.isolationLevel = isolationLevel; } - public FetchConfig(ConsumerConfig config, IsolationLevel isolationLevel) { + /** + * Constructs a new {@link FetchConfig} using values from the given {@link ConsumerConfig consumer configuration} + * settings: + * + *

      + *
    • {@link #minBytes}: {@link ConsumerConfig#FETCH_MIN_BYTES_CONFIG}
      • + *
    • {@link #maxBytes}: {@link ConsumerConfig#FETCH_MAX_BYTES_CONFIG}
      • + *
    • {@link #maxWaitMs}: {@link ConsumerConfig#FETCH_MAX_WAIT_MS_CONFIG}
      • + *
    • {@link #fetchSize}: {@link ConsumerConfig#MAX_PARTITION_FETCH_BYTES_CONFIG}
      • + *
    • {@link #maxPollRecords}: {@link ConsumerConfig#MAX_POLL_RECORDS_CONFIG}
      • + *
    • {@link #checkCrcs}: {@link ConsumerConfig#CHECK_CRCS_CONFIG}
      • + *
    • {@link #clientRackId}: {@link ConsumerConfig#CLIENT_RACK_CONFIG}
      • + *
    • {@link #isolationLevel}: {@link ConsumerConfig#ISOLATION_LEVEL_CONFIG}
      • + *
    + * + * @param config Consumer configuration + */ + public FetchConfig(ConsumerConfig config) { this.minBytes = config.getInt(ConsumerConfig.FETCH_MIN_BYTES_CONFIG); this.maxBytes = config.getInt(ConsumerConfig.FETCH_MAX_BYTES_CONFIG); this.maxWaitMs = config.getInt(ConsumerConfig.FETCH_MAX_WAIT_MS_CONFIG); @@ -87,7 +86,7 @@ public FetchConfig(ConsumerConfig config, IsolationLevel isolationLevel) { this.maxPollRecords = config.getInt(ConsumerConfig.MAX_POLL_RECORDS_CONFIG); this.checkCrcs = config.getBoolean(ConsumerConfig.CHECK_CRCS_CONFIG); this.clientRackId = config.getString(ConsumerConfig.CLIENT_RACK_CONFIG); - this.isolationLevel = isolationLevel; + this.isolationLevel = configuredIsolationLevel(config); } @Override diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java index 4059b2e6aef9c..de69f0adf68e0 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java @@ -100,7 +100,6 @@ public void clearBufferedDataForUnassignedPartitions(Collection public synchronized int sendFetches() { Map fetchRequestMap = prepareFetchRequests(); - for (Map.Entry entry : fetchRequestMap.entrySet()) { final Node fetchTarget = entry.getKey(); final FetchSessionHandler.FetchRequestData data = entry.getValue(); diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java index ac09ddf9c10b7..436af0182dd4e 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java @@ -97,13 +97,11 @@ import static org.apache.kafka.clients.consumer.ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_JMX_PREFIX; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRIC_GROUP_PREFIX; -import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createFetchConfig; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createFetchMetricsManager; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createLogContext; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createMetrics; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createSubscriptionState; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.configuredConsumerInterceptors; -import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.configuredIsolationLevel; import static org.apache.kafka.common.utils.Utils.closeQuietly; import static org.apache.kafka.common.utils.Utils.isBlank; import static org.apache.kafka.common.utils.Utils.join; @@ -152,9 +150,11 @@ public PrototypeAsyncConsumer(final Properties properties, } public PrototypeAsyncConsumer(final Map configs, - final Deserializer keyDeser, - final Deserializer valDeser) { - this(new ConsumerConfig(appendDeserializerToConfig(configs, keyDeser, valDeser)), keyDeser, valDeser); + final Deserializer keyDeserializer, + final Deserializer valueDeserializer) { + this(new ConsumerConfig(appendDeserializerToConfig(configs, keyDeserializer, valueDeserializer)), + keyDeserializer, + valueDeserializer); } public PrototypeAsyncConsumer(final ConsumerConfig config, @@ -200,7 +200,8 @@ public PrototypeAsyncConsumer(final Time time, metadata.bootstrap(addresses); FetchMetricsManager fetchMetricsManager = createFetchMetricsManager(metrics); - this.isolationLevel = configuredIsolationLevel(config); + FetchConfig fetchConfig = new FetchConfig(config); + this.isolationLevel = fetchConfig.isolationLevel; ApiVersions apiVersions = new ApiVersions(); final BlockingQueue applicationEventQueue = new LinkedBlockingQueue<>(); @@ -244,7 +245,6 @@ public PrototypeAsyncConsumer(final Time time, //config.ignore(ConsumerConfig.THROW_ON_FETCH_STABLE_OFFSET_UNSUPPORTED); } // These are specific to the foreground thread - FetchConfig fetchConfig = createFetchConfig(config); this.fetchBuffer = new FetchBuffer(logContext); this.fetchCollector = new FetchCollector<>(logContext, metadata, @@ -444,7 +444,7 @@ public void seekToBeginning(Collection partitions) { if (partitions == null) throw new IllegalArgumentException("Partitions collection cannot be null"); - Collection parts = partitions.size() == 0 ? this.subscriptions.assignedPartitions() : partitions; + Collection parts = partitions.isEmpty() ? this.subscriptions.assignedPartitions() : partitions; subscriptions.requestOffsetReset(parts, OffsetResetStrategy.EARLIEST); } @@ -453,7 +453,7 @@ public void seekToEnd(Collection partitions) { if (partitions == null) throw new IllegalArgumentException("Partitions collection cannot be null"); - Collection parts = partitions.size() == 0 ? this.subscriptions.assignedPartitions() : partitions; + Collection parts = partitions.isEmpty() ? this.subscriptions.assignedPartitions() : partitions; subscriptions.requestOffsetReset(parts, OffsetResetStrategy.LATEST); } @@ -623,15 +623,21 @@ private Map beginningOrEndOffset(Collection timestampToSearch = - partitions.stream().collect(Collectors.toMap(Function.identity(), tp -> timestamp)); + Map timestampToSearch = partitions + .stream() + .collect(Collectors.toMap(Function.identity(), tp -> timestamp)); final ListOffsetsApplicationEvent listOffsetsEvent = new ListOffsetsApplicationEvent( timestampToSearch, - false); - Map offsetAndTimestampMap = - eventHandler.addAndGet(listOffsetsEvent, time.timer(timeout)); - return offsetAndTimestampMap.entrySet().stream().collect(Collectors.toMap(e -> e.getKey(), - e -> e.getValue().offset())); + false + ); + Map offsetAndTimestampMap = eventHandler.addAndGet( + listOffsetsEvent, + time.timer(timeout) + ); + return offsetAndTimestampMap + .entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().offset())); } @Override @@ -796,7 +802,7 @@ public void subscribe(Collection topics, ConsumerRebalanceListener callb throw new IllegalArgumentException("Topic collection to subscribe to cannot be null"); if (topics.isEmpty()) { // treat subscribing to empty topic list as the same as unsubscribing - this.unsubscribe(); + unsubscribe(); } else { for (String topic : topics) { if (isBlank(topic)) @@ -859,7 +865,7 @@ public void assign(Collection partitions) { @Override public void subscribe(Pattern pattern, ConsumerRebalanceListener listener) { maybeThrowInvalidGroupIdException(); - if (pattern == null || pattern.toString().equals("")) + if (pattern == null || pattern.toString().isEmpty()) throw new IllegalArgumentException("Topic pattern to subscribe to cannot be " + (pattern == null ? "null" : "empty")); @@ -921,9 +927,6 @@ private void sendFetches() { }); } - /** - * @throws KafkaException if the rebalance callback throws exception - */ private Fetch pollForFetches(Timer timer) { long pollTimeout = timer.remainingMs(); @@ -949,7 +952,7 @@ private Fetch pollForFetches(Timer timer) { Timer pollTimer = time.timer(pollTimeout); - // Attempt to fetch any data. It's OK if we time out here; it's a best case effort. The + // Attempt to fetch any data. It's OK if we time out here; it's a 'best case' effort. The // data may not be immediately available, but the calling method (poll) will correctly // handle the overall timeout. try { diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java index c7fb1982b2489..7037cb174e4d6 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java @@ -35,8 +35,6 @@ import java.util.function.Supplier; import static java.util.Objects.requireNonNull; -import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createFetchConfig; -import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.configuredIsolationLevel; /** * {@code RequestManagers} provides a means to pass around the set of {@link RequestManager} instances in the system. @@ -119,14 +117,13 @@ public static Supplier supplier(final Time time, protected RequestManagers create() { final NetworkClientDelegate networkClientDelegate = networkClientDelegateSupplier.get(); final ErrorEventHandler errorEventHandler = new ErrorEventHandler(backgroundEventQueue); - final IsolationLevel isolationLevel = configuredIsolationLevel(config); - final FetchConfig fetchConfig = createFetchConfig(config); + final FetchConfig fetchConfig = new FetchConfig(config); long retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG); long retryBackoffMaxMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MAX_MS_CONFIG); final int requestTimeoutMs = config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG); final OffsetsRequestManager listOffsets = new OffsetsRequestManager(subscriptions, metadata, - isolationLevel, + fetchConfig.isolationLevel, time, retryBackoffMs, requestTimeoutMs, diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java index 02457b022162b..46af158e446c6 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java @@ -48,11 +48,9 @@ import static org.apache.kafka.clients.consumer.ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG; import static org.apache.kafka.clients.consumer.ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG; -import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createFetchConfig; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createFetchMetricsManager; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createMetrics; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createSubscriptionState; -import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.configuredIsolationLevel; import static org.mockito.Mockito.spy; public class ConsumerTestBuilder implements Closeable { @@ -106,14 +104,13 @@ public ConsumerTestBuilder() { properties.put(CommonClientConfigs.REQUEST_TIMEOUT_MS_CONFIG, REQUEST_TIMEOUT_MS); this.config = new ConsumerConfig(properties); - IsolationLevel isolationLevel = configuredIsolationLevel(config); + this.fetchConfig = new FetchConfig(config); this.retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG); final long requestTimeoutMs = config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG); this.metrics = createMetrics(config, time); this.subscriptions = spy(createSubscriptionState(config, logContext)); this.metadata = spy(new ConsumerMetadata(config, subscriptions, logContext, new ClusterResourceListeners())); - this.fetchConfig = createFetchConfig(config); this.metricsManager = createFetchMetricsManager(metrics); this.client = new MockClient(time, metadata); @@ -133,7 +130,7 @@ public ConsumerTestBuilder() { client)); this.offsetsRequestManager = spy(new OffsetsRequestManager(subscriptions, metadata, - isolationLevel, + fetchConfig.isolationLevel, time, retryBackoffMs, requestTimeoutMs, diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ErrorEventHandlerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ErrorEventHandlerTest.java index 61e2e95a15aef..f86203a755dbb 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ErrorEventHandlerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ErrorEventHandlerTest.java @@ -123,7 +123,7 @@ public void testMultipleErrorEvents() { errorEventHandler.handle(error2); errorEventHandler.handle(error3); - assertThrows(new RuntimeException(error1)); + assertThrows(new KafkaException(error1)); } @Test @@ -140,7 +140,7 @@ public void testMixedEventsWithErrorEvents() { errorEventHandler.handle(error3); backgroundEventQueue.add(new NoopBackgroundEvent("D")); - assertThrows(new RuntimeException(error1)); + assertThrows(new KafkaException(error1)); } private void assertPeeked(BackgroundEvent event) { diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchCollectorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchCollectorTest.java index 4a8126143ea2f..0ca4a18a48ae5 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchCollectorTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchCollectorTest.java @@ -51,7 +51,6 @@ import java.util.Set; import java.util.stream.Stream; -import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createFetchConfig; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createFetchMetricsManager; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createMetrics; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createSubscriptionState; @@ -432,7 +431,7 @@ private void buildDependencies(int maxPollRecords) { deserializers = new Deserializers<>(new StringDeserializer(), new StringDeserializer()); subscriptions = createSubscriptionState(config, logContext); - fetchConfig = createFetchConfig(config); + fetchConfig = new FetchConfig(config); Metrics metrics = createMetrics(config, time); metricsManager = createFetchMetricsManager(metrics); diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchConfigTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchConfigTest.java index c5d0755d2a107..f38a6294624ad 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchConfigTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchConfigTest.java @@ -42,7 +42,7 @@ private void newFetchConfigFromConsumerConfig() { p.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); p.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); ConsumerConfig config = new ConsumerConfig(p); - new FetchConfig(config, IsolationLevel.READ_UNCOMMITTED); + new FetchConfig(config); } private void newFetchConfigFromValues() { From 5eef0f7957f8f76c050bd55f1bdd96330affba22 Mon Sep 17 00:00:00 2001 From: Kirk True Date: Mon, 25 Sep 2023 17:09:20 -0700 Subject: [PATCH 08/72] Medium-sized refactor of the queue processing Changes: 1. Introduced a new generic queue processing class that both the ApplicationEventProcessor and the BackgroundEventProcessor use. 2. Moved event processing to each iteration of the internal loop inside PrototypeAsyncConsumer.poll, instead of just at the start. 3. Removed the NOOP event types. 4. Introduced the CompletableEvent interface which will be used by the BackgroundEvent hierarchy soon 5. Provided missing documentation around the queue processors. 6. No longer returning a flag from the event processor method as it was not used anywhere. --- .../internals/DefaultBackgroundThread.java | 44 +----- .../internals/DefaultEventHandler.java | 1 - .../internals/PrototypeAsyncConsumer.java | 41 ++++- .../consumer/internals/RequestManagers.java | 1 - .../internals/events/ApplicationEvent.java | 2 +- .../events/ApplicationEventProcessor.java | 141 +++++++++--------- .../internals/events/BackgroundEvent.java | 2 +- .../events/BackgroundEventProcessor.java | 91 +++-------- .../events/CompletableApplicationEvent.java | 2 +- .../internals/events/CompletableEvent.java | 25 ++++ .../internals/events/EventProcessor.java | 135 +++++++++++++++++ .../events/NoopApplicationEvent.java | 62 -------- .../internals/events/NoopBackgroundEvent.java | 62 -------- .../internals/ConsumerTestBuilder.java | 12 +- .../DefaultBackgroundThreadTest.java | 4 +- .../internals/DefaultEventHandlerTest.java | 4 +- .../internals/ErrorEventHandlerTest.java | 75 +++++----- 17 files changed, 347 insertions(+), 357 deletions(-) create mode 100644 clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/CompletableEvent.java create mode 100644 clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/EventProcessor.java delete mode 100644 clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/NoopApplicationEvent.java delete mode 100644 clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/NoopBackgroundEvent.java diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThread.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThread.java index df29e181a1813..40711c819ca28 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThread.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThread.java @@ -18,7 +18,6 @@ import org.apache.kafka.clients.consumer.internals.events.ApplicationEvent; import org.apache.kafka.clients.consumer.internals.events.ApplicationEventProcessor; -import org.apache.kafka.clients.consumer.internals.events.CompletableApplicationEvent; import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.errors.WakeupException; import org.apache.kafka.common.internals.IdempotentCloser; @@ -29,13 +28,7 @@ import org.slf4j.Logger; import java.io.Closeable; -import java.util.ArrayList; -import java.util.LinkedList; -import java.util.List; -import java.util.Objects; import java.util.Optional; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.CompletableFuture; import java.util.function.Supplier; /** @@ -44,7 +37,7 @@ * produce events, and poll the network client to handle network IO. *

    * It holds a reference to the {@link SubscriptionState}, which is - * initialized by the polling thread. + * initialized by the application thread. */ public class DefaultBackgroundThread extends KafkaThread implements Closeable { @@ -52,7 +45,6 @@ public class DefaultBackgroundThread extends KafkaThread implements Closeable { private static final String BACKGROUND_THREAD_NAME = "consumer_background_thread"; private final Time time; private final Logger log; - private final BlockingQueue applicationEventQueue; private final Supplier applicationEventProcessorSupplier; private final Supplier networkClientDelegateSupplier; private final Supplier requestManagersSupplier; @@ -64,14 +56,12 @@ public class DefaultBackgroundThread extends KafkaThread implements Closeable { public DefaultBackgroundThread(Time time, LogContext logContext, - BlockingQueue applicationEventQueue, Supplier applicationEventProcessorSupplier, Supplier networkClientDelegateSupplier, Supplier requestManagersSupplier) { super(BACKGROUND_THREAD_NAME, true); this.time = time; this.log = logContext.logger(getClass()); - this.applicationEventQueue = applicationEventQueue; this.applicationEventProcessorSupplier = applicationEventProcessorSupplier; this.networkClientDelegateSupplier = networkClientDelegateSupplier; this.requestManagersSupplier = requestManagersSupplier; @@ -118,14 +108,11 @@ void initializeResources() { * 3. Poll the networkClient to send and retrieve the response. */ void runOnce() { - LinkedList events = new LinkedList<>(); - applicationEventQueue.drainTo(events); - - for (ApplicationEvent event : events) { - log.trace("Dequeued event: {}", event); - Objects.requireNonNull(event); - applicationEventProcessor.process(event); - } + // If there are errors processing any events, we want to throw the error immediately. This will have + // the effect of closing the background thread. + applicationEventProcessor.process(error -> { + throw error; + }); final long currentTimeMs = time.milliseconds(); final long pollWaitTimeMs = requestManagers.entries().stream() @@ -160,25 +147,8 @@ public void close() { wakeup(); Utils.closeQuietly(requestManagers, "Request managers client"); Utils.closeQuietly(networkClientDelegate, "network client utils"); + Utils.closeQuietly(applicationEventProcessor, "application event processor"); log.debug("Closed the consumer background thread"); - drainAndComplete(); }, () -> log.warn("The consumer background thread was previously closed")); } - - - /** - * It is possible for the background thread to close before complete processing all the events in the queue. In - * this case, we need to throw an exception to notify the user the consumer is closed. - */ - private void drainAndComplete() { - List incompleteEvents = new ArrayList<>(); - applicationEventQueue.drainTo(incompleteEvents); - incompleteEvents.forEach(event -> { - if (event instanceof CompletableApplicationEvent) { - CompletableFuture future = ((CompletableApplicationEvent) event).future(); - future.completeExceptionally(new KafkaException("The consumer is closed")); - } - }); - log.debug("Discarding {} events because the consumer is closed", incompleteEvents.size()); - } } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/DefaultEventHandler.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/DefaultEventHandler.java index 342c13c4b51a6..33f3e9be2f4d5 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/DefaultEventHandler.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/DefaultEventHandler.java @@ -54,7 +54,6 @@ public DefaultEventHandler(final Time time, this.applicationEventQueue = applicationEventQueue; this.backgroundThread = new DefaultBackgroundThread(time, logContext, - applicationEventQueue, applicationEventProcessorSupplier, networkClientDelegateSupplier, requestManagersSupplier); diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java index 436af0182dd4e..1d8cfe7d5e31c 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java @@ -38,7 +38,9 @@ import org.apache.kafka.clients.consumer.internals.events.AssignmentChangeApplicationEvent; import org.apache.kafka.clients.consumer.internals.events.BackgroundEvent; import org.apache.kafka.clients.consumer.internals.events.BackgroundEventProcessor; +import org.apache.kafka.clients.consumer.internals.events.EventProcessor.ProcessErrorHandler; import org.apache.kafka.clients.consumer.internals.events.CommitApplicationEvent; +import org.apache.kafka.clients.consumer.internals.events.ErrorBackgroundEvent; import org.apache.kafka.clients.consumer.internals.events.EventHandler; import org.apache.kafka.clients.consumer.internals.events.FetchEvent; import org.apache.kafka.clients.consumer.internals.events.ListOffsetsApplicationEvent; @@ -225,7 +227,7 @@ public PrototypeAsyncConsumer(final Time time, networkClientDelegateSupplier); final Supplier applicationEventProcessorSupplier = ApplicationEventProcessor.supplier(logContext, metadata, - backgroundEventQueue, + applicationEventQueue, requestManagersSupplier); this.eventHandler = new DefaultEventHandler(time, logContext, @@ -323,8 +325,6 @@ public ConsumerRecords poll(final Duration timeout) { Timer timer = time.timer(timeout); try { - backgroundEventProcessor.process(); - this.kafkaConsumerMetrics.recordPollStart(timer.currentTimeMs()); if (this.subscriptions.hasNoSubscriptionOrUserAssignment()) { @@ -1069,9 +1069,44 @@ public void onComplete(Map offsets, Exception } boolean updateAssignmentMetadataIfNeeded(Timer timer) { + processBackgroundEvents(); + // Keeping this updateAssignmentMetadataIfNeeded wrapping up the updateFetchPositions as // in the previous implementation, because it will eventually involve group coordination // logic return updateFetchPositions(timer); } + + /** + * {@link BackgroundEventProcessor Process the events}—if any—that were produced by the + * {@link DefaultBackgroundThread background thread}. It is possible that when processing the events that a + * given event will {@link ErrorBackgroundEvent represent an error directly}, or it could be that processing + * an event generates an error. In such cases, the processor will continue to process the remaining events. In + * this case, we provide the caller to provide a callback handler that "collects" the errors. We grab the first + * error that occurred and throw it. + */ + private void processBackgroundEvents() { + BackgroundEventProcessHandler handler = new BackgroundEventProcessHandler(); + backgroundEventProcessor.process(handler); + if (handler.first != null) + throw handler.first; + } + + private class BackgroundEventProcessHandler implements ProcessErrorHandler { + + private KafkaException first; + private int errorCount; + + @Override + public void onProcessingError(KafkaException error) { + errorCount++; + + if (first == null) { + first = error; + log.warn("Error #{} from background thread (will be logged and thrown): {}", errorCount, error.getMessage(), error); + } else { + log.warn("Error #{} from background thread (will be logged only): {}", errorCount, error.getMessage(), error); + } + } + } } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java index 7037cb174e4d6..6a96a27caa41e 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java @@ -20,7 +20,6 @@ import org.apache.kafka.clients.GroupRebalanceConfig; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.internals.events.BackgroundEvent; -import org.apache.kafka.common.IsolationLevel; import org.apache.kafka.common.internals.IdempotentCloser; import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.Time; diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEvent.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEvent.java index 9badce91cfed0..3a1530fccdf9d 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEvent.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEvent.java @@ -24,7 +24,7 @@ public abstract class ApplicationEvent { public enum Type { - NOOP, COMMIT, POLL, FETCH_COMMITTED_OFFSET, METADATA_UPDATE, ASSIGNMENT_CHANGE, + COMMIT, POLL, FETCH_COMMITTED_OFFSET, METADATA_UPDATE, ASSIGNMENT_CHANGE, LIST_OFFSETS, RESET_POSITIONS, VALIDATE_POSITIONS, TOPIC_METADATA, FETCH } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventProcessor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventProcessor.java index 19462de520cfc..bc01130aa08e7 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventProcessor.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventProcessor.java @@ -21,169 +21,163 @@ import org.apache.kafka.clients.consumer.internals.CommitRequestManager; import org.apache.kafka.clients.consumer.internals.CompletedFetch; import org.apache.kafka.clients.consumer.internals.ConsumerMetadata; +import org.apache.kafka.clients.consumer.internals.DefaultBackgroundThread; import org.apache.kafka.clients.consumer.internals.RequestManagers; import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.PartitionInfo; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.utils.LogContext; -import org.slf4j.Logger; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Optional; import java.util.Queue; import java.util.concurrent.BlockingQueue; import java.util.concurrent.CompletableFuture; import java.util.function.Supplier; -public class ApplicationEventProcessor { - - private final BlockingQueue backgroundEventQueue; - - private final Logger log; +/** + * An {@link EventProcessor} that is created and executes in {@link DefaultBackgroundThread the background thread} + * which processes {@link ApplicationEvent application events} generated by the application thread. + */ +public class ApplicationEventProcessor extends EventProcessor { private final ConsumerMetadata metadata; - private final RequestManagers requestManagers; - public ApplicationEventProcessor(final BlockingQueue backgroundEventQueue, + public ApplicationEventProcessor(final LogContext logContext, + final BlockingQueue applicationEventQueue, final RequestManagers requestManagers, - final ConsumerMetadata metadata, - final LogContext logContext) { - this.log = logContext.logger(ApplicationEventProcessor.class); - this.backgroundEventQueue = backgroundEventQueue; + final ConsumerMetadata metadata) { + super(logContext, applicationEventQueue); this.requestManagers = requestManagers; this.metadata = metadata; } - public boolean process(final ApplicationEvent event) { - Objects.requireNonNull(event, "Attempt to process null ApplicationEvent"); - Objects.requireNonNull(event.type(), "Attempt to process ApplicationEvent with null type: " + event); - - log.trace("Processing event: {}", event); - - // Make sure to use the event's type() method, not the type variable directly. This causes problems when - // unit tests mock the EventType. + @Override + public void process(ApplicationEvent event) { switch (event.type()) { - case NOOP: - return process((NoopApplicationEvent) event); case COMMIT: - return process((CommitApplicationEvent) event); + process((CommitApplicationEvent) event); + return; + case POLL: - return process((PollApplicationEvent) event); + process((PollApplicationEvent) event); + return; + case FETCH_COMMITTED_OFFSET: - return process((OffsetFetchApplicationEvent) event); + process((OffsetFetchApplicationEvent) event); + return; + case METADATA_UPDATE: - return process((NewTopicsMetadataUpdateRequestEvent) event); + process((NewTopicsMetadataUpdateRequestEvent) event); + return; + case ASSIGNMENT_CHANGE: - return process((AssignmentChangeApplicationEvent) event); + process((AssignmentChangeApplicationEvent) event); + return; + case TOPIC_METADATA: - return process((TopicMetadataApplicationEvent) event); + process((TopicMetadataApplicationEvent) event); + return; + case LIST_OFFSETS: - return process((ListOffsetsApplicationEvent) event); + process((ListOffsetsApplicationEvent) event); + return; + case FETCH: - return process((FetchEvent) event); + process((FetchEvent) event); + return; + case RESET_POSITIONS: - return processResetPositionsEvent(); + processResetPositionsEvent(); + return; + case VALIDATE_POSITIONS: - return processValidatePositionsEvent(); + processValidatePositionsEvent(); + return; + + default: + throw new IllegalArgumentException("Application event type " + event.type() + " was not expected"); } - return false; } - /** - * Processes {@link NoopApplicationEvent} and enqueue a - * {@link NoopBackgroundEvent}. This is intentionally left here for - * demonstration purpose. - * - * @param event a {@link NoopApplicationEvent} - */ - private boolean process(final NoopApplicationEvent event) { - return backgroundEventQueue.add(new NoopBackgroundEvent(event.message())); + @Override + protected Class getEventClass() { + return ApplicationEvent.class; } - private boolean process(final PollApplicationEvent event) { + private void process(final PollApplicationEvent event) { if (!requestManagers.commitRequestManager.isPresent()) { - return true; + return; } CommitRequestManager manager = requestManagers.commitRequestManager.get(); manager.updateAutoCommitTimer(event.pollTimeMs()); - return true; } - private boolean process(final CommitApplicationEvent event) { + private void process(final CommitApplicationEvent event) { if (!requestManagers.commitRequestManager.isPresent()) { // Leaving this error handling here, but it is a bit strange as the commit API should enforce the group.id - // upfront so we should never get to this block. + // upfront, so we should never get to this block. Exception exception = new KafkaException("Unable to commit offset. Most likely because the group.id wasn't set"); event.future().completeExceptionally(exception); - return false; + return; } CommitRequestManager manager = requestManagers.commitRequestManager.get(); event.chain(manager.addOffsetCommitRequest(event.offsets())); - return true; } - private boolean process(final OffsetFetchApplicationEvent event) { + private void process(final OffsetFetchApplicationEvent event) { if (!requestManagers.commitRequestManager.isPresent()) { event.future().completeExceptionally(new KafkaException("Unable to fetch committed " + "offset because the CommittedRequestManager is not available. Check if group.id was set correctly")); - return false; + return; } CommitRequestManager manager = requestManagers.commitRequestManager.get(); event.chain(manager.addOffsetFetchRequest(event.partitions())); - return true; } - private boolean process(final NewTopicsMetadataUpdateRequestEvent event) { + private void process(final NewTopicsMetadataUpdateRequestEvent ignored) { metadata.requestUpdateForNewTopics(); - return true; } - private boolean process(final AssignmentChangeApplicationEvent event) { + private void process(final AssignmentChangeApplicationEvent event) { if (!requestManagers.commitRequestManager.isPresent()) { - return false; + return; } CommitRequestManager manager = requestManagers.commitRequestManager.get(); manager.updateAutoCommitTimer(event.currentTimeMs()); manager.maybeAutoCommit(event.offsets()); - return true; } - private boolean process(final ListOffsetsApplicationEvent event) { + private void process(final ListOffsetsApplicationEvent event) { final CompletableFuture> future = requestManagers.offsetsRequestManager.fetchOffsets(event.timestampsToSearch(), event.requireTimestamps()); event.chain(future); - return true; } - private boolean processResetPositionsEvent() { + private void processResetPositionsEvent() { requestManagers.offsetsRequestManager.resetPositionsIfNeeded(); - return true; } - private boolean processValidatePositionsEvent() { + private void processValidatePositionsEvent() { requestManagers.offsetsRequestManager.validatePositionsIfNeeded(); - return true; } - private boolean process(final TopicMetadataApplicationEvent event) { + private void process(final TopicMetadataApplicationEvent event) { final CompletableFuture>> future = - this.requestManagers.topicMetadataRequestManager.requestTopicMetadata(Optional.of(event.topic())); + this.requestManagers.topicMetadataRequestManager.requestTopicMetadata(Optional.of(event.topic())); event.chain(future); - return true; } - private boolean process(final FetchEvent event) { + private void process(final FetchEvent event) { // The request manager keeps track of the completed fetches, so we pull any that are ready off, and return // them to the application. Queue completedFetches = requestManagers.fetchRequestManager.drain(); event.future().complete(completedFetches); - return true; } /** @@ -192,13 +186,18 @@ private boolean process(final FetchEvent event) { */ public static Supplier supplier(final LogContext logContext, final ConsumerMetadata metadata, - final BlockingQueue backgroundEventQueue, + final BlockingQueue applicationEventQueue, final Supplier requestManagersSupplier) { return new CachedSupplier() { @Override protected ApplicationEventProcessor create() { RequestManagers requestManagers = requestManagersSupplier.get(); - return new ApplicationEventProcessor(backgroundEventQueue, requestManagers, metadata, logContext); + return new ApplicationEventProcessor( + logContext, + applicationEventQueue, + requestManagers, + metadata + ); } }; } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEvent.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEvent.java index b0f7c3454f665..91b0fc0a8ddbf 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEvent.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEvent.java @@ -24,7 +24,7 @@ public abstract class BackgroundEvent { public enum Type { - NOOP, ERROR, + ERROR, } protected final Type type; diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEventProcessor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEventProcessor.java index cdf7fa20d37df..dea43b28de59d 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEventProcessor.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEventProcessor.java @@ -16,85 +16,40 @@ */ package org.apache.kafka.clients.consumer.internals.events; +import org.apache.kafka.clients.consumer.ConsumerRebalanceListener; +import org.apache.kafka.clients.consumer.internals.DefaultBackgroundThread; import org.apache.kafka.common.utils.LogContext; -import org.slf4j.Logger; -import java.util.LinkedList; -import java.util.Objects; import java.util.concurrent.BlockingQueue; -public class BackgroundEventProcessor { - - private final Logger log; - private final BlockingQueue backgroundEventQueue; +/** + * An {@link EventProcessor} that is created and executes in the application thread for the purpose of processing + * {@link BackgroundEvent background events} generated by {@link DefaultBackgroundThread the background thread}. + * Those events are generally of two types: + * + *

      + *
    • Errors that occur in the background thread that need to be propagated to the application thread
      • + *
    • {@link ConsumerRebalanceListener} callbacks that are to be executed on the application thread
      • + *
    + */ +public class BackgroundEventProcessor extends EventProcessor { public BackgroundEventProcessor(final LogContext logContext, final BlockingQueue backgroundEventQueue) { - this.log = logContext.logger(BackgroundEventProcessor.class); - this.backgroundEventQueue = backgroundEventQueue; - } - - /** - * Drains all available {@link BackgroundEvent}s, and then processes them in order. If any - * errors are thrown as a result of a {@link ErrorBackgroundEvent} or an error occurs while processing - * another type of {@link BackgroundEvent}, only the first exception will be thrown, all - * subsequent errors will simply be logged at WARN level. - * - * @throws RuntimeException or subclass - */ - public void process() { - LinkedList events = new LinkedList<>(); - backgroundEventQueue.drainTo(events); - - RuntimeException first = null; - int errorCount = 0; - - for (BackgroundEvent event : events) { - log.debug("Consuming background event: {}", event); - - try { - process(event); - } catch (RuntimeException e) { - errorCount++; - - if (first == null) { - first = e; - log.warn("Error #{} from background thread (will be logged and thrown): {}", errorCount, e.getMessage(), e); - } else { - log.warn("Error #{} from background thread (will be logged only): {}", errorCount, e.getMessage(), e); - } - } - } - - if (first != null) { - throw first; - } + super(logContext, backgroundEventQueue); } - private void process(final BackgroundEvent event) { - Objects.requireNonNull(event, "Attempt to process null BackgroundEvent"); - Objects.requireNonNull(event.type(), "Attempt to process BackgroundEvent with null type: " + event); - - log.debug("Processing event {}", event); - - // Make sure to use the event's type() method, not the type variable directly. This causes problems when - // unit tests mock the EventType. - switch (event.type()) { - case NOOP: - process((NoopBackgroundEvent) event); - return; - - case ERROR: - process((ErrorBackgroundEvent) event); - return; - - default: - throw new IllegalArgumentException("Background event type " + event.type() + " was not expected"); - } + @Override + protected Class getEventClass() { + return BackgroundEvent.class; } - private void process(final NoopBackgroundEvent event) { - log.debug("Received no-op background event with message: {}", event.message()); + @Override + public void process(final BackgroundEvent event) { + if (event.type() == BackgroundEvent.Type.ERROR) + process((ErrorBackgroundEvent) event); + else + throw new IllegalArgumentException("Background event type " + event.type() + " was not expected"); } private void process(final ErrorBackgroundEvent event) { diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/CompletableApplicationEvent.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/CompletableApplicationEvent.java index 8146d9583ae2f..365c620e0c0c0 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/CompletableApplicationEvent.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/CompletableApplicationEvent.java @@ -27,7 +27,7 @@ * * @param */ -public abstract class CompletableApplicationEvent extends ApplicationEvent { +public abstract class CompletableApplicationEvent extends ApplicationEvent implements CompletableEvent { private final CompletableFuture future; diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/CompletableEvent.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/CompletableEvent.java new file mode 100644 index 0000000000000..8fdcc20fa8363 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/CompletableEvent.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.clients.consumer.internals.events; + +import java.util.concurrent.CompletableFuture; + +public interface CompletableEvent { + + CompletableFuture future(); + +} diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/EventProcessor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/EventProcessor.java new file mode 100644 index 0000000000000..72b74138e5c18 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/EventProcessor.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.clients.consumer.internals.events; + +import org.apache.kafka.common.KafkaException; +import org.apache.kafka.common.internals.IdempotentCloser; +import org.apache.kafka.common.utils.LogContext; +import org.slf4j.Logger; + +import java.io.Closeable; +import java.util.LinkedList; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.BlockingQueue; + +/** + * An {@link EventProcessor} is the means by which events produced by thread A are + * processed by thread B. By definition, threads A and B run in parallel to + * each other, so a mechanism is needed with which to receive and process the events from the other thread. That + * communication channel is formed around {@link BlockingQueue a shared queue} into which thread A + * enqueues events and thread B reads and processes those events. + */ +public abstract class EventProcessor implements Closeable { + + private final Logger log; + private final BlockingQueue eventQueue; + private final IdempotentCloser closer; + + protected EventProcessor(final LogContext logContext, final BlockingQueue eventQueue) { + this.log = logContext.logger(EventProcessor.class); + this.eventQueue = eventQueue; + this.closer = new IdempotentCloser(); + } + + /** + * Drains all available events from the queue, and then processes them in order. If any errors are thrown while + * processing the individual events, these are submitted to the given {@link ProcessErrorHandler}. + */ + public void process(ProcessErrorHandler processErrorHandler) { + String eventClassName = getEventClass().getSimpleName(); + closer.assertOpen(() -> String.format("The processor was previously closed, so no further %s processing can occur", eventClassName)); + + try { + List events = drain(); + log.debug("Starting processing of {} {}s", events.size(), eventClassName); + + for (T event : events) { + try { + Objects.requireNonNull(event, () -> String.format("Attempted to process a null %s", eventClassName)); + log.debug("Consuming {}: {}", eventClassName, event); + process(event); + } catch (Throwable t) { + log.warn("An error occurred when processing the {}: {}", eventClassName, t.getMessage(), t); + + KafkaException error; + + if (t instanceof KafkaException) + error = (KafkaException) t; + else + error = new KafkaException(t); + + processErrorHandler.onProcessingError(error); + } + } + } finally { + log.debug("Completed processing of {}s", eventClassName); + } + } + + @Override + public void close() { + closer.close(this::closeInternal, () -> log.warn("Already closed")); + } + + /** + * It is possible for the consumer to close before complete processing all the events in the queue. In + * this case, we need to throw an exception to notify the user the consumer is closed. + */ + private void closeInternal() { + String eventClassName = getEventClass().getSimpleName(); + log.debug("Closing {} processor", eventClassName); + List incompleteEvents = drain(); + + if (incompleteEvents.isEmpty()) + return; + + KafkaException exception = new KafkaException("The consumer is closed"); + + // Check each of the events and if it has a Future, complete it exceptionally. + incompleteEvents + .stream() + .filter(e -> e instanceof CompletableEvent) + .map(e -> ((CompletableEvent) e).future()) + .forEach(f -> { + log.debug("Completing {} with exception {}", f, exception.getMessage()); + f.completeExceptionally(exception); + }); + + log.debug("Discarding {} {}s because the consumer is closing", + incompleteEvents.size(), + eventClassName); + } + + public abstract void process(T event); + + protected abstract Class getEventClass(); + + /** + * Moves all the events from the queue to the returned list. + */ + private List drain() { + LinkedList events = new LinkedList<>(); + eventQueue.drainTo(events); + return events; + } + + public interface ProcessErrorHandler { + + void onProcessingError(KafkaException error); + } +} diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/NoopApplicationEvent.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/NoopApplicationEvent.java deleted file mode 100644 index 22817fb2bae9f..0000000000000 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/NoopApplicationEvent.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.kafka.clients.consumer.internals.events; - -import java.util.Objects; - -/** - * The event is a no-op, but is intentionally left here for demonstration and test purposes. - */ -public class NoopApplicationEvent extends ApplicationEvent { - - private final String message; - - public NoopApplicationEvent(final String message) { - super(Type.NOOP); - this.message = Objects.requireNonNull(message); - } - - public String message() { - return message; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - if (!super.equals(o)) return false; - - NoopApplicationEvent that = (NoopApplicationEvent) o; - - return message.equals(that.message); - } - - @Override - public int hashCode() { - int result = super.hashCode(); - result = 31 * result + message.hashCode(); - return result; - } - - @Override - public String toString() { - return "NoopApplicationEvent{" + - toStringBase() + - ",message='" + message + '\'' + - '}'; - } -} \ No newline at end of file diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/NoopBackgroundEvent.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/NoopBackgroundEvent.java deleted file mode 100644 index c1cbcc253a9c7..0000000000000 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/NoopBackgroundEvent.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.kafka.clients.consumer.internals.events; - -import java.util.Objects; - -/** - * No-op event. Intentionally left it here for demonstration purpose. - */ -public class NoopBackgroundEvent extends BackgroundEvent { - - private final String message; - - public NoopBackgroundEvent(final String message) { - super(Type.NOOP); - this.message = Objects.requireNonNull(message); - } - - public String message() { - return message; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - if (!super.equals(o)) return false; - - NoopBackgroundEvent that = (NoopBackgroundEvent) o; - - return message.equals(that.message); - } - - @Override - public int hashCode() { - int result = super.hashCode(); - result = 31 * result + message.hashCode(); - return result; - } - - @Override - public String toString() { - return "NoopBackgroundEvent{" + - toStringBase() + - ", message='" + message + '\'' + - '}'; - } -} \ No newline at end of file diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java index 46af158e446c6..0282064ec7a4c 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java @@ -27,7 +27,6 @@ import org.apache.kafka.clients.consumer.internals.events.BackgroundEvent; import org.apache.kafka.clients.consumer.internals.events.BackgroundEventProcessor; import org.apache.kafka.clients.consumer.internals.events.EventHandler; -import org.apache.kafka.common.IsolationLevel; import org.apache.kafka.common.internals.ClusterResourceListeners; import org.apache.kafka.common.metrics.Metrics; import org.apache.kafka.common.requests.MetadataResponse; @@ -166,10 +165,11 @@ public ConsumerTestBuilder() { Optional.of(coordinatorRequestManager), Optional.of(commitRequestManager)); this.applicationEventProcessor = spy(new ApplicationEventProcessor( - backgroundEventQueue, + logContext, + applicationEventQueue, requestManagers, - metadata, - logContext)); + metadata) + ); this.backgroundEventProcessor = spy(new BackgroundEventProcessor(logContext, backgroundEventQueue)); } @@ -186,10 +186,10 @@ public DefaultBackgroundThreadTestBuilder() { this.backgroundThread = new DefaultBackgroundThread( time, logContext, - applicationEventQueue, () -> applicationEventProcessor, () -> networkClientDelegate, - () -> requestManagers); + () -> requestManagers + ); } @Override diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThreadTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThreadTest.java index f6fb9d70c8735..80201075d2aa7 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThreadTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThreadTest.java @@ -24,9 +24,9 @@ import org.apache.kafka.clients.consumer.internals.events.AssignmentChangeApplicationEvent; import org.apache.kafka.clients.consumer.internals.events.CommitApplicationEvent; import org.apache.kafka.clients.consumer.internals.events.CompletableApplicationEvent; +import org.apache.kafka.clients.consumer.internals.events.FetchEvent; import org.apache.kafka.clients.consumer.internals.events.ListOffsetsApplicationEvent; import org.apache.kafka.clients.consumer.internals.events.NewTopicsMetadataUpdateRequestEvent; -import org.apache.kafka.clients.consumer.internals.events.NoopApplicationEvent; import org.apache.kafka.clients.consumer.internals.events.ResetPositionsApplicationEvent; import org.apache.kafka.clients.consumer.internals.events.TopicMetadataApplicationEvent; import org.apache.kafka.clients.consumer.internals.events.ValidatePositionsApplicationEvent; @@ -119,7 +119,7 @@ public void testStartupAndTearDown() throws InterruptedException { @Test public void testApplicationEvent() { - ApplicationEvent e = new NoopApplicationEvent("noop event"); + FetchEvent e = new FetchEvent(); applicationEventsQueue.add(e); backgroundThread.runOnce(); verify(applicationEventProcessor, times(1)).process(e); diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/DefaultEventHandlerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/DefaultEventHandlerTest.java index 1bbd1d1bab083..38cfafbde109b 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/DefaultEventHandlerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/DefaultEventHandlerTest.java @@ -18,7 +18,7 @@ import org.apache.kafka.clients.consumer.internals.events.ApplicationEvent; import org.apache.kafka.clients.consumer.internals.events.EventHandler; -import org.apache.kafka.clients.consumer.internals.events.NoopApplicationEvent; +import org.apache.kafka.clients.consumer.internals.events.FetchEvent; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -48,7 +48,7 @@ public void tearDown() { @Test public void testBasicHandlerOps() { - handler.add(new NoopApplicationEvent("test")); + handler.add(new FetchEvent()); assertEquals(1, aq.size()); } } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ErrorEventHandlerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ErrorEventHandlerTest.java index f86203a755dbb..0c74a7d8a284e 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ErrorEventHandlerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ErrorEventHandlerTest.java @@ -19,7 +19,7 @@ import org.apache.kafka.clients.consumer.internals.events.BackgroundEvent; import org.apache.kafka.clients.consumer.internals.events.BackgroundEventProcessor; import org.apache.kafka.clients.consumer.internals.events.ErrorBackgroundEvent; -import org.apache.kafka.clients.consumer.internals.events.NoopBackgroundEvent; +import org.apache.kafka.clients.consumer.internals.events.EventProcessor; import org.apache.kafka.common.KafkaException; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -57,16 +57,16 @@ public void tearDown() { @Test public void testNoEvents() { assertTrue(backgroundEventQueue.isEmpty()); - backgroundEventProcessor.process(); + backgroundEventProcessor.process(ignored -> { }); assertTrue(backgroundEventQueue.isEmpty()); } @Test public void testSingleEvent() { - BackgroundEvent event = new NoopBackgroundEvent("A"); + BackgroundEvent event = new ErrorBackgroundEvent(new RuntimeException("A")); backgroundEventQueue.add(event); assertPeeked(event); - backgroundEventProcessor.process(); + backgroundEventProcessor.process(ignored -> { }); assertTrue(backgroundEventQueue.isEmpty()); } @@ -76,40 +76,18 @@ public void testSingleErrorEvent() { BackgroundEvent event = new ErrorBackgroundEvent(error); errorEventHandler.handle(error); assertPeeked(event); - assertThrows(error); - } - - @Test - public void testInvalidEvent() { - String message = "I'm a naughty error!"; - BackgroundEvent event = new BackgroundEvent(BackgroundEvent.Type.NOOP) { - @Override - public Type type() { - return null; - } - - @Override - public String toString() { - return message; - } - }; - - backgroundEventQueue.add(event); - assertPeeked(event); - - Exception error = new NullPointerException(String.format("Attempt to process BackgroundEvent with null type: %s", message)); - assertThrows(error); + assertProcessThrows(error); } @Test public void testMultipleEvents() { - BackgroundEvent event1 = new NoopBackgroundEvent("A"); + BackgroundEvent event1 = new ErrorBackgroundEvent(new RuntimeException("A")); backgroundEventQueue.add(event1); - backgroundEventQueue.add(new NoopBackgroundEvent("B")); - backgroundEventQueue.add(new NoopBackgroundEvent("C")); + backgroundEventQueue.add(new ErrorBackgroundEvent(new RuntimeException("B"))); + backgroundEventQueue.add(new ErrorBackgroundEvent(new RuntimeException("C"))); assertPeeked(event1); - backgroundEventProcessor.process(); + backgroundEventProcessor.process(ignored -> { }); assertTrue(backgroundEventQueue.isEmpty()); } @@ -123,7 +101,7 @@ public void testMultipleErrorEvents() { errorEventHandler.handle(error2); errorEventHandler.handle(error3); - assertThrows(new KafkaException(error1)); + assertProcessThrows(new KafkaException(error1)); } @Test @@ -132,15 +110,16 @@ public void testMixedEventsWithErrorEvents() { KafkaException error2 = new KafkaException("error2"); KafkaException error3 = new KafkaException("error3"); - backgroundEventQueue.add(new NoopBackgroundEvent("A")); + RuntimeException errorToCheck = new RuntimeException("A"); + backgroundEventQueue.add(new ErrorBackgroundEvent(errorToCheck)); errorEventHandler.handle(error1); - backgroundEventQueue.add(new NoopBackgroundEvent("B")); + backgroundEventQueue.add(new ErrorBackgroundEvent(new RuntimeException("B"))); errorEventHandler.handle(error2); - backgroundEventQueue.add(new NoopBackgroundEvent("C")); + backgroundEventQueue.add(new ErrorBackgroundEvent(new RuntimeException("C"))); errorEventHandler.handle(error3); - backgroundEventQueue.add(new NoopBackgroundEvent("D")); + backgroundEventQueue.add(new ErrorBackgroundEvent(new RuntimeException("D"))); - assertThrows(new KafkaException(error1)); + assertProcessThrows(new KafkaException(errorToCheck)); } private void assertPeeked(BackgroundEvent event) { @@ -149,11 +128,13 @@ private void assertPeeked(BackgroundEvent event) { assertEquals(event, peekEvent); } - private void assertThrows(Throwable error) { + private void assertProcessThrows(Throwable error) { assertFalse(backgroundEventQueue.isEmpty()); try { - backgroundEventProcessor.process(); + TestProcessHandler processHandler = new TestProcessHandler(); + backgroundEventProcessor.process(processHandler); + processHandler.maybeThrow(); fail("Should have thrown error: " + error); } catch (Throwable t) { assertEquals(error.getClass(), t.getClass()); @@ -162,4 +143,20 @@ private void assertThrows(Throwable error) { assertTrue(backgroundEventQueue.isEmpty()); } + + private static class TestProcessHandler implements EventProcessor.ProcessErrorHandler { + + private KafkaException first; + + @Override + public void onProcessingError(KafkaException error) { + if (first == null) + first = error; + } + + void maybeThrow() { + if (first != null) + throw first; + } + } } From 1e18f989a3ae4d4c10abc8b60e37183809872a28 Mon Sep 17 00:00:00 2001 From: Kirk True Date: Tue, 26 Sep 2023 08:53:28 -0700 Subject: [PATCH 09/72] Changes to PrototypeAsyncConsumer.close() Refactored the close method so that it's much cleaner. Also fixed a confusing doc change to refreshCommittedOffsetsIfNeeded(). --- .../consumer/internals/FetchBuffer.java | 3 +- .../internals/PrototypeAsyncConsumer.java | 39 +++---------------- .../internals/ConsumerTestBuilder.java | 1 - 3 files changed, 7 insertions(+), 36 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchBuffer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchBuffer.java index d82b37ad99a90..c7944e294e2e0 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchBuffer.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchBuffer.java @@ -21,7 +21,6 @@ import org.apache.kafka.common.utils.LogContext; import org.slf4j.Logger; -import java.io.Closeable; import java.util.Collection; import java.util.HashSet; import java.util.Set; @@ -37,7 +36,7 @@ * * Note: this class is not thread-safe and is intended to only be used from a single thread. */ -public class FetchBuffer implements Closeable { +public class FetchBuffer implements AutoCloseable { private final Logger log; private final ConcurrentLinkedQueue completedFetches; diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java index 1d8cfe7d5e31c..33d9a25867502 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java @@ -68,7 +68,6 @@ import org.apache.kafka.common.utils.Time; import org.apache.kafka.common.utils.Timer; import org.slf4j.Logger; -import org.slf4j.event.Level; import java.net.InetSocketAddress; import java.time.Duration; @@ -108,7 +107,6 @@ import static org.apache.kafka.common.utils.Utils.isBlank; import static org.apache.kafka.common.utils.Utils.join; import static org.apache.kafka.common.utils.Utils.propsToMap; -import static org.apache.kafka.common.utils.Utils.swallow; /** * This prototype consumer uses the EventHandler to process application @@ -136,7 +134,6 @@ public class PrototypeAsyncConsumer implements Consumer { private final ConsumerMetadata metadata; private final Metrics metrics; private final long retryBackoffMs; - private final long requestTimeoutMs; private final long defaultApiTimeoutMs; private volatile boolean closed = false; private final List assignors; @@ -184,7 +181,6 @@ public PrototypeAsyncConsumer(final Time time, }); log.debug("Initializing the Kafka consumer"); - this.requestTimeoutMs = config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG); this.defaultApiTimeoutMs = config.getInt(ConsumerConfig.DEFAULT_API_TIMEOUT_MS_CONFIG); this.time = time; this.metrics = createMetrics(config, time); @@ -265,7 +261,7 @@ public PrototypeAsyncConsumer(final Time time, // call close methods if internal objects are already constructed; this is to prevent resource leak. see KAFKA-2121 // we do not need to call `close` at all when `log` is null, which means no internal objects were initialized. if (this.log != null) { - close(Duration.ZERO, true); + close(true); } // now propagate the exception throw new KafkaException("Failed to construct kafka consumer", t); @@ -285,7 +281,6 @@ public PrototypeAsyncConsumer(LogContext logContext, SubscriptionState subscriptions, ConsumerMetadata metadata, long retryBackoffMs, - long requestTimeoutMs, int defaultApiTimeoutMs, List assignors, String groupId) { @@ -302,7 +297,6 @@ public PrototypeAsyncConsumer(LogContext logContext, this.groupId = Optional.ofNullable(groupId); this.metadata = metadata; this.retryBackoffMs = retryBackoffMs; - this.requestTimeoutMs = requestTimeoutMs; this.defaultApiTimeoutMs = defaultApiTimeoutMs; this.deserializers = deserializers; this.eventHandler = eventHandler; @@ -683,12 +677,6 @@ public void close() { close(Duration.ofMillis(DEFAULT_CLOSE_TIMEOUT_MS)); } - private Timer createTimerForRequest(final Duration timeout) { - // this.time could be null if an exception occurs in constructor prior to setting the this.time field - final Time localTime = (time == null) ? Time.SYSTEM : time; - return localTime.timer(Math.min(timeout.toMillis(), requestTimeoutMs)); - } - @Override public void close(Duration timeout) { if (timeout.toMillis() < 0) @@ -698,36 +686,21 @@ public void close(Duration timeout) { if (!closed) { // need to close before setting the flag since the close function // itself may trigger rebalance callback that needs the consumer to be open still - close(timeout, false); + close(false); } } finally { closed = true; } } - private void close(Duration timeout, boolean swallowException) { + private void close(boolean swallowException) { log.trace("Closing the Kafka consumer"); AtomicReference firstException = new AtomicReference<>(); - - final Timer closeTimer = createTimerForRequest(timeout); - if (fetchBuffer != null) { - // the timeout for the session close is at-most the requestTimeoutMs - long remainingDurationInTimeout = Math.max(0, timeout.toMillis() - closeTimer.elapsedMs()); - if (remainingDurationInTimeout > 0) { - remainingDurationInTimeout = Math.min(requestTimeoutMs, remainingDurationInTimeout); - } - - closeTimer.reset(remainingDurationInTimeout); - - // This is a blocking call bound by the time remaining in closeTimer - swallow(log, Level.ERROR, "Failed to close fetcher", fetchBuffer::close, firstException); - } - - + closeQuietly(fetchBuffer, "Failed to close fetcher", firstException); closeQuietly(interceptors, "consumer interceptors", firstException); closeQuietly(kafkaConsumerMetrics, "kafka consumer metrics", firstException); closeQuietly(metrics, "consumer metrics", firstException); - closeQuietly(this.eventHandler, "event handler", firstException); + closeQuietly(eventHandler, "event handler", firstException); closeQuietly(deserializers, "consumer deserializers", firstException); AppInfoParser.unregisterAppInfo(CONSUMER_JMX_PREFIX, clientId, metrics); @@ -1014,7 +987,7 @@ private boolean isCommittedOffsetsManagementEnabled() { } /** - * Refresh the committed offsets for provided partitions. + * Refresh the committed offsets for partitions that require initialization. * * @param timer Timer bounding how long this method can block * @return true iff the operation completed within the timeout diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java index 0282064ec7a4c..b9479d930e036 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java @@ -250,7 +250,6 @@ public PrototypeAsyncConsumerTestBuilder(Optional groupIdOpt) { subscriptions, metadata, retryBackoffMs, - REQUEST_TIMEOUT_MS, 60000, assignors, groupIdOpt.orElse(null))); From 4a9cdd08773cba8d72c39a5a4e46b0e8c9f35019 Mon Sep 17 00:00:00 2001 From: Kirk True Date: Tue, 26 Sep 2023 12:01:54 -0700 Subject: [PATCH 10/72] Updates to the fetch results/buffering process FetchEvent now provides the Future to the FetchRequestManager which will directly respond with the results of the fetch. This allows the application thread to properly block in pollForFetches() as it waits for a response. A separate thread-safe blocking queue collects the results from the Future, as opposed to collecting the results from the future in the fetch buffer, as that is not thread safe and should not be used across threads. --- .../internals/FetchRequestManager.java | 50 ++++++++++++++++++- .../internals/PrototypeAsyncConsumer.java | 35 +++++++++---- .../events/ApplicationEventProcessor.java | 5 +- 3 files changed, 76 insertions(+), 14 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchRequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchRequestManager.java index 9c010d722f0b4..25b08035227bc 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchRequestManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchRequestManager.java @@ -16,15 +16,19 @@ */ package org.apache.kafka.clients.consumer.internals; +import java.util.ArrayList; import java.util.LinkedList; import java.util.List; import java.util.Queue; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Future; import java.util.function.BiConsumer; import java.util.function.Predicate; import java.util.stream.Collectors; import org.apache.kafka.clients.ClientResponse; import org.apache.kafka.clients.FetchSessionHandler; +import org.apache.kafka.clients.NetworkClient; import org.apache.kafka.clients.consumer.internals.NetworkClientDelegate.PollResult; import org.apache.kafka.clients.consumer.internals.NetworkClientDelegate.UnsentRequest; import org.apache.kafka.common.Node; @@ -43,6 +47,7 @@ public class FetchRequestManager extends AbstractFetch implements RequestManager private final Logger log; private final ErrorEventHandler errorEventHandler; private final NetworkClientDelegate networkClientDelegate; + private final List>> futures; FetchRequestManager(final LogContext logContext, final Time time, @@ -56,6 +61,7 @@ public class FetchRequestManager extends AbstractFetch implements RequestManager this.log = logContext.logger(FetchRequestManager.class); this.errorEventHandler = errorEventHandler; this.networkClientDelegate = networkClientDelegate; + this.futures = new ArrayList<>(); } @Override @@ -68,6 +74,17 @@ protected void maybeThrowAuthFailure(Node node) { networkClientDelegate.maybeThrowAuthFailure(node); } + /** + * Adds a new {@link Future future} to the list of futures awaiting results. Per the comments on + * {@link #forwardResults()}, there is no guarantee that this particular future will be provided with + * a non-empty result, but it is guaranteed to be completed with a result, assuming that it does not time out. + * + * @param future Future that will be {@link CompletableFuture#complete(Object) completed} if not timed out + */ + public void requestFetch(CompletableFuture> future) { + futures.add(future); + } + @Override public PollResult poll(long currentTimeMs) { List requests; @@ -85,6 +102,7 @@ public PollResult poll(long currentTimeMs) { errorEventHandler.handle(t); } else { handleFetchResponse(fetchTarget, data, clientResponse); + forwardResults(); } }; @@ -112,6 +130,36 @@ public PollResult poll(long currentTimeMs) { return new PollResult(Long.MAX_VALUE, requests); } + /** + * Drains any of the {@link CompletedFetch completed fetch} objects from the + * {@link FetchBuffer internal fetch buffer} and provides the results to awaiting {@link Future futures} + * from the application thread. + * + *

    + * + * For each future in {@link #futures} that isn't completed at the time that this method is invoked, we will + * drain the available, buffered data from the underlying fetch buffer and provide it to the future. As the + * {@link NetworkClient} loops through the completed I/O, each response handler is invoked. Those response handlers + * will then re-populate the internal fetch buffer before calling this method. + * + *

    + * + * It may be that some of the futures will be completed with an empty queue if the futures + * ahead of it read all the results. Empty results should be accounted for in the code. + */ + private void forwardResults() { + for (CompletableFuture> f : futures) { + if (f.isDone()) + continue; + + Queue q = drain(); + f.complete(q); + } + + // Clear the list of futures here as they have fulfilled their purpose. + futures.clear(); + } + /** * Drains any of the {@link CompletedFetch} objects from the internal buffer to the returned {@link Queue}. * @@ -122,7 +170,7 @@ public PollResult poll(long currentTimeMs) { * * @return {@link Queue} containing zero or more {@link CompletedFetch} */ - public Queue drain() { + private Queue drain() { Queue q = new LinkedList<>(); CompletedFetch completedFetch = fetchBuffer.poll(); diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java index 33d9a25867502..0afae7484d815 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java @@ -82,11 +82,11 @@ import java.util.Optional; import java.util.OptionalLong; import java.util.Properties; -import java.util.Queue; import java.util.Set; import java.util.concurrent.BlockingQueue; import java.util.concurrent.CompletableFuture; import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; import java.util.function.Supplier; @@ -142,6 +142,14 @@ public class PrototypeAsyncConsumer implements Consumer { private boolean cachedSubscriptionHasAllFetchPositions; private final WakeupTrigger wakeupTrigger = new WakeupTrigger(); + /** + * A thread-safe {@link BlockingQueue queue} for the results that are populated in the background thread + * when the fetch results are available. Because the {@link #fetchBuffer fetch buffer} is not thread-safe, we + * need to separate the results collection that we provide to the background thread from the collection that + * we read from on the application thread. + */ + private final BlockingQueue fetchResults = new LinkedBlockingQueue<>(); + public PrototypeAsyncConsumer(final Properties properties, final Deserializer keyDeserializer, final Deserializer valueDeserializer) { @@ -889,14 +897,19 @@ WakeupTrigger wakeupTrigger() { return wakeupTrigger; } + /** + * Send the requests for fetch data to the background thread and set up to collect the results in + * {@link #fetchResults}. + */ private void sendFetches() { FetchEvent event = new FetchEvent(); eventHandler.add(event); event.future().whenComplete((completedFetches, error) -> { - if (completedFetches != null && !completedFetches.isEmpty()) { - fetchBuffer.addAll(completedFetches); - } + if (error != null) + log.warn("An error occurred during poll: {}", error.getMessage(), error); + else + fetchResults.addAll(completedFetches); }); } @@ -925,15 +938,19 @@ private Fetch pollForFetches(Timer timer) { Timer pollTimer = time.timer(pollTimeout); - // Attempt to fetch any data. It's OK if we time out here; it's a 'best case' effort. The + // Attempt to fetch any data. It's OK if we don't have any waiting data here; it's a 'best case' effort. The // data may not be immediately available, but the calling method (poll) will correctly // handle the overall timeout. try { - Queue completedFetches = eventHandler.addAndGet(new FetchEvent(), pollTimer); - if (completedFetches != null && !completedFetches.isEmpty()) { - fetchBuffer.addAll(completedFetches); + while (pollTimer.notExpired()) { + CompletedFetch completedFetch = fetchResults.poll(pollTimer.remainingMs(), TimeUnit.MILLISECONDS); + + if (completedFetch != null) + fetchBuffer.add(completedFetch); + + pollTimer.update(); } - } catch (TimeoutException e) { + } catch (InterruptedException e) { log.trace("Timeout during fetch", e); } finally { timer.update(pollTimer.currentTimeMs()); diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventProcessor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventProcessor.java index bc01130aa08e7..64ece0186a5de 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventProcessor.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventProcessor.java @@ -174,10 +174,7 @@ private void process(final TopicMetadataApplicationEvent event) { } private void process(final FetchEvent event) { - // The request manager keeps track of the completed fetches, so we pull any that are ready off, and return - // them to the application. - Queue completedFetches = requestManagers.fetchRequestManager.drain(); - event.future().complete(completedFetches); + requestManagers.fetchRequestManager.requestFetch(event.future()); } /** From c8734630f6734eb82bc3048c943d27205008bb08 Mon Sep 17 00:00:00 2001 From: Kirk True Date: Tue, 26 Sep 2023 15:30:15 -0700 Subject: [PATCH 11/72] Updates for background event handler 1. Renamed DefaultEventHandler to ApplicationEventHandler 2. Renamed ErrorEventHandler to BackgroundEventHandler 3. Deleted EventHandler interface 4. Moved ApplicationEventHandler and BackgroundEventHandler (and their unit tests) into the "events" sub-package for consistency 5. Removed use of mocks in DefaultBackgroundThreadTest as they caused issues with various methods returning null --- .../internals/CoordinatorRequestManager.java | 14 ++-- .../consumer/internals/ErrorEventHandler.java | 40 ----------- .../internals/FetchRequestManager.java | 14 ++-- .../internals/PrototypeAsyncConsumer.java | 34 +++++----- .../consumer/internals/RequestManagers.java | 7 +- .../ApplicationEventHandler.java} | 68 ++++++++++++------- .../events/ApplicationEventProcessor.java | 2 - .../events/BackgroundEventHandler.java | 52 ++++++++++++++ .../events/ErrorBackgroundEvent.java | 6 +- .../internals/events/EventHandler.java | 60 ---------------- .../internals/ConsumerTestBuilder.java | 38 ++++++----- .../CoordinatorRequestManagerTest.java | 21 ++++-- .../DefaultBackgroundThreadTest.java | 11 ++- .../internals/FetchRequestManagerTest.java | 7 +- .../internals/PrototypeAsyncConsumerTest.java | 48 ++++++------- .../ApplicationEventHandlerTest.java} | 24 +++---- .../BackgroundEventHandlerTest.java} | 33 ++++----- 17 files changed, 234 insertions(+), 245 deletions(-) delete mode 100644 clients/src/main/java/org/apache/kafka/clients/consumer/internals/ErrorEventHandler.java rename clients/src/main/java/org/apache/kafka/clients/consumer/internals/{DefaultEventHandler.java => events/ApplicationEventHandler.java} (54%) create mode 100644 clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEventHandler.java delete mode 100644 clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/EventHandler.java rename clients/src/test/java/org/apache/kafka/clients/consumer/internals/{DefaultEventHandlerTest.java => events/ApplicationEventHandlerTest.java} (63%) rename clients/src/test/java/org/apache/kafka/clients/consumer/internals/{ErrorEventHandlerTest.java => events/BackgroundEventHandlerTest.java} (82%) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CoordinatorRequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CoordinatorRequestManager.java index c119a6198929c..bb83468734be8 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CoordinatorRequestManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CoordinatorRequestManager.java @@ -16,6 +16,9 @@ */ package org.apache.kafka.clients.consumer.internals; +import org.apache.kafka.clients.consumer.internals.events.BackgroundEventHandler; +import org.apache.kafka.clients.consumer.internals.events.ErrorBackgroundEvent; +import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.Node; import org.apache.kafka.common.errors.GroupAuthorizationException; import org.apache.kafka.common.errors.RetriableException; @@ -49,7 +52,7 @@ public class CoordinatorRequestManager implements RequestManager { private static final long COORDINATOR_DISCONNECT_LOGGING_INTERVAL_MS = 60 * 1000; private final Time time; private final Logger log; - private final ErrorEventHandler nonRetriableErrorHandler; + private final BackgroundEventHandler backgroundEventHandler; private final String groupId; private final RequestState coordinatorRequestState; @@ -62,13 +65,13 @@ public CoordinatorRequestManager( final LogContext logContext, final long retryBackoffMs, final long retryBackoffMaxMs, - final ErrorEventHandler errorHandler, + final BackgroundEventHandler errorHandler, final String groupId ) { Objects.requireNonNull(groupId); this.time = time; this.log = logContext.logger(this.getClass()); - this.nonRetriableErrorHandler = errorHandler; + this.backgroundEventHandler = errorHandler; this.groupId = groupId; this.coordinatorRequestState = new RequestState( logContext, @@ -179,12 +182,13 @@ private void onFailedResponse( if (exception == Errors.GROUP_AUTHORIZATION_FAILED.exception()) { log.debug("FindCoordinator request failed due to authorization error {}", exception.getMessage()); - nonRetriableErrorHandler.handle(GroupAuthorizationException.forGroupId(this.groupId)); + KafkaException groupAuthorizationException = GroupAuthorizationException.forGroupId(this.groupId); + backgroundEventHandler.add(new ErrorBackgroundEvent(groupAuthorizationException)); return; } log.warn("FindCoordinator request failed due to fatal exception", exception); - nonRetriableErrorHandler.handle(exception); + backgroundEventHandler.add(new ErrorBackgroundEvent(exception)); } /** diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ErrorEventHandler.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ErrorEventHandler.java deleted file mode 100644 index a6eaf9c3bc0c1..0000000000000 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ErrorEventHandler.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.kafka.clients.consumer.internals; - -import org.apache.kafka.clients.consumer.internals.events.BackgroundEvent; -import org.apache.kafka.clients.consumer.internals.events.ErrorBackgroundEvent; -import org.apache.kafka.common.KafkaException; - -import java.util.Queue; - -public class ErrorEventHandler { - - private final Queue backgroundEventQueue; - - public ErrorEventHandler(Queue backgroundEventQueue) { - this.backgroundEventQueue = backgroundEventQueue; - } - - public void handle(RuntimeException e) { - backgroundEventQueue.add(new ErrorBackgroundEvent(e)); - } - - public void handle(Throwable e) { - handle(new KafkaException(e)); - } -} diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchRequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchRequestManager.java index 25b08035227bc..e82ee1596105e 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchRequestManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchRequestManager.java @@ -31,6 +31,8 @@ import org.apache.kafka.clients.NetworkClient; import org.apache.kafka.clients.consumer.internals.NetworkClientDelegate.PollResult; import org.apache.kafka.clients.consumer.internals.NetworkClientDelegate.UnsentRequest; +import org.apache.kafka.clients.consumer.internals.events.BackgroundEventHandler; +import org.apache.kafka.clients.consumer.internals.events.ErrorBackgroundEvent; import org.apache.kafka.common.Node; import org.apache.kafka.common.requests.FetchRequest; import org.apache.kafka.common.utils.LogContext; @@ -45,13 +47,13 @@ public class FetchRequestManager extends AbstractFetch implements RequestManager { private final Logger log; - private final ErrorEventHandler errorEventHandler; + private final BackgroundEventHandler backgroundEventHandler; private final NetworkClientDelegate networkClientDelegate; private final List>> futures; FetchRequestManager(final LogContext logContext, final Time time, - final ErrorEventHandler errorEventHandler, + final BackgroundEventHandler backgroundEventHandler, final ConsumerMetadata metadata, final SubscriptionState subscriptions, final FetchConfig fetchConfig, @@ -59,7 +61,7 @@ public class FetchRequestManager extends AbstractFetch implements RequestManager final NetworkClientDelegate networkClientDelegate) { super(logContext, metadata, subscriptions, fetchConfig, metricsManager, time); this.log = logContext.logger(FetchRequestManager.class); - this.errorEventHandler = errorEventHandler; + this.backgroundEventHandler = backgroundEventHandler; this.networkClientDelegate = networkClientDelegate; this.futures = new ArrayList<>(); } @@ -98,8 +100,8 @@ public PollResult poll(long currentTimeMs) { final BiConsumer responseHandler = (clientResponse, t) -> { if (t != null) { handleFetchResponse(fetchTarget, t); - log.warn("Attempt to fetch data from node {} failed due to fatal exception", fetchTarget, t); - errorEventHandler.handle(t); + log.debug("Attempt to fetch data from node {} failed due to fatal exception", fetchTarget, t); + backgroundEventHandler.add(new ErrorBackgroundEvent(t)); } else { handleFetchResponse(fetchTarget, data, clientResponse); forwardResults(); @@ -117,7 +119,7 @@ public PollResult poll(long currentTimeMs) { if (t != null) { handleCloseFetchSessionResponse(fetchTarget, data, t); log.warn("Attempt to close fetch session on node {} failed due to fatal exception", fetchTarget, t); - errorEventHandler.handle(t); + backgroundEventHandler.add(new ErrorBackgroundEvent(t)); } else { handleCloseFetchSessionResponse(fetchTarget, data); } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java index 0afae7484d815..786e6ad12565a 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java @@ -38,10 +38,10 @@ import org.apache.kafka.clients.consumer.internals.events.AssignmentChangeApplicationEvent; import org.apache.kafka.clients.consumer.internals.events.BackgroundEvent; import org.apache.kafka.clients.consumer.internals.events.BackgroundEventProcessor; +import org.apache.kafka.clients.consumer.internals.events.ApplicationEventHandler; import org.apache.kafka.clients.consumer.internals.events.EventProcessor.ProcessErrorHandler; import org.apache.kafka.clients.consumer.internals.events.CommitApplicationEvent; import org.apache.kafka.clients.consumer.internals.events.ErrorBackgroundEvent; -import org.apache.kafka.clients.consumer.internals.events.EventHandler; import org.apache.kafka.clients.consumer.internals.events.FetchEvent; import org.apache.kafka.clients.consumer.internals.events.ListOffsetsApplicationEvent; import org.apache.kafka.clients.consumer.internals.events.NewTopicsMetadataUpdateRequestEvent; @@ -117,7 +117,7 @@ public class PrototypeAsyncConsumer implements Consumer { static final long DEFAULT_CLOSE_TIMEOUT_MS = 30 * 1000; - private final EventHandler eventHandler; + private final ApplicationEventHandler applicationEventHandler; private final Time time; private final Optional groupId; private final KafkaConsumerMetrics kafkaConsumerMetrics; @@ -233,7 +233,7 @@ public PrototypeAsyncConsumer(final Time time, metadata, applicationEventQueue, requestManagersSupplier); - this.eventHandler = new DefaultEventHandler(time, + this.applicationEventHandler = new ApplicationEventHandler(time, logContext, applicationEventQueue, applicationEventProcessorSupplier, @@ -283,7 +283,7 @@ public PrototypeAsyncConsumer(LogContext logContext, FetchCollector fetchCollector, ConsumerInterceptors interceptors, Time time, - EventHandler eventHandler, + ApplicationEventHandler applicationEventHandler, BlockingQueue backgroundEventQueue, Metrics metrics, SubscriptionState subscriptions, @@ -307,13 +307,13 @@ public PrototypeAsyncConsumer(LogContext logContext, this.retryBackoffMs = retryBackoffMs; this.defaultApiTimeoutMs = defaultApiTimeoutMs; this.deserializers = deserializers; - this.eventHandler = eventHandler; + this.applicationEventHandler = applicationEventHandler; this.assignors = assignors; this.kafkaConsumerMetrics = new KafkaConsumerMetrics(metrics, "consumer"); } /** - * poll implementation using {@link EventHandler}. + * poll implementation using {@link ApplicationEventHandler}. * 1. Poll for background events. If there's a fetch response event, process the record and return it. If it is * another type of event, process it. * 2. Send fetches if needed. @@ -402,7 +402,7 @@ CompletableFuture commit(Map offsets, f // the task can only be woken up if the top level API call is commitSync wakeupTrigger.setActiveTask(commitEvent.future()); } - eventHandler.add(commitEvent); + applicationEventHandler.add(commitEvent); return commitEvent.future(); } @@ -510,7 +510,7 @@ public Map committed(final Set offsetsForTimes(Map beginningOrEndOffset(Collection offsetAndTimestampMap = eventHandler.addAndGet( + Map offsetAndTimestampMap = applicationEventHandler.addAndGet( listOffsetsEvent, time.timer(timeout) ); @@ -708,7 +708,7 @@ private void close(boolean swallowException) { closeQuietly(interceptors, "consumer interceptors", firstException); closeQuietly(kafkaConsumerMetrics, "kafka consumer metrics", firstException); closeQuietly(metrics, "consumer metrics", firstException); - closeQuietly(eventHandler, "event handler", firstException); + closeQuietly(applicationEventHandler, "event handler", firstException); closeQuietly(deserializers, "consumer deserializers", firstException); AppInfoParser.unregisterAppInfo(CONSUMER_JMX_PREFIX, clientId, metrics); @@ -836,11 +836,11 @@ public void assign(Collection partitions) { // make sure the offsets of topic partitions the consumer is unsubscribing from // are committed since there will be no following rebalance - eventHandler.add(new AssignmentChangeApplicationEvent(this.subscriptions.allConsumed(), time.milliseconds())); + applicationEventHandler.add(new AssignmentChangeApplicationEvent(this.subscriptions.allConsumed(), time.milliseconds())); log.info("Assigned to partition(s): {}", join(partitions, ", ")); if (this.subscriptions.assignFromUser(new HashSet<>(partitions))) - eventHandler.add(new NewTopicsMetadataUpdateRequestEvent()); + applicationEventHandler.add(new NewTopicsMetadataUpdateRequestEvent()); } @Override @@ -903,7 +903,7 @@ WakeupTrigger wakeupTrigger() { */ private void sendFetches() { FetchEvent event = new FetchEvent(); - eventHandler.add(event); + applicationEventHandler.add(event); event.future().whenComplete((completedFetches, error) -> { if (error != null) @@ -970,7 +970,7 @@ private Fetch pollForFetches(Timer timer) { */ private boolean updateFetchPositions(final Timer timer) { // If any partitions have been truncated due to a leader change, we need to validate the offsets - eventHandler.add(new ValidatePositionsApplicationEvent()); + applicationEventHandler.add(new ValidatePositionsApplicationEvent()); cachedSubscriptionHasAllFetchPositions = subscriptions.hasAllFetchPositions(); if (cachedSubscriptionHasAllFetchPositions) return true; @@ -990,7 +990,7 @@ private boolean updateFetchPositions(final Timer timer) { // Finally send an asynchronous request to look up and update the positions of any // partitions which are awaiting reset. - eventHandler.add(new ResetPositionsApplicationEvent()); + applicationEventHandler.add(new ResetPositionsApplicationEvent()); return true; } @@ -1014,7 +1014,7 @@ private boolean refreshCommittedOffsetsIfNeeded(Timer timer) { log.debug("Refreshing committed offsets for partitions {}", initializingPartitions); try { - final Map offsets = eventHandler.addAndGet(new OffsetFetchApplicationEvent(initializingPartitions), timer); + final Map offsets = applicationEventHandler.addAndGet(new OffsetFetchApplicationEvent(initializingPartitions), timer); return ConsumerUtils.refreshCommittedOffsets(offsets, this.metadata, this.subscriptions); } catch (org.apache.kafka.common.errors.TimeoutException e) { log.error("Couldn't refresh committed offsets before timeout expired"); diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java index 6a96a27caa41e..e58678e4d0ecb 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java @@ -20,6 +20,7 @@ import org.apache.kafka.clients.GroupRebalanceConfig; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.internals.events.BackgroundEvent; +import org.apache.kafka.clients.consumer.internals.events.BackgroundEventHandler; import org.apache.kafka.common.internals.IdempotentCloser; import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.Time; @@ -115,7 +116,7 @@ public static Supplier supplier(final Time time, @Override protected RequestManagers create() { final NetworkClientDelegate networkClientDelegate = networkClientDelegateSupplier.get(); - final ErrorEventHandler errorEventHandler = new ErrorEventHandler(backgroundEventQueue); + final BackgroundEventHandler backgroundEventHandler = new BackgroundEventHandler(logContext, backgroundEventQueue); final FetchConfig fetchConfig = new FetchConfig(config); long retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG); long retryBackoffMaxMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MAX_MS_CONFIG); @@ -131,7 +132,7 @@ protected RequestManagers create() { logContext); final FetchRequestManager fetch = new FetchRequestManager(logContext, time, - errorEventHandler, + backgroundEventHandler, metadata, subscriptions, fetchConfig, @@ -149,7 +150,7 @@ protected RequestManagers create() { logContext, retryBackoffMs, retryBackoffMaxMs, - errorEventHandler, + backgroundEventHandler, groupState.groupId); commit = new CommitRequestManager(time, logContext, subscriptions, config, coordinator, groupState); } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/DefaultEventHandler.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventHandler.java similarity index 54% rename from clients/src/main/java/org/apache/kafka/clients/consumer/internals/DefaultEventHandler.java rename to clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventHandler.java index 33f3e9be2f4d5..a5bda46e6f60f 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/DefaultEventHandler.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventHandler.java @@ -14,13 +14,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.kafka.clients.consumer.internals; +package org.apache.kafka.clients.consumer.internals.events; -import org.apache.kafka.clients.consumer.internals.events.ApplicationEvent; -import org.apache.kafka.clients.consumer.internals.events.ApplicationEventProcessor; -import org.apache.kafka.clients.consumer.internals.events.BackgroundEvent; -import org.apache.kafka.clients.consumer.internals.events.CompletableApplicationEvent; -import org.apache.kafka.clients.consumer.internals.events.EventHandler; +import org.apache.kafka.clients.consumer.internals.DefaultBackgroundThread; +import org.apache.kafka.clients.consumer.internals.NetworkClientDelegate; +import org.apache.kafka.clients.consumer.internals.RequestManagers; import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.internals.IdempotentCloser; import org.apache.kafka.common.utils.LogContext; @@ -28,29 +26,32 @@ import org.apache.kafka.common.utils.Timer; import org.slf4j.Logger; +import java.io.Closeable; import java.time.Duration; import java.util.Objects; import java.util.concurrent.BlockingQueue; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; import java.util.function.Supplier; /** - * An {@link EventHandler} that uses a single background thread to consume {@link ApplicationEvent} and produce - * {@link BackgroundEvent} from the {@link DefaultBackgroundThread}. + * An event handler that receives {@link ApplicationEvent application events} from the application thread which + * are then readable from the {@link ApplicationEventProcessor} in the background thread. */ -public class DefaultEventHandler implements EventHandler { +public class ApplicationEventHandler implements Closeable { private final Logger log; private final BlockingQueue applicationEventQueue; private final DefaultBackgroundThread backgroundThread; private final IdempotentCloser closer = new IdempotentCloser(); - public DefaultEventHandler(final Time time, - final LogContext logContext, - final BlockingQueue applicationEventQueue, - final Supplier applicationEventProcessorSupplier, - final Supplier networkClientDelegateSupplier, - final Supplier requestManagersSupplier) { - this.log = logContext.logger(DefaultEventHandler.class); + public ApplicationEventHandler(final Time time, + final LogContext logContext, + final BlockingQueue applicationEventQueue, + final Supplier applicationEventProcessorSupplier, + final Supplier networkClientDelegateSupplier, + final Supplier requestManagersSupplier) { + this.log = logContext.logger(ApplicationEventHandler.class); this.applicationEventQueue = applicationEventQueue; this.backgroundThread = new DefaultBackgroundThread(time, logContext, @@ -60,15 +61,31 @@ public DefaultEventHandler(final Time time, this.backgroundThread.start(); } - @Override - public boolean add(final ApplicationEvent event) { + /** + * Add an {@link ApplicationEvent} to the handler. + * + * @param event An {@link ApplicationEvent} created by the application thread + */ + public void add(final ApplicationEvent event) { Objects.requireNonNull(event, "ApplicationEvent provided to add must be non-null"); log.trace("Enqueued event: {}", event); backgroundThread.wakeup(); - return applicationEventQueue.add(event); + applicationEventQueue.add(event); } - @Override + /** + * Add a {@link CompletableApplicationEvent} to the handler. The method blocks waiting for the result, and will + * return the result value upon successful completion; otherwise throws an error. + * + *

    + * + * See {@link CompletableApplicationEvent#get(Timer)} and {@link Future#get(long, TimeUnit)} for more details. + * + * @param event A {@link CompletableApplicationEvent} created by the polling thread. + * @param timer Timer for which to wait for the event to complete + * @return Value that is the result of the event + * @param Type of return value of the event + */ public T addAndGet(final CompletableApplicationEvent event, final Timer timer) { Objects.requireNonNull(event, "CompletableApplicationEvent provided to addAndGet must be non-null"); Objects.requireNonNull(timer, "Timer provided to addAndGet must be non-null"); @@ -76,12 +93,17 @@ public T addAndGet(final CompletableApplicationEvent event, final Timer t return event.get(timer); } + @Override + public void close() { + close(Duration.ofMillis(Long.MAX_VALUE)); + } + public void close(final Duration timeout) { Objects.requireNonNull(timeout, "Duration provided to close must be non-null"); closer.close( () -> { - log.info("Closing the default consumer event handler"); + log.info("Closing the consumer application event handler"); try { long timeoutMs = timeout.toMillis(); @@ -90,11 +112,11 @@ public void close(final Duration timeout) { throw new IllegalArgumentException("The timeout cannot be negative."); backgroundThread.close(); - log.info("The default consumer event handler was closed"); + log.info("The consumer application event handler was closed"); } catch (final Exception e) { throw new KafkaException(e); } }, - () -> log.info("The default consumer event handler was already closed")); + () -> log.info("The consumer application event handler was already closed")); } } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventProcessor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventProcessor.java index 64ece0186a5de..7d7e5c2bf25cf 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventProcessor.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventProcessor.java @@ -19,7 +19,6 @@ import org.apache.kafka.clients.consumer.OffsetAndTimestamp; import org.apache.kafka.clients.consumer.internals.CachedSupplier; import org.apache.kafka.clients.consumer.internals.CommitRequestManager; -import org.apache.kafka.clients.consumer.internals.CompletedFetch; import org.apache.kafka.clients.consumer.internals.ConsumerMetadata; import org.apache.kafka.clients.consumer.internals.DefaultBackgroundThread; import org.apache.kafka.clients.consumer.internals.RequestManagers; @@ -31,7 +30,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.Queue; import java.util.concurrent.BlockingQueue; import java.util.concurrent.CompletableFuture; import java.util.function.Supplier; diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEventHandler.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEventHandler.java new file mode 100644 index 0000000000000..514a57770c00a --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEventHandler.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.clients.consumer.internals.events; + +import org.apache.kafka.clients.consumer.internals.DefaultBackgroundThread; +import org.apache.kafka.common.utils.LogContext; +import org.slf4j.Logger; + +import java.util.Objects; +import java.util.Queue; + +/** + * An event handler that receives {@link BackgroundEvent background events} from the + * {@link DefaultBackgroundThread background thread} which are then made available to the application thread + * via the {@link BackgroundEventProcessor}. + */ + +public class BackgroundEventHandler { + + private final Logger log; + private final Queue backgroundEventQueue; + + public BackgroundEventHandler(final LogContext logContext, final Queue backgroundEventQueue) { + this.log = logContext.logger(BackgroundEventHandler.class); + this.backgroundEventQueue = backgroundEventQueue; + } + + /** + * Add a {@link BackgroundEvent} to the handler. + * + * @param event A {@link BackgroundEvent} created by the {@link DefaultBackgroundThread background thread} + */ + public void add(BackgroundEvent event) { + Objects.requireNonNull(event, "BackgroundEvent provided to add must be non-null"); + log.trace("Enqueued event: {}", event); + backgroundEventQueue.add(event); + } +} diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ErrorBackgroundEvent.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ErrorBackgroundEvent.java index 7f3d4e22429e3..910927f981b77 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ErrorBackgroundEvent.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ErrorBackgroundEvent.java @@ -16,13 +16,15 @@ */ package org.apache.kafka.clients.consumer.internals.events; +import org.apache.kafka.common.KafkaException; + public class ErrorBackgroundEvent extends BackgroundEvent { private final RuntimeException error; - public ErrorBackgroundEvent(RuntimeException error) { + public ErrorBackgroundEvent(Throwable t) { super(Type.ERROR); - this.error = error; + this.error = t instanceof RuntimeException ? (RuntimeException) t : new KafkaException(t); } public RuntimeException error() { diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/EventHandler.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/EventHandler.java deleted file mode 100644 index 4c76b5f7bf104..0000000000000 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/EventHandler.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.kafka.clients.consumer.internals.events; - -import org.apache.kafka.common.utils.Timer; - -import java.io.Closeable; -import java.time.Duration; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; - -/** - * This class interfaces with the KafkaConsumer and the background thread. It allows the caller to enqueue events via - * the {@code add()} method and to retrieve events via the {@code poll()} method. - */ -public interface EventHandler extends Closeable { - - /** - * Add an {@link ApplicationEvent} to the handler. The method returns true upon successful add; otherwise returns - * false. - * @param event An {@link ApplicationEvent} created by the polling thread. - * @return true upon successful add. - */ - boolean add(ApplicationEvent event); - - /** - * Add a {@link CompletableApplicationEvent} to the handler. The method blocks waiting for the result, and will - * return the result value upon successful completion; otherwise throws an error. - * - *

    - * - * See {@link CompletableApplicationEvent#get(Timer)} and {@link Future#get(long, TimeUnit)} for more details. - * - * @param event A {@link CompletableApplicationEvent} created by the polling thread. - * @param timer Timer for which to wait for the event to complete - * @return Value that is the result of the event - * @param Type of return value of the event - */ - T addAndGet(final CompletableApplicationEvent event, final Timer timer); - - default void close() { - close(Duration.ofMillis(Long.MAX_VALUE)); - } - - void close(Duration timeout); -} diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java index b9479d930e036..2d4016a8d45cc 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java @@ -23,10 +23,11 @@ import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.ConsumerPartitionAssignor; import org.apache.kafka.clients.consumer.internals.events.ApplicationEvent; +import org.apache.kafka.clients.consumer.internals.events.ApplicationEventHandler; import org.apache.kafka.clients.consumer.internals.events.ApplicationEventProcessor; import org.apache.kafka.clients.consumer.internals.events.BackgroundEvent; +import org.apache.kafka.clients.consumer.internals.events.BackgroundEventHandler; import org.apache.kafka.clients.consumer.internals.events.BackgroundEventProcessor; -import org.apache.kafka.clients.consumer.internals.events.EventHandler; import org.apache.kafka.common.internals.ClusterResourceListeners; import org.apache.kafka.common.metrics.Metrics; import org.apache.kafka.common.requests.MetadataResponse; @@ -50,6 +51,7 @@ import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createFetchMetricsManager; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createMetrics; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createSubscriptionState; +import static org.apache.kafka.common.utils.Utils.closeQuietly; import static org.mockito.Mockito.spy; public class ConsumerTestBuilder implements Closeable { @@ -60,8 +62,8 @@ public class ConsumerTestBuilder implements Closeable { final LogContext logContext = new LogContext(); final Time time = new MockTime(0); - final BlockingQueue applicationEventQueue; - final LinkedBlockingQueue backgroundEventQueue; + public final BlockingQueue applicationEventQueue; + public final BlockingQueue backgroundEventQueue; final ConsumerConfig config; final long retryBackoffMs; final SubscriptionState subscriptions; @@ -76,14 +78,15 @@ public class ConsumerTestBuilder implements Closeable { final TopicMetadataRequestManager topicMetadataRequestManager; final FetchRequestManager fetchRequestManager; final RequestManagers requestManagers; - final ApplicationEventProcessor applicationEventProcessor; - final BackgroundEventProcessor backgroundEventProcessor; + public final ApplicationEventProcessor applicationEventProcessor; + public final BackgroundEventProcessor backgroundEventProcessor; + public final BackgroundEventHandler backgroundEventHandler; final MockClient client; public ConsumerTestBuilder() { this.applicationEventQueue = new LinkedBlockingQueue<>(); this.backgroundEventQueue = new LinkedBlockingQueue<>(); - ErrorEventHandler errorEventHandler = new ErrorEventHandler(backgroundEventQueue); + this.backgroundEventHandler = new BackgroundEventHandler(logContext, backgroundEventQueue); GroupRebalanceConfig groupRebalanceConfig = new GroupRebalanceConfig( 100, 100, @@ -140,7 +143,7 @@ public ConsumerTestBuilder() { logContext, RETRY_BACKOFF_MS, RETRY_BACKOFF_MAX_MS, - errorEventHandler, + backgroundEventHandler, "group_id")); this.commitRequestManager = spy(new CommitRequestManager(time, logContext, @@ -150,7 +153,7 @@ public ConsumerTestBuilder() { groupState)); this.fetchRequestManager = spy(new FetchRequestManager(logContext, time, - errorEventHandler, + backgroundEventHandler, metadata, subscriptions, fetchConfig, @@ -175,6 +178,9 @@ public ConsumerTestBuilder() { @Override public void close() { + closeQuietly(requestManagers, RequestManagers.class.getSimpleName()); + closeQuietly(applicationEventProcessor, ApplicationEventProcessor.class.getSimpleName()); + closeQuietly(backgroundEventProcessor, BackgroundEventProcessor.class.getSimpleName()); requestManagers.close(); } @@ -194,16 +200,16 @@ public DefaultBackgroundThreadTestBuilder() { @Override public void close() { - backgroundThread.close(); + closeQuietly(backgroundThread, DefaultBackgroundThread.class.getSimpleName()); } } - public static class DefaultEventHandlerTestBuilder extends ConsumerTestBuilder { + public static class ApplicationEventHandlerTestBuilder extends ConsumerTestBuilder { - final EventHandler eventHandler; + public final ApplicationEventHandler applicationEventHandler; - public DefaultEventHandlerTestBuilder() { - this.eventHandler = spy(new DefaultEventHandler( + public ApplicationEventHandlerTestBuilder() { + this.applicationEventHandler = spy(new ApplicationEventHandler( time, logContext, applicationEventQueue, @@ -214,11 +220,11 @@ public DefaultEventHandlerTestBuilder() { @Override public void close() { - eventHandler.close(); + closeQuietly(applicationEventHandler, ApplicationEventHandler.class.getSimpleName()); } } - public static class PrototypeAsyncConsumerTestBuilder extends DefaultEventHandlerTestBuilder { + public static class PrototypeAsyncConsumerTestBuilder extends ApplicationEventHandlerTestBuilder { final PrototypeAsyncConsumer consumer; @@ -244,7 +250,7 @@ public PrototypeAsyncConsumerTestBuilder(Optional groupIdOpt) { fetchCollector, new ConsumerInterceptors<>(Collections.emptyList()), time, - eventHandler, + applicationEventHandler, backgroundEventQueue, metrics, subscriptions, diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/CoordinatorRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/CoordinatorRequestManagerTest.java index 8a5d3996966ee..2e326ee3e476b 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/CoordinatorRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/CoordinatorRequestManagerTest.java @@ -17,6 +17,8 @@ package org.apache.kafka.clients.consumer.internals; import org.apache.kafka.clients.ClientResponse; +import org.apache.kafka.clients.consumer.internals.events.BackgroundEventHandler; +import org.apache.kafka.clients.consumer.internals.events.ErrorBackgroundEvent; import org.apache.kafka.common.Node; import org.apache.kafka.common.errors.GroupAuthorizationException; import org.apache.kafka.common.protocol.ApiKeys; @@ -45,14 +47,14 @@ public class CoordinatorRequestManagerTest { private static final int RETRY_BACKOFF_MS = 500; private static final String GROUP_ID = "group-1"; private MockTime time; - private ErrorEventHandler errorEventHandler; + private BackgroundEventHandler backgroundEventHandler; private Node node; @BeforeEach public void setup() { this.time = new MockTime(0); this.node = new Node(1, "localhost", 9092); - this.errorEventHandler = mock(ErrorEventHandler.class); + this.backgroundEventHandler = mock(BackgroundEventHandler.class); } @Test @@ -96,7 +98,7 @@ public void testMarkCoordinatorUnknown() { public void testBackoffAfterRetriableFailure() { CoordinatorRequestManager coordinatorManager = setupCoordinatorManager(GROUP_ID); expectFindCoordinatorRequest(coordinatorManager, Errors.COORDINATOR_LOAD_IN_PROGRESS); - verifyNoInteractions(errorEventHandler); + verifyNoInteractions(backgroundEventHandler); time.sleep(RETRY_BACKOFF_MS - 1); assertEquals(Collections.emptyList(), coordinatorManager.poll(time.milliseconds()).unsentRequests); @@ -110,10 +112,15 @@ public void testPropagateAndBackoffAfterFatalError() { CoordinatorRequestManager coordinatorManager = setupCoordinatorManager(GROUP_ID); expectFindCoordinatorRequest(coordinatorManager, Errors.GROUP_AUTHORIZATION_FAILED); - verify(errorEventHandler).handle(argThat(exception -> { - if (!(exception instanceof GroupAuthorizationException)) { + verify(backgroundEventHandler).add(argThat(backgroundEvent -> { + if (!(backgroundEvent instanceof ErrorBackgroundEvent)) return false; - } + + RuntimeException exception = ((ErrorBackgroundEvent) backgroundEvent).error(); + + if (!(exception instanceof GroupAuthorizationException)) + return false; + GroupAuthorizationException groupAuthException = (GroupAuthorizationException) exception; return groupAuthException.groupId().equals(GROUP_ID); })); @@ -165,7 +172,7 @@ private CoordinatorRequestManager setupCoordinatorManager(String groupId) { new LogContext(), RETRY_BACKOFF_MS, RETRY_BACKOFF_MS, - this.errorEventHandler, + this.backgroundEventHandler, groupId ); } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThreadTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThreadTest.java index 80201075d2aa7..416230b559e0b 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThreadTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThreadTest.java @@ -23,7 +23,6 @@ import org.apache.kafka.clients.consumer.internals.events.ApplicationEventProcessor; import org.apache.kafka.clients.consumer.internals.events.AssignmentChangeApplicationEvent; import org.apache.kafka.clients.consumer.internals.events.CommitApplicationEvent; -import org.apache.kafka.clients.consumer.internals.events.CompletableApplicationEvent; import org.apache.kafka.clients.consumer.internals.events.FetchEvent; import org.apache.kafka.clients.consumer.internals.events.ListOffsetsApplicationEvent; import org.apache.kafka.clients.consumer.internals.events.NewTopicsMetadataUpdateRequestEvent; @@ -47,6 +46,7 @@ import java.util.HashMap; import java.util.Map; import java.util.Optional; +import java.util.Queue; import java.util.concurrent.BlockingQueue; import java.util.concurrent.CompletableFuture; @@ -58,7 +58,7 @@ import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -261,11 +261,10 @@ void testEnsureMetadataUpdateOnPoll() { } @Test - @SuppressWarnings("unchecked") void testEnsureEventsAreCompleted() { - CompletableApplicationEvent event = (CompletableApplicationEvent) mock(CompletableApplicationEvent.class); - ApplicationEvent e = mock(ApplicationEvent.class); - CompletableFuture future = new CompletableFuture<>(); + FetchEvent event = spy(new FetchEvent()); + ApplicationEvent e = new CommitApplicationEvent(Collections.emptyMap()); + CompletableFuture> future = new CompletableFuture<>(); when(event.future()).thenReturn(future); applicationEventsQueue.add(event); applicationEventsQueue.add(e); diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java index fd4f327aa964f..937e70881a987 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java @@ -29,6 +29,7 @@ import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.OffsetOutOfRangeException; import org.apache.kafka.clients.consumer.OffsetResetStrategy; +import org.apache.kafka.clients.consumer.internals.events.BackgroundEventHandler; import org.apache.kafka.common.Cluster; import org.apache.kafka.common.IsolationLevel; import org.apache.kafka.common.KafkaException; @@ -3383,7 +3384,7 @@ private void buildFetcher(MetricConfig metricConfig, fetcher = spy(new TestableFetchRequestManager<>( logContext, time, - new ErrorEventHandler(new LinkedBlockingQueue<>()), + new BackgroundEventHandler(logContext, new LinkedBlockingQueue<>()), metadata, subscriptionState, fetchConfig, @@ -3435,14 +3436,14 @@ private class TestableFetchRequestManager extends FetchRequestManager { public TestableFetchRequestManager(LogContext logContext, Time time, - ErrorEventHandler errorEventHandler, + BackgroundEventHandler backgroundEventHandler, ConsumerMetadata metadata, SubscriptionState subscriptions, FetchConfig fetchConfig, FetchMetricsManager metricsManager, NetworkClientDelegate networkClientDelegate, FetchCollector fetchCollector) { - super(logContext, time, errorEventHandler, metadata, subscriptions, fetchConfig, metricsManager, networkClientDelegate); + super(logContext, time, backgroundEventHandler, metadata, subscriptions, fetchConfig, metricsManager, networkClientDelegate); this.fetchCollector = fetchCollector; } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumerTest.java index 3d0177085d9e1..4add7f180f674 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumerTest.java @@ -20,8 +20,8 @@ import org.apache.kafka.clients.consumer.OffsetAndTimestamp; import org.apache.kafka.clients.consumer.OffsetCommitCallback; import org.apache.kafka.clients.consumer.internals.events.ApplicationEvent; +import org.apache.kafka.clients.consumer.internals.events.ApplicationEventHandler; import org.apache.kafka.clients.consumer.internals.events.AssignmentChangeApplicationEvent; -import org.apache.kafka.clients.consumer.internals.events.EventHandler; import org.apache.kafka.clients.consumer.internals.events.ListOffsetsApplicationEvent; import org.apache.kafka.clients.consumer.internals.events.NewTopicsMetadataUpdateRequestEvent; import org.apache.kafka.clients.consumer.internals.events.OffsetFetchApplicationEvent; @@ -74,7 +74,7 @@ public class PrototypeAsyncConsumerTest { private static final Optional DEFAULT_GROUP_ID = Optional.of("group.id"); private ConsumerTestBuilder.PrototypeAsyncConsumerTestBuilder testBuilder; - private EventHandler eventHandler; + private ApplicationEventHandler applicationEventHandler; @BeforeEach public void setup() { @@ -90,7 +90,7 @@ public void cleanup() { private void setup(Optional groupIdOpt) { testBuilder = new ConsumerTestBuilder.PrototypeAsyncConsumerTestBuilder(groupIdOpt); - eventHandler = testBuilder.eventHandler; + applicationEventHandler = testBuilder.applicationEventHandler; consumer = testBuilder.consumer; } @@ -146,7 +146,7 @@ public void testCommitted() { try (MockedConstruction ignored = offsetFetchEventMocker(committedFuture)) { assertDoesNotThrow(() -> consumer.committed(offsets.keySet(), Duration.ofMillis(1000))); - verify(eventHandler).add(ArgumentMatchers.isA(OffsetFetchApplicationEvent.class)); + verify(applicationEventHandler).add(ArgumentMatchers.isA(OffsetFetchApplicationEvent.class)); } } @@ -158,7 +158,7 @@ public void testCommitted_ExceptionThrown() { try (MockedConstruction ignored = offsetFetchEventMocker(committedFuture)) { assertThrows(KafkaException.class, () -> consumer.committed(offsets.keySet(), Duration.ofMillis(1000))); - verify(eventHandler).add(ArgumentMatchers.isA(OffsetFetchApplicationEvent.class)); + verify(applicationEventHandler).add(ArgumentMatchers.isA(OffsetFetchApplicationEvent.class)); } } @@ -202,8 +202,8 @@ public void testAssign() { consumer.assign(singleton(tp)); assertTrue(consumer.subscription().isEmpty()); assertTrue(consumer.assignment().contains(tp)); - verify(eventHandler).add(any(AssignmentChangeApplicationEvent.class)); - verify(eventHandler).add(any(NewTopicsMetadataUpdateRequestEvent.class)); + verify(applicationEventHandler).add(any(AssignmentChangeApplicationEvent.class)); + verify(applicationEventHandler).add(any(NewTopicsMetadataUpdateRequestEvent.class)); } @Test @@ -239,14 +239,14 @@ public void testBeginningOffsets() { Map expectedOffsetsAndTimestamp = mockOffsetAndTimestamp(); Set partitions = expectedOffsetsAndTimestamp.keySet(); - doReturn(expectedOffsetsAndTimestamp).when(eventHandler).addAndGet(any(), any()); + doReturn(expectedOffsetsAndTimestamp).when(applicationEventHandler).addAndGet(any(), any()); Map result = assertDoesNotThrow(() -> consumer.beginningOffsets(partitions, Duration.ofMillis(1))); Map expectedOffsets = expectedOffsetsAndTimestamp.entrySet().stream() .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().offset())); assertEquals(expectedOffsets, result); - verify(eventHandler).addAndGet(ArgumentMatchers.isA(ListOffsetsApplicationEvent.class), + verify(applicationEventHandler).addAndGet(ArgumentMatchers.isA(ListOffsetsApplicationEvent.class), ArgumentMatchers.isA(Timer.class)); } @@ -255,22 +255,22 @@ public void testBeginningOffsetsThrowsKafkaExceptionForUnderlyingExecutionFailur Set partitions = mockTopicPartitionOffset().keySet(); Throwable eventProcessingFailure = new KafkaException("Unexpected failure " + "processing List Offsets event"); - doThrow(eventProcessingFailure).when(eventHandler).addAndGet(any(), any()); + doThrow(eventProcessingFailure).when(applicationEventHandler).addAndGet(any(), any()); Throwable consumerError = assertThrows(KafkaException.class, () -> consumer.beginningOffsets(partitions, Duration.ofMillis(1))); assertEquals(eventProcessingFailure, consumerError); - verify(eventHandler).addAndGet(ArgumentMatchers.isA(ListOffsetsApplicationEvent.class), ArgumentMatchers.isA(Timer.class)); + verify(applicationEventHandler).addAndGet(ArgumentMatchers.isA(ListOffsetsApplicationEvent.class), ArgumentMatchers.isA(Timer.class)); } @Test public void testBeginningOffsetsTimeoutOnEventProcessingTimeout() { - doThrow(new TimeoutException()).when(eventHandler).addAndGet(any(), any()); + doThrow(new TimeoutException()).when(applicationEventHandler).addAndGet(any(), any()); assertThrows(TimeoutException.class, () -> consumer.beginningOffsets( Collections.singletonList(new TopicPartition("t1", 0)), Duration.ofMillis(1))); - verify(eventHandler).addAndGet(ArgumentMatchers.isA(ListOffsetsApplicationEvent.class), + verify(applicationEventHandler).addAndGet(ArgumentMatchers.isA(ListOffsetsApplicationEvent.class), ArgumentMatchers.isA(Timer.class)); } @@ -285,11 +285,11 @@ public void testOffsetsForTimes() { Map expectedResult = mockOffsetAndTimestamp(); Map timestampToSearch = mockTimestampToSearch(); - doReturn(expectedResult).when(eventHandler).addAndGet(any(), any()); + doReturn(expectedResult).when(applicationEventHandler).addAndGet(any(), any()); Map result = assertDoesNotThrow(() -> consumer.offsetsForTimes(timestampToSearch, Duration.ofMillis(1))); assertEquals(expectedResult, result); - verify(eventHandler).addAndGet(ArgumentMatchers.isA(ListOffsetsApplicationEvent.class), + verify(applicationEventHandler).addAndGet(ArgumentMatchers.isA(ListOffsetsApplicationEvent.class), ArgumentMatchers.isA(Timer.class)); } @@ -307,7 +307,7 @@ public void testOffsetsForTimesWithZeroTimeout() { assertDoesNotThrow(() -> consumer.offsetsForTimes(timestampToSearch, Duration.ofMillis(0))); assertEquals(expectedResult, result); - verify(eventHandler, never()).addAndGet(ArgumentMatchers.isA(ListOffsetsApplicationEvent.class), + verify(applicationEventHandler, never()).addAndGet(ArgumentMatchers.isA(ListOffsetsApplicationEvent.class), ArgumentMatchers.isA(Timer.class)); } @@ -363,18 +363,18 @@ private void testUpdateFetchPositionsWithFetchCommittedOffsetsTimeout(boolean co // Poll with 0 timeout to run a single iteration of the poll loop consumer.poll(Duration.ofMillis(0)); - verify(eventHandler).add(ArgumentMatchers.isA(ValidatePositionsApplicationEvent.class)); + verify(applicationEventHandler).add(ArgumentMatchers.isA(ValidatePositionsApplicationEvent.class)); if (committedOffsetsEnabled) { // Verify there was an OffsetFetch event and no ResetPositions event - verify(eventHandler).add(ArgumentMatchers.isA(OffsetFetchApplicationEvent.class)); - verify(eventHandler, + verify(applicationEventHandler).add(ArgumentMatchers.isA(OffsetFetchApplicationEvent.class)); + verify(applicationEventHandler, never()).add(ArgumentMatchers.isA(ResetPositionsApplicationEvent.class)); } else { // Verify there was not any OffsetFetch event but there should be a ResetPositions - verify(eventHandler, + verify(applicationEventHandler, never()).add(ArgumentMatchers.isA(OffsetFetchApplicationEvent.class)); - verify(eventHandler).add(ArgumentMatchers.isA(ResetPositionsApplicationEvent.class)); + verify(applicationEventHandler).add(ArgumentMatchers.isA(ResetPositionsApplicationEvent.class)); } } } @@ -393,9 +393,9 @@ private void testRefreshCommittedOffsetsSuccess(Set partitions, // Poll with 0 timeout to run a single iteration of the poll loop consumer.poll(Duration.ofMillis(0)); - verify(eventHandler).add(ArgumentMatchers.isA(ValidatePositionsApplicationEvent.class)); - verify(eventHandler).add(ArgumentMatchers.isA(OffsetFetchApplicationEvent.class)); - verify(eventHandler).add(ArgumentMatchers.isA(ResetPositionsApplicationEvent.class)); + verify(applicationEventHandler).add(ArgumentMatchers.isA(ValidatePositionsApplicationEvent.class)); + verify(applicationEventHandler).add(ArgumentMatchers.isA(OffsetFetchApplicationEvent.class)); + verify(applicationEventHandler).add(ArgumentMatchers.isA(ResetPositionsApplicationEvent.class)); } } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/DefaultEventHandlerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventHandlerTest.java similarity index 63% rename from clients/src/test/java/org/apache/kafka/clients/consumer/internals/DefaultEventHandlerTest.java rename to clients/src/test/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventHandlerTest.java index 38cfafbde109b..6dfeef40c3d49 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/DefaultEventHandlerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventHandlerTest.java @@ -14,11 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.kafka.clients.consumer.internals; +package org.apache.kafka.clients.consumer.internals.events; -import org.apache.kafka.clients.consumer.internals.events.ApplicationEvent; -import org.apache.kafka.clients.consumer.internals.events.EventHandler; -import org.apache.kafka.clients.consumer.internals.events.FetchEvent; +import org.apache.kafka.clients.consumer.internals.ConsumerTestBuilder; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -27,17 +25,17 @@ import static org.junit.jupiter.api.Assertions.assertEquals; -public class DefaultEventHandlerTest { +public class ApplicationEventHandlerTest { - private ConsumerTestBuilder.DefaultEventHandlerTestBuilder testBuilder; - private EventHandler handler; - private BlockingQueue aq; + private ConsumerTestBuilder.ApplicationEventHandlerTestBuilder testBuilder; + private ApplicationEventHandler applicationEventHandler; + private BlockingQueue applicationEventQueue; @BeforeEach public void setup() { - testBuilder = new ConsumerTestBuilder.DefaultEventHandlerTestBuilder(); - handler = testBuilder.eventHandler; - aq = testBuilder.applicationEventQueue; + testBuilder = new ConsumerTestBuilder.ApplicationEventHandlerTestBuilder(); + applicationEventHandler = testBuilder.applicationEventHandler; + applicationEventQueue = testBuilder.applicationEventQueue; } @AfterEach @@ -48,7 +46,7 @@ public void tearDown() { @Test public void testBasicHandlerOps() { - handler.add(new FetchEvent()); - assertEquals(1, aq.size()); + applicationEventHandler.add(new FetchEvent()); + assertEquals(1, applicationEventQueue.size()); } } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ErrorEventHandlerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEventHandlerTest.java similarity index 82% rename from clients/src/test/java/org/apache/kafka/clients/consumer/internals/ErrorEventHandlerTest.java rename to clients/src/test/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEventHandlerTest.java index 0c74a7d8a284e..349fe63f4dab6 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ErrorEventHandlerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEventHandlerTest.java @@ -14,12 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.kafka.clients.consumer.internals; +package org.apache.kafka.clients.consumer.internals.events; -import org.apache.kafka.clients.consumer.internals.events.BackgroundEvent; -import org.apache.kafka.clients.consumer.internals.events.BackgroundEventProcessor; -import org.apache.kafka.clients.consumer.internals.events.ErrorBackgroundEvent; -import org.apache.kafka.clients.consumer.internals.events.EventProcessor; +import org.apache.kafka.clients.consumer.internals.ConsumerTestBuilder; import org.apache.kafka.common.KafkaException; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -33,19 +30,19 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; -public class ErrorEventHandlerTest { +public class BackgroundEventHandlerTest { - private ConsumerTestBuilder.DefaultEventHandlerTestBuilder testBuilder; + private ConsumerTestBuilder testBuilder; private BlockingQueue backgroundEventQueue; - private ErrorEventHandler errorEventHandler; + private BackgroundEventHandler backgroundEventHandler; private BackgroundEventProcessor backgroundEventProcessor; @BeforeEach public void setup() { - testBuilder = new ConsumerTestBuilder.DefaultEventHandlerTestBuilder(); + testBuilder = new ConsumerTestBuilder(); backgroundEventQueue = testBuilder.backgroundEventQueue; - errorEventHandler = new ErrorEventHandler(backgroundEventQueue); - backgroundEventProcessor = new BackgroundEventProcessor(testBuilder.logContext, backgroundEventQueue); + backgroundEventHandler = testBuilder.backgroundEventHandler; + backgroundEventProcessor = testBuilder.backgroundEventProcessor; } @AfterEach @@ -74,7 +71,7 @@ public void testSingleEvent() { public void testSingleErrorEvent() { KafkaException error = new KafkaException("error"); BackgroundEvent event = new ErrorBackgroundEvent(error); - errorEventHandler.handle(error); + backgroundEventHandler.add(new ErrorBackgroundEvent(error)); assertPeeked(event); assertProcessThrows(error); } @@ -97,9 +94,9 @@ public void testMultipleErrorEvents() { KafkaException error2 = new KafkaException("error2"); KafkaException error3 = new KafkaException("error3"); - errorEventHandler.handle(error1); - errorEventHandler.handle(error2); - errorEventHandler.handle(error3); + backgroundEventHandler.add(new ErrorBackgroundEvent(error1)); + backgroundEventHandler.add(new ErrorBackgroundEvent(error2)); + backgroundEventHandler.add(new ErrorBackgroundEvent(error3)); assertProcessThrows(new KafkaException(error1)); } @@ -112,11 +109,11 @@ public void testMixedEventsWithErrorEvents() { RuntimeException errorToCheck = new RuntimeException("A"); backgroundEventQueue.add(new ErrorBackgroundEvent(errorToCheck)); - errorEventHandler.handle(error1); + backgroundEventHandler.add(new ErrorBackgroundEvent(error1)); backgroundEventQueue.add(new ErrorBackgroundEvent(new RuntimeException("B"))); - errorEventHandler.handle(error2); + backgroundEventHandler.add(new ErrorBackgroundEvent(error2)); backgroundEventQueue.add(new ErrorBackgroundEvent(new RuntimeException("C"))); - errorEventHandler.handle(error3); + backgroundEventHandler.add(new ErrorBackgroundEvent(error3)); backgroundEventQueue.add(new ErrorBackgroundEvent(new RuntimeException("D"))); assertProcessThrows(new KafkaException(errorToCheck)); From e8b990f150940e563bb1ab7a1e01eab827c81758 Mon Sep 17 00:00:00 2001 From: Kirk True Date: Tue, 26 Sep 2023 16:42:52 -0700 Subject: [PATCH 12/72] Minor refactoring for the event processor callbacks --- .../internals/DefaultBackgroundThread.java | 6 +-- .../internals/PrototypeAsyncConsumer.java | 37 +------------------ .../events/ApplicationEventProcessor.java | 12 ++++++ .../events/BackgroundEventProcessor.java | 23 +++++++++++- .../internals/events/EventProcessor.java | 34 +++++++++-------- .../events/BackgroundEventHandlerTest.java | 20 +--------- 6 files changed, 55 insertions(+), 77 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThread.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThread.java index 40711c819ca28..bc1f4478a1dc3 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThread.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThread.java @@ -108,11 +108,9 @@ void initializeResources() { * 3. Poll the networkClient to send and retrieve the response. */ void runOnce() { - // If there are errors processing any events, we want to throw the error immediately. This will have + // If there are errors processing any events, the error will be thrown immediately. This will have // the effect of closing the background thread. - applicationEventProcessor.process(error -> { - throw error; - }); + applicationEventProcessor.process(); final long currentTimeMs = time.milliseconds(); final long pollWaitTimeMs = requestManagers.entries().stream() diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java index 786e6ad12565a..3dfd73ffa4352 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java @@ -39,9 +39,7 @@ import org.apache.kafka.clients.consumer.internals.events.BackgroundEvent; import org.apache.kafka.clients.consumer.internals.events.BackgroundEventProcessor; import org.apache.kafka.clients.consumer.internals.events.ApplicationEventHandler; -import org.apache.kafka.clients.consumer.internals.events.EventProcessor.ProcessErrorHandler; import org.apache.kafka.clients.consumer.internals.events.CommitApplicationEvent; -import org.apache.kafka.clients.consumer.internals.events.ErrorBackgroundEvent; import org.apache.kafka.clients.consumer.internals.events.FetchEvent; import org.apache.kafka.clients.consumer.internals.events.ListOffsetsApplicationEvent; import org.apache.kafka.clients.consumer.internals.events.NewTopicsMetadataUpdateRequestEvent; @@ -1059,44 +1057,11 @@ public void onComplete(Map offsets, Exception } boolean updateAssignmentMetadataIfNeeded(Timer timer) { - processBackgroundEvents(); + backgroundEventProcessor.process(); // Keeping this updateAssignmentMetadataIfNeeded wrapping up the updateFetchPositions as // in the previous implementation, because it will eventually involve group coordination // logic return updateFetchPositions(timer); } - - /** - * {@link BackgroundEventProcessor Process the events}—if any—that were produced by the - * {@link DefaultBackgroundThread background thread}. It is possible that when processing the events that a - * given event will {@link ErrorBackgroundEvent represent an error directly}, or it could be that processing - * an event generates an error. In such cases, the processor will continue to process the remaining events. In - * this case, we provide the caller to provide a callback handler that "collects" the errors. We grab the first - * error that occurred and throw it. - */ - private void processBackgroundEvents() { - BackgroundEventProcessHandler handler = new BackgroundEventProcessHandler(); - backgroundEventProcessor.process(handler); - if (handler.first != null) - throw handler.first; - } - - private class BackgroundEventProcessHandler implements ProcessErrorHandler { - - private KafkaException first; - private int errorCount; - - @Override - public void onProcessingError(KafkaException error) { - errorCount++; - - if (first == null) { - first = error; - log.warn("Error #{} from background thread (will be logged and thrown): {}", errorCount, error.getMessage(), error); - } else { - log.warn("Error #{} from background thread (will be logged only): {}", errorCount, error.getMessage(), error); - } - } - } } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventProcessor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventProcessor.java index 7d7e5c2bf25cf..64bfa45202431 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventProcessor.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventProcessor.java @@ -52,6 +52,18 @@ public ApplicationEventProcessor(final LogContext logContext, this.metadata = metadata; } + /** + * Process the events—if any—that were produced by the application thread. It is possible that when processing + * an event generates an error. In such cases, the processor will immediately throw an exception, and not + * process the remaining events. + */ + @Override + public void process() { + process(error -> { + throw error; + }); + } + @Override public void process(ApplicationEvent event) { switch (event.type()) { diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEventProcessor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEventProcessor.java index dea43b28de59d..9343e9b636605 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEventProcessor.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEventProcessor.java @@ -18,9 +18,11 @@ import org.apache.kafka.clients.consumer.ConsumerRebalanceListener; import org.apache.kafka.clients.consumer.internals.DefaultBackgroundThread; +import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.utils.LogContext; import java.util.concurrent.BlockingQueue; +import java.util.concurrent.atomic.AtomicReference; /** * An {@link EventProcessor} that is created and executes in the application thread for the purpose of processing @@ -39,9 +41,21 @@ public BackgroundEventProcessor(final LogContext logContext, super(logContext, backgroundEventQueue); } + /** + * Process the events—if any—that were produced by the {@link DefaultBackgroundThread background thread}. + * It is possible that when processing the events that a given event will + * {@link ErrorBackgroundEvent represent an error directly}, or it could be that processing an event generates + * an error. In such cases, the processor will continue to process the remaining events. In this case, we + * provide the caller to provide a callback handler that "collects" the errors. We grab the first error that + * occurred and throw it. + */ @Override - protected Class getEventClass() { - return BackgroundEvent.class; + public void process() { + AtomicReference error = new AtomicReference<>(); + process(t -> error.compareAndSet(null, t)); + + if (error.get() != null) + throw error.get(); } @Override @@ -52,6 +66,11 @@ public void process(final BackgroundEvent event) { throw new IllegalArgumentException("Background event type " + event.type() + " was not expected"); } + @Override + protected Class getEventClass() { + return BackgroundEvent.class; + } + private void process(final ErrorBackgroundEvent event) { throw event.error(); } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/EventProcessor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/EventProcessor.java index 72b74138e5c18..88f1926e06850 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/EventProcessor.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/EventProcessor.java @@ -46,11 +46,27 @@ protected EventProcessor(final LogContext logContext, final BlockingQueue eve this.closer = new IdempotentCloser(); } + public abstract void process(); + + public abstract void process(T event); + + @Override + public void close() { + closer.close(this::closeInternal, () -> log.warn("Already closed")); + } + + protected abstract Class getEventClass(); + + protected interface ProcessErrorHandler { + + void onError(KafkaException error); + } + /** * Drains all available events from the queue, and then processes them in order. If any errors are thrown while * processing the individual events, these are submitted to the given {@link ProcessErrorHandler}. */ - public void process(ProcessErrorHandler processErrorHandler) { + protected void process(ProcessErrorHandler processErrorHandler) { String eventClassName = getEventClass().getSimpleName(); closer.assertOpen(() -> String.format("The processor was previously closed, so no further %s processing can occur", eventClassName)); @@ -73,7 +89,7 @@ public void process(ProcessErrorHandler processErrorHandler) { else error = new KafkaException(t); - processErrorHandler.onProcessingError(error); + processErrorHandler.onError(error); } } } finally { @@ -81,11 +97,6 @@ public void process(ProcessErrorHandler processErrorHandler) { } } - @Override - public void close() { - closer.close(this::closeInternal, () -> log.warn("Already closed")); - } - /** * It is possible for the consumer to close before complete processing all the events in the queue. In * this case, we need to throw an exception to notify the user the consumer is closed. @@ -115,10 +126,6 @@ private void closeInternal() { eventClassName); } - public abstract void process(T event); - - protected abstract Class getEventClass(); - /** * Moves all the events from the queue to the returned list. */ @@ -127,9 +134,4 @@ private List drain() { eventQueue.drainTo(events); return events; } - - public interface ProcessErrorHandler { - - void onProcessingError(KafkaException error); - } } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEventHandlerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEventHandlerTest.java index 349fe63f4dab6..dfd972dc33a99 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEventHandlerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEventHandlerTest.java @@ -129,9 +129,7 @@ private void assertProcessThrows(Throwable error) { assertFalse(backgroundEventQueue.isEmpty()); try { - TestProcessHandler processHandler = new TestProcessHandler(); - backgroundEventProcessor.process(processHandler); - processHandler.maybeThrow(); + backgroundEventProcessor.process(); fail("Should have thrown error: " + error); } catch (Throwable t) { assertEquals(error.getClass(), t.getClass()); @@ -140,20 +138,4 @@ private void assertProcessThrows(Throwable error) { assertTrue(backgroundEventQueue.isEmpty()); } - - private static class TestProcessHandler implements EventProcessor.ProcessErrorHandler { - - private KafkaException first; - - @Override - public void onProcessingError(KafkaException error) { - if (first == null) - first = error; - } - - void maybeThrow() { - if (first != null) - throw first; - } - } } From 731e716fb42de35109d01604ed3a36a86c60340b Mon Sep 17 00:00:00 2001 From: Kirk True Date: Tue, 26 Sep 2023 16:45:35 -0700 Subject: [PATCH 13/72] Added comment to explain a case in DefaultBackgroundThread.wakeup() --- .../clients/consumer/internals/DefaultBackgroundThread.java | 1 + 1 file changed, 1 insertion(+) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThread.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThread.java index bc1f4478a1dc3..00897fb1c5133 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThread.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThread.java @@ -133,6 +133,7 @@ public boolean isRunning() { } public void wakeup() { + // The network client can be null if the initializeResources method has not yet been called. if (networkClientDelegate != null) networkClientDelegate.wakeup(); } From 3f920798d221b94fbc3328da796d391a8f4745fc Mon Sep 17 00:00:00 2001 From: Kirk True Date: Wed, 27 Sep 2023 17:21:17 -0700 Subject: [PATCH 14/72] Closing the fetch sessions 1. Added pollOnClose() method to RequestManager to explicitly call out requests that need to be made at shutdown 2. Added runAtClose() method to DefaultBackgroundThread to perform network I/O on close 3. DefaultBackgroundThread.run() is no longer calling close() before it exits. This was causing conflicts when the close() method was called. 3. QoL enhancements to the PollResult constructor 4. RequestManager interface no longer extends Closeable interface, but individual implementations are free to implement that interface --- .../consumer/internals/AbstractFetch.java | 9 +- .../internals/CommitRequestManager.java | 15 +- .../internals/CoordinatorRequestManager.java | 14 +- .../internals/DefaultBackgroundThread.java | 153 +++++++++++++++--- .../consumer/internals/FetchBuffer.java | 2 +- .../internals/FetchRequestManager.java | 80 ++++----- .../internals/NetworkClientDelegate.java | 33 +++- .../internals/OffsetsRequestManager.java | 8 +- .../internals/PrototypeAsyncConsumer.java | 17 +- .../consumer/internals/RequestManager.java | 14 +- .../consumer/internals/RequestManagers.java | 17 +- .../TopicMetadataRequestManager.java | 6 +- .../events/ApplicationEventHandler.java | 33 ++-- .../internals/events/EventProcessor.java | 2 +- .../internals/ConsumerTestBuilder.java | 5 +- .../DefaultBackgroundThreadTest.java | 7 +- .../internals/FetchRequestManagerTest.java | 11 +- 17 files changed, 273 insertions(+), 153 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractFetch.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractFetch.java index 108be1999f6a8..1a32b414639a4 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractFetch.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractFetch.java @@ -40,6 +40,7 @@ import org.slf4j.helpers.MessageFormatter; import java.io.Closeable; +import java.time.Duration; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; @@ -56,6 +57,7 @@ public abstract class AbstractFetch implements Closeable { private final Logger log; + private final IdempotentCloser idempotentCloser = new IdempotentCloser(); protected final LogContext logContext; protected final ConsumerMetadata metadata; protected final SubscriptionState subscriptions; @@ -65,7 +67,6 @@ public abstract class AbstractFetch implements Closeable { protected final FetchBuffer fetchBuffer; protected final BufferSupplier decompressionBufferSupplier; protected final Set nodesWithPendingFetchRequests; - protected final IdempotentCloser idempotentCloser = new IdempotentCloser(); private final Map sessionHandlers; @@ -445,13 +446,11 @@ protected void closeInternal(Timer timer) { } public void close(final Timer timer) { - idempotentCloser.close(() -> { - closeInternal(timer); - }); + idempotentCloser.close(() -> closeInternal(timer)); } @Override public void close() { - close(time.timer(0)); + close(time.timer(Duration.ZERO)); } } \ No newline at end of file diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CommitRequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CommitRequestManager.java index 0fdeaef8b842b..5f0c42f19e38f 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CommitRequestManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CommitRequestManager.java @@ -48,6 +48,8 @@ import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; +import static org.apache.kafka.clients.consumer.internals.NetworkClientDelegate.PollResult.EMPTY; + public class CommitRequestManager implements RequestManager { // TODO: current in ConsumerConfig but inaccessible in the internal package. @@ -96,17 +98,14 @@ public CommitRequestManager( @Override public NetworkClientDelegate.PollResult poll(final long currentTimeMs) { // poll only when the coordinator node is known. - if (!coordinatorRequestManager.coordinator().isPresent()) { - return new NetworkClientDelegate.PollResult(Long.MAX_VALUE, Collections.emptyList()); - } + if (!coordinatorRequestManager.coordinator().isPresent()) + return EMPTY; maybeAutoCommit(this.subscriptions.allConsumed()); - if (!pendingRequests.hasUnsentRequests()) { - return new NetworkClientDelegate.PollResult(Long.MAX_VALUE, Collections.emptyList()); - } + if (!pendingRequests.hasUnsentRequests()) + return EMPTY; - return new NetworkClientDelegate.PollResult(Long.MAX_VALUE, - Collections.unmodifiableList(pendingRequests.drain(currentTimeMs))); + return new NetworkClientDelegate.PollResult(pendingRequests.drain(currentTimeMs)); } public void maybeAutoCommit(final Map offsets) { diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CoordinatorRequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CoordinatorRequestManager.java index bb83468734be8..ddbbc67f6ffc4 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CoordinatorRequestManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/CoordinatorRequestManager.java @@ -31,10 +31,11 @@ import org.apache.kafka.common.utils.Time; import org.slf4j.Logger; -import java.util.Collections; import java.util.Objects; import java.util.Optional; +import static org.apache.kafka.clients.consumer.internals.NetworkClientDelegate.PollResult.EMPTY; + /** * This is responsible for timing to send the next {@link FindCoordinatorRequest} based on the following criteria: *

    @@ -93,18 +94,15 @@ public CoordinatorRequestManager( */ @Override public NetworkClientDelegate.PollResult poll(final long currentTimeMs) { - if (this.coordinator != null) { - return new NetworkClientDelegate.PollResult(Long.MAX_VALUE, Collections.emptyList()); - } + if (this.coordinator != null) + return EMPTY; if (coordinatorRequestState.canSendRequest(currentTimeMs)) { NetworkClientDelegate.UnsentRequest request = makeFindCoordinatorRequest(currentTimeMs); - return new NetworkClientDelegate.PollResult(Long.MAX_VALUE, Collections.singletonList(request)); + return new NetworkClientDelegate.PollResult(request); } - return new NetworkClientDelegate.PollResult( - coordinatorRequestState.remainingBackoffMs(currentTimeMs), - Collections.emptyList()); + return new NetworkClientDelegate.PollResult(coordinatorRequestState.remainingBackoffMs(currentTimeMs)); } private NetworkClientDelegate.UnsentRequest makeFindCoordinatorRequest(final long currentTimeMs) { diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThread.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThread.java index 00897fb1c5133..66f1d9dd8ce5d 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThread.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThread.java @@ -16,20 +16,30 @@ */ package org.apache.kafka.clients.consumer.internals; +import org.apache.kafka.clients.KafkaClient; import org.apache.kafka.clients.consumer.internals.events.ApplicationEvent; import org.apache.kafka.clients.consumer.internals.events.ApplicationEventProcessor; import org.apache.kafka.common.KafkaException; +import org.apache.kafka.common.errors.InterruptException; import org.apache.kafka.common.errors.WakeupException; import org.apache.kafka.common.internals.IdempotentCloser; +import org.apache.kafka.common.requests.AbstractRequest; import org.apache.kafka.common.utils.KafkaThread; import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.Time; -import org.apache.kafka.common.utils.Utils; +import org.apache.kafka.common.utils.Timer; import org.slf4j.Logger; import java.io.Closeable; +import java.time.Duration; +import java.util.Collection; +import java.util.List; import java.util.Optional; +import java.util.concurrent.Future; import java.util.function.Supplier; +import java.util.stream.Collectors; + +import static org.apache.kafka.common.utils.Utils.closeQuietly; /** * Background thread runnable that consumes {@code ApplicationEvent} and @@ -54,8 +64,8 @@ public class DefaultBackgroundThread extends KafkaThread implements Closeable { private volatile boolean running; private final IdempotentCloser closer = new IdempotentCloser(); - public DefaultBackgroundThread(Time time, - LogContext logContext, + public DefaultBackgroundThread(LogContext logContext, + Time time, Supplier applicationEventProcessorSupplier, Supplier networkClientDelegateSupplier, Supplier requestManagersSupplier) { @@ -89,9 +99,6 @@ public void run() { } catch (final Throwable t) { log.error("The background thread failed due to unexpected error", t); throw new KafkaException(t); - } finally { - close(); - log.debug("Background thread closed"); } } @@ -102,10 +109,26 @@ void initializeResources() { } /** - * Poll and process an {@link ApplicationEvent}. It performs the following tasks: - * 1. Drains and try to process all the requests in the queue. - * 2. Iterate through the registry, poll, and get the next poll time for the network poll - * 3. Poll the networkClient to send and retrieve the response. + * Poll and process the {@link ApplicationEvent application events}. It performs the following tasks: + * + *

      + *
    1. + * Drains and processes all the events from the application thread's application event queue via + * {@link ApplicationEventProcessor} + *
      2. + *
    2. + * Iterate through the {@link RequestManager} list and invoke {@link RequestManager#poll(long)} to get + * the {@link NetworkClientDelegate.UnsentRequest} list and the poll time for the network poll + *
      3. + *
    3. + * Stage each {@link AbstractRequest.Builder request} to be sent via + * {@link NetworkClientDelegate#addAll(List)} + *
      4. + *
    4. + * Poll the client via {@link KafkaClient#poll(long, long)} to send the requests, as well as + * retrieve any available responses + *
      5. + *
    */ void runOnce() { // If there are errors processing any events, the error will be thrown immediately. This will have @@ -115,17 +138,66 @@ void runOnce() { final long currentTimeMs = time.milliseconds(); final long pollWaitTimeMs = requestManagers.entries().stream() .filter(Optional::isPresent) - .map(m -> m.get().poll(currentTimeMs)) - .map(this::handlePollResult) + .map(Optional::get) + .map(rm -> rm.poll(currentTimeMs)) + .map(networkClientDelegate::addAll) .reduce(MAX_POLL_TIMEOUT_MS, Math::min); networkClientDelegate.poll(pollWaitTimeMs, currentTimeMs); } - long handlePollResult(NetworkClientDelegate.PollResult res) { - if (!res.unsentRequests.isEmpty()) { - networkClientDelegate.addAll(res.unsentRequests); + /** + * Performs any network I/O that is needed at the time of close for the consumer: + * + *
      + *
    1. + * Iterate through the {@link RequestManager} list and invoke {@link RequestManager#pollOnClose()} + * to get the {@link NetworkClientDelegate.UnsentRequest} list and the poll time for the network poll + *
      2. + *
    2. + * Stage each {@link AbstractRequest.Builder request} to be sent via + * {@link NetworkClientDelegate#addAll(List)} + *
      3. + *
    3. + * {@link KafkaClient#poll(long, long) Poll the client} to send the requests, as well as + * retrieve any available responses + *
      4. + *
    4. + * Continuously {@link KafkaClient#poll(long, long) poll the client} as long as the + * {@link Timer#notExpired() timer hasn't expired} to retrieve the responses + *
      5. + *
    + */ + // Visible for testing + static void runAtClose(final Time time, + final Collection> requestManagers, + final NetworkClientDelegate networkClientDelegate, + final Timer timer) { + long currentTimeMs = time.milliseconds(); + + // These are the optional outgoing requests at the + List pollResults = requestManagers.stream() + .filter(Optional::isPresent) + .map(Optional::get) + .map(RequestManager::pollOnClose) + .collect(Collectors.toList()); + long pollWaitTimeMs = pollResults.stream() + .map(networkClientDelegate::addAll) + .reduce(MAX_POLL_TIMEOUT_MS, Math::min); + pollWaitTimeMs = Math.min(pollWaitTimeMs, timer.remainingMs()); + networkClientDelegate.poll(pollWaitTimeMs, currentTimeMs); + timer.update(); + + List> requestFutures = pollResults.stream() + .flatMap(fads -> fads.unsentRequests.stream()) + .map(NetworkClientDelegate.UnsentRequest::future) + .collect(Collectors.toList()); + + // Poll to ensure that request has been written to the socket. Wait until either the timer has expired or until + // all requests have received a response. + while (timer.notExpired() && !requestFutures.stream().allMatch(Future::isDone)) { + networkClientDelegate.poll(timer.remainingMs(), currentTimeMs); + timer.update(); } - return res.timeUntilNextPollMs; } public boolean isRunning() { @@ -140,14 +212,45 @@ public void wakeup() { @Override public void close() { - closer.close(() -> { - log.trace("Closing the consumer background thread"); - running = false; - wakeup(); - Utils.closeQuietly(requestManagers, "Request managers client"); - Utils.closeQuietly(networkClientDelegate, "network client utils"); - Utils.closeQuietly(applicationEventProcessor, "application event processor"); - log.debug("Closed the consumer background thread"); - }, () -> log.warn("The consumer background thread was previously closed")); + close(Duration.ZERO); + } + + public void close(final Duration timeout) { + closer.close( + () -> closeInternal(timeout), + () -> log.warn("The consumer background thread was already closed") + ); + } + + void closeInternal(final Duration timeout) { + log.trace("Closing the consumer background thread"); + boolean hadStarted = running; + running = false; + wakeup(); + + Timer timer = time.timer(timeout); + + if (hadStarted && timer.remainingMs() > 0) { + // If the thread had started, we need to wait for the run method to exit. It may take a little time + // for the thread to check the status of the running flag. + // + // We check the value of remainingMs because this method can be called with a duration of 0, which for + // the Thread.join method means "wait forever" which isn't what we want. + try { + long remainingMs = timer.remainingMs(); + log.warn("Waiting up to {} ms for the thread to complete", remainingMs); + join(remainingMs); + } catch (InterruptedException e) { + throw new InterruptException(e); + } finally { + timer.update(); + } + } + + runAtClose(time, requestManagers.entries(), networkClientDelegate, timer); + closeQuietly(requestManagers, "request managers"); + closeQuietly(networkClientDelegate, "network client delegate"); + closeQuietly(applicationEventProcessor, "application event processor"); + log.debug("Closed the consumer background thread"); } } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchBuffer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchBuffer.java index c7944e294e2e0..d1865117e3f52 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchBuffer.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchBuffer.java @@ -143,6 +143,6 @@ public void close() { completedFetches.forEach(CompletedFetch::drain); completedFetches.clear(); - }, () -> log.warn("The fetch buffer was previously closed")); + }, () -> log.warn("The fetch buffer was already closed")); } } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchRequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchRequestManager.java index e82ee1596105e..6c95a2c4a0440 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchRequestManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchRequestManager.java @@ -89,47 +89,47 @@ public void requestFetch(CompletableFuture> future) { @Override public PollResult poll(long currentTimeMs) { - List requests; - - if (!idempotentCloser.isClosed()) { - // If the fetcher is open (i.e. not closed), we will issue the normal fetch requests - requests = prepareFetchRequests().entrySet().stream().map(entry -> { - final Node fetchTarget = entry.getKey(); - final FetchSessionHandler.FetchRequestData data = entry.getValue(); - final FetchRequest.Builder request = createFetchRequest(fetchTarget, data); - final BiConsumer responseHandler = (clientResponse, t) -> { - if (t != null) { - handleFetchResponse(fetchTarget, t); - log.debug("Attempt to fetch data from node {} failed due to fatal exception", fetchTarget, t); - backgroundEventHandler.add(new ErrorBackgroundEvent(t)); - } else { - handleFetchResponse(fetchTarget, data, clientResponse); - forwardResults(); - } - }; - - return new UnsentRequest(request, fetchTarget, responseHandler); - }).collect(Collectors.toList()); - } else { - requests = prepareCloseFetchSessionRequests().entrySet().stream().map(entry -> { - final Node fetchTarget = entry.getKey(); - final FetchSessionHandler.FetchRequestData data = entry.getValue(); - final FetchRequest.Builder request = createFetchRequest(fetchTarget, data); - final BiConsumer responseHandler = (clientResponse, t) -> { - if (t != null) { - handleCloseFetchSessionResponse(fetchTarget, data, t); - log.warn("Attempt to close fetch session on node {} failed due to fatal exception", fetchTarget, t); - backgroundEventHandler.add(new ErrorBackgroundEvent(t)); - } else { - handleCloseFetchSessionResponse(fetchTarget, data); - } - }; - - return new UnsentRequest(request, fetchTarget, responseHandler); - }).collect(Collectors.toList()); - } + List requests = prepareFetchRequests().entrySet().stream().map(entry -> { + final Node fetchTarget = entry.getKey(); + final FetchSessionHandler.FetchRequestData data = entry.getValue(); + final FetchRequest.Builder request = createFetchRequest(fetchTarget, data); + final BiConsumer responseHandler = (clientResponse, t) -> { + if (t != null) { + handleFetchResponse(fetchTarget, t); + log.debug("Attempt to fetch data from node {} failed due to fatal exception", fetchTarget, t); + backgroundEventHandler.add(new ErrorBackgroundEvent(t)); + } else { + handleFetchResponse(fetchTarget, data, clientResponse); + forwardResults(); + } + }; + + return new UnsentRequest(request, fetchTarget, responseHandler); + }).collect(Collectors.toList()); + + return new PollResult(requests); + } - return new PollResult(Long.MAX_VALUE, requests); + @Override + public PollResult pollOnClose() { + List requests = prepareCloseFetchSessionRequests().entrySet().stream().map(entry -> { + final Node fetchTarget = entry.getKey(); + final FetchSessionHandler.FetchRequestData data = entry.getValue(); + final FetchRequest.Builder request = createFetchRequest(fetchTarget, data); + final BiConsumer responseHandler = (clientResponse, t) -> { + if (t != null) { + handleCloseFetchSessionResponse(fetchTarget, data, t); + log.warn("Attempt to close fetch session on node {} failed due to fatal exception", fetchTarget, t); + backgroundEventHandler.add(new ErrorBackgroundEvent(t)); + } else { + handleCloseFetchSessionResponse(fetchTarget, data); + } + }; + + return new UnsentRequest(request, fetchTarget, responseHandler); + }).collect(Collectors.toList()); + + return new PollResult(requests); } /** diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/NetworkClientDelegate.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/NetworkClientDelegate.java index c26541b67479e..2e3d8ba1895f1 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/NetworkClientDelegate.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/NetworkClientDelegate.java @@ -224,22 +224,43 @@ public void close() throws IOException { this.client.close(); } + public long addAll(PollResult pollResult) { + addAll(pollResult.unsentRequests); + return pollResult.timeUntilNextPollMs; + } + public void addAll(final List requests) { - requests.forEach(u -> { - u.setTimer(this.time, this.requestTimeoutMs); - }); - this.unsentRequests.addAll(requests); + if (!requests.isEmpty()) { + requests.forEach(ur -> ur.setTimer(time, requestTimeoutMs)); + unsentRequests.addAll(requests); + } } public static class PollResult { + + public static final long WAIT_FOREVER = Long.MAX_VALUE; + public static final PollResult EMPTY = new PollResult(WAIT_FOREVER); public final long timeUntilNextPollMs; public final List unsentRequests; - public PollResult(final long timeMsTillNextPoll, final List unsentRequests) { - this.timeUntilNextPollMs = timeMsTillNextPoll; + public PollResult(final long timeUntilNextPollMs, final List unsentRequests) { + this.timeUntilNextPollMs = timeUntilNextPollMs; this.unsentRequests = Collections.unmodifiableList(unsentRequests); } + + public PollResult(final List unsentRequests) { + this(WAIT_FOREVER, unsentRequests); + } + + public PollResult(final UnsentRequest unsentRequest) { + this(Collections.singletonList(unsentRequest)); + } + + public PollResult(final long timeUntilNextPollMs) { + this(timeUntilNextPollMs, Collections.emptyList()); + } } + public static class UnsentRequest { private final AbstractRequest.Builder requestBuilder; private final FutureCompletionHandler handler; diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/OffsetsRequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/OffsetsRequestManager.java index 0e33361825583..6e27c71fb6929 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/OffsetsRequestManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/OffsetsRequestManager.java @@ -127,10 +127,10 @@ public OffsetsRequestManager(final SubscriptionState subscriptionState, */ @Override public NetworkClientDelegate.PollResult poll(final long currentTimeMs) { - NetworkClientDelegate.PollResult pollResult = - new NetworkClientDelegate.PollResult(Long.MAX_VALUE, new ArrayList<>(requestsToSend)); - this.requestsToSend.clear(); - return pollResult; + // Copy the outgoing request list and clear it. + List unsentRequests = new ArrayList<>(requestsToSend); + requestsToSend.clear(); + return new NetworkClientDelegate.PollResult(unsentRequests); } /** diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java index 3dfd73ffa4352..eecb52f4eeae9 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java @@ -96,11 +96,11 @@ import static org.apache.kafka.clients.consumer.ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_JMX_PREFIX; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRIC_GROUP_PREFIX; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.configuredConsumerInterceptors; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createFetchMetricsManager; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createLogContext; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createMetrics; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createSubscriptionState; -import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.configuredConsumerInterceptors; import static org.apache.kafka.common.utils.Utils.closeQuietly; import static org.apache.kafka.common.utils.Utils.isBlank; import static org.apache.kafka.common.utils.Utils.join; @@ -231,8 +231,8 @@ public PrototypeAsyncConsumer(final Time time, metadata, applicationEventQueue, requestManagersSupplier); - this.applicationEventHandler = new ApplicationEventHandler(time, - logContext, + this.applicationEventHandler = new ApplicationEventHandler(logContext, + time, applicationEventQueue, applicationEventProcessorSupplier, networkClientDelegateSupplier, @@ -267,7 +267,7 @@ public PrototypeAsyncConsumer(final Time time, // call close methods if internal objects are already constructed; this is to prevent resource leak. see KAFKA-2121 // we do not need to call `close` at all when `log` is null, which means no internal objects were initialized. if (this.log != null) { - close(true); + close(Duration.ZERO, true); } // now propagate the exception throw new KafkaException("Failed to construct kafka consumer", t); @@ -692,21 +692,24 @@ public void close(Duration timeout) { if (!closed) { // need to close before setting the flag since the close function // itself may trigger rebalance callback that needs the consumer to be open still - close(false); + close(timeout, false); } } finally { closed = true; } } - private void close(boolean swallowException) { + private void close(Duration timeout, boolean swallowException) { log.trace("Closing the Kafka consumer"); AtomicReference firstException = new AtomicReference<>(); + + if (applicationEventHandler != null) + closeQuietly(() -> applicationEventHandler.close(timeout), "Failed to close application event handler with a timeout(ms)=" + timeout, firstException); + closeQuietly(fetchBuffer, "Failed to close fetcher", firstException); closeQuietly(interceptors, "consumer interceptors", firstException); closeQuietly(kafkaConsumerMetrics, "kafka consumer metrics", firstException); closeQuietly(metrics, "consumer metrics", firstException); - closeQuietly(applicationEventHandler, "event handler", firstException); closeQuietly(deserializers, "consumer deserializers", firstException); AppInfoParser.unregisterAppInfo(CONSUMER_JMX_PREFIX, clientId, metrics); diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManager.java index 5bd6e01c2bb15..4ebc698c2ee30 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManager.java @@ -16,20 +16,24 @@ */ package org.apache.kafka.clients.consumer.internals; +import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.consumer.internals.NetworkClientDelegate.PollResult; -import java.io.Closeable; +import static org.apache.kafka.clients.consumer.internals.NetworkClientDelegate.PollResult.EMPTY; /** * {@code PollResult} consist of {@code UnsentRequest} if there are requests to send; otherwise, return the time till * the next poll event. */ -public interface RequestManager extends Closeable { +public interface RequestManager { PollResult poll(long currentTimeMs); - @Override - default void close() { - // Do nothing... + /** + * On shutdown of the {@link Consumer}, a request manager may want/need to send out some requests. If so, + * implementations can signal that by returning the close requests here. + */ + default PollResult pollOnClose() { + return EMPTY; } } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java index e58678e4d0ecb..923d71e44d559 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java @@ -34,6 +34,8 @@ import java.util.concurrent.BlockingQueue; import java.util.function.Supplier; +import static org.apache.kafka.common.utils.Utils.closeQuietly; + import static java.util.Objects.requireNonNull; /** @@ -84,15 +86,12 @@ public void close() { () -> { log.debug("Closing RequestManagers"); - entries.forEach(rm -> { - rm.ifPresent(requestManager -> { - try { - requestManager.close(); - } catch (Throwable t) { - log.debug("Error closing request manager {}", requestManager.getClass().getSimpleName(), t); - } - }); - }); + entries.stream() + .filter(Optional::isPresent) + .map(Optional::get) + .filter(rm -> rm instanceof Closeable) + .map(rm -> (Closeable) rm) + .forEach(c -> closeQuietly(c, c.getClass().getSimpleName())); log.debug("RequestManagers has been closed"); }, () -> log.debug("RequestManagers was already closed")); diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/TopicMetadataRequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/TopicMetadataRequestManager.java index 2b0cbf5dcb0ff..b972f70c76f70 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/TopicMetadataRequestManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/TopicMetadataRequestManager.java @@ -40,6 +40,8 @@ import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; +import static org.apache.kafka.clients.consumer.internals.NetworkClientDelegate.PollResult.EMPTY; + /** *

    * Manages the state of topic metadata requests. This manager returns a @@ -84,9 +86,7 @@ public NetworkClientDelegate.PollResult poll(final long currentTimeMs) { .filter(Optional::isPresent) .map(Optional::get) .collect(Collectors.toList()); - return requests.isEmpty() ? - new NetworkClientDelegate.PollResult(Long.MAX_VALUE, new ArrayList<>()) : - new NetworkClientDelegate.PollResult(0, Collections.unmodifiableList(requests)); + return requests.isEmpty() ? EMPTY : new NetworkClientDelegate.PollResult(0, requests); } /** diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventHandler.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventHandler.java index a5bda46e6f60f..b6c0cfa92bbb5 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventHandler.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventHandler.java @@ -19,11 +19,11 @@ import org.apache.kafka.clients.consumer.internals.DefaultBackgroundThread; import org.apache.kafka.clients.consumer.internals.NetworkClientDelegate; import org.apache.kafka.clients.consumer.internals.RequestManagers; -import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.internals.IdempotentCloser; import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.Time; import org.apache.kafka.common.utils.Timer; +import org.apache.kafka.common.utils.Utils; import org.slf4j.Logger; import java.io.Closeable; @@ -45,16 +45,16 @@ public class ApplicationEventHandler implements Closeable { private final DefaultBackgroundThread backgroundThread; private final IdempotentCloser closer = new IdempotentCloser(); - public ApplicationEventHandler(final Time time, - final LogContext logContext, + public ApplicationEventHandler(final LogContext logContext, + final Time time, final BlockingQueue applicationEventQueue, final Supplier applicationEventProcessorSupplier, final Supplier networkClientDelegateSupplier, final Supplier requestManagersSupplier) { this.log = logContext.logger(ApplicationEventHandler.class); this.applicationEventQueue = applicationEventQueue; - this.backgroundThread = new DefaultBackgroundThread(time, - logContext, + this.backgroundThread = new DefaultBackgroundThread(logContext, + time, applicationEventProcessorSupplier, networkClientDelegateSupplier, requestManagersSupplier); @@ -95,28 +95,13 @@ public T addAndGet(final CompletableApplicationEvent event, final Timer t @Override public void close() { - close(Duration.ofMillis(Long.MAX_VALUE)); + close(Duration.ZERO); } public void close(final Duration timeout) { - Objects.requireNonNull(timeout, "Duration provided to close must be non-null"); - closer.close( - () -> { - log.info("Closing the consumer application event handler"); - - try { - long timeoutMs = timeout.toMillis(); - - if (timeoutMs < 0) - throw new IllegalArgumentException("The timeout cannot be negative."); - - backgroundThread.close(); - log.info("The consumer application event handler was closed"); - } catch (final Exception e) { - throw new KafkaException(e); - } - }, - () -> log.info("The consumer application event handler was already closed")); + () -> Utils.closeQuietly(() -> backgroundThread.close(timeout), "application event handler"), + () -> log.warn("The application event handler was already closed") + ); } } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/EventProcessor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/EventProcessor.java index 88f1926e06850..6223c7ed11918 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/EventProcessor.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/EventProcessor.java @@ -52,7 +52,7 @@ protected EventProcessor(final LogContext logContext, final BlockingQueue eve @Override public void close() { - closer.close(this::closeInternal, () -> log.warn("Already closed")); + closer.close(this::closeInternal, () -> log.warn("The event processor was already closed")); } protected abstract Class getEventClass(); diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java index 2d4016a8d45cc..b1b48f1eb6273 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java @@ -181,7 +181,6 @@ public void close() { closeQuietly(requestManagers, RequestManagers.class.getSimpleName()); closeQuietly(applicationEventProcessor, ApplicationEventProcessor.class.getSimpleName()); closeQuietly(backgroundEventProcessor, BackgroundEventProcessor.class.getSimpleName()); - requestManagers.close(); } public static class DefaultBackgroundThreadTestBuilder extends ConsumerTestBuilder { @@ -190,8 +189,8 @@ public static class DefaultBackgroundThreadTestBuilder extends ConsumerTestBuild public DefaultBackgroundThreadTestBuilder() { this.backgroundThread = new DefaultBackgroundThread( - time, logContext, + time, () -> applicationEventProcessor, () -> networkClientDelegate, () -> requestManagers @@ -210,8 +209,8 @@ public static class ApplicationEventHandlerTestBuilder extends ConsumerTestBuild public ApplicationEventHandlerTestBuilder() { this.applicationEventHandler = spy(new ApplicationEventHandler( - time, logContext, + time, applicationEventQueue, () -> applicationEventProcessor, () -> networkClientDelegate, diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThreadTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThreadTest.java index 416230b559e0b..2e377ac69098c 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThreadTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThreadTest.java @@ -41,6 +41,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import java.time.Duration; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -111,7 +112,7 @@ public void testStartupAndTearDown() throws InterruptedException { TestUtils.waitForCondition(backgroundThread::isRunning, maxWaitMs, "Thread did not start within " + maxWaitMs + " ms"); - backgroundThread.close(); + backgroundThread.close(Duration.ofMillis(maxWaitMs)); TestUtils.waitForCondition(() -> !backgroundThread.isRunning(), maxWaitMs, "Thread did not stop within " + maxWaitMs + " ms"); @@ -236,12 +237,12 @@ void testPollResultTimer() { NetworkClientDelegate.PollResult success = new NetworkClientDelegate.PollResult( 10, Collections.singletonList(req)); - assertEquals(10, backgroundThread.handlePollResult(success)); + assertEquals(10, networkClient.addAll(success)); NetworkClientDelegate.PollResult failure = new NetworkClientDelegate.PollResult( 10, new ArrayList<>()); - assertEquals(10, backgroundThread.handlePollResult(failure)); + assertEquals(10, networkClient.addAll(failure)); } @Test diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java index 937e70881a987..39067a0accb2b 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java @@ -312,7 +312,16 @@ public void testFetcherCloseClosesFetchSessionsInBroker() { // send request to close the fetcher Timer timer = time.timer(Duration.ofSeconds(10)); - this.fetcher.close(timer); + // this.fetcher.close(timer); + // + // NOTE: by design the FetchRequestManager doesn't perform network I/O internally. That means that calling + // close with a Timer will NOT send out the close session requests on close. The network I/O logic is + // handled inside DefaultBackgroundThread.runAtClose, so we need to run that logic here. + DefaultBackgroundThread.runAtClose(time, + Collections.singletonList(Optional.of(fetcher)), + consumerClient, + timer); + NetworkClientDelegate.PollResult pollResult = fetcher.poll(time.milliseconds()); consumerClient.addAll(pollResult.unsentRequests); consumerClient.poll(timer); From e08d01a6b0f636d17f9b7aaa59864a54f15cde04 Mon Sep 17 00:00:00 2001 From: Kirk True Date: Tue, 3 Oct 2023 14:46:15 -0700 Subject: [PATCH 15/72] Minor logic change for assignFromUserNoId Changed name to just assignFromUser and refactored slightly to reduce magic strings. --- .../consumer/internals/FetchRequestManagerTest.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java index 39067a0accb2b..6109185c45056 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java @@ -214,13 +214,13 @@ private void assignFromUser(Set partitions) { tp -> validLeaderEpoch, topicIds), false, 0L); } - private void assignFromUserNoId(Set partitions) { - subscriptions.assignFromUser(partitions); - client.updateMetadata(RequestTestUtils.metadataUpdateWithIds(1, singletonMap("noId", 1), Collections.emptyMap())); + private void assignFromUser(TopicPartition partition) { + subscriptions.assignFromUser(singleton(partition)); + client.updateMetadata(RequestTestUtils.metadataUpdateWithIds(1, singletonMap(partition.topic(), 1), Collections.emptyMap())); // A dummy metadata update to ensure valid leader epoch. metadata.update(9, RequestTestUtils.metadataUpdateWithIds("dummy", 1, - Collections.emptyMap(), singletonMap("noId", 1), + Collections.emptyMap(), singletonMap(partition.topic(), 1), tp -> validLeaderEpoch, topicIds), false, 0L); } @@ -368,7 +368,7 @@ public void testFetchWithNoTopicId() { buildFetcher(); TopicIdPartition noId = new TopicIdPartition(Uuid.ZERO_UUID, new TopicPartition("noId", 0)); - assignFromUserNoId(singleton(noId.topicPartition())); + assignFromUser(noId.topicPartition()); subscriptions.seek(noId.topicPartition(), 0); // Fetch should use request version 12 From d7c4d0c37e9f57f84dafa909a53ebc336aed58e7 Mon Sep 17 00:00:00 2001 From: Kirk True Date: Tue, 3 Oct 2023 14:49:43 -0700 Subject: [PATCH 16/72] Minor logic change for assignFromUserNoId (from FetchRequestManagerTest) Changed name to just assignFromUser and refactored slightly to reduce magic strings. --- .../kafka/clients/consumer/internals/FetcherTest.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java index 1fa4a6c2e5c8d..323ac74ee693b 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java @@ -209,13 +209,13 @@ private void assignFromUser(Set partitions) { tp -> validLeaderEpoch, topicIds), false, 0L); } - private void assignFromUserNoId(Set partitions) { - subscriptions.assignFromUser(partitions); - client.updateMetadata(RequestTestUtils.metadataUpdateWithIds(1, singletonMap("noId", 1), Collections.emptyMap())); + private void assignFromUser(TopicPartition partition) { + subscriptions.assignFromUser(singleton(partition)); + client.updateMetadata(RequestTestUtils.metadataUpdateWithIds(1, singletonMap(partition.topic(), 1), Collections.emptyMap())); // A dummy metadata update to ensure valid leader epoch. metadata.update(9, RequestTestUtils.metadataUpdateWithIds("dummy", 1, - Collections.emptyMap(), singletonMap("noId", 1), + Collections.emptyMap(), singletonMap(partition.topic(), 1), tp -> validLeaderEpoch, topicIds), false, 0L); } @@ -353,7 +353,7 @@ public void testFetchWithNoTopicId() { buildFetcher(); TopicIdPartition noId = new TopicIdPartition(Uuid.ZERO_UUID, new TopicPartition("noId", 0)); - assignFromUserNoId(singleton(noId.topicPartition())); + assignFromUser(noId.topicPartition()); subscriptions.seek(noId.topicPartition(), 0); // Fetch should use request version 12 From 6ffb7ec72527afdc1bd1aa1be00744dbabf9785d Mon Sep 17 00:00:00 2001 From: Kirk True Date: Tue, 3 Oct 2023 15:03:25 -0700 Subject: [PATCH 17/72] Removed unnecessary references to "this" in fetcher tests --- .../internals/FetchRequestManagerTest.java | 112 +++++++++--------- .../consumer/internals/FetcherTest.java | 112 +++++++++--------- 2 files changed, 112 insertions(+), 112 deletions(-) diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java index 6109185c45056..4a9aaecdbc517 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java @@ -227,9 +227,9 @@ private void assignFromUser(TopicPartition partition) { @AfterEach public void teardown() throws Exception { if (metrics != null) - this.metrics.close(); + metrics.close(); if (fetcher != null) - this.fetcher.close(); + fetcher.close(); } private int sendFetches() { @@ -248,7 +248,7 @@ public void testFetchNormal() { assertEquals(1, sendFetches()); assertFalse(fetcher.hasCompletedFetches()); - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); @@ -275,7 +275,7 @@ public void testInflightFetchOnPendingPartitions() { assertEquals(1, sendFetches()); subscriptions.markPendingRevocation(singleton(tp0)); - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); consumerClient.poll(time.timer(0)); assertNull(fetchedRecords().get(tp0)); } @@ -302,7 +302,7 @@ public void testFetcherCloseClosesFetchSessionsInBroker() { assertEquals(1, sendFetches()); assertFalse(fetcher.hasCompletedFetches()); - final FetchResponse fetchResponse = fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0); + final FetchResponse fetchResponse = fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0); client.prepareResponse(fetchResponse); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); @@ -312,7 +312,7 @@ public void testFetcherCloseClosesFetchSessionsInBroker() { // send request to close the fetcher Timer timer = time.timer(Duration.ofSeconds(10)); - // this.fetcher.close(timer); + // fetcher.close(timer); // // NOTE: by design the FetchRequestManager doesn't perform network I/O internally. That means that calling // close with a Timer will NOT send out the close session requests on close. The network I/O logic is @@ -347,7 +347,7 @@ public void testFetchingPendingPartitions() { // normal fetch assertEquals(1, sendFetches()); - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); fetchedRecords(); @@ -377,7 +377,7 @@ public void testFetchWithNoTopicId() { client.prepareResponse( fetchRequestMatcher((short) 12, noId, 0, Optional.of(validLeaderEpoch)), - fullFetchResponse(noId, this.records, Errors.NONE, 100L, 0) + fullFetchResponse(noId, records, Errors.NONE, 100L, 0) ); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); @@ -409,7 +409,7 @@ public void testFetchWithTopicId() { client.prepareResponse( fetchRequestMatcher(ApiKeys.FETCH.latestVersion(), tp, 0, Optional.of(validLeaderEpoch)), - fullFetchResponse(tp, this.records, Errors.NONE, 100L, 0) + fullFetchResponse(tp, records, Errors.NONE, 100L, 0) ); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); @@ -453,7 +453,7 @@ public void testFetchForgetTopicIdWhenUnassigned() { ), emptyList() ), - fullFetchResponse(1, foo, this.records, Errors.NONE, 100L, 0) + fullFetchResponse(1, foo, records, Errors.NONE, 100L, 0) ); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); @@ -479,7 +479,7 @@ public void testFetchForgetTopicIdWhenUnassigned() { ), singletonList(foo) ), - fullFetchResponse(1, bar, this.records, Errors.NONE, 100L, 0) + fullFetchResponse(1, bar, records, Errors.NONE, 100L, 0) ); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); @@ -512,7 +512,7 @@ public void testFetchForgetTopicIdWhenReplaced() { ), emptyList() ), - fullFetchResponse(1, fooWithOldTopicId, this.records, Errors.NONE, 100L, 0) + fullFetchResponse(1, fooWithOldTopicId, records, Errors.NONE, 100L, 0) ); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); @@ -539,7 +539,7 @@ public void testFetchForgetTopicIdWhenReplaced() { ), singletonList(fooWithOldTopicId) ), - fullFetchResponse(1, fooWithNewTopicId, this.records, Errors.NONE, 100L, 0) + fullFetchResponse(1, fooWithNewTopicId, records, Errors.NONE, 100L, 0) ); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); @@ -572,7 +572,7 @@ public void testFetchTopicIdUpgradeDowngrade() { ), emptyList() ), - fullFetchResponse(1, fooWithoutId, this.records, Errors.NONE, 100L, 0) + fullFetchResponse(1, fooWithoutId, records, Errors.NONE, 100L, 0) ); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); @@ -599,7 +599,7 @@ public void testFetchTopicIdUpgradeDowngrade() { ), emptyList() ), - fullFetchResponse(1, fooWithId, this.records, Errors.NONE, 100L, 0) + fullFetchResponse(1, fooWithId, records, Errors.NONE, 100L, 0) ); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); @@ -626,7 +626,7 @@ public void testFetchTopicIdUpgradeDowngrade() { ), emptyList() ), - fullFetchResponse(1, fooWithoutId, this.records, Errors.NONE, 100L, 0) + fullFetchResponse(1, fooWithoutId, records, Errors.NONE, 100L, 0) ); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); @@ -771,7 +771,7 @@ public void testClearBufferedDataForTopicPartitions() { assertEquals(1, sendFetches()); assertFalse(fetcher.hasCompletedFetches()); - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); Set newAssignedTopicPartitions = new HashSet<>(); @@ -848,7 +848,7 @@ public void testFetchError() { assertEquals(1, sendFetches()); assertFalse(fetcher.hasCompletedFetches()); - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NOT_LEADER_OR_FOLLOWER, 100L, 0)); + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NOT_LEADER_OR_FOLLOWER, 100L, 0)); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); @@ -887,7 +887,7 @@ public byte[] deserialize(String topic, byte[] data) { assignFromUser(singleton(tp0)); subscriptions.seek(tp0, 1); - client.prepareResponse(matchesOffset(tidp0, 1), fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + client.prepareResponse(matchesOffset(tidp0, 1), fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); assertEquals(1, sendFetches()); consumerClient.poll(time.timer(0)); @@ -1122,7 +1122,7 @@ public void testFetchMaxPollRecords() { subscriptions.seek(tp0, 1); client.prepareResponse(matchesOffset(tidp0, 1), fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); - client.prepareResponse(matchesOffset(tidp0, 4), fullFetchResponse(tidp0, this.nextRecords, Errors.NONE, 100L, 0)); + client.prepareResponse(matchesOffset(tidp0, 4), fullFetchResponse(tidp0, nextRecords, Errors.NONE, 100L, 0)); assertEquals(1, sendFetches()); consumerClient.poll(time.timer(0)); @@ -1177,7 +1177,7 @@ public void testFetchAfterPartitionWithFetchedRecordsIsUnassigned() { assertEquals(2, records.get(1).offset()); assignFromUser(singleton(tp1)); - client.prepareResponse(matchesOffset(tidp1, 4), fullFetchResponse(tidp1, this.nextRecords, Errors.NONE, 100L, 0)); + client.prepareResponse(matchesOffset(tidp1, 4), fullFetchResponse(tidp1, nextRecords, Errors.NONE, 100L, 0)); subscriptions.seek(tp1, 4); assertEquals(1, sendFetches()); @@ -1287,7 +1287,7 @@ public void testUnauthorizedTopic() { // resize the limit of the buffer to pretend it is only fetch-size large assertEquals(1, sendFetches()); - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.TOPIC_AUTHORIZATION_FAILED, 100L, 0)); + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.TOPIC_AUTHORIZATION_FAILED, 100L, 0)); consumerClient.poll(time.timer(0)); try { fetcher.collectFetch(); @@ -1314,7 +1314,7 @@ public void testFetchDuringEagerRebalance() { subscriptions.assignFromSubscribed(Collections.emptyList()); subscriptions.assignFromSubscribed(singleton(tp0)); - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); consumerClient.poll(time.timer(0)); // The active fetch should be ignored since its position is no longer valid @@ -1337,7 +1337,7 @@ public void testFetchDuringCooperativeRebalance() { // Now the cooperative rebalance happens and fetch positions are NOT cleared for unrevoked partitions subscriptions.assignFromSubscribed(singleton(tp0)); - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); consumerClient.poll(time.timer(0)); Map>> fetchedRecords = fetchedRecords(); @@ -1357,7 +1357,7 @@ public void testInFlightFetchOnPausedPartition() { assertEquals(1, sendFetches()); subscriptions.pause(tp0); - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); consumerClient.poll(time.timer(0)); assertNull(fetchedRecords().get(tp0)); } @@ -1385,7 +1385,7 @@ public void testFetchOnCompletedFetchesForPausedAndResumedPartitions() { subscriptions.pause(tp0); - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); consumerClient.poll(time.timer(0)); assertEmptyFetch("Should not return any records or advance position when partition is paused"); @@ -1422,13 +1422,13 @@ public void testFetchOnCompletedFetchesForSomePausedPartitions() { // #1 seek, request, poll, response subscriptions.seekUnvalidated(tp0, new SubscriptionState.FetchPosition(1, Optional.empty(), metadata.currentLeader(tp0))); assertEquals(1, sendFetches()); - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); consumerClient.poll(time.timer(0)); // #2 seek, request, poll, response subscriptions.seekUnvalidated(tp1, new SubscriptionState.FetchPosition(1, Optional.empty(), metadata.currentLeader(tp1))); assertEquals(1, sendFetches()); - client.prepareResponse(fullFetchResponse(tidp1, this.nextRecords, Errors.NONE, 100L, 0)); + client.prepareResponse(fullFetchResponse(tidp1, nextRecords, Errors.NONE, 100L, 0)); subscriptions.pause(tp0); consumerClient.poll(time.timer(0)); @@ -1454,13 +1454,13 @@ public void testFetchOnCompletedFetchesForAllPausedPartitions() { // #1 seek, request, poll, response subscriptions.seekUnvalidated(tp0, new SubscriptionState.FetchPosition(1, Optional.empty(), metadata.currentLeader(tp0))); assertEquals(1, sendFetches()); - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); consumerClient.poll(time.timer(0)); // #2 seek, request, poll, response subscriptions.seekUnvalidated(tp1, new SubscriptionState.FetchPosition(1, Optional.empty(), metadata.currentLeader(tp1))); assertEquals(1, sendFetches()); - client.prepareResponse(fullFetchResponse(tidp1, this.nextRecords, Errors.NONE, 100L, 0)); + client.prepareResponse(fullFetchResponse(tidp1, nextRecords, Errors.NONE, 100L, 0)); subscriptions.pause(tp0); subscriptions.pause(tp1); @@ -1485,7 +1485,7 @@ public void testPartialFetchWithPausedPartitions() { subscriptions.seek(tp0, 1); assertEquals(1, sendFetches()); - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); consumerClient.poll(time.timer(0)); fetchedRecords = fetchedRecords(); @@ -1520,7 +1520,7 @@ public void testFetchDiscardedAfterPausedPartitionResumedAndSeekedToNewOffset() assertEquals(1, sendFetches()); subscriptions.pause(tp0); - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); subscriptions.seek(tp0, 3); subscriptions.resume(tp0); @@ -1540,7 +1540,7 @@ public void testFetchNotLeaderOrFollower() { subscriptions.seek(tp0, 0); assertEquals(1, sendFetches()); - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NOT_LEADER_OR_FOLLOWER, 100L, 0)); + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NOT_LEADER_OR_FOLLOWER, 100L, 0)); consumerClient.poll(time.timer(0)); assertEmptyFetch("Should not return records or advance position on fetch error"); assertEquals(0L, metadata.timeToNextUpdate(time.milliseconds())); @@ -1553,7 +1553,7 @@ public void testFetchUnknownTopicOrPartition() { subscriptions.seek(tp0, 0); assertEquals(1, sendFetches()); - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.UNKNOWN_TOPIC_OR_PARTITION, 100L, 0)); + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.UNKNOWN_TOPIC_OR_PARTITION, 100L, 0)); consumerClient.poll(time.timer(0)); assertEmptyFetch("Should not return records or advance position on fetch error"); assertEquals(0L, metadata.timeToNextUpdate(time.milliseconds())); @@ -1566,7 +1566,7 @@ public void testFetchUnknownTopicId() { subscriptions.seek(tp0, 0); assertEquals(1, sendFetches()); - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.UNKNOWN_TOPIC_ID, -1L, 0)); + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.UNKNOWN_TOPIC_ID, -1L, 0)); consumerClient.poll(time.timer(0)); assertEmptyFetch("Should not return records or advance position on fetch error"); assertEquals(0L, metadata.timeToNextUpdate(time.milliseconds())); @@ -1592,7 +1592,7 @@ public void testFetchInconsistentTopicId() { subscriptions.seek(tp0, 0); assertEquals(1, sendFetches()); - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.INCONSISTENT_TOPIC_ID, -1L, 0)); + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.INCONSISTENT_TOPIC_ID, -1L, 0)); consumerClient.poll(time.timer(0)); assertEmptyFetch("Should not return records or advance position on fetch error"); assertEquals(0L, metadata.timeToNextUpdate(time.milliseconds())); @@ -1605,7 +1605,7 @@ public void testFetchFencedLeaderEpoch() { subscriptions.seek(tp0, 0); assertEquals(1, sendFetches()); - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.FENCED_LEADER_EPOCH, 100L, 0)); + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.FENCED_LEADER_EPOCH, 100L, 0)); consumerClient.poll(time.timer(0)); assertEmptyFetch("Should not return records or advance position on fetch error"); @@ -1619,7 +1619,7 @@ public void testFetchUnknownLeaderEpoch() { subscriptions.seek(tp0, 0); assertEquals(1, sendFetches()); - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.UNKNOWN_LEADER_EPOCH, 100L, 0)); + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.UNKNOWN_LEADER_EPOCH, 100L, 0)); consumerClient.poll(time.timer(0)); assertEmptyFetch("Should not return records or advance position on fetch error"); @@ -1651,7 +1651,7 @@ public void testEpochSetInFetchRequest() { return false; } }; - client.prepareResponse(matcher, fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + client.prepareResponse(matcher, fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); consumerClient.pollNoWakeup(); } @@ -1662,7 +1662,7 @@ public void testFetchOffsetOutOfRange() { subscriptions.seek(tp0, 0); assertEquals(1, sendFetches()); - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.OFFSET_OUT_OF_RANGE, 100L, 0)); + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.OFFSET_OUT_OF_RANGE, 100L, 0)); consumerClient.poll(time.timer(0)); assertEmptyFetch("Should not return records or advance position on fetch error"); assertTrue(subscriptions.isOffsetResetNeeded(tp0)); @@ -1679,7 +1679,7 @@ public void testStaleOutOfRangeError() { subscriptions.seek(tp0, 0); assertEquals(1, sendFetches()); - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.OFFSET_OUT_OF_RANGE, 100L, 0)); + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.OFFSET_OUT_OF_RANGE, 100L, 0)); subscriptions.seek(tp0, 1); consumerClient.poll(time.timer(0)); assertEmptyFetch("Should not return records or advance position on fetch error"); @@ -1696,7 +1696,7 @@ public void testFetchedRecordsAfterSeek() { subscriptions.seek(tp0, 0); assertTrue(sendFetches() > 0); - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.OFFSET_OUT_OF_RANGE, 100L, 0)); + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.OFFSET_OUT_OF_RANGE, 100L, 0)); consumerClient.poll(time.timer(0)); assertFalse(subscriptions.isOffsetResetNeeded(tp0)); subscriptions.seek(tp0, 2); @@ -1712,7 +1712,7 @@ public void testFetchOffsetOutOfRangeException() { subscriptions.seek(tp0, 0); sendFetches(); - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.OFFSET_OUT_OF_RANGE, 100L, 0)); + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.OFFSET_OUT_OF_RANGE, 100L, 0)); consumerClient.poll(time.timer(0)); assertFalse(subscriptions.isOffsetResetNeeded(tp0)); @@ -1869,7 +1869,7 @@ public void testSeekBeforeException() { .setPartitionIndex(tp0.partition()) .setHighWatermark(100) .setRecords(records)); - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); consumerClient.poll(time.timer(0)); assertEquals(2, fetchedRecords().get(tp0).size()); @@ -1900,7 +1900,7 @@ public void testFetchDisconnected() { subscriptions.seek(tp0, 0); assertEquals(1, sendFetches()); - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0), true); + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0), true); consumerClient.poll(time.timer(0)); assertEmptyFetch("Should not return records or advance position on disconnect"); @@ -2271,7 +2271,7 @@ public void testFetcherMetricsTemplates() { assignFromUser(singleton(tp0)); subscriptions.seek(tp0, 0); assertEquals(1, sendFetches()); - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); Map>> partitionRecords = fetchedRecords(); @@ -2818,7 +2818,7 @@ public void testConsumingViaIncrementalFetchRequests() { .setHighWatermark(100) .setLastStableOffset(4) .setLogStartOffset(0) - .setRecords(this.nextRecords)); + .setRecords(nextRecords)); FetchResponse resp3 = FetchResponse.of(Errors.NONE, 0, 123, partitions3); client.prepareResponse(resp3); assertEquals(1, sendFetches()); @@ -2978,7 +2978,7 @@ public void testPreferredReadReplica() { assertFalse(fetcher.hasCompletedFetches()); // Set preferred read replica to node=1 - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, FetchResponse.INVALID_LAST_STABLE_OFFSET, 0, Optional.of(1))); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); @@ -2995,7 +2995,7 @@ public void testPreferredReadReplica() { assertFalse(fetcher.hasCompletedFetches()); // Set preferred read replica to node=2, which isn't in our metadata, should revert to leader - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, FetchResponse.INVALID_LAST_STABLE_OFFSET, 0, Optional.of(2))); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); @@ -3015,7 +3015,7 @@ public void testFetchDisconnectedShouldClearPreferredReadReplica() { assertEquals(1, sendFetches()); // Set preferred read replica to node=1 - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, FetchResponse.INVALID_LAST_STABLE_OFFSET, 0, Optional.of(1))); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); @@ -3028,7 +3028,7 @@ public void testFetchDisconnectedShouldClearPreferredReadReplica() { assertFalse(fetcher.hasCompletedFetches()); // Disconnect - preferred read replica should be cleared. - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0), true); + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0), true); consumerClient.poll(time.timer(0)); assertFalse(fetcher.hasCompletedFetches()); @@ -3048,7 +3048,7 @@ public void testFetchDisconnectedShouldNotClearPreferredReadReplicaIfUnassigned( assertEquals(1, sendFetches()); // Set preferred read replica to node=1 - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, FetchResponse.INVALID_LAST_STABLE_OFFSET, 0, Optional.of(1))); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); @@ -3061,7 +3061,7 @@ public void testFetchDisconnectedShouldNotClearPreferredReadReplicaIfUnassigned( assertFalse(fetcher.hasCompletedFetches()); // Disconnect and remove tp0 from assignment - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0), true); + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0), true); subscriptions.assignFromUser(emptySet()); // Preferred read replica should not be cleared @@ -3083,7 +3083,7 @@ public void testFetchErrorShouldClearPreferredReadReplica() { assertEquals(1, sendFetches()); // Set preferred read replica to node=1 - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, FetchResponse.INVALID_LAST_STABLE_OFFSET, 0, Optional.of(1))); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); @@ -3120,7 +3120,7 @@ public void testPreferredReadReplicaOffsetError() { assertEquals(1, sendFetches()); assertFalse(fetcher.hasCompletedFetches()); - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, FetchResponse.INVALID_LAST_STABLE_OFFSET, 0, Optional.of(1))); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); @@ -3134,7 +3134,7 @@ public void testPreferredReadReplicaOffsetError() { assertEquals(1, sendFetches()); assertFalse(fetcher.hasCompletedFetches()); - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.OFFSET_OUT_OF_RANGE, 100L, + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.OFFSET_OUT_OF_RANGE, 100L, FetchResponse.INVALID_LAST_STABLE_OFFSET, 0, Optional.empty())); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java index 323ac74ee693b..7724226b17222 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java @@ -222,9 +222,9 @@ private void assignFromUser(TopicPartition partition) { @AfterEach public void teardown() throws Exception { if (metrics != null) - this.metrics.close(); + metrics.close(); if (fetcher != null) - this.fetcher.close(); + fetcher.close(); if (executorService != null) { executorService.shutdownNow(); assertTrue(executorService.awaitTermination(5, TimeUnit.SECONDS)); @@ -247,7 +247,7 @@ public void testFetchNormal() { assertEquals(1, sendFetches()); assertFalse(fetcher.hasCompletedFetches()); - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); @@ -274,7 +274,7 @@ public void testInflightFetchOnPendingPartitions() { assertEquals(1, sendFetches()); subscriptions.markPendingRevocation(singleton(tp0)); - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); consumerClient.poll(time.timer(0)); assertNull(fetchedRecords().get(tp0)); } @@ -301,7 +301,7 @@ public void testFetcherCloseClosesFetchSessionsInBroker() { assertEquals(1, sendFetches()); assertFalse(fetcher.hasCompletedFetches()); - final FetchResponse fetchResponse = fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0); + final FetchResponse fetchResponse = fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0); client.prepareResponse(fetchResponse); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); @@ -310,7 +310,7 @@ public void testFetcherCloseClosesFetchSessionsInBroker() { final ArgumentCaptor argument = ArgumentCaptor.forClass(FetchRequest.Builder.class); // send request to close the fetcher - this.fetcher.close(time.timer(Duration.ofSeconds(10))); + fetcher.close(time.timer(Duration.ofSeconds(10))); // validate that Fetcher.close() has sent a request with final epoch. 2 requests are sent, one for the normal // fetch earlier and another for the finish fetch here. @@ -332,7 +332,7 @@ public void testFetchingPendingPartitions() { // normal fetch assertEquals(1, sendFetches()); - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); fetchedRecords(); @@ -362,7 +362,7 @@ public void testFetchWithNoTopicId() { client.prepareResponse( fetchRequestMatcher((short) 12, noId, 0, Optional.of(validLeaderEpoch)), - fullFetchResponse(noId, this.records, Errors.NONE, 100L, 0) + fullFetchResponse(noId, records, Errors.NONE, 100L, 0) ); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); @@ -394,7 +394,7 @@ public void testFetchWithTopicId() { client.prepareResponse( fetchRequestMatcher(ApiKeys.FETCH.latestVersion(), tp, 0, Optional.of(validLeaderEpoch)), - fullFetchResponse(tp, this.records, Errors.NONE, 100L, 0) + fullFetchResponse(tp, records, Errors.NONE, 100L, 0) ); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); @@ -438,7 +438,7 @@ public void testFetchForgetTopicIdWhenUnassigned() { ), emptyList() ), - fullFetchResponse(1, foo, this.records, Errors.NONE, 100L, 0) + fullFetchResponse(1, foo, records, Errors.NONE, 100L, 0) ); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); @@ -464,7 +464,7 @@ public void testFetchForgetTopicIdWhenUnassigned() { ), singletonList(foo) ), - fullFetchResponse(1, bar, this.records, Errors.NONE, 100L, 0) + fullFetchResponse(1, bar, records, Errors.NONE, 100L, 0) ); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); @@ -497,7 +497,7 @@ public void testFetchForgetTopicIdWhenReplaced() { ), emptyList() ), - fullFetchResponse(1, fooWithOldTopicId, this.records, Errors.NONE, 100L, 0) + fullFetchResponse(1, fooWithOldTopicId, records, Errors.NONE, 100L, 0) ); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); @@ -524,7 +524,7 @@ public void testFetchForgetTopicIdWhenReplaced() { ), singletonList(fooWithOldTopicId) ), - fullFetchResponse(1, fooWithNewTopicId, this.records, Errors.NONE, 100L, 0) + fullFetchResponse(1, fooWithNewTopicId, records, Errors.NONE, 100L, 0) ); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); @@ -557,7 +557,7 @@ public void testFetchTopicIdUpgradeDowngrade() { ), emptyList() ), - fullFetchResponse(1, fooWithoutId, this.records, Errors.NONE, 100L, 0) + fullFetchResponse(1, fooWithoutId, records, Errors.NONE, 100L, 0) ); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); @@ -584,7 +584,7 @@ public void testFetchTopicIdUpgradeDowngrade() { ), emptyList() ), - fullFetchResponse(1, fooWithId, this.records, Errors.NONE, 100L, 0) + fullFetchResponse(1, fooWithId, records, Errors.NONE, 100L, 0) ); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); @@ -611,7 +611,7 @@ public void testFetchTopicIdUpgradeDowngrade() { ), emptyList() ), - fullFetchResponse(1, fooWithoutId, this.records, Errors.NONE, 100L, 0) + fullFetchResponse(1, fooWithoutId, records, Errors.NONE, 100L, 0) ); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); @@ -756,7 +756,7 @@ public void testClearBufferedDataForTopicPartitions() { assertEquals(1, sendFetches()); assertFalse(fetcher.hasCompletedFetches()); - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); Set newAssignedTopicPartitions = new HashSet<>(); @@ -833,7 +833,7 @@ public void testFetchError() { assertEquals(1, sendFetches()); assertFalse(fetcher.hasCompletedFetches()); - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NOT_LEADER_OR_FOLLOWER, 100L, 0)); + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NOT_LEADER_OR_FOLLOWER, 100L, 0)); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); @@ -872,7 +872,7 @@ public byte[] deserialize(String topic, byte[] data) { assignFromUser(singleton(tp0)); subscriptions.seek(tp0, 1); - client.prepareResponse(matchesOffset(tidp0, 1), fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + client.prepareResponse(matchesOffset(tidp0, 1), fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); assertEquals(1, sendFetches()); consumerClient.poll(time.timer(0)); @@ -1107,7 +1107,7 @@ public void testFetchMaxPollRecords() { subscriptions.seek(tp0, 1); client.prepareResponse(matchesOffset(tidp0, 1), fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); - client.prepareResponse(matchesOffset(tidp0, 4), fullFetchResponse(tidp0, this.nextRecords, Errors.NONE, 100L, 0)); + client.prepareResponse(matchesOffset(tidp0, 4), fullFetchResponse(tidp0, nextRecords, Errors.NONE, 100L, 0)); assertEquals(1, sendFetches()); consumerClient.poll(time.timer(0)); @@ -1162,7 +1162,7 @@ public void testFetchAfterPartitionWithFetchedRecordsIsUnassigned() { assertEquals(2, records.get(1).offset()); assignFromUser(singleton(tp1)); - client.prepareResponse(matchesOffset(tidp1, 4), fullFetchResponse(tidp1, this.nextRecords, Errors.NONE, 100L, 0)); + client.prepareResponse(matchesOffset(tidp1, 4), fullFetchResponse(tidp1, nextRecords, Errors.NONE, 100L, 0)); subscriptions.seek(tp1, 4); assertEquals(1, sendFetches()); @@ -1272,7 +1272,7 @@ public void testUnauthorizedTopic() { // resize the limit of the buffer to pretend it is only fetch-size large assertEquals(1, sendFetches()); - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.TOPIC_AUTHORIZATION_FAILED, 100L, 0)); + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.TOPIC_AUTHORIZATION_FAILED, 100L, 0)); consumerClient.poll(time.timer(0)); try { collectFetch(); @@ -1299,7 +1299,7 @@ public void testFetchDuringEagerRebalance() { subscriptions.assignFromSubscribed(Collections.emptyList()); subscriptions.assignFromSubscribed(singleton(tp0)); - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); consumerClient.poll(time.timer(0)); // The active fetch should be ignored since its position is no longer valid @@ -1322,7 +1322,7 @@ public void testFetchDuringCooperativeRebalance() { // Now the cooperative rebalance happens and fetch positions are NOT cleared for unrevoked partitions subscriptions.assignFromSubscribed(singleton(tp0)); - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); consumerClient.poll(time.timer(0)); Map>> fetchedRecords = fetchedRecords(); @@ -1342,7 +1342,7 @@ public void testInFlightFetchOnPausedPartition() { assertEquals(1, sendFetches()); subscriptions.pause(tp0); - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); consumerClient.poll(time.timer(0)); assertNull(fetchedRecords().get(tp0)); } @@ -1370,7 +1370,7 @@ public void testFetchOnCompletedFetchesForPausedAndResumedPartitions() { subscriptions.pause(tp0); - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); consumerClient.poll(time.timer(0)); assertEmptyFetch("Should not return any records or advance position when partition is paused"); @@ -1407,13 +1407,13 @@ public void testFetchOnCompletedFetchesForSomePausedPartitions() { // #1 seek, request, poll, response subscriptions.seekUnvalidated(tp0, new SubscriptionState.FetchPosition(1, Optional.empty(), metadata.currentLeader(tp0))); assertEquals(1, sendFetches()); - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); consumerClient.poll(time.timer(0)); // #2 seek, request, poll, response subscriptions.seekUnvalidated(tp1, new SubscriptionState.FetchPosition(1, Optional.empty(), metadata.currentLeader(tp1))); assertEquals(1, sendFetches()); - client.prepareResponse(fullFetchResponse(tidp1, this.nextRecords, Errors.NONE, 100L, 0)); + client.prepareResponse(fullFetchResponse(tidp1, nextRecords, Errors.NONE, 100L, 0)); subscriptions.pause(tp0); consumerClient.poll(time.timer(0)); @@ -1439,13 +1439,13 @@ public void testFetchOnCompletedFetchesForAllPausedPartitions() { // #1 seek, request, poll, response subscriptions.seekUnvalidated(tp0, new SubscriptionState.FetchPosition(1, Optional.empty(), metadata.currentLeader(tp0))); assertEquals(1, sendFetches()); - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); consumerClient.poll(time.timer(0)); // #2 seek, request, poll, response subscriptions.seekUnvalidated(tp1, new SubscriptionState.FetchPosition(1, Optional.empty(), metadata.currentLeader(tp1))); assertEquals(1, sendFetches()); - client.prepareResponse(fullFetchResponse(tidp1, this.nextRecords, Errors.NONE, 100L, 0)); + client.prepareResponse(fullFetchResponse(tidp1, nextRecords, Errors.NONE, 100L, 0)); subscriptions.pause(tp0); subscriptions.pause(tp1); @@ -1470,7 +1470,7 @@ public void testPartialFetchWithPausedPartitions() { subscriptions.seek(tp0, 1); assertEquals(1, sendFetches()); - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); consumerClient.poll(time.timer(0)); fetchedRecords = fetchedRecords(); @@ -1505,7 +1505,7 @@ public void testFetchDiscardedAfterPausedPartitionResumedAndSoughtToNewOffset() assertEquals(1, sendFetches()); subscriptions.pause(tp0); - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); subscriptions.seek(tp0, 3); subscriptions.resume(tp0); @@ -1525,7 +1525,7 @@ public void testFetchNotLeaderOrFollower() { subscriptions.seek(tp0, 0); assertEquals(1, sendFetches()); - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NOT_LEADER_OR_FOLLOWER, 100L, 0)); + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NOT_LEADER_OR_FOLLOWER, 100L, 0)); consumerClient.poll(time.timer(0)); assertEmptyFetch("Should not return records or advance position on fetch error"); assertEquals(0L, metadata.timeToNextUpdate(time.milliseconds())); @@ -1538,7 +1538,7 @@ public void testFetchUnknownTopicOrPartition() { subscriptions.seek(tp0, 0); assertEquals(1, sendFetches()); - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.UNKNOWN_TOPIC_OR_PARTITION, 100L, 0)); + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.UNKNOWN_TOPIC_OR_PARTITION, 100L, 0)); consumerClient.poll(time.timer(0)); assertEmptyFetch("Should not return records or advance position on fetch error"); assertEquals(0L, metadata.timeToNextUpdate(time.milliseconds())); @@ -1551,7 +1551,7 @@ public void testFetchUnknownTopicId() { subscriptions.seek(tp0, 0); assertEquals(1, sendFetches()); - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.UNKNOWN_TOPIC_ID, -1L, 0)); + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.UNKNOWN_TOPIC_ID, -1L, 0)); consumerClient.poll(time.timer(0)); assertEmptyFetch("Should not return records or advance position on fetch error"); assertEquals(0L, metadata.timeToNextUpdate(time.milliseconds())); @@ -1577,7 +1577,7 @@ public void testFetchInconsistentTopicId() { subscriptions.seek(tp0, 0); assertEquals(1, sendFetches()); - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.INCONSISTENT_TOPIC_ID, -1L, 0)); + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.INCONSISTENT_TOPIC_ID, -1L, 0)); consumerClient.poll(time.timer(0)); assertEmptyFetch("Should not return records or advance position on fetch error"); assertEquals(0L, metadata.timeToNextUpdate(time.milliseconds())); @@ -1590,7 +1590,7 @@ public void testFetchFencedLeaderEpoch() { subscriptions.seek(tp0, 0); assertEquals(1, sendFetches()); - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.FENCED_LEADER_EPOCH, 100L, 0)); + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.FENCED_LEADER_EPOCH, 100L, 0)); consumerClient.poll(time.timer(0)); assertEmptyFetch("Should not return records or advance position on fetch error"); @@ -1604,7 +1604,7 @@ public void testFetchUnknownLeaderEpoch() { subscriptions.seek(tp0, 0); assertEquals(1, sendFetches()); - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.UNKNOWN_LEADER_EPOCH, 100L, 0)); + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.UNKNOWN_LEADER_EPOCH, 100L, 0)); consumerClient.poll(time.timer(0)); assertEmptyFetch("Should not return records or advance position on fetch error"); @@ -1636,7 +1636,7 @@ public void testEpochSetInFetchRequest() { return false; } }; - client.prepareResponse(matcher, fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + client.prepareResponse(matcher, fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); consumerClient.pollNoWakeup(); } @@ -1647,7 +1647,7 @@ public void testFetchOffsetOutOfRange() { subscriptions.seek(tp0, 0); assertEquals(1, sendFetches()); - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.OFFSET_OUT_OF_RANGE, 100L, 0)); + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.OFFSET_OUT_OF_RANGE, 100L, 0)); consumerClient.poll(time.timer(0)); assertEmptyFetch("Should not return records or advance position on fetch error"); assertTrue(subscriptions.isOffsetResetNeeded(tp0)); @@ -1664,7 +1664,7 @@ public void testStaleOutOfRangeError() { subscriptions.seek(tp0, 0); assertEquals(1, sendFetches()); - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.OFFSET_OUT_OF_RANGE, 100L, 0)); + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.OFFSET_OUT_OF_RANGE, 100L, 0)); subscriptions.seek(tp0, 1); consumerClient.poll(time.timer(0)); assertEmptyFetch("Should not return records or advance position on fetch error"); @@ -1681,7 +1681,7 @@ public void testFetchedRecordsAfterSeek() { subscriptions.seek(tp0, 0); assertTrue(sendFetches() > 0); - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.OFFSET_OUT_OF_RANGE, 100L, 0)); + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.OFFSET_OUT_OF_RANGE, 100L, 0)); consumerClient.poll(time.timer(0)); assertFalse(subscriptions.isOffsetResetNeeded(tp0)); subscriptions.seek(tp0, 2); @@ -1697,7 +1697,7 @@ public void testFetchOffsetOutOfRangeException() { subscriptions.seek(tp0, 0); sendFetches(); - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.OFFSET_OUT_OF_RANGE, 100L, 0)); + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.OFFSET_OUT_OF_RANGE, 100L, 0)); consumerClient.poll(time.timer(0)); assertFalse(subscriptions.isOffsetResetNeeded(tp0)); @@ -1854,7 +1854,7 @@ public void testSeekBeforeException() { .setPartitionIndex(tp0.partition()) .setHighWatermark(100) .setRecords(records)); - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); consumerClient.poll(time.timer(0)); assertEquals(2, fetchedRecords().get(tp0).size()); @@ -1885,7 +1885,7 @@ public void testFetchDisconnected() { subscriptions.seek(tp0, 0); assertEquals(1, sendFetches()); - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0), true); + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0), true); consumerClient.poll(time.timer(0)); assertEmptyFetch("Should not return records or advance position on disconnect"); @@ -2256,7 +2256,7 @@ public void testFetcherMetricsTemplates() { assignFromUser(singleton(tp0)); subscriptions.seek(tp0, 0); assertEquals(1, sendFetches()); - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); Map>> partitionRecords = fetchedRecords(); @@ -2803,7 +2803,7 @@ public void testConsumingViaIncrementalFetchRequests() { .setHighWatermark(100) .setLastStableOffset(4) .setLogStartOffset(0) - .setRecords(this.nextRecords)); + .setRecords(nextRecords)); FetchResponse resp3 = FetchResponse.of(Errors.NONE, 0, 123, partitions3); client.prepareResponse(resp3); assertEquals(1, sendFetches()); @@ -3248,7 +3248,7 @@ public void testPreferredReadReplica() { assertFalse(fetcher.hasCompletedFetches()); // Set preferred read replica to node=1 - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, FetchResponse.INVALID_LAST_STABLE_OFFSET, 0, Optional.of(1))); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); @@ -3265,7 +3265,7 @@ public void testPreferredReadReplica() { assertFalse(fetcher.hasCompletedFetches()); // Set preferred read replica to node=2, which isn't in our metadata, should revert to leader - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, FetchResponse.INVALID_LAST_STABLE_OFFSET, 0, Optional.of(2))); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); @@ -3285,7 +3285,7 @@ public void testFetchDisconnectedShouldClearPreferredReadReplica() { assertEquals(1, sendFetches()); // Set preferred read replica to node=1 - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, FetchResponse.INVALID_LAST_STABLE_OFFSET, 0, Optional.of(1))); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); @@ -3298,7 +3298,7 @@ public void testFetchDisconnectedShouldClearPreferredReadReplica() { assertFalse(fetcher.hasCompletedFetches()); // Disconnect - preferred read replica should be cleared. - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0), true); + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0), true); consumerClient.poll(time.timer(0)); assertFalse(fetcher.hasCompletedFetches()); @@ -3318,7 +3318,7 @@ public void testFetchDisconnectedShouldNotClearPreferredReadReplicaIfUnassigned( assertEquals(1, sendFetches()); // Set preferred read replica to node=1 - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, FetchResponse.INVALID_LAST_STABLE_OFFSET, 0, Optional.of(1))); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); @@ -3331,7 +3331,7 @@ public void testFetchDisconnectedShouldNotClearPreferredReadReplicaIfUnassigned( assertFalse(fetcher.hasCompletedFetches()); // Disconnect and remove tp0 from assignment - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0), true); + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0), true); subscriptions.assignFromUser(emptySet()); // Preferred read replica should not be cleared @@ -3353,7 +3353,7 @@ public void testFetchErrorShouldClearPreferredReadReplica() { assertEquals(1, sendFetches()); // Set preferred read replica to node=1 - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, FetchResponse.INVALID_LAST_STABLE_OFFSET, 0, Optional.of(1))); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); @@ -3390,7 +3390,7 @@ public void testPreferredReadReplicaOffsetError() { assertEquals(1, sendFetches()); assertFalse(fetcher.hasCompletedFetches()); - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, FetchResponse.INVALID_LAST_STABLE_OFFSET, 0, Optional.of(1))); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); @@ -3404,7 +3404,7 @@ public void testPreferredReadReplicaOffsetError() { assertEquals(1, sendFetches()); assertFalse(fetcher.hasCompletedFetches()); - client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.OFFSET_OUT_OF_RANGE, 100L, + client.prepareResponse(fullFetchResponse(tidp0, records, Errors.OFFSET_OUT_OF_RANGE, 100L, FetchResponse.INVALID_LAST_STABLE_OFFSET, 0, Optional.empty())); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); From 81c0c7e5d32facc244c29092c689bca1c3818d8e Mon Sep 17 00:00:00 2001 From: Kirk True Date: Tue, 3 Oct 2023 15:06:36 -0700 Subject: [PATCH 18/72] Renamed FetchRequestManagerTest's consumerClient to networkClientDelegate --- .../internals/FetchRequestManagerTest.java | 220 +++++++++--------- 1 file changed, 110 insertions(+), 110 deletions(-) diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java index 4a9aaecdbc517..acbb5b3b08845 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java @@ -187,7 +187,7 @@ public class FetchRequestManagerTest { private ApiVersions apiVersions = new ApiVersions(); private ConsumerNetworkClient oldConsumerClient; private TestableFetchRequestManager fetcher; - private TestableNetworkClientDelegate consumerClient; + private TestableNetworkClientDelegate networkClientDelegate; private OffsetFetcher offsetFetcher; private MemoryRecords records; @@ -249,7 +249,7 @@ public void testFetchNormal() { assertFalse(fetcher.hasCompletedFetches()); client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); Map>> partitionRecords = fetchedRecords(); @@ -276,7 +276,7 @@ public void testInflightFetchOnPendingPartitions() { subscriptions.markPendingRevocation(singleton(tp0)); client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); assertNull(fetchedRecords().get(tp0)); } @@ -304,9 +304,9 @@ public void testFetcherCloseClosesFetchSessionsInBroker() { final FetchResponse fetchResponse = fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0); client.prepareResponse(fetchResponse); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - assertEquals(0, consumerClient.pendingRequestCount()); + assertEquals(0, networkClientDelegate.pendingRequestCount()); final ArgumentCaptor argument = ArgumentCaptor.forClass(NetworkClientDelegate.UnsentRequest.class); @@ -319,16 +319,16 @@ public void testFetcherCloseClosesFetchSessionsInBroker() { // handled inside DefaultBackgroundThread.runAtClose, so we need to run that logic here. DefaultBackgroundThread.runAtClose(time, Collections.singletonList(Optional.of(fetcher)), - consumerClient, + networkClientDelegate, timer); NetworkClientDelegate.PollResult pollResult = fetcher.poll(time.milliseconds()); - consumerClient.addAll(pollResult.unsentRequests); - consumerClient.poll(timer); + networkClientDelegate.addAll(pollResult.unsentRequests); + networkClientDelegate.poll(timer); // validate that Fetcher.close() has sent a request with final epoch. 2 requests are sent, one for the normal // fetch earlier and another for the finish fetch here. - verify(consumerClient, times(2)).doSend(argument.capture(), any(Long.class)); + verify(networkClientDelegate, times(2)).doSend(argument.capture(), any(Long.class)); NetworkClientDelegate.UnsentRequest unsentRequest = argument.getValue(); FetchRequest.Builder builder = (FetchRequest.Builder) unsentRequest.requestBuilder(); // session Id is the same @@ -348,7 +348,7 @@ public void testFetchingPendingPartitions() { // normal fetch assertEquals(1, sendFetches()); client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); fetchedRecords(); assertEquals(4L, subscriptions.position(tp0).offset); // this is the next fetching position @@ -356,7 +356,7 @@ public void testFetchingPendingPartitions() { // mark partition unfetchable subscriptions.markPendingRevocation(singleton(tp0)); assertEquals(0, sendFetches()); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); assertFalse(fetcher.hasCompletedFetches()); fetchedRecords(); assertEquals(4L, subscriptions.position(tp0).offset); @@ -379,7 +379,7 @@ public void testFetchWithNoTopicId() { fetchRequestMatcher((short) 12, noId, 0, Optional.of(validLeaderEpoch)), fullFetchResponse(noId, records, Errors.NONE, 100L, 0) ); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); Map>> partitionRecords = fetchedRecords(); @@ -411,7 +411,7 @@ public void testFetchWithTopicId() { fetchRequestMatcher(ApiKeys.FETCH.latestVersion(), tp, 0, Optional.of(validLeaderEpoch)), fullFetchResponse(tp, records, Errors.NONE, 100L, 0) ); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); Map>> partitionRecords = fetchedRecords(); @@ -455,7 +455,7 @@ public void testFetchForgetTopicIdWhenUnassigned() { ), fullFetchResponse(1, foo, records, Errors.NONE, 100L, 0) ); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); fetchedRecords(); @@ -481,7 +481,7 @@ public void testFetchForgetTopicIdWhenUnassigned() { ), fullFetchResponse(1, bar, records, Errors.NONE, 100L, 0) ); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); fetchedRecords(); } @@ -514,7 +514,7 @@ public void testFetchForgetTopicIdWhenReplaced() { ), fullFetchResponse(1, fooWithOldTopicId, records, Errors.NONE, 100L, 0) ); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); fetchedRecords(); @@ -541,7 +541,7 @@ public void testFetchForgetTopicIdWhenReplaced() { ), fullFetchResponse(1, fooWithNewTopicId, records, Errors.NONE, 100L, 0) ); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); fetchedRecords(); } @@ -574,7 +574,7 @@ public void testFetchTopicIdUpgradeDowngrade() { ), fullFetchResponse(1, fooWithoutId, records, Errors.NONE, 100L, 0) ); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); fetchedRecords(); @@ -601,7 +601,7 @@ public void testFetchTopicIdUpgradeDowngrade() { ), fullFetchResponse(1, fooWithId, records, Errors.NONE, 100L, 0) ); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); fetchedRecords(); @@ -628,7 +628,7 @@ public void testFetchTopicIdUpgradeDowngrade() { ), fullFetchResponse(1, fooWithoutId, records, Errors.NONE, 100L, 0) ); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); fetchedRecords(); } @@ -696,7 +696,7 @@ public void testMissingLeaderEpochInRecords() { assertFalse(fetcher.hasCompletedFetches()); client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); Map>> partitionRecords = fetchedRecords(); @@ -747,7 +747,7 @@ public void testLeaderEpochInConsumerRecord() { assertFalse(fetcher.hasCompletedFetches()); client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); Map>> partitionRecords = fetchedRecords(); @@ -772,7 +772,7 @@ public void testClearBufferedDataForTopicPartitions() { assertFalse(fetcher.hasCompletedFetches()); client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); Set newAssignedTopicPartitions = new HashSet<>(); newAssignedTopicPartitions.add(tp1); @@ -824,7 +824,7 @@ public void testFetcherIgnoresControlRecords() { buffer.flip(); client.prepareResponse(fullFetchResponse(tidp0, MemoryRecords.readableRecords(buffer), Errors.NONE, 100L, 0)); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); Map>> partitionRecords = fetchedRecords(); @@ -849,7 +849,7 @@ public void testFetchError() { assertFalse(fetcher.hasCompletedFetches()); client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NOT_LEADER_OR_FOLLOWER, 100L, 0)); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); Map>> partitionRecords = fetchedRecords(); @@ -890,7 +890,7 @@ public byte[] deserialize(String topic, byte[] data) { client.prepareResponse(matchesOffset(tidp0, 1), fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); assertEquals(1, sendFetches()); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); // The fetcher should block on Deserialization error for (int i = 0; i < 2; i++) { try { @@ -952,7 +952,7 @@ public void testParseCorruptedRecord() throws Exception { // normal fetch assertEquals(1, sendFetches()); client.prepareResponse(fullFetchResponse(tidp0, MemoryRecords.readableRecords(buffer), Errors.NONE, 100L, 0)); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); // the first fetchedRecords() should return the first valid message assertEquals(1, fetchedRecords().get(tp0).size()); @@ -990,7 +990,7 @@ private void seekAndConsumeRecord(ByteBuffer responseBuffer, long toOffset) { fetcher.collectFetch(); assertEquals(1, sendFetches()); client.prepareResponse(fullFetchResponse(tidp0, MemoryRecords.readableRecords(responseBuffer), Errors.NONE, 100L, 0)); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); Map>> recordsByPartition = fetchedRecords(); List> records = recordsByPartition.get(tp0); @@ -1026,7 +1026,7 @@ public void testInvalidDefaultRecordBatch() { // normal fetch assertEquals(1, sendFetches()); client.prepareResponse(fullFetchResponse(tidp0, MemoryRecords.readableRecords(buffer), Errors.NONE, 100L, 0)); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); // the fetchedRecords() should always throw exception due to the bad batch. for (int i = 0; i < 2; i++) { @@ -1058,7 +1058,7 @@ public void testParseInvalidRecordBatch() { // normal fetch assertEquals(1, sendFetches()); client.prepareResponse(fullFetchResponse(tidp0, MemoryRecords.readableRecords(buffer), Errors.NONE, 100L, 0)); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); try { fetcher.collectFetch(); fail("fetchedRecords should have raised"); @@ -1093,7 +1093,7 @@ public void testHeaders() { client.prepareResponse(matchesOffset(tidp0, 1), fullFetchResponse(tidp0, memoryRecords, Errors.NONE, 100L, 0)); assertEquals(1, sendFetches()); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); Map>> recordsByPartition = fetchedRecords(); records = recordsByPartition.get(tp0); @@ -1125,7 +1125,7 @@ public void testFetchMaxPollRecords() { client.prepareResponse(matchesOffset(tidp0, 4), fullFetchResponse(tidp0, nextRecords, Errors.NONE, 100L, 0)); assertEquals(1, sendFetches()); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); Map>> recordsByPartition = fetchedRecords(); records = recordsByPartition.get(tp0); assertEquals(2, records.size()); @@ -1134,7 +1134,7 @@ public void testFetchMaxPollRecords() { assertEquals(2, records.get(1).offset()); assertEquals(0, sendFetches()); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); recordsByPartition = fetchedRecords(); records = recordsByPartition.get(tp0); assertEquals(1, records.size()); @@ -1142,7 +1142,7 @@ public void testFetchMaxPollRecords() { assertEquals(3, records.get(0).offset()); assertTrue(sendFetches() > 0); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); recordsByPartition = fetchedRecords(); records = recordsByPartition.get(tp0); assertEquals(2, records.size()); @@ -1168,7 +1168,7 @@ public void testFetchAfterPartitionWithFetchedRecordsIsUnassigned() { client.prepareResponse(matchesOffset(tidp0, 1), fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); assertEquals(1, sendFetches()); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); Map>> recordsByPartition = fetchedRecords(); records = recordsByPartition.get(tp0); assertEquals(2, records.size()); @@ -1181,7 +1181,7 @@ public void testFetchAfterPartitionWithFetchedRecordsIsUnassigned() { subscriptions.seek(tp1, 4); assertEquals(1, sendFetches()); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); Map>> fetchedRecords = fetchedRecords(); assertNull(fetchedRecords.get(tp0)); records = fetchedRecords.get(tp1); @@ -1211,7 +1211,7 @@ public void testFetchNonContinuousRecords() { // normal fetch assertEquals(1, sendFetches()); client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); Map>> recordsByPartition = fetchedRecords(); consumerRecords = recordsByPartition.get(tp0); assertEquals(3, consumerRecords.size()); @@ -1274,7 +1274,7 @@ private void makeFetchRequestWithIncompleteRecord() { MemoryRecords partialRecord = MemoryRecords.readableRecords( ByteBuffer.wrap(new byte[]{0, 0, 0, 0, 0, 0, 0, 0})); client.prepareResponse(fullFetchResponse(tidp0, partialRecord, Errors.NONE, 100L, 0)); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); } @@ -1288,7 +1288,7 @@ public void testUnauthorizedTopic() { // resize the limit of the buffer to pretend it is only fetch-size large assertEquals(1, sendFetches()); client.prepareResponse(fullFetchResponse(tidp0, records, Errors.TOPIC_AUTHORIZATION_FAILED, 100L, 0)); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); try { fetcher.collectFetch(); fail("fetchedRecords should have thrown"); @@ -1315,7 +1315,7 @@ public void testFetchDuringEagerRebalance() { subscriptions.assignFromSubscribed(singleton(tp0)); client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); // The active fetch should be ignored since its position is no longer valid assertTrue(fetchedRecords().isEmpty()); @@ -1338,7 +1338,7 @@ public void testFetchDuringCooperativeRebalance() { subscriptions.assignFromSubscribed(singleton(tp0)); client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); Map>> fetchedRecords = fetchedRecords(); @@ -1358,7 +1358,7 @@ public void testInFlightFetchOnPausedPartition() { subscriptions.pause(tp0); client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); assertNull(fetchedRecords().get(tp0)); } @@ -1386,7 +1386,7 @@ public void testFetchOnCompletedFetchesForPausedAndResumedPartitions() { subscriptions.pause(tp0); client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); assertEmptyFetch("Should not return any records or advance position when partition is paused"); @@ -1398,13 +1398,13 @@ public void testFetchOnCompletedFetchesForPausedAndResumedPartitions() { assertTrue(fetcher.hasAvailableFetches(), "Should have available (non-paused) completed fetches"); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); Map>> fetchedRecords = fetchedRecords(); assertEquals(1, fetchedRecords.size(), "Should return records when partition is resumed"); assertNotNull(fetchedRecords.get(tp0)); assertEquals(3, fetchedRecords.get(tp0).size()); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); assertEmptyFetch("Should not return records or advance position after previously paused partitions are fetched"); assertFalse(fetcher.hasCompletedFetches(), "Should no longer contain completed fetches"); } @@ -1423,7 +1423,7 @@ public void testFetchOnCompletedFetchesForSomePausedPartitions() { subscriptions.seekUnvalidated(tp0, new SubscriptionState.FetchPosition(1, Optional.empty(), metadata.currentLeader(tp0))); assertEquals(1, sendFetches()); client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); // #2 seek, request, poll, response subscriptions.seekUnvalidated(tp1, new SubscriptionState.FetchPosition(1, Optional.empty(), metadata.currentLeader(tp1))); @@ -1431,7 +1431,7 @@ public void testFetchOnCompletedFetchesForSomePausedPartitions() { client.prepareResponse(fullFetchResponse(tidp1, nextRecords, Errors.NONE, 100L, 0)); subscriptions.pause(tp0); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); fetchedRecords = fetchedRecords(); assertEquals(1, fetchedRecords.size(), "Should return completed fetch for unpaused partitions"); @@ -1455,7 +1455,7 @@ public void testFetchOnCompletedFetchesForAllPausedPartitions() { subscriptions.seekUnvalidated(tp0, new SubscriptionState.FetchPosition(1, Optional.empty(), metadata.currentLeader(tp0))); assertEquals(1, sendFetches()); client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); // #2 seek, request, poll, response subscriptions.seekUnvalidated(tp1, new SubscriptionState.FetchPosition(1, Optional.empty(), metadata.currentLeader(tp1))); @@ -1465,7 +1465,7 @@ public void testFetchOnCompletedFetchesForAllPausedPartitions() { subscriptions.pause(tp0); subscriptions.pause(tp1); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); assertEmptyFetch("Should not return records or advance position for all paused partitions"); assertTrue(fetcher.hasCompletedFetches(), "Should still contain completed fetches"); @@ -1486,7 +1486,7 @@ public void testPartialFetchWithPausedPartitions() { subscriptions.seek(tp0, 1); assertEquals(1, sendFetches()); client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); fetchedRecords = fetchedRecords(); @@ -1494,7 +1494,7 @@ public void testPartialFetchWithPausedPartitions() { assertFalse(fetcher.hasCompletedFetches(), "Should have no completed fetches"); subscriptions.pause(tp0); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); fetchedRecords = fetchedRecords(); @@ -1504,7 +1504,7 @@ public void testPartialFetchWithPausedPartitions() { subscriptions.resume(tp0); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); fetchedRecords = fetchedRecords(); @@ -1524,7 +1524,7 @@ public void testFetchDiscardedAfterPausedPartitionResumedAndSeekedToNewOffset() subscriptions.seek(tp0, 3); subscriptions.resume(tp0); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches(), "Should have 1 entry in completed fetches"); Fetch fetch = collectFetch(); @@ -1541,7 +1541,7 @@ public void testFetchNotLeaderOrFollower() { assertEquals(1, sendFetches()); client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NOT_LEADER_OR_FOLLOWER, 100L, 0)); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); assertEmptyFetch("Should not return records or advance position on fetch error"); assertEquals(0L, metadata.timeToNextUpdate(time.milliseconds())); } @@ -1554,7 +1554,7 @@ public void testFetchUnknownTopicOrPartition() { assertEquals(1, sendFetches()); client.prepareResponse(fullFetchResponse(tidp0, records, Errors.UNKNOWN_TOPIC_OR_PARTITION, 100L, 0)); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); assertEmptyFetch("Should not return records or advance position on fetch error"); assertEquals(0L, metadata.timeToNextUpdate(time.milliseconds())); } @@ -1567,7 +1567,7 @@ public void testFetchUnknownTopicId() { assertEquals(1, sendFetches()); client.prepareResponse(fullFetchResponse(tidp0, records, Errors.UNKNOWN_TOPIC_ID, -1L, 0)); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); assertEmptyFetch("Should not return records or advance position on fetch error"); assertEquals(0L, metadata.timeToNextUpdate(time.milliseconds())); } @@ -1580,7 +1580,7 @@ public void testFetchSessionIdError() { assertEquals(1, sendFetches()); client.prepareResponse(fetchResponseWithTopLevelError(tidp0, Errors.FETCH_SESSION_TOPIC_ID_ERROR, 0)); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); assertEmptyFetch("Should not return records or advance position on fetch error"); assertEquals(0L, metadata.timeToNextUpdate(time.milliseconds())); } @@ -1593,7 +1593,7 @@ public void testFetchInconsistentTopicId() { assertEquals(1, sendFetches()); client.prepareResponse(fullFetchResponse(tidp0, records, Errors.INCONSISTENT_TOPIC_ID, -1L, 0)); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); assertEmptyFetch("Should not return records or advance position on fetch error"); assertEquals(0L, metadata.timeToNextUpdate(time.milliseconds())); } @@ -1606,7 +1606,7 @@ public void testFetchFencedLeaderEpoch() { assertEquals(1, sendFetches()); client.prepareResponse(fullFetchResponse(tidp0, records, Errors.FENCED_LEADER_EPOCH, 100L, 0)); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); assertEmptyFetch("Should not return records or advance position on fetch error"); assertEquals(0L, metadata.timeToNextUpdate(time.milliseconds()), "Should have requested metadata update"); @@ -1620,7 +1620,7 @@ public void testFetchUnknownLeaderEpoch() { assertEquals(1, sendFetches()); client.prepareResponse(fullFetchResponse(tidp0, records, Errors.UNKNOWN_LEADER_EPOCH, 100L, 0)); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); assertEmptyFetch("Should not return records or advance position on fetch error"); assertNotEquals(0L, metadata.timeToNextUpdate(time.milliseconds()), "Should not have requested metadata update"); @@ -1652,7 +1652,7 @@ public void testEpochSetInFetchRequest() { } }; client.prepareResponse(matcher, fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); - consumerClient.pollNoWakeup(); + networkClientDelegate.pollNoWakeup(); } @Test @@ -1663,7 +1663,7 @@ public void testFetchOffsetOutOfRange() { assertEquals(1, sendFetches()); client.prepareResponse(fullFetchResponse(tidp0, records, Errors.OFFSET_OUT_OF_RANGE, 100L, 0)); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); assertEmptyFetch("Should not return records or advance position on fetch error"); assertTrue(subscriptions.isOffsetResetNeeded(tp0)); assertNull(subscriptions.validPosition(tp0)); @@ -1681,7 +1681,7 @@ public void testStaleOutOfRangeError() { assertEquals(1, sendFetches()); client.prepareResponse(fullFetchResponse(tidp0, records, Errors.OFFSET_OUT_OF_RANGE, 100L, 0)); subscriptions.seek(tp0, 1); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); assertEmptyFetch("Should not return records or advance position on fetch error"); assertFalse(subscriptions.isOffsetResetNeeded(tp0)); assertEquals(1, subscriptions.position(tp0).offset); @@ -1697,7 +1697,7 @@ public void testFetchedRecordsAfterSeek() { assertTrue(sendFetches() > 0); client.prepareResponse(fullFetchResponse(tidp0, records, Errors.OFFSET_OUT_OF_RANGE, 100L, 0)); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); assertFalse(subscriptions.isOffsetResetNeeded(tp0)); subscriptions.seek(tp0, 2); assertEmptyFetch("Should not return records or advance position after seeking to end of topic partition"); @@ -1713,7 +1713,7 @@ public void testFetchOffsetOutOfRangeException() { sendFetches(); client.prepareResponse(fullFetchResponse(tidp0, records, Errors.OFFSET_OUT_OF_RANGE, 100L, 0)); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); assertFalse(subscriptions.isOffsetResetNeeded(tp0)); for (int i = 0; i < 2; i++) { @@ -1746,7 +1746,7 @@ public void testFetchPositionAfterException() { .setErrorCode(Errors.OFFSET_OUT_OF_RANGE.code()) .setHighWatermark(100)); client.prepareResponse(FetchResponse.of(Errors.NONE, 0, INVALID_SESSION_ID, new LinkedHashMap<>(partitions))); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); List> allFetchedRecords = new ArrayList<>(); fetchRecordsInto(allFetchedRecords); @@ -1807,7 +1807,7 @@ public void testCompletedFetchRemoval() { .setLogStartOffset(0) .setRecords(partialRecords)); client.prepareResponse(FetchResponse.of(Errors.NONE, 0, INVALID_SESSION_ID, new LinkedHashMap<>(partitions))); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); List> fetchedRecords = new ArrayList<>(); Map>> recordsByPartition = fetchedRecords(); @@ -1870,7 +1870,7 @@ public void testSeekBeforeException() { .setHighWatermark(100) .setRecords(records)); client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); assertEquals(2, fetchedRecords().get(tp0).size()); @@ -1884,7 +1884,7 @@ public void testSeekBeforeException() { .setErrorCode(Errors.OFFSET_OUT_OF_RANGE.code()) .setHighWatermark(100)); client.prepareResponse(FetchResponse.of(Errors.NONE, 0, INVALID_SESSION_ID, new LinkedHashMap<>(partitions))); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); assertEquals(1, fetchedRecords().get(tp0).size()); subscriptions.seek(tp1, 10); @@ -1901,7 +1901,7 @@ public void testFetchDisconnected() { assertEquals(1, sendFetches()); client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0), true); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); assertEmptyFetch("Should not return records or advance position on disconnect"); // disconnects should have no affect on subscription state @@ -2131,7 +2131,7 @@ public void testFetchResponseMetrics() { assertEquals(1, sendFetches()); client.prepareResponse(FetchResponse.of(Errors.NONE, 0, INVALID_SESSION_ID, fetchPartitionData)); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); Map>> fetchedRecords = fetchedRecords(); assertEquals(3, fetchedRecords.get(tp1).size()); @@ -2203,7 +2203,7 @@ public void testFetchResponseMetricsWithOnePartitionError() { assertEquals(1, sendFetches()); client.prepareResponse(FetchResponse.of(Errors.NONE, 0, INVALID_SESSION_ID, new LinkedHashMap<>(partitions))); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); fetcher.collectFetch(); int expectedBytes = 0; @@ -2249,7 +2249,7 @@ public void testFetchResponseMetricsWithOnePartitionAtTheWrongOffset() { .setRecords(MemoryRecords.withRecords(CompressionType.NONE, new SimpleRecord("val".getBytes())))); client.prepareResponse(FetchResponse.of(Errors.NONE, 0, INVALID_SESSION_ID, new LinkedHashMap<>(partitions))); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); fetcher.collectFetch(); // we should have ignored the record at the wrong offset @@ -2272,7 +2272,7 @@ public void testFetcherMetricsTemplates() { subscriptions.seek(tp0, 0); assertEquals(1, sendFetches()); client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); Map>> partitionRecords = fetchedRecords(); assertTrue(partitionRecords.containsKey(tp0)); @@ -2296,7 +2296,7 @@ private Map>> fetchRecords( TopicIdPartition tp, MemoryRecords records, Errors error, long hw, long lastStableOffset, int throttleTime) { assertEquals(1, sendFetches()); client.prepareResponse(fullFetchResponse(tp, records, error, hw, lastStableOffset, throttleTime)); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); return fetchedRecords(); } @@ -2304,7 +2304,7 @@ private Map>> fetchRecords( TopicIdPartition tp, MemoryRecords records, Errors error, long hw, long lastStableOffset, long logStartOffset, int throttleTime) { assertEquals(1, sendFetches()); client.prepareResponse(fetchResponse(tp, records, error, hw, lastStableOffset, logStartOffset, throttleTime)); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); return fetchedRecords(); } @@ -2335,7 +2335,7 @@ public void testSkippingAbortedTransactions() { assertFalse(fetcher.hasCompletedFetches()); client.prepareResponse(fullFetchResponseWithAbortedTransactions(records, abortedTransactions, Errors.NONE, 100L, 100L, 0)); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); Fetch fetch = collectFetch(); @@ -2371,7 +2371,7 @@ public void testReturnCommittedTransactions() { return true; }, fullFetchResponseWithAbortedTransactions(records, Collections.emptyList(), Errors.NONE, 100L, 100L, 0)); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); Map>> fetchedRecords = fetchedRecords(); @@ -2441,7 +2441,7 @@ public void testReadCommittedWithCommittedAndAbortedTransactions() { assertFalse(fetcher.hasCompletedFetches()); client.prepareResponse(fullFetchResponseWithAbortedTransactions(records, abortedTransactions, Errors.NONE, 100L, 100L, 0)); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); Map>> fetchedRecords = fetchedRecords(); @@ -2489,7 +2489,7 @@ public void testMultipleAbortMarkers() { assertFalse(fetcher.hasCompletedFetches()); client.prepareResponse(fullFetchResponseWithAbortedTransactions(records, abortedTransactions, Errors.NONE, 100L, 100L, 0)); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); Map>> fetchedRecords = fetchedRecords(); @@ -2534,7 +2534,7 @@ public void testReadCommittedAbortMarkerWithNoData() { client.prepareResponse(fullFetchResponseWithAbortedTransactions(MemoryRecords.readableRecords(buffer), abortedTransactions, Errors.NONE, 100L, 100L, 0)); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); Map>> allFetchedRecords = fetchedRecords(); @@ -2573,7 +2573,7 @@ protected boolean shouldRetainRecord(RecordBatch recordBatch, Record record) { subscriptions.seek(tp0, 0); assertEquals(1, sendFetches()); client.prepareResponse(fullFetchResponse(tidp0, compactedRecords, Errors.NONE, 100L, 0)); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); Map>> allFetchedRecords = fetchedRecords(); @@ -2610,7 +2610,7 @@ public void testUpdatePositionOnEmptyBatch() { subscriptions.seek(tp0, 0); assertEquals(1, sendFetches()); client.prepareResponse(fullFetchResponse(tidp0, recordsWithEmptyBatch, Errors.NONE, 100L, 0)); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); Fetch fetch = collectFetch(); @@ -2674,7 +2674,7 @@ public void testReadCommittedWithCompactedTopic() { client.prepareResponse(fullFetchResponseWithAbortedTransactions(MemoryRecords.readableRecords(buffer), abortedTransactions, Errors.NONE, 100L, 100L, 0)); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); Map>> allFetchedRecords = fetchedRecords(); @@ -2711,7 +2711,7 @@ public void testReturnAbortedTransactionsinUncommittedMode() { assertFalse(fetcher.hasCompletedFetches()); client.prepareResponse(fullFetchResponseWithAbortedTransactions(records, abortedTransactions, Errors.NONE, 100L, 100L, 0)); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); Map>> fetchedRecords = fetchedRecords(); @@ -2744,7 +2744,7 @@ public void testConsumerPositionUpdatedWhenSkippingAbortedTransactions() { assertFalse(fetcher.hasCompletedFetches()); client.prepareResponse(fullFetchResponseWithAbortedTransactions(records, abortedTransactions, Errors.NONE, 100L, 100L, 0)); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); Map>> fetchedRecords = fetchedRecords(); @@ -2780,7 +2780,7 @@ public void testConsumingViaIncrementalFetchRequests() { client.prepareResponse(resp1); assertEquals(1, sendFetches()); assertFalse(fetcher.hasCompletedFetches()); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); Map>> fetchedRecords = fetchedRecords(); assertFalse(fetchedRecords.containsKey(tp1)); @@ -2805,7 +2805,7 @@ public void testConsumingViaIncrementalFetchRequests() { FetchResponse resp2 = FetchResponse.of(Errors.NONE, 0, 123, partitions2); client.prepareResponse(resp2); assertEquals(1, sendFetches()); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); fetchedRecords = fetchedRecords(); assertTrue(fetchedRecords.isEmpty()); assertEquals(4L, subscriptions.position(tp0).offset); @@ -2822,7 +2822,7 @@ public void testConsumingViaIncrementalFetchRequests() { FetchResponse resp3 = FetchResponse.of(Errors.NONE, 0, 123, partitions3); client.prepareResponse(resp3); assertEquals(1, sendFetches()); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); fetchedRecords = fetchedRecords(); assertFalse(fetchedRecords.containsKey(tp1)); records = fetchedRecords.get(tp0); @@ -2867,7 +2867,7 @@ public void testEmptyControlBatch() { return true; }, fullFetchResponseWithAbortedTransactions(records, Collections.emptyList(), Errors.NONE, 100L, 100L, 0)); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); Map>> fetchedRecords = fetchedRecords(); @@ -2951,7 +2951,7 @@ public void testSubscriptionPositionUpdatedWithEpoch() { assertFalse(fetcher.hasCompletedFetches()); client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); - consumerClient.pollNoWakeup(); + networkClientDelegate.pollNoWakeup(); assertTrue(fetcher.hasCompletedFetches()); Map>> partitionRecords = fetchedRecords(); @@ -2980,7 +2980,7 @@ public void testPreferredReadReplica() { // Set preferred read replica to node=1 client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, FetchResponse.INVALID_LAST_STABLE_OFFSET, 0, Optional.of(1))); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); Map>> partitionRecords = fetchedRecords(); @@ -2997,7 +2997,7 @@ public void testPreferredReadReplica() { // Set preferred read replica to node=2, which isn't in our metadata, should revert to leader client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, FetchResponse.INVALID_LAST_STABLE_OFFSET, 0, Optional.of(2))); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); fetchedRecords(); selected = fetcher.selectReadReplica(tp0, Node.noNode(), time.milliseconds()); @@ -3017,7 +3017,7 @@ public void testFetchDisconnectedShouldClearPreferredReadReplica() { // Set preferred read replica to node=1 client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, FetchResponse.INVALID_LAST_STABLE_OFFSET, 0, Optional.of(1))); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); fetchedRecords(); @@ -3030,7 +3030,7 @@ public void testFetchDisconnectedShouldClearPreferredReadReplica() { // Disconnect - preferred read replica should be cleared. client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0), true); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); assertFalse(fetcher.hasCompletedFetches()); fetchedRecords(); selected = fetcher.selectReadReplica(tp0, Node.noNode(), time.milliseconds()); @@ -3050,7 +3050,7 @@ public void testFetchDisconnectedShouldNotClearPreferredReadReplicaIfUnassigned( // Set preferred read replica to node=1 client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, FetchResponse.INVALID_LAST_STABLE_OFFSET, 0, Optional.of(1))); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); fetchedRecords(); @@ -3065,7 +3065,7 @@ public void testFetchDisconnectedShouldNotClearPreferredReadReplicaIfUnassigned( subscriptions.assignFromUser(emptySet()); // Preferred read replica should not be cleared - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); assertFalse(fetcher.hasCompletedFetches()); fetchedRecords(); selected = fetcher.selectReadReplica(tp0, Node.noNode(), time.milliseconds()); @@ -3085,7 +3085,7 @@ public void testFetchErrorShouldClearPreferredReadReplica() { // Set preferred read replica to node=1 client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, FetchResponse.INVALID_LAST_STABLE_OFFSET, 0, Optional.of(1))); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); fetchedRecords(); @@ -3100,7 +3100,7 @@ public void testFetchErrorShouldClearPreferredReadReplica() { client.prepareResponse(fullFetchResponse(tidp0, MemoryRecords.EMPTY, Errors.NOT_LEADER_OR_FOLLOWER, -1L, FetchResponse.INVALID_LAST_STABLE_OFFSET, 0, Optional.of(1))); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); fetchedRecords(); selected = fetcher.selectReadReplica(tp0, Node.noNode(), time.milliseconds()); @@ -3122,7 +3122,7 @@ public void testPreferredReadReplicaOffsetError() { client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, FetchResponse.INVALID_LAST_STABLE_OFFSET, 0, Optional.of(1))); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); fetchedRecords(); @@ -3136,7 +3136,7 @@ public void testPreferredReadReplicaOffsetError() { client.prepareResponse(fullFetchResponse(tidp0, records, Errors.OFFSET_OUT_OF_RANGE, 100L, FetchResponse.INVALID_LAST_STABLE_OFFSET, 0, Optional.empty())); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); fetchedRecords(); @@ -3152,7 +3152,7 @@ public void testFetchCompletedBeforeHandlerAdded() { subscriptions.seek(tp0, 0); sendFetches(); client.prepareResponse(fullFetchResponse(tidp0, buildRecords(1L, 1, 1), Errors.NONE, 100L, 0)); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); fetchedRecords(); Metadata.LeaderAndEpoch leaderAndEpoch = subscriptions.position(tp0).currentLeader; @@ -3162,15 +3162,15 @@ public void testFetchCompletedBeforeHandlerAdded() { AtomicBoolean wokenUp = new AtomicBoolean(false); client.setWakeupHook(() -> { if (!wokenUp.getAndSet(true)) { - consumerClient.disconnectAsync(readReplica); - consumerClient.poll(time.timer(0)); + networkClientDelegate.disconnectAsync(readReplica); + networkClientDelegate.poll(time.timer(0)); } }); assertEquals(1, sendFetches()); - consumerClient.disconnectAsync(readReplica); - consumerClient.poll(time.timer(0)); + networkClientDelegate.disconnectAsync(readReplica); + networkClientDelegate.poll(time.timer(0)); assertEquals(1, sendFetches()); } @@ -3190,7 +3190,7 @@ public void testCorruptMessageError() { buildRecords(1L, 1, 1), Errors.CORRUPT_MESSAGE, 100L, 0)); - consumerClient.poll(time.timer(0)); + networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); // Trigger the exception. @@ -3398,7 +3398,7 @@ private void buildFetcher(MetricConfig metricConfig, subscriptionState, fetchConfig, metricsManager, - consumerClient, + networkClientDelegate, fetchCollector)); offsetFetcher = new OffsetFetcher(logContext, oldConsumerClient, @@ -3432,7 +3432,7 @@ private void buildDependencies(MetricConfig metricConfig, properties.setProperty(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG, String.valueOf(requestTimeoutMs)); properties.setProperty(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG, String.valueOf(retryBackoffMs)); ConsumerConfig config = new ConsumerConfig(properties); - consumerClient = spy(new TestableNetworkClientDelegate(time, config, logContext, client)); + networkClientDelegate = spy(new TestableNetworkClientDelegate(time, config, logContext, client)); } private List collectRecordOffsets(List> records) { @@ -3462,7 +3462,7 @@ private Fetch collectFetch() { private int sendFetches() { NetworkClientDelegate.PollResult pollResult = fetcher.poll(time.milliseconds()); - consumerClient.addAll(pollResult.unsentRequests); + networkClientDelegate.addAll(pollResult.unsentRequests); return pollResult.unsentRequests.size(); } From 76743041252ce523117d2d17ef9a13c159699783 Mon Sep 17 00:00:00 2001 From: Kirk True Date: Tue, 3 Oct 2023 15:08:09 -0700 Subject: [PATCH 19/72] Removed FetchRequestManagerTest's oldConsumerClient --- .../consumer/internals/FetchRequestManagerTest.java | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java index acbb5b3b08845..8fcef9b72c397 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java @@ -185,7 +185,6 @@ public class FetchRequestManagerTest { private MockClient client; private Metrics metrics; private ApiVersions apiVersions = new ApiVersions(); - private ConsumerNetworkClient oldConsumerClient; private TestableFetchRequestManager fetcher; private TestableNetworkClientDelegate networkClientDelegate; private OffsetFetcher offsetFetcher; @@ -3400,8 +3399,16 @@ private void buildFetcher(MetricConfig metricConfig, metricsManager, networkClientDelegate, fetchCollector)); + ConsumerNetworkClient consumerNetworkClient = new ConsumerNetworkClient( + logContext, + client, + metadata, + time, + retryBackoffMs, + (int) requestTimeoutMs, + Integer.MAX_VALUE); offsetFetcher = new OffsetFetcher(logContext, - oldConsumerClient, + consumerNetworkClient, metadata, subscriptions, time, @@ -3421,8 +3428,6 @@ private void buildDependencies(MetricConfig metricConfig, subscriptions, logContext, new ClusterResourceListeners()); client = new MockClient(time, metadata); metrics = new Metrics(metricConfig, time); - oldConsumerClient = spy(new ConsumerNetworkClient(logContext, client, metadata, time, - retryBackoffMs, (int) requestTimeoutMs, Integer.MAX_VALUE)); metricsRegistry = new FetchMetricsRegistry(metricConfig.tags().keySet(), "consumer" + groupId); metricsManager = new FetchMetricsManager(metrics, metricsRegistry); From 0cbb8d278c4db18e1a236f6c1e423d3de9493d61 Mon Sep 17 00:00:00 2001 From: Kirk True Date: Tue, 3 Oct 2023 15:09:32 -0700 Subject: [PATCH 20/72] Updated comment placement in old and new fetcher tests --- .../clients/consumer/internals/FetchRequestManagerTest.java | 2 +- .../apache/kafka/clients/consumer/internals/FetcherTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java index 8fcef9b72c397..28b31018b960f 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java @@ -402,10 +402,10 @@ public void testFetchWithTopicId() { assignFromUser(singleton(tp.topicPartition())); subscriptions.seek(tp.topicPartition(), 0); - // Fetch should use latest version assertEquals(1, sendFetches()); assertFalse(fetcher.hasCompletedFetches()); + // Fetch should use latest version client.prepareResponse( fetchRequestMatcher(ApiKeys.FETCH.latestVersion(), tp, 0, Optional.of(validLeaderEpoch)), fullFetchResponse(tp, records, Errors.NONE, 100L, 0) diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java index 7724226b17222..f3c14539ed15e 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java @@ -388,10 +388,10 @@ public void testFetchWithTopicId() { assignFromUser(singleton(tp.topicPartition())); subscriptions.seek(tp.topicPartition(), 0); - // Fetch should use latest version assertEquals(1, sendFetches()); assertFalse(fetcher.hasCompletedFetches()); + // Fetch should use latest version client.prepareResponse( fetchRequestMatcher(ApiKeys.FETCH.latestVersion(), tp, 0, Optional.of(validLeaderEpoch)), fullFetchResponse(tp, records, Errors.NONE, 100L, 0) From 9c7b4c4628cebffbc76fad885c63f76acc24e629 Mon Sep 17 00:00:00 2001 From: Kirk True Date: Tue, 3 Oct 2023 15:30:28 -0700 Subject: [PATCH 21/72] Added documentation to RequestManager methods and referenced from FetchRequestManager --- .../internals/FetchRequestManager.java | 6 ++++ .../consumer/internals/RequestManager.java | 28 +++++++++++++++++-- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchRequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchRequestManager.java index 6c95a2c4a0440..cf3d1b4d56c20 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchRequestManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchRequestManager.java @@ -87,6 +87,9 @@ public void requestFetch(CompletableFuture> future) { futures.add(future); } + /** + * @see RequestManager#poll(long) + */ @Override public PollResult poll(long currentTimeMs) { List requests = prepareFetchRequests().entrySet().stream().map(entry -> { @@ -110,6 +113,9 @@ public PollResult poll(long currentTimeMs) { return new PollResult(requests); } + /** + * @see RequestManager#pollOnClose() + */ @Override public PollResult pollOnClose() { List requests = prepareCloseFetchSessionRequests().entrySet().stream().map(entry -> { diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManager.java index 4ebc698c2ee30..80fbeaf3de6d2 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManager.java @@ -27,11 +27,35 @@ */ public interface RequestManager { + /** + * During normal operation of the {@link Consumer}, a request manager may need to send out network requests. + * Implementations can return {@link PollResult their need for network I/O} by returning the requests here. + * Because the {@code poll} method is called within the single-threaded context of the consumer's main network + * I/O thread, there should be no need for synchronization protection within itself or other state. + * + *

    + * + * Note: no network I/O occurs in this method. The method itself should not block on I/O or for any + * other reason. This method is called from by the consumer's main network I/O thread. So quick execution of + * this method in all request managers is critical to ensure that we can heartbeat in a timely fashion. + * + * @param currentTimeMs The current system time at which the method was called; useful for determining if + * time-sensitive operations should be performed + */ PollResult poll(long currentTimeMs); /** - * On shutdown of the {@link Consumer}, a request manager may want/need to send out some requests. If so, - * implementations can signal that by returning the close requests here. + * On shutdown of the {@link Consumer}, a request manager may need to send out network requests. Implementations + * can signal that by returning the {@link PollResult close} requests here. Unlike {@link #poll(long)}, the + * {@code pollOnClose} method is called from the application thread. Therefore, protection should be made + * when interacting with other any state that could be affected by other threads. + * + *

    + * + * Note: no network I/O occurs in this method. The method itself should not block on I/O or for any + * other reason. This method is called (indirectly) by the {@link Consumer#close() consumer's close method}. + * So quick execution of this method in all request managers is critical to ensure that we can + * complete as many of the shutdown tasks as possible given the user-provided timeout. */ default PollResult pollOnClose() { return EMPTY; From 65197a1c433bf80b6e6372e551d8d67c443e5071 Mon Sep 17 00:00:00 2001 From: Kirk True Date: Tue, 3 Oct 2023 15:33:41 -0700 Subject: [PATCH 22/72] Refactored AbstractFetch to make fetch request maps more succinct --- .../consumer/internals/AbstractFetch.java | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractFetch.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractFetch.java index 1a32b414639a4..c9657b1406803 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractFetch.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractFetch.java @@ -43,11 +43,11 @@ import java.time.Duration; import java.util.HashMap; import java.util.HashSet; -import java.util.LinkedHashMap; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.function.Predicate; +import java.util.stream.Collectors; import static org.apache.kafka.clients.consumer.internals.FetchUtils.requestMetadataUpdate; @@ -333,7 +333,7 @@ Node selectReadReplica(final TopicPartition partition, final Node leaderReplica, protected Map prepareCloseFetchSessionRequests() { final Cluster cluster = metadata.fetch(); - Map fetchable = new LinkedHashMap<>(); + Map fetchable = new HashMap<>(); try { sessionHandlers.forEach((fetchTargetNodeId, sessionHandler) -> { @@ -355,11 +355,7 @@ protected Map prepareCloseFetchSessi sessionHandlers.clear(); } - Map reqs = new LinkedHashMap<>(); - for (Map.Entry entry : fetchable.entrySet()) { - reqs.put(entry.getKey(), entry.getValue().build()); - } - return reqs; + return fetchable.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().build())); } /** @@ -370,7 +366,7 @@ protected Map prepareFetchRequests() // Update metrics in case there was an assignment change metricsManager.maybeUpdateAssignment(subscriptions); - Map fetchable = new LinkedHashMap<>(); + Map fetchable = new HashMap<>(); long currentTimeMs = time.milliseconds(); Map topicIds = metadata.topicIds(); @@ -419,11 +415,7 @@ protected Map prepareFetchRequests() } } - Map reqs = new LinkedHashMap<>(); - for (Map.Entry entry : fetchable.entrySet()) { - reqs.put(entry.getKey(), entry.getValue().build()); - } - return reqs; + return fetchable.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().build())); } // Visible for testing From 5387b7f7389f7b5e2f04da594fffb904f18b2ec8 Mon Sep 17 00:00:00 2001 From: Kirk True Date: Tue, 3 Oct 2023 17:08:43 -0700 Subject: [PATCH 23/72] Removed unnecessary "this" references. --- .../internals/PrototypeAsyncConsumer.java | 52 +++++++++---------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java index eecb52f4eeae9..c881a253c162e 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java @@ -325,9 +325,9 @@ public ConsumerRecords poll(final Duration timeout) { Timer timer = time.timer(timeout); try { - this.kafkaConsumerMetrics.recordPollStart(timer.currentTimeMs()); + kafkaConsumerMetrics.recordPollStart(timer.currentTimeMs()); - if (this.subscriptions.hasNoSubscriptionOrUserAssignment()) { + if (subscriptions.hasNoSubscriptionOrUserAssignment()) { throw new IllegalStateException("Consumer is not subscribed to any topics or assigned any partitions"); } @@ -343,14 +343,14 @@ public ConsumerRecords poll(final Duration timeout) { + "since the consumer's position has advanced for at least one topic partition"); } - return this.interceptors.onConsume(new ConsumerRecords<>(fetch.records())); + return interceptors.onConsume(new ConsumerRecords<>(fetch.records())); } // We will wait for retryBackoffMs } while (timer.notExpired()); return ConsumerRecords.empty(); } finally { - this.kafkaConsumerMetrics.recordPollEnd(timer.currentTimeMs()); + kafkaConsumerMetrics.recordPollEnd(timer.currentTimeMs()); } // TODO: Once we implement poll(), clear wakeupTrigger in a finally block: wakeupTrigger.clearActiveTask(); } @@ -413,8 +413,8 @@ public void seek(TopicPartition partition, long offset) { SubscriptionState.FetchPosition newPosition = new SubscriptionState.FetchPosition( offset, Optional.empty(), // This will ensure we skip validation - this.metadata.currentLeader(partition)); - this.subscriptions.seekUnvalidated(partition, newPosition); + metadata.currentLeader(partition)); + subscriptions.seekUnvalidated(partition, newPosition); } @Override @@ -430,13 +430,13 @@ public void seek(TopicPartition partition, OffsetAndMetadata offsetAndMetadata) } else { log.info("Seeking to offset {} for partition {}", offset, partition); } - Metadata.LeaderAndEpoch currentLeaderAndEpoch = this.metadata.currentLeader(partition); + Metadata.LeaderAndEpoch currentLeaderAndEpoch = metadata.currentLeader(partition); SubscriptionState.FetchPosition newPosition = new SubscriptionState.FetchPosition( offsetAndMetadata.offset(), offsetAndMetadata.leaderEpoch(), currentLeaderAndEpoch); - this.updateLastSeenEpochIfNewer(partition, offsetAndMetadata); - this.subscriptions.seekUnvalidated(partition, newPosition); + updateLastSeenEpochIfNewer(partition, offsetAndMetadata); + subscriptions.seekUnvalidated(partition, newPosition); } @Override @@ -444,7 +444,7 @@ public void seekToBeginning(Collection partitions) { if (partitions == null) throw new IllegalArgumentException("Partitions collection cannot be null"); - Collection parts = partitions.isEmpty() ? this.subscriptions.assignedPartitions() : partitions; + Collection parts = partitions.isEmpty() ? subscriptions.assignedPartitions() : partitions; subscriptions.requestOffsetReset(parts, OffsetResetStrategy.EARLIEST); } @@ -453,7 +453,7 @@ public void seekToEnd(Collection partitions) { if (partitions == null) throw new IllegalArgumentException("Partitions collection cannot be null"); - Collection parts = partitions.isEmpty() ? this.subscriptions.assignedPartitions() : partitions; + Collection parts = partitions.isEmpty() ? subscriptions.assignedPartitions() : partitions; subscriptions.requestOffsetReset(parts, OffsetResetStrategy.LATEST); } @@ -464,12 +464,12 @@ public long position(TopicPartition partition) { @Override public long position(TopicPartition partition, Duration timeout) { - if (!this.subscriptions.isAssigned(partition)) + if (!subscriptions.isAssigned(partition)) throw new IllegalStateException("You can only check the position for partitions assigned to this consumer."); Timer timer = time.timer(timeout); do { - SubscriptionState.FetchPosition position = this.subscriptions.validPosition(partition); + SubscriptionState.FetchPosition position = subscriptions.validPosition(partition); if (position != null) return position.offset; @@ -523,7 +523,7 @@ private void maybeThrowInvalidGroupIdException() { @Override public Map metrics() { - return Collections.unmodifiableMap(this.metrics.metrics()); + return Collections.unmodifiableMap(metrics.metrics()); } @Override @@ -706,7 +706,7 @@ private void close(Duration timeout, boolean swallowException) { if (applicationEventHandler != null) closeQuietly(() -> applicationEventHandler.close(timeout), "Failed to close application event handler with a timeout(ms)=" + timeout, firstException); - closeQuietly(fetchBuffer, "Failed to close fetcher", firstException); + closeQuietly(fetchBuffer, "Failed to close the fetch buffer", firstException); closeQuietly(interceptors, "consumer interceptors", firstException); closeQuietly(kafkaConsumerMetrics, "kafka consumer metrics", firstException); closeQuietly(metrics, "consumer metrics", firstException); @@ -759,7 +759,7 @@ public void commitSync(Map offsets, Duration @Override public Set assignment() { - return Collections.unmodifiableSet(this.subscriptions.assignedPartitions()); + return Collections.unmodifiableSet(subscriptions.assignedPartitions()); } /** @@ -769,7 +769,7 @@ public Set assignment() { */ @Override public Set subscription() { - return Collections.unmodifiableSet(this.subscriptions.subscription()); + return Collections.unmodifiableSet(subscriptions.subscription()); } @Override @@ -803,7 +803,7 @@ public void subscribe(Collection topics, ConsumerRebalanceListener callb fetchBuffer.retainAll(currentTopicPartitions); log.info("Subscribed to topic(s): {}", join(topics, ", ")); - if (this.subscriptions.subscribe(new HashSet<>(topics), callback)) + if (subscriptions.subscribe(new HashSet<>(topics), callback)) metadata.requestUpdateForNewTopics(); } } @@ -815,7 +815,7 @@ public void assign(Collection partitions) { } if (partitions.isEmpty()) { - this.unsubscribe(); + unsubscribe(); return; } @@ -837,10 +837,10 @@ public void assign(Collection partitions) { // make sure the offsets of topic partitions the consumer is unsubscribing from // are committed since there will be no following rebalance - applicationEventHandler.add(new AssignmentChangeApplicationEvent(this.subscriptions.allConsumed(), time.milliseconds())); + applicationEventHandler.add(new AssignmentChangeApplicationEvent(subscriptions.allConsumed(), time.milliseconds())); log.info("Assigned to partition(s): {}", join(partitions, ", ")); - if (this.subscriptions.assignFromUser(new HashSet<>(partitions))) + if (subscriptions.assignFromUser(new HashSet<>(partitions))) applicationEventHandler.add(new NewTopicsMetadataUpdateRequestEvent()); } @@ -853,9 +853,9 @@ public void subscribe(Pattern pattern, ConsumerRebalanceListener listener) { throwIfNoAssignorsConfigured(); log.info("Subscribed to pattern: '{}'", pattern); - this.subscriptions.subscribe(pattern, listener); - this.updatePatternSubscription(metadata.fetch()); - this.metadata.requestUpdateForNewTopics(); + subscriptions.subscribe(pattern, listener); + updatePatternSubscription(metadata.fetch()); + metadata.requestUpdateForNewTopics(); } /** @@ -884,7 +884,7 @@ public void subscribe(Pattern pattern) { @Override public void unsubscribe() { fetchBuffer.retainAll(Collections.emptySet()); - this.subscriptions.unsubscribe(); + subscriptions.unsubscribe(); } @Override @@ -1016,7 +1016,7 @@ private boolean refreshCommittedOffsetsIfNeeded(Timer timer) { log.debug("Refreshing committed offsets for partitions {}", initializingPartitions); try { final Map offsets = applicationEventHandler.addAndGet(new OffsetFetchApplicationEvent(initializingPartitions), timer); - return ConsumerUtils.refreshCommittedOffsets(offsets, this.metadata, this.subscriptions); + return ConsumerUtils.refreshCommittedOffsets(offsets, metadata, subscriptions); } catch (org.apache.kafka.common.errors.TimeoutException e) { log.error("Couldn't refresh committed offsets before timeout expired"); return false; From 8490681c50b993d13302e3bfda7e1438ebb277b8 Mon Sep 17 00:00:00 2001 From: Kirk True Date: Tue, 3 Oct 2023 17:11:25 -0700 Subject: [PATCH 24/72] Minor formatting change in beginningOrEndOffset --- .../consumer/internals/PrototypeAsyncConsumer.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java index c881a253c162e..6fe5066676219 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java @@ -626,14 +626,12 @@ private Map beginningOrEndOffset(Collection timestampToSearch = partitions .stream() .collect(Collectors.toMap(Function.identity(), tp -> timestamp)); - final ListOffsetsApplicationEvent listOffsetsEvent = new ListOffsetsApplicationEvent( + ListOffsetsApplicationEvent listOffsetsEvent = new ListOffsetsApplicationEvent( timestampToSearch, - false - ); + false); Map offsetAndTimestampMap = applicationEventHandler.addAndGet( listOffsetsEvent, - time.timer(timeout) - ); + time.timer(timeout)); return offsetAndTimestampMap .entrySet() .stream() From 4e463d2076f127a6621b381f798ea0f12fd6ddd3 Mon Sep 17 00:00:00 2001 From: Kirk True Date: Wed, 4 Oct 2023 10:29:53 -0700 Subject: [PATCH 25/72] Revised usage of Timer in DefaultBackgroundThread.runAtClose() --- .../consumer/internals/DefaultBackgroundThread.java | 11 ++++------- .../consumer/internals/FetchRequestManagerTest.java | 5 +---- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThread.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThread.java index 66f1d9dd8ce5d..f4efca20317d8 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThread.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThread.java @@ -168,12 +168,9 @@ void runOnce() { * */ // Visible for testing - static void runAtClose(final Time time, - final Collection> requestManagers, + static void runAtClose(final Collection> requestManagers, final NetworkClientDelegate networkClientDelegate, final Timer timer) { - long currentTimeMs = time.milliseconds(); - // These are the optional outgoing requests at the List pollResults = requestManagers.stream() .filter(Optional::isPresent) @@ -184,7 +181,7 @@ static void runAtClose(final Time time, .map(networkClientDelegate::addAll) .reduce(MAX_POLL_TIMEOUT_MS, Math::min); pollWaitTimeMs = Math.min(pollWaitTimeMs, timer.remainingMs()); - networkClientDelegate.poll(pollWaitTimeMs, currentTimeMs); + networkClientDelegate.poll(pollWaitTimeMs, timer.currentTimeMs()); timer.update(); List> requestFutures = pollResults.stream() @@ -195,7 +192,7 @@ static void runAtClose(final Time time, // Poll to ensure that request has been written to the socket. Wait until either the timer has expired or until // all requests have received a response. while (timer.notExpired() && !requestFutures.stream().allMatch(Future::isDone)) { - networkClientDelegate.poll(timer.remainingMs(), currentTimeMs); + networkClientDelegate.poll(timer.remainingMs(), timer.currentTimeMs()); timer.update(); } } @@ -247,7 +244,7 @@ void closeInternal(final Duration timeout) { } } - runAtClose(time, requestManagers.entries(), networkClientDelegate, timer); + runAtClose(requestManagers.entries(), networkClientDelegate, timer); closeQuietly(requestManagers, "request managers"); closeQuietly(networkClientDelegate, "network client delegate"); closeQuietly(applicationEventProcessor, "application event processor"); diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java index 28b31018b960f..5b7b571242396 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java @@ -316,10 +316,7 @@ public void testFetcherCloseClosesFetchSessionsInBroker() { // NOTE: by design the FetchRequestManager doesn't perform network I/O internally. That means that calling // close with a Timer will NOT send out the close session requests on close. The network I/O logic is // handled inside DefaultBackgroundThread.runAtClose, so we need to run that logic here. - DefaultBackgroundThread.runAtClose(time, - Collections.singletonList(Optional.of(fetcher)), - networkClientDelegate, - timer); + DefaultBackgroundThread.runAtClose(singletonList(Optional.of(fetcher)), networkClientDelegate, timer); NetworkClientDelegate.PollResult pollResult = fetcher.poll(time.milliseconds()); networkClientDelegate.addAll(pollResult.unsentRequests); From d3e920d10ae20bd8c0f228cbf7cbdd8ccf1d5a20 Mon Sep 17 00:00:00 2001 From: Kirk True Date: Wed, 4 Oct 2023 10:54:33 -0700 Subject: [PATCH 26/72] Got fed up and changed DefaultBackgroundThread to ConsumerNetworkThread --- ...Thread.java => ConsumerNetworkThread.java} | 39 +++++++------- .../internals/FetchRequestManager.java | 5 +- .../internals/NetworkClientDelegate.java | 2 +- .../internals/OffsetsRequestManager.java | 2 +- .../internals/PrototypeAsyncConsumer.java | 18 ++++--- .../consumer/internals/RequestManagers.java | 2 +- .../events/ApplicationEventHandler.java | 16 +++--- .../events/ApplicationEventProcessor.java | 6 +-- .../internals/events/BackgroundEvent.java | 4 +- .../events/BackgroundEventHandler.java | 6 +-- .../events/BackgroundEventProcessor.java | 8 +-- .../consumer/internals/events/FetchEvent.java | 3 +- ...st.java => ConsumerNetworkThreadTest.java} | 52 +++++++++---------- .../internals/ConsumerTestBuilder.java | 10 ++-- .../internals/FetchRequestManagerTest.java | 4 +- 15 files changed, 90 insertions(+), 87 deletions(-) rename clients/src/main/java/org/apache/kafka/clients/consumer/internals/{DefaultBackgroundThread.java => ConsumerNetworkThread.java} (86%) rename clients/src/test/java/org/apache/kafka/clients/consumer/internals/{DefaultBackgroundThreadTest.java => ConsumerNetworkThreadTest.java} (88%) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThread.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThread.java similarity index 86% rename from clients/src/main/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThread.java rename to clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThread.java index f4efca20317d8..d95b80b25a798 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThread.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThread.java @@ -19,6 +19,7 @@ import org.apache.kafka.clients.KafkaClient; import org.apache.kafka.clients.consumer.internals.events.ApplicationEvent; import org.apache.kafka.clients.consumer.internals.events.ApplicationEventProcessor; +import org.apache.kafka.clients.consumer.internals.events.BackgroundEvent; import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.errors.InterruptException; import org.apache.kafka.common.errors.WakeupException; @@ -42,14 +43,10 @@ import static org.apache.kafka.common.utils.Utils.closeQuietly; /** - * Background thread runnable that consumes {@code ApplicationEvent} and - * produces {@code BackgroundEvent}. It uses an event loop to consume and - * produce events, and poll the network client to handle network IO. - *

    - * It holds a reference to the {@link SubscriptionState}, which is - * initialized by the application thread. + * Background thread runnable that consumes {@link ApplicationEvent} and produces {@link BackgroundEvent}. It + * uses an event loop to consume and produce events, and poll the network client to handle network IO. */ -public class DefaultBackgroundThread extends KafkaThread implements Closeable { +public class ConsumerNetworkThread extends KafkaThread implements Closeable { private static final long MAX_POLL_TIMEOUT_MS = 5000; private static final String BACKGROUND_THREAD_NAME = "consumer_background_thread"; @@ -64,11 +61,11 @@ public class DefaultBackgroundThread extends KafkaThread implements Closeable { private volatile boolean running; private final IdempotentCloser closer = new IdempotentCloser(); - public DefaultBackgroundThread(LogContext logContext, - Time time, - Supplier applicationEventProcessorSupplier, - Supplier networkClientDelegateSupplier, - Supplier requestManagersSupplier) { + public ConsumerNetworkThread(LogContext logContext, + Time time, + Supplier applicationEventProcessorSupplier, + Supplier networkClientDelegateSupplier, + Supplier requestManagersSupplier) { super(BACKGROUND_THREAD_NAME, true); this.time = time; this.log = logContext.logger(getClass()); @@ -79,25 +76,25 @@ public DefaultBackgroundThread(LogContext logContext, @Override public void run() { - closer.assertOpen("Consumer background thread is already closed"); + closer.assertOpen("Consumer network thread is already closed"); running = true; try { - log.debug("Background thread started"); + log.debug("Consumer network thread started"); - // Wait until we're securely in the background thread to initialize these objects... + // Wait until we're securely in the background network thread to initialize these objects... initializeResources(); while (running) { try { runOnce(); } catch (final WakeupException e) { - log.debug("WakeupException caught, background thread won't be interrupted"); - // swallow the wakeup exception to prevent killing the background thread. + log.debug("WakeupException caught, consumer network thread won't be interrupted"); + // swallow the wakeup exception to prevent killing the thread. } } } catch (final Throwable t) { - log.error("The background thread failed due to unexpected error", t); + log.error("The consumer network thread failed due to unexpected error", t); throw new KafkaException(t); } } @@ -215,12 +212,12 @@ public void close() { public void close(final Duration timeout) { closer.close( () -> closeInternal(timeout), - () -> log.warn("The consumer background thread was already closed") + () -> log.warn("The consumer network thread was already closed") ); } void closeInternal(final Duration timeout) { - log.trace("Closing the consumer background thread"); + log.trace("Closing the consumer network thread"); boolean hadStarted = running; running = false; wakeup(); @@ -248,6 +245,6 @@ void closeInternal(final Duration timeout) { closeQuietly(requestManagers, "request managers"); closeQuietly(networkClientDelegate, "network client delegate"); closeQuietly(applicationEventProcessor, "application event processor"); - log.debug("Closed the consumer background thread"); + log.debug("Closed the consumer network thread"); } } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchRequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchRequestManager.java index cf3d1b4d56c20..42d5bad29c2b9 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchRequestManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchRequestManager.java @@ -31,6 +31,7 @@ import org.apache.kafka.clients.NetworkClient; import org.apache.kafka.clients.consumer.internals.NetworkClientDelegate.PollResult; import org.apache.kafka.clients.consumer.internals.NetworkClientDelegate.UnsentRequest; +import org.apache.kafka.clients.consumer.internals.events.ApplicationEventProcessor; import org.apache.kafka.clients.consumer.internals.events.BackgroundEventHandler; import org.apache.kafka.clients.consumer.internals.events.ErrorBackgroundEvent; import org.apache.kafka.common.Node; @@ -173,8 +174,8 @@ private void forwardResults() { * *

    * - * This is used by the {@link org.apache.kafka.clients.consumer.internals.events.ApplicationEventProcessor} to - * pull off any fetch results that are stored in the background thread to provide them to the application thread. + * This is used by the {@link ApplicationEventProcessor} to pull off any fetch results that are stored in + * the {@link ConsumerNetworkThread network thread} to provide them to the application thread. * * @return {@link Queue} containing zero or more {@link CompletedFetch} */ diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/NetworkClientDelegate.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/NetworkClientDelegate.java index 2e3d8ba1895f1..e2aa915612316 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/NetworkClientDelegate.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/NetworkClientDelegate.java @@ -348,7 +348,7 @@ public void onComplete(final ClientResponse response) { /** * Creates a {@link Supplier} for deferred creation during invocation by - * {@link org.apache.kafka.clients.consumer.internals.DefaultBackgroundThread}. + * {@link ConsumerNetworkThread}. */ public static Supplier supplier(final Time time, final LogContext logContext, diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/OffsetsRequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/OffsetsRequestManager.java index 6e27c71fb6929..e415cc122ae42 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/OffsetsRequestManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/OffsetsRequestManager.java @@ -116,7 +116,7 @@ public OffsetsRequestManager(final SubscriptionState subscriptionState, time, retryBackoffMs, apiVersions); // Register the cluster metadata update callback. Note this only relies on the // requestsToRetry initialized above, and won't be invoked until all managers are - // initialized and the background thread started. + // initialized and the network thread started. this.metadata.addClusterUpdateListener(this); } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java index 6fe5066676219..dc8b2047cc8ea 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java @@ -107,9 +107,10 @@ import static org.apache.kafka.common.utils.Utils.propsToMap; /** - * This prototype consumer uses the EventHandler to process application - * events so that the network IO can be processed in a background thread. Visit - * this document + * This prototype consumer uses an {@link ApplicationEventHandler event handler} to process + * {@link ApplicationEvent application events} so that the network IO can be processed in a dedicated + * {@link ConsumerNetworkThread network thread}. Visit + * this document * for detail implementation. */ public class PrototypeAsyncConsumer implements Consumer { @@ -141,9 +142,10 @@ public class PrototypeAsyncConsumer implements Consumer { private final WakeupTrigger wakeupTrigger = new WakeupTrigger(); /** - * A thread-safe {@link BlockingQueue queue} for the results that are populated in the background thread - * when the fetch results are available. Because the {@link #fetchBuffer fetch buffer} is not thread-safe, we - * need to separate the results collection that we provide to the background thread from the collection that + * A thread-safe {@link BlockingQueue queue} for the results that are populated in the + * {@link ConsumerNetworkThread network thread} when the fetch results are available. Because + * the {@link #fetchBuffer fetch buffer} is not thread-safe, we need to separate the results + * collection that we provide to the network thread from the collection that * we read from on the application thread. */ private final BlockingQueue fetchResults = new LinkedBlockingQueue<>(); @@ -897,8 +899,8 @@ WakeupTrigger wakeupTrigger() { } /** - * Send the requests for fetch data to the background thread and set up to collect the results in - * {@link #fetchResults}. + * Send the requests for fetch data to the {@link ConsumerNetworkThread network thread} and set up to + * collect the results in {@link #fetchResults}. */ private void sendFetches() { FetchEvent event = new FetchEvent(); diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java index 923d71e44d559..384f2515b505b 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java @@ -99,7 +99,7 @@ public void close() { /** * Creates a {@link Supplier} for deferred creation during invocation by - * {@link org.apache.kafka.clients.consumer.internals.DefaultBackgroundThread}. + * {@link ConsumerNetworkThread}. */ public static Supplier supplier(final Time time, final LogContext logContext, diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventHandler.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventHandler.java index b6c0cfa92bbb5..042d0af9c9e30 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventHandler.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventHandler.java @@ -16,7 +16,7 @@ */ package org.apache.kafka.clients.consumer.internals.events; -import org.apache.kafka.clients.consumer.internals.DefaultBackgroundThread; +import org.apache.kafka.clients.consumer.internals.ConsumerNetworkThread; import org.apache.kafka.clients.consumer.internals.NetworkClientDelegate; import org.apache.kafka.clients.consumer.internals.RequestManagers; import org.apache.kafka.common.internals.IdempotentCloser; @@ -36,13 +36,13 @@ /** * An event handler that receives {@link ApplicationEvent application events} from the application thread which - * are then readable from the {@link ApplicationEventProcessor} in the background thread. + * are then readable from the {@link ApplicationEventProcessor} in the {@link ConsumerNetworkThread network thread}. */ public class ApplicationEventHandler implements Closeable { private final Logger log; private final BlockingQueue applicationEventQueue; - private final DefaultBackgroundThread backgroundThread; + private final ConsumerNetworkThread networkThread; private final IdempotentCloser closer = new IdempotentCloser(); public ApplicationEventHandler(final LogContext logContext, @@ -53,12 +53,12 @@ public ApplicationEventHandler(final LogContext logContext, final Supplier requestManagersSupplier) { this.log = logContext.logger(ApplicationEventHandler.class); this.applicationEventQueue = applicationEventQueue; - this.backgroundThread = new DefaultBackgroundThread(logContext, + this.networkThread = new ConsumerNetworkThread(logContext, time, applicationEventProcessorSupplier, networkClientDelegateSupplier, requestManagersSupplier); - this.backgroundThread.start(); + this.networkThread.start(); } /** @@ -69,7 +69,7 @@ public ApplicationEventHandler(final LogContext logContext, public void add(final ApplicationEvent event) { Objects.requireNonNull(event, "ApplicationEvent provided to add must be non-null"); log.trace("Enqueued event: {}", event); - backgroundThread.wakeup(); + networkThread.wakeup(); applicationEventQueue.add(event); } @@ -81,7 +81,7 @@ public void add(final ApplicationEvent event) { * * See {@link CompletableApplicationEvent#get(Timer)} and {@link Future#get(long, TimeUnit)} for more details. * - * @param event A {@link CompletableApplicationEvent} created by the polling thread. + * @param event A {@link CompletableApplicationEvent} created by the polling thread * @param timer Timer for which to wait for the event to complete * @return Value that is the result of the event * @param Type of return value of the event @@ -100,7 +100,7 @@ public void close() { public void close(final Duration timeout) { closer.close( - () -> Utils.closeQuietly(() -> backgroundThread.close(timeout), "application event handler"), + () -> Utils.closeQuietly(() -> networkThread.close(timeout), "consumer network thread"), () -> log.warn("The application event handler was already closed") ); } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventProcessor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventProcessor.java index 64bfa45202431..373ba631fd999 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventProcessor.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventProcessor.java @@ -20,7 +20,7 @@ import org.apache.kafka.clients.consumer.internals.CachedSupplier; import org.apache.kafka.clients.consumer.internals.CommitRequestManager; import org.apache.kafka.clients.consumer.internals.ConsumerMetadata; -import org.apache.kafka.clients.consumer.internals.DefaultBackgroundThread; +import org.apache.kafka.clients.consumer.internals.ConsumerNetworkThread; import org.apache.kafka.clients.consumer.internals.RequestManagers; import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.PartitionInfo; @@ -35,7 +35,7 @@ import java.util.function.Supplier; /** - * An {@link EventProcessor} that is created and executes in {@link DefaultBackgroundThread the background thread} + * An {@link EventProcessor} that is created and executes in the {@link ConsumerNetworkThread network thread} * which processes {@link ApplicationEvent application events} generated by the application thread. */ public class ApplicationEventProcessor extends EventProcessor { @@ -189,7 +189,7 @@ private void process(final FetchEvent event) { /** * Creates a {@link Supplier} for deferred creation during invocation by - * {@link org.apache.kafka.clients.consumer.internals.DefaultBackgroundThread}. + * {@link ConsumerNetworkThread}. */ public static Supplier supplier(final LogContext logContext, final ConsumerMetadata metadata, diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEvent.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEvent.java index 91b0fc0a8ddbf..a7dc3e454a776 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEvent.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEvent.java @@ -16,10 +16,12 @@ */ package org.apache.kafka.clients.consumer.internals.events; +import org.apache.kafka.clients.consumer.internals.ConsumerNetworkThread; + import java.util.Objects; /** - * This is the abstract definition of the events created by the background thread. + * This is the abstract definition of the events created by the {@link ConsumerNetworkThread network thread}. */ public abstract class BackgroundEvent { diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEventHandler.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEventHandler.java index 514a57770c00a..cafa426d033aa 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEventHandler.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEventHandler.java @@ -16,7 +16,7 @@ */ package org.apache.kafka.clients.consumer.internals.events; -import org.apache.kafka.clients.consumer.internals.DefaultBackgroundThread; +import org.apache.kafka.clients.consumer.internals.ConsumerNetworkThread; import org.apache.kafka.common.utils.LogContext; import org.slf4j.Logger; @@ -25,7 +25,7 @@ /** * An event handler that receives {@link BackgroundEvent background events} from the - * {@link DefaultBackgroundThread background thread} which are then made available to the application thread + * {@link ConsumerNetworkThread network thread} which are then made available to the application thread * via the {@link BackgroundEventProcessor}. */ @@ -42,7 +42,7 @@ public BackgroundEventHandler(final LogContext logContext, final Queue - *

  • Errors that occur in the background thread that need to be propagated to the application thread
    • + *
  • Errors that occur in the network thread that need to be propagated to the application thread
    • *
  • {@link ConsumerRebalanceListener} callbacks that are to be executed on the application thread
    • * */ @@ -42,7 +42,7 @@ public BackgroundEventProcessor(final LogContext logContext, } /** - * Process the events—if any—that were produced by the {@link DefaultBackgroundThread background thread}. + * Process the events—if any—that were produced by the {@link ConsumerNetworkThread network thread}. * It is possible that when processing the events that a given event will * {@link ErrorBackgroundEvent represent an error directly}, or it could be that processing an event generates * an error. In such cases, the processor will continue to process the remaining events. In this case, we diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/FetchEvent.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/FetchEvent.java index ec68a3e4cda44..79628da140dd6 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/FetchEvent.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/FetchEvent.java @@ -17,11 +17,12 @@ package org.apache.kafka.clients.consumer.internals.events; import org.apache.kafka.clients.consumer.internals.CompletedFetch; +import org.apache.kafka.clients.consumer.internals.ConsumerNetworkThread; import java.util.Queue; /** - * This event signals the background thread to submit a fetch request. + * This event signals the {@link ConsumerNetworkThread network thread} to submit a fetch request. */ public class FetchEvent extends CompletableApplicationEvent> { diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThreadTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThreadTest.java similarity index 88% rename from clients/src/test/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThreadTest.java rename to clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThreadTest.java index 2e377ac69098c..f38090c8b94cc 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/DefaultBackgroundThreadTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThreadTest.java @@ -65,9 +65,9 @@ import static org.mockito.Mockito.when; @SuppressWarnings("ClassDataAbstractionCoupling") -public class DefaultBackgroundThreadTest { +public class ConsumerNetworkThreadTest { - private ConsumerTestBuilder.DefaultBackgroundThreadTestBuilder testBuilder; + private ConsumerTestBuilder.ConsumerNetworkThreadTestBuilder testBuilder; private Time time; private ConsumerMetadata metadata; private NetworkClientDelegate networkClient; @@ -76,12 +76,12 @@ public class DefaultBackgroundThreadTest { private CoordinatorRequestManager coordinatorManager; private OffsetsRequestManager offsetsRequestManager; private CommitRequestManager commitManager; - private DefaultBackgroundThread backgroundThread; + private ConsumerNetworkThread consumerNetworkThread; private MockClient client; @BeforeEach public void setup() { - testBuilder = new ConsumerTestBuilder.DefaultBackgroundThreadTestBuilder(); + testBuilder = new ConsumerTestBuilder.ConsumerNetworkThreadTestBuilder(); time = testBuilder.time; metadata = testBuilder.metadata; networkClient = testBuilder.networkClientDelegate; @@ -91,8 +91,8 @@ public void setup() { coordinatorManager = testBuilder.coordinatorRequestManager; commitManager = testBuilder.commitRequestManager; offsetsRequestManager = testBuilder.offsetsRequestManager; - backgroundThread = testBuilder.backgroundThread; - backgroundThread.initializeResources(); + consumerNetworkThread = testBuilder.consumerNetworkThread; + consumerNetworkThread.initializeResources(); } @AfterEach @@ -103,17 +103,17 @@ public void tearDown() { @Test public void testStartupAndTearDown() throws InterruptedException { - backgroundThread.start(); - TestUtils.waitForCondition(backgroundThread::isRunning, "Failed awaiting for the background thread to be running"); + consumerNetworkThread.start(); + TestUtils.waitForCondition(consumerNetworkThread::isRunning, "Failed awaiting for the background thread to be running"); // There's a nonzero amount of time between starting the thread and having it // begin to execute our code. Wait for a bit before checking... int maxWaitMs = 1000; - TestUtils.waitForCondition(backgroundThread::isRunning, + TestUtils.waitForCondition(consumerNetworkThread::isRunning, maxWaitMs, "Thread did not start within " + maxWaitMs + " ms"); - backgroundThread.close(Duration.ofMillis(maxWaitMs)); - TestUtils.waitForCondition(() -> !backgroundThread.isRunning(), + consumerNetworkThread.close(Duration.ofMillis(maxWaitMs)); + TestUtils.waitForCondition(() -> !consumerNetworkThread.isRunning(), maxWaitMs, "Thread did not stop within " + maxWaitMs + " ms"); } @@ -122,7 +122,7 @@ public void testStartupAndTearDown() throws InterruptedException { public void testApplicationEvent() { FetchEvent e = new FetchEvent(); applicationEventsQueue.add(e); - backgroundThread.runOnce(); + consumerNetworkThread.runOnce(); verify(applicationEventProcessor, times(1)).process(e); } @@ -130,7 +130,7 @@ public void testApplicationEvent() { public void testMetadataUpdateEvent() { ApplicationEvent e = new NewTopicsMetadataUpdateRequestEvent(); applicationEventsQueue.add(e); - backgroundThread.runOnce(); + consumerNetworkThread.runOnce(); verify(metadata).requestUpdateForNewTopics(); } @@ -138,7 +138,7 @@ public void testMetadataUpdateEvent() { public void testCommitEvent() { ApplicationEvent e = new CommitApplicationEvent(new HashMap<>()); applicationEventsQueue.add(e); - backgroundThread.runOnce(); + consumerNetworkThread.runOnce(); verify(applicationEventProcessor).process(any(CommitApplicationEvent.class)); } @@ -147,7 +147,7 @@ public void testListOffsetsEventIsProcessed() { Map timestamps = Collections.singletonMap(new TopicPartition("topic1", 1), 5L); ApplicationEvent e = new ListOffsetsApplicationEvent(timestamps, true); applicationEventsQueue.add(e); - backgroundThread.runOnce(); + consumerNetworkThread.runOnce(); verify(applicationEventProcessor).process(any(ListOffsetsApplicationEvent.class)); assertTrue(applicationEventsQueue.isEmpty()); } @@ -156,19 +156,19 @@ public void testListOffsetsEventIsProcessed() { public void testResetPositionsEventIsProcessed() { ResetPositionsApplicationEvent e = new ResetPositionsApplicationEvent(); applicationEventsQueue.add(e); - backgroundThread.runOnce(); + consumerNetworkThread.runOnce(); verify(applicationEventProcessor).process(any(ResetPositionsApplicationEvent.class)); assertTrue(applicationEventsQueue.isEmpty()); } @Test - public void testResetPositionsProcessFailureInterruptsBackgroundThread() { + public void testResetPositionsProcessFailureInterruptsConsumerNetworkThread() { TopicAuthorizationException authException = new TopicAuthorizationException("Topic authorization failed"); doThrow(authException).when(offsetsRequestManager).resetPositionsIfNeeded(); ResetPositionsApplicationEvent event = new ResetPositionsApplicationEvent(); applicationEventsQueue.add(event); - assertThrows(TopicAuthorizationException.class, () -> backgroundThread.runOnce()); + assertThrows(TopicAuthorizationException.class, () -> consumerNetworkThread.runOnce()); verify(applicationEventProcessor).process(any(ResetPositionsApplicationEvent.class)); } @@ -177,7 +177,7 @@ public void testResetPositionsProcessFailureInterruptsBackgroundThread() { public void testValidatePositionsEventIsProcessed() { ValidatePositionsApplicationEvent e = new ValidatePositionsApplicationEvent(); applicationEventsQueue.add(e); - backgroundThread.runOnce(); + consumerNetworkThread.runOnce(); verify(applicationEventProcessor).process(any(ValidatePositionsApplicationEvent.class)); assertTrue(applicationEventsQueue.isEmpty()); } @@ -189,7 +189,7 @@ public void testValidatePositionsProcessFailure() { ValidatePositionsApplicationEvent event = new ValidatePositionsApplicationEvent(); applicationEventsQueue.add(event); - assertThrows(LogTruncationException.class, backgroundThread::runOnce); + assertThrows(LogTruncationException.class, consumerNetworkThread::runOnce); verify(applicationEventProcessor).process(any(ValidatePositionsApplicationEvent.class)); } @@ -202,7 +202,7 @@ public void testAssignmentChangeEvent() { ApplicationEvent e = new AssignmentChangeApplicationEvent(offset, currentTimeMs); applicationEventsQueue.add(e); - backgroundThread.runOnce(); + consumerNetworkThread.runOnce(); verify(applicationEventProcessor).process(any(AssignmentChangeApplicationEvent.class)); verify(networkClient, times(1)).poll(anyLong(), anyLong()); verify(commitManager, times(1)).updateAutoCommitTimer(currentTimeMs); @@ -211,7 +211,7 @@ public void testAssignmentChangeEvent() { @Test void testFindCoordinator() { - backgroundThread.runOnce(); + consumerNetworkThread.runOnce(); verify(coordinatorManager, times(1)).poll(anyLong()); verify(networkClient, times(1)).poll(anyLong(), anyLong()); } @@ -219,7 +219,7 @@ void testFindCoordinator() { @Test void testFetchTopicMetadata() { applicationEventsQueue.add(new TopicMetadataApplicationEvent("topic")); - backgroundThread.runOnce(); + consumerNetworkThread.runOnce(); verify(applicationEventProcessor).process(any(TopicMetadataApplicationEvent.class)); } @@ -247,7 +247,7 @@ void testPollResultTimer() { @Test void testRequestManagersArePolledOnce() { - backgroundThread.runOnce(); + consumerNetworkThread.runOnce(); testBuilder.requestManagers.entries().forEach(rmo -> rmo.ifPresent(rm -> verify(rm, times(1)).poll(anyLong()))); verify(networkClient, times(1)).poll(anyLong(), anyLong()); } @@ -257,7 +257,7 @@ void testEnsureMetadataUpdateOnPoll() { MetadataResponse metadataResponse = RequestTestUtils.metadataUpdateWith(2, Collections.emptyMap()); client.prepareMetadataUpdate(metadataResponse); metadata.requestUpdate(false); - backgroundThread.runOnce(); + consumerNetworkThread.runOnce(); verify(metadata, times(1)).updateWithCurrentRequestVersion(eq(metadataResponse), eq(false), anyLong()); } @@ -272,7 +272,7 @@ void testEnsureEventsAreCompleted() { assertFalse(future.isDone()); assertFalse(applicationEventsQueue.isEmpty()); - backgroundThread.close(); + consumerNetworkThread.close(); assertTrue(future.isCompletedExceptionally()); assertTrue(applicationEventsQueue.isEmpty()); } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java index b1b48f1eb6273..aac1a35bf9ec4 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java @@ -183,12 +183,12 @@ public void close() { closeQuietly(backgroundEventProcessor, BackgroundEventProcessor.class.getSimpleName()); } - public static class DefaultBackgroundThreadTestBuilder extends ConsumerTestBuilder { + public static class ConsumerNetworkThreadTestBuilder extends ConsumerTestBuilder { - final DefaultBackgroundThread backgroundThread; + final ConsumerNetworkThread consumerNetworkThread; - public DefaultBackgroundThreadTestBuilder() { - this.backgroundThread = new DefaultBackgroundThread( + public ConsumerNetworkThreadTestBuilder() { + this.consumerNetworkThread = new ConsumerNetworkThread( logContext, time, () -> applicationEventProcessor, @@ -199,7 +199,7 @@ public DefaultBackgroundThreadTestBuilder() { @Override public void close() { - closeQuietly(backgroundThread, DefaultBackgroundThread.class.getSimpleName()); + closeQuietly(consumerNetworkThread, ConsumerNetworkThread.class.getSimpleName()); } } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java index 5b7b571242396..b7267ff23aa55 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java @@ -315,8 +315,8 @@ public void testFetcherCloseClosesFetchSessionsInBroker() { // // NOTE: by design the FetchRequestManager doesn't perform network I/O internally. That means that calling // close with a Timer will NOT send out the close session requests on close. The network I/O logic is - // handled inside DefaultBackgroundThread.runAtClose, so we need to run that logic here. - DefaultBackgroundThread.runAtClose(singletonList(Optional.of(fetcher)), networkClientDelegate, timer); + // handled inside ConsumerNetworkThread.runAtClose, so we need to run that logic here. + ConsumerNetworkThread.runAtClose(singletonList(Optional.of(fetcher)), networkClientDelegate, timer); NetworkClientDelegate.PollResult pollResult = fetcher.poll(time.milliseconds()); networkClientDelegate.addAll(pollResult.unsentRequests); From 60470ed390351b21556c203eaa3df9a1114d36a2 Mon Sep 17 00:00:00 2001 From: Kirk True Date: Wed, 4 Oct 2023 11:05:02 -0700 Subject: [PATCH 27/72] Asserting ConsumerNetworkThread is not closed on each call to runOnce() --- .../kafka/clients/consumer/internals/ConsumerNetworkThread.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThread.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThread.java index d95b80b25a798..9709b486a8ead 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThread.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThread.java @@ -128,6 +128,8 @@ void initializeResources() { * */ void runOnce() { + closer.assertOpen("Cannot process consumer application events because the network thread is closed"); + // If there are errors processing any events, the error will be thrown immediately. This will have // the effect of closing the background thread. applicationEventProcessor.process(); From 6c1004ff2bbb16c55cab31947c9eddf0e744facb Mon Sep 17 00:00:00 2001 From: Kirk True Date: Wed, 4 Oct 2023 11:13:01 -0700 Subject: [PATCH 28/72] Clean up of outdated comments for BackgroundEventProcessor.process() --- .../internals/events/BackgroundEventProcessor.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEventProcessor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEventProcessor.java index 8936cd9fc86dc..c2087c38700bf 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEventProcessor.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEventProcessor.java @@ -43,11 +43,9 @@ public BackgroundEventProcessor(final LogContext logContext, /** * Process the events—if any—that were produced by the {@link ConsumerNetworkThread network thread}. - * It is possible that when processing the events that a given event will - * {@link ErrorBackgroundEvent represent an error directly}, or it could be that processing an event generates - * an error. In such cases, the processor will continue to process the remaining events. In this case, we - * provide the caller to provide a callback handler that "collects" the errors. We grab the first error that - * occurred and throw it. + * It is possible that {@link ErrorBackgroundEvent an error} could occur when processing the events. + * In such cases, the processor will take a reference to the first error, continue to process the + * remaining events, and then throw the first error that occurred. */ @Override public void process() { From 849755809db3b998b7f6c3cb4b5ff1d8ad2a7a59 Mon Sep 17 00:00:00 2001 From: Kirk True Date: Wed, 4 Oct 2023 12:33:46 -0700 Subject: [PATCH 29/72] Clean up of logging, but a lot of other refactoring to reduce boilerplate too --- .../consumer/internals/AbstractFetch.java | 38 ++++-- .../internals/FetchRequestManager.java | 111 +++++++++--------- .../clients/consumer/internals/Fetcher.java | 106 +++++++++-------- 3 files changed, 142 insertions(+), 113 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractFetch.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractFetch.java index c9657b1406803..b62881d0cd3a2 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractFetch.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractFetch.java @@ -199,8 +199,7 @@ protected void handleFetchResponse(final Node fetchTarget, metricsManager.recordLatency(resp.requestLatencyMs()); } finally { - log.debug("Removing pending request for node {}", fetchTarget); - nodesWithPendingFetchRequests.remove(fetchTarget.id()); + removePendingFetchRequest(fetchTarget, data.metadata().sessionId()); } } @@ -208,9 +207,12 @@ protected void handleFetchResponse(final Node fetchTarget, * Implements the core logic for a failed fetch request/response. * * @param fetchTarget {@link Node} from which the fetch data was requested - * @param t {@link Throwable} representing the error that resulted in the failure + * @param data {@link FetchSessionHandler.FetchRequestData} from request + * @param t {@link Throwable} representing the error that resulted in the failure */ - protected void handleFetchResponse(final Node fetchTarget, final Throwable t) { + protected void handleFetchResponse(final Node fetchTarget, + final FetchSessionHandler.FetchRequestData data, + final Throwable t) { try { final FetchSessionHandler handler = sessionHandler(fetchTarget.id()); @@ -219,14 +221,15 @@ protected void handleFetchResponse(final Node fetchTarget, final Throwable t) { handler.sessionTopicPartitions().forEach(subscriptions::clearPreferredReadReplica); } } finally { - log.debug("Removing pending request for node {}", fetchTarget); - nodesWithPendingFetchRequests.remove(fetchTarget.id()); + removePendingFetchRequest(fetchTarget, data.metadata().sessionId()); } } protected void handleCloseFetchSessionResponse(final Node fetchTarget, - final FetchSessionHandler.FetchRequestData data) { + final FetchSessionHandler.FetchRequestData data, + final ClientResponse ignored) { int sessionId = data.metadata().sessionId(); + removePendingFetchRequest(fetchTarget, sessionId); log.debug("Successfully sent a close message for fetch session: {} to node: {}", sessionId, fetchTarget); } @@ -234,10 +237,16 @@ public void handleCloseFetchSessionResponse(final Node fetchTarget, final FetchSessionHandler.FetchRequestData data, final Throwable t) { int sessionId = data.metadata().sessionId(); - log.debug("Unable to a close message for fetch session: {} to node: {}. " + + removePendingFetchRequest(fetchTarget, sessionId); + log.debug("Unable to send a close message for fetch session: {} to node: {}. " + "This may result in unnecessary fetch sessions at the broker.", sessionId, fetchTarget, t); } + private void removePendingFetchRequest(Node fetchTarget, int sessionId) { + log.debug("Removing pending request for fetch session: {} for node: {}", sessionId, fetchTarget); + nodesWithPendingFetchRequests.remove(fetchTarget.id()); + } + /** * Creates a new {@link FetchRequest fetch request} in preparation for sending to the Kafka cluster. * @@ -445,4 +454,17 @@ public void close(final Timer timer) { public void close() { close(time.timer(Duration.ZERO)); } + + /** + * Defines the contract for handling fetch responses from brokers. + * @param Type of response, usually either {@link ClientResponse} or {@link Throwable} + */ + @FunctionalInterface + protected interface ResponseHandler { + + /** + * Handle the response from the given {@link Node target} + */ + void handle(Node target, FetchSessionHandler.FetchRequestData data, T response); + } } \ No newline at end of file diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchRequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchRequestManager.java index 42d5bad29c2b9..093d18d8322ef 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchRequestManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchRequestManager.java @@ -19,6 +19,7 @@ import java.util.ArrayList; import java.util.LinkedList; import java.util.List; +import java.util.Map; import java.util.Queue; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Future; @@ -26,14 +27,13 @@ import java.util.function.Predicate; import java.util.stream.Collectors; +import org.apache.kafka.clients.ClientRequest; import org.apache.kafka.clients.ClientResponse; import org.apache.kafka.clients.FetchSessionHandler; import org.apache.kafka.clients.NetworkClient; import org.apache.kafka.clients.consumer.internals.NetworkClientDelegate.PollResult; import org.apache.kafka.clients.consumer.internals.NetworkClientDelegate.UnsentRequest; -import org.apache.kafka.clients.consumer.internals.events.ApplicationEventProcessor; import org.apache.kafka.clients.consumer.internals.events.BackgroundEventHandler; -import org.apache.kafka.clients.consumer.internals.events.ErrorBackgroundEvent; import org.apache.kafka.common.Node; import org.apache.kafka.common.requests.FetchRequest; import org.apache.kafka.common.utils.LogContext; @@ -79,8 +79,9 @@ protected void maybeThrowAuthFailure(Node node) { /** * Adds a new {@link Future future} to the list of futures awaiting results. Per the comments on - * {@link #forwardResults()}, there is no guarantee that this particular future will be provided with - * a non-empty result, but it is guaranteed to be completed with a result, assuming that it does not time out. + * {@link #handleFetchResponse(Node, FetchSessionHandler.FetchRequestData, ClientResponse)}}, there is no + * guarantee that this particular future will be provided with a non-empty result, but it is guaranteed + * to be completed with a result, assuming that it does not time out. * * @param future Future that will be {@link CompletableFuture#complete(Object) completed} if not timed out */ @@ -89,48 +90,52 @@ public void requestFetch(CompletableFuture> future) { } /** - * @see RequestManager#poll(long) + * {@inheritDoc} */ @Override public PollResult poll(long currentTimeMs) { - List requests = prepareFetchRequests().entrySet().stream().map(entry -> { - final Node fetchTarget = entry.getKey(); - final FetchSessionHandler.FetchRequestData data = entry.getValue(); - final FetchRequest.Builder request = createFetchRequest(fetchTarget, data); - final BiConsumer responseHandler = (clientResponse, t) -> { - if (t != null) { - handleFetchResponse(fetchTarget, t); - log.debug("Attempt to fetch data from node {} failed due to fatal exception", fetchTarget, t); - backgroundEventHandler.add(new ErrorBackgroundEvent(t)); - } else { - handleFetchResponse(fetchTarget, data, clientResponse); - forwardResults(); - } - }; - - return new UnsentRequest(request, fetchTarget, responseHandler); - }).collect(Collectors.toList()); - - return new PollResult(requests); + return pollInternal( + prepareFetchRequests(), + this::handleFetchResponse, + this::handleFetchResponse + ); } /** - * @see RequestManager#pollOnClose() + * {@inheritDoc} */ @Override public PollResult pollOnClose() { - List requests = prepareCloseFetchSessionRequests().entrySet().stream().map(entry -> { + return pollInternal( + prepareCloseFetchSessionRequests(), + this::handleCloseFetchSessionResponse, + this::handleCloseFetchSessionResponse + ); + } + + /** + * Creates the {@link PollResult poll result} that contains a list of zero or more + * {@link FetchRequest.Builder fetch requests} fetch request}, + * {@link NetworkClient#send(ClientRequest, long) enqueues/sends it, and adds the {@link RequestFuture callback} + * for the response. + * + * @param fetchRequests {@link Map} of {@link Node nodes} to their {@link FetchSessionHandler.FetchRequestData} + * @param successHandler {@link ResponseHandler Handler for successful responses} + * @param errorHandler {@link ResponseHandler Handler for failure responses} + * @return {@link PollResult} + */ + private PollResult pollInternal(Map fetchRequests, + ResponseHandler successHandler, + ResponseHandler errorHandler) { + List requests = fetchRequests.entrySet().stream().map(entry -> { final Node fetchTarget = entry.getKey(); final FetchSessionHandler.FetchRequestData data = entry.getValue(); final FetchRequest.Builder request = createFetchRequest(fetchTarget, data); - final BiConsumer responseHandler = (clientResponse, t) -> { - if (t != null) { - handleCloseFetchSessionResponse(fetchTarget, data, t); - log.warn("Attempt to close fetch session on node {} failed due to fatal exception", fetchTarget, t); - backgroundEventHandler.add(new ErrorBackgroundEvent(t)); - } else { - handleCloseFetchSessionResponse(fetchTarget, data); - } + final BiConsumer responseHandler = (clientResponse, error) -> { + if (error != null) + errorHandler.handle(fetchTarget, data, error); + else + successHandler.handle(fetchTarget, data, clientResponse); }; return new UnsentRequest(request, fetchTarget, responseHandler); @@ -156,38 +161,28 @@ public PollResult pollOnClose() { * It may be that some of the futures will be completed with an empty queue if the futures * ahead of it read all the results. Empty results should be accounted for in the code. */ - private void forwardResults() { + @Override + protected void handleFetchResponse(final Node fetchTarget, + final FetchSessionHandler.FetchRequestData data, + final ClientResponse response) { + super.handleFetchResponse(fetchTarget, data, response); + for (CompletableFuture> f : futures) { if (f.isDone()) continue; - Queue q = drain(); + Queue q = new LinkedList<>(); + CompletedFetch completedFetch = fetchBuffer.poll(); + + while (completedFetch != null) { + q.add(completedFetch); + completedFetch = fetchBuffer.poll(); + } + f.complete(q); } // Clear the list of futures here as they have fulfilled their purpose. futures.clear(); } - - /** - * Drains any of the {@link CompletedFetch} objects from the internal buffer to the returned {@link Queue}. - * - *

    - * - * This is used by the {@link ApplicationEventProcessor} to pull off any fetch results that are stored in - * the {@link ConsumerNetworkThread network thread} to provide them to the application thread. - * - * @return {@link Queue} containing zero or more {@link CompletedFetch} - */ - private Queue drain() { - Queue q = new LinkedList<>(); - CompletedFetch completedFetch = fetchBuffer.poll(); - - while (completedFetch != null) { - q.add(completedFetch); - completedFetch = fetchBuffer.poll(); - } - - return q; - } } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java index de69f0adf68e0..1d8be5192cf43 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java @@ -16,8 +16,10 @@ */ package org.apache.kafka.clients.consumer.internals; +import org.apache.kafka.clients.ClientRequest; import org.apache.kafka.clients.ClientResponse; import org.apache.kafka.clients.FetchSessionHandler; +import org.apache.kafka.clients.NetworkClient; import org.apache.kafka.common.Node; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.internals.IdempotentCloser; @@ -93,69 +95,39 @@ public void clearBufferedDataForUnassignedPartitions(Collection } /** - * Set-up a fetch request for any node that we have assigned partitions for which doesn't already have + * Set up a fetch request for any node that we have assigned partitions for which doesn't already have * an in-flight fetch or pending fetch data. * @return number of fetches sent */ public synchronized int sendFetches() { - Map fetchRequestMap = prepareFetchRequests(); - - for (Map.Entry entry : fetchRequestMap.entrySet()) { - final Node fetchTarget = entry.getKey(); - final FetchSessionHandler.FetchRequestData data = entry.getValue(); - final FetchRequest.Builder request = createFetchRequest(fetchTarget, data); - RequestFutureListener listener = new RequestFutureListener() { - @Override - public void onSuccess(ClientResponse resp) { + final Map fetchRequests = prepareFetchRequests(); + sendFetchesInternal( + fetchRequests, + (fetchTarget, data, clientResponse) -> { synchronized (Fetcher.this) { - handleFetchResponse(fetchTarget, data, resp); + handleFetchResponse(fetchTarget, data, clientResponse); } - } - - @Override - public void onFailure(RuntimeException e) { + }, + (fetchTarget, data, error) -> { synchronized (Fetcher.this) { - handleFetchResponse(fetchTarget, e); + handleFetchResponse(fetchTarget, data, error); } - } - }; - - final RequestFuture future = client.send(fetchTarget, request); - future.addListener(listener); - } - - return fetchRequestMap.size(); + }); + return fetchRequests.size(); } protected void maybeCloseFetchSessions(final Timer timer) { - final List> requestFutures = new ArrayList<>(); - Map fetchRequestMap = prepareCloseFetchSessionRequests(); - - for (Map.Entry entry : fetchRequestMap.entrySet()) { - final Node fetchTarget = entry.getKey(); - final FetchSessionHandler.FetchRequestData data = entry.getValue(); - final FetchRequest.Builder request = createFetchRequest(fetchTarget, data); - final RequestFuture responseFuture = client.send(fetchTarget, request); - - responseFuture.addListener(new RequestFutureListener() { - @Override - public void onSuccess(ClientResponse value) { - handleCloseFetchSessionResponse(fetchTarget, data); - } - - @Override - public void onFailure(RuntimeException e) { - handleCloseFetchSessionResponse(fetchTarget, data, e); - } - }); - - requestFutures.add(responseFuture); - } + final List> requestFutures = sendFetchesInternal( + prepareCloseFetchSessionRequests(), + this::handleCloseFetchSessionResponse, + this::handleCloseFetchSessionResponse + ); // Poll to ensure that request has been written to the socket. Wait until either the timer has expired or until // all requests have received a response. while (timer.notExpired() && !requestFutures.stream().allMatch(RequestFuture::isDone)) { client.poll(timer, null, true); + timer.update(); } if (!requestFutures.stream().allMatch(RequestFuture::isDone)) { @@ -185,4 +157,44 @@ protected void closeInternal(Timer timer) { maybeCloseFetchSessions(timer); super.closeInternal(timer); } + + /** + * Creates the {@link FetchRequest.Builder fetch request}, + * {@link NetworkClient#send(ClientRequest, long) enqueues/sends it, and adds the {@link RequestFuture callback} + * for the response. + * + * @param fetchRequests {@link Map} of {@link Node nodes} to their + * {@link FetchSessionHandler.FetchRequestData request data} + * @param successHandler {@link ResponseHandler Handler for successful responses} + * @param errorHandler {@link ResponseHandler Handler for failure responses} + * @return List of {@link RequestFuture callbacks} + */ + private List> sendFetchesInternal(Map fetchRequests, + ResponseHandler successHandler, + ResponseHandler errorHandler) { + final List> requestFutures = new ArrayList<>(); + + for (Map.Entry entry : fetchRequests.entrySet()) { + final Node fetchTarget = entry.getKey(); + final FetchSessionHandler.FetchRequestData data = entry.getValue(); + final FetchRequest.Builder request = createFetchRequest(fetchTarget, data); + final RequestFuture responseFuture = client.send(fetchTarget, request); + + responseFuture.addListener(new RequestFutureListener() { + @Override + public void onSuccess(ClientResponse resp) { + successHandler.handle(fetchTarget, data, resp); + } + + @Override + public void onFailure(RuntimeException e) { + errorHandler.handle(fetchTarget, data, e); + } + }); + + requestFutures.add(responseFuture); + } + + return requestFutures; + } } \ No newline at end of file From 6338733bfad130579a1b302d38c45e6cf664fc53 Mon Sep 17 00:00:00 2001 From: Kirk True Date: Wed, 4 Oct 2023 12:35:05 -0700 Subject: [PATCH 30/72] Removed now unneeded BackgroundEventHandler from FetchRequestManager --- .../clients/consumer/internals/FetchRequestManager.java | 7 ------- .../kafka/clients/consumer/internals/RequestManagers.java | 1 - .../clients/consumer/internals/ConsumerTestBuilder.java | 1 - .../consumer/internals/FetchRequestManagerTest.java | 6 +----- 4 files changed, 1 insertion(+), 14 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchRequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchRequestManager.java index 093d18d8322ef..b8db9a09641ce 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchRequestManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchRequestManager.java @@ -33,12 +33,10 @@ import org.apache.kafka.clients.NetworkClient; import org.apache.kafka.clients.consumer.internals.NetworkClientDelegate.PollResult; import org.apache.kafka.clients.consumer.internals.NetworkClientDelegate.UnsentRequest; -import org.apache.kafka.clients.consumer.internals.events.BackgroundEventHandler; import org.apache.kafka.common.Node; import org.apache.kafka.common.requests.FetchRequest; import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.Time; -import org.slf4j.Logger; /** * {@code FetchRequestManager} is responsible for generating {@link FetchRequest} that represent the @@ -47,22 +45,17 @@ */ public class FetchRequestManager extends AbstractFetch implements RequestManager { - private final Logger log; - private final BackgroundEventHandler backgroundEventHandler; private final NetworkClientDelegate networkClientDelegate; private final List>> futures; FetchRequestManager(final LogContext logContext, final Time time, - final BackgroundEventHandler backgroundEventHandler, final ConsumerMetadata metadata, final SubscriptionState subscriptions, final FetchConfig fetchConfig, final FetchMetricsManager metricsManager, final NetworkClientDelegate networkClientDelegate) { super(logContext, metadata, subscriptions, fetchConfig, metricsManager, time); - this.log = logContext.logger(FetchRequestManager.class); - this.backgroundEventHandler = backgroundEventHandler; this.networkClientDelegate = networkClientDelegate; this.futures = new ArrayList<>(); } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java index 384f2515b505b..484f24eb98159 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java @@ -131,7 +131,6 @@ protected RequestManagers create() { logContext); final FetchRequestManager fetch = new FetchRequestManager(logContext, time, - backgroundEventHandler, metadata, subscriptions, fetchConfig, diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java index aac1a35bf9ec4..11b79cd108ee6 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java @@ -153,7 +153,6 @@ public ConsumerTestBuilder() { groupState)); this.fetchRequestManager = spy(new FetchRequestManager(logContext, time, - backgroundEventHandler, metadata, subscriptions, fetchConfig, diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java index b7267ff23aa55..acbc09597f2ae 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java @@ -29,7 +29,6 @@ import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.OffsetOutOfRangeException; import org.apache.kafka.clients.consumer.OffsetResetStrategy; -import org.apache.kafka.clients.consumer.internals.events.BackgroundEventHandler; import org.apache.kafka.common.Cluster; import org.apache.kafka.common.IsolationLevel; import org.apache.kafka.common.KafkaException; @@ -115,7 +114,6 @@ import java.util.Properties; import java.util.Set; import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; @@ -3389,7 +3387,6 @@ private void buildFetcher(MetricConfig metricConfig, fetcher = spy(new TestableFetchRequestManager<>( logContext, time, - new BackgroundEventHandler(logContext, new LinkedBlockingQueue<>()), metadata, subscriptionState, fetchConfig, @@ -3447,14 +3444,13 @@ private class TestableFetchRequestManager extends FetchRequestManager { public TestableFetchRequestManager(LogContext logContext, Time time, - BackgroundEventHandler backgroundEventHandler, ConsumerMetadata metadata, SubscriptionState subscriptions, FetchConfig fetchConfig, FetchMetricsManager metricsManager, NetworkClientDelegate networkClientDelegate, FetchCollector fetchCollector) { - super(logContext, time, backgroundEventHandler, metadata, subscriptions, fetchConfig, metricsManager, networkClientDelegate); + super(logContext, time, metadata, subscriptions, fetchConfig, metricsManager, networkClientDelegate); this.fetchCollector = fetchCollector; } From 634e15be04ee6bd44c2606850527576fdc27da81 Mon Sep 17 00:00:00 2001 From: Kirk True Date: Wed, 4 Oct 2023 17:02:39 -0700 Subject: [PATCH 31/72] Turned down the logging level in a couple of places --- .../kafka/clients/consumer/internals/ConsumerNetworkThread.java | 2 +- .../kafka/clients/consumer/internals/events/EventProcessor.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThread.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThread.java index 9709b486a8ead..dd72a4e4710ce 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThread.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThread.java @@ -234,7 +234,7 @@ void closeInternal(final Duration timeout) { // the Thread.join method means "wait forever" which isn't what we want. try { long remainingMs = timer.remainingMs(); - log.warn("Waiting up to {} ms for the thread to complete", remainingMs); + log.debug("Waiting up to {} ms for the thread to complete", remainingMs); join(remainingMs); } catch (InterruptedException e) { throw new InterruptException(e); diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/EventProcessor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/EventProcessor.java index 6223c7ed11918..a8ea23e6a8be0 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/EventProcessor.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/EventProcessor.java @@ -103,7 +103,7 @@ protected void process(ProcessErrorHandler processErrorHandler) { */ private void closeInternal() { String eventClassName = getEventClass().getSimpleName(); - log.debug("Closing {} processor", eventClassName); + log.trace("Closing {} processor", eventClassName); List incompleteEvents = drain(); if (incompleteEvents.isEmpty()) From 5f9aa585a931ed231fc475494f1a41d1de4868c9 Mon Sep 17 00:00:00 2001 From: Kirk True Date: Thu, 5 Oct 2023 14:39:05 -0700 Subject: [PATCH 32/72] Removed comment about wakeups from poll() (filed KAFKA-15555 to handle) --- .../kafka/clients/consumer/internals/PrototypeAsyncConsumer.java | 1 - 1 file changed, 1 deletion(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java index dc8b2047cc8ea..08c202bca28e1 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java @@ -354,7 +354,6 @@ public ConsumerRecords poll(final Duration timeout) { } finally { kafkaConsumerMetrics.recordPollEnd(timer.currentTimeMs()); } - // TODO: Once we implement poll(), clear wakeupTrigger in a finally block: wakeupTrigger.clearActiveTask(); } /** From 193af8230d0c61853d764cbbe29bca2fc6361af9 Mon Sep 17 00:00:00 2001 From: Kirk True Date: Thu, 5 Oct 2023 14:42:15 -0700 Subject: [PATCH 33/72] Fixed comment in assign() method --- .../clients/consumer/internals/PrototypeAsyncConsumer.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java index 08c202bca28e1..199ea64eed15c 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java @@ -834,8 +834,11 @@ public void assign(Collection partitions) { fetchBuffer.retainAll(currentTopicPartitions); - // make sure the offsets of topic partitions the consumer is unsubscribing from - // are committed since there will be no following rebalance + // assignment change event will trigger autocommit if it is configured and the group id is specified. This is + // to make sure offsets of topic partitions the consumer is unsubscribing from are committed since there will + // be no following rebalance. + // + // See the ApplicationEventProcessor.process() method that handles this event for more detail. applicationEventHandler.add(new AssignmentChangeApplicationEvent(subscriptions.allConsumed(), time.milliseconds())); log.info("Assigned to partition(s): {}", join(partitions, ", ")); From cda57410f6d3a0dcd2cf24a0695a07615591006f Mon Sep 17 00:00:00 2001 From: Kirk True Date: Fri, 6 Oct 2023 14:44:12 -0700 Subject: [PATCH 34/72] Made FetchBuffer thread safe so that it can be shared by application and consumer network thread --- .../consumer/internals/AbstractFetch.java | 3 +- .../consumer/internals/FetchBuffer.java | 166 +++++++++++++++--- .../internals/FetchRequestManager.java | 64 +------ .../clients/consumer/internals/Fetcher.java | 2 +- .../internals/PrototypeAsyncConsumer.java | 45 ++--- .../consumer/internals/RequestManagers.java | 2 + .../events/ApplicationEventProcessor.java | 2 +- .../internals/ConsumerTestBuilder.java | 3 + .../internals/FetchRequestManagerTest.java | 4 +- 9 files changed, 167 insertions(+), 124 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractFetch.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractFetch.java index b62881d0cd3a2..7482062b1b157 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractFetch.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractFetch.java @@ -74,6 +74,7 @@ public AbstractFetch(final LogContext logContext, final ConsumerMetadata metadata, final SubscriptionState subscriptions, final FetchConfig fetchConfig, + final FetchBuffer fetchBuffer, final FetchMetricsManager metricsManager, final Time time) { this.log = logContext.logger(AbstractFetch.class); @@ -81,8 +82,8 @@ public AbstractFetch(final LogContext logContext, this.metadata = metadata; this.subscriptions = subscriptions; this.fetchConfig = fetchConfig; + this.fetchBuffer = fetchBuffer; this.decompressionBufferSupplier = BufferSupplier.create(); - this.fetchBuffer = new FetchBuffer(logContext); this.sessionHandlers = new HashMap<>(); this.nodesWithPendingFetchRequests = new HashSet<>(); this.metricsManager = metricsManager; diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchBuffer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchBuffer.java index d1865117e3f52..9b27ea6438dbe 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchBuffer.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchBuffer.java @@ -17,14 +17,21 @@ package org.apache.kafka.clients.consumer.internals; import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.errors.InterruptException; import org.apache.kafka.common.internals.IdempotentCloser; import org.apache.kafka.common.utils.LogContext; +import org.apache.kafka.common.utils.Timer; import org.slf4j.Logger; import java.util.Collection; +import java.util.Collections; import java.util.HashSet; import java.util.Set; import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Condition; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; import java.util.function.Predicate; /** @@ -34,12 +41,15 @@ * *

    * - * Note: this class is not thread-safe and is intended to only be used from a single thread. + * Note: this class is thread-safe with the intention that {@link CompletedFetch the data} will be + * "produced" by a background thread and consumed by the application thread. */ public class FetchBuffer implements AutoCloseable { private final Logger log; private final ConcurrentLinkedQueue completedFetches; + private final Lock lock; + private final Condition notEmptyCondition; private final IdempotentCloser idempotentCloser = new IdempotentCloser(); private CompletedFetch nextInLineFetch; @@ -47,6 +57,8 @@ public class FetchBuffer implements AutoCloseable { public FetchBuffer(final LogContext logContext) { this.log = logContext.logger(FetchBuffer.class); this.completedFetches = new ConcurrentLinkedQueue<>(); + this.lock = new ReentrantLock(); + this.notEmptyCondition = lock.newCondition(); } /** @@ -55,7 +67,12 @@ public FetchBuffer(final LogContext logContext) { * @return {@code true} if the buffer is empty, {@code false} otherwise */ boolean isEmpty() { - return completedFetches.isEmpty(); + try { + lock.lock(); + return completedFetches.isEmpty(); + } finally { + lock.unlock(); + } } /** @@ -65,31 +82,107 @@ boolean isEmpty() { * @return {@code true} if there are completed fetches that match the {@link Predicate}, {@code false} otherwise */ boolean hasCompletedFetches(Predicate predicate) { - return completedFetches.stream().anyMatch(predicate); + try { + lock.lock(); + return completedFetches.stream().anyMatch(predicate); + } finally { + lock.unlock(); + } } void add(CompletedFetch completedFetch) { - completedFetches.add(completedFetch); + try { + lock.lock(); + completedFetches.add(completedFetch); + notEmptyCondition.signalAll(); + } finally { + lock.unlock(); + } } void addAll(Collection completedFetches) { - this.completedFetches.addAll(completedFetches); + if (completedFetches == null || completedFetches.isEmpty()) + return; + + try { + lock.lock(); + this.completedFetches.addAll(completedFetches); + notEmptyCondition.signalAll(); + } finally { + lock.unlock(); + } } CompletedFetch nextInLineFetch() { - return nextInLineFetch; + try { + lock.lock(); + return nextInLineFetch; + } finally { + lock.unlock(); + } } - void setNextInLineFetch(CompletedFetch completedFetch) { - this.nextInLineFetch = completedFetch; + void setNextInLineFetch(CompletedFetch nextInLineFetch) { + try { + lock.lock(); + this.nextInLineFetch = nextInLineFetch; + } finally { + lock.unlock(); + } } CompletedFetch peek() { - return completedFetches.peek(); + try { + lock.lock(); + return completedFetches.peek(); + } finally { + lock.unlock(); + } } CompletedFetch poll() { - return completedFetches.poll(); + try { + lock.lock(); + return completedFetches.poll(); + } finally { + lock.unlock(); + } + } + + /** + * Allows the caller to await presence of data in the buffer. The method will block, returning only + * under one of the following conditions: + * + *

      + *
    1. The buffer was already non-empty on entry
      2. + *
    2. The buffer was populated during the wait
      3. + *
    3. The remaining time on the {@link Timer timer} elapsed
      4. + *
    4. The thread was interrupted
      5. + *
    + * + * @param timer Timer that provides time to wait + */ + void awaitNotEmpty(Timer timer) { + try { + lock.lock(); + + while (isEmpty()) { + // Update the timer before we head into the loop in case it took a while to get the lock. + timer.update(); + + if (timer.isExpired()) + break; + + if (notEmptyCondition.await(timer.remainingMs(), TimeUnit.MILLISECONDS)) { + break; + } + } + } catch (InterruptedException e) { + throw new InterruptException("Timeout waiting for results from fetching records", e); + } finally { + lock.unlock(); + timer.update(); + } } /** @@ -99,12 +192,22 @@ CompletedFetch poll() { * @param partitions {@link Set} of {@link TopicPartition}s for which any buffered data should be kept */ void retainAll(final Set partitions) { - completedFetches.removeIf(cf -> maybeDrain(partitions, cf)); + try { + lock.lock(); + + completedFetches.removeIf(cf -> maybeDrain(partitions, cf)); - if (maybeDrain(partitions, nextInLineFetch)) - nextInLineFetch = null; + if (maybeDrain(partitions, nextInLineFetch)) + nextInLineFetch = null; + } finally { + lock.unlock(); + } } + /** + * Drains (i.e. removes) the contents of the given {@link CompletedFetch} as its data should not + * be returned to the user. + */ private boolean maybeDrain(final Set partitions, final CompletedFetch completedFetch) { if (completedFetch != null && !partitions.contains(completedFetch.partition)) { log.debug("Removing {} from buffered fetch data as it is not in the set of partitions to retain ({})", completedFetch.partition, partitions); @@ -121,28 +224,33 @@ private boolean maybeDrain(final Set partitions, final Completed * @return {@link TopicPartition Partition} set */ Set bufferedPartitions() { - final Set partitions = new HashSet<>(); + try { + lock.lock(); - if (nextInLineFetch != null && !nextInLineFetch.isConsumed()) { - partitions.add(nextInLineFetch.partition); - } + final Set partitions = new HashSet<>(); + + if (nextInLineFetch != null && !nextInLineFetch.isConsumed()) { + partitions.add(nextInLineFetch.partition); + } - completedFetches.forEach(cf -> partitions.add(cf.partition)); - return partitions; + completedFetches.forEach(cf -> partitions.add(cf.partition)); + return partitions; + } finally { + lock.unlock(); + } } @Override public void close() { - idempotentCloser.close(() -> { - log.debug("Closing the fetch buffer"); + try { + lock.lock(); - if (nextInLineFetch != null) { - nextInLineFetch.drain(); - nextInLineFetch = null; - } - - completedFetches.forEach(CompletedFetch::drain); - completedFetches.clear(); - }, () -> log.warn("The fetch buffer was already closed")); + idempotentCloser.close( + () -> retainAll(Collections.emptySet()), + () -> log.warn("The fetch buffer was already closed") + ); + } finally { + lock.unlock(); + } } } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchRequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchRequestManager.java index b8db9a09641ce..09c4ead0746cb 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchRequestManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchRequestManager.java @@ -16,13 +16,8 @@ */ package org.apache.kafka.clients.consumer.internals; -import java.util.ArrayList; -import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.Queue; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.Future; import java.util.function.BiConsumer; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -46,18 +41,17 @@ public class FetchRequestManager extends AbstractFetch implements RequestManager { private final NetworkClientDelegate networkClientDelegate; - private final List>> futures; FetchRequestManager(final LogContext logContext, final Time time, final ConsumerMetadata metadata, final SubscriptionState subscriptions, final FetchConfig fetchConfig, + final FetchBuffer fetchBuffer, final FetchMetricsManager metricsManager, final NetworkClientDelegate networkClientDelegate) { - super(logContext, metadata, subscriptions, fetchConfig, metricsManager, time); + super(logContext, metadata, subscriptions, fetchConfig, fetchBuffer, metricsManager, time); this.networkClientDelegate = networkClientDelegate; - this.futures = new ArrayList<>(); } @Override @@ -70,18 +64,6 @@ protected void maybeThrowAuthFailure(Node node) { networkClientDelegate.maybeThrowAuthFailure(node); } - /** - * Adds a new {@link Future future} to the list of futures awaiting results. Per the comments on - * {@link #handleFetchResponse(Node, FetchSessionHandler.FetchRequestData, ClientResponse)}}, there is no - * guarantee that this particular future will be provided with a non-empty result, but it is guaranteed - * to be completed with a result, assuming that it does not time out. - * - * @param future Future that will be {@link CompletableFuture#complete(Object) completed} if not timed out - */ - public void requestFetch(CompletableFuture> future) { - futures.add(future); - } - /** * {@inheritDoc} */ @@ -136,46 +118,4 @@ private PollResult pollInternal(Map return new PollResult(requests); } - - /** - * Drains any of the {@link CompletedFetch completed fetch} objects from the - * {@link FetchBuffer internal fetch buffer} and provides the results to awaiting {@link Future futures} - * from the application thread. - * - *

    - * - * For each future in {@link #futures} that isn't completed at the time that this method is invoked, we will - * drain the available, buffered data from the underlying fetch buffer and provide it to the future. As the - * {@link NetworkClient} loops through the completed I/O, each response handler is invoked. Those response handlers - * will then re-populate the internal fetch buffer before calling this method. - * - *

    - * - * It may be that some of the futures will be completed with an empty queue if the futures - * ahead of it read all the results. Empty results should be accounted for in the code. - */ - @Override - protected void handleFetchResponse(final Node fetchTarget, - final FetchSessionHandler.FetchRequestData data, - final ClientResponse response) { - super.handleFetchResponse(fetchTarget, data, response); - - for (CompletableFuture> f : futures) { - if (f.isDone()) - continue; - - Queue q = new LinkedList<>(); - CompletedFetch completedFetch = fetchBuffer.poll(); - - while (completedFetch != null) { - q.add(completedFetch); - completedFetch = fetchBuffer.poll(); - } - - f.complete(q); - } - - // Clear the list of futures here as they have fulfilled their purpose. - futures.clear(); - } } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java index 1d8be5192cf43..3449790cd3318 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java @@ -68,7 +68,7 @@ public Fetcher(LogContext logContext, Deserializers deserializers, FetchMetricsManager metricsManager, Time time) { - super(logContext, metadata, subscriptions, fetchConfig, metricsManager, time); + super(logContext, metadata, subscriptions, fetchConfig, new FetchBuffer(logContext), metricsManager, time); this.log = logContext.logger(Fetcher.class); this.client = client; this.fetchCollector = new FetchCollector<>(logContext, diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java index 199ea64eed15c..9d3087a5b8efa 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java @@ -84,7 +84,6 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.CompletableFuture; import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; import java.util.function.Supplier; @@ -124,6 +123,13 @@ public class PrototypeAsyncConsumer implements Consumer { private final String clientId; private final BackgroundEventProcessor backgroundEventProcessor; private final Deserializers deserializers; + + /** + * A thread-safe {@link FetchBuffer fetch buffer} for the results that are populated in the + * {@link ConsumerNetworkThread network thread} when the results are available. Because of the interaction + * of the fetch buffer in the application thread and the network I/O thread, this is shared between the + * two threads and is thus designed to be thread-safe. + */ private final FetchBuffer fetchBuffer; private final FetchCollector fetchCollector; private final ConsumerInterceptors interceptors; @@ -141,15 +147,6 @@ public class PrototypeAsyncConsumer implements Consumer { private boolean cachedSubscriptionHasAllFetchPositions; private final WakeupTrigger wakeupTrigger = new WakeupTrigger(); - /** - * A thread-safe {@link BlockingQueue queue} for the results that are populated in the - * {@link ConsumerNetworkThread network thread} when the fetch results are available. Because - * the {@link #fetchBuffer fetch buffer} is not thread-safe, we need to separate the results - * collection that we provide to the network thread from the collection that - * we read from on the application thread. - */ - private final BlockingQueue fetchResults = new LinkedBlockingQueue<>(); - public PrototypeAsyncConsumer(final Properties properties, final Deserializer keyDeserializer, final Deserializer valueDeserializer) { @@ -212,6 +209,9 @@ public PrototypeAsyncConsumer(final Time time, ApiVersions apiVersions = new ApiVersions(); final BlockingQueue applicationEventQueue = new LinkedBlockingQueue<>(); final BlockingQueue backgroundEventQueue = new LinkedBlockingQueue<>(); + + // This FetchBuffer is shared between the application and network threads. + this.fetchBuffer = new FetchBuffer(logContext); final Supplier networkClientDelegateSupplier = NetworkClientDelegate.supplier(time, logContext, metadata, @@ -224,6 +224,7 @@ public PrototypeAsyncConsumer(final Time time, backgroundEventQueue, metadata, subscriptions, + fetchBuffer, config, groupRebalanceConfig, apiVersions, @@ -250,8 +251,8 @@ public PrototypeAsyncConsumer(final Time time, config.ignore(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG); //config.ignore(ConsumerConfig.THROW_ON_FETCH_STABLE_OFFSET_UNSUPPORTED); } - // These are specific to the foreground thread - this.fetchBuffer = new FetchBuffer(logContext); + + // The FetchCollector is only used on the application thread. this.fetchCollector = new FetchCollector<>(logContext, metadata, subscriptions, @@ -902,18 +903,11 @@ WakeupTrigger wakeupTrigger() { /** * Send the requests for fetch data to the {@link ConsumerNetworkThread network thread} and set up to - * collect the results in {@link #fetchResults}. + * collect the results in {@link #fetchBuffer}. */ private void sendFetches() { FetchEvent event = new FetchEvent(); applicationEventHandler.add(event); - - event.future().whenComplete((completedFetches, error) -> { - if (error != null) - log.warn("An error occurred during poll: {}", error.getMessage(), error); - else - fetchResults.addAll(completedFetches); - }); } private Fetch pollForFetches(Timer timer) { @@ -945,15 +939,8 @@ private Fetch pollForFetches(Timer timer) { // data may not be immediately available, but the calling method (poll) will correctly // handle the overall timeout. try { - while (pollTimer.notExpired()) { - CompletedFetch completedFetch = fetchResults.poll(pollTimer.remainingMs(), TimeUnit.MILLISECONDS); - - if (completedFetch != null) - fetchBuffer.add(completedFetch); - - pollTimer.update(); - } - } catch (InterruptedException e) { + fetchBuffer.awaitNotEmpty(pollTimer); + } catch (InterruptException e) { log.trace("Timeout during fetch", e); } finally { timer.update(pollTimer.currentTimeMs()); diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java index 484f24eb98159..7c704990d18e5 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java @@ -106,6 +106,7 @@ public static Supplier supplier(final Time time, final BlockingQueue backgroundEventQueue, final ConsumerMetadata metadata, final SubscriptionState subscriptions, + final FetchBuffer fetchBuffer, final ConsumerConfig config, final GroupRebalanceConfig groupRebalanceConfig, final ApiVersions apiVersions, @@ -134,6 +135,7 @@ protected RequestManagers create() { metadata, subscriptions, fetchConfig, + fetchBuffer, fetchMetricsManager, networkClientDelegate); final TopicMetadataRequestManager topic = new TopicMetadataRequestManager( diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventProcessor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventProcessor.java index 373ba631fd999..264858b51bc8b 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventProcessor.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventProcessor.java @@ -184,7 +184,7 @@ private void process(final TopicMetadataApplicationEvent event) { } private void process(final FetchEvent event) { - requestManagers.fetchRequestManager.requestFetch(event.future()); + // This is currently unused. } /** diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java index 11b79cd108ee6..168ec6d9c1b65 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java @@ -69,6 +69,7 @@ public class ConsumerTestBuilder implements Closeable { final SubscriptionState subscriptions; final ConsumerMetadata metadata; final FetchConfig fetchConfig; + final FetchBuffer fetchBuffer; final Metrics metrics; final FetchMetricsManager metricsManager; final NetworkClientDelegate networkClientDelegate; @@ -151,11 +152,13 @@ public ConsumerTestBuilder() { config, coordinatorRequestManager, groupState)); + this.fetchBuffer = new FetchBuffer(logContext); this.fetchRequestManager = spy(new FetchRequestManager(logContext, time, metadata, subscriptions, fetchConfig, + fetchBuffer, metricsManager, networkClientDelegate)); this.topicMetadataRequestManager = spy(new TopicMetadataRequestManager(logContext, diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java index acbc09597f2ae..d9a639a814fc3 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java @@ -3390,6 +3390,7 @@ private void buildFetcher(MetricConfig metricConfig, metadata, subscriptionState, fetchConfig, + new FetchBuffer(logContext), metricsManager, networkClientDelegate, fetchCollector)); @@ -3447,10 +3448,11 @@ public TestableFetchRequestManager(LogContext logContext, ConsumerMetadata metadata, SubscriptionState subscriptions, FetchConfig fetchConfig, + FetchBuffer fetchBuffer, FetchMetricsManager metricsManager, NetworkClientDelegate networkClientDelegate, FetchCollector fetchCollector) { - super(logContext, time, metadata, subscriptions, fetchConfig, metricsManager, networkClientDelegate); + super(logContext, time, metadata, subscriptions, fetchConfig, fetchBuffer, metricsManager, networkClientDelegate); this.fetchCollector = fetchCollector; } From 0d56cacabb66da14aa025f86adb74a6ead53fb20 Mon Sep 17 00:00:00 2001 From: Kirk True Date: Fri, 6 Oct 2023 17:26:38 -0700 Subject: [PATCH 35/72] Minor: removed unnecessarily qualified reference to object from within itself --- .../clients/consumer/internals/FetchRequestManagerTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java index d9a639a814fc3..bc9454e6c3c17 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java @@ -3461,7 +3461,7 @@ private Fetch collectFetch() { } private int sendFetches() { - NetworkClientDelegate.PollResult pollResult = fetcher.poll(time.milliseconds()); + NetworkClientDelegate.PollResult pollResult = poll(time.milliseconds()); networkClientDelegate.addAll(pollResult.unsentRequests); return pollResult.unsentRequests.size(); } From 17458c465df05cafe07c5dfb74bcb286c76093b8 Mon Sep 17 00:00:00 2001 From: Kirk True Date: Mon, 9 Oct 2023 15:01:05 -0700 Subject: [PATCH 36/72] Resolving merge conflicts with changes to trunk --- .../internals/HeartbeatRequestManager.java | 17 +- .../consumer/internals/RequestManagers.java | 2 +- .../internals/CommitRequestManagerTest.java | 3 +- .../internals/ConsumerNetworkThreadTest.java | 7 +- .../internals/ConsumerTestBuilder.java | 163 ++++++++++---- .../HeartbeatRequestManagerTest.java | 202 +++++++----------- .../internals/PrototypeAsyncConsumerTest.java | 35 ++- 7 files changed, 226 insertions(+), 203 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java index 0cf7cf6fa6234..4311819146e17 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManager.java @@ -60,6 +60,7 @@ *

    See {@link HeartbeatRequestState} for more details.

    */ public class HeartbeatRequestManager implements RequestManager { + private final Logger logger; private final Time time; @@ -95,8 +96,8 @@ public class HeartbeatRequestManager implements RequestManager { private final BackgroundEventHandler backgroundEventHandler; public HeartbeatRequestManager( - final Time time, final LogContext logContext, + final Time time, final ConsumerConfig config, final CoordinatorRequestManager coordinatorRequestManager, final SubscriptionState subscriptions, @@ -148,18 +149,14 @@ public HeartbeatRequestManager( */ @Override public NetworkClientDelegate.PollResult poll(long currentTimeMs) { - if (!coordinatorRequestManager.coordinator().isPresent() || !membershipManager.shouldSendHeartbeat()) { - return new NetworkClientDelegate.PollResult( - Long.MAX_VALUE, Collections.emptyList()); - } + if (!coordinatorRequestManager.coordinator().isPresent() || !membershipManager.shouldSendHeartbeat()) + return NetworkClientDelegate.PollResult.EMPTY; // TODO: We will need to send a heartbeat response after partitions being revoke. This needs to be // implemented either with or after the partition reconciliation logic. - if (!heartbeatRequestState.canSendRequest(currentTimeMs)) { - return new NetworkClientDelegate.PollResult( - heartbeatRequestState.nextHeartbeatMs(currentTimeMs), - Collections.emptyList()); - } + if (!heartbeatRequestState.canSendRequest(currentTimeMs)) + return new NetworkClientDelegate.PollResult(heartbeatRequestState.nextHeartbeatMs(currentTimeMs)); + this.heartbeatRequestState.onSendAttempt(currentTimeMs); NetworkClientDelegate.UnsentRequest request = makeHeartbeatRequest(); return new NetworkClientDelegate.PollResult(heartbeatRequestState.heartbeatIntervalMs, Collections.singletonList(request)); diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java index 3f48d7b910fd7..dfe4bb971ada1 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java @@ -161,8 +161,8 @@ protected RequestManagers create() { commit = new CommitRequestManager(time, logContext, subscriptions, config, coordinator, groupState); MembershipManager membershipManager = new MembershipManagerImpl(groupState.groupId); heartbeatRequestManager = new HeartbeatRequestManager( - time, logContext, + time, config, coordinator, subscriptions, diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/CommitRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/CommitRequestManagerTest.java index 8b2e08bd5207a..fa55cbfffe65e 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/CommitRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/CommitRequestManagerTest.java @@ -55,6 +55,7 @@ import static org.apache.kafka.clients.consumer.ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG; import static org.apache.kafka.clients.consumer.ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG; import static org.apache.kafka.clients.consumer.ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG; +import static org.apache.kafka.clients.consumer.internals.ConsumerTestBuilder.DEFAULT_GROUP_ID; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -76,7 +77,7 @@ public void setup() { this.time = new MockTime(0); this.subscriptionState = mock(SubscriptionState.class); this.coordinatorRequestManager = mock(CoordinatorRequestManager.class); - this.groupState = new GroupState("group-1", Optional.empty()); + this.groupState = new GroupState(DEFAULT_GROUP_ID, Optional.empty()); this.props = new Properties(); this.props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, 100); diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThreadTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThreadTest.java index f38090c8b94cc..4eff5a905e806 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThreadTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThreadTest.java @@ -51,6 +51,7 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.CompletableFuture; +import static org.apache.kafka.clients.consumer.internals.ConsumerTestBuilder.DEFAULT_REQUEST_TIMEOUT_MS; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -88,8 +89,8 @@ public void setup() { client = testBuilder.client; applicationEventsQueue = testBuilder.applicationEventQueue; applicationEventProcessor = testBuilder.applicationEventProcessor; - coordinatorManager = testBuilder.coordinatorRequestManager; - commitManager = testBuilder.commitRequestManager; + coordinatorManager = testBuilder.coordinatorRequestManager.orElseThrow(IllegalStateException::new); + commitManager = testBuilder.commitRequestManager.orElseThrow(IllegalStateException::new); offsetsRequestManager = testBuilder.offsetsRequestManager; consumerNetworkThread = testBuilder.consumerNetworkThread; consumerNetworkThread.initializeResources(); @@ -231,7 +232,7 @@ void testPollResultTimer() { .setKeyType(FindCoordinatorRequest.CoordinatorType.TRANSACTION.id()) .setKey("foobar")), Optional.empty()); - req.setTimer(time, ConsumerTestBuilder.REQUEST_TIMEOUT_MS); + req.setTimer(time, DEFAULT_REQUEST_TIMEOUT_MS); // purposely setting a non-MAX time to ensure it is returning Long.MAX_VALUE upon success NetworkClientDelegate.PollResult success = new NetworkClientDelegate.PollResult( diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java index 5c08ceaabca41..d56261e48ae25 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java @@ -46,6 +46,8 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; +import static org.apache.kafka.clients.consumer.ConsumerConfig.GROUP_ID_CONFIG; +import static org.apache.kafka.clients.consumer.ConsumerConfig.GROUP_INSTANCE_ID_CONFIG; import static org.apache.kafka.clients.consumer.ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG; import static org.apache.kafka.clients.consumer.ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createFetchMetricsManager; @@ -54,11 +56,17 @@ import static org.apache.kafka.common.utils.Utils.closeQuietly; import static org.mockito.Mockito.spy; +@SuppressWarnings("ClassDataAbstractionCoupling") public class ConsumerTestBuilder implements Closeable { - static final long RETRY_BACKOFF_MS = 80; - static final long RETRY_BACKOFF_MAX_MS = 1000; - static final int REQUEST_TIMEOUT_MS = 500; + static final long DEFAULT_RETRY_BACKOFF_MS = 80; + static final long DEFAULT_RETRY_BACKOFF_MAX_MS = 1000; + static final int DEFAULT_REQUEST_TIMEOUT_MS = 500; + static final int DEFAULT_MAX_POLL_INTERVAL_MS = 10000; + static final String DEFAULT_GROUP_INSTANCE_ID = "group-instance-id"; + static final String DEFAULT_GROUP_ID = "group-id"; + static final int DEFAULT_HEARTBEAT_INTERVAL_MS = 1000; + static final double DEFAULT_HEARTBEAT_JITTER_MS = 0.0; final LogContext logContext = new LogContext(); final Time time = new MockTime(0); @@ -74,9 +82,11 @@ public class ConsumerTestBuilder implements Closeable { final FetchMetricsManager metricsManager; final NetworkClientDelegate networkClientDelegate; final OffsetsRequestManager offsetsRequestManager; - final CoordinatorRequestManager coordinatorRequestManager; - final CommitRequestManager commitRequestManager; - final HeartbeatRequestManager heartbeatRequestManager; + final Optional coordinatorRequestManager; + final Optional commitRequestManager; + final Optional heartbeatRequestManager; + final Optional membershipManager; + final Optional heartbeatRequestState; final TopicMetadataRequestManager topicMetadataRequestManager; final FetchRequestManager fetchRequestManager; final RequestManagers requestManagers; @@ -84,19 +94,25 @@ public class ConsumerTestBuilder implements Closeable { public final BackgroundEventProcessor backgroundEventProcessor; public final BackgroundEventHandler backgroundEventHandler; final MockClient client; + final Optional groupInfo; public ConsumerTestBuilder() { + this(Optional.empty()); + } + + public ConsumerTestBuilder(Optional groupInfo) { + this.groupInfo = groupInfo; this.applicationEventQueue = new LinkedBlockingQueue<>(); this.backgroundEventQueue = new LinkedBlockingQueue<>(); - this.backgroundEventHandler = new BackgroundEventHandler(logContext, backgroundEventQueue); + this.backgroundEventHandler = spy(new BackgroundEventHandler(logContext, backgroundEventQueue)); GroupRebalanceConfig groupRebalanceConfig = new GroupRebalanceConfig( 100, - 100, - 100, - "group_id", - Optional.empty(), - RETRY_BACKOFF_MS, - RETRY_BACKOFF_MAX_MS, + DEFAULT_MAX_POLL_INTERVAL_MS, + DEFAULT_HEARTBEAT_INTERVAL_MS, + groupInfo.map(gi -> gi.groupState.groupId).orElse(null), + groupInfo.flatMap(gi -> gi.groupState.groupInstanceId), + DEFAULT_RETRY_BACKOFF_MS, + DEFAULT_RETRY_BACKOFF_MAX_MS, true); GroupState groupState = new GroupState(groupRebalanceConfig); ApiVersions apiVersions = new ApiVersions(); @@ -104,10 +120,17 @@ public ConsumerTestBuilder() { Properties properties = new Properties(); properties.put(KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); properties.put(VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); - properties.put(CommonClientConfigs.RETRY_BACKOFF_MS_CONFIG, RETRY_BACKOFF_MS); - properties.put(CommonClientConfigs.REQUEST_TIMEOUT_MS_CONFIG, REQUEST_TIMEOUT_MS); + properties.put(CommonClientConfigs.RETRY_BACKOFF_MS_CONFIG, DEFAULT_RETRY_BACKOFF_MS); + properties.put(CommonClientConfigs.REQUEST_TIMEOUT_MS_CONFIG, DEFAULT_REQUEST_TIMEOUT_MS); + properties.put(CommonClientConfigs.MAX_POLL_INTERVAL_MS_CONFIG, DEFAULT_MAX_POLL_INTERVAL_MS); + + groupInfo.ifPresent(gi -> { + properties.put(GROUP_ID_CONFIG, gi.groupState.groupId); + gi.groupState.groupInstanceId.ifPresent(groupInstanceId -> properties.put(GROUP_INSTANCE_ID_CONFIG, groupInstanceId)); + }); this.config = new ConsumerConfig(properties); + this.fetchConfig = new FetchConfig(config); this.retryBackoffMs = config.getLong(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG); final long requestTimeoutMs = config.getInt(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG); @@ -141,27 +164,53 @@ public ConsumerTestBuilder() { apiVersions, networkClientDelegate, logContext)); - this.coordinatorRequestManager = spy(new CoordinatorRequestManager(time, - logContext, - RETRY_BACKOFF_MS, - RETRY_BACKOFF_MAX_MS, - backgroundEventHandler, - "group_id")); - this.commitRequestManager = spy(new CommitRequestManager(time, - logContext, - subscriptions, - config, - coordinatorRequestManager, - groupState)); - MembershipManager membershipManager = new MembershipManagerImpl(groupState.groupId); - this.heartbeatRequestManager = new HeartbeatRequestManager( - time, - logContext, - config, - coordinatorRequestManager, - subscriptions, - membershipManager, - backgroundEventHandler); + + if (groupInfo.isPresent()) { + GroupInformation gi = groupInfo.get(); + CoordinatorRequestManager coordinator = spy(new CoordinatorRequestManager( + time, + logContext, + DEFAULT_RETRY_BACKOFF_MS, + DEFAULT_RETRY_BACKOFF_MAX_MS, + backgroundEventHandler, + gi.groupState.groupId + )); + CommitRequestManager commit = spy(new CommitRequestManager(time, + logContext, + subscriptions, + config, + coordinator, + groupState)); + MembershipManager mm = spy(new MembershipManagerImpl(gi.groupState.groupId, gi.groupState.groupInstanceId.orElse(null), null)); + HeartbeatRequestManager.HeartbeatRequestState state = spy(new HeartbeatRequestManager.HeartbeatRequestState(logContext, + time, + gi.heartbeatIntervalMs, + retryBackoffMs, + DEFAULT_RETRY_BACKOFF_MAX_MS, + gi.heartbeatJitterMs)); + HeartbeatRequestManager heartbeat = spy(new HeartbeatRequestManager( + logContext, + time, + config, + coordinator, + subscriptions, + mm, + state, + backgroundEventHandler)); + + this.coordinatorRequestManager = Optional.of(coordinator); + this.commitRequestManager = Optional.of(commit); + this.heartbeatRequestManager = Optional.of(heartbeat); + this.heartbeatRequestState = Optional.of(state); + this.membershipManager = Optional.of(mm); + } else { + this.coordinatorRequestManager = Optional.empty(); + this.commitRequestManager = Optional.empty(); + this.heartbeatRequestManager = Optional.empty(); + this.heartbeatRequestState = Optional.empty(); + this.membershipManager = Optional.empty(); + } + this.fetchBuffer = new FetchBuffer(logContext); this.fetchRequestManager = spy(new FetchRequestManager(logContext, time, @@ -177,9 +226,9 @@ public ConsumerTestBuilder() { offsetsRequestManager, topicMetadataRequestManager, fetchRequestManager, - Optional.of(coordinatorRequestManager), - Optional.of(commitRequestManager), - Optional.of(heartbeatRequestManager)); + coordinatorRequestManager, + commitRequestManager, + heartbeatRequestManager); this.applicationEventProcessor = spy(new ApplicationEventProcessor( logContext, applicationEventQueue, @@ -201,6 +250,11 @@ public static class ConsumerNetworkThreadTestBuilder extends ConsumerTestBuilder final ConsumerNetworkThread consumerNetworkThread; public ConsumerNetworkThreadTestBuilder() { + this(createDefaultGroupInformation()); + } + + public ConsumerNetworkThreadTestBuilder(Optional groupInfo) { + super(groupInfo); this.consumerNetworkThread = new ConsumerNetworkThread( logContext, time, @@ -221,6 +275,11 @@ public static class ApplicationEventHandlerTestBuilder extends ConsumerTestBuild public final ApplicationEventHandler applicationEventHandler; public ApplicationEventHandlerTestBuilder() { + this(createDefaultGroupInformation()); + } + + public ApplicationEventHandlerTestBuilder(Optional groupInfo) { + super(groupInfo); this.applicationEventHandler = spy(new ApplicationEventHandler( logContext, time, @@ -240,7 +299,8 @@ public static class PrototypeAsyncConsumerTestBuilder extends ApplicationEventHa final PrototypeAsyncConsumer consumer; - public PrototypeAsyncConsumerTestBuilder(Optional groupIdOpt) { + public PrototypeAsyncConsumerTestBuilder(Optional groupInfo) { + super(groupInfo); String clientId = config.getString(CommonClientConfigs.CLIENT_ID_CONFIG); List assignors = ConsumerPartitionAssignor.getAssignorInstances( config.getList(ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG), @@ -270,7 +330,7 @@ public PrototypeAsyncConsumerTestBuilder(Optional groupIdOpt) { retryBackoffMs, 60000, assignors, - groupIdOpt.orElse(null))); + groupInfo.map(groupInformation -> groupInformation.groupState.groupId).orElse(null))); } @Override @@ -278,4 +338,25 @@ public void close() { consumer.close(); } } + + public static class GroupInformation { + + final GroupState groupState; + final int heartbeatIntervalMs; + final double heartbeatJitterMs; + + public GroupInformation(GroupState groupState) { + this(groupState, DEFAULT_HEARTBEAT_INTERVAL_MS, DEFAULT_HEARTBEAT_JITTER_MS); + } + + public GroupInformation(GroupState groupState, int heartbeatIntervalMs, double heartbeatJitterMs) { + this.groupState = groupState; + this.heartbeatIntervalMs = heartbeatIntervalMs; + this.heartbeatJitterMs = heartbeatJitterMs; + } + } + + static Optional createDefaultGroupInformation() { + return Optional.of(new GroupInformation(new GroupState(DEFAULT_GROUP_ID, Optional.empty()))); + } } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java index 598deace7f588..e383634d7a619 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/HeartbeatRequestManagerTest.java @@ -17,8 +17,7 @@ package org.apache.kafka.clients.consumer.internals; import org.apache.kafka.clients.ClientResponse; -import org.apache.kafka.clients.consumer.ConsumerConfig; -import org.apache.kafka.clients.consumer.OffsetResetStrategy; +import org.apache.kafka.clients.consumer.internals.events.BackgroundEventHandler; import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.Node; import org.apache.kafka.common.errors.TimeoutException; @@ -28,11 +27,9 @@ import org.apache.kafka.common.requests.ConsumerGroupHeartbeatRequest; import org.apache.kafka.common.requests.ConsumerGroupHeartbeatResponse; import org.apache.kafka.common.requests.RequestHeader; -import org.apache.kafka.common.serialization.StringDeserializer; -import org.apache.kafka.common.utils.LogContext; -import org.apache.kafka.common.utils.MockTime; import org.apache.kafka.common.utils.Time; import org.apache.kafka.common.utils.annotation.ApiKeyVersionsSource; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -46,80 +43,77 @@ import java.util.HashSet; import java.util.List; import java.util.Optional; -import java.util.Properties; -import static org.apache.kafka.clients.consumer.ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG; -import static org.apache.kafka.clients.consumer.ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG; -import static org.apache.kafka.clients.consumer.ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG; -import static org.apache.kafka.clients.consumer.ConsumerConfig.RETRY_BACKOFF_MS_CONFIG; -import static org.apache.kafka.clients.consumer.ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG; +import static org.apache.kafka.clients.consumer.internals.ConsumerTestBuilder.DEFAULT_GROUP_ID; +import static org.apache.kafka.clients.consumer.internals.ConsumerTestBuilder.DEFAULT_GROUP_INSTANCE_ID; +import static org.apache.kafka.clients.consumer.internals.ConsumerTestBuilder.DEFAULT_HEARTBEAT_INTERVAL_MS; +import static org.apache.kafka.clients.consumer.internals.ConsumerTestBuilder.DEFAULT_MAX_POLL_INTERVAL_MS; +import static org.apache.kafka.clients.consumer.internals.ConsumerTestBuilder.DEFAULT_RETRY_BACKOFF_MS; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; -import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; public class HeartbeatRequestManagerTest { - private static final int HEARTBEAT_INTERVAL_MS = 1000; - private static final long RETRY_BACKOFF_MAX_MS = 3000; - private static final long RETRY_BACKOFF_MS = 100; - private static final String GROUP_INSTANCE_ID = "group-instance-id"; - private static final String GROUP_ID = "group-id"; - + private ConsumerTestBuilder testBuilder; private Time time; - private LogContext logContext; private CoordinatorRequestManager coordinatorRequestManager; - private SubscriptionState subscriptionState; + private SubscriptionState subscriptions; private HeartbeatRequestManager heartbeatRequestManager; private MembershipManager membershipManager; private HeartbeatRequestManager.HeartbeatRequestState heartbeatRequestState; - private ConsumerConfig config; - - private String memberId = "member-id"; - private int memberEpoch = 1; - private ErrorEventHandler errorEventHandler; + private final String memberId = "member-id"; + private final int memberEpoch = 1; + private BackgroundEventHandler backgroundEventHandler; @BeforeEach public void setUp() { - time = new MockTime(); - logContext = new LogContext(); - config = new ConsumerConfig(createConsumerConfig()); - coordinatorRequestManager = mock(CoordinatorRequestManager.class); + setUp(ConsumerTestBuilder.createDefaultGroupInformation()); + } + + private void setUp(Optional groupInfo) { + testBuilder = new ConsumerTestBuilder(groupInfo); + time = testBuilder.time; + coordinatorRequestManager = testBuilder.coordinatorRequestManager.orElseThrow(IllegalStateException::new); + heartbeatRequestManager = testBuilder.heartbeatRequestManager.orElseThrow(IllegalStateException::new); + heartbeatRequestState = testBuilder.heartbeatRequestState.orElseThrow(IllegalStateException::new); + backgroundEventHandler = testBuilder.backgroundEventHandler; + subscriptions = testBuilder.subscriptions; + membershipManager = testBuilder.membershipManager.orElseThrow(IllegalStateException::new); + when(coordinatorRequestManager.coordinator()).thenReturn(Optional.of(new Node(1, "localhost", 9999))); - subscriptionState = mock(SubscriptionState.class); - membershipManager = spy(new MembershipManagerImpl(GROUP_ID)); - heartbeatRequestState = mock(HeartbeatRequestManager.HeartbeatRequestState.class); - errorEventHandler = mock(ErrorEventHandler.class); - heartbeatRequestManager = createManager(); } - private Properties createConsumerConfig() { - Properties properties = new Properties(); - properties.put(BOOTSTRAP_SERVERS_CONFIG, "localhost:9999"); - properties.put(KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); - properties.put(VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); - properties.put(RETRY_BACKOFF_MS_CONFIG, "100"); - return properties; + private void resetWithZeroHeartbeatInterval(Optional groupInstanceId) { + cleanup(); + + ConsumerTestBuilder.GroupInformation gi = new ConsumerTestBuilder.GroupInformation( + new GroupState(DEFAULT_GROUP_ID, groupInstanceId), + 0, + 0.0 + ); + + setUp(Optional.of(gi)); + } + + @AfterEach + public void cleanup() { + if (testBuilder != null) { + testBuilder.close(); + } } @Test public void testHeartbeatOnStartup() { // The initial heartbeatInterval is set to 0 - heartbeatRequestState = new HeartbeatRequestManager.HeartbeatRequestState( - logContext, - time, - 0, - RETRY_BACKOFF_MS, - RETRY_BACKOFF_MAX_MS, - 0); - heartbeatRequestManager = createManager(); + resetWithZeroHeartbeatInterval(Optional.empty()); + NetworkClientDelegate.PollResult result = heartbeatRequestManager.poll(time.milliseconds()); assertEquals(1, result.unsentRequests.size()); @@ -131,6 +125,9 @@ public void testHeartbeatOnStartup() { @ParameterizedTest @ValueSource(booleans = {true, false}) public void testSendHeartbeatOnMemberState(final boolean shouldSendHeartbeat) { + // The initial heartbeatInterval is set to 0 + resetWithZeroHeartbeatInterval(Optional.empty()); + // Mocking notInGroup when(membershipManager.shouldSendHeartbeat()).thenReturn(shouldSendHeartbeat); when(heartbeatRequestState.canSendRequest(anyLong())).thenReturn(true); @@ -150,21 +147,13 @@ public void testSendHeartbeatOnMemberState(final boolean shouldSendHeartbeat) { @ParameterizedTest @MethodSource("stateProvider") public void testTimerNotDue(final MemberState state) { - heartbeatRequestState = new HeartbeatRequestManager.HeartbeatRequestState( - logContext, - time, - HEARTBEAT_INTERVAL_MS, - RETRY_BACKOFF_MS, - RETRY_BACKOFF_MAX_MS); - heartbeatRequestManager = createManager(); - when(membershipManager.state()).thenReturn(state); time.sleep(100); // time elapsed < heartbeatInterval, no heartbeat should be sent NetworkClientDelegate.PollResult result = heartbeatRequestManager.poll(time.milliseconds()); assertEquals(0, result.unsentRequests.size()); if (membershipManager.shouldSendHeartbeat()) { - assertEquals(HEARTBEAT_INTERVAL_MS - 100, result.timeUntilNextPollMs); + assertEquals(DEFAULT_HEARTBEAT_INTERVAL_MS - 100, result.timeUntilNextPollMs); } else { assertEquals(Long.MAX_VALUE, result.timeUntilNextPollMs); } @@ -172,14 +161,9 @@ public void testTimerNotDue(final MemberState state) { @Test public void testBackoffOnTimeout() { - heartbeatRequestState = new HeartbeatRequestManager.HeartbeatRequestState( - logContext, - time, - 0, - RETRY_BACKOFF_MS, - RETRY_BACKOFF_MAX_MS, - 0); - heartbeatRequestManager = createManager(); + // The initial heartbeatInterval is set to 0 + resetWithZeroHeartbeatInterval(Optional.empty()); + when(coordinatorRequestManager.coordinator()).thenReturn(Optional.of(new Node(1, "localhost", 9999))); when(membershipManager.shouldSendHeartbeat()).thenReturn(true); NetworkClientDelegate.PollResult result = heartbeatRequestManager.poll(time.milliseconds()); @@ -187,7 +171,7 @@ public void testBackoffOnTimeout() { result.unsentRequests.get(0).future().completeExceptionally(new TimeoutException("timeout")); // Assure the manager will backoff on timeout - time.sleep(RETRY_BACKOFF_MS - 1); + time.sleep(DEFAULT_RETRY_BACKOFF_MS - 1); result = heartbeatRequestManager.poll(time.milliseconds()); assertEquals(0, result.unsentRequests.size()); @@ -198,21 +182,16 @@ public void testBackoffOnTimeout() { @Test public void testFailureOnFatalException() { - heartbeatRequestState = spy(new HeartbeatRequestManager.HeartbeatRequestState( - logContext, - time, - 0, - RETRY_BACKOFF_MS, - RETRY_BACKOFF_MAX_MS, - 0)); - heartbeatRequestManager = createManager(); + // The initial heartbeatInterval is set to 0 + resetWithZeroHeartbeatInterval(Optional.empty()); + when(coordinatorRequestManager.coordinator()).thenReturn(Optional.of(new Node(1, "localhost", 9999))); when(membershipManager.shouldSendHeartbeat()).thenReturn(true); NetworkClientDelegate.PollResult result = heartbeatRequestManager.poll(time.milliseconds()); assertEquals(1, result.unsentRequests.size()); result.unsentRequests.get(0).future().completeExceptionally(new KafkaException("fatal")); verify(membershipManager).transitionToFailed(); - verify(errorEventHandler).handle(any()); + verify(backgroundEventHandler).add(any()); } @Test @@ -227,22 +206,12 @@ public void testNoCoordinator() { @ParameterizedTest @ApiKeyVersionsSource(apiKey = ApiKeys.CONSUMER_GROUP_HEARTBEAT) public void testValidateConsumerGroupHeartbeatRequest(final short version) { + // The initial heartbeatInterval is set to 0, but we're testing + resetWithZeroHeartbeatInterval(Optional.of(DEFAULT_GROUP_INSTANCE_ID)); + List subscribedTopics = Collections.singletonList("topic"); - subscriptionState = new SubscriptionState(logContext, OffsetResetStrategy.NONE); - subscriptionState.subscribe(new HashSet<>(subscribedTopics), new NoOpConsumerRebalanceListener()); - - Properties prop = createConsumerConfig(); - prop.setProperty(MAX_POLL_INTERVAL_MS_CONFIG, "10000"); - config = new ConsumerConfig(prop); - membershipManager = new MembershipManagerImpl(GROUP_ID, GROUP_INSTANCE_ID, null); - heartbeatRequestState = new HeartbeatRequestManager.HeartbeatRequestState( - logContext, - time, - 0, - RETRY_BACKOFF_MS, - RETRY_BACKOFF_MAX_MS, - 0); - heartbeatRequestManager = createManager(); + subscriptions.subscribe(new HashSet<>(subscribedTopics), new NoOpConsumerRebalanceListener()); + // Update membershipManager's memberId and memberEpoch ConsumerGroupHeartbeatResponse result = new ConsumerGroupHeartbeatResponse(new ConsumerGroupHeartbeatResponseData() @@ -258,12 +227,12 @@ public void testValidateConsumerGroupHeartbeatRequest(final short version) { ConsumerGroupHeartbeatRequest heartbeatRequest = (ConsumerGroupHeartbeatRequest) request.requestBuilder().build(version); - assertEquals(GROUP_ID, heartbeatRequest.data().groupId()); + assertEquals(DEFAULT_GROUP_ID, heartbeatRequest.data().groupId()); assertEquals(memberId, heartbeatRequest.data().memberId()); assertEquals(memberEpoch, heartbeatRequest.data().memberEpoch()); - assertEquals(10000, heartbeatRequest.data().rebalanceTimeoutMs()); + assertEquals(DEFAULT_MAX_POLL_INTERVAL_MS, heartbeatRequest.data().rebalanceTimeoutMs()); assertEquals(subscribedTopics, heartbeatRequest.data().subscribedTopicNames()); - assertEquals(GROUP_INSTANCE_ID, heartbeatRequest.data().instanceId()); + assertEquals(DEFAULT_GROUP_INSTANCE_ID, heartbeatRequest.data().instanceId()); // TODO: Test pattern subscription and user provided assignor selection. assertNull(heartbeatRequest.data().serverAssignor()); assertNull(heartbeatRequest.data().subscribedTopicRegex()); @@ -272,25 +241,16 @@ public void testValidateConsumerGroupHeartbeatRequest(final short version) { @ParameterizedTest @MethodSource("errorProvider") public void testHeartbeatResponseOnErrorHandling(final Errors error, final boolean isFatal) { - heartbeatRequestState = new HeartbeatRequestManager.HeartbeatRequestState( - logContext, - time, - HEARTBEAT_INTERVAL_MS, - RETRY_BACKOFF_MS, - RETRY_BACKOFF_MAX_MS, - 0); - heartbeatRequestManager = createManager(); - // Sending first heartbeat w/o assignment to set the state to STABLE ConsumerGroupHeartbeatResponse rs1 = new ConsumerGroupHeartbeatResponse(new ConsumerGroupHeartbeatResponseData() - .setHeartbeatIntervalMs(HEARTBEAT_INTERVAL_MS) + .setHeartbeatIntervalMs(DEFAULT_HEARTBEAT_INTERVAL_MS) .setMemberId(memberId) .setMemberEpoch(memberEpoch)); membershipManager.updateState(rs1.data()); assertEquals(MemberState.STABLE, membershipManager.state()); // Handling errors on the second heartbeat - time.sleep(HEARTBEAT_INTERVAL_MS); + time.sleep(DEFAULT_HEARTBEAT_INTERVAL_MS); NetworkClientDelegate.PollResult result = heartbeatRequestManager.poll(time.milliseconds()); assertEquals(1, result.unsentRequests.size()); @@ -303,19 +263,19 @@ public void testHeartbeatResponseOnErrorHandling(final Errors error, final boole switch (error) { case NONE: - verify(errorEventHandler, never()).handle(any()); + verify(backgroundEventHandler, never()).add(any()); verify(membershipManager, times(2)).updateState(mockResponse.data()); - assertEquals(HEARTBEAT_INTERVAL_MS, heartbeatRequestState.nextHeartbeatMs(time.milliseconds())); + assertEquals(DEFAULT_HEARTBEAT_INTERVAL_MS, heartbeatRequestState.nextHeartbeatMs(time.milliseconds())); break; case COORDINATOR_LOAD_IN_PROGRESS: - verify(errorEventHandler, never()).handle(any()); - assertEquals(RETRY_BACKOFF_MS, heartbeatRequestState.nextHeartbeatMs(time.milliseconds())); + verify(backgroundEventHandler, never()).add(any()); + assertEquals(DEFAULT_RETRY_BACKOFF_MS, heartbeatRequestState.nextHeartbeatMs(time.milliseconds())); break; case COORDINATOR_NOT_AVAILABLE: case NOT_COORDINATOR: - verify(errorEventHandler, never()).handle(any()); + verify(backgroundEventHandler, never()).add(any()); verify(coordinatorRequestManager).markCoordinatorUnknown(any(), anyLong()); assertEquals(0, heartbeatRequestState.nextHeartbeatMs(time.milliseconds())); break; @@ -325,7 +285,7 @@ public void testHeartbeatResponseOnErrorHandling(final Errors error, final boole // The memberStateManager should have stopped heartbeat at this point ensureFatalError(); } else { - verify(errorEventHandler, never()).handle(any()); + verify(backgroundEventHandler, never()).add(any()); assertEquals(0, heartbeatRequestState.nextHeartbeatMs(time.milliseconds())); } break; @@ -334,12 +294,12 @@ public void testHeartbeatResponseOnErrorHandling(final Errors error, final boole private void ensureFatalError() { verify(membershipManager).transitionToFailed(); - verify(errorEventHandler).handle(any()); + verify(backgroundEventHandler).add(any()); ensureHeartbeatStopped(); } private void ensureHeartbeatStopped() { - time.sleep(HEARTBEAT_INTERVAL_MS); + time.sleep(DEFAULT_HEARTBEAT_INTERVAL_MS); assertEquals(MemberState.FAILED, membershipManager.state()); NetworkClientDelegate.PollResult result = heartbeatRequestManager.poll(time.milliseconds()); assertEquals(0, result.unsentRequests.size()); @@ -376,7 +336,7 @@ private ClientResponse createHeartbeatResponse( final Errors error) { ConsumerGroupHeartbeatResponseData data = new ConsumerGroupHeartbeatResponseData() .setErrorCode(error.code()) - .setHeartbeatIntervalMs(HEARTBEAT_INTERVAL_MS) + .setHeartbeatIntervalMs(DEFAULT_HEARTBEAT_INTERVAL_MS) .setMemberId(memberId) .setMemberEpoch(memberEpoch); if (error != Errors.NONE) { @@ -394,16 +354,4 @@ private ClientResponse createHeartbeatResponse( null, response); } - - private HeartbeatRequestManager createManager() { - return new HeartbeatRequestManager( - logContext, - time, - config, - coordinatorRequestManager, - subscriptionState, - membershipManager, - heartbeatRequestState, - errorEventHandler); - } } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumerTest.java index 4add7f180f674..2ea9917894e51 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumerTest.java @@ -71,14 +71,19 @@ public class PrototypeAsyncConsumerTest { private PrototypeAsyncConsumer consumer; - private static final Optional DEFAULT_GROUP_ID = Optional.of("group.id"); - private ConsumerTestBuilder.PrototypeAsyncConsumerTestBuilder testBuilder; private ApplicationEventHandler applicationEventHandler; @BeforeEach public void setup() { - setup(DEFAULT_GROUP_ID); + // By default, the consumer is part of a group. + setup(ConsumerTestBuilder.createDefaultGroupInformation()); + } + + private void setup(Optional groupInfo) { + testBuilder = new ConsumerTestBuilder.PrototypeAsyncConsumerTestBuilder(groupInfo); + applicationEventHandler = testBuilder.applicationEventHandler; + consumer = testBuilder.consumer; } @AfterEach @@ -88,10 +93,10 @@ public void cleanup() { } } - private void setup(Optional groupIdOpt) { - testBuilder = new ConsumerTestBuilder.PrototypeAsyncConsumerTestBuilder(groupIdOpt); - applicationEventHandler = testBuilder.applicationEventHandler; - consumer = testBuilder.consumer; + private void resetWithEmptyGroupId() { + // Create a consumer that is not configured as part of a group. + cleanup(); + setup(Optional.empty()); } @Test @@ -101,8 +106,8 @@ public void testSuccessfulStartupShutdown() { @Test public void testInvalidGroupId() { - cleanup(); - setup(Optional.empty()); + // Create consumer without group id + resetWithEmptyGroupId(); assertThrows(InvalidGroupIdException.class, () -> consumer.committed(new HashSet<>())); } @@ -336,19 +341,13 @@ public void testRefreshCommittedOffsetsSuccessButNoCommittedOffsetsFound() { @Test public void testRefreshCommittedOffsetsShouldNotResetIfFailedWithTimeout() { - // Create consumer with group id to enable committed offset usage - cleanup(); - setup(Optional.of("consumer-group-1")); - testUpdateFetchPositionsWithFetchCommittedOffsetsTimeout(true); } @Test public void testRefreshCommittedOffsetsNotCalledIfNoGroupId() { // Create consumer without group id so committed offsets are not used for updating positions - cleanup(); - setup(Optional.empty()); - + resetWithEmptyGroupId(); testUpdateFetchPositionsWithFetchCommittedOffsetsTimeout(false); } @@ -383,10 +382,6 @@ private void testRefreshCommittedOffsetsSuccess(Set partitions, Map committedOffsets) { CompletableFuture> committedFuture = new CompletableFuture<>(); committedFuture.complete(committedOffsets); - - // Create consumer with group id to enable committed offset usage - cleanup(); - setup(Optional.of("consumer-group-id")); consumer.assign(partitions); try (MockedConstruction ignored = offsetFetchEventMocker(committedFuture)) { From 281f28ced99f3e74eb6dbfc4bc327fc25ae5a6bd Mon Sep 17 00:00:00 2001 From: Kirk True Date: Mon, 9 Oct 2023 16:58:13 -0700 Subject: [PATCH 37/72] Removed assertion from ConsumerNetworkThread.runOnce to avoid logging --- .../kafka/clients/consumer/internals/ConsumerNetworkThread.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThread.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThread.java index dd72a4e4710ce..99aff8f1f5031 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThread.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThread.java @@ -128,8 +128,6 @@ void initializeResources() { * */ void runOnce() { - closer.assertOpen("Cannot process consumer application events because the network thread is closed"); - // If there are errors processing any events, the error will be thrown immediately. This will have // the effect of closing the background thread. applicationEventProcessor.process(); From 15738bd554544708be4bc7ed5343101e649f782a Mon Sep 17 00:00:00 2001 From: Kirk True Date: Mon, 9 Oct 2023 17:03:40 -0700 Subject: [PATCH 38/72] Updated the fetch response handler method names for clarity --- .../consumer/internals/AbstractFetch.java | 28 +++++++++---------- .../internals/FetchRequestManager.java | 14 ++++------ .../clients/consumer/internals/Fetcher.java | 8 +++--- 3 files changed, 23 insertions(+), 27 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractFetch.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractFetch.java index 7482062b1b157..ffca2cee35dba 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractFetch.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractFetch.java @@ -126,15 +126,15 @@ public boolean hasAvailableFetches() { } /** - * Implements the core logic for a successful fetch request/response. + * Implements the core logic for a successful fetch response. * * @param fetchTarget {@link Node} from which the fetch data was requested * @param data {@link FetchSessionHandler.FetchRequestData} that represents the session data * @param resp {@link ClientResponse} from which the {@link FetchResponse} will be retrieved */ - protected void handleFetchResponse(final Node fetchTarget, - final FetchSessionHandler.FetchRequestData data, - final ClientResponse resp) { + protected void handleFetchSuccess(final Node fetchTarget, + final FetchSessionHandler.FetchRequestData data, + final ClientResponse resp) { try { final FetchResponse response = (FetchResponse) resp.responseBody(); final FetchSessionHandler handler = sessionHandler(fetchTarget.id()); @@ -205,15 +205,15 @@ protected void handleFetchResponse(final Node fetchTarget, } /** - * Implements the core logic for a failed fetch request/response. + * Implements the core logic for a failed fetch response. * * @param fetchTarget {@link Node} from which the fetch data was requested * @param data {@link FetchSessionHandler.FetchRequestData} from request * @param t {@link Throwable} representing the error that resulted in the failure */ - protected void handleFetchResponse(final Node fetchTarget, - final FetchSessionHandler.FetchRequestData data, - final Throwable t) { + protected void handleFetchFailure(final Node fetchTarget, + final FetchSessionHandler.FetchRequestData data, + final Throwable t) { try { final FetchSessionHandler handler = sessionHandler(fetchTarget.id()); @@ -226,17 +226,17 @@ protected void handleFetchResponse(final Node fetchTarget, } } - protected void handleCloseFetchSessionResponse(final Node fetchTarget, - final FetchSessionHandler.FetchRequestData data, - final ClientResponse ignored) { + protected void handleCloseFetchSessionSuccess(final Node fetchTarget, + final FetchSessionHandler.FetchRequestData data, + final ClientResponse ignored) { int sessionId = data.metadata().sessionId(); removePendingFetchRequest(fetchTarget, sessionId); log.debug("Successfully sent a close message for fetch session: {} to node: {}", sessionId, fetchTarget); } - public void handleCloseFetchSessionResponse(final Node fetchTarget, - final FetchSessionHandler.FetchRequestData data, - final Throwable t) { + public void handleCloseFetchSessionFailure(final Node fetchTarget, + final FetchSessionHandler.FetchRequestData data, + final Throwable t) { int sessionId = data.metadata().sessionId(); removePendingFetchRequest(fetchTarget, sessionId); log.debug("Unable to send a close message for fetch session: {} to node: {}. " + diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchRequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchRequestManager.java index 09c4ead0746cb..f98d7b3d9fafe 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchRequestManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchRequestManager.java @@ -22,10 +22,8 @@ import java.util.function.Predicate; import java.util.stream.Collectors; -import org.apache.kafka.clients.ClientRequest; import org.apache.kafka.clients.ClientResponse; import org.apache.kafka.clients.FetchSessionHandler; -import org.apache.kafka.clients.NetworkClient; import org.apache.kafka.clients.consumer.internals.NetworkClientDelegate.PollResult; import org.apache.kafka.clients.consumer.internals.NetworkClientDelegate.UnsentRequest; import org.apache.kafka.common.Node; @@ -71,8 +69,8 @@ protected void maybeThrowAuthFailure(Node node) { public PollResult poll(long currentTimeMs) { return pollInternal( prepareFetchRequests(), - this::handleFetchResponse, - this::handleFetchResponse + this::handleFetchSuccess, + this::handleFetchFailure ); } @@ -83,16 +81,14 @@ public PollResult poll(long currentTimeMs) { public PollResult pollOnClose() { return pollInternal( prepareCloseFetchSessionRequests(), - this::handleCloseFetchSessionResponse, - this::handleCloseFetchSessionResponse + this::handleCloseFetchSessionSuccess, + this::handleCloseFetchSessionFailure ); } /** * Creates the {@link PollResult poll result} that contains a list of zero or more - * {@link FetchRequest.Builder fetch requests} fetch request}, - * {@link NetworkClient#send(ClientRequest, long) enqueues/sends it, and adds the {@link RequestFuture callback} - * for the response. + * {@link FetchRequest.Builder fetch requests}. * * @param fetchRequests {@link Map} of {@link Node nodes} to their {@link FetchSessionHandler.FetchRequestData} * @param successHandler {@link ResponseHandler Handler for successful responses} diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java index 3449790cd3318..05df99373e121 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java @@ -105,12 +105,12 @@ public synchronized int sendFetches() { fetchRequests, (fetchTarget, data, clientResponse) -> { synchronized (Fetcher.this) { - handleFetchResponse(fetchTarget, data, clientResponse); + handleFetchSuccess(fetchTarget, data, clientResponse); } }, (fetchTarget, data, error) -> { synchronized (Fetcher.this) { - handleFetchResponse(fetchTarget, data, error); + handleFetchFailure(fetchTarget, data, error); } }); return fetchRequests.size(); @@ -119,8 +119,8 @@ public synchronized int sendFetches() { protected void maybeCloseFetchSessions(final Timer timer) { final List> requestFutures = sendFetchesInternal( prepareCloseFetchSessionRequests(), - this::handleCloseFetchSessionResponse, - this::handleCloseFetchSessionResponse + this::handleCloseFetchSessionSuccess, + this::handleCloseFetchSessionFailure ); // Poll to ensure that request has been written to the socket. Wait until either the timer has expired or until From 24e014f7d02942a30b2784be8c80d1bd118742c4 Mon Sep 17 00:00:00 2001 From: Kirk True Date: Mon, 9 Oct 2023 17:15:01 -0700 Subject: [PATCH 39/72] ApplicationEventProcessor will now NOT throw errors on event processing exceptions --- .../internals/events/ApplicationEventProcessor.java | 13 +++++++------ .../internals/events/BackgroundEventProcessor.java | 8 ++++---- .../consumer/internals/events/EventProcessor.java | 8 ++++---- .../internals/ConsumerNetworkThreadTest.java | 10 +++++----- .../events/BackgroundEventHandlerTest.java | 6 +++--- 5 files changed, 23 insertions(+), 22 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventProcessor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventProcessor.java index 264858b51bc8b..486b1ff1cf4a9 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventProcessor.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventProcessor.java @@ -26,6 +26,7 @@ import org.apache.kafka.common.PartitionInfo; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.utils.LogContext; +import org.slf4j.Logger; import java.util.List; import java.util.Map; @@ -40,6 +41,7 @@ */ public class ApplicationEventProcessor extends EventProcessor { + private final Logger log; private final ConsumerMetadata metadata; private final RequestManagers requestManagers; @@ -48,20 +50,19 @@ public ApplicationEventProcessor(final LogContext logContext, final RequestManagers requestManagers, final ConsumerMetadata metadata) { super(logContext, applicationEventQueue); + this.log = logContext.logger(ApplicationEventProcessor.class); this.requestManagers = requestManagers; this.metadata = metadata; } /** * Process the events—if any—that were produced by the application thread. It is possible that when processing - * an event generates an error. In such cases, the processor will immediately throw an exception, and not - * process the remaining events. + * an event generates an error. In such cases, the processor will log an exception, but we do not want those + * errors to be propagated to the caller. */ @Override public void process() { - process(error -> { - throw error; - }); + process((event, error) -> { }); } @Override @@ -108,7 +109,7 @@ public void process(ApplicationEvent event) { return; default: - throw new IllegalArgumentException("Application event type " + event.type() + " was not expected"); + log.warn("Application event type " + event.type() + " was not expected"); } } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEventProcessor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEventProcessor.java index c2087c38700bf..cafd4fba492ec 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEventProcessor.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEventProcessor.java @@ -49,11 +49,11 @@ public BackgroundEventProcessor(final LogContext logContext, */ @Override public void process() { - AtomicReference error = new AtomicReference<>(); - process(t -> error.compareAndSet(null, t)); + AtomicReference firstError = new AtomicReference<>(); + process((event, error) -> firstError.compareAndSet(null, error)); - if (error.get() != null) - throw error.get(); + if (firstError.get() != null) + throw firstError.get(); } @Override diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/EventProcessor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/EventProcessor.java index a8ea23e6a8be0..523797d96fbd8 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/EventProcessor.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/EventProcessor.java @@ -57,16 +57,16 @@ public void close() { protected abstract Class getEventClass(); - protected interface ProcessErrorHandler { + protected interface ProcessErrorHandler { - void onError(KafkaException error); + void onError(T event, KafkaException error); } /** * Drains all available events from the queue, and then processes them in order. If any errors are thrown while * processing the individual events, these are submitted to the given {@link ProcessErrorHandler}. */ - protected void process(ProcessErrorHandler processErrorHandler) { + protected void process(ProcessErrorHandler processErrorHandler) { String eventClassName = getEventClass().getSimpleName(); closer.assertOpen(() -> String.format("The processor was previously closed, so no further %s processing can occur", eventClassName)); @@ -89,7 +89,7 @@ protected void process(ProcessErrorHandler processErrorHandler) { else error = new KafkaException(t); - processErrorHandler.onError(error); + processErrorHandler.onError(event, error); } } } finally { diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThreadTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThreadTest.java index 4eff5a905e806..ef77b5c1d1d0a 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThreadTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThreadTest.java @@ -52,9 +52,9 @@ import java.util.concurrent.CompletableFuture; import static org.apache.kafka.clients.consumer.internals.ConsumerTestBuilder.DEFAULT_REQUEST_TIMEOUT_MS; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; @@ -163,13 +163,13 @@ public void testResetPositionsEventIsProcessed() { } @Test - public void testResetPositionsProcessFailureInterruptsConsumerNetworkThread() { + public void testResetPositionsProcessFailureIsIgnored() { TopicAuthorizationException authException = new TopicAuthorizationException("Topic authorization failed"); doThrow(authException).when(offsetsRequestManager).resetPositionsIfNeeded(); ResetPositionsApplicationEvent event = new ResetPositionsApplicationEvent(); applicationEventsQueue.add(event); - assertThrows(TopicAuthorizationException.class, () -> consumerNetworkThread.runOnce()); + assertDoesNotThrow(() -> consumerNetworkThread.runOnce()); verify(applicationEventProcessor).process(any(ResetPositionsApplicationEvent.class)); } @@ -184,13 +184,13 @@ public void testValidatePositionsEventIsProcessed() { } @Test - public void testValidatePositionsProcessFailure() { + public void testValidatePositionsProcessFailureIsIgnored() { LogTruncationException logTruncationException = new LogTruncationException(Collections.emptyMap(), Collections.emptyMap()); doThrow(logTruncationException).when(offsetsRequestManager).validatePositionsIfNeeded(); ValidatePositionsApplicationEvent event = new ValidatePositionsApplicationEvent(); applicationEventsQueue.add(event); - assertThrows(LogTruncationException.class, consumerNetworkThread::runOnce); + assertDoesNotThrow(consumerNetworkThread::runOnce); verify(applicationEventProcessor).process(any(ValidatePositionsApplicationEvent.class)); } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEventHandlerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEventHandlerTest.java index dfd972dc33a99..0670b8bdb7c3f 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEventHandlerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/events/BackgroundEventHandlerTest.java @@ -54,7 +54,7 @@ public void tearDown() { @Test public void testNoEvents() { assertTrue(backgroundEventQueue.isEmpty()); - backgroundEventProcessor.process(ignored -> { }); + backgroundEventProcessor.process((event, error) -> { }); assertTrue(backgroundEventQueue.isEmpty()); } @@ -63,7 +63,7 @@ public void testSingleEvent() { BackgroundEvent event = new ErrorBackgroundEvent(new RuntimeException("A")); backgroundEventQueue.add(event); assertPeeked(event); - backgroundEventProcessor.process(ignored -> { }); + backgroundEventProcessor.process((e, error) -> { }); assertTrue(backgroundEventQueue.isEmpty()); } @@ -84,7 +84,7 @@ public void testMultipleEvents() { backgroundEventQueue.add(new ErrorBackgroundEvent(new RuntimeException("C"))); assertPeeked(event1); - backgroundEventProcessor.process(ignored -> { }); + backgroundEventProcessor.process((event, error) -> { }); assertTrue(backgroundEventQueue.isEmpty()); } From 2eaca7e3ad29891e09f6bec0e809711455f1f016 Mon Sep 17 00:00:00 2001 From: Kirk True Date: Mon, 9 Oct 2023 17:24:02 -0700 Subject: [PATCH 40/72] Removed redundant tests from ConsumerNetworkThreadTest --- .../internals/ConsumerNetworkThreadTest.java | 26 +------------------ 1 file changed, 1 insertion(+), 25 deletions(-) diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThreadTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThreadTest.java index ef77b5c1d1d0a..f556e33c2a7cd 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThreadTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThreadTest.java @@ -17,7 +17,6 @@ package org.apache.kafka.clients.consumer.internals; import org.apache.kafka.clients.MockClient; -import org.apache.kafka.clients.consumer.LogTruncationException; import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.clients.consumer.internals.events.ApplicationEvent; import org.apache.kafka.clients.consumer.internals.events.ApplicationEventProcessor; @@ -30,7 +29,6 @@ import org.apache.kafka.clients.consumer.internals.events.TopicMetadataApplicationEvent; import org.apache.kafka.clients.consumer.internals.events.ValidatePositionsApplicationEvent; import org.apache.kafka.common.TopicPartition; -import org.apache.kafka.common.errors.TopicAuthorizationException; import org.apache.kafka.common.message.FindCoordinatorRequestData; import org.apache.kafka.common.requests.FindCoordinatorRequest; import org.apache.kafka.common.requests.MetadataResponse; @@ -74,7 +72,6 @@ public class ConsumerNetworkThreadTest { private NetworkClientDelegate networkClient; private BlockingQueue applicationEventsQueue; private ApplicationEventProcessor applicationEventProcessor; - private CoordinatorRequestManager coordinatorManager; private OffsetsRequestManager offsetsRequestManager; private CommitRequestManager commitManager; private ConsumerNetworkThread consumerNetworkThread; @@ -89,7 +86,6 @@ public void setup() { client = testBuilder.client; applicationEventsQueue = testBuilder.applicationEventQueue; applicationEventProcessor = testBuilder.applicationEventProcessor; - coordinatorManager = testBuilder.coordinatorRequestManager.orElseThrow(IllegalStateException::new); commitManager = testBuilder.commitRequestManager.orElseThrow(IllegalStateException::new); offsetsRequestManager = testBuilder.offsetsRequestManager; consumerNetworkThread = testBuilder.consumerNetworkThread; @@ -164,8 +160,7 @@ public void testResetPositionsEventIsProcessed() { @Test public void testResetPositionsProcessFailureIsIgnored() { - TopicAuthorizationException authException = new TopicAuthorizationException("Topic authorization failed"); - doThrow(authException).when(offsetsRequestManager).resetPositionsIfNeeded(); + doThrow(new NullPointerException()).when(offsetsRequestManager).resetPositionsIfNeeded(); ResetPositionsApplicationEvent event = new ResetPositionsApplicationEvent(); applicationEventsQueue.add(event); @@ -183,18 +178,6 @@ public void testValidatePositionsEventIsProcessed() { assertTrue(applicationEventsQueue.isEmpty()); } - @Test - public void testValidatePositionsProcessFailureIsIgnored() { - LogTruncationException logTruncationException = new LogTruncationException(Collections.emptyMap(), Collections.emptyMap()); - doThrow(logTruncationException).when(offsetsRequestManager).validatePositionsIfNeeded(); - - ValidatePositionsApplicationEvent event = new ValidatePositionsApplicationEvent(); - applicationEventsQueue.add(event); - assertDoesNotThrow(consumerNetworkThread::runOnce); - - verify(applicationEventProcessor).process(any(ValidatePositionsApplicationEvent.class)); - } - @Test public void testAssignmentChangeEvent() { HashMap offset = mockTopicPartitionOffset(); @@ -210,13 +193,6 @@ public void testAssignmentChangeEvent() { verify(commitManager, times(1)).maybeAutoCommit(offset); } - @Test - void testFindCoordinator() { - consumerNetworkThread.runOnce(); - verify(coordinatorManager, times(1)).poll(anyLong()); - verify(networkClient, times(1)).poll(anyLong(), anyLong()); - } - @Test void testFetchTopicMetadata() { applicationEventsQueue.add(new TopicMetadataApplicationEvent("topic")); From d2ab01ed91006707c613b6621bf94ce4aa9fe09f Mon Sep 17 00:00:00 2001 From: Kirk True Date: Mon, 9 Oct 2023 19:02:24 -0700 Subject: [PATCH 41/72] Now issuing both the poll() and pollOnClose() network I/O on the ConsumerNetworkThread --- .../kafka/clients/consumer/KafkaConsumer.java | 2 +- .../kafka/clients/consumer/MockConsumer.java | 2 +- .../internals/ConsumerNetworkThread.java | 39 +++++++------------ .../consumer/internals/ConsumerUtils.java | 1 + .../internals/PrototypeAsyncConsumer.java | 2 +- .../consumer/internals/RequestManager.java | 26 +++++++------ .../internals/events/EventProcessor.java | 16 ++++---- .../internals/ConsumerNetworkThreadTest.java | 25 +++++++----- 8 files changed, 56 insertions(+), 57 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java index 7ce4c2d367a90..c2d77596c9e64 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java @@ -77,6 +77,7 @@ import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_JMX_PREFIX; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRIC_GROUP_PREFIX; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.DEFAULT_CLOSE_TIMEOUT_MS; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createConsumerNetworkClient; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createFetchMetricsManager; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createLogContext; @@ -567,7 +568,6 @@ public class KafkaConsumer implements Consumer { private static final long NO_CURRENT_THREAD = -1L; - static final long DEFAULT_CLOSE_TIMEOUT_MS = 30 * 1000; static final String DEFAULT_REASON = "rebalance enforced by user"; // Visible for testing diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/MockConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/MockConsumer.java index df6a34737e6ef..25f96a42a1a47 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/MockConsumer.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/MockConsumer.java @@ -45,7 +45,7 @@ import java.util.stream.Collectors; import static java.util.Collections.singleton; -import static org.apache.kafka.clients.consumer.KafkaConsumer.DEFAULT_CLOSE_TIMEOUT_MS; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.DEFAULT_CLOSE_TIMEOUT_MS; /** diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThread.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThread.java index 99aff8f1f5031..9abb8eafd39c4 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThread.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThread.java @@ -21,7 +21,6 @@ import org.apache.kafka.clients.consumer.internals.events.ApplicationEventProcessor; import org.apache.kafka.clients.consumer.internals.events.BackgroundEvent; import org.apache.kafka.common.KafkaException; -import org.apache.kafka.common.errors.InterruptException; import org.apache.kafka.common.errors.WakeupException; import org.apache.kafka.common.internals.IdempotentCloser; import org.apache.kafka.common.requests.AbstractRequest; @@ -35,11 +34,13 @@ import java.time.Duration; import java.util.Collection; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.concurrent.Future; import java.util.function.Supplier; import java.util.stream.Collectors; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.DEFAULT_CLOSE_TIMEOUT_MS; import static org.apache.kafka.common.utils.Utils.closeQuietly; /** @@ -60,6 +61,7 @@ public class ConsumerNetworkThread extends KafkaThread implements Closeable { private RequestManagers requestManagers; private volatile boolean running; private final IdempotentCloser closer = new IdempotentCloser(); + private volatile Duration closeTimeout = Duration.ofMillis(DEFAULT_CLOSE_TIMEOUT_MS); public ConsumerNetworkThread(LogContext logContext, Time time, @@ -96,6 +98,8 @@ public void run() { } catch (final Throwable t) { log.error("The consumer network thread failed due to unexpected error", t); throw new KafkaException(t); + } finally { + cleanup(); } } @@ -206,41 +210,28 @@ public void wakeup() { @Override public void close() { - close(Duration.ZERO); + close(closeTimeout); } public void close(final Duration timeout) { + Objects.requireNonNull(timeout, "Close timeout for consumer network thread must be non-null"); + closer.close( () -> closeInternal(timeout), () -> log.warn("The consumer network thread was already closed") ); } - void closeInternal(final Duration timeout) { - log.trace("Closing the consumer network thread"); - boolean hadStarted = running; + private void closeInternal(final Duration timeout) { + log.trace("Signaling the consumer network thread to close in {}ms", timeout.toMillis()); running = false; + closeTimeout = timeout; wakeup(); + } - Timer timer = time.timer(timeout); - - if (hadStarted && timer.remainingMs() > 0) { - // If the thread had started, we need to wait for the run method to exit. It may take a little time - // for the thread to check the status of the running flag. - // - // We check the value of remainingMs because this method can be called with a duration of 0, which for - // the Thread.join method means "wait forever" which isn't what we want. - try { - long remainingMs = timer.remainingMs(); - log.debug("Waiting up to {} ms for the thread to complete", remainingMs); - join(remainingMs); - } catch (InterruptedException e) { - throw new InterruptException(e); - } finally { - timer.update(); - } - } - + void cleanup() { + log.trace("Closing the consumer network thread"); + Timer timer = time.timer(closeTimeout); runAtClose(requestManagers.entries(), networkClientDelegate, timer); closeQuietly(requestManagers, "request managers"); closeQuietly(networkClientDelegate, "network client delegate"); diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerUtils.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerUtils.java index 01c4d908eebbc..7f42934fa4f7c 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerUtils.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerUtils.java @@ -56,6 +56,7 @@ public final class ConsumerUtils { + public static final long DEFAULT_CLOSE_TIMEOUT_MS = 30 * 1000; public static final String CONSUMER_JMX_PREFIX = "kafka.consumer"; public static final String CONSUMER_METRIC_GROUP_PREFIX = "consumer"; diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java index 9d3087a5b8efa..cbb2de1ef69f3 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java @@ -95,6 +95,7 @@ import static org.apache.kafka.clients.consumer.ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_JMX_PREFIX; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.CONSUMER_METRIC_GROUP_PREFIX; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.DEFAULT_CLOSE_TIMEOUT_MS; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.configuredConsumerInterceptors; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createFetchMetricsManager; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createLogContext; @@ -113,7 +114,6 @@ * for detail implementation. */ public class PrototypeAsyncConsumer implements Consumer { - static final long DEFAULT_CLOSE_TIMEOUT_MS = 30 * 1000; private final ApplicationEventHandler applicationEventHandler; private final Time time; diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManager.java index 80fbeaf3de6d2..f8a0065aa261e 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManager.java @@ -30,14 +30,15 @@ public interface RequestManager { /** * During normal operation of the {@link Consumer}, a request manager may need to send out network requests. * Implementations can return {@link PollResult their need for network I/O} by returning the requests here. - * Because the {@code poll} method is called within the single-threaded context of the consumer's main network - * I/O thread, there should be no need for synchronization protection within itself or other state. + * This method is called within a single-threaded context from + * {@link ConsumerNetworkThread the consumer's network I/O thread}. As such, there should be no need for + * synchronization protection in this method's implementation. * *

    * - * Note: no network I/O occurs in this method. The method itself should not block on I/O or for any - * other reason. This method is called from by the consumer's main network I/O thread. So quick execution of - * this method in all request managers is critical to ensure that we can heartbeat in a timely fashion. + * Note: no network I/O occurs in this method. The method itself should not block for any reason. This + * method is called from by the consumer's network I/O thread, so quick execution of this method in all + * request managers is critical to ensure that we can heartbeat in a timely fashion. * * @param currentTimeMs The current system time at which the method was called; useful for determining if * time-sensitive operations should be performed @@ -46,16 +47,17 @@ public interface RequestManager { /** * On shutdown of the {@link Consumer}, a request manager may need to send out network requests. Implementations - * can signal that by returning the {@link PollResult close} requests here. Unlike {@link #poll(long)}, the - * {@code pollOnClose} method is called from the application thread. Therefore, protection should be made - * when interacting with other any state that could be affected by other threads. + * can signal that by returning the {@link PollResult close} requests here. Like {@link #poll(long)}, this method + * is called within a single-threaded context from {@link ConsumerNetworkThread the consumer's network I/O thread}. + * As such, there should be no need for synchronization protection in this method's implementation. * *

    * - * Note: no network I/O occurs in this method. The method itself should not block on I/O or for any - * other reason. This method is called (indirectly) by the {@link Consumer#close() consumer's close method}. - * So quick execution of this method in all request managers is critical to ensure that we can - * complete as many of the shutdown tasks as possible given the user-provided timeout. + * Note: no network I/O occurs in this method. The method itself should not block for any reason. This + * method is called as an (indirect) result of {@link Consumer#close() the consumer's close method} being invoked. + * (Note that it is still invoked on the consumer's network I/O thread). Quick execution of this method in + * all request managers is critical to ensure that we can complete as many of the consumer's shutdown + * tasks as possible within the user-provided timeout. */ default PollResult pollOnClose() { return EMPTY; diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/EventProcessor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/EventProcessor.java index 523797d96fbd8..06e7ec28dd7e3 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/EventProcessor.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/EventProcessor.java @@ -70,9 +70,10 @@ protected void process(ProcessErrorHandler processErrorHandler) { String eventClassName = getEventClass().getSimpleName(); closer.assertOpen(() -> String.format("The processor was previously closed, so no further %s processing can occur", eventClassName)); + List events = drain(); + try { - List events = drain(); - log.debug("Starting processing of {} {}s", events.size(), eventClassName); + log.debug("Starting processing of {} {}(s)", events.size(), eventClassName); for (T event : events) { try { @@ -93,7 +94,7 @@ protected void process(ProcessErrorHandler processErrorHandler) { } } } finally { - log.debug("Completed processing of {}s", eventClassName); + log.debug("Completed processing of {} {}(s)", events.size(), eventClassName); } } @@ -103,7 +104,7 @@ protected void process(ProcessErrorHandler processErrorHandler) { */ private void closeInternal() { String eventClassName = getEventClass().getSimpleName(); - log.trace("Closing {} processor", eventClassName); + log.trace("Closing event processor for {}", eventClassName); List incompleteEvents = drain(); if (incompleteEvents.isEmpty()) @@ -111,19 +112,18 @@ private void closeInternal() { KafkaException exception = new KafkaException("The consumer is closed"); - // Check each of the events and if it has a Future, complete it exceptionally. + // Check each of the events and if it has a Future that is incomplete, complete it exceptionally. incompleteEvents .stream() .filter(e -> e instanceof CompletableEvent) .map(e -> ((CompletableEvent) e).future()) + .filter(f -> !f.isDone()) .forEach(f -> { log.debug("Completing {} with exception {}", f, exception.getMessage()); f.completeExceptionally(exception); }); - log.debug("Discarding {} {}s because the consumer is closing", - incompleteEvents.size(), - eventClassName); + log.debug("Discarding {} {}s because the consumer is closing", incompleteEvents.size(), eventClassName); } /** diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThreadTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThreadTest.java index f556e33c2a7cd..093dd1ec64d34 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThreadTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThreadTest.java @@ -34,6 +34,7 @@ import org.apache.kafka.common.requests.MetadataResponse; import org.apache.kafka.common.requests.RequestTestUtils; import org.apache.kafka.common.utils.Time; +import org.apache.kafka.test.TestCondition; import org.apache.kafka.test.TestUtils; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -50,6 +51,7 @@ import java.util.concurrent.CompletableFuture; import static org.apache.kafka.clients.consumer.internals.ConsumerTestBuilder.DEFAULT_REQUEST_TIMEOUT_MS; +import static org.apache.kafka.test.TestUtils.DEFAULT_MAX_WAIT_MS; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -100,19 +102,22 @@ public void tearDown() { @Test public void testStartupAndTearDown() throws InterruptedException { + // The consumer is closed in ConsumerTestBuilder.ConsumerNetworkThreadTestBuilder.close() + // which is called from tearDown(). consumerNetworkThread.start(); - TestUtils.waitForCondition(consumerNetworkThread::isRunning, "Failed awaiting for the background thread to be running"); + + TestCondition isStarted = () -> consumerNetworkThread.isRunning(); + TestCondition isClosed = () ->!(consumerNetworkThread.isRunning() || consumerNetworkThread.isAlive()); // There's a nonzero amount of time between starting the thread and having it // begin to execute our code. Wait for a bit before checking... - int maxWaitMs = 1000; - TestUtils.waitForCondition(consumerNetworkThread::isRunning, - maxWaitMs, - "Thread did not start within " + maxWaitMs + " ms"); - consumerNetworkThread.close(Duration.ofMillis(maxWaitMs)); - TestUtils.waitForCondition(() -> !consumerNetworkThread.isRunning(), - maxWaitMs, - "Thread did not stop within " + maxWaitMs + " ms"); + TestUtils.waitForCondition(isStarted, + "The consumer network thread did not start within " + DEFAULT_MAX_WAIT_MS + " ms"); + + consumerNetworkThread.close(Duration.ofMillis(DEFAULT_MAX_WAIT_MS)); + + TestUtils.waitForCondition(isClosed, + "The consumer network thread did not stop within " + DEFAULT_MAX_WAIT_MS + " ms"); } @Test @@ -249,7 +254,7 @@ void testEnsureEventsAreCompleted() { assertFalse(future.isDone()); assertFalse(applicationEventsQueue.isEmpty()); - consumerNetworkThread.close(); + consumerNetworkThread.cleanup(); assertTrue(future.isCompletedExceptionally()); assertTrue(applicationEventsQueue.isEmpty()); } From 19327fa90785af3393460051331aa90abc162301 Mon Sep 17 00:00:00 2001 From: Kirk True Date: Mon, 9 Oct 2023 19:04:29 -0700 Subject: [PATCH 42/72] Fixed minor whitespace checkstyle violation in ConsumerNetworkThreadTest --- .../clients/consumer/internals/ConsumerNetworkThreadTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThreadTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThreadTest.java index 093dd1ec64d34..ef1ad00503dab 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThreadTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThreadTest.java @@ -107,7 +107,7 @@ public void testStartupAndTearDown() throws InterruptedException { consumerNetworkThread.start(); TestCondition isStarted = () -> consumerNetworkThread.isRunning(); - TestCondition isClosed = () ->!(consumerNetworkThread.isRunning() || consumerNetworkThread.isAlive()); + TestCondition isClosed = () -> !(consumerNetworkThread.isRunning() || consumerNetworkThread.isAlive()); // There's a nonzero amount of time between starting the thread and having it // begin to execute our code. Wait for a bit before checking... From 621fc7d95fbb3ee111479420f23cf6996b882a6d Mon Sep 17 00:00:00 2001 From: Kirk True Date: Tue, 10 Oct 2023 09:31:29 -0700 Subject: [PATCH 43/72] Fixed typo in RequestManager comment --- .../apache/kafka/clients/consumer/internals/RequestManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManager.java index f8a0065aa261e..8592035dda27d 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManager.java @@ -37,7 +37,7 @@ public interface RequestManager { *

    * * Note: no network I/O occurs in this method. The method itself should not block for any reason. This - * method is called from by the consumer's network I/O thread, so quick execution of this method in all + * method is called from the consumer's network I/O thread, so quick execution of this method in all * request managers is critical to ensure that we can heartbeat in a timely fashion. * * @param currentTimeMs The current system time at which the method was called; useful for determining if From 5a8037111fdfdbe3275baf0ee6ed9755caee1a59 Mon Sep 17 00:00:00 2001 From: Kirk True Date: Fri, 13 Oct 2023 11:16:20 -0700 Subject: [PATCH 44/72] Comments change to remove close() method in testFetcherCloseClosesFetchSessionsInBroker --- .../clients/consumer/internals/FetchRequestManagerTest.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java index bc9454e6c3c17..739c6c27faca0 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java @@ -309,11 +309,9 @@ public void testFetcherCloseClosesFetchSessionsInBroker() { // send request to close the fetcher Timer timer = time.timer(Duration.ofSeconds(10)); - // fetcher.close(timer); - // // NOTE: by design the FetchRequestManager doesn't perform network I/O internally. That means that calling - // close with a Timer will NOT send out the close session requests on close. The network I/O logic is - // handled inside ConsumerNetworkThread.runAtClose, so we need to run that logic here. + // the close() method with a Timer will NOT send out the close session requests on close. The network + // I/O logic is handled inside ConsumerNetworkThread.runAtClose, so we need to run that logic here. ConsumerNetworkThread.runAtClose(singletonList(Optional.of(fetcher)), networkClientDelegate, timer); NetworkClientDelegate.PollResult pollResult = fetcher.poll(time.milliseconds()); From 2ba9e8576dcfa689eb0ba5006ecb47feaa98644a Mon Sep 17 00:00:00 2001 From: Kirk True Date: Fri, 13 Oct 2023 11:18:43 -0700 Subject: [PATCH 45/72] Removed superfluous poll() from testFetcherCloseClosesFetchSessionsInBroker after runOnClose called --- .../consumer/internals/FetchRequestManagerTest.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java index 739c6c27faca0..d8e6b3e1454a1 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java @@ -314,12 +314,8 @@ public void testFetcherCloseClosesFetchSessionsInBroker() { // I/O logic is handled inside ConsumerNetworkThread.runAtClose, so we need to run that logic here. ConsumerNetworkThread.runAtClose(singletonList(Optional.of(fetcher)), networkClientDelegate, timer); - NetworkClientDelegate.PollResult pollResult = fetcher.poll(time.milliseconds()); - networkClientDelegate.addAll(pollResult.unsentRequests); - networkClientDelegate.poll(timer); - - // validate that Fetcher.close() has sent a request with final epoch. 2 requests are sent, one for the normal - // fetch earlier and another for the finish fetch here. + // validate that closing the fetcher has sent a request with final epoch. 2 requests are sent, one for the + // normal fetch earlier and another for the finish fetch here. verify(networkClientDelegate, times(2)).doSend(argument.capture(), any(Long.class)); NetworkClientDelegate.UnsentRequest unsentRequest = argument.getValue(); FetchRequest.Builder builder = (FetchRequest.Builder) unsentRequest.requestBuilder(); From 627624c23d41234d0dc77fd773afa9d91328307d Mon Sep 17 00:00:00 2001 From: Kirk True Date: Fri, 13 Oct 2023 11:44:36 -0700 Subject: [PATCH 46/72] Updated method names in FetcherTest and FetchRequestManagerTest fetchedRecords() calls fetcher.collectFetch(), but that wasn't very clear. I updated the call sites and changed fetchedRecords() to just fetchRecords(). --- .../internals/FetchRequestManagerTest.java | 176 +++++++++--------- .../consumer/internals/FetcherTest.java | 163 ++++++++-------- 2 files changed, 170 insertions(+), 169 deletions(-) diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java index d8e6b3e1454a1..f1458067705ac 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java @@ -249,7 +249,7 @@ public void testFetchNormal() { networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - Map>> partitionRecords = fetchedRecords(); + Map>> partitionRecords = fetchRecords(); assertTrue(partitionRecords.containsKey(tp0)); List> records = partitionRecords.get(tp0); @@ -274,7 +274,7 @@ public void testInflightFetchOnPendingPartitions() { client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); networkClientDelegate.poll(time.timer(0)); - assertNull(fetchedRecords().get(tp0)); + assertNull(fetchRecords().get(tp0)); } @Test @@ -338,7 +338,7 @@ public void testFetchingPendingPartitions() { client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - fetchedRecords(); + fetchRecords(); assertEquals(4L, subscriptions.position(tp0).offset); // this is the next fetching position // mark partition unfetchable @@ -346,7 +346,7 @@ public void testFetchingPendingPartitions() { assertEquals(0, sendFetches()); networkClientDelegate.poll(time.timer(0)); assertFalse(fetcher.hasCompletedFetches()); - fetchedRecords(); + fetchRecords(); assertEquals(4L, subscriptions.position(tp0).offset); } @@ -359,10 +359,10 @@ public void testFetchWithNoTopicId() { assignFromUser(noId.topicPartition()); subscriptions.seek(noId.topicPartition(), 0); - // Fetch should use request version 12 assertEquals(1, sendFetches()); assertFalse(fetcher.hasCompletedFetches()); + // Fetch should use request version 12 client.prepareResponse( fetchRequestMatcher((short) 12, noId, 0, Optional.of(validLeaderEpoch)), fullFetchResponse(noId, records, Errors.NONE, 100L, 0) @@ -370,7 +370,7 @@ public void testFetchWithNoTopicId() { networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - Map>> partitionRecords = fetchedRecords(); + Map>> partitionRecords = fetchRecords(); assertTrue(partitionRecords.containsKey(noId.topicPartition())); List> records = partitionRecords.get(noId.topicPartition()); @@ -402,7 +402,7 @@ public void testFetchWithTopicId() { networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - Map>> partitionRecords = fetchedRecords(); + Map>> partitionRecords = fetchRecords(); assertTrue(partitionRecords.containsKey(tp.topicPartition())); List> records = partitionRecords.get(tp.topicPartition()); @@ -427,9 +427,9 @@ public void testFetchForgetTopicIdWhenUnassigned() { client.updateMetadata(RequestTestUtils.metadataUpdateWithIds(1, singleton(foo), tp -> validLeaderEpoch)); subscriptions.seek(foo.topicPartition(), 0); - // Fetch should use latest version. assertEquals(1, sendFetches()); + // Fetch should use latest version. client.prepareResponse( fetchRequestMatcher(ApiKeys.FETCH.latestVersion(), singletonMap(foo, new PartitionData( @@ -445,9 +445,9 @@ public void testFetchForgetTopicIdWhenUnassigned() { ); networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - fetchedRecords(); + fetchRecords(); - // Assign bar and unassign foo. + // Assign bar and un-assign foo. subscriptions.assignFromUser(singleton(bar.topicPartition())); client.updateMetadata(RequestTestUtils.metadataUpdateWithIds(1, singleton(bar), tp -> validLeaderEpoch)); subscriptions.seek(bar.topicPartition(), 0); @@ -471,7 +471,7 @@ public void testFetchForgetTopicIdWhenUnassigned() { ); networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - fetchedRecords(); + fetchRecords(); } @Test @@ -504,7 +504,7 @@ public void testFetchForgetTopicIdWhenReplaced() { ); networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - fetchedRecords(); + fetchRecords(); // Replace foo with old topic id with foo with new topic id. subscriptions.assignFromUser(singleton(fooWithNewTopicId.topicPartition())); @@ -531,7 +531,7 @@ public void testFetchForgetTopicIdWhenReplaced() { ); networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - fetchedRecords(); + fetchRecords(); } @Test @@ -539,7 +539,6 @@ public void testFetchTopicIdUpgradeDowngrade() { buildFetcher(); TopicIdPartition fooWithoutId = new TopicIdPartition(Uuid.ZERO_UUID, new TopicPartition("foo", 0)); - TopicIdPartition fooWithId = new TopicIdPartition(Uuid.randomUuid(), new TopicPartition("foo", 0)); // Assign foo without a topic id. subscriptions.assignFromUser(singleton(fooWithoutId.topicPartition())); @@ -564,9 +563,10 @@ public void testFetchTopicIdUpgradeDowngrade() { ); networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - fetchedRecords(); + fetchRecords(); // Upgrade. + TopicIdPartition fooWithId = new TopicIdPartition(Uuid.randomUuid(), new TopicPartition("foo", 0)); subscriptions.assignFromUser(singleton(fooWithId.topicPartition())); client.updateMetadata(RequestTestUtils.metadataUpdateWithIds(1, singleton(fooWithId), tp -> validLeaderEpoch)); subscriptions.seek(fooWithId.topicPartition(), 0); @@ -591,7 +591,7 @@ public void testFetchTopicIdUpgradeDowngrade() { ); networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - fetchedRecords(); + fetchRecords(); // Downgrade. subscriptions.assignFromUser(singleton(fooWithoutId.topicPartition())); @@ -618,7 +618,7 @@ public void testFetchTopicIdUpgradeDowngrade() { ); networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - fetchedRecords(); + fetchRecords(); } private MockClient.RequestMatcher fetchRequestMatcher( @@ -687,7 +687,7 @@ public void testMissingLeaderEpochInRecords() { networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - Map>> partitionRecords = fetchedRecords(); + Map>> partitionRecords = fetchRecords(); assertTrue(partitionRecords.containsKey(tp0)); assertEquals(2, partitionRecords.get(tp0).size()); @@ -738,7 +738,7 @@ public void testLeaderEpochInConsumerRecord() { networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - Map>> partitionRecords = fetchedRecords(); + Map>> partitionRecords = fetchRecords(); assertTrue(partitionRecords.containsKey(tp0)); assertEquals(6, partitionRecords.get(tp0).size()); @@ -815,7 +815,7 @@ public void testFetcherIgnoresControlRecords() { networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - Map>> partitionRecords = fetchedRecords(); + Map>> partitionRecords = fetchRecords(); assertTrue(partitionRecords.containsKey(tp0)); List> records = partitionRecords.get(tp0); @@ -840,7 +840,7 @@ public void testFetchError() { networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - Map>> partitionRecords = fetchedRecords(); + Map>> partitionRecords = fetchRecords(); assertFalse(partitionRecords.containsKey(tp0)); } @@ -882,7 +882,7 @@ public byte[] deserialize(String topic, byte[] data) { // The fetcher should block on Deserialization error for (int i = 0; i < 2; i++) { try { - fetcher.collectFetch(); + collectFetch(); fail("fetchedRecords should have raised"); } catch (SerializationException e) { // the position should not advance since no data has been returned @@ -943,7 +943,7 @@ public void testParseCorruptedRecord() throws Exception { networkClientDelegate.poll(time.timer(0)); // the first fetchedRecords() should return the first valid message - assertEquals(1, fetchedRecords().get(tp0).size()); + assertEquals(1, fetchRecords().get(tp0).size()); assertEquals(1, subscriptions.position(tp0).offset); ensureBlockOnRecord(1L); @@ -963,7 +963,7 @@ private void ensureBlockOnRecord(long blockedOffset) { // the fetchedRecords() should always throw exception due to the invalid message at the starting offset. for (int i = 0; i < 2; i++) { try { - fetcher.collectFetch(); + fetchRecords(); fail("fetchedRecords should have raised KafkaException"); } catch (KafkaException e) { assertEquals(blockedOffset, subscriptions.position(tp0).offset); @@ -975,12 +975,12 @@ private void seekAndConsumeRecord(ByteBuffer responseBuffer, long toOffset) { // Seek to skip the bad record and fetch again. subscriptions.seekUnvalidated(tp0, new SubscriptionState.FetchPosition(toOffset, Optional.empty(), metadata.currentLeader(tp0))); // Should not throw exception after the seek. - fetcher.collectFetch(); + collectFetch(); assertEquals(1, sendFetches()); client.prepareResponse(fullFetchResponse(tidp0, MemoryRecords.readableRecords(responseBuffer), Errors.NONE, 100L, 0)); networkClientDelegate.poll(time.timer(0)); - Map>> recordsByPartition = fetchedRecords(); + Map>> recordsByPartition = fetchRecords(); List> records = recordsByPartition.get(tp0); assertEquals(1, records.size()); assertEquals(toOffset, records.get(0).offset()); @@ -1019,7 +1019,7 @@ public void testInvalidDefaultRecordBatch() { // the fetchedRecords() should always throw exception due to the bad batch. for (int i = 0; i < 2; i++) { try { - fetcher.collectFetch(); + collectFetch(); fail("fetchedRecords should have raised KafkaException"); } catch (KafkaException e) { assertEquals(0, subscriptions.position(tp0).offset); @@ -1048,7 +1048,7 @@ public void testParseInvalidRecordBatch() { client.prepareResponse(fullFetchResponse(tidp0, MemoryRecords.readableRecords(buffer), Errors.NONE, 100L, 0)); networkClientDelegate.poll(time.timer(0)); try { - fetcher.collectFetch(); + collectFetch(); fail("fetchedRecords should have raised"); } catch (KafkaException e) { // the position should not advance since no data has been returned @@ -1082,7 +1082,7 @@ public void testHeaders() { assertEquals(1, sendFetches()); networkClientDelegate.poll(time.timer(0)); - Map>> recordsByPartition = fetchedRecords(); + Map>> recordsByPartition = fetchRecords(); records = recordsByPartition.get(tp0); assertEquals(3, records.size()); @@ -1114,7 +1114,7 @@ public void testFetchMaxPollRecords() { assertEquals(1, sendFetches()); networkClientDelegate.poll(time.timer(0)); - Map>> recordsByPartition = fetchedRecords(); + Map>> recordsByPartition = fetchRecords(); records = recordsByPartition.get(tp0); assertEquals(2, records.size()); assertEquals(3L, subscriptions.position(tp0).offset); @@ -1123,7 +1123,7 @@ public void testFetchMaxPollRecords() { assertEquals(0, sendFetches()); networkClientDelegate.poll(time.timer(0)); - recordsByPartition = fetchedRecords(); + recordsByPartition = fetchRecords(); records = recordsByPartition.get(tp0); assertEquals(1, records.size()); assertEquals(4L, subscriptions.position(tp0).offset); @@ -1131,7 +1131,7 @@ public void testFetchMaxPollRecords() { assertTrue(sendFetches() > 0); networkClientDelegate.poll(time.timer(0)); - recordsByPartition = fetchedRecords(); + recordsByPartition = fetchRecords(); records = recordsByPartition.get(tp0); assertEquals(2, records.size()); assertEquals(6L, subscriptions.position(tp0).offset); @@ -1157,7 +1157,7 @@ public void testFetchAfterPartitionWithFetchedRecordsIsUnassigned() { assertEquals(1, sendFetches()); networkClientDelegate.poll(time.timer(0)); - Map>> recordsByPartition = fetchedRecords(); + Map>> recordsByPartition = fetchRecords(); records = recordsByPartition.get(tp0); assertEquals(2, records.size()); assertEquals(3L, subscriptions.position(tp0).offset); @@ -1170,7 +1170,7 @@ public void testFetchAfterPartitionWithFetchedRecordsIsUnassigned() { assertEquals(1, sendFetches()); networkClientDelegate.poll(time.timer(0)); - Map>> fetchedRecords = fetchedRecords(); + Map>> fetchedRecords = fetchRecords(); assertNull(fetchedRecords.get(tp0)); records = fetchedRecords.get(tp1); assertEquals(2, records.size()); @@ -1200,7 +1200,7 @@ public void testFetchNonContinuousRecords() { assertEquals(1, sendFetches()); client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); networkClientDelegate.poll(time.timer(0)); - Map>> recordsByPartition = fetchedRecords(); + Map>> recordsByPartition = fetchRecords(); consumerRecords = recordsByPartition.get(tp0); assertEquals(3, consumerRecords.size()); assertEquals(31L, subscriptions.position(tp0).offset); // this is the next fetching position @@ -1222,7 +1222,7 @@ public void testFetchRequestWhenRecordTooLarge() { client.setNodeApiVersions(NodeApiVersions.create(ApiKeys.FETCH.id, (short) 2, (short) 2)); makeFetchRequestWithIncompleteRecord(); try { - fetcher.collectFetch(); + collectFetch(); fail("RecordTooLargeException should have been raised"); } catch (RecordTooLargeException e) { assertTrue(e.getMessage().startsWith("There are some messages at [Partition=Offset]: ")); @@ -1245,7 +1245,7 @@ public void testFetchRequestInternalError() { buildFetcher(); makeFetchRequestWithIncompleteRecord(); try { - fetcher.collectFetch(); + collectFetch(); fail("RecordTooLargeException should have been raised"); } catch (KafkaException e) { assertTrue(e.getMessage().startsWith("Failed to make progress reading messages")); @@ -1278,7 +1278,7 @@ public void testUnauthorizedTopic() { client.prepareResponse(fullFetchResponse(tidp0, records, Errors.TOPIC_AUTHORIZATION_FAILED, 100L, 0)); networkClientDelegate.poll(time.timer(0)); try { - fetcher.collectFetch(); + collectFetch(); fail("fetchedRecords should have thrown"); } catch (TopicAuthorizationException e) { assertEquals(singleton(topicName), e.unauthorizedTopics()); @@ -1306,7 +1306,7 @@ public void testFetchDuringEagerRebalance() { networkClientDelegate.poll(time.timer(0)); // The active fetch should be ignored since its position is no longer valid - assertTrue(fetchedRecords().isEmpty()); + assertTrue(fetchRecords().isEmpty()); } @Test @@ -1328,7 +1328,7 @@ public void testFetchDuringCooperativeRebalance() { client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); networkClientDelegate.poll(time.timer(0)); - Map>> fetchedRecords = fetchedRecords(); + Map>> fetchedRecords = fetchRecords(); // The active fetch should NOT be ignored since the position for tp0 is still valid assertEquals(1, fetchedRecords.size()); @@ -1347,7 +1347,7 @@ public void testInFlightFetchOnPausedPartition() { client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); networkClientDelegate.poll(time.timer(0)); - assertNull(fetchedRecords().get(tp0)); + assertNull(fetchRecords().get(tp0)); } @Test @@ -1387,7 +1387,7 @@ public void testFetchOnCompletedFetchesForPausedAndResumedPartitions() { assertTrue(fetcher.hasAvailableFetches(), "Should have available (non-paused) completed fetches"); networkClientDelegate.poll(time.timer(0)); - Map>> fetchedRecords = fetchedRecords(); + Map>> fetchedRecords = fetchRecords(); assertEquals(1, fetchedRecords.size(), "Should return records when partition is resumed"); assertNotNull(fetchedRecords.get(tp0)); assertEquals(3, fetchedRecords.get(tp0).size()); @@ -1421,7 +1421,7 @@ public void testFetchOnCompletedFetchesForSomePausedPartitions() { subscriptions.pause(tp0); networkClientDelegate.poll(time.timer(0)); - fetchedRecords = fetchedRecords(); + fetchedRecords = fetchRecords(); assertEquals(1, fetchedRecords.size(), "Should return completed fetch for unpaused partitions"); assertTrue(fetcher.hasCompletedFetches(), "Should still contain completed fetches"); assertNotNull(fetchedRecords.get(tp1)); @@ -1476,7 +1476,7 @@ public void testPartialFetchWithPausedPartitions() { client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); networkClientDelegate.poll(time.timer(0)); - fetchedRecords = fetchedRecords(); + fetchedRecords = fetchRecords(); assertEquals(2, fetchedRecords.get(tp0).size(), "Should return 2 records from fetch with 3 records"); assertFalse(fetcher.hasCompletedFetches(), "Should have no completed fetches"); @@ -1484,7 +1484,7 @@ public void testPartialFetchWithPausedPartitions() { subscriptions.pause(tp0); networkClientDelegate.poll(time.timer(0)); - fetchedRecords = fetchedRecords(); + fetchedRecords = fetchRecords(); assertEmptyFetch("Should not return records or advance position for paused partitions"); assertTrue(fetcher.hasCompletedFetches(), "Should have 1 entry in completed fetches"); @@ -1494,7 +1494,7 @@ public void testPartialFetchWithPausedPartitions() { networkClientDelegate.poll(time.timer(0)); - fetchedRecords = fetchedRecords(); + fetchedRecords = fetchRecords(); assertEquals(1, fetchedRecords.get(tp0).size(), "Should return last remaining record"); assertFalse(fetcher.hasCompletedFetches(), "Should have no completed fetches"); @@ -1706,7 +1706,7 @@ public void testFetchOffsetOutOfRangeException() { assertFalse(subscriptions.isOffsetResetNeeded(tp0)); for (int i = 0; i < 2; i++) { OffsetOutOfRangeException e = assertThrows(OffsetOutOfRangeException.class, () -> - fetcher.collectFetch()); + collectFetch()); assertEquals(singleton(tp0), e.offsetOutOfRangePartitions().keySet()); assertEquals(0L, e.offsetOutOfRangePartitions().get(tp0).longValue()); } @@ -1755,7 +1755,7 @@ public void testFetchPositionAfterException() { } private void fetchRecordsInto(List> allFetchedRecords) { - Map>> fetchedRecords = fetchedRecords(); + Map>> fetchedRecords = fetchRecords(); fetchedRecords.values().forEach(allFetchedRecords::addAll); } @@ -1798,7 +1798,7 @@ public void testCompletedFetchRemoval() { networkClientDelegate.poll(time.timer(0)); List> fetchedRecords = new ArrayList<>(); - Map>> recordsByPartition = fetchedRecords(); + Map>> recordsByPartition = fetchRecords(); for (List> records : recordsByPartition.values()) fetchedRecords.addAll(records); @@ -1808,7 +1808,7 @@ public void testCompletedFetchRemoval() { List oorExceptions = new ArrayList<>(); try { - recordsByPartition = fetchedRecords(); + recordsByPartition = fetchRecords(); for (List> records : recordsByPartition.values()) fetchedRecords.addAll(records); } catch (OffsetOutOfRangeException oor) { @@ -1821,7 +1821,7 @@ public void testCompletedFetchRemoval() { assertTrue(oor.offsetOutOfRangePartitions().containsKey(tp0)); assertEquals(oor.offsetOutOfRangePartitions().size(), 1); - recordsByPartition = fetchedRecords(); + recordsByPartition = fetchRecords(); for (List> records : recordsByPartition.values()) fetchedRecords.addAll(records); @@ -1833,7 +1833,7 @@ public void testCompletedFetchRemoval() { List kafkaExceptions = new ArrayList<>(); for (int i = 1; i <= numExceptionsExpected; i++) { try { - recordsByPartition = fetchedRecords(); + recordsByPartition = fetchRecords(); for (List> records : recordsByPartition.values()) fetchedRecords.addAll(records); } catch (KafkaException e) { @@ -1860,7 +1860,7 @@ public void testSeekBeforeException() { client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); networkClientDelegate.poll(time.timer(0)); - assertEquals(2, fetchedRecords().get(tp0).size()); + assertEquals(2, fetchRecords().get(tp0).size()); subscriptions.assignFromUser(mkSet(tp0, tp1)); subscriptions.seekUnvalidated(tp1, new SubscriptionState.FetchPosition(1, Optional.empty(), metadata.currentLeader(tp1))); @@ -1873,7 +1873,7 @@ public void testSeekBeforeException() { .setHighWatermark(100)); client.prepareResponse(FetchResponse.of(Errors.NONE, 0, INVALID_SESSION_ID, new LinkedHashMap<>(partitions))); networkClientDelegate.poll(time.timer(0)); - assertEquals(1, fetchedRecords().get(tp0).size()); + assertEquals(1, fetchRecords().get(tp0).size()); subscriptions.seek(tp1, 10); // Should not throw OffsetOutOfRangeException after the seek @@ -2121,7 +2121,7 @@ public void testFetchResponseMetrics() { client.prepareResponse(FetchResponse.of(Errors.NONE, 0, INVALID_SESSION_ID, fetchPartitionData)); networkClientDelegate.poll(time.timer(0)); - Map>> fetchedRecords = fetchedRecords(); + Map>> fetchedRecords = fetchRecords(); assertEquals(3, fetchedRecords.get(tp1).size()); assertEquals(3, fetchedRecords.get(tp2).size()); @@ -2192,7 +2192,7 @@ public void testFetchResponseMetricsWithOnePartitionError() { assertEquals(1, sendFetches()); client.prepareResponse(FetchResponse.of(Errors.NONE, 0, INVALID_SESSION_ID, new LinkedHashMap<>(partitions))); networkClientDelegate.poll(time.timer(0)); - fetcher.collectFetch(); + collectFetch(); int expectedBytes = 0; for (Record record : records.records()) @@ -2238,7 +2238,7 @@ public void testFetchResponseMetricsWithOnePartitionAtTheWrongOffset() { client.prepareResponse(FetchResponse.of(Errors.NONE, 0, INVALID_SESSION_ID, new LinkedHashMap<>(partitions))); networkClientDelegate.poll(time.timer(0)); - fetcher.collectFetch(); + collectFetch(); // we should have ignored the record at the wrong offset int expectedBytes = 0; @@ -2262,7 +2262,7 @@ public void testFetcherMetricsTemplates() { client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - Map>> partitionRecords = fetchedRecords(); + Map>> partitionRecords = fetchRecords(); assertTrue(partitionRecords.containsKey(tp0)); // Verify that all metrics except metrics-count have registered templates @@ -2285,7 +2285,7 @@ private Map>> fetchRecords( assertEquals(1, sendFetches()); client.prepareResponse(fullFetchResponse(tp, records, error, hw, lastStableOffset, throttleTime)); networkClientDelegate.poll(time.timer(0)); - return fetchedRecords(); + return fetchRecords(); } private Map>> fetchRecords( @@ -2293,7 +2293,7 @@ private Map>> fetchRecords( assertEquals(1, sendFetches()); client.prepareResponse(fetchResponse(tp, records, error, hw, lastStableOffset, logStartOffset, throttleTime)); networkClientDelegate.poll(time.timer(0)); - return fetchedRecords(); + return fetchRecords(); } @Test @@ -2362,7 +2362,7 @@ public void testReturnCommittedTransactions() { networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - Map>> fetchedRecords = fetchedRecords(); + Map>> fetchedRecords = fetchRecords(); assertTrue(fetchedRecords.containsKey(tp0)); assertEquals(fetchedRecords.get(tp0).size(), 2); } @@ -2432,7 +2432,7 @@ public void testReadCommittedWithCommittedAndAbortedTransactions() { networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - Map>> fetchedRecords = fetchedRecords(); + Map>> fetchedRecords = fetchRecords(); assertTrue(fetchedRecords.containsKey(tp0)); // There are only 3 committed records List> fetchedConsumerRecords = fetchedRecords.get(tp0); @@ -2480,7 +2480,7 @@ public void testMultipleAbortMarkers() { networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - Map>> fetchedRecords = fetchedRecords(); + Map>> fetchedRecords = fetchRecords(); assertTrue(fetchedRecords.containsKey(tp0)); assertEquals(fetchedRecords.get(tp0).size(), 2); List> fetchedConsumerRecords = fetchedRecords.get(tp0); @@ -2525,7 +2525,7 @@ public void testReadCommittedAbortMarkerWithNoData() { networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - Map>> allFetchedRecords = fetchedRecords(); + Map>> allFetchedRecords = fetchRecords(); assertTrue(allFetchedRecords.containsKey(tp0)); List> fetchedRecords = allFetchedRecords.get(tp0); assertEquals(3, fetchedRecords.size()); @@ -2564,7 +2564,7 @@ protected boolean shouldRetainRecord(RecordBatch recordBatch, Record record) { networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - Map>> allFetchedRecords = fetchedRecords(); + Map>> allFetchedRecords = fetchRecords(); assertTrue(allFetchedRecords.containsKey(tp0)); List> fetchedRecords = allFetchedRecords.get(tp0); assertEquals(3, fetchedRecords.size()); @@ -2665,7 +2665,7 @@ public void testReadCommittedWithCompactedTopic() { networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - Map>> allFetchedRecords = fetchedRecords(); + Map>> allFetchedRecords = fetchRecords(); assertTrue(allFetchedRecords.containsKey(tp0)); List> fetchedRecords = allFetchedRecords.get(tp0); assertEquals(5, fetchedRecords.size()); @@ -2702,7 +2702,7 @@ public void testReturnAbortedTransactionsinUncommittedMode() { networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - Map>> fetchedRecords = fetchedRecords(); + Map>> fetchedRecords = fetchRecords(); assertTrue(fetchedRecords.containsKey(tp0)); } @@ -2735,7 +2735,7 @@ public void testConsumerPositionUpdatedWhenSkippingAbortedTransactions() { networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - Map>> fetchedRecords = fetchedRecords(); + Map>> fetchedRecords = fetchRecords(); // Ensure that we don't return any of the aborted records, but yet advance the consumer position. assertFalse(fetchedRecords.containsKey(tp0)); @@ -2770,7 +2770,7 @@ public void testConsumingViaIncrementalFetchRequests() { assertFalse(fetcher.hasCompletedFetches()); networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - Map>> fetchedRecords = fetchedRecords(); + Map>> fetchedRecords = fetchRecords(); assertFalse(fetchedRecords.containsKey(tp1)); records = fetchedRecords.get(tp0); assertEquals(2, records.size()); @@ -2781,7 +2781,7 @@ public void testConsumingViaIncrementalFetchRequests() { // There is still a buffered record. assertEquals(0, sendFetches()); - fetchedRecords = fetchedRecords(); + fetchedRecords = fetchRecords(); assertFalse(fetchedRecords.containsKey(tp1)); records = fetchedRecords.get(tp0); assertEquals(1, records.size()); @@ -2794,7 +2794,7 @@ public void testConsumingViaIncrementalFetchRequests() { client.prepareResponse(resp2); assertEquals(1, sendFetches()); networkClientDelegate.poll(time.timer(0)); - fetchedRecords = fetchedRecords(); + fetchedRecords = fetchRecords(); assertTrue(fetchedRecords.isEmpty()); assertEquals(4L, subscriptions.position(tp0).offset); assertEquals(1L, subscriptions.position(tp1).offset); @@ -2811,7 +2811,7 @@ public void testConsumingViaIncrementalFetchRequests() { client.prepareResponse(resp3); assertEquals(1, sendFetches()); networkClientDelegate.poll(time.timer(0)); - fetchedRecords = fetchedRecords(); + fetchedRecords = fetchRecords(); assertFalse(fetchedRecords.containsKey(tp1)); records = fetchedRecords.get(tp0); assertEquals(2, records.size()); @@ -2858,7 +2858,7 @@ public void testEmptyControlBatch() { networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - Map>> fetchedRecords = fetchedRecords(); + Map>> fetchedRecords = fetchRecords(); assertTrue(fetchedRecords.containsKey(tp0)); assertEquals(fetchedRecords.get(tp0).size(), 2); } @@ -2942,7 +2942,7 @@ public void testSubscriptionPositionUpdatedWithEpoch() { networkClientDelegate.pollNoWakeup(); assertTrue(fetcher.hasCompletedFetches()); - Map>> partitionRecords = fetchedRecords(); + Map>> partitionRecords = fetchRecords(); assertTrue(partitionRecords.containsKey(tp0)); assertEquals(subscriptions.position(tp0).offset, 3L); @@ -2971,7 +2971,7 @@ public void testPreferredReadReplica() { networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - Map>> partitionRecords = fetchedRecords(); + Map>> partitionRecords = fetchRecords(); assertTrue(partitionRecords.containsKey(tp0)); // Verify @@ -2987,7 +2987,7 @@ public void testPreferredReadReplica() { FetchResponse.INVALID_LAST_STABLE_OFFSET, 0, Optional.of(2))); networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - fetchedRecords(); + fetchRecords(); selected = fetcher.selectReadReplica(tp0, Node.noNode(), time.milliseconds()); assertEquals(-1, selected.id()); } @@ -3007,7 +3007,7 @@ public void testFetchDisconnectedShouldClearPreferredReadReplica() { FetchResponse.INVALID_LAST_STABLE_OFFSET, 0, Optional.of(1))); networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - fetchedRecords(); + fetchRecords(); // Verify Node selected = fetcher.selectReadReplica(tp0, Node.noNode(), time.milliseconds()); @@ -3020,7 +3020,7 @@ public void testFetchDisconnectedShouldClearPreferredReadReplica() { networkClientDelegate.poll(time.timer(0)); assertFalse(fetcher.hasCompletedFetches()); - fetchedRecords(); + fetchRecords(); selected = fetcher.selectReadReplica(tp0, Node.noNode(), time.milliseconds()); assertEquals(-1, selected.id()); } @@ -3040,7 +3040,7 @@ public void testFetchDisconnectedShouldNotClearPreferredReadReplicaIfUnassigned( FetchResponse.INVALID_LAST_STABLE_OFFSET, 0, Optional.of(1))); networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - fetchedRecords(); + fetchRecords(); // Verify Node selected = fetcher.selectReadReplica(tp0, Node.noNode(), time.milliseconds()); @@ -3055,7 +3055,7 @@ public void testFetchDisconnectedShouldNotClearPreferredReadReplicaIfUnassigned( // Preferred read replica should not be cleared networkClientDelegate.poll(time.timer(0)); assertFalse(fetcher.hasCompletedFetches()); - fetchedRecords(); + fetchRecords(); selected = fetcher.selectReadReplica(tp0, Node.noNode(), time.milliseconds()); assertEquals(-1, selected.id()); } @@ -3075,7 +3075,7 @@ public void testFetchErrorShouldClearPreferredReadReplica() { FetchResponse.INVALID_LAST_STABLE_OFFSET, 0, Optional.of(1))); networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - fetchedRecords(); + fetchRecords(); // Verify Node selected = fetcher.selectReadReplica(tp0, Node.noNode(), time.milliseconds()); @@ -3090,7 +3090,7 @@ public void testFetchErrorShouldClearPreferredReadReplica() { networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - fetchedRecords(); + fetchRecords(); selected = fetcher.selectReadReplica(tp0, Node.noNode(), time.milliseconds()); assertEquals(-1, selected.id()); } @@ -3113,7 +3113,7 @@ public void testPreferredReadReplicaOffsetError() { networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - fetchedRecords(); + fetchRecords(); Node selected = fetcher.selectReadReplica(tp0, Node.noNode(), time.milliseconds()); assertEquals(selected.id(), 1); @@ -3127,7 +3127,7 @@ public void testPreferredReadReplicaOffsetError() { networkClientDelegate.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - fetchedRecords(); + fetchRecords(); selected = fetcher.selectReadReplica(tp0, Node.noNode(), time.milliseconds()); assertEquals(selected.id(), -1); @@ -3141,7 +3141,7 @@ public void testFetchCompletedBeforeHandlerAdded() { sendFetches(); client.prepareResponse(fullFetchResponse(tidp0, buildRecords(1L, 1, 1), Errors.NONE, 100L, 0)); networkClientDelegate.poll(time.timer(0)); - fetchedRecords(); + fetchRecords(); Metadata.LeaderAndEpoch leaderAndEpoch = subscriptions.position(tp0).currentLeader; assertTrue(leaderAndEpoch.leader.isPresent()); @@ -3182,7 +3182,7 @@ public void testCorruptMessageError() { assertTrue(fetcher.hasCompletedFetches()); // Trigger the exception. - assertThrows(KafkaException.class, this::fetchedRecords); + assertThrows(KafkaException.class, this::fetchRecords); } private OffsetsForLeaderEpochResponse prepareOffsetsForLeaderEpochResponse( @@ -3296,14 +3296,14 @@ private void assertEmptyFetch(String reason) { assertTrue(fetch.isEmpty(), reason); } - private Map>> fetchedRecords() { + private Map>> fetchRecords() { Fetch fetch = collectFetch(); return fetch.records(); } @SuppressWarnings("unchecked") private Fetch collectFetch() { - return (Fetch) fetcher.collectFetch(); + return (Fetch) fetcher.collectFetch(); } private void buildFetcher(int maxPollRecords) { diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java index f3c14539ed15e..7e7af8f75608c 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java @@ -251,7 +251,7 @@ public void testFetchNormal() { consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - Map>> partitionRecords = fetchedRecords(); + Map>> partitionRecords = fetchRecords(); assertTrue(partitionRecords.containsKey(tp0)); List> records = partitionRecords.get(tp0); @@ -276,7 +276,7 @@ public void testInflightFetchOnPendingPartitions() { client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); consumerClient.poll(time.timer(0)); - assertNull(fetchedRecords().get(tp0)); + assertNull(fetchRecords().get(tp0)); } @Test @@ -335,7 +335,7 @@ public void testFetchingPendingPartitions() { client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - fetchedRecords(); + fetchRecords(); assertEquals(4L, subscriptions.position(tp0).offset); // this is the next fetching position // mark partition unfetchable @@ -343,7 +343,7 @@ public void testFetchingPendingPartitions() { assertEquals(0, sendFetches()); consumerClient.poll(time.timer(0)); assertFalse(fetcher.hasCompletedFetches()); - fetchedRecords(); + fetchRecords(); assertEquals(4L, subscriptions.position(tp0).offset); } @@ -356,10 +356,10 @@ public void testFetchWithNoTopicId() { assignFromUser(noId.topicPartition()); subscriptions.seek(noId.topicPartition(), 0); - // Fetch should use request version 12 assertEquals(1, sendFetches()); assertFalse(fetcher.hasCompletedFetches()); + // Fetch should use request version 12 client.prepareResponse( fetchRequestMatcher((short) 12, noId, 0, Optional.of(validLeaderEpoch)), fullFetchResponse(noId, records, Errors.NONE, 100L, 0) @@ -367,7 +367,7 @@ public void testFetchWithNoTopicId() { consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - Map>> partitionRecords = fetchedRecords(); + Map>> partitionRecords = fetchRecords(); assertTrue(partitionRecords.containsKey(noId.topicPartition())); List> records = partitionRecords.get(noId.topicPartition()); @@ -399,7 +399,7 @@ public void testFetchWithTopicId() { consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - Map>> partitionRecords = fetchedRecords(); + Map>> partitionRecords = fetchRecords(); assertTrue(partitionRecords.containsKey(tp.topicPartition())); List> records = partitionRecords.get(tp.topicPartition()); @@ -424,9 +424,9 @@ public void testFetchForgetTopicIdWhenUnassigned() { client.updateMetadata(RequestTestUtils.metadataUpdateWithIds(1, singleton(foo), tp -> validLeaderEpoch)); subscriptions.seek(foo.topicPartition(), 0); - // Fetch should use latest version. assertEquals(1, sendFetches()); + // Fetch should use latest version. client.prepareResponse( fetchRequestMatcher(ApiKeys.FETCH.latestVersion(), singletonMap(foo, new PartitionData( @@ -442,9 +442,9 @@ public void testFetchForgetTopicIdWhenUnassigned() { ); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - fetchedRecords(); + fetchRecords(); - // Assign bar and unassign foo. + // Assign bar and un-assign foo. subscriptions.assignFromUser(singleton(bar.topicPartition())); client.updateMetadata(RequestTestUtils.metadataUpdateWithIds(1, singleton(bar), tp -> validLeaderEpoch)); subscriptions.seek(bar.topicPartition(), 0); @@ -468,7 +468,7 @@ public void testFetchForgetTopicIdWhenUnassigned() { ); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - fetchedRecords(); + fetchRecords(); } @Test @@ -501,7 +501,7 @@ public void testFetchForgetTopicIdWhenReplaced() { ); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - fetchedRecords(); + fetchRecords(); // Replace foo with old topic id with foo with new topic id. subscriptions.assignFromUser(singleton(fooWithNewTopicId.topicPartition())); @@ -528,7 +528,7 @@ public void testFetchForgetTopicIdWhenReplaced() { ); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - fetchedRecords(); + fetchRecords(); } @Test @@ -536,7 +536,6 @@ public void testFetchTopicIdUpgradeDowngrade() { buildFetcher(); TopicIdPartition fooWithoutId = new TopicIdPartition(Uuid.ZERO_UUID, new TopicPartition("foo", 0)); - TopicIdPartition fooWithId = new TopicIdPartition(Uuid.randomUuid(), new TopicPartition("foo", 0)); // Assign foo without a topic id. subscriptions.assignFromUser(singleton(fooWithoutId.topicPartition())); @@ -561,9 +560,10 @@ public void testFetchTopicIdUpgradeDowngrade() { ); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - fetchedRecords(); + fetchRecords(); // Upgrade. + TopicIdPartition fooWithId = new TopicIdPartition(Uuid.randomUuid(), new TopicPartition("foo", 0)); subscriptions.assignFromUser(singleton(fooWithId.topicPartition())); client.updateMetadata(RequestTestUtils.metadataUpdateWithIds(1, singleton(fooWithId), tp -> validLeaderEpoch)); subscriptions.seek(fooWithId.topicPartition(), 0); @@ -588,7 +588,7 @@ public void testFetchTopicIdUpgradeDowngrade() { ); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - fetchedRecords(); + fetchRecords(); // Downgrade. subscriptions.assignFromUser(singleton(fooWithoutId.topicPartition())); @@ -615,7 +615,7 @@ public void testFetchTopicIdUpgradeDowngrade() { ); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - fetchedRecords(); + fetchRecords(); } private MockClient.RequestMatcher fetchRequestMatcher( @@ -684,7 +684,7 @@ public void testMissingLeaderEpochInRecords() { consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - Map>> partitionRecords = fetchedRecords(); + Map>> partitionRecords = fetchRecords(); assertTrue(partitionRecords.containsKey(tp0)); assertEquals(2, partitionRecords.get(tp0).size()); @@ -735,7 +735,7 @@ public void testLeaderEpochInConsumerRecord() { consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - Map>> partitionRecords = fetchedRecords(); + Map>> partitionRecords = fetchRecords(); assertTrue(partitionRecords.containsKey(tp0)); assertEquals(6, partitionRecords.get(tp0).size()); @@ -812,7 +812,7 @@ public void testFetcherIgnoresControlRecords() { consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - Map>> partitionRecords = fetchedRecords(); + Map>> partitionRecords = fetchRecords(); assertTrue(partitionRecords.containsKey(tp0)); List> records = partitionRecords.get(tp0); @@ -837,7 +837,7 @@ public void testFetchError() { consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - Map>> partitionRecords = fetchedRecords(); + Map>> partitionRecords = fetchRecords(); assertFalse(partitionRecords.containsKey(tp0)); } @@ -940,7 +940,7 @@ public void testParseCorruptedRecord() throws Exception { consumerClient.poll(time.timer(0)); // the first fetchedRecords() should return the first valid message - assertEquals(1, fetchedRecords().get(tp0).size()); + assertEquals(1, fetchRecords().get(tp0).size()); assertEquals(1, subscriptions.position(tp0).offset); ensureBlockOnRecord(1L); @@ -957,10 +957,11 @@ public void testParseCorruptedRecord() throws Exception { } private void ensureBlockOnRecord(long blockedOffset) { + // // the fetchedRecords() should always throw exception due to the invalid message at the starting offset. for (int i = 0; i < 2; i++) { try { - collectFetch(); + fetchRecords(); fail("fetchedRecords should have raised KafkaException"); } catch (KafkaException e) { assertEquals(blockedOffset, subscriptions.position(tp0).offset); @@ -977,7 +978,7 @@ private void seekAndConsumeRecord(ByteBuffer responseBuffer, long toOffset) { client.prepareResponse(fullFetchResponse(tidp0, MemoryRecords.readableRecords(responseBuffer), Errors.NONE, 100L, 0)); consumerClient.poll(time.timer(0)); - Map>> recordsByPartition = fetchedRecords(); + Map>> recordsByPartition = fetchRecords(); List> records = recordsByPartition.get(tp0); assertEquals(1, records.size()); assertEquals(toOffset, records.get(0).offset()); @@ -1079,7 +1080,7 @@ public void testHeaders() { assertEquals(1, sendFetches()); consumerClient.poll(time.timer(0)); - Map>> recordsByPartition = fetchedRecords(); + Map>> recordsByPartition = fetchRecords(); records = recordsByPartition.get(tp0); assertEquals(3, records.size()); @@ -1111,7 +1112,7 @@ public void testFetchMaxPollRecords() { assertEquals(1, sendFetches()); consumerClient.poll(time.timer(0)); - Map>> recordsByPartition = fetchedRecords(); + Map>> recordsByPartition = fetchRecords(); records = recordsByPartition.get(tp0); assertEquals(2, records.size()); assertEquals(3L, subscriptions.position(tp0).offset); @@ -1120,7 +1121,7 @@ public void testFetchMaxPollRecords() { assertEquals(0, sendFetches()); consumerClient.poll(time.timer(0)); - recordsByPartition = fetchedRecords(); + recordsByPartition = fetchRecords(); records = recordsByPartition.get(tp0); assertEquals(1, records.size()); assertEquals(4L, subscriptions.position(tp0).offset); @@ -1128,7 +1129,7 @@ public void testFetchMaxPollRecords() { assertTrue(sendFetches() > 0); consumerClient.poll(time.timer(0)); - recordsByPartition = fetchedRecords(); + recordsByPartition = fetchRecords(); records = recordsByPartition.get(tp0); assertEquals(2, records.size()); assertEquals(6L, subscriptions.position(tp0).offset); @@ -1154,7 +1155,7 @@ public void testFetchAfterPartitionWithFetchedRecordsIsUnassigned() { assertEquals(1, sendFetches()); consumerClient.poll(time.timer(0)); - Map>> recordsByPartition = fetchedRecords(); + Map>> recordsByPartition = fetchRecords(); records = recordsByPartition.get(tp0); assertEquals(2, records.size()); assertEquals(3L, subscriptions.position(tp0).offset); @@ -1167,7 +1168,7 @@ public void testFetchAfterPartitionWithFetchedRecordsIsUnassigned() { assertEquals(1, sendFetches()); consumerClient.poll(time.timer(0)); - Map>> fetchedRecords = fetchedRecords(); + Map>> fetchedRecords = fetchRecords(); assertNull(fetchedRecords.get(tp0)); records = fetchedRecords.get(tp1); assertEquals(2, records.size()); @@ -1197,7 +1198,7 @@ public void testFetchNonContinuousRecords() { assertEquals(1, sendFetches()); client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); consumerClient.poll(time.timer(0)); - Map>> recordsByPartition = fetchedRecords(); + Map>> recordsByPartition = fetchRecords(); consumerRecords = recordsByPartition.get(tp0); assertEquals(3, consumerRecords.size()); assertEquals(31L, subscriptions.position(tp0).offset); // this is the next fetching position @@ -1303,7 +1304,7 @@ public void testFetchDuringEagerRebalance() { consumerClient.poll(time.timer(0)); // The active fetch should be ignored since its position is no longer valid - assertTrue(fetchedRecords().isEmpty()); + assertTrue(fetchRecords().isEmpty()); } @Test @@ -1325,7 +1326,7 @@ public void testFetchDuringCooperativeRebalance() { client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); consumerClient.poll(time.timer(0)); - Map>> fetchedRecords = fetchedRecords(); + Map>> fetchedRecords = fetchRecords(); // The active fetch should NOT be ignored since the position for tp0 is still valid assertEquals(1, fetchedRecords.size()); @@ -1344,7 +1345,7 @@ public void testInFlightFetchOnPausedPartition() { client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); consumerClient.poll(time.timer(0)); - assertNull(fetchedRecords().get(tp0)); + assertNull(fetchRecords().get(tp0)); } @Test @@ -1384,7 +1385,7 @@ public void testFetchOnCompletedFetchesForPausedAndResumedPartitions() { assertTrue(fetcher.hasAvailableFetches(), "Should have available (non-paused) completed fetches"); consumerClient.poll(time.timer(0)); - Map>> fetchedRecords = fetchedRecords(); + Map>> fetchedRecords = fetchRecords(); assertEquals(1, fetchedRecords.size(), "Should return records when partition is resumed"); assertNotNull(fetchedRecords.get(tp0)); assertEquals(3, fetchedRecords.get(tp0).size()); @@ -1418,7 +1419,7 @@ public void testFetchOnCompletedFetchesForSomePausedPartitions() { subscriptions.pause(tp0); consumerClient.poll(time.timer(0)); - fetchedRecords = fetchedRecords(); + fetchedRecords = fetchRecords(); assertEquals(1, fetchedRecords.size(), "Should return completed fetch for unpaused partitions"); assertTrue(fetcher.hasCompletedFetches(), "Should still contain completed fetches"); assertNotNull(fetchedRecords.get(tp1)); @@ -1473,7 +1474,7 @@ public void testPartialFetchWithPausedPartitions() { client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); consumerClient.poll(time.timer(0)); - fetchedRecords = fetchedRecords(); + fetchedRecords = fetchRecords(); assertEquals(2, fetchedRecords.get(tp0).size(), "Should return 2 records from fetch with 3 records"); assertFalse(fetcher.hasCompletedFetches(), "Should have no completed fetches"); @@ -1481,7 +1482,7 @@ public void testPartialFetchWithPausedPartitions() { subscriptions.pause(tp0); consumerClient.poll(time.timer(0)); - fetchedRecords = fetchedRecords(); + fetchedRecords = fetchRecords(); assertEmptyFetch("Should not return records or advance position for paused partitions"); assertTrue(fetcher.hasCompletedFetches(), "Should have 1 entry in completed fetches"); @@ -1491,7 +1492,7 @@ public void testPartialFetchWithPausedPartitions() { consumerClient.poll(time.timer(0)); - fetchedRecords = fetchedRecords(); + fetchedRecords = fetchRecords(); assertEquals(1, fetchedRecords.get(tp0).size(), "Should return last remaining record"); assertFalse(fetcher.hasCompletedFetches(), "Should have no completed fetches"); @@ -1752,7 +1753,7 @@ public void testFetchPositionAfterException() { } private void fetchRecordsInto(List> allFetchedRecords) { - Map>> fetchedRecords = fetchedRecords(); + Map>> fetchedRecords = fetchRecords(); fetchedRecords.values().forEach(allFetchedRecords::addAll); } @@ -1795,7 +1796,7 @@ public void testCompletedFetchRemoval() { consumerClient.poll(time.timer(0)); List> fetchedRecords = new ArrayList<>(); - Map>> recordsByPartition = fetchedRecords(); + Map>> recordsByPartition = fetchRecords(); for (List> records : recordsByPartition.values()) fetchedRecords.addAll(records); @@ -1805,7 +1806,7 @@ public void testCompletedFetchRemoval() { List oorExceptions = new ArrayList<>(); try { - recordsByPartition = fetchedRecords(); + recordsByPartition = fetchRecords(); for (List> records : recordsByPartition.values()) fetchedRecords.addAll(records); } catch (OffsetOutOfRangeException oor) { @@ -1818,7 +1819,7 @@ public void testCompletedFetchRemoval() { assertTrue(oor.offsetOutOfRangePartitions().containsKey(tp0)); assertEquals(oor.offsetOutOfRangePartitions().size(), 1); - recordsByPartition = fetchedRecords(); + recordsByPartition = fetchRecords(); for (List> records : recordsByPartition.values()) fetchedRecords.addAll(records); @@ -1830,7 +1831,7 @@ public void testCompletedFetchRemoval() { List kafkaExceptions = new ArrayList<>(); for (int i = 1; i <= numExceptionsExpected; i++) { try { - recordsByPartition = fetchedRecords(); + recordsByPartition = fetchRecords(); for (List> records : recordsByPartition.values()) fetchedRecords.addAll(records); } catch (KafkaException e) { @@ -1857,7 +1858,7 @@ public void testSeekBeforeException() { client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); consumerClient.poll(time.timer(0)); - assertEquals(2, fetchedRecords().get(tp0).size()); + assertEquals(2, fetchRecords().get(tp0).size()); subscriptions.assignFromUser(mkSet(tp0, tp1)); subscriptions.seekUnvalidated(tp1, new SubscriptionState.FetchPosition(1, Optional.empty(), metadata.currentLeader(tp1))); @@ -1870,7 +1871,7 @@ public void testSeekBeforeException() { .setHighWatermark(100)); client.prepareResponse(FetchResponse.of(Errors.NONE, 0, INVALID_SESSION_ID, new LinkedHashMap<>(partitions))); consumerClient.poll(time.timer(0)); - assertEquals(1, fetchedRecords().get(tp0).size()); + assertEquals(1, fetchRecords().get(tp0).size()); subscriptions.seek(tp1, 10); // Should not throw OffsetOutOfRangeException after the seek @@ -2118,7 +2119,7 @@ public void testFetchResponseMetrics() { client.prepareResponse(FetchResponse.of(Errors.NONE, 0, INVALID_SESSION_ID, fetchPartitionData)); consumerClient.poll(time.timer(0)); - Map>> fetchedRecords = fetchedRecords(); + Map>> fetchedRecords = fetchRecords(); assertEquals(3, fetchedRecords.get(tp1).size()); assertEquals(3, fetchedRecords.get(tp2).size()); @@ -2259,7 +2260,7 @@ public void testFetcherMetricsTemplates() { client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - Map>> partitionRecords = fetchedRecords(); + Map>> partitionRecords = fetchRecords(); assertTrue(partitionRecords.containsKey(tp0)); // Verify that all metrics except metrics-count have registered templates @@ -2282,7 +2283,7 @@ private Map>> fetchRecords( assertEquals(1, sendFetches()); client.prepareResponse(fullFetchResponse(tp, records, error, hw, lastStableOffset, throttleTime)); consumerClient.poll(time.timer(0)); - return fetchedRecords(); + return fetchRecords(); } private Map>> fetchRecords( @@ -2290,7 +2291,7 @@ private Map>> fetchRecords( assertEquals(1, sendFetches()); client.prepareResponse(fetchResponse(tp, records, error, hw, lastStableOffset, logStartOffset, throttleTime)); consumerClient.poll(time.timer(0)); - return fetchedRecords(); + return fetchRecords(); } @Test @@ -2359,7 +2360,7 @@ public void testReturnCommittedTransactions() { consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - Map>> fetchedRecords = fetchedRecords(); + Map>> fetchedRecords = fetchRecords(); assertTrue(fetchedRecords.containsKey(tp0)); assertEquals(fetchedRecords.get(tp0).size(), 2); } @@ -2429,7 +2430,7 @@ public void testReadCommittedWithCommittedAndAbortedTransactions() { consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - Map>> fetchedRecords = fetchedRecords(); + Map>> fetchedRecords = fetchRecords(); assertTrue(fetchedRecords.containsKey(tp0)); // There are only 3 committed records List> fetchedConsumerRecords = fetchedRecords.get(tp0); @@ -2477,7 +2478,7 @@ public void testMultipleAbortMarkers() { consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - Map>> fetchedRecords = fetchedRecords(); + Map>> fetchedRecords = fetchRecords(); assertTrue(fetchedRecords.containsKey(tp0)); assertEquals(fetchedRecords.get(tp0).size(), 2); List> fetchedConsumerRecords = fetchedRecords.get(tp0); @@ -2522,7 +2523,7 @@ public void testReadCommittedAbortMarkerWithNoData() { consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - Map>> allFetchedRecords = fetchedRecords(); + Map>> allFetchedRecords = fetchRecords(); assertTrue(allFetchedRecords.containsKey(tp0)); List> fetchedRecords = allFetchedRecords.get(tp0); assertEquals(3, fetchedRecords.size()); @@ -2561,7 +2562,7 @@ protected boolean shouldRetainRecord(RecordBatch recordBatch, Record record) { consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - Map>> allFetchedRecords = fetchedRecords(); + Map>> allFetchedRecords = fetchRecords(); assertTrue(allFetchedRecords.containsKey(tp0)); List> fetchedRecords = allFetchedRecords.get(tp0); assertEquals(3, fetchedRecords.size()); @@ -2662,7 +2663,7 @@ public void testReadCommittedWithCompactedTopic() { consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - Map>> allFetchedRecords = fetchedRecords(); + Map>> allFetchedRecords = fetchRecords(); assertTrue(allFetchedRecords.containsKey(tp0)); List> fetchedRecords = allFetchedRecords.get(tp0); assertEquals(5, fetchedRecords.size()); @@ -2699,7 +2700,7 @@ public void testReturnAbortedTransactionsinUncommittedMode() { consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - Map>> fetchedRecords = fetchedRecords(); + Map>> fetchedRecords = fetchRecords(); assertTrue(fetchedRecords.containsKey(tp0)); } @@ -2732,7 +2733,7 @@ public void testConsumerPositionUpdatedWhenSkippingAbortedTransactions() { consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - Map>> fetchedRecords = fetchedRecords(); + Map>> fetchedRecords = fetchRecords(); // Ensure that we don't return any of the aborted records, but yet advance the consumer position. assertFalse(fetchedRecords.containsKey(tp0)); @@ -2767,7 +2768,7 @@ public void testConsumingViaIncrementalFetchRequests() { assertFalse(fetcher.hasCompletedFetches()); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - Map>> fetchedRecords = fetchedRecords(); + Map>> fetchedRecords = fetchRecords(); assertFalse(fetchedRecords.containsKey(tp1)); records = fetchedRecords.get(tp0); assertEquals(2, records.size()); @@ -2778,7 +2779,7 @@ public void testConsumingViaIncrementalFetchRequests() { // There is still a buffered record. assertEquals(0, sendFetches()); - fetchedRecords = fetchedRecords(); + fetchedRecords = fetchRecords(); assertFalse(fetchedRecords.containsKey(tp1)); records = fetchedRecords.get(tp0); assertEquals(1, records.size()); @@ -2791,7 +2792,7 @@ public void testConsumingViaIncrementalFetchRequests() { client.prepareResponse(resp2); assertEquals(1, sendFetches()); consumerClient.poll(time.timer(0)); - fetchedRecords = fetchedRecords(); + fetchedRecords = fetchRecords(); assertTrue(fetchedRecords.isEmpty()); assertEquals(4L, subscriptions.position(tp0).offset); assertEquals(1L, subscriptions.position(tp1).offset); @@ -2808,7 +2809,7 @@ public void testConsumingViaIncrementalFetchRequests() { client.prepareResponse(resp3); assertEquals(1, sendFetches()); consumerClient.poll(time.timer(0)); - fetchedRecords = fetchedRecords(); + fetchedRecords = fetchRecords(); assertFalse(fetchedRecords.containsKey(tp1)); records = fetchedRecords.get(tp0); assertEquals(2, records.size()); @@ -2948,7 +2949,7 @@ private void verifySessionPartitions() { } } if (fetcher.hasCompletedFetches()) { - Map>> fetchedRecords = fetchedRecords(); + Map>> fetchedRecords = fetchRecords(); if (!fetchedRecords.isEmpty()) { fetchesRemaining.decrementAndGet(); fetchedRecords.forEach((tp, records) -> { @@ -3010,7 +3011,7 @@ public void testFetcherSessionEpochUpdate() throws Exception { } } if (fetcher.hasCompletedFetches()) { - Map>> fetchedRecords = fetchedRecords(); + Map>> fetchedRecords = fetchRecords(); if (!fetchedRecords.isEmpty()) { fetchesRemaining.decrementAndGet(); List> records = fetchedRecords.get(tp0); @@ -3019,7 +3020,7 @@ public void testFetcherSessionEpochUpdate() throws Exception { assertEquals(nextFetchOffset + 1, records.get(1).offset()); nextFetchOffset += 2; } - assertTrue(fetchedRecords().isEmpty()); + assertTrue(fetchRecords().isEmpty()); } } assertEquals(0, future.get()); @@ -3062,7 +3063,7 @@ public void testEmptyControlBatch() { consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - Map>> fetchedRecords = fetchedRecords(); + Map>> fetchedRecords = fetchRecords(); assertTrue(fetchedRecords.containsKey(tp0)); assertEquals(fetchedRecords.get(tp0).size(), 2); } @@ -3146,7 +3147,7 @@ public void testSubscriptionPositionUpdatedWithEpoch() { consumerClient.pollNoWakeup(); assertTrue(fetcher.hasCompletedFetches()); - Map>> partitionRecords = fetchedRecords(); + Map>> partitionRecords = fetchRecords(); assertTrue(partitionRecords.containsKey(tp0)); assertEquals(subscriptions.position(tp0).offset, 3L); @@ -3224,7 +3225,7 @@ public void testTruncationDetected() { consumerClient.pollNoWakeup(); assertTrue(fetcher.hasCompletedFetches()); - Map>> partitionRecords = fetchedRecords(); + Map>> partitionRecords = fetchRecords(); assertTrue(partitionRecords.containsKey(tp0)); assertEquals(subscriptions.position(tp0).offset, 3L); @@ -3253,7 +3254,7 @@ public void testPreferredReadReplica() { consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - Map>> partitionRecords = fetchedRecords(); + Map>> partitionRecords = fetchRecords(); assertTrue(partitionRecords.containsKey(tp0)); // Verify @@ -3269,7 +3270,7 @@ public void testPreferredReadReplica() { FetchResponse.INVALID_LAST_STABLE_OFFSET, 0, Optional.of(2))); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - fetchedRecords(); + fetchRecords(); selected = fetcher.selectReadReplica(tp0, Node.noNode(), time.milliseconds()); assertEquals(-1, selected.id()); } @@ -3289,7 +3290,7 @@ public void testFetchDisconnectedShouldClearPreferredReadReplica() { FetchResponse.INVALID_LAST_STABLE_OFFSET, 0, Optional.of(1))); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - fetchedRecords(); + fetchRecords(); // Verify Node selected = fetcher.selectReadReplica(tp0, Node.noNode(), time.milliseconds()); @@ -3302,7 +3303,7 @@ public void testFetchDisconnectedShouldClearPreferredReadReplica() { consumerClient.poll(time.timer(0)); assertFalse(fetcher.hasCompletedFetches()); - fetchedRecords(); + fetchRecords(); selected = fetcher.selectReadReplica(tp0, Node.noNode(), time.milliseconds()); assertEquals(-1, selected.id()); } @@ -3322,7 +3323,7 @@ public void testFetchDisconnectedShouldNotClearPreferredReadReplicaIfUnassigned( FetchResponse.INVALID_LAST_STABLE_OFFSET, 0, Optional.of(1))); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - fetchedRecords(); + fetchRecords(); // Verify Node selected = fetcher.selectReadReplica(tp0, Node.noNode(), time.milliseconds()); @@ -3337,7 +3338,7 @@ public void testFetchDisconnectedShouldNotClearPreferredReadReplicaIfUnassigned( // Preferred read replica should not be cleared consumerClient.poll(time.timer(0)); assertFalse(fetcher.hasCompletedFetches()); - fetchedRecords(); + fetchRecords(); selected = fetcher.selectReadReplica(tp0, Node.noNode(), time.milliseconds()); assertEquals(-1, selected.id()); } @@ -3357,7 +3358,7 @@ public void testFetchErrorShouldClearPreferredReadReplica() { FetchResponse.INVALID_LAST_STABLE_OFFSET, 0, Optional.of(1))); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - fetchedRecords(); + fetchRecords(); // Verify Node selected = fetcher.selectReadReplica(tp0, Node.noNode(), time.milliseconds()); @@ -3372,7 +3373,7 @@ public void testFetchErrorShouldClearPreferredReadReplica() { consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - fetchedRecords(); + fetchRecords(); selected = fetcher.selectReadReplica(tp0, Node.noNode(), time.milliseconds()); assertEquals(-1, selected.id()); } @@ -3395,7 +3396,7 @@ public void testPreferredReadReplicaOffsetError() { consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - fetchedRecords(); + fetchRecords(); Node selected = fetcher.selectReadReplica(tp0, Node.noNode(), time.milliseconds()); assertEquals(selected.id(), 1); @@ -3409,7 +3410,7 @@ public void testPreferredReadReplicaOffsetError() { consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); - fetchedRecords(); + fetchRecords(); selected = fetcher.selectReadReplica(tp0, Node.noNode(), time.milliseconds()); assertEquals(selected.id(), -1); @@ -3423,7 +3424,7 @@ public void testFetchCompletedBeforeHandlerAdded() { sendFetches(); client.prepareResponse(fullFetchResponse(tidp0, buildRecords(1L, 1, 1), Errors.NONE, 100L, 0)); consumerClient.poll(time.timer(0)); - fetchedRecords(); + fetchRecords(); Metadata.LeaderAndEpoch leaderAndEpoch = subscriptions.position(tp0).currentLeader; assertTrue(leaderAndEpoch.leader.isPresent()); @@ -3464,7 +3465,7 @@ public void testCorruptMessageError() { assertTrue(fetcher.hasCompletedFetches()); // Trigger the exception. - assertThrows(KafkaException.class, this::fetchedRecords); + assertThrows(KafkaException.class, this::fetchRecords); } private OffsetsForLeaderEpochResponse prepareOffsetsForLeaderEpochResponse( @@ -3578,7 +3579,7 @@ private void assertEmptyFetch(String reason) { assertTrue(fetch.isEmpty(), reason); } - private Map>> fetchedRecords() { + private Map>> fetchRecords() { Fetch fetch = collectFetch(); return fetch.records(); } From 6ebd657cf3346998d9e469b37a3a6e7749f697a5 Mon Sep 17 00:00:00 2001 From: Kirk True Date: Fri, 13 Oct 2023 11:53:24 -0700 Subject: [PATCH 47/72] Changed locally-scoped "records" to "testRecords" to avoid naming collision I had removed most unnecessary uses of `this` in a previous round of updates, but left these because they were necessary. There are three tests that create a locally-scoped variable named `records` that masks the instance-scoped variable of the same name, hence the use of `this` to distinguish them. I've updated the tests to change the name of the local variable to `testRecords` so it's a) more clear that they're separate from `records`, and b) removes the use of `this`. --- .../internals/FetchRequestManagerTest.java | 69 +++++++++---------- .../consumer/internals/FetcherTest.java | 49 +++++++------ 2 files changed, 56 insertions(+), 62 deletions(-) diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java index f1458067705ac..e6f7e6c3180a2 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java @@ -1105,38 +1105,37 @@ record = recordIterator.next(); public void testFetchMaxPollRecords() { buildFetcher(2); - List> records; assignFromUser(singleton(tp0)); subscriptions.seek(tp0, 1); - client.prepareResponse(matchesOffset(tidp0, 1), fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + client.prepareResponse(matchesOffset(tidp0, 1), fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); client.prepareResponse(matchesOffset(tidp0, 4), fullFetchResponse(tidp0, nextRecords, Errors.NONE, 100L, 0)); assertEquals(1, sendFetches()); networkClientDelegate.poll(time.timer(0)); Map>> recordsByPartition = fetchRecords(); - records = recordsByPartition.get(tp0); - assertEquals(2, records.size()); + List> recordsToTest = recordsByPartition.get(tp0); + assertEquals(2, recordsToTest.size()); assertEquals(3L, subscriptions.position(tp0).offset); - assertEquals(1, records.get(0).offset()); - assertEquals(2, records.get(1).offset()); + assertEquals(1, recordsToTest.get(0).offset()); + assertEquals(2, recordsToTest.get(1).offset()); assertEquals(0, sendFetches()); networkClientDelegate.poll(time.timer(0)); recordsByPartition = fetchRecords(); - records = recordsByPartition.get(tp0); - assertEquals(1, records.size()); + recordsToTest = recordsByPartition.get(tp0); + assertEquals(1, recordsToTest.size()); assertEquals(4L, subscriptions.position(tp0).offset); - assertEquals(3, records.get(0).offset()); + assertEquals(3, recordsToTest.get(0).offset()); assertTrue(sendFetches() > 0); networkClientDelegate.poll(time.timer(0)); recordsByPartition = fetchRecords(); - records = recordsByPartition.get(tp0); - assertEquals(2, records.size()); + recordsToTest = recordsByPartition.get(tp0); + assertEquals(2, recordsToTest.size()); assertEquals(6L, subscriptions.position(tp0).offset); - assertEquals(4, records.get(0).offset()); - assertEquals(5, records.get(1).offset()); + assertEquals(4, recordsToTest.get(0).offset()); + assertEquals(5, recordsToTest.get(1).offset()); } /** @@ -1148,21 +1147,20 @@ public void testFetchMaxPollRecords() { public void testFetchAfterPartitionWithFetchedRecordsIsUnassigned() { buildFetcher(2); - List> records; assignFromUser(singleton(tp0)); subscriptions.seek(tp0, 1); // Returns 3 records while `max.poll.records` is configured to 2 - client.prepareResponse(matchesOffset(tidp0, 1), fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + client.prepareResponse(matchesOffset(tidp0, 1), fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); assertEquals(1, sendFetches()); networkClientDelegate.poll(time.timer(0)); Map>> recordsByPartition = fetchRecords(); - records = recordsByPartition.get(tp0); - assertEquals(2, records.size()); + List> recordsToTest = recordsByPartition.get(tp0); + assertEquals(2, recordsToTest.size()); assertEquals(3L, subscriptions.position(tp0).offset); - assertEquals(1, records.get(0).offset()); - assertEquals(2, records.get(1).offset()); + assertEquals(1, recordsToTest.get(0).offset()); + assertEquals(2, recordsToTest.get(1).offset()); assignFromUser(singleton(tp1)); client.prepareResponse(matchesOffset(tidp1, 4), fullFetchResponse(tidp1, nextRecords, Errors.NONE, 100L, 0)); @@ -1172,11 +1170,11 @@ public void testFetchAfterPartitionWithFetchedRecordsIsUnassigned() { networkClientDelegate.poll(time.timer(0)); Map>> fetchedRecords = fetchRecords(); assertNull(fetchedRecords.get(tp0)); - records = fetchedRecords.get(tp1); - assertEquals(2, records.size()); + recordsToTest = fetchedRecords.get(tp1); + assertEquals(2, recordsToTest.size()); assertEquals(6L, subscriptions.position(tp1).offset); - assertEquals(4, records.get(0).offset()); - assertEquals(5, records.get(1).offset()); + assertEquals(4, recordsToTest.get(0).offset()); + assertEquals(5, recordsToTest.get(1).offset()); } @Test @@ -2746,7 +2744,6 @@ public void testConsumerPositionUpdatedWhenSkippingAbortedTransactions() { public void testConsumingViaIncrementalFetchRequests() { buildFetcher(2); - List> records; assignFromUser(new HashSet<>(Arrays.asList(tp0, tp1))); subscriptions.seekValidated(tp0, new SubscriptionState.FetchPosition(0, Optional.empty(), metadata.currentLeader(tp0))); subscriptions.seekValidated(tp1, new SubscriptionState.FetchPosition(1, Optional.empty(), metadata.currentLeader(tp1))); @@ -2758,7 +2755,7 @@ public void testConsumingViaIncrementalFetchRequests() { .setHighWatermark(2) .setLastStableOffset(2) .setLogStartOffset(0) - .setRecords(this.records)); + .setRecords(records)); partitions1.put(tidp1, new FetchResponseData.PartitionData() .setPartitionIndex(tp1.partition()) .setHighWatermark(100) @@ -2772,20 +2769,20 @@ public void testConsumingViaIncrementalFetchRequests() { assertTrue(fetcher.hasCompletedFetches()); Map>> fetchedRecords = fetchRecords(); assertFalse(fetchedRecords.containsKey(tp1)); - records = fetchedRecords.get(tp0); - assertEquals(2, records.size()); + List> recordsToTest = fetchedRecords.get(tp0); + assertEquals(2, recordsToTest.size()); assertEquals(3L, subscriptions.position(tp0).offset); assertEquals(1L, subscriptions.position(tp1).offset); - assertEquals(1, records.get(0).offset()); - assertEquals(2, records.get(1).offset()); + assertEquals(1, recordsToTest.get(0).offset()); + assertEquals(2, recordsToTest.get(1).offset()); // There is still a buffered record. assertEquals(0, sendFetches()); fetchedRecords = fetchRecords(); assertFalse(fetchedRecords.containsKey(tp1)); - records = fetchedRecords.get(tp0); - assertEquals(1, records.size()); - assertEquals(3, records.get(0).offset()); + recordsToTest = fetchedRecords.get(tp0); + assertEquals(1, recordsToTest.size()); + assertEquals(3, recordsToTest.get(0).offset()); assertEquals(4L, subscriptions.position(tp0).offset); // The second response contains no new records. @@ -2813,12 +2810,12 @@ public void testConsumingViaIncrementalFetchRequests() { networkClientDelegate.poll(time.timer(0)); fetchedRecords = fetchRecords(); assertFalse(fetchedRecords.containsKey(tp1)); - records = fetchedRecords.get(tp0); - assertEquals(2, records.size()); + recordsToTest = fetchedRecords.get(tp0); + assertEquals(2, recordsToTest.size()); assertEquals(6L, subscriptions.position(tp0).offset); assertEquals(1L, subscriptions.position(tp1).offset); - assertEquals(4, records.get(0).offset()); - assertEquals(5, records.get(1).offset()); + assertEquals(4, recordsToTest.get(0).offset()); + assertEquals(5, recordsToTest.get(1).offset()); } @Test diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java index 7e7af8f75608c..5474a4b783e5e 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java @@ -1103,38 +1103,37 @@ record = recordIterator.next(); public void testFetchMaxPollRecords() { buildFetcher(2); - List> records; assignFromUser(singleton(tp0)); subscriptions.seek(tp0, 1); - client.prepareResponse(matchesOffset(tidp0, 1), fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + client.prepareResponse(matchesOffset(tidp0, 1), fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); client.prepareResponse(matchesOffset(tidp0, 4), fullFetchResponse(tidp0, nextRecords, Errors.NONE, 100L, 0)); assertEquals(1, sendFetches()); consumerClient.poll(time.timer(0)); Map>> recordsByPartition = fetchRecords(); - records = recordsByPartition.get(tp0); - assertEquals(2, records.size()); + List> recordsToTest = recordsByPartition.get(tp0); + assertEquals(2, recordsToTest.size()); assertEquals(3L, subscriptions.position(tp0).offset); - assertEquals(1, records.get(0).offset()); - assertEquals(2, records.get(1).offset()); + assertEquals(1, recordsToTest.get(0).offset()); + assertEquals(2, recordsToTest.get(1).offset()); assertEquals(0, sendFetches()); consumerClient.poll(time.timer(0)); recordsByPartition = fetchRecords(); - records = recordsByPartition.get(tp0); - assertEquals(1, records.size()); + recordsToTest = recordsByPartition.get(tp0); + assertEquals(1, recordsToTest.size()); assertEquals(4L, subscriptions.position(tp0).offset); - assertEquals(3, records.get(0).offset()); + assertEquals(3, recordsToTest.get(0).offset()); assertTrue(sendFetches() > 0); consumerClient.poll(time.timer(0)); recordsByPartition = fetchRecords(); - records = recordsByPartition.get(tp0); - assertEquals(2, records.size()); + recordsToTest = recordsByPartition.get(tp0); + assertEquals(2, recordsToTest.size()); assertEquals(6L, subscriptions.position(tp0).offset); - assertEquals(4, records.get(0).offset()); - assertEquals(5, records.get(1).offset()); + assertEquals(4, recordsToTest.get(0).offset()); + assertEquals(5, recordsToTest.get(1).offset()); } /** @@ -1146,21 +1145,20 @@ public void testFetchMaxPollRecords() { public void testFetchAfterPartitionWithFetchedRecordsIsUnassigned() { buildFetcher(2); - List> records; assignFromUser(singleton(tp0)); subscriptions.seek(tp0, 1); // Returns 3 records while `max.poll.records` is configured to 2 - client.prepareResponse(matchesOffset(tidp0, 1), fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0)); + client.prepareResponse(matchesOffset(tidp0, 1), fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); assertEquals(1, sendFetches()); consumerClient.poll(time.timer(0)); Map>> recordsByPartition = fetchRecords(); - records = recordsByPartition.get(tp0); - assertEquals(2, records.size()); + List> recordsToTest = recordsByPartition.get(tp0); + assertEquals(2, recordsToTest.size()); assertEquals(3L, subscriptions.position(tp0).offset); - assertEquals(1, records.get(0).offset()); - assertEquals(2, records.get(1).offset()); + assertEquals(1, recordsToTest.get(0).offset()); + assertEquals(2, recordsToTest.get(1).offset()); assignFromUser(singleton(tp1)); client.prepareResponse(matchesOffset(tidp1, 4), fullFetchResponse(tidp1, nextRecords, Errors.NONE, 100L, 0)); @@ -1170,11 +1168,11 @@ public void testFetchAfterPartitionWithFetchedRecordsIsUnassigned() { consumerClient.poll(time.timer(0)); Map>> fetchedRecords = fetchRecords(); assertNull(fetchedRecords.get(tp0)); - records = fetchedRecords.get(tp1); - assertEquals(2, records.size()); + recordsToTest = fetchedRecords.get(tp1); + assertEquals(2, recordsToTest.size()); assertEquals(6L, subscriptions.position(tp1).offset); - assertEquals(4, records.get(0).offset()); - assertEquals(5, records.get(1).offset()); + assertEquals(4, recordsToTest.get(0).offset()); + assertEquals(5, recordsToTest.get(1).offset()); } @Test @@ -2744,7 +2742,6 @@ public void testConsumerPositionUpdatedWhenSkippingAbortedTransactions() { public void testConsumingViaIncrementalFetchRequests() { buildFetcher(2); - List> records; assignFromUser(new HashSet<>(Arrays.asList(tp0, tp1))); subscriptions.seekValidated(tp0, new SubscriptionState.FetchPosition(0, Optional.empty(), metadata.currentLeader(tp0))); subscriptions.seekValidated(tp1, new SubscriptionState.FetchPosition(1, Optional.empty(), metadata.currentLeader(tp1))); @@ -2756,7 +2753,7 @@ public void testConsumingViaIncrementalFetchRequests() { .setHighWatermark(2) .setLastStableOffset(2) .setLogStartOffset(0) - .setRecords(this.records)); + .setRecords(records)); partitions1.put(tidp1, new FetchResponseData.PartitionData() .setPartitionIndex(tp1.partition()) .setHighWatermark(100) @@ -2770,7 +2767,7 @@ public void testConsumingViaIncrementalFetchRequests() { assertTrue(fetcher.hasCompletedFetches()); Map>> fetchedRecords = fetchRecords(); assertFalse(fetchedRecords.containsKey(tp1)); - records = fetchedRecords.get(tp0); + List> records = fetchedRecords.get(tp0); assertEquals(2, records.size()); assertEquals(3L, subscriptions.position(tp0).offset); assertEquals(1L, subscriptions.position(tp1).offset); From e9719012ddbcb278d9fa5d8a08935bdd00050b02 Mon Sep 17 00:00:00 2001 From: Kirk True Date: Fri, 13 Oct 2023 12:08:58 -0700 Subject: [PATCH 48/72] Grammar fixes for comments in fetch tests --- .../clients/consumer/internals/FetchRequestManagerTest.java | 2 +- .../apache/kafka/clients/consumer/internals/FetcherTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java index e6f7e6c3180a2..4e3125758bf36 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java @@ -1462,7 +1462,7 @@ public void testFetchOnCompletedFetchesForAllPausedPartitions() { public void testPartialFetchWithPausedPartitions() { // this test sends creates a completed fetch with 3 records and a max poll of 2 records to assert // that a fetch that must be returned over at least 2 polls can be cached successfully when its partition is - // paused, then returned successfully after its been resumed again later + // paused, then returned successfully after it has been resumed again later buildFetcher(2); Map>> fetchedRecords; diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java index 5474a4b783e5e..49e2409142c48 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java @@ -1460,7 +1460,7 @@ public void testFetchOnCompletedFetchesForAllPausedPartitions() { public void testPartialFetchWithPausedPartitions() { // this test sends creates a completed fetch with 3 records and a max poll of 2 records to assert // that a fetch that must be returned over at least 2 polls can be cached successfully when its partition is - // paused, then returned successfully after its been resumed again later + // paused, then returned successfully after it has been resumed again later buildFetcher(2); Map>> fetchedRecords; From 97d833e87df59487b38af3a02c7a0ba74fa565c1 Mon Sep 17 00:00:00 2001 From: Kirk True Date: Fri, 13 Oct 2023 12:13:41 -0700 Subject: [PATCH 49/72] Removed unnecessary fetchRecords call from testPartialFetchWithPausedPartitions() --- .../clients/consumer/internals/FetchRequestManagerTest.java | 2 -- .../apache/kafka/clients/consumer/internals/FetcherTest.java | 2 -- 2 files changed, 4 deletions(-) diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java index 4e3125758bf36..edfd2993d6c08 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java @@ -1482,8 +1482,6 @@ public void testPartialFetchWithPausedPartitions() { subscriptions.pause(tp0); networkClientDelegate.poll(time.timer(0)); - fetchedRecords = fetchRecords(); - assertEmptyFetch("Should not return records or advance position for paused partitions"); assertTrue(fetcher.hasCompletedFetches(), "Should have 1 entry in completed fetches"); assertFalse(fetcher.hasAvailableFetches(), "Should not have any available (non-paused) completed fetches"); diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java index 49e2409142c48..1e60332c7c51a 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java @@ -1480,8 +1480,6 @@ public void testPartialFetchWithPausedPartitions() { subscriptions.pause(tp0); consumerClient.poll(time.timer(0)); - fetchedRecords = fetchRecords(); - assertEmptyFetch("Should not return records or advance position for paused partitions"); assertTrue(fetcher.hasCompletedFetches(), "Should have 1 entry in completed fetches"); assertFalse(fetcher.hasAvailableFetches(), "Should not have any available (non-paused) completed fetches"); From 5629d2c2f89d7d1f980b0cbab4ccf4f86cad9762 Mon Sep 17 00:00:00 2001 From: Kirk True Date: Fri, 13 Oct 2023 12:43:25 -0700 Subject: [PATCH 50/72] Refactoring to use parameterized test for different fetch response errors --- .../internals/FetchRequestManagerTest.java | 101 ++++++---------- .../consumer/internals/FetcherTest.java | 110 ++++++------------ 2 files changed, 70 insertions(+), 141 deletions(-) diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java index edfd2993d6c08..e09d9ddb94cf3 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java @@ -93,6 +93,9 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import org.mockito.ArgumentCaptor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -116,6 +119,7 @@ import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; +import java.util.stream.Stream; import static java.util.Collections.emptyList; import static java.util.Collections.emptyMap; @@ -1517,45 +1521,6 @@ public void testFetchDiscardedAfterPausedPartitionResumedAndSeekedToNewOffset() assertFalse(fetcher.hasCompletedFetches(), "Should have no completed fetches"); } - @Test - public void testFetchNotLeaderOrFollower() { - buildFetcher(); - assignFromUser(singleton(tp0)); - subscriptions.seek(tp0, 0); - - assertEquals(1, sendFetches()); - client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NOT_LEADER_OR_FOLLOWER, 100L, 0)); - networkClientDelegate.poll(time.timer(0)); - assertEmptyFetch("Should not return records or advance position on fetch error"); - assertEquals(0L, metadata.timeToNextUpdate(time.milliseconds())); - } - - @Test - public void testFetchUnknownTopicOrPartition() { - buildFetcher(); - assignFromUser(singleton(tp0)); - subscriptions.seek(tp0, 0); - - assertEquals(1, sendFetches()); - client.prepareResponse(fullFetchResponse(tidp0, records, Errors.UNKNOWN_TOPIC_OR_PARTITION, 100L, 0)); - networkClientDelegate.poll(time.timer(0)); - assertEmptyFetch("Should not return records or advance position on fetch error"); - assertEquals(0L, metadata.timeToNextUpdate(time.milliseconds())); - } - - @Test - public void testFetchUnknownTopicId() { - buildFetcher(); - assignFromUser(singleton(tp0)); - subscriptions.seek(tp0, 0); - - assertEquals(1, sendFetches()); - client.prepareResponse(fullFetchResponse(tidp0, records, Errors.UNKNOWN_TOPIC_ID, -1L, 0)); - networkClientDelegate.poll(time.timer(0)); - assertEmptyFetch("Should not return records or advance position on fetch error"); - assertEquals(0L, metadata.timeToNextUpdate(time.milliseconds())); - } - @Test public void testFetchSessionIdError() { buildFetcher(); @@ -1569,45 +1534,51 @@ public void testFetchSessionIdError() { assertEquals(0L, metadata.timeToNextUpdate(time.milliseconds())); } - @Test - public void testFetchInconsistentTopicId() { + @ParameterizedTest + @MethodSource("handleFetchResponseErrorSupplier") + public void testHandleFetchResponseError(Errors error, + long highWatermark, + boolean hasTopLevelError, + boolean shouldRequestMetadataUpdate) { buildFetcher(); assignFromUser(singleton(tp0)); subscriptions.seek(tp0, 0); assertEquals(1, sendFetches()); - client.prepareResponse(fullFetchResponse(tidp0, records, Errors.INCONSISTENT_TOPIC_ID, -1L, 0)); - networkClientDelegate.poll(time.timer(0)); - assertEmptyFetch("Should not return records or advance position on fetch error"); - assertEquals(0L, metadata.timeToNextUpdate(time.milliseconds())); - } - @Test - public void testFetchFencedLeaderEpoch() { - buildFetcher(); - assignFromUser(singleton(tp0)); - subscriptions.seek(tp0, 0); + final FetchResponse fetchResponse; - assertEquals(1, sendFetches()); - client.prepareResponse(fullFetchResponse(tidp0, records, Errors.FENCED_LEADER_EPOCH, 100L, 0)); + if (hasTopLevelError) + fetchResponse = fetchResponseWithTopLevelError(tidp0, error, 0); + else + fetchResponse = fullFetchResponse(tidp0, records, error, highWatermark, 0); + + client.prepareResponse(fetchResponse); networkClientDelegate.poll(time.timer(0)); assertEmptyFetch("Should not return records or advance position on fetch error"); - assertEquals(0L, metadata.timeToNextUpdate(time.milliseconds()), "Should have requested metadata update"); - } - @Test - public void testFetchUnknownLeaderEpoch() { - buildFetcher(); - assignFromUser(singleton(tp0)); - subscriptions.seek(tp0, 0); + long timeToNextUpdate = metadata.timeToNextUpdate(time.milliseconds()); - assertEquals(1, sendFetches()); - client.prepareResponse(fullFetchResponse(tidp0, records, Errors.UNKNOWN_LEADER_EPOCH, 100L, 0)); - networkClientDelegate.poll(time.timer(0)); + if (shouldRequestMetadataUpdate) + assertEquals(0L, timeToNextUpdate, "Should have requested metadata update"); + else + assertNotEquals(0L, timeToNextUpdate, "Should not have requested metadata update"); + } - assertEmptyFetch("Should not return records or advance position on fetch error"); - assertNotEquals(0L, metadata.timeToNextUpdate(time.milliseconds()), "Should not have requested metadata update"); + /** + * Supplies parameters to {@link #testHandleFetchResponseError(Errors, long, boolean, boolean)}. + */ + private static Stream handleFetchResponseErrorSupplier() { + return Stream.of( + Arguments.of(Errors.NOT_LEADER_OR_FOLLOWER, 100L, false, true), + Arguments.of(Errors.UNKNOWN_TOPIC_OR_PARTITION, 100L, false, true), + Arguments.of(Errors.UNKNOWN_TOPIC_ID, -1L, false, true), + Arguments.of(Errors.FETCH_SESSION_TOPIC_ID_ERROR, -1L, true, true), + Arguments.of(Errors.INCONSISTENT_TOPIC_ID, -1L, false, true), + Arguments.of(Errors.FENCED_LEADER_EPOCH, 100L, false, true), + Arguments.of(Errors.UNKNOWN_LEADER_EPOCH, 100L, false, false) + ); } @Test diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java index 1e60332c7c51a..86ba691d26a95 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java @@ -90,6 +90,9 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import org.mockito.ArgumentCaptor; import java.io.DataOutputStream; @@ -116,6 +119,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; import java.util.stream.Collectors; +import java.util.stream.Stream; import static java.util.Collections.emptyList; import static java.util.Collections.emptyMap; @@ -1515,97 +1519,51 @@ public void testFetchDiscardedAfterPausedPartitionResumedAndSoughtToNewOffset() assertFalse(fetcher.hasCompletedFetches(), "Should have no completed fetches"); } - @Test - public void testFetchNotLeaderOrFollower() { + @ParameterizedTest + @MethodSource("handleFetchResponseErrorSupplier") + public void testHandleFetchResponseError(Errors error, + long highWatermark, + boolean hasTopLevelError, + boolean shouldRequestMetadataUpdate) { buildFetcher(); assignFromUser(singleton(tp0)); subscriptions.seek(tp0, 0); assertEquals(1, sendFetches()); - client.prepareResponse(fullFetchResponse(tidp0, records, Errors.NOT_LEADER_OR_FOLLOWER, 100L, 0)); - consumerClient.poll(time.timer(0)); - assertEmptyFetch("Should not return records or advance position on fetch error"); - assertEquals(0L, metadata.timeToNextUpdate(time.milliseconds())); - } - - @Test - public void testFetchUnknownTopicOrPartition() { - buildFetcher(); - assignFromUser(singleton(tp0)); - subscriptions.seek(tp0, 0); - - assertEquals(1, sendFetches()); - client.prepareResponse(fullFetchResponse(tidp0, records, Errors.UNKNOWN_TOPIC_OR_PARTITION, 100L, 0)); - consumerClient.poll(time.timer(0)); - assertEmptyFetch("Should not return records or advance position on fetch error"); - assertEquals(0L, metadata.timeToNextUpdate(time.milliseconds())); - } - @Test - public void testFetchUnknownTopicId() { - buildFetcher(); - assignFromUser(singleton(tp0)); - subscriptions.seek(tp0, 0); + final FetchResponse fetchResponse; - assertEquals(1, sendFetches()); - client.prepareResponse(fullFetchResponse(tidp0, records, Errors.UNKNOWN_TOPIC_ID, -1L, 0)); - consumerClient.poll(time.timer(0)); - assertEmptyFetch("Should not return records or advance position on fetch error"); - assertEquals(0L, metadata.timeToNextUpdate(time.milliseconds())); - } + if (hasTopLevelError) + fetchResponse = fetchResponseWithTopLevelError(tidp0, error, 0); + else + fetchResponse = fullFetchResponse(tidp0, records, error, highWatermark, 0); - @Test - public void testFetchSessionIdError() { - buildFetcher(); - assignFromUser(singleton(tp0)); - subscriptions.seek(tp0, 0); - - assertEquals(1, sendFetches()); - client.prepareResponse(fetchResponseWithTopLevelError(tidp0, Errors.FETCH_SESSION_TOPIC_ID_ERROR, 0)); + client.prepareResponse(fetchResponse); consumerClient.poll(time.timer(0)); - assertEmptyFetch("Should not return records or advance position on fetch error"); - assertEquals(0L, metadata.timeToNextUpdate(time.milliseconds())); - } - - @Test - public void testFetchInconsistentTopicId() { - buildFetcher(); - assignFromUser(singleton(tp0)); - subscriptions.seek(tp0, 0); - assertEquals(1, sendFetches()); - client.prepareResponse(fullFetchResponse(tidp0, records, Errors.INCONSISTENT_TOPIC_ID, -1L, 0)); - consumerClient.poll(time.timer(0)); assertEmptyFetch("Should not return records or advance position on fetch error"); - assertEquals(0L, metadata.timeToNextUpdate(time.milliseconds())); - } - - @Test - public void testFetchFencedLeaderEpoch() { - buildFetcher(); - assignFromUser(singleton(tp0)); - subscriptions.seek(tp0, 0); - assertEquals(1, sendFetches()); - client.prepareResponse(fullFetchResponse(tidp0, records, Errors.FENCED_LEADER_EPOCH, 100L, 0)); - consumerClient.poll(time.timer(0)); + long timeToNextUpdate = metadata.timeToNextUpdate(time.milliseconds()); - assertEmptyFetch("Should not return records or advance position on fetch error"); - assertEquals(0L, metadata.timeToNextUpdate(time.milliseconds()), "Should have requested metadata update"); + if (shouldRequestMetadataUpdate) + assertEquals(0L, timeToNextUpdate, "Should have requested metadata update"); + else + assertNotEquals(0L, timeToNextUpdate, "Should not have requested metadata update"); } - @Test - public void testFetchUnknownLeaderEpoch() { - buildFetcher(); - assignFromUser(singleton(tp0)); - subscriptions.seek(tp0, 0); - - assertEquals(1, sendFetches()); - client.prepareResponse(fullFetchResponse(tidp0, records, Errors.UNKNOWN_LEADER_EPOCH, 100L, 0)); - consumerClient.poll(time.timer(0)); - - assertEmptyFetch("Should not return records or advance position on fetch error"); - assertNotEquals(0L, metadata.timeToNextUpdate(time.milliseconds()), "Should not have requested metadata update"); + /** + * Supplies parameters to {@link #testHandleFetchResponseError(Errors, long, boolean, boolean)}. + */ + private static Stream handleFetchResponseErrorSupplier() { + return Stream.of( + Arguments.of(Errors.NOT_LEADER_OR_FOLLOWER, 100L, false, true), + Arguments.of(Errors.UNKNOWN_TOPIC_OR_PARTITION, 100L, false, true), + Arguments.of(Errors.UNKNOWN_TOPIC_ID, -1L, false, true), + Arguments.of(Errors.FETCH_SESSION_TOPIC_ID_ERROR, -1L, true, true), + Arguments.of(Errors.INCONSISTENT_TOPIC_ID, -1L, false, true), + Arguments.of(Errors.FENCED_LEADER_EPOCH, 100L, false, true), + Arguments.of(Errors.UNKNOWN_LEADER_EPOCH, 100L, false, false) + ); } @Test From 913fc8b3cad31dc698a43f63c8c8c73a4eec5bb2 Mon Sep 17 00:00:00 2001 From: Kirk True Date: Fri, 13 Oct 2023 12:48:10 -0700 Subject: [PATCH 51/72] Added check for no pending requests in testEpochSetInFetchRequest --- .../clients/consumer/internals/FetchRequestManagerTest.java | 1 + .../org/apache/kafka/clients/consumer/internals/FetcherTest.java | 1 + 2 files changed, 2 insertions(+) diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java index e09d9ddb94cf3..340ca0392aa7c 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java @@ -1608,6 +1608,7 @@ public void testEpochSetInFetchRequest() { }; client.prepareResponse(matcher, fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); networkClientDelegate.pollNoWakeup(); + assertEquals(0, networkClientDelegate.pendingRequestCount()); } @Test diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java index 86ba691d26a95..db12894129c87 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java @@ -1593,6 +1593,7 @@ public void testEpochSetInFetchRequest() { }; client.prepareResponse(matcher, fullFetchResponse(tidp0, records, Errors.NONE, 100L, 0)); consumerClient.pollNoWakeup(); + assertEquals(0, consumerClient.pendingRequestCount()); } @Test From e4c416c8d3ded4fb6b8a4b02035fb193f2bc17b2 Mon Sep 17 00:00:00 2001 From: Kirk True Date: Fri, 13 Oct 2023 13:20:35 -0700 Subject: [PATCH 52/72] Added clarifying comments to testFetchedRecordsAfterSeek Added this blurb about why offsets are not marked as being needed in the SubscriptionState: // The partition is not marked as needing its offset reset because that error handling logic is // performed during the fetch collection. When we call seek() before we collect the fetch, the // partition's position is updated (to offset 2) which is different from the offset from which // we fetched the data (from offset 0). --- .../clients/consumer/internals/FetchRequestManagerTest.java | 5 +++++ .../apache/kafka/clients/consumer/internals/FetcherTest.java | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java index 340ca0392aa7c..c1e8e89a9461d 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java @@ -1654,6 +1654,11 @@ public void testFetchedRecordsAfterSeek() { assertTrue(sendFetches() > 0); client.prepareResponse(fullFetchResponse(tidp0, records, Errors.OFFSET_OUT_OF_RANGE, 100L, 0)); networkClientDelegate.poll(time.timer(0)); + + // The partition is not marked as needing its offset reset because that error handling logic is + // performed during the fetch collection. When we call seek() before we collect the fetch, the + // partition's position is updated (to offset 2) which is different from the offset from which + // we fetched the data (from offset 0). assertFalse(subscriptions.isOffsetResetNeeded(tp0)); subscriptions.seek(tp0, 2); assertEmptyFetch("Should not return records or advance position after seeking to end of topic partition"); diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java index db12894129c87..0db827fc07a5b 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java @@ -1639,6 +1639,11 @@ public void testFetchedRecordsAfterSeek() { assertTrue(sendFetches() > 0); client.prepareResponse(fullFetchResponse(tidp0, records, Errors.OFFSET_OUT_OF_RANGE, 100L, 0)); consumerClient.poll(time.timer(0)); + + // The partition is not marked as needing its offset reset because that error handling logic is + // performed during the fetch collection. When we call seek() before we collect the fetch, the + // partition's position is updated (to offset 2) which is different from the offset from which + // we fetched the data (from offset 0). assertFalse(subscriptions.isOffsetResetNeeded(tp0)); subscriptions.seek(tp0, 2); assertEmptyFetch("Should not return records or advance position after seeking to end of topic partition"); From dd956615efd33eaf99b30b133ca93434db9dd552 Mon Sep 17 00:00:00 2001 From: Kirk True Date: Fri, 13 Oct 2023 15:35:57 -0700 Subject: [PATCH 53/72] Updated fetch tests to use fetchRecordsInto utility method to reduce boilerplate code --- .../internals/FetchRequestManagerTest.java | 16 ++++------------ .../clients/consumer/internals/FetcherTest.java | 16 ++++------------ 2 files changed, 8 insertions(+), 24 deletions(-) diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java index c1e8e89a9461d..f0bad8cf15a96 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java @@ -1771,9 +1771,7 @@ public void testCompletedFetchRemoval() { networkClientDelegate.poll(time.timer(0)); List> fetchedRecords = new ArrayList<>(); - Map>> recordsByPartition = fetchRecords(); - for (List> records : recordsByPartition.values()) - fetchedRecords.addAll(records); + fetchRecordsInto(fetchedRecords); assertEquals(fetchedRecords.size(), subscriptions.position(tp1).offset - 1); assertEquals(4, subscriptions.position(tp1).offset); @@ -1781,9 +1779,7 @@ public void testCompletedFetchRemoval() { List oorExceptions = new ArrayList<>(); try { - recordsByPartition = fetchRecords(); - for (List> records : recordsByPartition.values()) - fetchedRecords.addAll(records); + fetchRecordsInto(fetchedRecords); } catch (OffsetOutOfRangeException oor) { oorExceptions.add(oor); } @@ -1794,9 +1790,7 @@ public void testCompletedFetchRemoval() { assertTrue(oor.offsetOutOfRangePartitions().containsKey(tp0)); assertEquals(oor.offsetOutOfRangePartitions().size(), 1); - recordsByPartition = fetchRecords(); - for (List> records : recordsByPartition.values()) - fetchedRecords.addAll(records); + fetchRecordsInto(fetchedRecords); // Should not have received an Exception for tp2. assertEquals(6, subscriptions.position(tp2).offset); @@ -1806,9 +1800,7 @@ public void testCompletedFetchRemoval() { List kafkaExceptions = new ArrayList<>(); for (int i = 1; i <= numExceptionsExpected; i++) { try { - recordsByPartition = fetchRecords(); - for (List> records : recordsByPartition.values()) - fetchedRecords.addAll(records); + fetchRecordsInto(fetchedRecords); } catch (KafkaException e) { kafkaExceptions.add(e); } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java index 0db827fc07a5b..234791a663d58 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java @@ -1756,9 +1756,7 @@ public void testCompletedFetchRemoval() { consumerClient.poll(time.timer(0)); List> fetchedRecords = new ArrayList<>(); - Map>> recordsByPartition = fetchRecords(); - for (List> records : recordsByPartition.values()) - fetchedRecords.addAll(records); + fetchRecordsInto(fetchedRecords); assertEquals(fetchedRecords.size(), subscriptions.position(tp1).offset - 1); assertEquals(4, subscriptions.position(tp1).offset); @@ -1766,9 +1764,7 @@ public void testCompletedFetchRemoval() { List oorExceptions = new ArrayList<>(); try { - recordsByPartition = fetchRecords(); - for (List> records : recordsByPartition.values()) - fetchedRecords.addAll(records); + fetchRecordsInto(fetchedRecords); } catch (OffsetOutOfRangeException oor) { oorExceptions.add(oor); } @@ -1779,9 +1775,7 @@ public void testCompletedFetchRemoval() { assertTrue(oor.offsetOutOfRangePartitions().containsKey(tp0)); assertEquals(oor.offsetOutOfRangePartitions().size(), 1); - recordsByPartition = fetchRecords(); - for (List> records : recordsByPartition.values()) - fetchedRecords.addAll(records); + fetchRecordsInto(fetchedRecords); // Should not have received an Exception for tp2. assertEquals(6, subscriptions.position(tp2).offset); @@ -1791,9 +1785,7 @@ public void testCompletedFetchRemoval() { List kafkaExceptions = new ArrayList<>(); for (int i = 1; i <= numExceptionsExpected; i++) { try { - recordsByPartition = fetchRecords(); - for (List> records : recordsByPartition.values()) - fetchedRecords.addAll(records); + fetchRecordsInto(fetchedRecords); } catch (KafkaException e) { kafkaExceptions.add(e); } From b752bf84621102562127a07ea4a9801dd1432a0d Mon Sep 17 00:00:00 2001 From: Kirk True Date: Fri, 13 Oct 2023 16:17:44 -0700 Subject: [PATCH 54/72] Fixed typo in comments --- .../clients/consumer/internals/FetchRequestManagerTest.java | 2 +- .../apache/kafka/clients/consumer/internals/FetcherTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java index f0bad8cf15a96..00e8c9476b6fa 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java @@ -1857,7 +1857,7 @@ public void testFetchDisconnected() { networkClientDelegate.poll(time.timer(0)); assertEmptyFetch("Should not return records or advance position on disconnect"); - // disconnects should have no affect on subscription state + // disconnects should have no effect on subscription state assertFalse(subscriptions.isOffsetResetNeeded(tp0)); assertTrue(subscriptions.isFetchable(tp0)); assertEquals(0, subscriptions.position(tp0).offset); diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java index 234791a663d58..131eefee5e87c 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java @@ -1842,7 +1842,7 @@ public void testFetchDisconnected() { consumerClient.poll(time.timer(0)); assertEmptyFetch("Should not return records or advance position on disconnect"); - // disconnects should have no affect on subscription state + // disconnects should have no effect on subscription state assertFalse(subscriptions.isOffsetResetNeeded(tp0)); assertTrue(subscriptions.isFetchable(tp0)); assertEquals(0, subscriptions.position(tp0).offset); From 8458f2c9fe58112443fad21e8575a31491557abe Mon Sep 17 00:00:00 2001 From: Kirk True Date: Fri, 13 Oct 2023 16:25:27 -0700 Subject: [PATCH 55/72] Renamed and swapped 'expected' vs. 'actual' comparison in testMultipleAbortMarkers --- .../clients/consumer/internals/FetchRequestManagerTest.java | 4 ++-- .../apache/kafka/clients/consumer/internals/FetcherTest.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java index 00e8c9476b6fa..0213693b3e0bf 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java @@ -2449,12 +2449,12 @@ public void testMultipleAbortMarkers() { assertTrue(fetchedRecords.containsKey(tp0)); assertEquals(fetchedRecords.get(tp0).size(), 2); List> fetchedConsumerRecords = fetchedRecords.get(tp0); - Set committedKeys = new HashSet<>(Arrays.asList("commit1-1", "commit1-2")); + Set expectedCommittedKeys = new HashSet<>(Arrays.asList("commit1-1", "commit1-2")); Set actuallyCommittedKeys = new HashSet<>(); for (ConsumerRecord consumerRecord : fetchedConsumerRecords) { actuallyCommittedKeys.add(new String(consumerRecord.key(), StandardCharsets.UTF_8)); } - assertEquals(actuallyCommittedKeys, committedKeys); + assertEquals(expectedCommittedKeys, actuallyCommittedKeys); } @Test diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java index 131eefee5e87c..bd60beba7f15c 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java @@ -2434,12 +2434,12 @@ public void testMultipleAbortMarkers() { assertTrue(fetchedRecords.containsKey(tp0)); assertEquals(fetchedRecords.get(tp0).size(), 2); List> fetchedConsumerRecords = fetchedRecords.get(tp0); - Set committedKeys = new HashSet<>(Arrays.asList("commit1-1", "commit1-2")); + Set expectedCommittedKeys = new HashSet<>(Arrays.asList("commit1-1", "commit1-2")); Set actuallyCommittedKeys = new HashSet<>(); for (ConsumerRecord consumerRecord : fetchedConsumerRecords) { actuallyCommittedKeys.add(new String(consumerRecord.key(), StandardCharsets.UTF_8)); } - assertEquals(actuallyCommittedKeys, committedKeys); + assertEquals(expectedCommittedKeys, actuallyCommittedKeys); } @Test From 7e39a74764798b912e78adcb62f636fd6e03b3b2 Mon Sep 17 00:00:00 2001 From: Kirk True Date: Fri, 13 Oct 2023 16:28:55 -0700 Subject: [PATCH 56/72] Changed testReturnAbortedTransactionsinUncommittedMode() to testReturnAbortedTransactionsInUncommittedMode() --- .../clients/consumer/internals/FetchRequestManagerTest.java | 4 ++-- .../apache/kafka/clients/consumer/internals/FetcherTest.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java index 0213693b3e0bf..5268633b70715 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java @@ -2638,7 +2638,7 @@ public void testReadCommittedWithCompactedTopic() { } @Test - public void testReturnAbortedTransactionsinUncommittedMode() { + public void testReturnAbortedTransactionsInUncommittedMode() { buildFetcher(OffsetResetStrategy.EARLIEST, new ByteArrayDeserializer(), new ByteArrayDeserializer(), Integer.MAX_VALUE, IsolationLevel.READ_UNCOMMITTED); ByteBuffer buffer = ByteBuffer.allocate(1024); @@ -3247,7 +3247,7 @@ private FetchResponse fetchResponse(TopicIdPartition tp, MemoryRecords records, } /** - * Assert that the {@link Fetcher#collectFetch()} latest fetch} does not contain any + * Assert that the {@link Fetcher#collectFetch() latest fetch} does not contain any * {@link Fetch#records() user-visible records}, did not * {@link Fetch#positionAdvanced() advance the consumer's position}, * and is {@link Fetch#isEmpty() empty}. diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java index bd60beba7f15c..2040d5801bdbe 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java @@ -2623,7 +2623,7 @@ public void testReadCommittedWithCompactedTopic() { } @Test - public void testReturnAbortedTransactionsinUncommittedMode() { + public void testReturnAbortedTransactionsInUncommittedMode() { buildFetcher(OffsetResetStrategy.EARLIEST, new ByteArrayDeserializer(), new ByteArrayDeserializer(), Integer.MAX_VALUE, IsolationLevel.READ_UNCOMMITTED); ByteBuffer buffer = ByteBuffer.allocate(1024); From 2bd3635041a67d0e00912d08063cea3e147100a4 Mon Sep 17 00:00:00 2001 From: Kirk True Date: Fri, 13 Oct 2023 16:29:40 -0700 Subject: [PATCH 57/72] Grammar/typo fix --- .../clients/consumer/internals/FetchRequestManagerTest.java | 2 +- .../apache/kafka/clients/consumer/internals/FetcherTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java index 5268633b70715..36821abde4d81 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java @@ -2922,7 +2922,7 @@ public void testPreferredReadReplica() { client.updateMetadata(RequestTestUtils.metadataUpdateWithIds(2, singletonMap(topicName, 4), tp -> validLeaderEpoch, topicIds, false)); subscriptions.seek(tp0, 0); - // Node preferred replica before first fetch response + // Take note of the preferred replica before the first fetch response Node selected = fetcher.selectReadReplica(tp0, Node.noNode(), time.milliseconds()); assertEquals(-1, selected.id()); diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java index 2040d5801bdbe..01997e2b8d56f 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java @@ -3192,7 +3192,7 @@ public void testPreferredReadReplica() { client.updateMetadata(RequestTestUtils.metadataUpdateWithIds(2, singletonMap(topicName, 4), tp -> validLeaderEpoch, topicIds, false)); subscriptions.seek(tp0, 0); - // Node preferred replica before first fetch response + // Take note of the preferred replica before the first fetch response Node selected = fetcher.selectReadReplica(tp0, Node.noNode(), time.milliseconds()); assertEquals(-1, selected.id()); From cd0e9aea2b5b7d3c0aa1c50e06fbd2a1053b31e1 Mon Sep 17 00:00:00 2001 From: Kirk True Date: Fri, 13 Oct 2023 16:30:56 -0700 Subject: [PATCH 58/72] Moved comment in testPreferredReadReplicaOffsetError closer to applicable code --- .../clients/consumer/internals/FetchRequestManagerTest.java | 2 +- .../apache/kafka/clients/consumer/internals/FetcherTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java index 36821abde4d81..44b1ab0406c1a 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java @@ -3082,10 +3082,10 @@ public void testPreferredReadReplicaOffsetError() { Node selected = fetcher.selectReadReplica(tp0, Node.noNode(), time.milliseconds()); assertEquals(selected.id(), 1); - // Return an error, should unset the preferred read replica assertEquals(1, sendFetches()); assertFalse(fetcher.hasCompletedFetches()); + // Return an error, should unset the preferred read replica client.prepareResponse(fullFetchResponse(tidp0, records, Errors.OFFSET_OUT_OF_RANGE, 100L, FetchResponse.INVALID_LAST_STABLE_OFFSET, 0, Optional.empty())); networkClientDelegate.poll(time.timer(0)); diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java index 01997e2b8d56f..8b9c4fa97544f 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java @@ -3352,10 +3352,10 @@ public void testPreferredReadReplicaOffsetError() { Node selected = fetcher.selectReadReplica(tp0, Node.noNode(), time.milliseconds()); assertEquals(selected.id(), 1); - // Return an error, should unset the preferred read replica assertEquals(1, sendFetches()); assertFalse(fetcher.hasCompletedFetches()); + // Return an error, should unset the preferred read replica client.prepareResponse(fullFetchResponse(tidp0, records, Errors.OFFSET_OUT_OF_RANGE, 100L, FetchResponse.INVALID_LAST_STABLE_OFFSET, 0, Optional.empty())); consumerClient.poll(time.timer(0)); From b18720fe64d76c72331a98782f28423c19e08bf8 Mon Sep 17 00:00:00 2001 From: Kirk True Date: Fri, 13 Oct 2023 17:22:42 -0700 Subject: [PATCH 59/72] Removed FetchEvent and added wakeup() method to ApplicationEventHandler --- .../consumer/internals/FetchBuffer.java | 2 +- .../internals/PrototypeAsyncConsumer.java | 23 +++----- .../internals/events/ApplicationEvent.java | 2 +- .../events/ApplicationEventHandler.java | 12 ++++- .../events/ApplicationEventProcessor.java | 8 --- .../consumer/internals/events/FetchEvent.java | 32 ------------ .../internals/ConsumerNetworkThreadTest.java | 17 +++--- .../events/ApplicationEventHandlerTest.java | 52 ------------------- 8 files changed, 27 insertions(+), 121 deletions(-) delete mode 100644 clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/FetchEvent.java delete mode 100644 clients/src/test/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventHandlerTest.java diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchBuffer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchBuffer.java index 9b27ea6438dbe..de7f88ab72596 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchBuffer.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/FetchBuffer.java @@ -173,7 +173,7 @@ void awaitNotEmpty(Timer timer) { if (timer.isExpired()) break; - if (notEmptyCondition.await(timer.remainingMs(), TimeUnit.MILLISECONDS)) { + if (!notEmptyCondition.await(timer.remainingMs(), TimeUnit.MILLISECONDS)) { break; } } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java index 4b8105d6a880a..5041dca51133a 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java @@ -40,7 +40,6 @@ import org.apache.kafka.clients.consumer.internals.events.BackgroundEventProcessor; import org.apache.kafka.clients.consumer.internals.events.ApplicationEventHandler; import org.apache.kafka.clients.consumer.internals.events.CommitApplicationEvent; -import org.apache.kafka.clients.consumer.internals.events.FetchEvent; import org.apache.kafka.clients.consumer.internals.events.ListOffsetsApplicationEvent; import org.apache.kafka.clients.consumer.internals.events.NewTopicsMetadataUpdateRequestEvent; import org.apache.kafka.clients.consumer.internals.events.OffsetFetchApplicationEvent; @@ -339,7 +338,8 @@ public ConsumerRecords poll(final Duration timeout) { final Fetch fetch = pollForFetches(timer); if (!fetch.isEmpty()) { - sendFetches(); + // Notify the network thread to wake up and start the next round of fetching. + applicationEventHandler.wakeup(); if (fetch.records().isEmpty()) { log.trace("Returning empty records from `poll()` " @@ -908,15 +908,6 @@ WakeupTrigger wakeupTrigger() { return wakeupTrigger; } - /** - * Send the requests for fetch data to the {@link ConsumerNetworkThread network thread} and set up to - * collect the results in {@link #fetchBuffer}. - */ - private void sendFetches() { - FetchEvent event = new FetchEvent(); - applicationEventHandler.add(event); - } - private Fetch pollForFetches(Timer timer) { long pollTimeout = timer.remainingMs(); @@ -926,8 +917,8 @@ private Fetch pollForFetches(Timer timer) { return fetch; } - // send any new fetches (won't resend pending fetches) - sendFetches(); + // Wake up the network thread to send any new fetches (won't resend pending fetches) + applicationEventHandler.wakeup(); // We do not want to be stuck blocking in poll if we are missing some positions // since the offset lookup may be backing off after a failure @@ -942,9 +933,9 @@ private Fetch pollForFetches(Timer timer) { Timer pollTimer = time.timer(pollTimeout); - // Attempt to fetch any data. It's OK if we don't have any waiting data here; it's a 'best case' effort. The - // data may not be immediately available, but the calling method (poll) will correctly - // handle the overall timeout. + // Wait a bit for some fetched data to arrive, as there may not be anything immediately available. Note the + // use of a shorter, dedicated "pollTimer" here which updates "timer" so that calling method (poll) will + // correctly handle the overall timeout. try { fetchBuffer.awaitNotEmpty(pollTimer); } catch (InterruptException e) { diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEvent.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEvent.java index 3a1530fccdf9d..133836da3b753 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEvent.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEvent.java @@ -25,7 +25,7 @@ public abstract class ApplicationEvent { public enum Type { COMMIT, POLL, FETCH_COMMITTED_OFFSET, METADATA_UPDATE, ASSIGNMENT_CHANGE, - LIST_OFFSETS, RESET_POSITIONS, VALIDATE_POSITIONS, TOPIC_METADATA, FETCH + LIST_OFFSETS, RESET_POSITIONS, VALIDATE_POSITIONS, TOPIC_METADATA } private final Type type; diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventHandler.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventHandler.java index 042d0af9c9e30..18dcf94c0acfe 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventHandler.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventHandler.java @@ -62,15 +62,23 @@ public ApplicationEventHandler(final LogContext logContext, } /** - * Add an {@link ApplicationEvent} to the handler. + * Add an {@link ApplicationEvent} to the handler and then internally invoke {@link #wakeup} to alert the + * network I/O thread that it has something to process. * * @param event An {@link ApplicationEvent} created by the application thread */ public void add(final ApplicationEvent event) { Objects.requireNonNull(event, "ApplicationEvent provided to add must be non-null"); log.trace("Enqueued event: {}", event); - networkThread.wakeup(); applicationEventQueue.add(event); + wakeup(); + } + + /** + * Wakeup the {@link ConsumerNetworkThread network I/O thread} to pull the event from the queue. + */ + public void wakeup() { + networkThread.wakeup(); } /** diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventProcessor.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventProcessor.java index 486b1ff1cf4a9..ccbdd21b9dc24 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventProcessor.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventProcessor.java @@ -96,10 +96,6 @@ public void process(ApplicationEvent event) { process((ListOffsetsApplicationEvent) event); return; - case FETCH: - process((FetchEvent) event); - return; - case RESET_POSITIONS: processResetPositionsEvent(); return; @@ -184,10 +180,6 @@ private void process(final TopicMetadataApplicationEvent event) { event.chain(future); } - private void process(final FetchEvent event) { - // This is currently unused. - } - /** * Creates a {@link Supplier} for deferred creation during invocation by * {@link ConsumerNetworkThread}. diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/FetchEvent.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/FetchEvent.java deleted file mode 100644 index 79628da140dd6..0000000000000 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/FetchEvent.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.kafka.clients.consumer.internals.events; - -import org.apache.kafka.clients.consumer.internals.CompletedFetch; -import org.apache.kafka.clients.consumer.internals.ConsumerNetworkThread; - -import java.util.Queue; - -/** - * This event signals the {@link ConsumerNetworkThread network thread} to submit a fetch request. - */ -public class FetchEvent extends CompletableApplicationEvent> { - - public FetchEvent() { - super(Type.FETCH); - } -} diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThreadTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThreadTest.java index ef1ad00503dab..812999b3ca340 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThreadTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThreadTest.java @@ -22,7 +22,7 @@ import org.apache.kafka.clients.consumer.internals.events.ApplicationEventProcessor; import org.apache.kafka.clients.consumer.internals.events.AssignmentChangeApplicationEvent; import org.apache.kafka.clients.consumer.internals.events.CommitApplicationEvent; -import org.apache.kafka.clients.consumer.internals.events.FetchEvent; +import org.apache.kafka.clients.consumer.internals.events.CompletableApplicationEvent; import org.apache.kafka.clients.consumer.internals.events.ListOffsetsApplicationEvent; import org.apache.kafka.clients.consumer.internals.events.NewTopicsMetadataUpdateRequestEvent; import org.apache.kafka.clients.consumer.internals.events.ResetPositionsApplicationEvent; @@ -46,7 +46,6 @@ import java.util.HashMap; import java.util.Map; import java.util.Optional; -import java.util.Queue; import java.util.concurrent.BlockingQueue; import java.util.concurrent.CompletableFuture; @@ -122,7 +121,7 @@ public void testStartupAndTearDown() throws InterruptedException { @Test public void testApplicationEvent() { - FetchEvent e = new FetchEvent(); + ApplicationEvent e = new CommitApplicationEvent(new HashMap<>()); applicationEventsQueue.add(e); consumerNetworkThread.runOnce(); verify(applicationEventProcessor, times(1)).process(e); @@ -245,12 +244,12 @@ void testEnsureMetadataUpdateOnPoll() { @Test void testEnsureEventsAreCompleted() { - FetchEvent event = spy(new FetchEvent()); - ApplicationEvent e = new CommitApplicationEvent(Collections.emptyMap()); - CompletableFuture> future = new CompletableFuture<>(); - when(event.future()).thenReturn(future); - applicationEventsQueue.add(event); - applicationEventsQueue.add(e); + CompletableApplicationEvent event1 = spy(new CommitApplicationEvent(Collections.emptyMap())); + ApplicationEvent event2 = new CommitApplicationEvent(Collections.emptyMap()); + CompletableFuture future = new CompletableFuture<>(); + when(event1.future()).thenReturn(future); + applicationEventsQueue.add(event1); + applicationEventsQueue.add(event2); assertFalse(future.isDone()); assertFalse(applicationEventsQueue.isEmpty()); diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventHandlerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventHandlerTest.java deleted file mode 100644 index 6dfeef40c3d49..0000000000000 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventHandlerTest.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.kafka.clients.consumer.internals.events; - -import org.apache.kafka.clients.consumer.internals.ConsumerTestBuilder; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.util.concurrent.BlockingQueue; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -public class ApplicationEventHandlerTest { - - private ConsumerTestBuilder.ApplicationEventHandlerTestBuilder testBuilder; - private ApplicationEventHandler applicationEventHandler; - private BlockingQueue applicationEventQueue; - - @BeforeEach - public void setup() { - testBuilder = new ConsumerTestBuilder.ApplicationEventHandlerTestBuilder(); - applicationEventHandler = testBuilder.applicationEventHandler; - applicationEventQueue = testBuilder.applicationEventQueue; - } - - @AfterEach - public void tearDown() { - if (testBuilder != null) - testBuilder.close(); - } - - @Test - public void testBasicHandlerOps() { - applicationEventHandler.add(new FetchEvent()); - assertEquals(1, applicationEventQueue.size()); - } -} From f6093d6a61ed164a0437d5a0d5a5f344ed6a6cf0 Mon Sep 17 00:00:00 2001 From: Kirk True Date: Mon, 16 Oct 2023 16:06:49 -0700 Subject: [PATCH 60/72] Updates to change references to method namews A number of the tests reference a nonexistent method named fetchedRequests. The correct name is fetchRequests. References were changed to either fetchRequests or collectFetch as appropriate. Some minor refactoring was also introduced to use assertThrows where possible. --- .../internals/FetchRequestManagerTest.java | 61 +++++++----------- .../consumer/internals/FetcherTest.java | 62 +++++++------------ 2 files changed, 42 insertions(+), 81 deletions(-) diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java index 44b1ab0406c1a..f95f2f3a1770f 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java @@ -883,15 +883,12 @@ public byte[] deserialize(String topic, byte[] data) { assertEquals(1, sendFetches()); networkClientDelegate.poll(time.timer(0)); - // The fetcher should block on Deserialization error + for (int i = 0; i < 2; i++) { - try { - collectFetch(); - fail("fetchedRecords should have raised"); - } catch (SerializationException e) { - // the position should not advance since no data has been returned - assertEquals(1, subscriptions.position(tp0).offset); - } + // The fetcher should block on Deserialization error + assertThrows(SerializationException.class, this::collectFetch); + // the position should not advance since no data has been returned + assertEquals(1, subscriptions.position(tp0).offset); } } @@ -946,7 +943,7 @@ public void testParseCorruptedRecord() throws Exception { client.prepareResponse(fullFetchResponse(tidp0, MemoryRecords.readableRecords(buffer), Errors.NONE, 100L, 0)); networkClientDelegate.poll(time.timer(0)); - // the first fetchedRecords() should return the first valid message + // the first fetchRecords() should return the first valid message assertEquals(1, fetchRecords().get(tp0).size()); assertEquals(1, subscriptions.position(tp0).offset); @@ -964,14 +961,10 @@ public void testParseCorruptedRecord() throws Exception { } private void ensureBlockOnRecord(long blockedOffset) { - // the fetchedRecords() should always throw exception due to the invalid message at the starting offset. for (int i = 0; i < 2; i++) { - try { - fetchRecords(); - fail("fetchedRecords should have raised KafkaException"); - } catch (KafkaException e) { - assertEquals(blockedOffset, subscriptions.position(tp0).offset); - } + // the fetchRecords() should always throw exception due to the invalid message at the starting offset. + assertThrows(KafkaException.class, this::fetchRecords); + assertEquals(blockedOffset, subscriptions.position(tp0).offset); } } @@ -1020,14 +1013,10 @@ public void testInvalidDefaultRecordBatch() { client.prepareResponse(fullFetchResponse(tidp0, MemoryRecords.readableRecords(buffer), Errors.NONE, 100L, 0)); networkClientDelegate.poll(time.timer(0)); - // the fetchedRecords() should always throw exception due to the bad batch. for (int i = 0; i < 2; i++) { - try { - collectFetch(); - fail("fetchedRecords should have raised KafkaException"); - } catch (KafkaException e) { - assertEquals(0, subscriptions.position(tp0).offset); - } + // the fetchRecords() should always throw exception due to the bad batch. + assertThrows(KafkaException.class, this::collectFetch); + assertEquals(0, subscriptions.position(tp0).offset); } } @@ -1051,13 +1040,10 @@ public void testParseInvalidRecordBatch() { assertEquals(1, sendFetches()); client.prepareResponse(fullFetchResponse(tidp0, MemoryRecords.readableRecords(buffer), Errors.NONE, 100L, 0)); networkClientDelegate.poll(time.timer(0)); - try { - collectFetch(); - fail("fetchedRecords should have raised"); - } catch (KafkaException e) { - // the position should not advance since no data has been returned - assertEquals(0, subscriptions.position(tp0).offset); - } + + assertThrows(KafkaException.class, this::collectFetch); + // the position should not advance since no data has been returned + assertEquals(0, subscriptions.position(tp0).offset); } @Test @@ -1223,14 +1209,9 @@ public void testFetchRequestWhenRecordTooLarge() { client.setNodeApiVersions(NodeApiVersions.create(ApiKeys.FETCH.id, (short) 2, (short) 2)); makeFetchRequestWithIncompleteRecord(); - try { - collectFetch(); - fail("RecordTooLargeException should have been raised"); - } catch (RecordTooLargeException e) { - assertTrue(e.getMessage().startsWith("There are some messages at [Partition=Offset]: ")); - // the position should not advance since no data has been returned - assertEquals(0, subscriptions.position(tp0).offset); - } + assertThrows(RecordTooLargeException.class, this::collectFetch); + // the position should not advance since no data has been returned + assertEquals(0, subscriptions.position(tp0).offset); } finally { client.setNodeApiVersions(NodeApiVersions.create()); } @@ -1248,7 +1229,7 @@ public void testFetchRequestInternalError() { makeFetchRequestWithIncompleteRecord(); try { collectFetch(); - fail("RecordTooLargeException should have been raised"); + fail("collectFetch should have thrown a KafkaException"); } catch (KafkaException e) { assertTrue(e.getMessage().startsWith("Failed to make progress reading messages")); // the position should not advance since no data has been returned @@ -1281,7 +1262,7 @@ public void testUnauthorizedTopic() { networkClientDelegate.poll(time.timer(0)); try { collectFetch(); - fail("fetchedRecords should have thrown"); + fail("collectFetch should have thrown a TopicAuthorizationException"); } catch (TopicAuthorizationException e) { assertEquals(singleton(topicName), e.unauthorizedTopics()); } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java index 8b9c4fa97544f..55e79eb64c2fc 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java @@ -880,15 +880,12 @@ public byte[] deserialize(String topic, byte[] data) { assertEquals(1, sendFetches()); consumerClient.poll(time.timer(0)); - // The fetcher should block on Deserialization error + for (int i = 0; i < 2; i++) { - try { - collectFetch(); - fail("fetchedRecords should have raised"); - } catch (SerializationException e) { - // the position should not advance since no data has been returned - assertEquals(1, subscriptions.position(tp0).offset); - } + // The fetcher should block on Deserialization error + assertThrows(SerializationException.class, this::collectFetch); + // the position should not advance since no data has been returned + assertEquals(1, subscriptions.position(tp0).offset); } } @@ -943,7 +940,7 @@ public void testParseCorruptedRecord() throws Exception { client.prepareResponse(fullFetchResponse(tidp0, MemoryRecords.readableRecords(buffer), Errors.NONE, 100L, 0)); consumerClient.poll(time.timer(0)); - // the first fetchedRecords() should return the first valid message + // the first fetchRecords() should return the first valid message assertEquals(1, fetchRecords().get(tp0).size()); assertEquals(1, subscriptions.position(tp0).offset); @@ -961,15 +958,10 @@ public void testParseCorruptedRecord() throws Exception { } private void ensureBlockOnRecord(long blockedOffset) { - // - // the fetchedRecords() should always throw exception due to the invalid message at the starting offset. for (int i = 0; i < 2; i++) { - try { - fetchRecords(); - fail("fetchedRecords should have raised KafkaException"); - } catch (KafkaException e) { - assertEquals(blockedOffset, subscriptions.position(tp0).offset); - } + // the fetchRecords() should always throw exception due to the invalid message at the starting offset. + assertThrows(KafkaException.class, this::fetchRecords); + assertEquals(blockedOffset, subscriptions.position(tp0).offset); } } @@ -1018,14 +1010,10 @@ public void testInvalidDefaultRecordBatch() { client.prepareResponse(fullFetchResponse(tidp0, MemoryRecords.readableRecords(buffer), Errors.NONE, 100L, 0)); consumerClient.poll(time.timer(0)); - // the fetchedRecords() should always throw exception due to the bad batch. for (int i = 0; i < 2; i++) { - try { - collectFetch(); - fail("fetchedRecords should have raised KafkaException"); - } catch (KafkaException e) { - assertEquals(0, subscriptions.position(tp0).offset); - } + // collectFetch() should always throw exception due to the bad batch. + assertThrows(KafkaException.class, this::collectFetch); + assertEquals(0, subscriptions.position(tp0).offset); } } @@ -1049,13 +1037,10 @@ public void testParseInvalidRecordBatch() { assertEquals(1, sendFetches()); client.prepareResponse(fullFetchResponse(tidp0, MemoryRecords.readableRecords(buffer), Errors.NONE, 100L, 0)); consumerClient.poll(time.timer(0)); - try { - collectFetch(); - fail("fetchedRecords should have raised"); - } catch (KafkaException e) { - // the position should not advance since no data has been returned - assertEquals(0, subscriptions.position(tp0).offset); - } + + assertThrows(KafkaException.class, this::collectFetch); + // the position should not advance since no data has been returned + assertEquals(0, subscriptions.position(tp0).offset); } @Test @@ -1221,14 +1206,9 @@ public void testFetchRequestWhenRecordTooLarge() { client.setNodeApiVersions(NodeApiVersions.create(ApiKeys.FETCH.id, (short) 2, (short) 2)); makeFetchRequestWithIncompleteRecord(); - try { - collectFetch(); - fail("RecordTooLargeException should have been raised"); - } catch (RecordTooLargeException e) { - assertTrue(e.getMessage().startsWith("There are some messages at [Partition=Offset]: ")); - // the position should not advance since no data has been returned - assertEquals(0, subscriptions.position(tp0).offset); - } + assertThrows(RecordTooLargeException.class, this::collectFetch); + // the position should not advance since no data has been returned + assertEquals(0, subscriptions.position(tp0).offset); } finally { client.setNodeApiVersions(NodeApiVersions.create()); } @@ -1246,7 +1226,7 @@ public void testFetchRequestInternalError() { makeFetchRequestWithIncompleteRecord(); try { collectFetch(); - fail("RecordTooLargeException should have been raised"); + fail("collectFetch should have thrown a KafkaException"); } catch (KafkaException e) { assertTrue(e.getMessage().startsWith("Failed to make progress reading messages")); // the position should not advance since no data has been returned @@ -1279,7 +1259,7 @@ public void testUnauthorizedTopic() { consumerClient.poll(time.timer(0)); try { collectFetch(); - fail("fetchedRecords should have thrown"); + fail("collectFetch should have thrown a TopicAuthorizationException"); } catch (TopicAuthorizationException e) { assertEquals(singleton(topicName), e.unauthorizedTopics()); } From 589bcc8b9ad197d9aa936efd0daecb7e7e81f47c Mon Sep 17 00:00:00 2001 From: Kirk True Date: Mon, 16 Oct 2023 16:08:18 -0700 Subject: [PATCH 61/72] Removed outdated comment in testUnauthorizedTopic() --- .../clients/consumer/internals/FetchRequestManagerTest.java | 1 - .../org/apache/kafka/clients/consumer/internals/FetcherTest.java | 1 - 2 files changed, 2 deletions(-) diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java index f95f2f3a1770f..e2498c4c30256 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java @@ -1256,7 +1256,6 @@ public void testUnauthorizedTopic() { assignFromUser(singleton(tp0)); subscriptions.seek(tp0, 0); - // resize the limit of the buffer to pretend it is only fetch-size large assertEquals(1, sendFetches()); client.prepareResponse(fullFetchResponse(tidp0, records, Errors.TOPIC_AUTHORIZATION_FAILED, 100L, 0)); networkClientDelegate.poll(time.timer(0)); diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java index 55e79eb64c2fc..12949016b16ae 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java @@ -1253,7 +1253,6 @@ public void testUnauthorizedTopic() { assignFromUser(singleton(tp0)); subscriptions.seek(tp0, 0); - // resize the limit of the buffer to pretend it is only fetch-size large assertEquals(1, sendFetches()); client.prepareResponse(fullFetchResponse(tidp0, records, Errors.TOPIC_AUTHORIZATION_FAILED, 100L, 0)); consumerClient.poll(time.timer(0)); From e9d2bef0e886907cabe438b9ceb42b6defe34780 Mon Sep 17 00:00:00 2001 From: Kirk True Date: Mon, 16 Oct 2023 16:09:35 -0700 Subject: [PATCH 62/72] Test method name changed for clarity Changed name of test from testFetchResponseMetricsPartialResponse to testFetchResponseMetricsWithSkippedOffset. --- .../clients/consumer/internals/FetchRequestManagerTest.java | 2 +- .../apache/kafka/clients/consumer/internals/FetcherTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java index e2498c4c30256..8e0d82a84cc1f 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java @@ -2078,7 +2078,7 @@ public void testFetchResponseMetrics() { } @Test - public void testFetchResponseMetricsPartialResponse() { + public void testFetchResponseMetricsWithSkippedOffset() { buildFetcher(); assignFromUser(singleton(tp0)); diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java index 12949016b16ae..b2a1e6af403ad 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java @@ -2062,7 +2062,7 @@ public void testFetchResponseMetrics() { } @Test - public void testFetchResponseMetricsPartialResponse() { + public void testFetchResponseMetricsWithSkippedOffset() { buildFetcher(); assignFromUser(singleton(tp0)); From 44e7789c04f74f58f2313d1fe45b169308d0fb32 Mon Sep 17 00:00:00 2001 From: Kirk True Date: Wed, 18 Oct 2023 17:02:03 -0700 Subject: [PATCH 63/72] Removed unnecessarily comment from FetchRequestManagerTest.testFetcherCloseClosesFetchSessionsInBroker() --- .../clients/consumer/internals/FetchRequestManagerTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java index 8e0d82a84cc1f..1fed36922f30a 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java @@ -311,7 +311,6 @@ public void testFetcherCloseClosesFetchSessionsInBroker() { final ArgumentCaptor argument = ArgumentCaptor.forClass(NetworkClientDelegate.UnsentRequest.class); - // send request to close the fetcher Timer timer = time.timer(Duration.ofSeconds(10)); // NOTE: by design the FetchRequestManager doesn't perform network I/O internally. That means that calling // the close() method with a Timer will NOT send out the close session requests on close. The network From 1eaad84e368bcffe342659aa485e9a4069e9507f Mon Sep 17 00:00:00 2001 From: Kirk True Date: Wed, 18 Oct 2023 17:04:21 -0700 Subject: [PATCH 64/72] Updated variable name in FetchRequestManagerTest.testFetcherCloseClosesFetchSessionsInBroker for clarity. --- .../clients/consumer/internals/FetchRequestManagerTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java index 1fed36922f30a..c99e180cecea6 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java @@ -320,8 +320,8 @@ public void testFetcherCloseClosesFetchSessionsInBroker() { // validate that closing the fetcher has sent a request with final epoch. 2 requests are sent, one for the // normal fetch earlier and another for the finish fetch here. verify(networkClientDelegate, times(2)).doSend(argument.capture(), any(Long.class)); - NetworkClientDelegate.UnsentRequest unsentRequest = argument.getValue(); - FetchRequest.Builder builder = (FetchRequest.Builder) unsentRequest.requestBuilder(); + NetworkClientDelegate.UnsentRequest lastUnsentRequest = argument.getValue(); + FetchRequest.Builder builder = (FetchRequest.Builder) lastUnsentRequest.requestBuilder(); // session Id is the same assertEquals(fetchResponse.sessionId(), builder.metadata().sessionId()); // contains final epoch From 94833630c424147eea30a3a7dc9fe2666d9f4d2b Mon Sep 17 00:00:00 2001 From: Kirk True Date: Thu, 19 Oct 2023 16:59:43 -0700 Subject: [PATCH 65/72] On close, make the application thread wait for the network thread to finish, up to the given timeout --- .../internals/ConsumerNetworkThread.java | 29 ++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThread.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThread.java index 9abb8eafd39c4..77a2952d1b2f9 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThread.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerNetworkThread.java @@ -222,11 +222,38 @@ public void close(final Duration timeout) { ); } + /** + * Starts the closing process. + * + *

    + * + * This method is called from the application thread, but our resources are owned by the network thread. As such, + * we don't actually close any of those resources here, immediately, on the application thread. Instead, we just + * update our internal state on the application thread. When the network thread next + * {@link #run() executes its loop}, it will notice that state, cease processing any further events, and begin + * {@link #cleanup() closing its resources}. + * + *

    + * + * This method will wait (i.e. block the application thread) for up to the duration of the given timeout to give + * the network thread the time to close down cleanly. + * + * @param timeout Upper bound of time to wait for the network thread to close its resources + */ private void closeInternal(final Duration timeout) { - log.trace("Signaling the consumer network thread to close in {}ms", timeout.toMillis()); + long timeoutMs = timeout.toMillis(); + log.trace("Signaling the consumer network thread to close in {}ms", timeoutMs); running = false; closeTimeout = timeout; wakeup(); + + if (timeoutMs > 0) { + try { + join(timeoutMs); + } catch (InterruptedException e) { + log.error("Interrupted while waiting for consumer network thread to complete", e); + } + } } void cleanup() { From 99a65eca06f617d06c3cf136c3f4879d7d6295bd Mon Sep 17 00:00:00 2001 From: Kirk True Date: Thu, 19 Oct 2023 17:06:20 -0700 Subject: [PATCH 66/72] Handle previously cached errors from getOffsetResetTimestamp Changes: 1. Now OffsetsRequestManager correctly forwards previously cached errors from getOffsetResetTimestamp to the application thread 2. Update OffsetFetcherTest's testRestOffsetsAuthorizationFailure() -> testResetOffsetsAuthorizationFailure() 3. Update OffsetsRequestManagerTest's testResetPositionsThrowsPreviousException -> testResetOffsetsAuthorizationFailure() and updated logic to ensure error is present in queue for the application thread to check --- .../internals/NetworkClientDelegate.java | 4 +- .../internals/OffsetsRequestManager.java | 15 +++++- .../consumer/internals/RequestManagers.java | 1 + .../internals/ConsumerTestBuilder.java | 1 + .../consumer/internals/OffsetFetcherTest.java | 2 +- .../internals/OffsetsRequestManagerTest.java | 54 +++++++++++++++---- 6 files changed, 63 insertions(+), 14 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/NetworkClientDelegate.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/NetworkClientDelegate.java index e2aa915612316..7034f8d2a1884 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/NetworkClientDelegate.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/NetworkClientDelegate.java @@ -284,9 +284,9 @@ public UnsentRequest(final AbstractRequest.Builder requestBuilder, public UnsentRequest(final AbstractRequest.Builder requestBuilder, final Node node, - final BiConsumer responseHandler) { + final BiConsumer callback) { this(requestBuilder, Optional.of(node)); - future().whenComplete(responseHandler); + this.handler.future.whenComplete(callback); } public void setTimer(final Time time, final long requestTimeoutMs) { diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/OffsetsRequestManager.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/OffsetsRequestManager.java index e415cc122ae42..957b18a30e8ad 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/OffsetsRequestManager.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/OffsetsRequestManager.java @@ -24,6 +24,8 @@ import org.apache.kafka.clients.consumer.OffsetAndTimestamp; import org.apache.kafka.clients.consumer.internals.OffsetFetcherUtils.ListOffsetData; import org.apache.kafka.clients.consumer.internals.OffsetFetcherUtils.ListOffsetResult; +import org.apache.kafka.clients.consumer.internals.events.BackgroundEventHandler; +import org.apache.kafka.clients.consumer.internals.events.ErrorBackgroundEvent; import org.apache.kafka.common.ClusterResource; import org.apache.kafka.common.ClusterResourceListener; import org.apache.kafka.common.IsolationLevel; @@ -83,6 +85,7 @@ public class OffsetsRequestManager implements RequestManager, ClusterResourceLis private final Time time; private final ApiVersions apiVersions; private final NetworkClientDelegate networkClientDelegate; + private final BackgroundEventHandler backgroundEventHandler; @SuppressWarnings("this-escape") public OffsetsRequestManager(final SubscriptionState subscriptionState, @@ -93,6 +96,7 @@ public OffsetsRequestManager(final SubscriptionState subscriptionState, final long requestTimeoutMs, final ApiVersions apiVersions, final NetworkClientDelegate networkClientDelegate, + final BackgroundEventHandler backgroundEventHandler, final LogContext logContext) { requireNonNull(subscriptionState); requireNonNull(metadata); @@ -100,6 +104,7 @@ public OffsetsRequestManager(final SubscriptionState subscriptionState, requireNonNull(time); requireNonNull(apiVersions); requireNonNull(networkClientDelegate); + requireNonNull(backgroundEventHandler); requireNonNull(logContext); this.metadata = metadata; @@ -112,6 +117,7 @@ public OffsetsRequestManager(final SubscriptionState subscriptionState, this.requestTimeoutMs = requestTimeoutMs; this.apiVersions = apiVersions; this.networkClientDelegate = networkClientDelegate; + this.backgroundEventHandler = backgroundEventHandler; this.offsetFetcherUtils = new OffsetFetcherUtils(logContext, metadata, subscriptionState, time, retryBackoffMs, apiVersions); // Register the cluster metadata update callback. Note this only relies on the @@ -188,7 +194,14 @@ public CompletableFuture> fetchOffsets( * this function (ex. {@link org.apache.kafka.common.errors.TopicAuthorizationException}) */ public void resetPositionsIfNeeded() { - Map offsetResetTimestamps = offsetFetcherUtils.getOffsetResetTimestamp(); + Map offsetResetTimestamps; + + try { + offsetResetTimestamps = offsetFetcherUtils.getOffsetResetTimestamp(); + } catch (Exception e) { + backgroundEventHandler.add(new ErrorBackgroundEvent(e)); + return; + } if (offsetResetTimestamps.isEmpty()) return; diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java index dfe4bb971ada1..b5bd7e2e3a9ef 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/RequestManagers.java @@ -134,6 +134,7 @@ protected RequestManagers create() { requestTimeoutMs, apiVersions, networkClientDelegate, + backgroundEventHandler, logContext); final FetchRequestManager fetch = new FetchRequestManager(logContext, time, diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java index d56261e48ae25..6b82c4519a137 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerTestBuilder.java @@ -163,6 +163,7 @@ public ConsumerTestBuilder(Optional groupInfo) { requestTimeoutMs, apiVersions, networkClientDelegate, + backgroundEventHandler, logContext)); if (groupInfo.isPresent()) { diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/OffsetFetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/OffsetFetcherTest.java index fe3535eda64b4..a118b20297a6b 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/OffsetFetcherTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/OffsetFetcherTest.java @@ -564,7 +564,7 @@ public void testIdempotentResetWithInFlightReset() { } @Test - public void testRestOffsetsAuthorizationFailure() { + public void testResetOffsetsAuthorizationFailure() { buildFetcher(); assignFromUser(singleton(tp0)); subscriptions.requestOffsetReset(tp0, OffsetResetStrategy.LATEST); diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/OffsetsRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/OffsetsRequestManagerTest.java index 0fec9a9e0892e..f0eb55f1c7921 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/OffsetsRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/OffsetsRequestManagerTest.java @@ -22,6 +22,9 @@ import org.apache.kafka.clients.NodeApiVersions; import org.apache.kafka.clients.consumer.OffsetAndTimestamp; import org.apache.kafka.clients.consumer.OffsetResetStrategy; +import org.apache.kafka.clients.consumer.internals.events.BackgroundEvent; +import org.apache.kafka.clients.consumer.internals.events.BackgroundEventHandler; +import org.apache.kafka.clients.consumer.internals.events.ErrorBackgroundEvent; import org.apache.kafka.common.Cluster; import org.apache.kafka.common.ClusterResource; import org.apache.kafka.common.IsolationLevel; @@ -57,15 +60,20 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.concurrent.BlockingQueue; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; +import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; import java.util.stream.Stream; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; @@ -84,6 +92,7 @@ public class OffsetsRequestManagerTest { private SubscriptionState subscriptionState; private MockTime time; private ApiVersions apiVersions; + private BlockingQueue backgroundEventQueue; private static final String TEST_TOPIC = "t1"; private static final TopicPartition TEST_PARTITION_1 = new TopicPartition(TEST_TOPIC, 1); private static final TopicPartition TEST_PARTITION_2 = new TopicPartition(TEST_TOPIC, 2); @@ -95,13 +104,25 @@ public class OffsetsRequestManagerTest { @BeforeEach public void setup() { + LogContext logContext = new LogContext(); + backgroundEventQueue = new LinkedBlockingQueue<>(); + BackgroundEventHandler backgroundEventHandler = new BackgroundEventHandler(logContext, backgroundEventQueue); metadata = mock(ConsumerMetadata.class); subscriptionState = mock(SubscriptionState.class); - this.time = new MockTime(0); + time = new MockTime(0); apiVersions = mock(ApiVersions.class); - requestManager = new OffsetsRequestManager(subscriptionState, metadata, - DEFAULT_ISOLATION_LEVEL, time, RETRY_BACKOFF_MS, REQUEST_TIMEOUT_MS, - apiVersions, mock(NetworkClientDelegate.class), new LogContext()); + requestManager = new OffsetsRequestManager( + subscriptionState, + metadata, + DEFAULT_ISOLATION_LEVEL, + time, + RETRY_BACKOFF_MS, + REQUEST_TIMEOUT_MS, + apiVersions, + mock(NetworkClientDelegate.class), + backgroundEventHandler, + logContext + ); } @Test @@ -500,7 +521,7 @@ public void testResetPositionsSuccess_LeaderEpochInResponse() { } @Test - public void testResetPositionsThrowsPreviousException() { + public void testResetOffsetsAuthorizationFailure() { when(subscriptionState.partitionsNeedingReset(time.milliseconds())).thenReturn(Collections.singleton(TEST_PARTITION_1)); when(subscriptionState.resetStrategy(any())).thenReturn(OffsetResetStrategy.EARLIEST); mockSuccessfulRequest(Collections.singletonMap(TEST_PARTITION_1, LEADER_1)); @@ -510,8 +531,9 @@ public void testResetPositionsThrowsPreviousException() { // Reset positions response with TopicAuthorizationException NetworkClientDelegate.PollResult res = requestManager.poll(time.milliseconds()); NetworkClientDelegate.UnsentRequest unsentRequest = res.unsentRequests.get(0); + Errors topicAuthorizationFailedError = Errors.TOPIC_AUTHORIZATION_FAILED; ClientResponse clientResponse = buildClientResponseWithErrors( - unsentRequest, Collections.singletonMap(TEST_PARTITION_1, Errors.TOPIC_AUTHORIZATION_FAILED)); + unsentRequest, Collections.singletonMap(TEST_PARTITION_1, topicAuthorizationFailedError)); clientResponse.onComplete(); assertTrue(unsentRequest.future().isDone()); @@ -520,11 +542,23 @@ public void testResetPositionsThrowsPreviousException() { verify(subscriptionState).requestFailed(any(), anyLong()); verify(metadata).requestUpdate(false); - // Following resetPositions should raise the previous exception without performing any - // request - assertThrows(TopicAuthorizationException.class, - () -> requestManager.resetPositionsIfNeeded()); + // Following resetPositions should enqueue the previous exception in the background event queue + // without performing any request + assertDoesNotThrow(() -> requestManager.resetPositionsIfNeeded()); assertEquals(0, requestManager.requestsToSend()); + + // Check that the event was enqueued during resetPositionsIfNeeded + assertEquals(1, backgroundEventQueue.size()); + BackgroundEvent event = backgroundEventQueue.poll(); + assertNotNull(event); + + // Check that the event itself is of the expected type + assertInstanceOf(ErrorBackgroundEvent.class, event); + ErrorBackgroundEvent errorEvent = (ErrorBackgroundEvent) event; + assertNotNull(errorEvent.error()); + + // Check that the error held in the event is of the expected type + assertInstanceOf(topicAuthorizationFailedError.exception().getClass(), errorEvent.error()); } @Test From 8372505f7874b79af4881a0dd72861a424f41fa7 Mon Sep 17 00:00:00 2001 From: Kirk True Date: Fri, 20 Oct 2023 14:47:59 -0700 Subject: [PATCH 67/72] Updated wakeup() to be invoked only after receiving a non-empty Fetch --- .../internals/PrototypeAsyncConsumer.java | 32 ++++++++++++++----- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java index 5041dca51133a..3c55f4ef4a457 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java @@ -338,9 +338,6 @@ public ConsumerRecords poll(final Duration timeout) { final Fetch fetch = pollForFetches(timer); if (!fetch.isEmpty()) { - // Notify the network thread to wake up and start the next round of fetching. - applicationEventHandler.wakeup(); - if (fetch.records().isEmpty()) { log.trace("Returning empty records from `poll()` " + "since the consumer's position has advanced for at least one topic partition"); @@ -912,14 +909,11 @@ private Fetch pollForFetches(Timer timer) { long pollTimeout = timer.remainingMs(); // if data is available already, return it immediately - final Fetch fetch = fetchCollector.collectFetch(fetchBuffer); + final Fetch fetch = collectFetch(); if (!fetch.isEmpty()) { return fetch; } - // Wake up the network thread to send any new fetches (won't resend pending fetches) - applicationEventHandler.wakeup(); - // We do not want to be stuck blocking in poll if we are missing some positions // since the offset lookup may be backing off after a failure @@ -944,9 +938,31 @@ private Fetch pollForFetches(Timer timer) { timer.update(pollTimer.currentTimeMs()); } - return fetchCollector.collectFetch(fetchBuffer); + return collectFetch(); } + /** + * Perform the "{@link FetchCollector#collectFetch(FetchBuffer) fetch collection}" step by reading raw data out + * of the {@link #fetchBuffer}, converting it to a well-formed {@link CompletedFetch}, validating that it and + * the internal {@link SubscriptionState state} are correct, and then converting it all into a {@link Fetch} + * for returning. + * + *

    + * + * Note that if the {@link Fetch#isEmpty() is not empty}, this method will + * {@link ApplicationEventHandler#wakeup() wake up} the {@link ConsumerNetworkThread} before retuning. This is + * done as an optimization so that the next round of data can be pre-fetched. + */ + private Fetch collectFetch() { + final Fetch fetch = fetchCollector.collectFetch(fetchBuffer); + + if (!fetch.isEmpty()) { + // Notify the network thread to wake up and start the next round of fetching. + applicationEventHandler.wakeup(); + } + + return fetch; + } /** * Set the fetch position to the committed position (if there is one) * or reset it using the offset reset policy the user has configured. From fb4d1c2624ed56bb5fa40a558a863510ded662ec Mon Sep 17 00:00:00 2001 From: Kirk True Date: Fri, 20 Oct 2023 14:49:53 -0700 Subject: [PATCH 68/72] Fixed misleading comment in FetcherTest/FetchRequestManagerTest testFetchedRecordsRaisesOnSerializationErrors() --- .../clients/consumer/internals/FetchRequestManagerTest.java | 2 +- .../apache/kafka/clients/consumer/internals/FetcherTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java index c99e180cecea6..ebefa8bbc92e8 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetchRequestManagerTest.java @@ -884,7 +884,7 @@ public byte[] deserialize(String topic, byte[] data) { networkClientDelegate.poll(time.timer(0)); for (int i = 0; i < 2; i++) { - // The fetcher should block on Deserialization error + // The fetcher should throw a Deserialization error assertThrows(SerializationException.class, this::collectFetch); // the position should not advance since no data has been returned assertEquals(1, subscriptions.position(tp0).offset); diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java index b2a1e6af403ad..a66d153f098e4 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java @@ -882,7 +882,7 @@ public byte[] deserialize(String topic, byte[] data) { consumerClient.poll(time.timer(0)); for (int i = 0; i < 2; i++) { - // The fetcher should block on Deserialization error + // The fetcher should throw a Deserialization error assertThrows(SerializationException.class, this::collectFetch); // the position should not advance since no data has been returned assertEquals(1, subscriptions.position(tp0).offset); From f40d4d2831d377d8630cebc45868a377fced8a7a Mon Sep 17 00:00:00 2001 From: Kirk True Date: Fri, 20 Oct 2023 14:53:21 -0700 Subject: [PATCH 69/72] Making NetworkClientDelegate constructor call another to eliminate redundancy --- .../clients/consumer/internals/NetworkClientDelegate.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/NetworkClientDelegate.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/NetworkClientDelegate.java index 7034f8d2a1884..a68562736399e 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/NetworkClientDelegate.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/NetworkClientDelegate.java @@ -285,8 +285,7 @@ public UnsentRequest(final AbstractRequest.Builder requestBuilder, public UnsentRequest(final AbstractRequest.Builder requestBuilder, final Node node, final BiConsumer callback) { - this(requestBuilder, Optional.of(node)); - this.handler.future.whenComplete(callback); + this(requestBuilder, Optional.of(node), callback); } public void setTimer(final Time time, final long requestTimeoutMs) { From 13cb3b6e0ce88d183276872655d01aca8a3d9492 Mon Sep 17 00:00:00 2001 From: Kirk True Date: Fri, 20 Oct 2023 15:45:32 -0700 Subject: [PATCH 70/72] Ensure wakeupNetworkThread() is called even on empty Fetch results --- .../consumer/internals/PrototypeAsyncConsumer.java | 11 ++++------- .../internals/events/ApplicationEventHandler.java | 10 +++++----- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java index a71cb10b6a3d8..a895bc0a12044 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java @@ -955,17 +955,14 @@ private Fetch pollForFetches(Timer timer) { * *

    * - * Note that if the {@link Fetch#isEmpty() is not empty}, this method will - * {@link ApplicationEventHandler#wakeup() wake up} the {@link ConsumerNetworkThread} before retuning. This is - * done as an optimization so that the next round of data can be pre-fetched. + * This method will {@link ApplicationEventHandler#wakeupNetworkThread() wake up} the {@link ConsumerNetworkThread} before + * retuning. This is done as an optimization so that the next round of data can be pre-fetched. */ private Fetch collectFetch() { final Fetch fetch = fetchCollector.collectFetch(fetchBuffer); - if (!fetch.isEmpty()) { - // Notify the network thread to wake up and start the next round of fetching. - applicationEventHandler.wakeup(); - } + // Notify the network thread to wake up and start the next round of fetching. + applicationEventHandler.wakeupNetworkThread(); return fetch; } diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventHandler.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventHandler.java index 18dcf94c0acfe..2917d507d7f02 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventHandler.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/events/ApplicationEventHandler.java @@ -62,8 +62,8 @@ public ApplicationEventHandler(final LogContext logContext, } /** - * Add an {@link ApplicationEvent} to the handler and then internally invoke {@link #wakeup} to alert the - * network I/O thread that it has something to process. + * Add an {@link ApplicationEvent} to the handler and then internally invoke {@link #wakeupNetworkThread} + * to alert the network I/O thread that it has something to process. * * @param event An {@link ApplicationEvent} created by the application thread */ @@ -71,13 +71,13 @@ public void add(final ApplicationEvent event) { Objects.requireNonNull(event, "ApplicationEvent provided to add must be non-null"); log.trace("Enqueued event: {}", event); applicationEventQueue.add(event); - wakeup(); + wakeupNetworkThread(); } /** - * Wakeup the {@link ConsumerNetworkThread network I/O thread} to pull the event from the queue. + * Wakeup the {@link ConsumerNetworkThread network I/O thread} to pull the next event(s) from the queue. */ - public void wakeup() { + public void wakeupNetworkThread() { networkThread.wakeup(); } From 9525fcae20a69594df47f4cbca73ffcd867af280 Mon Sep 17 00:00:00 2001 From: Kirk True Date: Fri, 20 Oct 2023 15:50:27 -0700 Subject: [PATCH 71/72] Changed OffsetFetcherUtils.offsetResetStrategyTimestamp to throw a NoOffsetForPartitionException if there's no reset strategy --- .../kafka/clients/consumer/internals/OffsetFetcherUtils.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/OffsetFetcherUtils.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/OffsetFetcherUtils.java index b7fdefeb0d100..9239811f7d602 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/OffsetFetcherUtils.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/OffsetFetcherUtils.java @@ -19,6 +19,7 @@ import org.apache.kafka.clients.ApiVersions; import org.apache.kafka.clients.NodeApiVersions; import org.apache.kafka.clients.consumer.LogTruncationException; +import org.apache.kafka.clients.consumer.NoOffsetForPartitionException; import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.clients.consumer.OffsetAndTimestamp; import org.apache.kafka.clients.consumer.OffsetResetStrategy; @@ -262,7 +263,7 @@ private Long offsetResetStrategyTimestamp(final TopicPartition partition) { else if (strategy == OffsetResetStrategy.LATEST) return ListOffsetsRequest.LATEST_TIMESTAMP; else - return null; + throw new NoOffsetForPartitionException(partition); } static Set topicsForPartitions(Collection partitions) { From c5d5fc6a24f508b6da91e749cec970c8a82a2d75 Mon Sep 17 00:00:00 2001 From: Kirk True Date: Fri, 20 Oct 2023 17:02:44 -0700 Subject: [PATCH 72/72] Renamed refreshCommittedOffsetsIfNeeded to initWithCommittedOffsetsIfNeeded Also removed the check for null from ConsumerUtils.refreshCommittedOffsets() and put it on the ConsumerCoordinator.initWithCommittedOffsetsIfNeeded to check for null offsets map before calling. --- .../kafka/clients/consumer/KafkaConsumer.java | 2 +- .../internals/ConsumerCoordinator.java | 10 +++++++-- .../consumer/internals/ConsumerUtils.java | 11 +++------- .../internals/PrototypeAsyncConsumer.java | 13 ++++++----- .../internals/ConsumerCoordinatorTest.java | 22 +++++++++---------- 5 files changed, 31 insertions(+), 27 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java index c2a939c148f6c..dd273c38c437d 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/KafkaConsumer.java @@ -2482,7 +2482,7 @@ private boolean updateFetchPositions(final Timer timer) { // coordinator lookup if there are partitions which have missing positions, so // a consumer with manually assigned partitions can avoid a coordinator dependence // by always ensuring that assigned partitions have an initial position. - if (coordinator != null && !coordinator.refreshCommittedOffsetsIfNeeded(timer)) return false; + if (coordinator != null && !coordinator.initWithCommittedOffsetsIfNeeded(timer)) return false; // If there are partitions still needing a position and a reset policy is defined, // request reset using the default policy. If no reset strategy is defined and there diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java index b906a02fef908..bdcbfc39dfc27 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinator.java @@ -955,10 +955,16 @@ public boolean rejoinNeededOrPending() { * @param timer Timer bounding how long this method can block * @return true iff the operation completed within the timeout */ - public boolean refreshCommittedOffsetsIfNeeded(Timer timer) { + public boolean initWithCommittedOffsetsIfNeeded(Timer timer) { final Set initializingPartitions = subscriptions.initializingPartitions(); final Map offsets = fetchCommittedOffsets(initializingPartitions, timer); - return refreshCommittedOffsets(offsets, this.metadata, this.subscriptions); + + // "offsets" will be null if the offset fetch requests did not receive responses within the given timeout + if (offsets == null) + return false; + + refreshCommittedOffsets(offsets, this.metadata, this.subscriptions); + return true; } /** diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerUtils.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerUtils.java index 7f42934fa4f7c..92b098213b061 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerUtils.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerUtils.java @@ -163,14 +163,10 @@ public static List> configuredConsumerIntercept * committed offsets' metadata. * @param subscriptions Subscription state to update, setting partitions' offsets to the * committed offsets. - * @return False if null offsetsAndMetadata is provided, indicating that the - * refresh operation could not be performed. True in any other case. */ - public static boolean refreshCommittedOffsets(final Map offsetsAndMetadata, - final ConsumerMetadata metadata, - final SubscriptionState subscriptions) { - if (offsetsAndMetadata == null) return false; - + public static void refreshCommittedOffsets(final Map offsetsAndMetadata, + final ConsumerMetadata metadata, + final SubscriptionState subscriptions) { for (final Map.Entry entry : offsetsAndMetadata.entrySet()) { final TopicPartition tp = entry.getKey(); final OffsetAndMetadata offsetAndMetadata = entry.getValue(); @@ -195,7 +191,6 @@ public static boolean refreshCommittedOffsets(final Map T getResult(CompletableFuture future, Timer timer) { diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java index a895bc0a12044..949616daa8551 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/PrototypeAsyncConsumer.java @@ -101,6 +101,7 @@ import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createLogContext; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createMetrics; import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.createSubscriptionState; +import static org.apache.kafka.clients.consumer.internals.ConsumerUtils.refreshCommittedOffsets; import static org.apache.kafka.common.utils.Utils.closeQuietly; import static org.apache.kafka.common.utils.Utils.isBlank; import static org.apache.kafka.common.utils.Utils.join; @@ -990,7 +991,7 @@ private boolean updateFetchPositions(final Timer timer) { // will only do a coordinator lookup if there are partitions which have missing // positions, so a consumer with manually assigned partitions can avoid a coordinator // dependence by always ensuring that assigned partitions have an initial position. - if (isCommittedOffsetsManagementEnabled() && !refreshCommittedOffsetsIfNeeded(timer)) + if (isCommittedOffsetsManagementEnabled() && !initWithCommittedOffsetsIfNeeded(timer)) return false; // If there are partitions still needing a position and a reset policy is defined, @@ -1021,14 +1022,16 @@ private boolean isCommittedOffsetsManagementEnabled() { * @param timer Timer bounding how long this method can block * @return true iff the operation completed within the timeout */ - private boolean refreshCommittedOffsetsIfNeeded(Timer timer) { + private boolean initWithCommittedOffsetsIfNeeded(Timer timer) { final Set initializingPartitions = subscriptions.initializingPartitions(); log.debug("Refreshing committed offsets for partitions {}", initializingPartitions); try { - final Map offsets = applicationEventHandler.addAndGet(new OffsetFetchApplicationEvent(initializingPartitions), timer); - return ConsumerUtils.refreshCommittedOffsets(offsets, metadata, subscriptions); - } catch (org.apache.kafka.common.errors.TimeoutException e) { + final OffsetFetchApplicationEvent event = new OffsetFetchApplicationEvent(initializingPartitions); + final Map offsets = applicationEventHandler.addAndGet(event, timer); + refreshCommittedOffsets(offsets, metadata, subscriptions); + return true; + } catch (TimeoutException e) { log.error("Couldn't refresh committed offsets before timeout expired"); return false; } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java index 43ceca65bb7f9..cc0b1294fc222 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java @@ -3082,7 +3082,7 @@ public void testRefreshOffset() { subscriptions.assignFromUser(singleton(t1p)); client.prepareResponse(offsetFetchResponse(t1p, Errors.NONE, "", 100L)); - coordinator.refreshCommittedOffsetsIfNeeded(time.timer(Long.MAX_VALUE)); + coordinator.initWithCommittedOffsetsIfNeeded(time.timer(Long.MAX_VALUE)); assertEquals(Collections.emptySet(), subscriptions.initializingPartitions()); assertTrue(subscriptions.hasAllFetchPositions()); @@ -3103,7 +3103,7 @@ public void testRefreshOffsetWithValidation() { // Load offsets from previous epoch client.prepareResponse(offsetFetchResponse(t1p, Errors.NONE, "", 100L, Optional.of(3))); - coordinator.refreshCommittedOffsetsIfNeeded(time.timer(Long.MAX_VALUE)); + coordinator.initWithCommittedOffsetsIfNeeded(time.timer(Long.MAX_VALUE)); // Offset gets loaded, but requires validation assertEquals(Collections.emptySet(), subscriptions.initializingPartitions()); @@ -3155,7 +3155,7 @@ public void testRefreshOffsetLoadInProgress() { subscriptions.assignFromUser(singleton(t1p)); client.prepareResponse(offsetFetchResponse(Errors.COORDINATOR_LOAD_IN_PROGRESS, Collections.emptyMap())); client.prepareResponse(offsetFetchResponse(t1p, Errors.NONE, "", 100L)); - coordinator.refreshCommittedOffsetsIfNeeded(time.timer(Long.MAX_VALUE)); + coordinator.initWithCommittedOffsetsIfNeeded(time.timer(Long.MAX_VALUE)); assertEquals(Collections.emptySet(), subscriptions.initializingPartitions()); assertTrue(subscriptions.hasAllFetchPositions()); @@ -3170,7 +3170,7 @@ public void testRefreshOffsetsGroupNotAuthorized() { subscriptions.assignFromUser(singleton(t1p)); client.prepareResponse(offsetFetchResponse(Errors.GROUP_AUTHORIZATION_FAILED, Collections.emptyMap())); try { - coordinator.refreshCommittedOffsetsIfNeeded(time.timer(Long.MAX_VALUE)); + coordinator.initWithCommittedOffsetsIfNeeded(time.timer(Long.MAX_VALUE)); fail("Expected group authorization error"); } catch (GroupAuthorizationException e) { assertEquals(groupId, e.groupId()); @@ -3186,9 +3186,9 @@ public void testRefreshOffsetWithPendingTransactions() { client.prepareResponse(offsetFetchResponse(t1p, Errors.UNSTABLE_OFFSET_COMMIT, "", -1L)); client.prepareResponse(offsetFetchResponse(t1p, Errors.NONE, "", 100L)); assertEquals(Collections.singleton(t1p), subscriptions.initializingPartitions()); - coordinator.refreshCommittedOffsetsIfNeeded(time.timer(0L)); + coordinator.initWithCommittedOffsetsIfNeeded(time.timer(0L)); assertEquals(Collections.singleton(t1p), subscriptions.initializingPartitions()); - coordinator.refreshCommittedOffsetsIfNeeded(time.timer(0L)); + coordinator.initWithCommittedOffsetsIfNeeded(time.timer(0L)); assertEquals(Collections.emptySet(), subscriptions.initializingPartitions()); assertTrue(subscriptions.hasAllFetchPositions()); @@ -3202,7 +3202,7 @@ public void testRefreshOffsetUnknownTopicOrPartition() { subscriptions.assignFromUser(singleton(t1p)); client.prepareResponse(offsetFetchResponse(t1p, Errors.UNKNOWN_TOPIC_OR_PARTITION, "", 100L)); - assertThrows(KafkaException.class, () -> coordinator.refreshCommittedOffsetsIfNeeded(time.timer(Long.MAX_VALUE))); + assertThrows(KafkaException.class, () -> coordinator.initWithCommittedOffsetsIfNeeded(time.timer(Long.MAX_VALUE))); } @Test @@ -3214,7 +3214,7 @@ public void testRefreshOffsetNotCoordinatorForConsumer() { client.prepareResponse(offsetFetchResponse(Errors.NOT_COORDINATOR, Collections.emptyMap())); client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE)); client.prepareResponse(offsetFetchResponse(t1p, Errors.NONE, "", 100L)); - coordinator.refreshCommittedOffsetsIfNeeded(time.timer(Long.MAX_VALUE)); + coordinator.initWithCommittedOffsetsIfNeeded(time.timer(Long.MAX_VALUE)); assertEquals(Collections.emptySet(), subscriptions.initializingPartitions()); assertTrue(subscriptions.hasAllFetchPositions()); @@ -3228,7 +3228,7 @@ public void testRefreshOffsetWithNoFetchableOffsets() { subscriptions.assignFromUser(singleton(t1p)); client.prepareResponse(offsetFetchResponse(t1p, Errors.NONE, "", -1L)); - coordinator.refreshCommittedOffsetsIfNeeded(time.timer(Long.MAX_VALUE)); + coordinator.initWithCommittedOffsetsIfNeeded(time.timer(Long.MAX_VALUE)); assertEquals(Collections.singleton(t1p), subscriptions.initializingPartitions()); assertEquals(Collections.emptySet(), subscriptions.partitionsNeedingReset(time.milliseconds())); @@ -3242,7 +3242,7 @@ public void testNoCoordinatorDiscoveryIfPositionsKnown() { subscriptions.assignFromUser(singleton(t1p)); subscriptions.seek(t1p, 500L); - coordinator.refreshCommittedOffsetsIfNeeded(time.timer(Long.MAX_VALUE)); + coordinator.initWithCommittedOffsetsIfNeeded(time.timer(Long.MAX_VALUE)); assertEquals(Collections.emptySet(), subscriptions.initializingPartitions()); assertTrue(subscriptions.hasAllFetchPositions()); @@ -3256,7 +3256,7 @@ public void testNoCoordinatorDiscoveryIfPartitionAwaitingReset() { subscriptions.assignFromUser(singleton(t1p)); subscriptions.requestOffsetReset(t1p, OffsetResetStrategy.EARLIEST); - coordinator.refreshCommittedOffsetsIfNeeded(time.timer(Long.MAX_VALUE)); + coordinator.initWithCommittedOffsetsIfNeeded(time.timer(Long.MAX_VALUE)); assertEquals(Collections.emptySet(), subscriptions.initializingPartitions()); assertFalse(subscriptions.hasAllFetchPositions());