From cfb60064ec50f52f7deeda3c3581259d07670a59 Mon Sep 17 00:00:00 2001 From: dengziming Date: Wed, 3 Mar 2021 12:38:06 +0800 Subject: [PATCH 01/59] MINOR: Fix null exception in coordinator log (#10250) Reviewers: A. Sophie Blee-Goldman , Chia-Ping Tsai --- .../kafka/clients/consumer/internals/AbstractCoordinator.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java index 34f81db306d33..ab3c33beb876e 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java @@ -256,8 +256,8 @@ protected synchronized boolean ensureCoordinatorReady(final Timer timer) { log.debug("Coordinator discovery failed, refreshing metadata", future.exception()); client.awaitMetadataUpdate(timer); } else { - log.info("FindCoordinator request hit fatal exception", fatalException); fatalException = future.exception(); + log.info("FindCoordinator request hit fatal exception", fatalException); } } else if (coordinator != null && client.isUnavailable(coordinator)) { // we found the coordinator, but the connection has failed, so mark @@ -267,7 +267,7 @@ protected synchronized boolean ensureCoordinatorReady(final Timer timer) { } clearFindCoordinatorFuture(); - if (fatalException != null) + if (fatalException != null) throw fatalException; } while (coordinatorUnknown() && timer.notExpired()); From b77deece1db3fca5575e336e157677f83bf3b506 Mon Sep 17 00:00:00 2001 From: Lee Dongjin Date: Wed, 3 Mar 2021 10:13:40 +0530 Subject: [PATCH 02/59] KAFKA-12400: Upgrade jetty to fix CVE-2020-27223 Here is the fix. The reason of [CVE-2020-27223](https://nvd.nist.gov/vuln/detail/CVE-2020-27223) was DOS vulnerability for Quoted Quality CSV headers and [patched in 9.4.37.v20210219](https://github.com/eclipse/jetty.project/security/advisories/GHSA-m394-8rww-3jr7). This PR updates Jetty dependency into the following version, 9.4.38.v20210224. Author: Lee Dongjin Reviewers: Manikumar Reddy Closes #10245 from dongjinleekr/feature/KAFKA-12400 --- gradle/dependencies.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index 90bbadf32aae7..5974e76896a93 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -70,7 +70,7 @@ versions += [ jacksonDatabind: "2.10.5.1", jacoco: "0.8.5", javassist: "3.27.0-GA", - jetty: "9.4.36.v20210114", + jetty: "9.4.38.v20210224", jersey: "2.31", jline: "3.12.1", jmh: "1.27", From 0d9a95a7d0ab06aecc4480901707e29dd2a3147e Mon Sep 17 00:00:00 2001 From: Satish Duggana Date: Wed, 3 Mar 2021 22:25:13 +0530 Subject: [PATCH 03/59] KAFKA-9548 Added SPIs and public classes/interfaces introduced in KIP-405 for tiered storage feature in Kafka. (#10173) KIP-405 introduces tiered storage feature in Kafka. With this feature, Kafka cluster is configured with two tiers of storage - local and remote. The local tier is the same as the current Kafka that uses the local disks on the Kafka brokers to store the log segments. The new remote tier uses systems, such as HDFS or S3 or other cloud storages to store the completed log segments. Consumers fetch the records stored in remote storage through the brokers with the existing protocol. We introduced a few SPIs for plugging in log/index store and remote log metadata store. This involves two parts 1. Storing the actual data in remote storage like HDFS, S3, or other cloud storages. 2. Storing the metadata about where the remote segments are stored. The default implementation uses an internal Kafka topic. You can read KIP for more details at https://cwiki.apache.org/confluence/display/KAFKA/KIP-405%3A+Kafka+Tiered+Storage Reviewers: Jun Rao --- build.gradle | 1 + .../apache/kafka/common/TopicIdPartition.java | 74 +++++ .../log/remote/storage/LogSegmentData.java | 139 +++++++++ .../storage/RemoteLogMetadataManager.java | 200 +++++++++++++ .../remote/storage/RemoteLogSegmentId.java | 80 +++++ .../storage/RemoteLogSegmentMetadata.java | 282 ++++++++++++++++++ .../RemoteLogSegmentMetadataUpdate.java | 120 ++++++++ .../remote/storage/RemoteLogSegmentState.java | 90 ++++++ .../RemotePartitionDeleteMetadata.java | 110 +++++++ .../storage/RemotePartitionDeleteState.java | 86 ++++++ .../RemoteResourceNotFoundException.java | 39 +++ .../storage/RemoteStorageException.java | 36 +++ .../remote/storage/RemoteStorageManager.java | 141 +++++++++ 13 files changed, 1398 insertions(+) create mode 100644 clients/src/main/java/org/apache/kafka/common/TopicIdPartition.java create mode 100644 clients/src/main/java/org/apache/kafka/server/log/remote/storage/LogSegmentData.java create mode 100644 clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogMetadataManager.java create mode 100644 clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogSegmentId.java create mode 100644 clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogSegmentMetadata.java create mode 100644 clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogSegmentMetadataUpdate.java create mode 100644 clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogSegmentState.java create mode 100644 clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemotePartitionDeleteMetadata.java create mode 100644 clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemotePartitionDeleteState.java create mode 100644 clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteResourceNotFoundException.java create mode 100644 clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteStorageException.java create mode 100644 clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteStorageManager.java diff --git a/build.gradle b/build.gradle index 4838a35bf7c03..acd8a84e6377b 100644 --- a/build.gradle +++ b/build.gradle @@ -1225,6 +1225,7 @@ project(':clients') { include "**/org/apache/kafka/server/authorizer/*" include "**/org/apache/kafka/server/policy/*" include "**/org/apache/kafka/server/quota/*" + include "**/org/apache/kafka/server/log/remote/storage/*" } } diff --git a/clients/src/main/java/org/apache/kafka/common/TopicIdPartition.java b/clients/src/main/java/org/apache/kafka/common/TopicIdPartition.java new file mode 100644 index 0000000000000..9fb570a67f438 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/common/TopicIdPartition.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.common; + +import java.util.Objects; + +/** + * This represents universally unique identifier with topic id for a topic partition. This makes sure that topics + * recreated with the same name will always have unique topic identifiers. + */ +public class TopicIdPartition { + + private final Uuid topicId; + private final TopicPartition topicPartition; + + public TopicIdPartition(Uuid topicId, TopicPartition topicPartition) { + this.topicId = Objects.requireNonNull(topicId, "topicId can not be null"); + this.topicPartition = Objects.requireNonNull(topicPartition, "topicPartition can not be null"); + } + + /** + * @return Universally unique id representing this topic partition. + */ + public Uuid topicId() { + return topicId; + } + + /** + * @return Topic partition representing this instance. + */ + public TopicPartition topicPartition() { + return topicPartition; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + TopicIdPartition that = (TopicIdPartition) o; + return Objects.equals(topicId, that.topicId) && + Objects.equals(topicPartition, that.topicPartition); + } + + @Override + public int hashCode() { + return Objects.hash(topicId, topicPartition); + } + + @Override + public String toString() { + return "TopicIdPartition{" + + "topicId=" + topicId + + ", topicPartition=" + topicPartition + + '}'; + } +} diff --git a/clients/src/main/java/org/apache/kafka/server/log/remote/storage/LogSegmentData.java b/clients/src/main/java/org/apache/kafka/server/log/remote/storage/LogSegmentData.java new file mode 100644 index 0000000000000..662e396352142 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/server/log/remote/storage/LogSegmentData.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.server.log.remote.storage; + +import org.apache.kafka.common.annotation.InterfaceStability; + +import java.io.File; +import java.nio.ByteBuffer; +import java.util.Objects; + +/** + * This represents all the required data and indexes for a specific log segment that needs to be stored in the remote + * storage. This is passed with {@link RemoteStorageManager#copyLogSegmentData(RemoteLogSegmentMetadata, LogSegmentData)} + * while copying a specific log segment to the remote storage. + */ +@InterfaceStability.Evolving +public class LogSegmentData { + + private final File logSegment; + private final File offsetIndex; + private final File timeIndex; + private final File txnIndex; + private final File producerSnapshotIndex; + private final ByteBuffer leaderEpochIndex; + + /** + * Creates a LogSegmentData instance with data and indexes. + * + * @param logSegment actual log segment file + * @param offsetIndex offset index file + * @param timeIndex time index file + * @param txnIndex transaction index file + * @param producerSnapshotIndex producer snapshot until this segment + * @param leaderEpochIndex leader-epoch-index until this segment + */ + public LogSegmentData(File logSegment, + File offsetIndex, + File timeIndex, + File txnIndex, + File producerSnapshotIndex, + ByteBuffer leaderEpochIndex) { + this.logSegment = Objects.requireNonNull(logSegment, "logSegment can not be null"); + this.offsetIndex = Objects.requireNonNull(offsetIndex, "offsetIndex can not be null"); + this.timeIndex = Objects.requireNonNull(timeIndex, "timeIndex can not be null"); + this.txnIndex = Objects.requireNonNull(txnIndex, "txnIndex can not be null"); + this.producerSnapshotIndex = Objects.requireNonNull(producerSnapshotIndex, "producerSnapshotIndex can not be null"); + this.leaderEpochIndex = Objects.requireNonNull(leaderEpochIndex, "leaderEpochIndex can not be null"); + } + + /** + * @return Log segment file of this segment. + */ + public File logSegment() { + return logSegment; + } + + /** + * @return Offset index file. + */ + public File offsetIndex() { + return offsetIndex; + } + + /** + * @return Time index file of this segment. + */ + public File timeIndex() { + return timeIndex; + } + + /** + * @return Transaction index file of this segment. + */ + public File txnIndex() { + return txnIndex; + } + + /** + * @return Producer snapshot file until this segment. + */ + public File producerSnapshotIndex() { + return producerSnapshotIndex; + } + + /** + * @return Leader epoch index until this segment. + */ + public ByteBuffer leaderEpochIndex() { + return leaderEpochIndex; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + LogSegmentData that = (LogSegmentData) o; + return Objects.equals(logSegment, that.logSegment) && Objects + .equals(offsetIndex, that.offsetIndex) && Objects + .equals(timeIndex, that.timeIndex) && Objects + .equals(txnIndex, that.txnIndex) && Objects + .equals(producerSnapshotIndex, that.producerSnapshotIndex) && Objects + .equals(leaderEpochIndex, that.leaderEpochIndex); + } + + @Override + public int hashCode() { + return Objects.hash(logSegment, offsetIndex, timeIndex, txnIndex, producerSnapshotIndex, leaderEpochIndex); + } + + @Override + public String toString() { + return "LogSegmentData{" + + "logSegment=" + logSegment + + ", offsetIndex=" + offsetIndex + + ", timeIndex=" + timeIndex + + ", txnIndex=" + txnIndex + + ", producerSnapshotIndex=" + producerSnapshotIndex + + ", leaderEpochIndex=" + leaderEpochIndex + + '}'; + } +} \ No newline at end of file diff --git a/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogMetadataManager.java b/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogMetadataManager.java new file mode 100644 index 0000000000000..f53858d7a4540 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogMetadataManager.java @@ -0,0 +1,200 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.server.log.remote.storage; + +import org.apache.kafka.common.Configurable; +import org.apache.kafka.common.TopicIdPartition; +import org.apache.kafka.common.annotation.InterfaceStability; + +import java.io.Closeable; +import java.util.Iterator; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +/** + * This interface provides storing and fetching remote log segment metadata with strongly consistent semantics. + *

+ * This class can be plugged in to Kafka cluster by adding the implementation class as + * remote.log.metadata.manager.class.name property value. There is an inbuilt implementation backed by + * topic storage in the local cluster. This is used as the default implementation if + * remote.log.metadata.manager.class.name is not configured. + *

+ *

+ * remote.log.metadata.manager.class.path property is about the class path of the RemoteLogStorageManager + * implementation. If specified, the RemoteLogStorageManager implementation and its dependent libraries will be loaded + * by a dedicated classloader which searches this class path before the Kafka broker class path. The syntax of this + * parameter is same with the standard Java class path string. + *

+ *

+ * remote.log.metadata.manager.listener.name property is about listener name of the local broker to which + * it should get connected if needed by RemoteLogMetadataManager implementation. When this is configured all other + * required properties can be passed as properties with prefix of 'remote.log.metadata.manager.listener. + *

+ * "cluster.id", "broker.id" and all other properties prefixed with "remote.log.metadata." are passed when + * {@link #configure(Map)} is invoked on this instance. + *

+ */ +@InterfaceStability.Evolving +public interface RemoteLogMetadataManager extends Configurable, Closeable { + + /** + * Adds {@link RemoteLogSegmentMetadata} with the containing {@link RemoteLogSegmentId} into {@link RemoteLogMetadataManager}. + *

+ * RemoteLogSegmentMetadata is identified by RemoteLogSegmentId and it should have the initial state which is {@link RemoteLogSegmentState#COPY_SEGMENT_STARTED}. + *

+ * {@link #updateRemoteLogSegmentMetadata(RemoteLogSegmentMetadataUpdate)} should be used to update an existing RemoteLogSegmentMetadata. + * + * @param remoteLogSegmentMetadata metadata about the remote log segment. + * @throws RemoteStorageException if there are any storage related errors occurred. + * @throws IllegalArgumentException if the given metadata instance does not have the state as {@link RemoteLogSegmentState#COPY_SEGMENT_STARTED} + */ + void addRemoteLogSegmentMetadata(RemoteLogSegmentMetadata remoteLogSegmentMetadata) throws RemoteStorageException; + + /** + * This method is used to update the {@link RemoteLogSegmentMetadata}. Currently, it allows to update with the new + * state based on the life cycle of the segment. It can go through the below state transitions. + *

+ * 

+     * +---------------------+            +----------------------+
+     * |COPY_SEGMENT_STARTED |----------->|COPY_SEGMENT_FINISHED |
+     * +-------------------+-+            +--+-------------------+
+     *                     |                 |
+     *                     |                 |
+     *                     v                 v
+     *                  +--+-----------------+-+
+     *                  |DELETE_SEGMENT_STARTED|
+     *                  +-----------+----------+
+     *                              |
+     *                              |
+     *                              v
+     *                  +-----------+-----------+
+     *                  |DELETE_SEGMENT_FINISHED|
+     *                  +-----------------------+
+     *
+ *

+ * {@link RemoteLogSegmentState#COPY_SEGMENT_STARTED} - This state indicates that the segment copying to remote storage is started but not yet finished. + * {@link RemoteLogSegmentState#COPY_SEGMENT_FINISHED} - This state indicates that the segment copying to remote storage is finished. + *
+ * The leader broker copies the log segments to the remote storage and puts the remote log segment metadata with the + * state as “COPY_SEGMENT_STARTED” and updates the state as “COPY_SEGMENT_FINISHED” once the copy is successful. + *

+ * {@link RemoteLogSegmentState#DELETE_SEGMENT_STARTED} - This state indicates that the segment deletion is started but not yet finished. + * {@link RemoteLogSegmentState#DELETE_SEGMENT_FINISHED} - This state indicates that the segment is deleted successfully. + *
+ * Leader partitions publish both the above delete segment events when remote log retention is reached for the + * respective segments. Remote Partition Removers also publish these events when a segment is deleted as part of + * the remote partition deletion. + * + * @param remoteLogSegmentMetadataUpdate update of the remote log segment metadata. + * @throws RemoteStorageException if there are any storage related errors occurred. + * @throws RemoteResourceNotFoundException when there are no resources associated with the given remoteLogSegmentMetadataUpdate. + * @throws IllegalArgumentException if the given metadata instance has the state as {@link RemoteLogSegmentState#COPY_SEGMENT_STARTED} + */ + void updateRemoteLogSegmentMetadata(RemoteLogSegmentMetadataUpdate remoteLogSegmentMetadataUpdate) + throws RemoteStorageException; + + /** + * Returns {@link RemoteLogSegmentMetadata} if it exists for the given topic partition containing the offset with + * the given leader-epoch for the offset, else returns {@link Optional#empty()}. + * + * @param topicIdPartition topic partition + * @param offset offset + * @param epochForOffset leader epoch for the given offset + * @return the requested remote log segment metadata if it exists. + * @throws RemoteStorageException if there are any storage related errors occurred. + */ + Optional remoteLogSegmentMetadata(TopicIdPartition topicIdPartition, + long offset, + int epochForOffset) + throws RemoteStorageException; + + /** + * Returns the highest log offset of topic partition for the given leader epoch in remote storage. This is used by + * remote log management subsystem to know up to which offset the segments have been copied to remote storage for + * a given leader epoch. + * + * @param topicIdPartition topic partition + * @param leaderEpoch leader epoch + * @return the requested highest log offset if exists. + * @throws RemoteStorageException if there are any storage related errors occurred. + */ + Optional highestLogOffset(TopicIdPartition topicIdPartition, + int leaderEpoch) throws RemoteStorageException; + + /** + * This method is used to update the metadata about remote partition delete event. Currently, it allows updating the + * state ({@link RemotePartitionDeleteState}) of a topic partition in remote metadata storage. Controller invokes + * this method with {@link RemotePartitionDeleteMetadata} having state as {@link RemotePartitionDeleteState#DELETE_PARTITION_MARKED}. + * So, remote partition removers can act on this event to clean the respective remote log segments of the partition. + *


+ * In the case of default RLMM implementation, remote partition remover processes {@link RemotePartitionDeleteState#DELETE_PARTITION_MARKED} + *

    + *
  • sends an event with state as {@link RemotePartitionDeleteState#DELETE_PARTITION_STARTED} + *
  • gets all the remote log segments and deletes them. + *
  • sends an event with state as {@link RemotePartitionDeleteState#DELETE_PARTITION_FINISHED} once all the remote log segments are + * deleted. + *
+ * + * @param remotePartitionDeleteMetadata update on delete state of a partition. + * @throws RemoteStorageException if there are any storage related errors occurred. + * @throws RemoteResourceNotFoundException when there are no resources associated with the given remotePartitionDeleteMetadata. + */ + void putRemotePartitionDeleteMetadata(RemotePartitionDeleteMetadata remotePartitionDeleteMetadata) + throws RemoteStorageException; + + /** + * List all the remote log segment metadata of the given topicIdPartition. + *

+ * Remote Partition Removers uses this method to fetch all the segments for a given topic partition, so that they + * can delete them. + * + * @return Iterator of remote log segment metadata for the given topic partition. + */ + Iterator listRemoteLogSegments(TopicIdPartition topicIdPartition) + throws RemoteStorageException; + + /** + * Returns iterator of remote log segment metadata, sorted by {@link RemoteLogSegmentMetadata#startOffset()} in + * ascending order which contains the given leader epoch. This is used by remote log retention management subsystem + * to fetch the segment metadata for a given leader epoch. + * + * @param topicIdPartition topic partition + * @param leaderEpoch leader epoch + * @return Iterator of remote segments, sorted by start offset in ascending order. + */ + Iterator listRemoteLogSegments(TopicIdPartition topicIdPartition, + int leaderEpoch) throws RemoteStorageException; + + /** + * This method is invoked only when there are changes in leadership of the topic partitions that this broker is + * responsible for. + * + * @param leaderPartitions partitions that have become leaders on this broker. + * @param followerPartitions partitions that have become followers on this broker. + */ + void onPartitionLeadershipChanges(Set leaderPartitions, + Set followerPartitions); + + /** + * This method is invoked only when the topic partitions are stopped on this broker. This can happen when a + * partition is emigrated to other broker or a partition is deleted. + * + * @param partitions topic partitions that have been stopped. + */ + void onStopPartitions(Set partitions); +} \ No newline at end of file diff --git a/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogSegmentId.java b/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogSegmentId.java new file mode 100644 index 0000000000000..cbebd9f606d32 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogSegmentId.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.server.log.remote.storage; + +import org.apache.kafka.common.TopicIdPartition; +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.annotation.InterfaceStability; + +import java.util.Objects; + +/** + * This class represents a universally unique identifier associated to a topic partition's log segment. This will be + * regenerated for every attempt of copying a specific log segment in {@link RemoteStorageManager#copyLogSegmentData(RemoteLogSegmentMetadata, LogSegmentData)}. + * Once it is stored in remote storage, it is used to access that segment later from remote log metadata storage. + */ +@InterfaceStability.Evolving +public class RemoteLogSegmentId { + + private final TopicIdPartition topicIdPartition; + private final Uuid id; + + public RemoteLogSegmentId(TopicIdPartition topicIdPartition, Uuid id) { + this.topicIdPartition = Objects.requireNonNull(topicIdPartition, "topicIdPartition can not be null"); + this.id = Objects.requireNonNull(id, "id can not be null"); + } + + /** + * @return TopicIdPartition of this remote log segment. + */ + public TopicIdPartition topicIdPartition() { + return topicIdPartition; + } + + /** + * @return Universally Unique Id of this remote log segment. + */ + public Uuid id() { + return id; + } + + @Override + public String toString() { + return "RemoteLogSegmentId{" + + "topicIdPartition=" + topicIdPartition + + ", id=" + id + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + RemoteLogSegmentId that = (RemoteLogSegmentId) o; + return Objects.equals(topicIdPartition, that.topicIdPartition) && Objects.equals(id, that.id); + } + + @Override + public int hashCode() { + return Objects.hash(topicIdPartition, id); + } + +} diff --git a/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogSegmentMetadata.java b/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogSegmentMetadata.java new file mode 100644 index 0000000000000..9f0dd817be849 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogSegmentMetadata.java @@ -0,0 +1,282 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.server.log.remote.storage; + +import org.apache.kafka.common.annotation.InterfaceStability; + +import java.util.Collections; +import java.util.Map; +import java.util.NavigableMap; +import java.util.Objects; +import java.util.TreeMap; + +/** + * It describes the metadata about a topic partition's remote log segment in the remote storage. This is uniquely + * represented with {@link RemoteLogSegmentId}. + *

+ * New instance is always created with the state as {@link RemoteLogSegmentState#COPY_SEGMENT_STARTED}. This can be + * updated by applying {@link RemoteLogSegmentMetadataUpdate} for the respective {@link RemoteLogSegmentId} of the + * {@code RemoteLogSegmentMetadata}. + */ +@InterfaceStability.Evolving +public class RemoteLogSegmentMetadata { + + /** + * Universally unique remote log segment id. + */ + private final RemoteLogSegmentId remoteLogSegmentId; + + /** + * Start offset of this segment. + */ + private final long startOffset; + + /** + * End offset of this segment. + */ + private final long endOffset; + + /** + * Broker id from which this event is generated. + */ + private final int brokerId; + + /** + * Maximum timestamp in the segment + */ + private final long maxTimestamp; + + /** + * Epoch time at which the respective {@link #state} is set. + */ + private final long eventTimestamp; + + /** + * LeaderEpoch vs offset for messages within this segment. + */ + private final NavigableMap segmentLeaderEpochs; + + /** + * Size of the segment in bytes. + */ + private final int segmentSizeInBytes; + + /** + * It indicates the state in which the action is executed on this segment. + */ + private final RemoteLogSegmentState state; + + /** + * Creates an instance with the given metadata of remote log segment. + * + * {@code segmentLeaderEpochs} can not be empty. If all the records in this segment belong to the same leader epoch + * then it should have an entry with epoch mapping to start-offset of this segment. + * + * @param remoteLogSegmentId Universally unique remote log segment id. + * @param startOffset Start offset of this segment (inclusive). + * @param endOffset End offset of this segment (inclusive). + * @param maxTimestamp Maximum timestamp in this segment. + * @param brokerId Broker id from which this event is generated. + * @param eventTimestamp Epoch time in milli seconds at which the remote log segment is copied to the remote tier storage. + * @param segmentSizeInBytes Size of this segment in bytes. + * @param state State of the respective segment of remoteLogSegmentId. + * @param segmentLeaderEpochs leader epochs occurred within this segment. + */ + private RemoteLogSegmentMetadata(RemoteLogSegmentId remoteLogSegmentId, + long startOffset, + long endOffset, + long maxTimestamp, + int brokerId, + long eventTimestamp, + int segmentSizeInBytes, + RemoteLogSegmentState state, + Map segmentLeaderEpochs) { + this.remoteLogSegmentId = Objects.requireNonNull(remoteLogSegmentId, "remoteLogSegmentId can not be null"); + this.state = Objects.requireNonNull(state, "state can not be null"); + + this.startOffset = startOffset; + this.endOffset = endOffset; + this.brokerId = brokerId; + this.maxTimestamp = maxTimestamp; + this.eventTimestamp = eventTimestamp; + this.segmentSizeInBytes = segmentSizeInBytes; + + if (segmentLeaderEpochs == null || segmentLeaderEpochs.isEmpty()) { + throw new IllegalArgumentException("segmentLeaderEpochs can not be null or empty"); + } + + this.segmentLeaderEpochs = Collections.unmodifiableNavigableMap(new TreeMap<>(segmentLeaderEpochs)); + } + + /** + * Creates an instance with the given metadata of remote log segment and its state as {@link RemoteLogSegmentState#COPY_SEGMENT_STARTED}. + * + * {@code segmentLeaderEpochs} can not be empty. If all the records in this segment belong to the same leader epoch + * then it should have an entry with epoch mapping to start-offset of this segment. + * + * @param remoteLogSegmentId Universally unique remote log segment id. + * @param startOffset Start offset of this segment (inclusive). + * @param endOffset End offset of this segment (inclusive). + * @param maxTimestamp Maximum timestamp in this segment + * @param brokerId Broker id from which this event is generated. + * @param eventTimestamp Epoch time in milli seconds at which the remote log segment is copied to the remote tier storage. + * @param segmentSizeInBytes Size of this segment in bytes. + * @param segmentLeaderEpochs leader epochs occurred within this segment + */ + public RemoteLogSegmentMetadata(RemoteLogSegmentId remoteLogSegmentId, + long startOffset, + long endOffset, + long maxTimestamp, + int brokerId, + long eventTimestamp, + int segmentSizeInBytes, + Map segmentLeaderEpochs) { + this(remoteLogSegmentId, + startOffset, + endOffset, + maxTimestamp, + brokerId, + eventTimestamp, segmentSizeInBytes, + RemoteLogSegmentState.COPY_SEGMENT_STARTED, + segmentLeaderEpochs); + } + + + /** + * @return unique id of this segment. + */ + public RemoteLogSegmentId remoteLogSegmentId() { + return remoteLogSegmentId; + } + + /** + * @return Start offset of this segment (inclusive). + */ + public long startOffset() { + return startOffset; + } + + /** + * @return End offset of this segment (inclusive). + */ + public long endOffset() { + return endOffset; + } + + /** + * @return Epoch time at which this event is occurred. + */ + public long eventTimestamp() { + return eventTimestamp; + } + + /** + * @return Total size of this segment in bytes. + */ + public int segmentSizeInBytes() { + return segmentSizeInBytes; + } + + /** + * @return Maximum timestamp of a record within this segment. + */ + public long maxTimestamp() { + return maxTimestamp; + } + + /** + * @return Map of leader epoch vs offset for the records available in this segment. + */ + public NavigableMap segmentLeaderEpochs() { + return segmentLeaderEpochs; + } + + /** + * @return Broker id from which this event is generated. + */ + public int brokerId() { + return brokerId; + } + + /** + * Returns the current state of this remote log segment. It can be any of the below + *

    + * {@link RemoteLogSegmentState#COPY_SEGMENT_STARTED} + * {@link RemoteLogSegmentState#COPY_SEGMENT_FINISHED} + * {@link RemoteLogSegmentState#DELETE_SEGMENT_STARTED} + * {@link RemoteLogSegmentState#DELETE_SEGMENT_FINISHED} + *
+ */ + public RemoteLogSegmentState state() { + return state; + } + + /** + * Creates a new RemoteLogSegmentMetadata applying the given {@code rlsmUpdate} on this instance. This method will + * not update this instance. + * + * @param rlsmUpdate update to be applied. + * @return a new instance created by applying the given update on this instance. + */ + public RemoteLogSegmentMetadata createRemoteLogSegmentWithUpdates(RemoteLogSegmentMetadataUpdate rlsmUpdate) { + if (!remoteLogSegmentId.equals(rlsmUpdate.remoteLogSegmentId())) { + throw new IllegalArgumentException("Given rlsmUpdate does not have this instance's remoteLogSegmentId."); + } + + return new RemoteLogSegmentMetadata(remoteLogSegmentId, startOffset, + endOffset, maxTimestamp, rlsmUpdate.brokerId(), rlsmUpdate.eventTimestamp(), + segmentSizeInBytes, rlsmUpdate.state(), segmentLeaderEpochs); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + RemoteLogSegmentMetadata that = (RemoteLogSegmentMetadata) o; + return startOffset == that.startOffset && endOffset == that.endOffset && brokerId == that.brokerId + && maxTimestamp == that.maxTimestamp && eventTimestamp == that.eventTimestamp + && segmentSizeInBytes == that.segmentSizeInBytes + && Objects.equals(remoteLogSegmentId, that.remoteLogSegmentId) + && Objects.equals(segmentLeaderEpochs, that.segmentLeaderEpochs) && state == that.state; + } + + @Override + public int hashCode() { + return Objects.hash(remoteLogSegmentId, startOffset, endOffset, brokerId, maxTimestamp, eventTimestamp, + segmentLeaderEpochs, segmentSizeInBytes, state); + } + + @Override + public String toString() { + return "RemoteLogSegmentMetadata{" + + "remoteLogSegmentId=" + remoteLogSegmentId + + ", startOffset=" + startOffset + + ", endOffset=" + endOffset + + ", brokerId=" + brokerId + + ", maxTimestamp=" + maxTimestamp + + ", eventTimestamp=" + eventTimestamp + + ", segmentLeaderEpochs=" + segmentLeaderEpochs + + ", segmentSizeInBytes=" + segmentSizeInBytes + + ", state=" + state + + '}'; + } + +} diff --git a/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogSegmentMetadataUpdate.java b/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogSegmentMetadataUpdate.java new file mode 100644 index 0000000000000..4fc1ea1b4cc95 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogSegmentMetadataUpdate.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.server.log.remote.storage; + +import org.apache.kafka.common.annotation.InterfaceStability; + +import java.util.Objects; + +/** + * It describes the metadata update about the log segment in the remote storage. This is currently used to update the + * state of the remote log segment by using {@link RemoteLogMetadataManager#updateRemoteLogSegmentMetadata(RemoteLogSegmentMetadataUpdate)}. + * This also includes the timestamp of this event. + */ +@InterfaceStability.Evolving +public class RemoteLogSegmentMetadataUpdate { + + /** + * Universally unique remote log segment id. + */ + private final RemoteLogSegmentId remoteLogSegmentId; + + /** + * Epoch time at which this event is generated. + */ + private final long eventTimestamp; + + /** + * It indicates the state in which the action is executed on this segment. + */ + private final RemoteLogSegmentState state; + + /** + * Broker id from which this event is generated. + */ + private final int brokerId; + + /** + * @param remoteLogSegmentId Universally unique remote log segment id. + * @param eventTimestamp Epoch time in milli seconds at which the remote log segment is copied to the remote tier storage. + * @param state State of the remote log segment. + * @param brokerId Broker id from which this event is generated. + */ + public RemoteLogSegmentMetadataUpdate(RemoteLogSegmentId remoteLogSegmentId, long eventTimestamp, + RemoteLogSegmentState state, int brokerId) { + this.remoteLogSegmentId = Objects.requireNonNull(remoteLogSegmentId, "remoteLogSegmentId can not be null"); + this.state = Objects.requireNonNull(state, "state can not be null"); + this.brokerId = brokerId; + this.eventTimestamp = eventTimestamp; + } + + /** + * @return Universally unique id of this remote log segment. + */ + public RemoteLogSegmentId remoteLogSegmentId() { + return remoteLogSegmentId; + } + + /** + * @return Epoch time at which this event is generated. + */ + public long eventTimestamp() { + return eventTimestamp; + } + + /** + * It represents the state of the remote log segment. It can be one of the values of {@link RemoteLogSegmentState}. + */ + public RemoteLogSegmentState state() { + return state; + } + + /** + * @return Broker id from which this event is generated. + */ + public int brokerId() { + return brokerId; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + RemoteLogSegmentMetadataUpdate that = (RemoteLogSegmentMetadataUpdate) o; + return eventTimestamp == that.eventTimestamp && Objects + .equals(remoteLogSegmentId, that.remoteLogSegmentId) && state == that.state && brokerId == that.brokerId; + } + + @Override + public int hashCode() { + return Objects.hash(remoteLogSegmentId, eventTimestamp, state, brokerId); + } + + @Override + public String toString() { + return "RemoteLogSegmentMetadataUpdate{" + + "remoteLogSegmentId=" + remoteLogSegmentId + + ", eventTimestamp=" + eventTimestamp + + ", state=" + state + + ", brokerId=" + brokerId + + '}'; + } +} diff --git a/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogSegmentState.java b/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogSegmentState.java new file mode 100644 index 0000000000000..e27fdd19cd43b --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogSegmentState.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.server.log.remote.storage; + +import org.apache.kafka.common.annotation.InterfaceStability; + +import java.util.Arrays; +import java.util.Collections; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * It indicates the state of the remote log segment. This will be based on the action executed on this + * segment by the remote log service implementation. + *

+ * It goes through the below state transitions. + *

+ * 

+ * +---------------------+            +----------------------+
+ * |COPY_SEGMENT_STARTED |----------->|COPY_SEGMENT_FINISHED |
+ * +-------------------+-+            +--+-------------------+
+ *                     |                 |
+ *                     |                 |
+ *                     v                 v
+ *                  +--+-----------------+-+
+ *                  |DELETE_SEGMENT_STARTED|
+ *                  +-----------+----------+
+ *                              |
+ *                              |
+ *                              v
+ *                  +-----------+-----------+
+ *                  |DELETE_SEGMENT_FINISHED|
+ *                  +-----------------------+
+ *
+ */ +@InterfaceStability.Evolving +public enum RemoteLogSegmentState { + + /** + * This state indicates that the segment copying to remote storage is started but not yet finished. + */ + COPY_SEGMENT_STARTED((byte) 0), + + /** + * This state indicates that the segment copying to remote storage is finished. + */ + COPY_SEGMENT_FINISHED((byte) 1), + + /** + * This state indicates that the segment deletion is started but not yet finished. + */ + DELETE_SEGMENT_STARTED((byte) 2), + + /** + * This state indicates that the segment is deleted successfully. + */ + DELETE_SEGMENT_FINISHED((byte) 3); + + private static final Map STATE_TYPES = Collections.unmodifiableMap( + Arrays.stream(values()).collect(Collectors.toMap(RemoteLogSegmentState::id, Function.identity()))); + + private final byte id; + + RemoteLogSegmentState(byte id) { + this.id = id; + } + + public byte id() { + return id; + } + + public static RemoteLogSegmentState forId(byte id) { + return STATE_TYPES.get(id); + } +} diff --git a/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemotePartitionDeleteMetadata.java b/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemotePartitionDeleteMetadata.java new file mode 100644 index 0000000000000..fdf4e61c1dbec --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemotePartitionDeleteMetadata.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.server.log.remote.storage; + +import org.apache.kafka.common.TopicIdPartition; +import org.apache.kafka.common.annotation.InterfaceStability; + +import java.util.Objects; + +/** + * This class represents the metadata about the remote partition. It can be updated with {@link RemoteLogMetadataManager#putRemotePartitionDeleteMetadata(RemotePartitionDeleteMetadata)}. + * Possible state transitions are mentioned at {@link RemotePartitionDeleteState}. + */ +@InterfaceStability.Evolving +public class RemotePartitionDeleteMetadata { + + private final TopicIdPartition topicIdPartition; + private final RemotePartitionDeleteState state; + private final long eventTimestamp; + private final int brokerId; + + /** + * + * @param topicIdPartition + * @param state + * @param eventTimestamp + * @param brokerId + */ + public RemotePartitionDeleteMetadata(TopicIdPartition topicIdPartition, + RemotePartitionDeleteState state, + long eventTimestamp, + int brokerId) { + this.topicIdPartition = Objects.requireNonNull(topicIdPartition); + this.state = Objects.requireNonNull(state); + this.eventTimestamp = eventTimestamp; + this.brokerId = brokerId; + } + + /** + * @return TopicIdPartition for which this event is meant for. + */ + public TopicIdPartition topicIdPartition() { + return topicIdPartition; + } + + /** + * It represents the state of the remote partition. It can be one of the values of {@link RemotePartitionDeleteState}. + */ + public RemotePartitionDeleteState state() { + return state; + } + + /** + * @return Epoch time at which this event is occurred. + */ + public long eventTimestamp() { + return eventTimestamp; + } + + /** + * @return broker id from which this event is generated. + */ + public int brokerId() { + return brokerId; + } + + @Override + public String toString() { + return "RemotePartitionDeleteMetadata{" + + "topicPartition=" + topicIdPartition + + ", state=" + state + + ", eventTimestamp=" + eventTimestamp + + ", brokerId=" + brokerId + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + RemotePartitionDeleteMetadata that = (RemotePartitionDeleteMetadata) o; + return eventTimestamp == that.eventTimestamp && + brokerId == that.brokerId && + Objects.equals(topicIdPartition, that.topicIdPartition) && + state == that.state; + } + + @Override + public int hashCode() { + return Objects.hash(topicIdPartition, state, eventTimestamp, brokerId); + } +} diff --git a/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemotePartitionDeleteState.java b/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemotePartitionDeleteState.java new file mode 100644 index 0000000000000..d5172fc1f942e --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemotePartitionDeleteState.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.server.log.remote.storage; + +import org.apache.kafka.common.annotation.InterfaceStability; + +import java.util.Arrays; +import java.util.Collections; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * It indicates the deletion state of the remote topic partition. This will be based on the action executed on this + * partition by the remote log service implementation. + * State transitions are mentioned below. + *

+ * 

+ * +-------------------------+
+ * |DELETE_PARTITION_MARKED  |
+ * +-----------+-------------+
+ * |
+ * |
+ * +-----------v--------------+
+ * |DELETE_PARTITION_STARTED  |
+ * +-----------+--------------+
+ * |
+ * |
+ * +-----------v--------------+
+ * |DELETE_PARTITION_FINISHED |
+ * +--------------------------+
+ *
+ *

+ */ +@InterfaceStability.Evolving +public enum RemotePartitionDeleteState { + + /** + * This is used when a topic/partition is marked for delete by the controller. + * That means, all its remote log segments are eligible for deletion so that remote partition removers can + * start deleting them. + */ + DELETE_PARTITION_MARKED((byte) 0), + + /** + * This state indicates that the partition deletion is started but not yet finished. + */ + DELETE_PARTITION_STARTED((byte) 1), + + /** + * This state indicates that the partition is deleted successfully. + */ + DELETE_PARTITION_FINISHED((byte) 2); + + private static final Map STATE_TYPES = Collections.unmodifiableMap( + Arrays.stream(values()).collect(Collectors.toMap(RemotePartitionDeleteState::id, Function.identity()))); + + private final byte id; + + RemotePartitionDeleteState(byte id) { + this.id = id; + } + + public byte id() { + return id; + } + + public static RemotePartitionDeleteState forId(byte id) { + return STATE_TYPES.get(id); + } + +} \ No newline at end of file diff --git a/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteResourceNotFoundException.java b/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteResourceNotFoundException.java new file mode 100644 index 0000000000000..f6ac4ec5ccc19 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteResourceNotFoundException.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.server.log.remote.storage; + +/** + * Exception thrown when a resource is not found on the remote storage. + *

+ * A resource can be a log segment, any of the indexes or any which was stored in remote storage for a particular log + * segment. + */ +public class RemoteResourceNotFoundException extends RemoteStorageException { + private static final long serialVersionUID = 1L; + + public RemoteResourceNotFoundException(final String message) { + super(message); + } + + public RemoteResourceNotFoundException(final Throwable cause) { + super("Requested remote resource was not found", cause); + } + + public RemoteResourceNotFoundException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteStorageException.java b/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteStorageException.java new file mode 100644 index 0000000000000..f2488cb2957fb --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteStorageException.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.server.log.remote.storage; + +/** + * Exception thrown when there is a remote storage error. + * This can be used as the base exception by implementors of + * {@link RemoteStorageManager} or {@link RemoteLogMetadataManager} to create extended exceptions. + */ +public class RemoteStorageException extends Exception { + private static final long serialVersionUID = 1L; + + public RemoteStorageException(final String message) { + super(message); + } + + public RemoteStorageException(final String message, + final Throwable cause) { + super(message, cause); + } + +} diff --git a/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteStorageManager.java b/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteStorageManager.java new file mode 100644 index 0000000000000..64e3fdd026616 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteStorageManager.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.server.log.remote.storage; + +import org.apache.kafka.common.Configurable; +import org.apache.kafka.common.annotation.InterfaceStability; + +import java.io.Closeable; +import java.io.InputStream; + +/** + * This interface provides the lifecycle of remote log segments that includes copy, fetch, and delete from remote + * storage. + *

+ * Each upload or copy of a segment is initiated with {@link RemoteLogSegmentMetadata} containing {@link RemoteLogSegmentId} + * which is universally unique even for the same topic partition and offsets. + *

+ * {@link RemoteLogSegmentMetadata} is stored in {@link RemoteLogMetadataManager} before and after copy/delete operations on + * {@link RemoteStorageManager} with the respective {@link RemoteLogSegmentState}. {@link RemoteLogMetadataManager} is + * responsible for storing and fetching metadata about the remote log segments in a strongly consistent manner. + * This allows {@link RemoteStorageManager} to have eventual consistency on metadata (although the data is stored + * in strongly consistent semantics). + */ +@InterfaceStability.Evolving +public interface RemoteStorageManager extends Configurable, Closeable { + + /** + * Type of the index file. + */ + enum IndexType { + /** + * Represents offset index. + */ + Offset, + + /** + * Represents timestamp index. + */ + Timestamp, + + /** + * Represents producer snapshot index. + */ + ProducerSnapshot, + + /** + * Represents transaction index. + */ + Transaction, + + /** + * Represents leader epoch index. + */ + LeaderEpoch, + } + + /** + * Copies the given {@link LogSegmentData} provided for the given {@code remoteLogSegmentMetadata}. This includes + * log segment and its auxiliary indexes like offset index, time index, transaction index, leader epoch index, and + * producer snapshot index. + *

+ * Invoker of this API should always send a unique id as part of {@link RemoteLogSegmentMetadata#remoteLogSegmentId()} + * even when it retries to invoke this method for the same log segment data. + * + * @param remoteLogSegmentMetadata metadata about the remote log segment. + * @param logSegmentData data to be copied to tiered storage. + * @throws RemoteStorageException if there are any errors in storing the data of the segment. + */ + void copyLogSegmentData(RemoteLogSegmentMetadata remoteLogSegmentMetadata, + LogSegmentData logSegmentData) + throws RemoteStorageException; + + /** + * Returns the remote log segment data file/object as InputStream for the given {@link RemoteLogSegmentMetadata} + * starting from the given startPosition. The stream will end at the end of the remote log segment data file/object. + * + * @param remoteLogSegmentMetadata metadata about the remote log segment. + * @param startPosition start position of log segment to be read, inclusive. + * @return input stream of the requested log segment data. + * @throws RemoteStorageException if there are any errors while fetching the desired segment. + * @throws RemoteResourceNotFoundException when there are no resources associated with the given remoteLogSegmentMetadata. + */ + InputStream fetchLogSegment(RemoteLogSegmentMetadata remoteLogSegmentMetadata, + int startPosition) throws RemoteStorageException; + + /** + * Returns the remote log segment data file/object as InputStream for the given {@link RemoteLogSegmentMetadata} + * starting from the given startPosition. The stream will end at the smaller of endPosition and the end of the + * remote log segment data file/object. + * + * @param remoteLogSegmentMetadata metadata about the remote log segment. + * @param startPosition start position of log segment to be read, inclusive. + * @param endPosition end position of log segment to be read, inclusive. + * @return input stream of the requested log segment data. + * @throws RemoteStorageException if there are any errors while fetching the desired segment. + * @throws RemoteResourceNotFoundException when there are no resources associated with the given remoteLogSegmentMetadata. + */ + InputStream fetchLogSegment(RemoteLogSegmentMetadata remoteLogSegmentMetadata, + int startPosition, + int endPosition) throws RemoteStorageException; + + /** + * Returns the index for the respective log segment of {@link RemoteLogSegmentMetadata}. + * + * @param remoteLogSegmentMetadata metadata about the remote log segment. + * @param indexType type of the index to be fetched for the segment. + * @return input stream of the requested index. + * @throws RemoteStorageException if there are any errors while fetching the index. + * @throws RemoteResourceNotFoundException when there are no resources associated with the given remoteLogSegmentMetadata. + */ + InputStream fetchIndex(RemoteLogSegmentMetadata remoteLogSegmentMetadata, + IndexType indexType) throws RemoteStorageException; + + /** + * Deletes the resources associated with the given {@code remoteLogSegmentMetadata}. Deletion is considered as + * successful if this call returns successfully without any errors. It will throw {@link RemoteStorageException} if + * there are any errors in deleting the file. + *

+ * + * @param remoteLogSegmentMetadata metadata about the remote log segment to be deleted. + * @throws RemoteResourceNotFoundException if the requested resource is not found + * @throws RemoteStorageException if there are any storage related errors occurred. + * @throws RemoteResourceNotFoundException when there are no resources associated with the given remoteLogSegmentMetadata. + */ + void deleteLogSegmentData(RemoteLogSegmentMetadata remoteLogSegmentMetadata) throws RemoteStorageException; + +} \ No newline at end of file From f6929637b9222a64e8037348ed85c1088dfc1efb Mon Sep 17 00:00:00 2001 From: Luke Chen <43372967+showuon@users.noreply.github.com> Date: Thu, 4 Mar 2021 03:20:08 +0800 Subject: [PATCH 04/59] KAFKA-12284: Wait for mirrored topics to be created (#10185) Reviewers: Mickael Maison --- .../MirrorConnectorsIntegrationBaseTest.java | 105 ++++++++++++------ 1 file changed, 68 insertions(+), 37 deletions(-) diff --git a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/integration/MirrorConnectorsIntegrationBaseTest.java b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/integration/MirrorConnectorsIntegrationBaseTest.java index f2cc850a4f07d..7aae7d51eb25b 100644 --- a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/integration/MirrorConnectorsIntegrationBaseTest.java +++ b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/integration/MirrorConnectorsIntegrationBaseTest.java @@ -47,6 +47,7 @@ import java.util.HashMap; import java.util.Map; import java.util.Properties; +import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -82,6 +83,7 @@ public abstract class MirrorConnectorsIntegrationBaseTest { private static final int CHECKPOINT_DURATION_MS = 20_000; private static final int RECORD_CONSUME_DURATION_MS = 20_000; private static final int OFFSET_SYNC_DURATION_MS = 30_000; + private static final int TOPIC_SYNC_DURATION_MS = 30_000; private static final int NUM_WORKERS = 3; private static final Duration CONSUMER_POLL_TIMEOUT_MS = Duration.ofMillis(500); protected static final String PRIMARY_CLUSTER_ALIAS = "primary"; @@ -227,7 +229,10 @@ public void testReplication() throws Exception { MirrorClient primaryClient = new MirrorClient(mm2Config.clientConfig(PRIMARY_CLUSTER_ALIAS)); MirrorClient backupClient = new MirrorClient(mm2Config.clientConfig(BACKUP_CLUSTER_ALIAS)); - + + // make sure the topic is auto-created in the other cluster + waitForTopicCreated(primary.kafka(), "backup.test-topic-1"); + waitForTopicCreated(backup.kafka(), "primary.test-topic-1"); assertEquals(TopicConfig.CLEANUP_POLICY_COMPACT, getTopicConfig(backup.kafka(), "primary.test-topic-1", TopicConfig.CLEANUP_POLICY_CONFIG), "topic config was not synced"); @@ -312,6 +317,10 @@ public void testReplication() throws Exception { primary.kafka().createTopic("test-topic-2", NUM_PARTITIONS); backup.kafka().createTopic("test-topic-3", NUM_PARTITIONS); + // make sure the topic is auto-created in the other cluster + waitForTopicCreated(backup.kafka(), "primary.test-topic-2"); + waitForTopicCreated(primary.kafka(), "backup.test-topic-3"); + // only produce messages to the first partition produceMessages(primary, "test-topic-2", 1); produceMessages(backup, "test-topic-3", 1); @@ -360,15 +369,17 @@ public void testReplicationWithEmptyPartition() throws Exception { waitForConsumingAllRecords(backupConsumer, expectedRecords); } - Admin backupClient = backup.kafka().createAdminClient(); - // retrieve the consumer group offset from backup cluster - Map remoteOffsets = + try (Admin backupClient = backup.kafka().createAdminClient()) { + // retrieve the consumer group offset from backup cluster + Map remoteOffsets = backupClient.listConsumerGroupOffsets(consumerGroupName).partitionsToOffsetAndMetadata().get(); - // pinpoint the offset of the last partition which does not receive records - OffsetAndMetadata offset = remoteOffsets.get(new TopicPartition(PRIMARY_CLUSTER_ALIAS + "." + topic, NUM_PARTITIONS - 1)); - // offset of the last partition should exist, but its value should be 0 - assertNotNull(offset, "Offset of last partition was not replicated"); - assertEquals(0, offset.offset(), "Offset of last partition is not zero"); + + // pinpoint the offset of the last partition which does not receive records + OffsetAndMetadata offset = remoteOffsets.get(new TopicPartition(PRIMARY_CLUSTER_ALIAS + "." + topic, NUM_PARTITIONS - 1)); + // offset of the last partition should exist, but its value should be 0 + assertNotNull(offset, "Offset of last partition was not replicated"); + assertEquals(0, offset.offset(), "Offset of last partition is not zero"); + } } @Test @@ -396,6 +407,9 @@ public void testOneWayReplicationWithAutoOffsetSync() throws InterruptedExceptio waitUntilMirrorMakerIsRunning(backup, CONNECTOR_LIST, mm2Config, PRIMARY_CLUSTER_ALIAS, BACKUP_CLUSTER_ALIAS); + // make sure the topic is created in the other cluster + waitForTopicCreated(primary.kafka(), "backup.test-topic-1"); + waitForTopicCreated(backup.kafka(), "primary.test-topic-1"); // create a consumer at backup cluster with same consumer group Id to consume 1 topic Consumer backupConsumer = backup.kafka().createConsumerAndSubscribeTo( consumerProps, "primary.test-topic-1"); @@ -412,7 +426,9 @@ public void testOneWayReplicationWithAutoOffsetSync() throws InterruptedExceptio // now create a new topic in primary cluster primary.kafka().createTopic("test-topic-2", NUM_PARTITIONS); - backup.kafka().createTopic("primary.test-topic-2", 1); + // make sure the topic is created in backup cluster + waitForTopicCreated(backup.kafka(), "primary.test-topic-2"); + // produce some records to the new topic in primary cluster produceMessages(primary, "test-topic-2"); @@ -455,27 +471,41 @@ private static void waitUntilMirrorMakerIsRunning(EmbeddedConnectCluster connect "Connector " + connector.getSimpleName() + " tasks did not start in time on cluster: " + connectCluster); } } - + + /* + * wait for the topic created on the cluster + */ + private static void waitForTopicCreated(EmbeddedKafkaCluster cluster, String topicName) throws InterruptedException { + try (final Admin adminClient = cluster.createAdminClient()) { + waitForCondition(() -> adminClient.listTopics().names().get().contains(topicName), TOPIC_SYNC_DURATION_MS, + "Topic: " + topicName + " didn't get created in the cluster" + ); + } + } + /* * delete all topics of the input kafka cluster */ private static void deleteAllTopics(EmbeddedKafkaCluster cluster) throws Exception { - Admin client = cluster.createAdminClient(); - client.deleteTopics(client.listTopics().names().get()); + try (final Admin adminClient = cluster.createAdminClient()) { + Set topicsToBeDeleted = adminClient.listTopics().names().get(); + log.debug("Deleting topics: {} ", topicsToBeDeleted); + adminClient.deleteTopics(topicsToBeDeleted).all().get(); + } } /* * retrieve the config value based on the input cluster, topic and config name */ - private static String getTopicConfig(EmbeddedKafkaCluster cluster, String topic, String configName) - throws Exception { - Admin client = cluster.createAdminClient(); - Collection cr = Collections.singleton( - new ConfigResource(ConfigResource.Type.TOPIC, topic)); - - DescribeConfigsResult configsResult = client.describeConfigs(cr); - Config allConfigs = (Config) configsResult.all().get().values().toArray()[0]; - return allConfigs.get(configName).value(); + private static String getTopicConfig(EmbeddedKafkaCluster cluster, String topic, String configName) throws Exception { + try (Admin client = cluster.createAdminClient()) { + Collection cr = Collections.singleton( + new ConfigResource(ConfigResource.Type.TOPIC, topic)); + + DescribeConfigsResult configsResult = client.describeConfigs(cr); + Config allConfigs = (Config) configsResult.all().get().values().toArray()[0]; + return allConfigs.get(configName).value(); + } } /* @@ -505,27 +535,28 @@ protected void produceMessages(EmbeddedConnectCluster cluster, String topicName, private static void waitForConsumerGroupOffsetSync(EmbeddedConnectCluster connect, Consumer consumer, List topics, String consumerGroupId, int numRecords) throws InterruptedException { - Admin adminClient = connect.kafka().createAdminClient(); - List tps = new ArrayList<>(NUM_PARTITIONS * topics.size()); - for (int partitionIndex = 0; partitionIndex < NUM_PARTITIONS; partitionIndex++) { - for (String topic : topics) { - tps.add(new TopicPartition(topic, partitionIndex)); + try (Admin adminClient = connect.kafka().createAdminClient()) { + List tps = new ArrayList<>(NUM_PARTITIONS * topics.size()); + for (int partitionIndex = 0; partitionIndex < NUM_PARTITIONS; partitionIndex++) { + for (String topic : topics) { + tps.add(new TopicPartition(topic, partitionIndex)); + } } - } - long expectedTotalOffsets = numRecords * topics.size(); + long expectedTotalOffsets = numRecords * topics.size(); - waitForCondition(() -> { - Map consumerGroupOffsets = + waitForCondition(() -> { + Map consumerGroupOffsets = adminClient.listConsumerGroupOffsets(consumerGroupId).partitionsToOffsetAndMetadata().get(); - long consumerGroupOffsetTotal = consumerGroupOffsets.values().stream() + long consumerGroupOffsetTotal = consumerGroupOffsets.values().stream() .mapToLong(OffsetAndMetadata::offset).sum(); - Map offsets = consumer.endOffsets(tps, CONSUMER_POLL_TIMEOUT_MS); - long totalOffsets = offsets.values().stream().mapToLong(l -> l).sum(); + Map offsets = consumer.endOffsets(tps, CONSUMER_POLL_TIMEOUT_MS); + long totalOffsets = offsets.values().stream().mapToLong(l -> l).sum(); - // make sure the consumer group offsets are synced to expected number - return totalOffsets == expectedTotalOffsets && consumerGroupOffsetTotal > 0; - }, OFFSET_SYNC_DURATION_MS, "Consumer group offset sync is not complete in time"); + // make sure the consumer group offsets are synced to expected number + return totalOffsets == expectedTotalOffsets && consumerGroupOffsetTotal > 0; + }, OFFSET_SYNC_DURATION_MS, "Consumer group offset sync is not complete in time"); + } } /* From f40a82e05440e579c69363dc62c2dd9c8520eb02 Mon Sep 17 00:00:00 2001 From: David Arthur Date: Wed, 3 Mar 2021 15:57:27 -0500 Subject: [PATCH 05/59] KAFKA-10759 Add ARM build stage (#9992) Only validation and unit test stages are enabled Co-authored-by: Peng.Lei <73098678+xiao-penglei@users.noreply.github.com> --- Jenkinsfile | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 87840d54b79a9..557988788cd4d 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -33,12 +33,12 @@ def doValidation() { ''' } -def doTest() { - sh ''' - ./gradlew -PscalaVersion=$SCALA_VERSION unitTest integrationTest \ +def doTest(target = "unitTest integrationTest") { + sh """ + ./gradlew -PscalaVersion=$SCALA_VERSION ${target} \ --profile --no-daemon --continue -PtestLoggingEvents=started,passed,skipped,failed \ -PignoreFailures=true -PmaxParallelForks=2 -PmaxTestRetries=1 -PmaxTestRetryFailures=5 - ''' + """ junit '**/build/test-results/**/TEST-*.xml' } @@ -158,6 +158,25 @@ pipeline { echo 'Skipping Kafka Streams archetype test for Java 15' } } + + stage('ARM') { + agent { label 'arm4' } + options { + timeout(time: 2, unit: 'HOURS') + timestamps() + } + environment { + SCALA_VERSION=2.12 + } + steps { + setupGradle() + doValidation() + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + doTest('unitTest') + } + echo 'Skipping Kafka Streams archetype test for ARM build' + } + } } } } From f06a47a7bba9baf035691feadbfeb3ff21802e82 Mon Sep 17 00:00:00 2001 From: Sven Erik Knop Date: Wed, 3 Mar 2021 22:17:49 +0000 Subject: [PATCH 06/59] KAFKA-12170: Fix for Connect Cast SMT to correctly transform a Byte array into a string (#9950) Cast SMT transformation for bytes -> string. Without this fix, the conversion becomes ByteBuffer.toString(), which always gives this useless result: "java.nio.HeapByteBuffer[pos=0 lim=4 cap=4]" With this change, the byte array is converted into a base64 string of the byte buffer content. Reviewers: Mickael Maison , Randall Hauch , Konstantine Karantasis --- .../apache/kafka/connect/transforms/Cast.java | 14 ++++++++++++-- .../kafka/connect/transforms/CastTest.java | 19 ++++++++++++++++++- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/connect/transforms/src/main/java/org/apache/kafka/connect/transforms/Cast.java b/connect/transforms/src/main/java/org/apache/kafka/connect/transforms/Cast.java index e872b336e8573..0a763cc24f85b 100644 --- a/connect/transforms/src/main/java/org/apache/kafka/connect/transforms/Cast.java +++ b/connect/transforms/src/main/java/org/apache/kafka/connect/transforms/Cast.java @@ -22,6 +22,7 @@ import org.apache.kafka.common.cache.SynchronizedCache; import org.apache.kafka.common.config.ConfigDef; import org.apache.kafka.common.config.ConfigException; +import org.apache.kafka.common.utils.Utils; import org.apache.kafka.connect.connector.ConnectRecord; import org.apache.kafka.connect.data.ConnectSchema; import org.apache.kafka.connect.data.Date; @@ -39,6 +40,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.nio.ByteBuffer; +import java.util.Base64; import java.util.EnumSet; import java.util.HashMap; import java.util.List; @@ -56,7 +59,8 @@ public abstract class Cast> implements Transformation // allow casting nested fields. public static final String OVERVIEW_DOC = "Cast fields or the entire key or value to a specific type, e.g. to force an integer field to a smaller " - + "width. Only simple primitive types are supported -- integers, floats, boolean, and string. " + + "width. Cast from integers, floats, boolean and string to any other type, " + + "and cast binary to string (base64 encoded)." + "

Use the concrete transformation type designed for the record key (" + Key.class.getName() + ") " + "or value (" + Value.class.getName() + ")."; @@ -82,7 +86,7 @@ public String toString() { ConfigDef.Importance.HIGH, "List of fields and the type to cast them to of the form field1:type,field2:type to cast fields of " + "Maps or Structs. A single type to cast the entire value. Valid types are int8, int16, int32, " - + "int64, float32, float64, boolean, and string."); + + "int64, float32, float64, boolean, and string. Note that binary fields can only be cast to string."); private static final String PURPOSE = "cast types"; @@ -364,6 +368,12 @@ private static String castToString(Object value) { if (value instanceof java.util.Date) { java.util.Date dateValue = (java.util.Date) value; return Values.dateFormatFor(dateValue).format(dateValue); + } else if (value instanceof ByteBuffer) { + ByteBuffer byteBuffer = (ByteBuffer) value; + return Base64.getEncoder().encodeToString(Utils.readBytes(byteBuffer)); + } else if (value instanceof byte[]) { + byte[] rawBytes = (byte[]) value; + return Base64.getEncoder().encodeToString(rawBytes); } else { return value.toString(); } diff --git a/connect/transforms/src/test/java/org/apache/kafka/connect/transforms/CastTest.java b/connect/transforms/src/test/java/org/apache/kafka/connect/transforms/CastTest.java index ae90c1956b9f0..764b904ea3e24 100644 --- a/connect/transforms/src/test/java/org/apache/kafka/connect/transforms/CastTest.java +++ b/connect/transforms/src/test/java/org/apache/kafka/connect/transforms/CastTest.java @@ -17,6 +17,7 @@ package org.apache.kafka.connect.transforms; +import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; import java.util.concurrent.TimeUnit; @@ -425,7 +426,11 @@ public void castLogicalToString() { @Test public void castFieldsWithSchema() { Date day = new Date(MILLIS_PER_DAY); - xformValue.configure(Collections.singletonMap(Cast.SPEC_CONFIG, "int8:int16,int16:int32,int32:int64,int64:boolean,float32:float64,float64:boolean,boolean:int8,string:int32,bigdecimal:string,date:string,optional:int32")); + byte[] byteArray = new byte[] {(byte) 0xFE, (byte) 0xDC, (byte) 0xBA, (byte) 0x98, 0x76, 0x54, 0x32, 0x10}; + ByteBuffer byteBuffer = ByteBuffer.wrap(Arrays.copyOf(byteArray, byteArray.length)); + + xformValue.configure(Collections.singletonMap(Cast.SPEC_CONFIG, + "int8:int16,int16:int32,int32:int64,int64:boolean,float32:float64,float64:boolean,boolean:int8,string:int32,bigdecimal:string,date:string,optional:int32,bytes:string,byteArray:string")); // Include an optional fields and fields with defaults to validate their values are passed through properly SchemaBuilder builder = SchemaBuilder.struct(); @@ -442,6 +447,9 @@ public void castFieldsWithSchema() { builder.field("date", org.apache.kafka.connect.data.Date.SCHEMA); builder.field("optional", Schema.OPTIONAL_FLOAT32_SCHEMA); builder.field("timestamp", Timestamp.SCHEMA); + builder.field("bytes", Schema.BYTES_SCHEMA); + builder.field("byteArray", Schema.BYTES_SCHEMA); + Schema supportedTypesSchema = builder.build(); Struct recordValue = new Struct(supportedTypesSchema); @@ -456,6 +464,9 @@ public void castFieldsWithSchema() { recordValue.put("date", day); recordValue.put("string", "42"); recordValue.put("timestamp", new Date(0)); + recordValue.put("bytes", byteBuffer); + recordValue.put("byteArray", byteArray); + // optional field intentionally omitted SourceRecord transformed = xformValue.apply(new SourceRecord(null, null, "topic", 0, @@ -475,6 +486,9 @@ public void castFieldsWithSchema() { assertEquals("42", ((Struct) transformed.value()).get("bigdecimal")); assertEquals(Values.dateFormatFor(day).format(day), ((Struct) transformed.value()).get("date")); assertEquals(new Date(0), ((Struct) transformed.value()).get("timestamp")); + assertEquals("/ty6mHZUMhA=", ((Struct) transformed.value()).get("bytes")); + assertEquals("/ty6mHZUMhA=", ((Struct) transformed.value()).get("byteArray")); + assertNull(((Struct) transformed.value()).get("optional")); Schema transformedSchema = ((Struct) transformed.value()).schema(); @@ -489,6 +503,9 @@ public void castFieldsWithSchema() { assertEquals(Schema.STRING_SCHEMA.type(), transformedSchema.field("bigdecimal").schema().type()); assertEquals(Schema.STRING_SCHEMA.type(), transformedSchema.field("date").schema().type()); assertEquals(Schema.OPTIONAL_INT32_SCHEMA.type(), transformedSchema.field("optional").schema().type()); + assertEquals(Schema.STRING_SCHEMA.type(), transformedSchema.field("bytes").schema().type()); + assertEquals(Schema.STRING_SCHEMA.type(), transformedSchema.field("byteArray").schema().type()); + // The following fields are not changed assertEquals(Timestamp.SCHEMA.type(), transformedSchema.field("timestamp").schema().type()); } From 3ef39e13658be2ea0c79d44fba241fdf9257e51c Mon Sep 17 00:00:00 2001 From: David Jacot Date: Thu, 4 Mar 2021 10:31:35 +0100 Subject: [PATCH 07/59] MINOR; Clean up LeaderAndIsrResponse construction in `ReplicaManager#becomeLeaderOrFollower` (#10234) This patch refactors the code, which constructs the `LeaderAndIsrResponse` in `ReplicaManager#becomeLeaderOrFollower`, to improve the readability and to remove unnecessary operations. Reviewers: Chia-Ping Tsai --- .../common/requests/LeaderAndIsrRequest.java | 25 +++++------ .../common/requests/LeaderAndIsrResponse.java | 6 +-- .../common/message/LeaderAndIsrResponse.json | 3 +- .../requests/LeaderAndIsrRequestTest.java | 41 ++++++++++++++---- .../requests/LeaderAndIsrResponseTest.java | 11 ++--- .../common/requests/RequestResponseTest.java | 3 +- .../scala/kafka/server/ReplicaManager.scala | 43 ++++++++----------- .../ControllerChannelManagerTest.scala | 12 +++--- 8 files changed, 80 insertions(+), 64 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java index 939212a16ea30..d738286317616 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java @@ -145,24 +145,21 @@ public LeaderAndIsrResponse getErrorResponse(int throttleTimeMs, Throwable e) { .setErrorCode(error.code())); } responseData.setPartitionErrors(partitions); - return new LeaderAndIsrResponse(responseData, version()); - } - - List topics = new ArrayList<>(data.topicStates().size()); - Map topicIds = topicIds(); - for (LeaderAndIsrTopicState topicState : data.topicStates()) { - LeaderAndIsrTopicError topicError = new LeaderAndIsrTopicError(); - topicError.setTopicId(topicIds.get(topicState.topicName())); - List partitions = new ArrayList<>(topicState.partitionStates().size()); - for (LeaderAndIsrPartitionState partition : topicState.partitionStates()) { - partitions.add(new LeaderAndIsrPartitionError() + } else { + for (LeaderAndIsrTopicState topicState : data.topicStates()) { + List partitions = new ArrayList<>( + topicState.partitionStates().size()); + for (LeaderAndIsrPartitionState partition : topicState.partitionStates()) { + partitions.add(new LeaderAndIsrPartitionError() .setPartitionIndex(partition.partitionIndex()) .setErrorCode(error.code())); + } + responseData.topics().add(new LeaderAndIsrTopicError() + .setTopicId(topicState.topicId()) + .setPartitionErrors(partitions)); } - topicError.setPartitionErrors(partitions); - topics.add(topicError); } - responseData.setTopics(topics); + return new LeaderAndIsrResponse(responseData, version()); } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrResponse.java index 490983f1b5d21..c7c04e2d99b41 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrResponse.java @@ -20,13 +20,13 @@ import org.apache.kafka.common.Uuid; import org.apache.kafka.common.message.LeaderAndIsrResponseData; import org.apache.kafka.common.message.LeaderAndIsrResponseData.LeaderAndIsrTopicError; +import org.apache.kafka.common.message.LeaderAndIsrResponseData.LeaderAndIsrTopicErrorCollection; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import java.nio.ByteBuffer; import java.util.Collections; -import java.util.List; import java.util.HashMap; import java.util.Map; @@ -39,7 +39,7 @@ public class LeaderAndIsrResponse extends AbstractResponse { * STALE_BROKER_EPOCH (77) */ private final LeaderAndIsrResponseData data; - private short version; + private final short version; public LeaderAndIsrResponse(LeaderAndIsrResponseData data, short version) { super(ApiKeys.LEADER_AND_ISR); @@ -47,7 +47,7 @@ public LeaderAndIsrResponse(LeaderAndIsrResponseData data, short version) { this.version = version; } - public List topics() { + public LeaderAndIsrTopicErrorCollection topics() { return this.data.topics(); } diff --git a/clients/src/main/resources/common/message/LeaderAndIsrResponse.json b/clients/src/main/resources/common/message/LeaderAndIsrResponse.json index dc5879b143671..958448be2744b 100644 --- a/clients/src/main/resources/common/message/LeaderAndIsrResponse.json +++ b/clients/src/main/resources/common/message/LeaderAndIsrResponse.json @@ -36,7 +36,8 @@ "about": "Each partition in v0 to v4 message."}, { "name": "Topics", "type": "[]LeaderAndIsrTopicError", "versions": "5+", "about": "Each topic", "fields": [ - { "name": "TopicId", "type": "uuid", "versions": "5+", "about": "The unique topic ID" }, + { "name": "TopicId", "type": "uuid", "versions": "5+", "mapKey": true, + "about": "The unique topic ID" }, { "name": "PartitionErrors", "type": "[]LeaderAndIsrPartitionError", "versions": "5+", "about": "Each partition."} ]} diff --git a/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrRequestTest.java index 9f636d7ef8be3..4fe51a0dccd3c 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrRequestTest.java @@ -24,6 +24,8 @@ import org.apache.kafka.common.message.LeaderAndIsrRequestData; import org.apache.kafka.common.message.LeaderAndIsrRequestData.LeaderAndIsrLiveLeader; import org.apache.kafka.common.message.LeaderAndIsrRequestData.LeaderAndIsrPartitionState; +import org.apache.kafka.common.message.LeaderAndIsrResponseData.LeaderAndIsrPartitionError; +import org.apache.kafka.common.message.LeaderAndIsrResponseData.LeaderAndIsrTopicError; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.test.TestUtils; @@ -59,18 +61,41 @@ public void testUnsupportedVersion() { @Test public void testGetErrorResponse() { - Uuid id = Uuid.randomUuid(); - for (short version = LEADER_AND_ISR.oldestVersion(); version < LEADER_AND_ISR.latestVersion(); version++) { - LeaderAndIsrRequest.Builder builder = new LeaderAndIsrRequest.Builder(version, 0, 0, 0, - Collections.emptyList(), Collections.singletonMap("topic", id), Collections.emptySet()); - LeaderAndIsrRequest request = builder.build(); + Uuid topicId = Uuid.randomUuid(); + String topicName = "topic"; + int partition = 0; + + for (short version = LEADER_AND_ISR.oldestVersion(); version <= LEADER_AND_ISR.latestVersion(); version++) { + LeaderAndIsrRequest request = new LeaderAndIsrRequest.Builder(version, 0, 0, 0, + Collections.singletonList(new LeaderAndIsrPartitionState() + .setTopicName(topicName) + .setPartitionIndex(partition)), + Collections.singletonMap(topicName, topicId), + Collections.emptySet() + ).build(version); + LeaderAndIsrResponse response = request.getErrorResponse(0, - new ClusterAuthorizationException("Not authorized")); + new ClusterAuthorizationException("Not authorized")); + assertEquals(Errors.CLUSTER_AUTHORIZATION_FAILED, response.error()); + if (version < 5) { - assertEquals(0, response.topics().size()); + assertEquals( + Collections.singletonList(new LeaderAndIsrPartitionError() + .setTopicName(topicName) + .setPartitionIndex(partition) + .setErrorCode(Errors.CLUSTER_AUTHORIZATION_FAILED.code())), + response.data().partitionErrors()); + assertEquals(0, response.data().topics().size()); } else { - assertEquals(id, response.topics().get(0).topicId()); + LeaderAndIsrTopicError topicState = response.topics().find(topicId); + assertEquals(topicId, topicState.topicId()); + assertEquals( + Collections.singletonList(new LeaderAndIsrPartitionError() + .setPartitionIndex(partition) + .setErrorCode(Errors.CLUSTER_AUTHORIZATION_FAILED.code())), + topicState.partitionErrors()); + assertEquals(0, response.data().partitionErrors().size()); } } } diff --git a/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrResponseTest.java index 3f6a22496ec2b..9ae2fdb4204fc 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrResponseTest.java @@ -21,6 +21,7 @@ import org.apache.kafka.common.message.LeaderAndIsrResponseData; import org.apache.kafka.common.message.LeaderAndIsrResponseData.LeaderAndIsrTopicError; import org.apache.kafka.common.message.LeaderAndIsrResponseData.LeaderAndIsrPartitionError; +import org.apache.kafka.common.message.LeaderAndIsrResponseData.LeaderAndIsrTopicErrorCollection; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; import org.junit.jupiter.api.Test; @@ -80,7 +81,7 @@ public void testErrorCountsWithTopLevelError() { .setPartitionErrors(partitions), version); } else { Uuid id = Uuid.randomUuid(); - List topics = createTopic(id, asList(Errors.NONE, Errors.NOT_LEADER_OR_FOLLOWER)); + LeaderAndIsrTopicErrorCollection topics = createTopic(id, asList(Errors.NONE, Errors.NOT_LEADER_OR_FOLLOWER)); response = new LeaderAndIsrResponse(new LeaderAndIsrResponseData() .setErrorCode(Errors.UNKNOWN_SERVER_ERROR.code()) .setTopics(topics), version); @@ -101,7 +102,7 @@ public void testErrorCountsNoTopLevelError() { .setPartitionErrors(partitions), version); } else { Uuid id = Uuid.randomUuid(); - List topics = createTopic(id, asList(Errors.NONE, Errors.CLUSTER_AUTHORIZATION_FAILED)); + LeaderAndIsrTopicErrorCollection topics = createTopic(id, asList(Errors.NONE, Errors.CLUSTER_AUTHORIZATION_FAILED)); response = new LeaderAndIsrResponse(new LeaderAndIsrResponseData() .setErrorCode(Errors.NONE.code()) .setTopics(topics), version); @@ -130,7 +131,7 @@ public void testToString() { } else { Uuid id = Uuid.randomUuid(); - List topics = createTopic(id, asList(Errors.NONE, Errors.CLUSTER_AUTHORIZATION_FAILED)); + LeaderAndIsrTopicErrorCollection topics = createTopic(id, asList(Errors.NONE, Errors.CLUSTER_AUTHORIZATION_FAILED)); response = new LeaderAndIsrResponse(new LeaderAndIsrResponseData() .setErrorCode(Errors.NONE.code()) .setTopics(topics), version); @@ -155,8 +156,8 @@ private List createPartitions(String topicName, List return partitions; } - private List createTopic(Uuid id, List errors) { - List topics = new ArrayList<>(); + private LeaderAndIsrTopicErrorCollection createTopic(Uuid id, List errors) { + LeaderAndIsrTopicErrorCollection topics = new LeaderAndIsrTopicErrorCollection(); LeaderAndIsrTopicError topic = new LeaderAndIsrTopicError(); topic.setTopicId(id); List partitions = new ArrayList<>(); diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java index d873d15207374..228f32afcea94 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java @@ -126,6 +126,7 @@ import org.apache.kafka.common.message.JoinGroupResponseData.JoinGroupResponseMember; import org.apache.kafka.common.message.LeaderAndIsrRequestData.LeaderAndIsrPartitionState; import org.apache.kafka.common.message.LeaderAndIsrResponseData; +import org.apache.kafka.common.message.LeaderAndIsrResponseData.LeaderAndIsrTopicErrorCollection; import org.apache.kafka.common.message.LeaveGroupRequestData.MemberIdentity; import org.apache.kafka.common.message.LeaveGroupResponseData; import org.apache.kafka.common.message.ListGroupsRequestData; @@ -1685,7 +1686,7 @@ private LeaderAndIsrResponse createLeaderAndIsrResponse(int version) { new LeaderAndIsrResponseData.LeaderAndIsrPartitionError() .setPartitionIndex(0) .setErrorCode(Errors.NONE.code())); - List topics = new ArrayList<>(); + LeaderAndIsrTopicErrorCollection topics = new LeaderAndIsrTopicErrorCollection(); topics.add(new LeaderAndIsrResponseData.LeaderAndIsrTopicError() .setTopicId(Uuid.randomUuid()) .setPartitionErrors(partition)); diff --git a/core/src/main/scala/kafka/server/ReplicaManager.scala b/core/src/main/scala/kafka/server/ReplicaManager.scala index 820af7b6ece2c..87b446208536c 100644 --- a/core/src/main/scala/kafka/server/ReplicaManager.scala +++ b/core/src/main/scala/kafka/server/ReplicaManager.scala @@ -1441,38 +1441,29 @@ class ReplicaManager(val config: KafkaConfig, replicaFetcherManager.shutdownIdleFetcherThreads() replicaAlterLogDirsManager.shutdownIdleFetcherThreads() onLeadershipChange(partitionsBecomeLeader, partitionsBecomeFollower) - if (leaderAndIsrRequest.version() < 5) { - val responsePartitions = responseMap.iterator.map { case (tp, error) => - new LeaderAndIsrPartitionError() + + val data = new LeaderAndIsrResponseData().setErrorCode(Errors.NONE.code) + if (leaderAndIsrRequest.version < 5) { + responseMap.forKeyValue { (tp, error) => + data.partitionErrors.add(new LeaderAndIsrPartitionError() .setTopicName(tp.topic) .setPartitionIndex(tp.partition) - .setErrorCode(error.code) - }.toBuffer - new LeaderAndIsrResponse(new LeaderAndIsrResponseData() - .setErrorCode(Errors.NONE.code) - .setPartitionErrors(responsePartitions.asJava), leaderAndIsrRequest.version()) + .setErrorCode(error.code)) + } } else { - val topics = new mutable.HashMap[String, List[LeaderAndIsrPartitionError]] - responseMap.asJava.forEach { case (tp, error) => - if (!topics.contains(tp.topic)) { - topics.put(tp.topic, List(new LeaderAndIsrPartitionError() - .setPartitionIndex(tp.partition) - .setErrorCode(error.code))) - } else { - topics.put(tp.topic, new LeaderAndIsrPartitionError() - .setPartitionIndex(tp.partition) - .setErrorCode(error.code)::topics(tp.topic)) + responseMap.forKeyValue { (tp, error) => + val topicId = topicIds.get(tp.topic) + var topic = data.topics.find(topicId) + if (topic == null) { + topic = new LeaderAndIsrTopicError().setTopicId(topicId) + data.topics.add(topic) } + topic.partitionErrors.add(new LeaderAndIsrPartitionError() + .setPartitionIndex(tp.partition) + .setErrorCode(error.code)) } - val topicErrors = topics.iterator.map { case (topic, partitionError) => - new LeaderAndIsrTopicError() - .setTopicId(topicIds.get(topic)) - .setPartitionErrors(partitionError.asJava) - }.toBuffer - new LeaderAndIsrResponse(new LeaderAndIsrResponseData() - .setErrorCode(Errors.NONE.code) - .setTopics(topicErrors.asJava), leaderAndIsrRequest.version()) } + new LeaderAndIsrResponse(data, leaderAndIsrRequest.version) } } val endMs = time.milliseconds() diff --git a/core/src/test/scala/unit/kafka/controller/ControllerChannelManagerTest.scala b/core/src/test/scala/unit/kafka/controller/ControllerChannelManagerTest.scala index cb494e62338d0..8a816865b45f4 100644 --- a/core/src/test/scala/unit/kafka/controller/ControllerChannelManagerTest.scala +++ b/core/src/test/scala/unit/kafka/controller/ControllerChannelManagerTest.scala @@ -847,17 +847,17 @@ class ControllerChannelManagerTest { sentRequests.filter(_.request.apiKey == ApiKeys.LEADER_AND_ISR).filter(_.responseCallback != null).foreach { sentRequest => val leaderAndIsrRequest = sentRequest.request.build().asInstanceOf[LeaderAndIsrRequest] val topicIds = leaderAndIsrRequest.topicIds - val topicErrors = leaderAndIsrRequest.data.topicStates.asScala.map(t => - new LeaderAndIsrTopicError() + val data = new LeaderAndIsrResponseData() + .setErrorCode(error.code) + leaderAndIsrRequest.data.topicStates.asScala.foreach { t => + data.topics.add(new LeaderAndIsrTopicError() .setTopicId(topicIds.get(t.topicName)) .setPartitionErrors(t.partitionStates.asScala.map(p => new LeaderAndIsrPartitionError() .setPartitionIndex(p.partitionIndex) .setErrorCode(error.code)).asJava)) - val leaderAndIsrResponse = new LeaderAndIsrResponse( - new LeaderAndIsrResponseData() - .setErrorCode(error.code) - .setTopics(topicErrors.toBuffer.asJava), leaderAndIsrRequest.version()) + } + val leaderAndIsrResponse = new LeaderAndIsrResponse(data, leaderAndIsrRequest.version) sentRequest.responseCallback(leaderAndIsrResponse) } } From 8205051e90e3ea16165f8dc1f5c81af744bb1b9a Mon Sep 17 00:00:00 2001 From: Chia-Ping Tsai Date: Thu, 4 Mar 2021 18:06:50 +0800 Subject: [PATCH 08/59] =?UTF-8?q?MINOR:=20remove=20FetchResponse.AbortedTr?= =?UTF-8?q?ansaction=20and=20redundant=20construc=E2=80=A6=20(#9758)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. rename INVALID_HIGHWATERMARK to INVALID_HIGH_WATERMARK 2. replace FetchResponse.AbortedTransaction by FetchResponseData.AbortedTransaction 3. remove redundant constructors from FetchResponse.PartitionData 4. rename recordSet to records 5. add helpers "recordsOrFail" and "recordsSize" to FetchResponse to process record casting Reviewers: Ismael Juma --- .../kafka/clients/FetchSessionHandler.java | 8 +- .../clients/consumer/internals/Fetcher.java | 48 +-- .../kafka/common/requests/FetchRequest.java | 11 +- .../kafka/common/requests/FetchResponse.java | 380 ++++++------------ .../common/message/FetchResponse.json | 6 +- .../clients/FetchSessionHandlerTest.java | 65 ++- .../clients/consumer/KafkaConsumerTest.java | 27 +- .../consumer/internals/FetcherTest.java | 301 +++++++++----- .../common/requests/RequestResponseTest.java | 114 ++++-- core/src/main/scala/kafka/log/Log.scala | 5 +- .../scala/kafka/log/TransactionIndex.scala | 7 +- .../scala/kafka/network/RequestChannel.scala | 2 +- .../kafka/network/RequestConvertToJson.scala | 2 +- .../kafka/server/AbstractFetcherThread.scala | 51 ++- .../scala/kafka/server/ControllerApis.scala | 3 +- .../scala/kafka/server/FetchDataInfo.scala | 4 +- .../scala/kafka/server/FetchSession.scala | 64 +-- .../main/scala/kafka/server/KafkaApis.scala | 123 +++--- .../server/ReplicaAlterLogDirsThread.scala | 32 +- .../kafka/server/ReplicaFetcherThread.scala | 6 +- .../scala/kafka/server/ReplicaManager.scala | 3 +- .../kafka/tools/ReplicaVerificationTool.scala | 45 +-- .../kafka/tools/TestRaftRequestHandler.scala | 3 +- .../kafka/api/AuthorizerIntegrationTest.scala | 44 +- .../tools/ReplicaVerificationToolTest.scala | 10 +- .../test/scala/unit/kafka/log/LogTest.scala | 5 +- .../unit/kafka/log/TransactionIndexTest.scala | 4 +- .../server/AbstractFetcherThreadTest.scala | 22 +- ...FetchRequestDownConversionConfigTest.scala | 23 +- .../server/FetchRequestMaxBytesTest.scala | 9 +- .../unit/kafka/server/FetchRequestTest.scala | 112 +++--- .../unit/kafka/server/FetchSessionTest.scala | 326 ++++++++++----- .../unit/kafka/server/KafkaApisTest.scala | 58 +-- .../unit/kafka/server/LogOffsetTest.scala | 22 +- .../server/ReplicaFetcherThreadTest.scala | 40 +- .../kafka/server/ReplicaManagerTest.scala | 31 +- .../kafka/server/UpdateFeaturesTest.scala | 2 +- .../util/ReplicaFetcherMockBlockingSend.scala | 16 +- .../jmh/common/FetchResponseBenchmark.java | 18 +- .../ReplicaFetcherThreadBenchmark.java | 19 +- .../fetchsession/FetchSessionBenchmark.java | 17 +- .../apache/kafka/raft/KafkaRaftClient.java | 15 +- .../java/org/apache/kafka/raft/RaftUtil.java | 14 +- .../raft/KafkaRaftClientSnapshotTest.java | 12 +- .../kafka/raft/RaftClientTestContext.java | 20 +- 45 files changed, 1102 insertions(+), 1047 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/FetchSessionHandler.java b/clients/src/main/java/org/apache/kafka/clients/FetchSessionHandler.java index 50252698c456a..b516e01923c58 100644 --- a/clients/src/main/java/org/apache/kafka/clients/FetchSessionHandler.java +++ b/clients/src/main/java/org/apache/kafka/clients/FetchSessionHandler.java @@ -318,7 +318,7 @@ static Set findMissing(Set toFind, Set response) { + String verifyFullFetchResponsePartitions(FetchResponse response) { StringBuilder bld = new StringBuilder(); Set extra = findMissing(response.responseData().keySet(), sessionPartitions.keySet()); @@ -343,7 +343,7 @@ String verifyFullFetchResponsePartitions(FetchResponse response) { * @param response The response. * @return True if the incremental fetch response partitions are valid. */ - String verifyIncrementalFetchResponsePartitions(FetchResponse response) { + String verifyIncrementalFetchResponsePartitions(FetchResponse response) { Set extra = findMissing(response.responseData().keySet(), sessionPartitions.keySet()); if (!extra.isEmpty()) { @@ -362,7 +362,7 @@ String verifyIncrementalFetchResponsePartitions(FetchResponse response) { * @param response The FetchResponse. * @return The string to log. */ - private String responseDataToLogString(FetchResponse response) { + private String responseDataToLogString(FetchResponse response) { if (!log.isTraceEnabled()) { int implied = sessionPartitions.size() - response.responseData().size(); if (implied > 0) { @@ -398,7 +398,7 @@ private String responseDataToLogString(FetchResponse response) { * @return True if the response is well-formed; false if it can't be processed * because of missing or unexpected partitions. */ - public boolean handleResponse(FetchResponse response) { + public boolean handleResponse(FetchResponse response) { if (response.error() != Errors.NONE) { log.info("Node {} was unable to process the fetch request with {}: {}.", node, nextMetadata, response.error()); diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java index 5e8ad3c2454e2..257cff6b206bf 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java @@ -47,6 +47,7 @@ import org.apache.kafka.common.errors.TopicAuthorizationException; import org.apache.kafka.common.header.Headers; import org.apache.kafka.common.header.internals.RecordHeaders; +import org.apache.kafka.common.message.FetchResponseData; import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersion; import org.apache.kafka.common.message.ListOffsetsRequestData.ListOffsetsPartition; import org.apache.kafka.common.message.ListOffsetsResponseData.ListOffsetsPartitionResponse; @@ -66,7 +67,6 @@ import org.apache.kafka.common.record.ControlRecordType; import org.apache.kafka.common.record.Record; import org.apache.kafka.common.record.RecordBatch; -import org.apache.kafka.common.record.Records; import org.apache.kafka.common.record.TimestampType; import org.apache.kafka.common.requests.FetchRequest; import org.apache.kafka.common.requests.FetchResponse; @@ -277,7 +277,7 @@ public void onSuccess(ClientResponse resp) { synchronized (Fetcher.this) { try { @SuppressWarnings("unchecked") - FetchResponse response = (FetchResponse) resp.responseBody(); + FetchResponse response = (FetchResponse) resp.responseBody(); FetchSessionHandler handler = sessionHandler(fetchTarget.id()); if (handler == null) { log.error("Unable to find FetchSessionHandler for node {}. Ignoring fetch response.", @@ -291,7 +291,7 @@ public void onSuccess(ClientResponse resp) { Set partitions = new HashSet<>(response.responseData().keySet()); FetchResponseMetricAggregator metricAggregator = new FetchResponseMetricAggregator(sensors, partitions); - for (Map.Entry> entry : response.responseData().entrySet()) { + for (Map.Entry entry : response.responseData().entrySet()) { TopicPartition partition = entry.getKey(); FetchRequest.PartitionData requestData = data.sessionPartitions().get(partition); if (requestData == null) { @@ -310,12 +310,12 @@ public void onSuccess(ClientResponse resp) { throw new IllegalStateException(message); } else { long fetchOffset = requestData.fetchOffset; - FetchResponse.PartitionData partitionData = entry.getValue(); + FetchResponseData.PartitionData partitionData = entry.getValue(); log.debug("Fetch {} at offset {} for partition {} returned fetch data {}", isolationLevel, fetchOffset, partition, partitionData); - Iterator batches = partitionData.records().batches().iterator(); + Iterator batches = FetchResponse.recordsOrFail(partitionData).batches().iterator(); short responseVersion = resp.requestHeader().apiVersion(); completedFetches.add(new CompletedFetch(partition, partitionData, @@ -618,8 +618,8 @@ public Map>> fetchedRecords() { // The first condition ensures that the completedFetches is not stuck with the same completedFetch // in cases such as the TopicAuthorizationException, and the second condition ensures that no // potential data loss due to an exception in a following record. - FetchResponse.PartitionData partition = records.partitionData; - if (fetched.isEmpty() && (partition.records() == null || partition.records().sizeInBytes() == 0)) { + FetchResponseData.PartitionData partition = records.partitionData; + if (fetched.isEmpty() && FetchResponse.recordsOrFail(partition).sizeInBytes() == 0) { completedFetches.poll(); } throw e; @@ -1229,10 +1229,10 @@ private Map> regroupPartitionMapByNode(Map partition = nextCompletedFetch.partitionData; + FetchResponseData.PartitionData partition = nextCompletedFetch.partitionData; long fetchOffset = nextCompletedFetch.nextFetchOffset; CompletedFetch completedFetch = null; - Errors error = partition.error(); + Errors error = Errors.forCode(partition.errorCode()); try { if (!subscriptions.hasValidPosition(tp)) { @@ -1249,11 +1249,11 @@ private CompletedFetch initializeCompletedFetch(CompletedFetch nextCompletedFetc } log.trace("Preparing to read {} bytes of data for partition {} with offset {}", - partition.records().sizeInBytes(), tp, position); - Iterator batches = partition.records().batches().iterator(); + FetchResponse.recordsSize(partition), tp, position); + Iterator batches = FetchResponse.recordsOrFail(partition).batches().iterator(); completedFetch = nextCompletedFetch; - if (!batches.hasNext() && partition.records().sizeInBytes() > 0) { + if (!batches.hasNext() && FetchResponse.recordsSize(partition) > 0) { if (completedFetch.responseVersion < 3) { // Implement the pre KIP-74 behavior of throwing a RecordTooLargeException. Map recordTooLargePartitions = Collections.singletonMap(tp, fetchOffset); @@ -1286,11 +1286,11 @@ private CompletedFetch initializeCompletedFetch(CompletedFetch nextCompletedFetc subscriptions.updateLastStableOffset(tp, partition.lastStableOffset()); } - if (partition.preferredReadReplica().isPresent()) { - subscriptions.updatePreferredReadReplica(completedFetch.partition, partition.preferredReadReplica().get(), () -> { + if (FetchResponse.isPreferredReplica(partition)) { + subscriptions.updatePreferredReadReplica(completedFetch.partition, partition.preferredReadReplica(), () -> { long expireTimeMs = time.milliseconds() + metadata.metadataExpireMs(); log.debug("Updating preferred read replica for partition {} to {}, set to expire at {}", - tp, partition.preferredReadReplica().get(), expireTimeMs); + tp, partition.preferredReadReplica(), expireTimeMs); return expireTimeMs; }); } @@ -1455,8 +1455,8 @@ private class CompletedFetch { private final TopicPartition partition; private final Iterator batches; private final Set abortedProducerIds; - private final PriorityQueue abortedTransactions; - private final FetchResponse.PartitionData partitionData; + private final PriorityQueue abortedTransactions; + private final FetchResponseData.PartitionData partitionData; private final FetchResponseMetricAggregator metricAggregator; private final short responseVersion; @@ -1473,7 +1473,7 @@ private class CompletedFetch { private boolean initialized = false; private CompletedFetch(TopicPartition partition, - FetchResponse.PartitionData partitionData, + FetchResponseData.PartitionData partitionData, FetchResponseMetricAggregator metricAggregator, Iterator batches, Long fetchOffset, @@ -1641,9 +1641,9 @@ private void consumeAbortedTransactionsUpTo(long offset) { if (abortedTransactions == null) return; - while (!abortedTransactions.isEmpty() && abortedTransactions.peek().firstOffset <= offset) { - FetchResponse.AbortedTransaction abortedTransaction = abortedTransactions.poll(); - abortedProducerIds.add(abortedTransaction.producerId); + while (!abortedTransactions.isEmpty() && abortedTransactions.peek().firstOffset() <= offset) { + FetchResponseData.AbortedTransaction abortedTransaction = abortedTransactions.poll(); + abortedProducerIds.add(abortedTransaction.producerId()); } } @@ -1651,12 +1651,12 @@ private boolean isBatchAborted(RecordBatch batch) { return batch.isTransactional() && abortedProducerIds.contains(batch.producerId()); } - private PriorityQueue abortedTransactions(FetchResponse.PartitionData partition) { + private PriorityQueue abortedTransactions(FetchResponseData.PartitionData partition) { if (partition.abortedTransactions() == null || partition.abortedTransactions().isEmpty()) return null; - PriorityQueue abortedTransactions = new PriorityQueue<>( - partition.abortedTransactions().size(), Comparator.comparingLong(o -> o.firstOffset) + PriorityQueue abortedTransactions = new PriorityQueue<>( + partition.abortedTransactions().size(), Comparator.comparingLong(FetchResponseData.AbortedTransaction::firstOffset) ); abortedTransactions.addAll(partition.abortedTransactions()); return abortedTransactions; diff --git a/clients/src/main/java/org/apache/kafka/common/requests/FetchRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/FetchRequest.java index 591c6e19a1977..05caaa405833f 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/FetchRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/FetchRequest.java @@ -19,10 +19,10 @@ import org.apache.kafka.common.IsolationLevel; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.message.FetchRequestData; +import org.apache.kafka.common.message.FetchResponseData; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.record.MemoryRecords; import org.apache.kafka.common.record.RecordBatch; import org.apache.kafka.common.utils.Utils; @@ -296,14 +296,11 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { // may not be any partitions at all in the response. For this reason, the top-level error code // is essential for them. Errors error = Errors.forException(e); - LinkedHashMap> responseData = new LinkedHashMap<>(); + LinkedHashMap responseData = new LinkedHashMap<>(); for (Map.Entry entry : fetchData.entrySet()) { - FetchResponse.PartitionData partitionResponse = new FetchResponse.PartitionData<>(error, - FetchResponse.INVALID_HIGHWATERMARK, FetchResponse.INVALID_LAST_STABLE_OFFSET, - FetchResponse.INVALID_LOG_START_OFFSET, Optional.empty(), null, MemoryRecords.EMPTY); - responseData.put(entry.getKey(), partitionResponse); + responseData.put(entry.getKey(), FetchResponse.partitionResponse(entry.getKey().partition(), error)); } - return new FetchResponse<>(error, responseData, throttleTimeMs, data.sessionId()); + return FetchResponse.of(error, throttleTimeMs, data.sessionId(), responseData); } public int replicaId() { diff --git a/clients/src/main/java/org/apache/kafka/common/requests/FetchResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/FetchResponse.java index 4e8dce5f32031..5b1f1f7e80eb8 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/FetchResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/FetchResponse.java @@ -22,8 +22,8 @@ import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.ObjectSerializationCache; -import org.apache.kafka.common.record.BaseRecords; import org.apache.kafka.common.record.MemoryRecords; +import org.apache.kafka.common.record.Records; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -33,7 +33,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.stream.Collectors; import static org.apache.kafka.common.requests.FetchMetadata.INVALID_SESSION_ID; @@ -57,238 +56,43 @@ * the fetch offset after the index lookup * - {@link Errors#UNKNOWN_SERVER_ERROR} For any unexpected errors */ -public class FetchResponse extends AbstractResponse { - - public static final long INVALID_HIGHWATERMARK = -1L; +public class FetchResponse extends AbstractResponse { + public static final long INVALID_HIGH_WATERMARK = -1L; public static final long INVALID_LAST_STABLE_OFFSET = -1L; public static final long INVALID_LOG_START_OFFSET = -1L; public static final int INVALID_PREFERRED_REPLICA_ID = -1; private final FetchResponseData data; - private final LinkedHashMap> responseDataMap; + // we build responseData when needed. + private volatile LinkedHashMap responseData = null; @Override public FetchResponseData data() { return data; } - public static final class AbortedTransaction { - public final long producerId; - public final long firstOffset; - - public AbortedTransaction(long producerId, long firstOffset) { - this.producerId = producerId; - this.firstOffset = firstOffset; - } - - @Override - public boolean equals(Object o) { - if (this == o) - return true; - if (o == null || getClass() != o.getClass()) - return false; - - AbortedTransaction that = (AbortedTransaction) o; - - return producerId == that.producerId && firstOffset == that.firstOffset; - } - - @Override - public int hashCode() { - int result = Long.hashCode(producerId); - result = 31 * result + Long.hashCode(firstOffset); - return result; - } - - @Override - public String toString() { - return "(producerId=" + producerId + ", firstOffset=" + firstOffset + ")"; - } - - static AbortedTransaction fromMessage(FetchResponseData.AbortedTransaction abortedTransaction) { - return new AbortedTransaction(abortedTransaction.producerId(), abortedTransaction.firstOffset()); - } - } - - public static final class PartitionData { - private final FetchResponseData.FetchablePartitionResponse partitionResponse; - - // Derived fields - private final Optional preferredReplica; - private final List abortedTransactions; - private final Errors error; - - private PartitionData(FetchResponseData.FetchablePartitionResponse partitionResponse) { - // We partially construct FetchablePartitionResponse since we don't know the partition ID at this point - // When we convert the PartitionData (and other fields) into FetchResponseData down in toMessage, we - // set the partition IDs. - this.partitionResponse = partitionResponse; - this.preferredReplica = Optional.of(partitionResponse.preferredReadReplica()) - .filter(replicaId -> replicaId != INVALID_PREFERRED_REPLICA_ID); - - if (partitionResponse.abortedTransactions() == null) { - this.abortedTransactions = null; - } else { - this.abortedTransactions = partitionResponse.abortedTransactions().stream() - .map(AbortedTransaction::fromMessage) - .collect(Collectors.toList()); - } - - this.error = Errors.forCode(partitionResponse.errorCode()); - } - - public PartitionData(Errors error, - long highWatermark, - long lastStableOffset, - long logStartOffset, - Optional preferredReadReplica, - List abortedTransactions, - Optional divergingEpoch, - T records) { - this.preferredReplica = preferredReadReplica; - this.abortedTransactions = abortedTransactions; - this.error = error; - - FetchResponseData.FetchablePartitionResponse partitionResponse = - new FetchResponseData.FetchablePartitionResponse(); - partitionResponse.setErrorCode(error.code()) - .setHighWatermark(highWatermark) - .setLastStableOffset(lastStableOffset) - .setLogStartOffset(logStartOffset); - if (abortedTransactions != null) { - partitionResponse.setAbortedTransactions(abortedTransactions.stream().map( - aborted -> new FetchResponseData.AbortedTransaction() - .setProducerId(aborted.producerId) - .setFirstOffset(aborted.firstOffset)) - .collect(Collectors.toList())); - } else { - partitionResponse.setAbortedTransactions(null); - } - partitionResponse.setPreferredReadReplica(preferredReadReplica.orElse(INVALID_PREFERRED_REPLICA_ID)); - partitionResponse.setRecordSet(records); - divergingEpoch.ifPresent(partitionResponse::setDivergingEpoch); - - this.partitionResponse = partitionResponse; - } - - public PartitionData(Errors error, - long highWatermark, - long lastStableOffset, - long logStartOffset, - Optional preferredReadReplica, - List abortedTransactions, - T records) { - this(error, highWatermark, lastStableOffset, logStartOffset, preferredReadReplica, - abortedTransactions, Optional.empty(), records); - } - - public PartitionData(Errors error, - long highWatermark, - long lastStableOffset, - long logStartOffset, - List abortedTransactions, - T records) { - this(error, highWatermark, lastStableOffset, logStartOffset, Optional.empty(), abortedTransactions, records); - } - - @Override - public boolean equals(Object o) { - if (this == o) - return true; - if (o == null || getClass() != o.getClass()) - return false; - - PartitionData that = (PartitionData) o; - - return this.partitionResponse.equals(that.partitionResponse); - } - - @Override - public int hashCode() { - return this.partitionResponse.hashCode(); - } - - @Override - public String toString() { - return "(error=" + error() + - ", highWaterMark=" + highWatermark() + - ", lastStableOffset = " + lastStableOffset() + - ", logStartOffset = " + logStartOffset() + - ", preferredReadReplica = " + preferredReadReplica().map(Object::toString).orElse("absent") + - ", abortedTransactions = " + abortedTransactions() + - ", divergingEpoch =" + divergingEpoch() + - ", recordsSizeInBytes=" + records().sizeInBytes() + ")"; - } - - public Errors error() { - return error; - } - - public long highWatermark() { - return partitionResponse.highWatermark(); - } - - public long lastStableOffset() { - return partitionResponse.lastStableOffset(); - } - - public long logStartOffset() { - return partitionResponse.logStartOffset(); - } - - public Optional preferredReadReplica() { - return preferredReplica; - } - - public List abortedTransactions() { - return abortedTransactions; - } - - public Optional divergingEpoch() { - FetchResponseData.EpochEndOffset epochEndOffset = partitionResponse.divergingEpoch(); - if (epochEndOffset.epoch() < 0) { - return Optional.empty(); - } else { - return Optional.of(epochEndOffset); - } - } - - @SuppressWarnings("unchecked") - public T records() { - return (T) partitionResponse.recordSet(); - } - } - - /** - * From version 3 or later, the entries in `responseData` should be in the same order as the entries in - * `FetchRequest.fetchData`. - * - * @param error The top-level error code. - * @param responseData The fetched data grouped by partition. - * @param throttleTimeMs The time in milliseconds that the response was throttled - * @param sessionId The fetch session id. - */ - public FetchResponse(Errors error, - LinkedHashMap> responseData, - int throttleTimeMs, - int sessionId) { - super(ApiKeys.FETCH); - this.data = toMessage(throttleTimeMs, error, responseData.entrySet().iterator(), sessionId); - this.responseDataMap = responseData; - } - public FetchResponse(FetchResponseData fetchResponseData) { super(ApiKeys.FETCH); this.data = fetchResponseData; - this.responseDataMap = toResponseDataMap(fetchResponseData); } public Errors error() { return Errors.forCode(data.errorCode()); } - public LinkedHashMap> responseData() { - return responseDataMap; + public LinkedHashMap responseData() { + if (responseData == null) { + synchronized (this) { + if (responseData == null) { + responseData = new LinkedHashMap<>(); + data.responses().forEach(topicResponse -> + topicResponse.partitions().forEach(partition -> + responseData.put(new TopicPartition(topicResponse.topic(), partition.partitionIndex()), partition)) + ); + } + } + } + return responseData; } @Override @@ -304,58 +108,15 @@ public int sessionId() { public Map errorCounts() { Map errorCounts = new HashMap<>(); updateErrorCounts(errorCounts, error()); - responseDataMap.values().forEach(response -> - updateErrorCounts(errorCounts, response.error()) + data.responses().forEach(topicResponse -> + topicResponse.partitions().forEach(partition -> + updateErrorCounts(errorCounts, Errors.forCode(partition.errorCode()))) ); return errorCounts; } - public static FetchResponse parse(ByteBuffer buffer, short version) { - return new FetchResponse<>(new FetchResponseData(new ByteBufferAccessor(buffer), version)); - } - - @SuppressWarnings("unchecked") - private static LinkedHashMap> toResponseDataMap( - FetchResponseData message) { - LinkedHashMap> responseMap = new LinkedHashMap<>(); - message.responses().forEach(topicResponse -> { - topicResponse.partitionResponses().forEach(partitionResponse -> { - TopicPartition tp = new TopicPartition(topicResponse.topic(), partitionResponse.partition()); - PartitionData partitionData = new PartitionData<>(partitionResponse); - responseMap.put(tp, partitionData); - }); - }); - return responseMap; - } - - private static FetchResponseData toMessage(int throttleTimeMs, Errors error, - Iterator>> partIterator, - int sessionId) { - List topicResponseList = new ArrayList<>(); - partIterator.forEachRemaining(entry -> { - PartitionData partitionData = entry.getValue(); - // Since PartitionData alone doesn't know the partition ID, we set it here - partitionData.partitionResponse.setPartition(entry.getKey().partition()); - // We have to keep the order of input topic-partition. Hence, we batch the partitions only if the last - // batch is in the same topic group. - FetchResponseData.FetchableTopicResponse previousTopic = topicResponseList.isEmpty() ? null - : topicResponseList.get(topicResponseList.size() - 1); - if (previousTopic != null && previousTopic.topic().equals(entry.getKey().topic())) - previousTopic.partitionResponses().add(partitionData.partitionResponse); - else { - List partitionResponses = new ArrayList<>(); - partitionResponses.add(partitionData.partitionResponse); - topicResponseList.add(new FetchResponseData.FetchableTopicResponse() - .setTopic(entry.getKey().topic()) - .setPartitionResponses(partitionResponses)); - } - }); - - return new FetchResponseData() - .setThrottleTimeMs(throttleTimeMs) - .setErrorCode(error.code()) - .setSessionId(sessionId) - .setResponses(topicResponseList); + public static FetchResponse parse(ByteBuffer buffer, short version) { + return new FetchResponse(new FetchResponseData(new ByteBufferAccessor(buffer), version)); } /** @@ -365,11 +126,11 @@ private static FetchResponseData toMessage(int throttleT * @param partIterator The partition iterator. * @return The response size in bytes. */ - public static int sizeOf(short version, - Iterator>> partIterator) { + public static int sizeOf(short version, + Iterator> partIterator) { // Since the throttleTimeMs and metadata field sizes are constant and fixed, we can // use arbitrary values here without affecting the result. - FetchResponseData data = toMessage(0, Errors.NONE, partIterator, INVALID_SESSION_ID); + FetchResponseData data = toMessage(Errors.NONE, 0, INVALID_SESSION_ID, partIterator); ObjectSerializationCache cache = new ObjectSerializationCache(); return 4 + data.size(cache, version); } @@ -378,4 +139,91 @@ public static int sizeOf(short version, public boolean shouldClientThrottle(short version) { return version >= 8; } -} + + public static Optional divergingEpoch(FetchResponseData.PartitionData partitionResponse) { + return partitionResponse.divergingEpoch().epoch() < 0 ? Optional.empty() + : Optional.of(partitionResponse.divergingEpoch()); + } + + public static boolean isDivergingEpoch(FetchResponseData.PartitionData partitionResponse) { + return partitionResponse.divergingEpoch().epoch() >= 0; + } + + public static Optional preferredReadReplica(FetchResponseData.PartitionData partitionResponse) { + return partitionResponse.preferredReadReplica() == INVALID_PREFERRED_REPLICA_ID ? Optional.empty() + : Optional.of(partitionResponse.preferredReadReplica()); + } + + public static boolean isPreferredReplica(FetchResponseData.PartitionData partitionResponse) { + return partitionResponse.preferredReadReplica() != INVALID_PREFERRED_REPLICA_ID; + } + + public static FetchResponseData.PartitionData partitionResponse(int partition, Errors error) { + return new FetchResponseData.PartitionData() + .setPartitionIndex(partition) + .setErrorCode(error.code()) + .setHighWatermark(FetchResponse.INVALID_HIGH_WATERMARK); + } + + /** + * Returns `partition.records` as `Records` (instead of `BaseRecords`). If `records` is `null`, returns `MemoryRecords.EMPTY`. + * + * If this response was deserialized after a fetch, this method should never fail. An example where this would + * fail is a down-converted response (e.g. LazyDownConversionRecords) on the broker (before it's serialized and + * sent on the wire). + * + * @param partition partition data + * @return Records or empty record if the records in PartitionData is null. + */ + public static Records recordsOrFail(FetchResponseData.PartitionData partition) { + if (partition.records() == null) return MemoryRecords.EMPTY; + if (partition.records() instanceof Records) return (Records) partition.records(); + throw new ClassCastException("The record type is " + partition.records().getClass().getSimpleName() + ", which is not a subtype of " + + Records.class.getSimpleName() + ". This method is only safe to call if the `FetchResponse` was deserialized from bytes."); + } + + /** + * @return The size in bytes of the records. 0 is returned if records of input partition is null. + */ + public static int recordsSize(FetchResponseData.PartitionData partition) { + return partition.records() == null ? 0 : partition.records().sizeInBytes(); + } + + public static FetchResponse of(Errors error, + int throttleTimeMs, + int sessionId, + LinkedHashMap responseData) { + return new FetchResponse(toMessage(error, throttleTimeMs, sessionId, responseData.entrySet().iterator())); + } + + private static FetchResponseData toMessage(Errors error, + int throttleTimeMs, + int sessionId, + Iterator> partIterator) { + List topicResponseList = new ArrayList<>(); + partIterator.forEachRemaining(entry -> { + FetchResponseData.PartitionData partitionData = entry.getValue(); + // Since PartitionData alone doesn't know the partition ID, we set it here + partitionData.setPartitionIndex(entry.getKey().partition()); + // We have to keep the order of input topic-partition. Hence, we batch the partitions only if the last + // batch is in the same topic group. + FetchResponseData.FetchableTopicResponse previousTopic = topicResponseList.isEmpty() ? null + : topicResponseList.get(topicResponseList.size() - 1); + if (previousTopic != null && previousTopic.topic().equals(entry.getKey().topic())) + previousTopic.partitions().add(partitionData); + else { + List partitionResponses = new ArrayList<>(); + partitionResponses.add(partitionData); + topicResponseList.add(new FetchResponseData.FetchableTopicResponse() + .setTopic(entry.getKey().topic()) + .setPartitions(partitionResponses)); + } + }); + + return new FetchResponseData() + .setThrottleTimeMs(throttleTimeMs) + .setErrorCode(error.code()) + .setSessionId(sessionId) + .setResponses(topicResponseList); + } +} \ No newline at end of file diff --git a/clients/src/main/resources/common/message/FetchResponse.json b/clients/src/main/resources/common/message/FetchResponse.json index 0aa9a3a029e33..22807255bfcf1 100644 --- a/clients/src/main/resources/common/message/FetchResponse.json +++ b/clients/src/main/resources/common/message/FetchResponse.json @@ -53,9 +53,9 @@ "about": "The response topics.", "fields": [ { "name": "Topic", "type": "string", "versions": "0+", "entityType": "topicName", "about": "The topic name." }, - { "name": "PartitionResponses", "type": "[]FetchablePartitionResponse", "versions": "0+", + { "name": "Partitions", "type": "[]PartitionData", "versions": "0+", "about": "The topic partitions.", "fields": [ - { "name": "Partition", "type": "int32", "versions": "0+", + { "name": "PartitionIndex", "type": "int32", "versions": "0+", "about": "The partition index." }, { "name": "ErrorCode", "type": "int16", "versions": "0+", "about": "The error code, or 0 if there was no fetch error." }, @@ -94,7 +94,7 @@ ]}, { "name": "PreferredReadReplica", "type": "int32", "versions": "11+", "default": "-1", "ignorable": false, "about": "The preferred read replica for the consumer to use on its next fetch request"}, - { "name": "RecordSet", "type": "records", "versions": "0+", "nullableVersions": "0+", "about": "The record data."} + { "name": "Records", "type": "records", "versions": "0+", "nullableVersions": "0+", "about": "The record data."} ]} ]} ] diff --git a/clients/src/test/java/org/apache/kafka/clients/FetchSessionHandlerTest.java b/clients/src/test/java/org/apache/kafka/clients/FetchSessionHandlerTest.java index 97a1b177ecad0..72cedd0fa0cae 100644 --- a/clients/src/test/java/org/apache/kafka/clients/FetchSessionHandlerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/FetchSessionHandlerTest.java @@ -17,8 +17,8 @@ package org.apache.kafka.clients; import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.message.FetchResponseData; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.record.MemoryRecords; import org.apache.kafka.common.requests.FetchRequest; import org.apache.kafka.common.requests.FetchResponse; import org.apache.kafka.common.utils.LogContext; @@ -150,22 +150,21 @@ private static void assertListEquals(List expected, List data; + final FetchResponseData.PartitionData data; RespEntry(String topic, int partition, long highWatermark, long lastStableOffset) { this.part = new TopicPartition(topic, partition); - this.data = new FetchResponse.PartitionData<>( - Errors.NONE, - highWatermark, - lastStableOffset, - 0, - null, - null); + + this.data = new FetchResponseData.PartitionData() + .setPartitionIndex(partition) + .setHighWatermark(highWatermark) + .setLastStableOffset(lastStableOffset) + .setLogStartOffset(0); } } - private static LinkedHashMap> respMap(RespEntry... entries) { - LinkedHashMap> map = new LinkedHashMap<>(); + private static LinkedHashMap respMap(RespEntry... entries) { + LinkedHashMap map = new LinkedHashMap<>(); for (RespEntry entry : entries) { map.put(entry.part, entry.data); } @@ -191,10 +190,10 @@ public void testSessionless() { assertEquals(INVALID_SESSION_ID, data.metadata().sessionId()); assertEquals(INITIAL_EPOCH, data.metadata().epoch()); - FetchResponse resp = new FetchResponse<>(Errors.NONE, + FetchResponse resp = FetchResponse.of(Errors.NONE, 0, INVALID_SESSION_ID, respMap(new RespEntry("foo", 0, 0, 0), - new RespEntry("foo", 1, 0, 0)), - 0, INVALID_SESSION_ID); + new RespEntry("foo", 1, 0, 0)) + ); handler.handleResponse(resp); FetchSessionHandler.Builder builder2 = handler.newBuilder(); @@ -225,10 +224,9 @@ public void testIncrementals() { assertEquals(INVALID_SESSION_ID, data.metadata().sessionId()); assertEquals(INITIAL_EPOCH, data.metadata().epoch()); - FetchResponse resp = new FetchResponse<>(Errors.NONE, + FetchResponse resp = FetchResponse.of(Errors.NONE, 0, 123, respMap(new RespEntry("foo", 0, 10, 20), - new RespEntry("foo", 1, 10, 20)), - 0, 123); + new RespEntry("foo", 1, 10, 20))); handler.handleResponse(resp); // Test an incremental fetch request which adds one partition and modifies another. @@ -249,15 +247,14 @@ public void testIncrementals() { new ReqEntry("foo", 1, 10, 120, 210)), data2.toSend()); - FetchResponse resp2 = new FetchResponse<>(Errors.NONE, - respMap(new RespEntry("foo", 1, 20, 20)), - 0, 123); + FetchResponse resp2 = FetchResponse.of(Errors.NONE, 0, 123, + respMap(new RespEntry("foo", 1, 20, 20))); handler.handleResponse(resp2); // Skip building a new request. Test that handling an invalid fetch session epoch response results // in a request which closes the session. - FetchResponse resp3 = new FetchResponse<>(Errors.INVALID_FETCH_SESSION_EPOCH, respMap(), - 0, INVALID_SESSION_ID); + FetchResponse resp3 = FetchResponse.of(Errors.INVALID_FETCH_SESSION_EPOCH, + 0, INVALID_SESSION_ID, respMap()); handler.handleResponse(resp3); FetchSessionHandler.Builder builder4 = handler.newBuilder(); @@ -312,11 +309,10 @@ public void testIncrementalPartitionRemoval() { data.toSend(), data.sessionPartitions()); assertTrue(data.metadata().isFull()); - FetchResponse resp = new FetchResponse<>(Errors.NONE, + FetchResponse resp = FetchResponse.of(Errors.NONE, 0, 123, respMap(new RespEntry("foo", 0, 10, 20), new RespEntry("foo", 1, 10, 20), - new RespEntry("bar", 0, 10, 20)), - 0, 123); + new RespEntry("bar", 0, 10, 20))); handler.handleResponse(resp); // Test an incremental fetch request which removes two partitions. @@ -337,8 +333,8 @@ public void testIncrementalPartitionRemoval() { // A FETCH_SESSION_ID_NOT_FOUND response triggers us to close the session. // The next request is a session establishing FULL request. - FetchResponse resp2 = new FetchResponse<>(Errors.FETCH_SESSION_ID_NOT_FOUND, - respMap(), 0, INVALID_SESSION_ID); + FetchResponse resp2 = FetchResponse.of(Errors.FETCH_SESSION_ID_NOT_FOUND, + 0, INVALID_SESSION_ID, respMap()); handler.handleResponse(resp2); FetchSessionHandler.Builder builder3 = handler.newBuilder(); builder3.add(new TopicPartition("foo", 0), @@ -354,11 +350,10 @@ public void testIncrementalPartitionRemoval() { @Test public void testVerifyFullFetchResponsePartitions() throws Exception { FetchSessionHandler handler = new FetchSessionHandler(LOG_CONTEXT, 1); - String issue = handler.verifyFullFetchResponsePartitions(new FetchResponse<>(Errors.NONE, + String issue = handler.verifyFullFetchResponsePartitions(FetchResponse.of(Errors.NONE, 0, INVALID_SESSION_ID, respMap(new RespEntry("foo", 0, 10, 20), new RespEntry("foo", 1, 10, 20), - new RespEntry("bar", 0, 10, 20)), - 0, INVALID_SESSION_ID)); + new RespEntry("bar", 0, 10, 20)))); assertTrue(issue.contains("extra")); assertFalse(issue.contains("omitted")); FetchSessionHandler.Builder builder = handler.newBuilder(); @@ -369,16 +364,14 @@ public void testVerifyFullFetchResponsePartitions() throws Exception { builder.add(new TopicPartition("bar", 0), new FetchRequest.PartitionData(20, 120, 220, Optional.empty())); builder.build(); - String issue2 = handler.verifyFullFetchResponsePartitions(new FetchResponse<>(Errors.NONE, + String issue2 = handler.verifyFullFetchResponsePartitions(FetchResponse.of(Errors.NONE, 0, INVALID_SESSION_ID, respMap(new RespEntry("foo", 0, 10, 20), new RespEntry("foo", 1, 10, 20), - new RespEntry("bar", 0, 10, 20)), - 0, INVALID_SESSION_ID)); + new RespEntry("bar", 0, 10, 20)))); assertTrue(issue2 == null); - String issue3 = handler.verifyFullFetchResponsePartitions(new FetchResponse<>(Errors.NONE, + String issue3 = handler.verifyFullFetchResponsePartitions(FetchResponse.of(Errors.NONE, 0, INVALID_SESSION_ID, respMap(new RespEntry("foo", 0, 10, 20), - new RespEntry("foo", 1, 10, 20)), - 0, INVALID_SESSION_ID)); + new RespEntry("foo", 1, 10, 20)))); assertFalse(issue3.contains("extra")); assertTrue(issue3.contains("omitted")); } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java index cfdc2bf29de9f..251170747cf9f 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java @@ -45,18 +45,20 @@ import org.apache.kafka.common.errors.InvalidTopicException; import org.apache.kafka.common.errors.UnsupportedVersionException; import org.apache.kafka.common.errors.WakeupException; +import org.apache.kafka.common.internals.ClusterResourceListeners; +import org.apache.kafka.common.message.FetchResponseData; import org.apache.kafka.common.message.HeartbeatResponseData; import org.apache.kafka.common.message.JoinGroupRequestData; import org.apache.kafka.common.message.JoinGroupResponseData; import org.apache.kafka.common.message.LeaveGroupResponseData; +import org.apache.kafka.common.message.ListOffsetsRequestData.ListOffsetsPartition; import org.apache.kafka.common.message.ListOffsetsResponseData; -import org.apache.kafka.common.message.ListOffsetsResponseData.ListOffsetsTopicResponse; import org.apache.kafka.common.message.ListOffsetsResponseData.ListOffsetsPartitionResponse; -import org.apache.kafka.common.internals.ClusterResourceListeners; +import org.apache.kafka.common.message.ListOffsetsResponseData.ListOffsetsTopicResponse; import org.apache.kafka.common.message.SyncGroupResponseData; -import org.apache.kafka.common.message.ListOffsetsRequestData.ListOffsetsPartition; import org.apache.kafka.common.metrics.Metrics; import org.apache.kafka.common.metrics.Sensor; +import org.apache.kafka.common.metrics.stats.Avg; import org.apache.kafka.common.network.Selectable; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; @@ -91,8 +93,6 @@ import org.apache.kafka.test.MockConsumerInterceptor; import org.apache.kafka.test.MockMetricsReporter; import org.apache.kafka.test.TestUtils; -import org.apache.kafka.common.metrics.stats.Avg; - import org.junit.jupiter.api.Test; import javax.management.MBeanServer; @@ -2276,8 +2276,8 @@ private ListOffsetsResponse listOffsetsResponse(Map partit return new ListOffsetsResponse(data); } - private FetchResponse fetchResponse(Map fetches) { - LinkedHashMap> tpResponses = new LinkedHashMap<>(); + private FetchResponse fetchResponse(Map fetches) { + LinkedHashMap tpResponses = new LinkedHashMap<>(); for (Map.Entry fetchEntry : fetches.entrySet()) { TopicPartition partition = fetchEntry.getKey(); long fetchOffset = fetchEntry.getValue().offset; @@ -2294,14 +2294,17 @@ private FetchResponse fetchResponse(Map( - Errors.NONE, highWatermark, FetchResponse.INVALID_LAST_STABLE_OFFSET, - logStartOffset, null, records)); + tpResponses.put(partition, + new FetchResponseData.PartitionData() + .setPartitionIndex(partition.partition()) + .setHighWatermark(highWatermark) + .setLogStartOffset(logStartOffset) + .setRecords(records)); } - return new FetchResponse<>(Errors.NONE, tpResponses, 0, INVALID_SESSION_ID); + return FetchResponse.of(Errors.NONE, 0, INVALID_SESSION_ID, tpResponses); } - private FetchResponse fetchResponse(TopicPartition partition, long fetchOffset, int count) { + private FetchResponse fetchResponse(TopicPartition partition, long fetchOffset, int count) { FetchInfo fetchInfo = new FetchInfo(fetchOffset, count); return fetchResponse(Collections.singletonMap(partition, fetchInfo)); } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java index 0a7d5cfde49b4..985d95947c236 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java @@ -48,6 +48,7 @@ import org.apache.kafka.common.header.Header; import org.apache.kafka.common.header.internals.RecordHeader; import org.apache.kafka.common.internals.ClusterResourceListeners; +import org.apache.kafka.common.message.FetchResponseData; import org.apache.kafka.common.message.ApiMessageType; import org.apache.kafka.common.message.ListOffsetsRequestData.ListOffsetsPartition; import org.apache.kafka.common.message.ListOffsetsRequestData.ListOffsetsTopic; @@ -1270,13 +1271,18 @@ public void testFetchPositionAfterException() { assertEquals(1, fetcher.sendFetches()); - Map> partitions = new LinkedHashMap<>(); - partitions.put(tp1, new FetchResponse.PartitionData<>(Errors.NONE, 100, - FetchResponse.INVALID_LAST_STABLE_OFFSET, FetchResponse.INVALID_LOG_START_OFFSET, null, records)); - partitions.put(tp0, new FetchResponse.PartitionData<>(Errors.OFFSET_OUT_OF_RANGE, 100, - FetchResponse.INVALID_LAST_STABLE_OFFSET, FetchResponse.INVALID_LOG_START_OFFSET, null, MemoryRecords.EMPTY)); - client.prepareResponse(new FetchResponse<>(Errors.NONE, new LinkedHashMap<>(partitions), - 0, INVALID_SESSION_ID)); + + Map partitions = new LinkedHashMap<>(); + partitions.put(tp1, new FetchResponseData.PartitionData() + .setPartitionIndex(tp1.partition()) + .setHighWatermark(100) + .setRecords(records)); + partitions.put(tp0, new FetchResponseData.PartitionData() + .setPartitionIndex(tp0.partition()) + .setErrorCode(Errors.OFFSET_OUT_OF_RANGE.code()) + .setHighWatermark(100)); + client.prepareResponse(FetchResponse.of(Errors.NONE, + 0, INVALID_SESSION_ID, new LinkedHashMap<>(partitions))); consumerClient.poll(time.timer(0)); List> allFetchedRecords = new ArrayList<>(); @@ -1316,17 +1322,29 @@ public void testCompletedFetchRemoval() { assertEquals(1, fetcher.sendFetches()); - Map> partitions = new LinkedHashMap<>(); - partitions.put(tp1, new FetchResponse.PartitionData<>(Errors.NONE, 100, FetchResponse.INVALID_LAST_STABLE_OFFSET, - FetchResponse.INVALID_LOG_START_OFFSET, null, records)); - partitions.put(tp0, new FetchResponse.PartitionData<>(Errors.OFFSET_OUT_OF_RANGE, 100, - FetchResponse.INVALID_LAST_STABLE_OFFSET, FetchResponse.INVALID_LOG_START_OFFSET, null, MemoryRecords.EMPTY)); - partitions.put(tp2, new FetchResponse.PartitionData<>(Errors.NONE, 100L, 4, - 0L, null, nextRecords)); - partitions.put(tp3, new FetchResponse.PartitionData<>(Errors.NONE, 100L, 4, - 0L, null, partialRecords)); - client.prepareResponse(new FetchResponse<>(Errors.NONE, new LinkedHashMap<>(partitions), - 0, INVALID_SESSION_ID)); + Map partitions = new LinkedHashMap<>(); + partitions.put(tp1, new FetchResponseData.PartitionData() + .setPartitionIndex(tp1.partition()) + .setHighWatermark(100) + .setRecords(records)); + partitions.put(tp0, new FetchResponseData.PartitionData() + .setPartitionIndex(tp0.partition()) + .setErrorCode(Errors.OFFSET_OUT_OF_RANGE.code()) + .setHighWatermark(100)); + partitions.put(tp2, new FetchResponseData.PartitionData() + .setPartitionIndex(tp2.partition()) + .setHighWatermark(100) + .setLastStableOffset(4) + .setLogStartOffset(0) + .setRecords(nextRecords)); + partitions.put(tp3, new FetchResponseData.PartitionData() + .setPartitionIndex(tp3.partition()) + .setHighWatermark(100) + .setLastStableOffset(4) + .setLogStartOffset(0) + .setRecords(partialRecords)); + client.prepareResponse(FetchResponse.of(Errors.NONE, + 0, INVALID_SESSION_ID, new LinkedHashMap<>(partitions))); consumerClient.poll(time.timer(0)); List> fetchedRecords = new ArrayList<>(); @@ -1384,9 +1402,11 @@ public void testSeekBeforeException() { assignFromUser(Utils.mkSet(tp0)); subscriptions.seek(tp0, 1); assertEquals(1, fetcher.sendFetches()); - Map> partitions = new HashMap<>(); - partitions.put(tp0, new FetchResponse.PartitionData<>(Errors.NONE, 100, - FetchResponse.INVALID_LAST_STABLE_OFFSET, FetchResponse.INVALID_LOG_START_OFFSET, Optional.empty(), null, records)); + Map partitions = new HashMap<>(); + partitions.put(tp0, new FetchResponseData.PartitionData() + .setPartitionIndex(tp0.partition()) + .setHighWatermark(100) + .setRecords(records)); client.prepareResponse(fullFetchResponse(tp0, this.records, Errors.NONE, 100L, 0)); consumerClient.poll(time.timer(0)); @@ -1397,9 +1417,11 @@ public void testSeekBeforeException() { assertEquals(1, fetcher.sendFetches()); partitions = new HashMap<>(); - partitions.put(tp1, new FetchResponse.PartitionData<>(Errors.OFFSET_OUT_OF_RANGE, 100, - FetchResponse.INVALID_LAST_STABLE_OFFSET, FetchResponse.INVALID_LOG_START_OFFSET, Optional.empty(), null, MemoryRecords.EMPTY)); - client.prepareResponse(new FetchResponse<>(Errors.NONE, new LinkedHashMap<>(partitions), 0, INVALID_SESSION_ID)); + partitions.put(tp1, new FetchResponseData.PartitionData() + .setPartitionIndex(tp1.partition()) + .setErrorCode(Errors.OFFSET_OUT_OF_RANGE.code()) + .setHighWatermark(100)); + client.prepareResponse(FetchResponse.of(Errors.NONE, 0, INVALID_SESSION_ID, new LinkedHashMap<>(partitions))); consumerClient.poll(time.timer(0)); assertEquals(1, fetcher.fetchedRecords().get(tp0).size()); @@ -2094,7 +2116,7 @@ public void testQuotaMetrics() { ClientRequest request = client.newClientRequest(node.idString(), builder, time.milliseconds(), true); client.send(request, time.milliseconds()); client.poll(1, time.milliseconds()); - FetchResponse response = fullFetchResponse(tp0, nextRecords, Errors.NONE, i, throttleTimeMs); + FetchResponse response = fullFetchResponse(tp0, nextRecords, Errors.NONE, i, throttleTimeMs); buffer = RequestTestUtils.serializeResponseWithHeader(response, ApiKeys.FETCH.latestVersion(), request.correlationId()); selector.completeReceive(new NetworkReceive(node.idString(), buffer)); client.poll(1, time.milliseconds()); @@ -2256,7 +2278,7 @@ public void testFetchResponseMetrics() { client.updateMetadata(RequestTestUtils.metadataUpdateWith(1, partitionCounts, tp -> validLeaderEpoch)); int expectedBytes = 0; - LinkedHashMap> fetchPartitionData = new LinkedHashMap<>(); + LinkedHashMap fetchPartitionData = new LinkedHashMap<>(); for (TopicPartition tp : Utils.mkSet(tp1, tp2)) { subscriptions.seek(tp, 0); @@ -2269,12 +2291,15 @@ public void testFetchResponseMetrics() { for (Record record : records.records()) expectedBytes += record.sizeInBytes(); - fetchPartitionData.put(tp, new FetchResponse.PartitionData<>(Errors.NONE, 15L, - FetchResponse.INVALID_LAST_STABLE_OFFSET, 0L, null, records)); + fetchPartitionData.put(tp, new FetchResponseData.PartitionData() + .setPartitionIndex(tp.partition()) + .setHighWatermark(15) + .setLogStartOffset(0) + .setRecords(records)); } assertEquals(1, fetcher.sendFetches()); - client.prepareResponse(new FetchResponse<>(Errors.NONE, fetchPartitionData, 0, INVALID_SESSION_ID)); + client.prepareResponse(FetchResponse.of(Errors.NONE, 0, INVALID_SESSION_ID, fetchPartitionData)); consumerClient.poll(time.timer(0)); Map>> fetchedRecords = fetchedRecords(); @@ -2333,15 +2358,21 @@ public void testFetchResponseMetricsWithOnePartitionError() { builder.appendWithOffset(v, RecordBatch.NO_TIMESTAMP, "key".getBytes(), ("value-" + v).getBytes()); MemoryRecords records = builder.build(); - Map> partitions = new HashMap<>(); - partitions.put(tp0, new FetchResponse.PartitionData<>(Errors.NONE, 100, - FetchResponse.INVALID_LAST_STABLE_OFFSET, 0L, null, records)); - partitions.put(tp1, new FetchResponse.PartitionData<>(Errors.OFFSET_OUT_OF_RANGE, 100, - FetchResponse.INVALID_LAST_STABLE_OFFSET, 0L, null, MemoryRecords.EMPTY)); + Map partitions = new HashMap<>(); + partitions.put(tp0, new FetchResponseData.PartitionData() + .setPartitionIndex(tp0.partition()) + .setHighWatermark(100) + .setLogStartOffset(0) + .setRecords(records)); + partitions.put(tp1, new FetchResponseData.PartitionData() + .setPartitionIndex(tp1.partition()) + .setErrorCode(Errors.OFFSET_OUT_OF_RANGE.code()) + .setHighWatermark(100) + .setLogStartOffset(0)); assertEquals(1, fetcher.sendFetches()); - client.prepareResponse(new FetchResponse<>(Errors.NONE, new LinkedHashMap<>(partitions), - 0, INVALID_SESSION_ID)); + client.prepareResponse(FetchResponse.of(Errors.NONE, + 0, INVALID_SESSION_ID, new LinkedHashMap<>(partitions))); consumerClient.poll(time.timer(0)); fetcher.fetchedRecords(); @@ -2375,15 +2406,19 @@ public void testFetchResponseMetricsWithOnePartitionAtTheWrongOffset() { builder.appendWithOffset(v, RecordBatch.NO_TIMESTAMP, "key".getBytes(), ("value-" + v).getBytes()); MemoryRecords records = builder.build(); - Map> partitions = new HashMap<>(); - partitions.put(tp0, new FetchResponse.PartitionData<>(Errors.NONE, 100, - FetchResponse.INVALID_LAST_STABLE_OFFSET, 0L, null, records)); - partitions.put(tp1, new FetchResponse.PartitionData<>(Errors.NONE, 100, - FetchResponse.INVALID_LAST_STABLE_OFFSET, 0L, null, - MemoryRecords.withRecords(CompressionType.NONE, new SimpleRecord("val".getBytes())))); - - client.prepareResponse(new FetchResponse<>(Errors.NONE, new LinkedHashMap<>(partitions), - 0, INVALID_SESSION_ID)); + Map partitions = new HashMap<>(); + partitions.put(tp0, new FetchResponseData.PartitionData() + .setPartitionIndex(tp0.partition()) + .setHighWatermark(100) + .setLogStartOffset(0) + .setRecords(records)); + partitions.put(tp1, new FetchResponseData.PartitionData() + .setPartitionIndex(tp1.partition()) + .setHighWatermark(100) + .setLogStartOffset(0) + .setRecords(MemoryRecords.withRecords(CompressionType.NONE, new SimpleRecord("val".getBytes())))); + + client.prepareResponse(FetchResponse.of(Errors.NONE, 0, INVALID_SESSION_ID, new LinkedHashMap<>(partitions))); consumerClient.poll(time.timer(0)); fetcher.fetchedRecords(); @@ -2778,8 +2813,8 @@ public void testSkippingAbortedTransactions() { buffer.flip(); - List abortedTransactions = new ArrayList<>(); - abortedTransactions.add(new FetchResponse.AbortedTransaction(1, 0)); + List abortedTransactions = Collections.singletonList( + new FetchResponseData.AbortedTransaction().setProducerId(1).setFirstOffset(0)); MemoryRecords records = MemoryRecords.readableRecords(buffer); assignFromUser(singleton(tp0)); @@ -2811,7 +2846,6 @@ public void testReturnCommittedTransactions() { commitTransaction(buffer, 1L, currentOffset); buffer.flip(); - List abortedTransactions = new ArrayList<>(); MemoryRecords records = MemoryRecords.readableRecords(buffer); assignFromUser(singleton(tp0)); @@ -2824,7 +2858,7 @@ public void testReturnCommittedTransactions() { FetchRequest request = (FetchRequest) body; assertEquals(IsolationLevel.READ_COMMITTED, request.isolationLevel()); return true; - }, fullFetchResponseWithAbortedTransactions(records, abortedTransactions, Errors.NONE, 100L, 100L, 0)); + }, fullFetchResponseWithAbortedTransactions(records, Collections.emptyList(), Errors.NONE, 100L, 100L, 0)); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); @@ -2840,7 +2874,7 @@ public void testReadCommittedWithCommittedAndAbortedTransactions() { new ByteArrayDeserializer(), Integer.MAX_VALUE, IsolationLevel.READ_COMMITTED); ByteBuffer buffer = ByteBuffer.allocate(1024); - List abortedTransactions = new ArrayList<>(); + List abortedTransactions = new ArrayList<>(); long pid1 = 1L; long pid2 = 2L; @@ -2863,7 +2897,7 @@ public void testReadCommittedWithCommittedAndAbortedTransactions() { // abort producer 2 abortTransaction(buffer, pid2, 5L); - abortedTransactions.add(new FetchResponse.AbortedTransaction(pid2, 2L)); + abortedTransactions.add(new FetchResponseData.AbortedTransaction().setProducerId(pid2).setFirstOffset(2L)); // New transaction for producer 1 (eventually aborted) appendTransactionalRecords(buffer, pid1, 6L, @@ -2879,7 +2913,7 @@ public void testReadCommittedWithCommittedAndAbortedTransactions() { // abort producer 1 abortTransaction(buffer, pid1, 9L); - abortedTransactions.add(new FetchResponse.AbortedTransaction(1, 6)); + abortedTransactions.add(new FetchResponseData.AbortedTransaction().setProducerId(1).setFirstOffset(6)); // commit producer 2 commitTransaction(buffer, pid2, 10L); @@ -2931,8 +2965,9 @@ public void testMultipleAbortMarkers() { commitTransaction(buffer, 1L, currentOffset); buffer.flip(); - List abortedTransactions = new ArrayList<>(); - abortedTransactions.add(new FetchResponse.AbortedTransaction(1, 0)); + List abortedTransactions = Collections.singletonList( + new FetchResponseData.AbortedTransaction().setProducerId(1).setFirstOffset(0) + ); MemoryRecords records = MemoryRecords.readableRecords(buffer); assignFromUser(singleton(tp0)); @@ -2983,8 +3018,8 @@ public void testReadCommittedAbortMarkerWithNoData() { assertEquals(1, fetcher.sendFetches()); // prepare the response. the aborted transactions begin at offsets which are no longer in the log - List abortedTransactions = new ArrayList<>(); - abortedTransactions.add(new FetchResponse.AbortedTransaction(producerId, 0L)); + List abortedTransactions = Collections.singletonList( + new FetchResponseData.AbortedTransaction().setProducerId(producerId).setFirstOffset(0L)); client.prepareResponse(fullFetchResponseWithAbortedTransactions(MemoryRecords.readableRecords(buffer), abortedTransactions, Errors.NONE, 100L, 100L, 0)); @@ -3120,9 +3155,10 @@ public void testReadCommittedWithCompactedTopic() { assertEquals(1, fetcher.sendFetches()); // prepare the response. the aborted transactions begin at offsets which are no longer in the log - List abortedTransactions = new ArrayList<>(); - abortedTransactions.add(new FetchResponse.AbortedTransaction(pid2, 6L)); - abortedTransactions.add(new FetchResponse.AbortedTransaction(pid1, 0L)); + List abortedTransactions = Arrays.asList( + new FetchResponseData.AbortedTransaction().setProducerId(pid2).setFirstOffset(6), + new FetchResponseData.AbortedTransaction().setProducerId(pid1).setFirstOffset(0) + ); client.prepareResponse(fullFetchResponseWithAbortedTransactions(MemoryRecords.readableRecords(buffer), abortedTransactions, Errors.NONE, 100L, 100L, 0)); @@ -3151,8 +3187,8 @@ public void testReturnAbortedTransactionsinUncommittedMode() { buffer.flip(); - List abortedTransactions = new ArrayList<>(); - abortedTransactions.add(new FetchResponse.AbortedTransaction(1, 0)); + List abortedTransactions = Collections.singletonList( + new FetchResponseData.AbortedTransaction().setProducerId(1).setFirstOffset(0)); MemoryRecords records = MemoryRecords.readableRecords(buffer); assignFromUser(singleton(tp0)); @@ -3184,8 +3220,8 @@ public void testConsumerPositionUpdatedWhenSkippingAbortedTransactions() { currentOffset += abortTransaction(buffer, 1L, currentOffset); buffer.flip(); - List abortedTransactions = new ArrayList<>(); - abortedTransactions.add(new FetchResponse.AbortedTransaction(1, 0)); + List abortedTransactions = Collections.singletonList( + new FetchResponseData.AbortedTransaction().setProducerId(1).setFirstOffset(0)); MemoryRecords records = MemoryRecords.readableRecords(buffer); assignFromUser(singleton(tp0)); @@ -3216,12 +3252,19 @@ public void testConsumingViaIncrementalFetchRequests() { subscriptions.seekValidated(tp1, new SubscriptionState.FetchPosition(1, Optional.empty(), metadata.currentLeader(tp1))); // Fetch some records and establish an incremental fetch session. - LinkedHashMap> partitions1 = new LinkedHashMap<>(); - partitions1.put(tp0, new FetchResponse.PartitionData<>(Errors.NONE, 2L, - 2, 0L, null, this.records)); - partitions1.put(tp1, new FetchResponse.PartitionData<>(Errors.NONE, 100L, - FetchResponse.INVALID_LAST_STABLE_OFFSET, 0L, null, emptyRecords)); - FetchResponse resp1 = new FetchResponse<>(Errors.NONE, partitions1, 0, 123); + LinkedHashMap partitions1 = new LinkedHashMap<>(); + partitions1.put(tp0, new FetchResponseData.PartitionData() + .setPartitionIndex(tp0.partition()) + .setHighWatermark(2) + .setLastStableOffset(2) + .setLogStartOffset(0) + .setRecords(this.records)); + partitions1.put(tp1, new FetchResponseData.PartitionData() + .setPartitionIndex(tp1.partition()) + .setHighWatermark(100) + .setLogStartOffset(0) + .setRecords(emptyRecords)); + FetchResponse resp1 = FetchResponse.of(Errors.NONE, 0, 123, partitions1); client.prepareResponse(resp1); assertEquals(1, fetcher.sendFetches()); assertFalse(fetcher.hasCompletedFetches()); @@ -3246,8 +3289,8 @@ public void testConsumingViaIncrementalFetchRequests() { assertEquals(4L, subscriptions.position(tp0).offset); // The second response contains no new records. - LinkedHashMap> partitions2 = new LinkedHashMap<>(); - FetchResponse resp2 = new FetchResponse<>(Errors.NONE, partitions2, 0, 123); + LinkedHashMap partitions2 = new LinkedHashMap<>(); + FetchResponse resp2 = FetchResponse.of(Errors.NONE, 0, 123, partitions2); client.prepareResponse(resp2); assertEquals(1, fetcher.sendFetches()); consumerClient.poll(time.timer(0)); @@ -3257,10 +3300,14 @@ public void testConsumingViaIncrementalFetchRequests() { assertEquals(1L, subscriptions.position(tp1).offset); // The third response contains some new records for tp0. - LinkedHashMap> partitions3 = new LinkedHashMap<>(); - partitions3.put(tp0, new FetchResponse.PartitionData<>(Errors.NONE, 100L, - 4, 0L, null, this.nextRecords)); - FetchResponse resp3 = new FetchResponse<>(Errors.NONE, partitions3, 0, 123); + LinkedHashMap partitions3 = new LinkedHashMap<>(); + partitions3.put(tp0, new FetchResponseData.PartitionData() + .setPartitionIndex(tp0.partition()) + .setHighWatermark(100) + .setLastStableOffset(4) + .setLogStartOffset(0) + .setRecords(this.nextRecords)); + FetchResponse resp3 = FetchResponse.of(Errors.NONE, 0, 123, partitions3); client.prepareResponse(resp3); assertEquals(1, fetcher.sendFetches()); consumerClient.poll(time.timer(0)); @@ -3319,7 +3366,7 @@ public Builder newBuilder() { } @Override - public boolean handleResponse(FetchResponse response) { + public boolean handleResponse(FetchResponse response) { verifySessionPartitions(); return handler.handleResponse(response); } @@ -3367,14 +3414,18 @@ private void verifySessionPartitions() { if (!client.requests().isEmpty()) { ClientRequest request = client.requests().peek(); FetchRequest fetchRequest = (FetchRequest) request.requestBuilder().build(); - LinkedHashMap> responseMap = new LinkedHashMap<>(); + LinkedHashMap responseMap = new LinkedHashMap<>(); for (Map.Entry entry : fetchRequest.fetchData().entrySet()) { TopicPartition tp = entry.getKey(); long offset = entry.getValue().fetchOffset; - responseMap.put(tp, new FetchResponse.PartitionData<>(Errors.NONE, offset + 2L, offset + 2, - 0L, null, buildRecords(offset, 2, offset))); + responseMap.put(tp, new FetchResponseData.PartitionData() + .setPartitionIndex(tp.partition()) + .setHighWatermark(offset + 2) + .setLastStableOffset(offset + 2) + .setLogStartOffset(0) + .setRecords(buildRecords(offset, 2, offset))); } - client.respondToRequest(request, new FetchResponse<>(Errors.NONE, responseMap, 0, 123)); + client.respondToRequest(request, FetchResponse.of(Errors.NONE, 0, 123, responseMap)); consumerClient.poll(time.timer(0)); } } @@ -3429,11 +3480,15 @@ public void testFetcherSessionEpochUpdate() throws Exception { assertTrue(epoch == 0 || epoch == nextEpoch, String.format("Unexpected epoch expected %d got %d", nextEpoch, epoch)); nextEpoch++; - LinkedHashMap> responseMap = new LinkedHashMap<>(); - responseMap.put(tp0, new FetchResponse.PartitionData<>(Errors.NONE, nextOffset + 2L, nextOffset + 2, - 0L, null, buildRecords(nextOffset, 2, nextOffset))); + LinkedHashMap responseMap = new LinkedHashMap<>(); + responseMap.put(tp0, new FetchResponseData.PartitionData() + .setPartitionIndex(tp0.partition()) + .setHighWatermark(nextOffset + 2) + .setLastStableOffset(nextOffset + 2) + .setLogStartOffset(0) + .setRecords(buildRecords(nextOffset, 2, nextOffset))); nextOffset += 2; - client.respondToRequest(request, new FetchResponse<>(Errors.NONE, responseMap, 0, 123)); + client.respondToRequest(request, FetchResponse.of(Errors.NONE, 0, 123, responseMap)); consumerClient.poll(time.timer(0)); } } @@ -3483,7 +3538,6 @@ public void testEmptyControlBatch() { commitTransaction(buffer, 1L, currentOffset); buffer.flip(); - List abortedTransactions = new ArrayList<>(); MemoryRecords records = MemoryRecords.readableRecords(buffer); assignFromUser(singleton(tp0)); @@ -3496,7 +3550,7 @@ public void testEmptyControlBatch() { FetchRequest request = (FetchRequest) body; assertEquals(IsolationLevel.READ_COMMITTED, request.isolationLevel()); return true; - }, fullFetchResponseWithAbortedTransactions(records, abortedTransactions, Errors.NONE, 100L, 100L, 0)); + }, fullFetchResponseWithAbortedTransactions(records, Collections.emptyList(), Errors.NONE, 100L, 100L, 0)); consumerClient.poll(time.timer(0)); assertTrue(fetcher.hasCompletedFetches()); @@ -4463,41 +4517,66 @@ private ListOffsetsResponse listOffsetResponse(Map offsets return new ListOffsetsResponse(data); } - private FetchResponse fullFetchResponseWithAbortedTransactions(MemoryRecords records, - List abortedTransactions, + private FetchResponse fullFetchResponseWithAbortedTransactions(MemoryRecords records, + List abortedTransactions, Errors error, long lastStableOffset, long hw, int throttleTime) { - Map> partitions = Collections.singletonMap(tp0, - new FetchResponse.PartitionData<>(error, hw, lastStableOffset, 0L, abortedTransactions, records)); - return new FetchResponse<>(Errors.NONE, new LinkedHashMap<>(partitions), throttleTime, INVALID_SESSION_ID); - } - - private FetchResponse fullFetchResponse(TopicPartition tp, MemoryRecords records, Errors error, long hw, int throttleTime) { + Map partitions = Collections.singletonMap(tp0, + new FetchResponseData.PartitionData() + .setPartitionIndex(tp0.partition()) + .setErrorCode(error.code()) + .setHighWatermark(hw) + .setLastStableOffset(lastStableOffset) + .setLogStartOffset(0) + .setAbortedTransactions(abortedTransactions) + .setRecords(records)); + return FetchResponse.of(Errors.NONE, throttleTime, INVALID_SESSION_ID, new LinkedHashMap<>(partitions)); + } + + private FetchResponse fullFetchResponse(TopicPartition tp, MemoryRecords records, Errors error, long hw, int throttleTime) { return fullFetchResponse(tp, records, error, hw, FetchResponse.INVALID_LAST_STABLE_OFFSET, throttleTime); } - private FetchResponse fullFetchResponse(TopicPartition tp, MemoryRecords records, Errors error, long hw, + private FetchResponse fullFetchResponse(TopicPartition tp, MemoryRecords records, Errors error, long hw, long lastStableOffset, int throttleTime) { - Map> partitions = Collections.singletonMap(tp, - new FetchResponse.PartitionData<>(error, hw, lastStableOffset, 0L, null, records)); - return new FetchResponse<>(Errors.NONE, new LinkedHashMap<>(partitions), throttleTime, INVALID_SESSION_ID); - } - - private FetchResponse fullFetchResponse(TopicPartition tp, MemoryRecords records, Errors error, long hw, + Map partitions = Collections.singletonMap(tp, + new FetchResponseData.PartitionData() + .setPartitionIndex(tp.partition()) + .setErrorCode(error.code()) + .setHighWatermark(hw) + .setLastStableOffset(lastStableOffset) + .setLogStartOffset(0) + .setRecords(records)); + return FetchResponse.of(Errors.NONE, throttleTime, INVALID_SESSION_ID, new LinkedHashMap<>(partitions)); + } + + private FetchResponse fullFetchResponse(TopicPartition tp, MemoryRecords records, Errors error, long hw, long lastStableOffset, int throttleTime, Optional preferredReplicaId) { - Map> partitions = Collections.singletonMap(tp, - new FetchResponse.PartitionData<>(error, hw, lastStableOffset, 0L, - preferredReplicaId, null, records)); - return new FetchResponse<>(Errors.NONE, new LinkedHashMap<>(partitions), throttleTime, INVALID_SESSION_ID); - } - - private FetchResponse fetchResponse(TopicPartition tp, MemoryRecords records, Errors error, long hw, + Map partitions = Collections.singletonMap(tp, + new FetchResponseData.PartitionData() + .setPartitionIndex(tp.partition()) + .setErrorCode(error.code()) + .setHighWatermark(hw) + .setLastStableOffset(lastStableOffset) + .setLogStartOffset(0) + .setRecords(records) + .setPreferredReadReplica(preferredReplicaId.orElse(FetchResponse.INVALID_PREFERRED_REPLICA_ID))); + return FetchResponse.of(Errors.NONE, throttleTime, INVALID_SESSION_ID, new LinkedHashMap<>(partitions)); + } + + private FetchResponse fetchResponse(TopicPartition tp, MemoryRecords records, Errors error, long hw, long lastStableOffset, long logStartOffset, int throttleTime) { - Map> partitions = Collections.singletonMap(tp, - new FetchResponse.PartitionData<>(error, hw, lastStableOffset, logStartOffset, null, records)); - return new FetchResponse<>(Errors.NONE, new LinkedHashMap<>(partitions), throttleTime, INVALID_SESSION_ID); + Map partitions = Collections.singletonMap(tp, + new FetchResponseData.PartitionData() + .setPartitionIndex(tp.partition()) + .setErrorCode(error.code()) + .setHighWatermark(hw) + .setLastStableOffset(lastStableOffset) + .setLogStartOffset(logStartOffset) + .setRecords(records)); + return FetchResponse.of(Errors.NONE, throttleTime, INVALID_SESSION_ID, new LinkedHashMap<>(partitions)); } private MetadataResponse newMetadataResponse(String topic, Errors error) { diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java index 228f32afcea94..cebb99d17ef22 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java @@ -803,15 +803,18 @@ public void produceRequestGetErrorResponseTest() { @Test public void fetchResponseVersionTest() { - LinkedHashMap> responseData = new LinkedHashMap<>(); + LinkedHashMap responseData = new LinkedHashMap<>(); MemoryRecords records = MemoryRecords.readableRecords(ByteBuffer.allocate(10)); - responseData.put(new TopicPartition("test", 0), new FetchResponse.PartitionData<>( - Errors.NONE, 1000000, FetchResponse.INVALID_LAST_STABLE_OFFSET, - 0L, Optional.empty(), Collections.emptyList(), records)); - - FetchResponse v0Response = new FetchResponse<>(Errors.NONE, responseData, 0, INVALID_SESSION_ID); - FetchResponse v1Response = new FetchResponse<>(Errors.NONE, responseData, 10, INVALID_SESSION_ID); + responseData.put(new TopicPartition("test", 0), + new FetchResponseData.PartitionData() + .setPartitionIndex(0) + .setHighWatermark(1000000) + .setLogStartOffset(0) + .setRecords(records)); + + FetchResponse v0Response = FetchResponse.of(Errors.NONE, 0, INVALID_SESSION_ID, responseData); + FetchResponse v1Response = FetchResponse.of(Errors.NONE, 10, INVALID_SESSION_ID, responseData); assertEquals(0, v0Response.throttleTimeMs(), "Throttle time must be zero"); assertEquals(10, v1Response.throttleTimeMs(), "Throttle time must be 10"); assertEquals(responseData, v0Response.responseData(), "Response data does not match"); @@ -820,22 +823,34 @@ public void fetchResponseVersionTest() { @Test public void testFetchResponseV4() { - LinkedHashMap> responseData = new LinkedHashMap<>(); + LinkedHashMap responseData = new LinkedHashMap<>(); MemoryRecords records = MemoryRecords.readableRecords(ByteBuffer.allocate(10)); - List abortedTransactions = asList( - new FetchResponse.AbortedTransaction(10, 100), - new FetchResponse.AbortedTransaction(15, 50) + List abortedTransactions = asList( + new FetchResponseData.AbortedTransaction().setProducerId(10).setFirstOffset(100), + new FetchResponseData.AbortedTransaction().setProducerId(15).setFirstOffset(50) ); - responseData.put(new TopicPartition("bar", 0), new FetchResponse.PartitionData<>(Errors.NONE, 100000, - FetchResponse.INVALID_LAST_STABLE_OFFSET, FetchResponse.INVALID_LOG_START_OFFSET, Optional.empty(), abortedTransactions, records)); - responseData.put(new TopicPartition("bar", 1), new FetchResponse.PartitionData<>(Errors.NONE, 900000, - 5, FetchResponse.INVALID_LOG_START_OFFSET, Optional.empty(), null, records)); - responseData.put(new TopicPartition("foo", 0), new FetchResponse.PartitionData<>(Errors.NONE, 70000, - 6, FetchResponse.INVALID_LOG_START_OFFSET, Optional.empty(), emptyList(), records)); - - FetchResponse response = new FetchResponse<>(Errors.NONE, responseData, 10, INVALID_SESSION_ID); - FetchResponse deserialized = FetchResponse.parse(response.serialize((short) 4), (short) 4); + responseData.put(new TopicPartition("bar", 0), + new FetchResponseData.PartitionData() + .setPartitionIndex(0) + .setHighWatermark(1000000) + .setAbortedTransactions(abortedTransactions) + .setRecords(records)); + responseData.put(new TopicPartition("bar", 1), + new FetchResponseData.PartitionData() + .setPartitionIndex(1) + .setHighWatermark(900000) + .setLastStableOffset(5) + .setRecords(records)); + responseData.put(new TopicPartition("foo", 0), + new FetchResponseData.PartitionData() + .setPartitionIndex(0) + .setHighWatermark(70000) + .setLastStableOffset(6) + .setRecords(records)); + + FetchResponse response = FetchResponse.of(Errors.NONE, 10, INVALID_SESSION_ID, responseData); + FetchResponse deserialized = FetchResponse.parse(response.serialize((short) 4), (short) 4); assertEquals(responseData, deserialized.responseData()); } @@ -849,7 +864,7 @@ public void verifyFetchResponseFullWrites() throws Exception { } } - private void verifyFetchResponseFullWrite(short apiVersion, FetchResponse fetchResponse) throws Exception { + private void verifyFetchResponseFullWrite(short apiVersion, FetchResponse fetchResponse) throws Exception { int correlationId = 15; short responseHeaderVersion = FETCH.responseHeaderVersion(apiVersion); @@ -1158,38 +1173,49 @@ private FetchRequest createFetchRequest(int version) { return FetchRequest.Builder.forConsumer(100, 100000, fetchData).setMaxBytes(1000).build((short) version); } - private FetchResponse createFetchResponse(Errors error, int sessionId) { - return new FetchResponse<>(error, new LinkedHashMap<>(), 25, sessionId); + private FetchResponse createFetchResponse(Errors error, int sessionId) { + return FetchResponse.of(error, 25, sessionId, new LinkedHashMap<>()); } - private FetchResponse createFetchResponse(int sessionId) { - LinkedHashMap> responseData = new LinkedHashMap<>(); + private FetchResponse createFetchResponse(int sessionId) { + LinkedHashMap responseData = new LinkedHashMap<>(); MemoryRecords records = MemoryRecords.withRecords(CompressionType.NONE, new SimpleRecord("blah".getBytes())); - responseData.put(new TopicPartition("test", 0), new FetchResponse.PartitionData<>(Errors.NONE, - 1000000, FetchResponse.INVALID_LAST_STABLE_OFFSET, 0L, Optional.empty(), Collections.emptyList(), records)); - List abortedTransactions = Collections.singletonList( - new FetchResponse.AbortedTransaction(234L, 999L)); - responseData.put(new TopicPartition("test", 1), new FetchResponse.PartitionData<>(Errors.NONE, - 1000000, FetchResponse.INVALID_LAST_STABLE_OFFSET, 0L, Optional.empty(), abortedTransactions, MemoryRecords.EMPTY)); - return new FetchResponse<>(Errors.NONE, responseData, 25, sessionId); - } - - private FetchResponse createFetchResponse(boolean includeAborted) { - LinkedHashMap> responseData = new LinkedHashMap<>(); + responseData.put(new TopicPartition("test", 0), new FetchResponseData.PartitionData() + .setPartitionIndex(0) + .setHighWatermark(1000000) + .setLogStartOffset(0) + .setRecords(records)); + List abortedTransactions = Collections.singletonList( + new FetchResponseData.AbortedTransaction().setProducerId(234L).setFirstOffset(999L)); + responseData.put(new TopicPartition("test", 1), new FetchResponseData.PartitionData() + .setPartitionIndex(1) + .setHighWatermark(1000000) + .setLogStartOffset(0) + .setAbortedTransactions(abortedTransactions)); + return FetchResponse.of(Errors.NONE, 25, sessionId, responseData); + } + + private FetchResponse createFetchResponse(boolean includeAborted) { + LinkedHashMap responseData = new LinkedHashMap<>(); MemoryRecords records = MemoryRecords.withRecords(CompressionType.NONE, new SimpleRecord("blah".getBytes())); + responseData.put(new TopicPartition("test", 0), new FetchResponseData.PartitionData() + .setPartitionIndex(0) + .setHighWatermark(1000000) + .setLogStartOffset(0) + .setRecords(records)); - responseData.put(new TopicPartition("test", 0), new FetchResponse.PartitionData<>(Errors.NONE, - 1000000, FetchResponse.INVALID_LAST_STABLE_OFFSET, 0L, Optional.empty(), Collections.emptyList(), records)); - - List abortedTransactions = Collections.emptyList(); + List abortedTransactions = Collections.emptyList(); if (includeAborted) { abortedTransactions = Collections.singletonList( - new FetchResponse.AbortedTransaction(234L, 999L)); + new FetchResponseData.AbortedTransaction().setProducerId(234L).setFirstOffset(999L)); } - responseData.put(new TopicPartition("test", 1), new FetchResponse.PartitionData<>(Errors.NONE, - 1000000, FetchResponse.INVALID_LAST_STABLE_OFFSET, 0L, Optional.empty(), abortedTransactions, MemoryRecords.EMPTY)); + responseData.put(new TopicPartition("test", 1), new FetchResponseData.PartitionData() + .setPartitionIndex(1) + .setHighWatermark(1000000) + .setLogStartOffset(0) + .setAbortedTransactions(abortedTransactions)); - return new FetchResponse<>(Errors.NONE, responseData, 25, INVALID_SESSION_ID); + return FetchResponse.of(Errors.NONE, 25, INVALID_SESSION_ID, responseData); } private HeartbeatRequest createHeartBeatRequest() { diff --git a/core/src/main/scala/kafka/log/Log.scala b/core/src/main/scala/kafka/log/Log.scala index 5ab2fac3e75b2..0ad82f46b9d4b 100644 --- a/core/src/main/scala/kafka/log/Log.scala +++ b/core/src/main/scala/kafka/log/Log.scala @@ -40,7 +40,6 @@ import org.apache.kafka.common.errors._ import org.apache.kafka.common.message.{DescribeProducersResponseData, FetchResponseData} import org.apache.kafka.common.record.FileRecords.TimestampAndOffset import org.apache.kafka.common.record._ -import org.apache.kafka.common.requests.FetchResponse.AbortedTransaction import org.apache.kafka.common.requests.ListOffsetsRequest import org.apache.kafka.common.requests.OffsetsForLeaderEpochResponse.UNDEFINED_EPOCH_OFFSET import org.apache.kafka.common.requests.ProduceResponse.RecordError @@ -1572,7 +1571,7 @@ class Log(@volatile private var _dir: File, private def emptyFetchDataInfo(fetchOffsetMetadata: LogOffsetMetadata, includeAbortedTxns: Boolean): FetchDataInfo = { val abortedTransactions = - if (includeAbortedTxns) Some(List.empty[AbortedTransaction]) + if (includeAbortedTxns) Some(List.empty[FetchResponseData.AbortedTransaction]) else None FetchDataInfo(fetchOffsetMetadata, MemoryRecords.EMPTY, @@ -1676,7 +1675,7 @@ class Log(@volatile private var _dir: File, logEndOffset } - val abortedTransactions = ListBuffer.empty[AbortedTransaction] + val abortedTransactions = ListBuffer.empty[FetchResponseData.AbortedTransaction] def accumulator(abortedTxns: List[AbortedTxn]): Unit = abortedTransactions ++= abortedTxns.map(_.asAbortedTransaction) collectAbortedTransactions(startOffset, upperBoundOffset, segmentEntry, accumulator) diff --git a/core/src/main/scala/kafka/log/TransactionIndex.scala b/core/src/main/scala/kafka/log/TransactionIndex.scala index 9152bc41ab353..565c4eb574060 100644 --- a/core/src/main/scala/kafka/log/TransactionIndex.scala +++ b/core/src/main/scala/kafka/log/TransactionIndex.scala @@ -20,10 +20,9 @@ import java.io.{File, IOException} import java.nio.ByteBuffer import java.nio.channels.FileChannel import java.nio.file.{Files, StandardOpenOption} - import kafka.utils.{Logging, nonthreadsafe} import org.apache.kafka.common.KafkaException -import org.apache.kafka.common.requests.FetchResponse.AbortedTransaction +import org.apache.kafka.common.message.FetchResponseData import org.apache.kafka.common.utils.Utils import scala.collection.mutable.ListBuffer @@ -245,7 +244,9 @@ private[log] class AbortedTxn(val buffer: ByteBuffer) { def lastStableOffset: Long = buffer.getLong(LastStableOffsetOffset) - def asAbortedTransaction: AbortedTransaction = new AbortedTransaction(producerId, firstOffset) + def asAbortedTransaction: FetchResponseData.AbortedTransaction = new FetchResponseData.AbortedTransaction() + .setProducerId(producerId) + .setFirstOffset(firstOffset) override def toString: String = s"AbortedTxn(version=$version, producerId=$producerId, firstOffset=$firstOffset, " + diff --git a/core/src/main/scala/kafka/network/RequestChannel.scala b/core/src/main/scala/kafka/network/RequestChannel.scala index 40d0bd62e622d..a2a2144f0128e 100644 --- a/core/src/main/scala/kafka/network/RequestChannel.scala +++ b/core/src/main/scala/kafka/network/RequestChannel.scala @@ -192,7 +192,7 @@ object RequestChannel extends Logging { resources.add(newResource) } val data = new IncrementalAlterConfigsRequestData() - .setValidateOnly(alterConfigs.data().validateOnly()) + .setValidateOnly(alterConfigs.data.validateOnly()) .setResources(resources) new IncrementalAlterConfigsRequest.Builder(data).build(alterConfigs.version) diff --git a/core/src/main/scala/kafka/network/RequestConvertToJson.scala b/core/src/main/scala/kafka/network/RequestConvertToJson.scala index 2f23dbc2b2f79..c0fbd68e75bf2 100644 --- a/core/src/main/scala/kafka/network/RequestConvertToJson.scala +++ b/core/src/main/scala/kafka/network/RequestConvertToJson.scala @@ -135,7 +135,7 @@ object RequestConvertToJson { case res: EndQuorumEpochResponse => EndQuorumEpochResponseDataJsonConverter.write(res.data, version) case res: EnvelopeResponse => EnvelopeResponseDataJsonConverter.write(res.data, version) case res: ExpireDelegationTokenResponse => ExpireDelegationTokenResponseDataJsonConverter.write(res.data, version) - case res: FetchResponse[_] => FetchResponseDataJsonConverter.write(res.data, version, false) + case res: FetchResponse => FetchResponseDataJsonConverter.write(res.data, version, false) case res: FindCoordinatorResponse => FindCoordinatorResponseDataJsonConverter.write(res.data, version) case res: HeartbeatResponse => HeartbeatResponseDataJsonConverter.write(res.data, version) case res: IncrementalAlterConfigsResponse => IncrementalAlterConfigsResponseDataJsonConverter.write(res.data, version) diff --git a/core/src/main/scala/kafka/server/AbstractFetcherThread.scala b/core/src/main/scala/kafka/server/AbstractFetcherThread.scala index 5e716e29e94f0..6118c5f85cbcf 100755 --- a/core/src/main/scala/kafka/server/AbstractFetcherThread.scala +++ b/core/src/main/scala/kafka/server/AbstractFetcherThread.scala @@ -17,37 +17,33 @@ package kafka.server -import java.nio.ByteBuffer -import java.util -import java.util.Optional -import java.util.concurrent.locks.ReentrantLock - import kafka.cluster.BrokerEndPoint -import kafka.utils.{DelayedItem, Pool, ShutdownableThread} -import kafka.utils.Implicits._ -import org.apache.kafka.common.errors._ import kafka.common.ClientIdAndBroker +import kafka.log.LogAppendInfo import kafka.metrics.KafkaMetricsGroup +import kafka.server.AbstractFetcherThread.{ReplicaFetch, ResultWithPartitions} import kafka.utils.CoreUtils.inLock -import org.apache.kafka.common.protocol.Errors - -import scala.collection.{Map, Set, mutable} -import scala.compat.java8.OptionConverters._ -import scala.jdk.CollectionConverters._ -import java.util.concurrent.TimeUnit -import java.util.concurrent.atomic.AtomicLong - -import kafka.log.LogAppendInfo -import kafka.server.AbstractFetcherThread.ReplicaFetch -import kafka.server.AbstractFetcherThread.ResultWithPartitions -import org.apache.kafka.common.{InvalidRecordException, TopicPartition} +import kafka.utils.Implicits._ +import kafka.utils.{DelayedItem, Pool, ShutdownableThread} +import org.apache.kafka.common.errors._ import org.apache.kafka.common.internals.PartitionStates -import org.apache.kafka.common.message.OffsetForLeaderEpochRequestData import org.apache.kafka.common.message.OffsetForLeaderEpochResponseData.EpochEndOffset +import org.apache.kafka.common.message.{FetchResponseData, OffsetForLeaderEpochRequestData} +import org.apache.kafka.common.protocol.Errors import org.apache.kafka.common.record.{FileRecords, MemoryRecords, Records} -import org.apache.kafka.common.requests._ import org.apache.kafka.common.requests.OffsetsForLeaderEpochResponse.{UNDEFINED_EPOCH, UNDEFINED_EPOCH_OFFSET} +import org.apache.kafka.common.requests._ +import org.apache.kafka.common.{InvalidRecordException, TopicPartition} +import java.nio.ByteBuffer +import java.util +import java.util.Optional +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicLong +import java.util.concurrent.locks.ReentrantLock +import scala.collection.{Map, Set, mutable} +import scala.compat.java8.OptionConverters._ +import scala.jdk.CollectionConverters._ import scala.math._ /** @@ -62,7 +58,7 @@ abstract class AbstractFetcherThread(name: String, val brokerTopicStats: BrokerTopicStats) //BrokerTopicStats's lifecycle managed by ReplicaManager extends ShutdownableThread(name, isInterruptible) { - type FetchData = FetchResponse.PartitionData[Records] + type FetchData = FetchResponseData.PartitionData type EpochData = OffsetForLeaderEpochRequestData.OffsetForLeaderPartition private val partitionStates = new PartitionStates[PartitionFetchState] @@ -340,7 +336,7 @@ abstract class AbstractFetcherThread(name: String, // the current offset is the same as the offset requested. val fetchPartitionData = sessionPartitions.get(topicPartition) if (fetchPartitionData != null && fetchPartitionData.fetchOffset == currentFetchState.fetchOffset && currentFetchState.isReadyForFetch) { - partitionData.error match { + Errors.forCode(partitionData.errorCode) match { case Errors.NONE => try { // Once we hand off the partition data to the subclass, we can't mess with it any more in this thread @@ -364,7 +360,7 @@ abstract class AbstractFetcherThread(name: String, } } if (isTruncationOnFetchSupported) { - partitionData.divergingEpoch.ifPresent { divergingEpoch => + FetchResponse.divergingEpoch(partitionData).ifPresent { divergingEpoch => divergingEndOffsets += topicPartition -> new EpochEndOffset() .setPartition(topicPartition.partition) .setErrorCode(Errors.NONE.code) @@ -416,9 +412,8 @@ abstract class AbstractFetcherThread(name: String, "expected to persist.") partitionsWithError += topicPartition - case _ => - error(s"Error for partition $topicPartition at offset ${currentFetchState.fetchOffset}", - partitionData.error.exception) + case partitionError => + error(s"Error for partition $topicPartition at offset ${currentFetchState.fetchOffset}", partitionError.exception) partitionsWithError += topicPartition } } diff --git a/core/src/main/scala/kafka/server/ControllerApis.scala b/core/src/main/scala/kafka/server/ControllerApis.scala index 53a41dc919a1a..5670448765b29 100644 --- a/core/src/main/scala/kafka/server/ControllerApis.scala +++ b/core/src/main/scala/kafka/server/ControllerApis.scala @@ -34,7 +34,6 @@ import org.apache.kafka.common.message.CreateTopicsResponseData.CreatableTopicRe import org.apache.kafka.common.message.MetadataResponseData.MetadataResponseBroker import org.apache.kafka.common.message.{BeginQuorumEpochResponseData, BrokerHeartbeatResponseData, BrokerRegistrationResponseData, CreateTopicsResponseData, DescribeQuorumResponseData, EndQuorumEpochResponseData, FetchResponseData, MetadataResponseData, SaslAuthenticateResponseData, SaslHandshakeResponseData, UnregisterBrokerResponseData, VoteResponseData} import org.apache.kafka.common.protocol.{ApiKeys, ApiMessage, Errors} -import org.apache.kafka.common.record.BaseRecords import org.apache.kafka.common.requests._ import org.apache.kafka.common.resource.Resource import org.apache.kafka.common.resource.Resource.CLUSTER_NAME @@ -116,7 +115,7 @@ class ControllerApis(val requestChannel: RequestChannel, private def handleFetch(request: RequestChannel.Request): Unit = { authHelper.authorizeClusterOperation(request, CLUSTER_ACTION) - handleRaftRequest(request, response => new FetchResponse[BaseRecords](response.asInstanceOf[FetchResponseData])) + handleRaftRequest(request, response => new FetchResponse(response.asInstanceOf[FetchResponseData])) } def handleMetadataRequest(request: RequestChannel.Request): Unit = { diff --git a/core/src/main/scala/kafka/server/FetchDataInfo.scala b/core/src/main/scala/kafka/server/FetchDataInfo.scala index a55fe23e1eca6..f6cf725843ef9 100644 --- a/core/src/main/scala/kafka/server/FetchDataInfo.scala +++ b/core/src/main/scala/kafka/server/FetchDataInfo.scala @@ -17,8 +17,8 @@ package kafka.server +import org.apache.kafka.common.message.FetchResponseData import org.apache.kafka.common.record.Records -import org.apache.kafka.common.requests.FetchResponse.AbortedTransaction sealed trait FetchIsolation case object FetchLogEnd extends FetchIsolation @@ -28,4 +28,4 @@ case object FetchTxnCommitted extends FetchIsolation case class FetchDataInfo(fetchOffsetMetadata: LogOffsetMetadata, records: Records, firstEntryIncomplete: Boolean = false, - abortedTransactions: Option[List[AbortedTransaction]] = None) + abortedTransactions: Option[List[FetchResponseData.AbortedTransaction]] = None) diff --git a/core/src/main/scala/kafka/server/FetchSession.scala b/core/src/main/scala/kafka/server/FetchSession.scala index c6280f384dfd7..a6f6fa8e4f72b 100644 --- a/core/src/main/scala/kafka/server/FetchSession.scala +++ b/core/src/main/scala/kafka/server/FetchSession.scala @@ -17,27 +17,26 @@ package kafka.server -import java.util -import java.util.Optional -import java.util.concurrent.{ThreadLocalRandom, TimeUnit} - import kafka.metrics.KafkaMetricsGroup import kafka.utils.Logging import org.apache.kafka.common.TopicPartition +import org.apache.kafka.common.message.FetchResponseData import org.apache.kafka.common.protocol.Errors -import org.apache.kafka.common.record.Records import org.apache.kafka.common.requests.FetchMetadata.{FINAL_EPOCH, INITIAL_EPOCH, INVALID_SESSION_ID} import org.apache.kafka.common.requests.{FetchRequest, FetchResponse, FetchMetadata => JFetchMetadata} import org.apache.kafka.common.utils.{ImplicitLinkedHashCollection, Time, Utils} -import scala.math.Ordered.orderingToOrdered +import java.util +import java.util.Optional +import java.util.concurrent.{ThreadLocalRandom, TimeUnit} import scala.collection.{mutable, _} +import scala.math.Ordered.orderingToOrdered object FetchSession { type REQ_MAP = util.Map[TopicPartition, FetchRequest.PartitionData] - type RESP_MAP = util.LinkedHashMap[TopicPartition, FetchResponse.PartitionData[Records]] + type RESP_MAP = util.LinkedHashMap[TopicPartition, FetchResponseData.PartitionData] type CACHE_MAP = ImplicitLinkedHashCollection[CachedPartition] - type RESP_MAP_ITER = util.Iterator[util.Map.Entry[TopicPartition, FetchResponse.PartitionData[Records]]] + type RESP_MAP_ITER = util.Iterator[util.Map.Entry[TopicPartition, FetchResponseData.PartitionData]] val NUM_INCREMENTAL_FETCH_SESSISONS = "NumIncrementalFetchSessions" val NUM_INCREMENTAL_FETCH_PARTITIONS_CACHED = "NumIncrementalFetchPartitionsCached" @@ -100,7 +99,7 @@ class CachedPartition(val topic: String, reqData.currentLeaderEpoch, reqData.logStartOffset, -1, reqData.lastFetchedEpoch) def this(part: TopicPartition, reqData: FetchRequest.PartitionData, - respData: FetchResponse.PartitionData[Records]) = + respData: FetchResponseData.PartitionData) = this(part.topic, part.partition, reqData.maxBytes, reqData.fetchOffset, respData.highWatermark, reqData.currentLeaderEpoch, reqData.logStartOffset, respData.logStartOffset, reqData.lastFetchedEpoch) @@ -125,10 +124,10 @@ class CachedPartition(val topic: String, * @param updateResponseData if set to true, update this CachedPartition with new request and response data. * @return True if this partition should be included in the response; false if it can be omitted. */ - def maybeUpdateResponseData(respData: FetchResponse.PartitionData[Records], updateResponseData: Boolean): Boolean = { + def maybeUpdateResponseData(respData: FetchResponseData.PartitionData, updateResponseData: Boolean): Boolean = { // Check the response data. var mustRespond = false - if ((respData.records != null) && (respData.records.sizeInBytes > 0)) { + if (FetchResponse.recordsSize(respData) > 0) { // Partitions with new data are always included in the response. mustRespond = true } @@ -142,11 +141,11 @@ class CachedPartition(val topic: String, if (updateResponseData) localLogStartOffset = respData.logStartOffset } - if (respData.preferredReadReplica.isPresent) { + if (FetchResponse.isPreferredReplica(respData)) { // If the broker computed a preferred read replica, we need to include it in the response mustRespond = true } - if (respData.error.code != 0) { + if (respData.errorCode != Errors.NONE.code) { // Partitions with errors are always included in the response. // We also set the cached highWatermark to an invalid offset, -1. // This ensures that when the error goes away, we re-send the partition. @@ -154,7 +153,8 @@ class CachedPartition(val topic: String, highWatermark = -1 mustRespond = true } - if (respData.divergingEpoch.isPresent) { + + if (FetchResponse.isDivergingEpoch(respData)) { // Partitions with diverging epoch are always included in response to trigger truncation. mustRespond = true } @@ -163,7 +163,7 @@ class CachedPartition(val topic: String, override def hashCode: Int = (31 * partition) + topic.hashCode - def canEqual(that: Any) = that.isInstanceOf[CachedPartition] + def canEqual(that: Any): Boolean = that.isInstanceOf[CachedPartition] override def equals(that: Any): Boolean = that match { @@ -292,7 +292,7 @@ trait FetchContext extends Logging { * Updates the fetch context with new partition information. Generates response data. * The response data may require subsequent down-conversion. */ - def updateAndGenerateResponseData(updates: FetchSession.RESP_MAP): FetchResponse[Records] + def updateAndGenerateResponseData(updates: FetchSession.RESP_MAP): FetchResponse def partitionsToLogString(partitions: util.Collection[TopicPartition]): String = FetchSession.partitionsToLogString(partitions, isTraceEnabled) @@ -300,8 +300,8 @@ trait FetchContext extends Logging { /** * Return an empty throttled response due to quota violation. */ - def getThrottledResponse(throttleTimeMs: Int): FetchResponse[Records] = - new FetchResponse(Errors.NONE, new FetchSession.RESP_MAP, throttleTimeMs, INVALID_SESSION_ID) + def getThrottledResponse(throttleTimeMs: Int): FetchResponse = + FetchResponse.of(Errors.NONE, throttleTimeMs, INVALID_SESSION_ID, new FetchSession.RESP_MAP) } /** @@ -318,9 +318,9 @@ class SessionErrorContext(val error: Errors, } // Because of the fetch session error, we don't know what partitions were supposed to be in this request. - override def updateAndGenerateResponseData(updates: FetchSession.RESP_MAP): FetchResponse[Records] = { + override def updateAndGenerateResponseData(updates: FetchSession.RESP_MAP): FetchResponse = { debug(s"Session error fetch context returning $error") - new FetchResponse(error, new FetchSession.RESP_MAP, 0, INVALID_SESSION_ID) + FetchResponse.of(error, 0, INVALID_SESSION_ID, new FetchSession.RESP_MAP) } } @@ -341,9 +341,9 @@ class SessionlessFetchContext(val fetchData: util.Map[TopicPartition, FetchReque FetchResponse.sizeOf(versionId, updates.entrySet.iterator) } - override def updateAndGenerateResponseData(updates: FetchSession.RESP_MAP): FetchResponse[Records] = { + override def updateAndGenerateResponseData(updates: FetchSession.RESP_MAP): FetchResponse = { debug(s"Sessionless fetch context returning ${partitionsToLogString(updates.keySet)}") - new FetchResponse(Errors.NONE, updates, 0, INVALID_SESSION_ID) + FetchResponse.of(Errors.NONE, 0, INVALID_SESSION_ID, updates) } } @@ -372,7 +372,7 @@ class FullFetchContext(private val time: Time, FetchResponse.sizeOf(versionId, updates.entrySet.iterator) } - override def updateAndGenerateResponseData(updates: FetchSession.RESP_MAP): FetchResponse[Records] = { + override def updateAndGenerateResponseData(updates: FetchSession.RESP_MAP): FetchResponse = { def createNewSession: FetchSession.CACHE_MAP = { val cachedPartitions = new FetchSession.CACHE_MAP(updates.size) updates.forEach { (part, respData) => @@ -385,7 +385,7 @@ class FullFetchContext(private val time: Time, updates.size, () => createNewSession) debug(s"Full fetch context with session id $responseSessionId returning " + s"${partitionsToLogString(updates.keySet)}") - new FetchResponse(Errors.NONE, updates, 0, responseSessionId) + FetchResponse.of(Errors.NONE, 0, responseSessionId, updates) } } @@ -417,7 +417,7 @@ class IncrementalFetchContext(private val time: Time, private class PartitionIterator(val iter: FetchSession.RESP_MAP_ITER, val updateFetchContextAndRemoveUnselected: Boolean) extends FetchSession.RESP_MAP_ITER { - var nextElement: util.Map.Entry[TopicPartition, FetchResponse.PartitionData[Records]] = null + var nextElement: util.Map.Entry[TopicPartition, FetchResponseData.PartitionData] = null override def hasNext: Boolean = { while ((nextElement == null) && iter.hasNext) { @@ -441,7 +441,7 @@ class IncrementalFetchContext(private val time: Time, nextElement != null } - override def next(): util.Map.Entry[TopicPartition, FetchResponse.PartitionData[Records]] = { + override def next(): util.Map.Entry[TopicPartition, FetchResponseData.PartitionData] = { if (!hasNext) throw new NoSuchElementException val element = nextElement nextElement = null @@ -463,7 +463,7 @@ class IncrementalFetchContext(private val time: Time, } } - override def updateAndGenerateResponseData(updates: FetchSession.RESP_MAP): FetchResponse[Records] = { + override def updateAndGenerateResponseData(updates: FetchSession.RESP_MAP): FetchResponse = { session.synchronized { // Check to make sure that the session epoch didn't change in between // creating this fetch context and generating this response. @@ -471,7 +471,7 @@ class IncrementalFetchContext(private val time: Time, if (session.epoch != expectedEpoch) { info(s"Incremental fetch session ${session.id} expected epoch $expectedEpoch, but " + s"got ${session.epoch}. Possible duplicate request.") - new FetchResponse(Errors.INVALID_FETCH_SESSION_EPOCH, new FetchSession.RESP_MAP, 0, session.id) + FetchResponse.of(Errors.INVALID_FETCH_SESSION_EPOCH, 0, session.id, new FetchSession.RESP_MAP) } else { // Iterate over the update list using PartitionIterator. This will prune updates which don't need to be sent val partitionIter = new PartitionIterator(updates.entrySet.iterator, true) @@ -480,12 +480,12 @@ class IncrementalFetchContext(private val time: Time, } debug(s"Incremental fetch context with session id ${session.id} returning " + s"${partitionsToLogString(updates.keySet)}") - new FetchResponse(Errors.NONE, updates, 0, session.id) + FetchResponse.of(Errors.NONE, 0, session.id, updates) } } } - override def getThrottledResponse(throttleTimeMs: Int): FetchResponse[Records] = { + override def getThrottledResponse(throttleTimeMs: Int): FetchResponse = { session.synchronized { // Check to make sure that the session epoch didn't change in between // creating this fetch context and generating this response. @@ -493,9 +493,9 @@ class IncrementalFetchContext(private val time: Time, if (session.epoch != expectedEpoch) { info(s"Incremental fetch session ${session.id} expected epoch $expectedEpoch, but " + s"got ${session.epoch}. Possible duplicate request.") - new FetchResponse(Errors.INVALID_FETCH_SESSION_EPOCH, new FetchSession.RESP_MAP, throttleTimeMs, session.id) + FetchResponse.of(Errors.INVALID_FETCH_SESSION_EPOCH, throttleTimeMs, session.id, new FetchSession.RESP_MAP) } else { - new FetchResponse(Errors.NONE, new FetchSession.RESP_MAP, throttleTimeMs, session.id) + FetchResponse.of(Errors.NONE, throttleTimeMs, session.id, new FetchSession.RESP_MAP) } } } diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala index 22054a3f57388..de7bb0b58fe10 100644 --- a/core/src/main/scala/kafka/server/KafkaApis.scala +++ b/core/src/main/scala/kafka/server/KafkaApis.scala @@ -17,24 +17,18 @@ package kafka.server -import java.lang.{Long => JLong} -import java.nio.ByteBuffer -import java.util -import java.util.concurrent.ConcurrentHashMap -import java.util.concurrent.atomic.AtomicInteger -import java.util.{Collections, Optional} - import kafka.admin.AdminUtils import kafka.api.{ApiVersion, ElectLeadersRequestOps, KAFKA_0_11_0_IV0, KAFKA_2_3_IV0} import kafka.common.OffsetAndMetadata import kafka.controller.ReplicaAssignment -import kafka.coordinator.group.{GroupCoordinator, JoinGroupResult, LeaveGroupResult, SyncGroupResult} +import kafka.coordinator.group._ import kafka.coordinator.transaction.{InitProducerIdResult, TransactionCoordinator} import kafka.log.AppendOrigin import kafka.message.ZStdCompressionCodec import kafka.network.RequestChannel import kafka.security.authorizer.AuthorizerUtils import kafka.server.QuotaFactory.{QuotaManagers, UnboundedQuota} +import kafka.server.metadata.ConfigRepository import kafka.utils.Implicits._ import kafka.utils.{CoreUtils, Logging} import org.apache.kafka.clients.admin.AlterConfigOp.OpType @@ -61,7 +55,7 @@ import org.apache.kafka.common.message.ListOffsetsResponseData.{ListOffsetsParti import org.apache.kafka.common.message.MetadataResponseData.{MetadataResponsePartition, MetadataResponseTopic} import org.apache.kafka.common.message.OffsetForLeaderEpochRequestData.OffsetForLeaderTopic import org.apache.kafka.common.message.OffsetForLeaderEpochResponseData.{EpochEndOffset, OffsetForLeaderTopicResult, OffsetForLeaderTopicResultCollection} -import org.apache.kafka.common.message.{AddOffsetsToTxnResponseData, AlterClientQuotasResponseData, AlterConfigsResponseData, AlterPartitionReassignmentsResponseData, AlterReplicaLogDirsResponseData, CreateAclsResponseData, CreatePartitionsResponseData, CreateTopicsResponseData, DeleteAclsResponseData, DeleteGroupsResponseData, DeleteRecordsResponseData, DeleteTopicsResponseData, DescribeAclsResponseData, DescribeClientQuotasResponseData, DescribeClusterResponseData, DescribeConfigsResponseData, DescribeGroupsResponseData, DescribeLogDirsResponseData, DescribeProducersResponseData, DescribeTransactionsResponseData, EndTxnResponseData, ExpireDelegationTokenResponseData, FindCoordinatorResponseData, HeartbeatResponseData, InitProducerIdResponseData, JoinGroupResponseData, LeaveGroupResponseData, ListGroupsResponseData, ListOffsetsResponseData, ListPartitionReassignmentsResponseData, OffsetCommitRequestData, OffsetCommitResponseData, OffsetDeleteResponseData, OffsetForLeaderEpochResponseData, RenewDelegationTokenResponseData, SaslAuthenticateResponseData, SaslHandshakeResponseData, StopReplicaResponseData, SyncGroupResponseData, UpdateMetadataResponseData} +import org.apache.kafka.common.message._ import org.apache.kafka.common.metrics.Metrics import org.apache.kafka.common.network.{ListenerName, Send} import org.apache.kafka.common.protocol.{ApiKeys, Errors} @@ -80,14 +74,18 @@ import org.apache.kafka.common.utils.{ProducerIdAndEpoch, Time} import org.apache.kafka.common.{Node, TopicPartition, Uuid} import org.apache.kafka.server.authorizer._ +import java.lang.{Long => JLong} +import java.nio.ByteBuffer +import java.util +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicInteger +import java.util.{Collections, Optional} import scala.annotation.nowarn import scala.collection.mutable.ArrayBuffer import scala.collection.{Map, Seq, Set, immutable, mutable} import scala.compat.java8.OptionConverters._ import scala.jdk.CollectionConverters._ import scala.util.{Failure, Success, Try} -import kafka.coordinator.group.GroupOverview -import kafka.server.metadata.ConfigRepository /** * Logic to handle the various Kafka requests @@ -681,25 +679,20 @@ class KafkaApis(val requestChannel: RequestChannel, None } - def errorResponse[T >: MemoryRecords <: BaseRecords](error: Errors): FetchResponse.PartitionData[T] = { - new FetchResponse.PartitionData[T](error, FetchResponse.INVALID_HIGHWATERMARK, FetchResponse.INVALID_LAST_STABLE_OFFSET, - FetchResponse.INVALID_LOG_START_OFFSET, null, MemoryRecords.EMPTY) - } - - val erroneous = mutable.ArrayBuffer[(TopicPartition, FetchResponse.PartitionData[Records])]() + val erroneous = mutable.ArrayBuffer[(TopicPartition, FetchResponseData.PartitionData)]() val interesting = mutable.ArrayBuffer[(TopicPartition, FetchRequest.PartitionData)]() if (fetchRequest.isFromFollower) { // The follower must have ClusterAction on ClusterResource in order to fetch partition data. if (authHelper.authorize(request.context, CLUSTER_ACTION, CLUSTER, CLUSTER_NAME)) { fetchContext.foreachPartition { (topicPartition, data) => if (!metadataCache.contains(topicPartition)) - erroneous += topicPartition -> errorResponse(Errors.UNKNOWN_TOPIC_OR_PARTITION) + erroneous += topicPartition -> FetchResponse.partitionResponse(topicPartition.partition, Errors.UNKNOWN_TOPIC_OR_PARTITION) else interesting += (topicPartition -> data) } } else { fetchContext.foreachPartition { (part, _) => - erroneous += part -> errorResponse(Errors.TOPIC_AUTHORIZATION_FAILED) + erroneous += part -> FetchResponse.partitionResponse(part.partition, Errors.TOPIC_AUTHORIZATION_FAILED) } } } else { @@ -711,9 +704,9 @@ class KafkaApis(val requestChannel: RequestChannel, val authorizedTopics = authHelper.filterByAuthorized(request.context, READ, TOPIC, partitionDatas)(_._1.topic) partitionDatas.foreach { case (topicPartition, data) => if (!authorizedTopics.contains(topicPartition.topic)) - erroneous += topicPartition -> errorResponse(Errors.TOPIC_AUTHORIZATION_FAILED) + erroneous += topicPartition -> FetchResponse.partitionResponse(topicPartition.partition, Errors.TOPIC_AUTHORIZATION_FAILED) else if (!metadataCache.contains(topicPartition)) - erroneous += topicPartition -> errorResponse(Errors.UNKNOWN_TOPIC_OR_PARTITION) + erroneous += topicPartition -> FetchResponse.partitionResponse(topicPartition.partition, Errors.UNKNOWN_TOPIC_OR_PARTITION) else interesting += (topicPartition -> data) } @@ -732,12 +725,12 @@ class KafkaApis(val requestChannel: RequestChannel, } def maybeConvertFetchedData(tp: TopicPartition, - partitionData: FetchResponse.PartitionData[Records]): FetchResponse.PartitionData[BaseRecords] = { + partitionData: FetchResponseData.PartitionData): FetchResponseData.PartitionData = { val logConfig = replicaManager.getLogConfig(tp) if (logConfig.exists(_.compressionType == ZStdCompressionCodec.name) && versionId < 10) { trace(s"Fetching messages is disabled for ZStandard compressed partition $tp. Sending unsupported version response to $clientId.") - errorResponse(Errors.UNSUPPORTED_COMPRESSION_TYPE) + FetchResponse.partitionResponse(tp.partition, Errors.UNSUPPORTED_COMPRESSION_TYPE) } else { // Down-conversion of the fetched records is needed when the stored magic version is // greater than that supported by the client (as indicated by the fetch request version). If the @@ -746,7 +739,7 @@ class KafkaApis(val requestChannel: RequestChannel, // know it must be supported. However, if the magic version is changed from a higher version back to a // lower version, this check will no longer be valid and we will fail to down-convert the messages // which were written in the new format prior to the version downgrade. - val unconvertedRecords = partitionData.records + val unconvertedRecords = FetchResponse.recordsOrFail(partitionData) val downConvertMagic = logConfig.map(_.messageFormatVersion.recordVersion.value).flatMap { magic => if (magic > RecordBatch.MAGIC_VALUE_V0 && versionId <= 1 && !unconvertedRecords.hasCompatibleMagic(RecordBatch.MAGIC_VALUE_V0)) @@ -762,7 +755,7 @@ class KafkaApis(val requestChannel: RequestChannel, // For fetch requests from clients, check if down-conversion is disabled for the particular partition if (!fetchRequest.isFromFollower && !logConfig.forall(_.messageDownConversionEnable)) { trace(s"Conversion to message format ${downConvertMagic.get} is disabled for partition $tp. Sending unsupported version response to $clientId.") - errorResponse(Errors.UNSUPPORTED_VERSION) + FetchResponse.partitionResponse(tp.partition, Errors.UNSUPPORTED_VERSION) } else { try { trace(s"Down converting records from partition $tp to message format version $magic for fetch request from $clientId") @@ -770,71 +763,77 @@ class KafkaApis(val requestChannel: RequestChannel, // as possible. With KIP-283, we have the ability to lazily down-convert in a chunked manner. The lazy, chunked // down-conversion always guarantees that at least one batch of messages is down-converted and sent out to the // client. - val error = maybeDownConvertStorageError(partitionData.error) - new FetchResponse.PartitionData[BaseRecords](error, partitionData.highWatermark, - partitionData.lastStableOffset, partitionData.logStartOffset, - partitionData.preferredReadReplica, partitionData.abortedTransactions, - new LazyDownConversionRecords(tp, unconvertedRecords, magic, fetchContext.getFetchOffset(tp).get, time)) + new FetchResponseData.PartitionData() + .setPartitionIndex(tp.partition) + .setErrorCode(maybeDownConvertStorageError(Errors.forCode(partitionData.errorCode)).code) + .setHighWatermark(partitionData.highWatermark) + .setLastStableOffset(partitionData.lastStableOffset) + .setLogStartOffset(partitionData.logStartOffset) + .setAbortedTransactions(partitionData.abortedTransactions) + .setRecords(new LazyDownConversionRecords(tp, unconvertedRecords, magic, fetchContext.getFetchOffset(tp).get, time)) + .setPreferredReadReplica(partitionData.preferredReadReplica()) } catch { case e: UnsupportedCompressionTypeException => trace("Received unsupported compression type error during down-conversion", e) - errorResponse(Errors.UNSUPPORTED_COMPRESSION_TYPE) + FetchResponse.partitionResponse(tp.partition, Errors.UNSUPPORTED_COMPRESSION_TYPE) } } case None => - val error = maybeDownConvertStorageError(partitionData.error) - new FetchResponse.PartitionData[BaseRecords](error, - partitionData.highWatermark, - partitionData.lastStableOffset, - partitionData.logStartOffset, - partitionData.preferredReadReplica, - partitionData.abortedTransactions, - partitionData.divergingEpoch, - unconvertedRecords) + new FetchResponseData.PartitionData() + .setPartitionIndex(tp.partition) + .setErrorCode(maybeDownConvertStorageError(Errors.forCode(partitionData.errorCode)).code) + .setHighWatermark(partitionData.highWatermark) + .setLastStableOffset(partitionData.lastStableOffset) + .setLogStartOffset(partitionData.logStartOffset) + .setAbortedTransactions(partitionData.abortedTransactions) + .setRecords(unconvertedRecords) + .setPreferredReadReplica(partitionData.preferredReadReplica) + .setDivergingEpoch(partitionData.divergingEpoch) } } } // the callback for process a fetch response, invoked before throttling def processResponseCallback(responsePartitionData: Seq[(TopicPartition, FetchPartitionData)]): Unit = { - val partitions = new util.LinkedHashMap[TopicPartition, FetchResponse.PartitionData[Records]] + val partitions = new util.LinkedHashMap[TopicPartition, FetchResponseData.PartitionData] val reassigningPartitions = mutable.Set[TopicPartition]() responsePartitionData.foreach { case (tp, data) => val abortedTransactions = data.abortedTransactions.map(_.asJava).orNull val lastStableOffset = data.lastStableOffset.getOrElse(FetchResponse.INVALID_LAST_STABLE_OFFSET) - if (data.isReassignmentFetch) - reassigningPartitions.add(tp) - val error = maybeDownConvertStorageError(data.error) - partitions.put(tp, new FetchResponse.PartitionData( - error, - data.highWatermark, - lastStableOffset, - data.logStartOffset, - data.preferredReadReplica.map(int2Integer).asJava, - abortedTransactions, - data.divergingEpoch.asJava, - data.records)) + if (data.isReassignmentFetch) reassigningPartitions.add(tp) + val partitionData = new FetchResponseData.PartitionData() + .setPartitionIndex(tp.partition) + .setErrorCode(maybeDownConvertStorageError(data.error).code) + .setHighWatermark(data.highWatermark) + .setLastStableOffset(lastStableOffset) + .setLogStartOffset(data.logStartOffset) + .setAbortedTransactions(abortedTransactions) + .setRecords(data.records) + .setPreferredReadReplica(data.preferredReadReplica.getOrElse(FetchResponse.INVALID_PREFERRED_REPLICA_ID)) + data.divergingEpoch.foreach(partitionData.setDivergingEpoch) + partitions.put(tp, partitionData) } erroneous.foreach { case (tp, data) => partitions.put(tp, data) } - var unconvertedFetchResponse: FetchResponse[Records] = null + var unconvertedFetchResponse: FetchResponse = null - def createResponse(throttleTimeMs: Int): FetchResponse[BaseRecords] = { + def createResponse(throttleTimeMs: Int): FetchResponse = { // Down-convert messages for each partition if required - val convertedData = new util.LinkedHashMap[TopicPartition, FetchResponse.PartitionData[BaseRecords]] + val convertedData = new util.LinkedHashMap[TopicPartition, FetchResponseData.PartitionData] unconvertedFetchResponse.responseData.forEach { (tp, unconvertedPartitionData) => - if (unconvertedPartitionData.error != Errors.NONE) + val error = Errors.forCode(unconvertedPartitionData.errorCode) + if (error != Errors.NONE) debug(s"Fetch request with correlation id ${request.header.correlationId} from client $clientId " + - s"on partition $tp failed due to ${unconvertedPartitionData.error.exceptionName}") + s"on partition $tp failed due to ${error.exceptionName}") convertedData.put(tp, maybeConvertFetchedData(tp, unconvertedPartitionData)) } // Prepare fetch response from converted data - val response = new FetchResponse(unconvertedFetchResponse.error, convertedData, throttleTimeMs, - unconvertedFetchResponse.sessionId) + val response = FetchResponse.of(unconvertedFetchResponse.error, throttleTimeMs, unconvertedFetchResponse.sessionId, convertedData) // record the bytes out metrics only when the response is being sent response.responseData.forEach { (tp, data) => - brokerTopicStats.updateBytesOut(tp.topic, fetchRequest.isFromFollower, reassigningPartitions.contains(tp), data.records.sizeInBytes) + brokerTopicStats.updateBytesOut(tp.topic, fetchRequest.isFromFollower, + reassigningPartitions.contains(tp), FetchResponse.recordsSize(data)) } response } @@ -3367,7 +3366,7 @@ object KafkaApis { // Traffic from both in-sync and out of sync replicas are accounted for in replication quota to ensure total replication // traffic doesn't exceed quota. private[server] def sizeOfThrottledPartitions(versionId: Short, - unconvertedResponse: FetchResponse[Records], + unconvertedResponse: FetchResponse, quota: ReplicationQuotaManager): Int = { FetchResponse.sizeOf(versionId, unconvertedResponse.responseData.entrySet .iterator.asScala.filter(element => quota.isThrottled(element.getKey)).asJava) diff --git a/core/src/main/scala/kafka/server/ReplicaAlterLogDirsThread.scala b/core/src/main/scala/kafka/server/ReplicaAlterLogDirsThread.scala index dae3fc226d15c..f12ae9e4ca2f3 100644 --- a/core/src/main/scala/kafka/server/ReplicaAlterLogDirsThread.scala +++ b/core/src/main/scala/kafka/server/ReplicaAlterLogDirsThread.scala @@ -17,28 +17,24 @@ package kafka.server -import java.util -import java.util.Optional - import kafka.api.Request import kafka.cluster.BrokerEndPoint import kafka.log.{LeaderOffsetIncremented, LogAppendInfo} -import kafka.server.AbstractFetcherThread.ReplicaFetch -import kafka.server.AbstractFetcherThread.ResultWithPartitions +import kafka.server.AbstractFetcherThread.{ReplicaFetch, ResultWithPartitions} import kafka.server.QuotaFactory.UnboundedQuota import org.apache.kafka.common.TopicPartition import org.apache.kafka.common.errors.KafkaStorageException +import org.apache.kafka.common.message.FetchResponseData import org.apache.kafka.common.message.OffsetForLeaderEpochResponseData.EpochEndOffset import org.apache.kafka.common.protocol.{ApiKeys, Errors} -import org.apache.kafka.common.record.Records -import org.apache.kafka.common.requests.FetchResponse.PartitionData -import org.apache.kafka.common.requests.{FetchRequest, FetchResponse} import org.apache.kafka.common.requests.OffsetsForLeaderEpochResponse.UNDEFINED_EPOCH -import org.apache.kafka.common.requests.RequestUtils +import org.apache.kafka.common.requests.{FetchRequest, FetchResponse, RequestUtils} -import scala.jdk.CollectionConverters._ +import java.util +import java.util.Optional import scala.collection.{Map, Seq, Set, mutable} import scala.compat.java8.OptionConverters._ +import scala.jdk.CollectionConverters._ class ReplicaAlterLogDirsThread(name: String, sourceBroker: BrokerEndPoint, @@ -77,15 +73,21 @@ class ReplicaAlterLogDirsThread(name: String, } def fetchFromLeader(fetchRequest: FetchRequest.Builder): Map[TopicPartition, FetchData] = { - var partitionData: Seq[(TopicPartition, FetchResponse.PartitionData[Records])] = null + var partitionData: Seq[(TopicPartition, FetchData)] = null val request = fetchRequest.build() def processResponseCallback(responsePartitionData: Seq[(TopicPartition, FetchPartitionData)]): Unit = { partitionData = responsePartitionData.map { case (tp, data) => val abortedTransactions = data.abortedTransactions.map(_.asJava).orNull val lastStableOffset = data.lastStableOffset.getOrElse(FetchResponse.INVALID_LAST_STABLE_OFFSET) - tp -> new FetchResponse.PartitionData(data.error, data.highWatermark, lastStableOffset, - data.logStartOffset, abortedTransactions, data.records) + tp -> new FetchResponseData.PartitionData() + .setPartitionIndex(tp.partition) + .setErrorCode(data.error.code) + .setHighWatermark(data.highWatermark) + .setLastStableOffset(lastStableOffset) + .setLogStartOffset(data.logStartOffset) + .setAbortedTransactions(abortedTransactions) + .setRecords(data.records) } } @@ -110,10 +112,10 @@ class ReplicaAlterLogDirsThread(name: String, // process fetched data override def processPartitionData(topicPartition: TopicPartition, fetchOffset: Long, - partitionData: PartitionData[Records]): Option[LogAppendInfo] = { + partitionData: FetchData): Option[LogAppendInfo] = { val partition = replicaMgr.getPartitionOrException(topicPartition) val futureLog = partition.futureLocalLogOrException - val records = toMemoryRecords(partitionData.records) + val records = toMemoryRecords(FetchResponse.recordsOrFail(partitionData)) if (fetchOffset != futureLog.logEndOffset) throw new IllegalStateException("Offset mismatch for the future replica %s: fetched offset = %d, log end offset = %d.".format( diff --git a/core/src/main/scala/kafka/server/ReplicaFetcherThread.scala b/core/src/main/scala/kafka/server/ReplicaFetcherThread.scala index 103fdb81eefbc..170b5b9a889d5 100644 --- a/core/src/main/scala/kafka/server/ReplicaFetcherThread.scala +++ b/core/src/main/scala/kafka/server/ReplicaFetcherThread.scala @@ -35,7 +35,7 @@ import org.apache.kafka.common.message.OffsetForLeaderEpochRequestData.OffsetFor import org.apache.kafka.common.message.OffsetForLeaderEpochResponseData.EpochEndOffset import org.apache.kafka.common.metrics.Metrics import org.apache.kafka.common.protocol.Errors -import org.apache.kafka.common.record.{MemoryRecords, Records} +import org.apache.kafka.common.record.MemoryRecords import org.apache.kafka.common.requests._ import org.apache.kafka.common.utils.{LogContext, Time} @@ -162,7 +162,7 @@ class ReplicaFetcherThread(name: String, val logTrace = isTraceEnabled val partition = replicaMgr.getPartitionOrException(topicPartition) val log = partition.localLogOrException - val records = toMemoryRecords(partitionData.records) + val records = toMemoryRecords(FetchResponse.recordsOrFail(partitionData)) maybeWarnIfOversizedRecords(records, topicPartition) @@ -215,7 +215,7 @@ class ReplicaFetcherThread(name: String, override protected def fetchFromLeader(fetchRequest: FetchRequest.Builder): Map[TopicPartition, FetchData] = { try { val clientResponse = leaderEndpoint.sendRequest(fetchRequest) - val fetchResponse = clientResponse.responseBody.asInstanceOf[FetchResponse[Records]] + val fetchResponse = clientResponse.responseBody.asInstanceOf[FetchResponse] if (!fetchSessionHandler.handleResponse(fetchResponse)) { Map.empty } else { diff --git a/core/src/main/scala/kafka/server/ReplicaManager.scala b/core/src/main/scala/kafka/server/ReplicaManager.scala index 87b446208536c..069059b335e79 100644 --- a/core/src/main/scala/kafka/server/ReplicaManager.scala +++ b/core/src/main/scala/kafka/server/ReplicaManager.scala @@ -57,7 +57,6 @@ import org.apache.kafka.common.replica.PartitionView.DefaultPartitionView import org.apache.kafka.common.replica.ReplicaView.DefaultReplicaView import org.apache.kafka.common.replica.{ClientMetadata, _} import org.apache.kafka.common.requests.FetchRequest.PartitionData -import org.apache.kafka.common.requests.FetchResponse.AbortedTransaction import org.apache.kafka.common.requests.ProduceResponse.PartitionResponse import org.apache.kafka.common.requests._ import org.apache.kafka.common.utils.Time @@ -150,7 +149,7 @@ case class FetchPartitionData(error: Errors = Errors.NONE, records: Records, divergingEpoch: Option[FetchResponseData.EpochEndOffset], lastStableOffset: Option[Long], - abortedTransactions: Option[List[AbortedTransaction]], + abortedTransactions: Option[List[FetchResponseData.AbortedTransaction]], preferredReadReplica: Option[Int], isReassignmentFetch: Boolean) diff --git a/core/src/main/scala/kafka/tools/ReplicaVerificationTool.scala b/core/src/main/scala/kafka/tools/ReplicaVerificationTool.scala index a00753eb273f7..9dd1b05b5c1fe 100644 --- a/core/src/main/scala/kafka/tools/ReplicaVerificationTool.scala +++ b/core/src/main/scala/kafka/tools/ReplicaVerificationTool.scala @@ -17,33 +17,31 @@ package kafka.tools -import java.net.SocketTimeoutException -import java.text.SimpleDateFormat -import java.util -import java.util.concurrent.CountDownLatch -import java.util.concurrent.atomic.{AtomicInteger, AtomicReference} -import java.util.regex.{Pattern, PatternSyntaxException} -import java.util.{Date, Optional, Properties} - import joptsimple.OptionParser import kafka.api._ -import kafka.utils.IncludeList -import kafka.utils._ +import kafka.utils.{IncludeList, _} import org.apache.kafka.clients._ import org.apache.kafka.clients.admin.{Admin, ListTopicsOptions, TopicDescription} import org.apache.kafka.clients.consumer.{ConsumerConfig, KafkaConsumer} +import org.apache.kafka.common.message.FetchResponseData import org.apache.kafka.common.metrics.Metrics import org.apache.kafka.common.network.{NetworkReceive, Selectable, Selector} import org.apache.kafka.common.protocol.{ApiKeys, Errors} -import org.apache.kafka.common.record.MemoryRecords import org.apache.kafka.common.requests.AbstractRequest.Builder import org.apache.kafka.common.requests.{AbstractRequest, FetchResponse, ListOffsetsRequest, FetchRequest => JFetchRequest} import org.apache.kafka.common.serialization.StringDeserializer import org.apache.kafka.common.utils.{LogContext, Time} import org.apache.kafka.common.{Node, TopicPartition} -import scala.jdk.CollectionConverters._ +import java.net.SocketTimeoutException +import java.text.SimpleDateFormat +import java.util +import java.util.concurrent.CountDownLatch +import java.util.concurrent.atomic.{AtomicInteger, AtomicReference} +import java.util.regex.{Pattern, PatternSyntaxException} +import java.util.{Date, Optional, Properties} import scala.collection.Seq +import scala.jdk.CollectionConverters._ /** * For verifying the consistency among replicas. @@ -261,7 +259,7 @@ private class ReplicaBuffer(expectedReplicasPerTopicPartition: collection.Map[To expectedNumFetchers: Int, reportInterval: Long) extends Logging { private val fetchOffsetMap = new Pool[TopicPartition, Long] - private val recordsCache = new Pool[TopicPartition, Pool[Int, FetchResponse.PartitionData[MemoryRecords]]] + private val recordsCache = new Pool[TopicPartition, Pool[Int, FetchResponseData.PartitionData]] private val fetcherBarrier = new AtomicReference(new CountDownLatch(expectedNumFetchers)) private val verificationBarrier = new AtomicReference(new CountDownLatch(1)) @volatile private var lastReportTime = Time.SYSTEM.milliseconds @@ -284,7 +282,7 @@ private class ReplicaBuffer(expectedReplicasPerTopicPartition: collection.Map[To private def initialize(): Unit = { for (topicPartition <- expectedReplicasPerTopicPartition.keySet) - recordsCache.put(topicPartition, new Pool[Int, FetchResponse.PartitionData[MemoryRecords]]) + recordsCache.put(topicPartition, new Pool[Int, FetchResponseData.PartitionData]) setInitialOffsets() } @@ -294,7 +292,7 @@ private class ReplicaBuffer(expectedReplicasPerTopicPartition: collection.Map[To fetchOffsetMap.put(tp, offset) } - def addFetchedData(topicAndPartition: TopicPartition, replicaId: Int, partitionData: FetchResponse.PartitionData[MemoryRecords]): Unit = { + def addFetchedData(topicAndPartition: TopicPartition, replicaId: Int, partitionData: FetchResponseData.PartitionData): Unit = { recordsCache.get(topicAndPartition).put(replicaId, partitionData) } @@ -311,7 +309,7 @@ private class ReplicaBuffer(expectedReplicasPerTopicPartition: collection.Map[To "fetched " + fetchResponsePerReplica.size + " replicas for " + topicPartition + ", but expected " + expectedReplicasPerTopicPartition(topicPartition) + " replicas") val recordBatchIteratorMap = fetchResponsePerReplica.map { case (replicaId, fetchResponse) => - replicaId -> fetchResponse.records.batches.iterator + replicaId -> FetchResponse.recordsOrFail(fetchResponse).batches.iterator } val maxHw = fetchResponsePerReplica.values.map(_.highWatermark).max @@ -403,10 +401,10 @@ private class ReplicaFetcher(name: String, sourceBroker: Node, topicPartitions: debug("Issuing fetch request ") - var fetchResponse: FetchResponse[MemoryRecords] = null + var fetchResponse: FetchResponse = null try { val clientResponse = fetchEndpoint.sendRequest(fetchRequestBuilder) - fetchResponse = clientResponse.responseBody.asInstanceOf[FetchResponse[MemoryRecords]] + fetchResponse = clientResponse.responseBody.asInstanceOf[FetchResponse] } catch { case t: Throwable => if (!isRunning) @@ -414,14 +412,13 @@ private class ReplicaFetcher(name: String, sourceBroker: Node, topicPartitions: } if (fetchResponse != null) { - fetchResponse.responseData.forEach { (tp, partitionData) => - replicaBuffer.addFetchedData(tp, sourceBroker.id, partitionData) - } + fetchResponse.data.responses.forEach(topicResponse => + topicResponse.partitions.forEach(partitionResponse => + replicaBuffer.addFetchedData(new TopicPartition(topicResponse.topic, partitionResponse.partitionIndex), + sourceBroker.id, partitionResponse))) } else { - val emptyResponse = new FetchResponse.PartitionData(Errors.NONE, FetchResponse.INVALID_HIGHWATERMARK, - FetchResponse.INVALID_LAST_STABLE_OFFSET, FetchResponse.INVALID_LOG_START_OFFSET, null, MemoryRecords.EMPTY) for (topicAndPartition <- topicPartitions) - replicaBuffer.addFetchedData(topicAndPartition, sourceBroker.id, emptyResponse) + replicaBuffer.addFetchedData(topicAndPartition, sourceBroker.id, FetchResponse.partitionResponse(topicAndPartition.partition, Errors.NONE)) } fetcherBarrier.countDown() diff --git a/core/src/main/scala/kafka/tools/TestRaftRequestHandler.scala b/core/src/main/scala/kafka/tools/TestRaftRequestHandler.scala index db825ff7d91d2..91d5ecd3997ac 100644 --- a/core/src/main/scala/kafka/tools/TestRaftRequestHandler.scala +++ b/core/src/main/scala/kafka/tools/TestRaftRequestHandler.scala @@ -24,7 +24,6 @@ import kafka.utils.Logging import org.apache.kafka.common.internals.FatalExitError import org.apache.kafka.common.message.{BeginQuorumEpochResponseData, EndQuorumEpochResponseData, FetchResponseData, FetchSnapshotResponseData, VoteResponseData} import org.apache.kafka.common.protocol.{ApiKeys, ApiMessage} -import org.apache.kafka.common.record.BaseRecords import org.apache.kafka.common.requests.{AbstractRequest, AbstractResponse, BeginQuorumEpochResponse, EndQuorumEpochResponse, FetchResponse, FetchSnapshotResponse, VoteResponse} import org.apache.kafka.common.utils.Time @@ -81,7 +80,7 @@ class TestRaftRequestHandler( } private def handleFetch(request: RequestChannel.Request): Unit = { - handle(request, response => new FetchResponse[BaseRecords](response.asInstanceOf[FetchResponseData])) + handle(request, response => new FetchResponse(response.asInstanceOf[FetchResponseData])) } private def handleFetchSnapshot(request: RequestChannel.Request): Unit = { diff --git a/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala index bb4266c98a09b..2eefa4419e5bf 100644 --- a/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala +++ b/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala @@ -49,7 +49,7 @@ import org.apache.kafka.common.message.UpdateMetadataRequestData.{UpdateMetadata import org.apache.kafka.common.message.{AddOffsetsToTxnRequestData, AlterPartitionReassignmentsRequestData, AlterReplicaLogDirsRequestData, ControlledShutdownRequestData, CreateAclsRequestData, CreatePartitionsRequestData, CreateTopicsRequestData, DeleteAclsRequestData, DeleteGroupsRequestData, DeleteRecordsRequestData, DeleteTopicsRequestData, DescribeClusterRequestData, DescribeConfigsRequestData, DescribeGroupsRequestData, DescribeLogDirsRequestData, DescribeProducersRequestData, DescribeTransactionsRequestData, FindCoordinatorRequestData, HeartbeatRequestData, IncrementalAlterConfigsRequestData, JoinGroupRequestData, ListPartitionReassignmentsRequestData, ListTransactionsRequestData, MetadataRequestData, OffsetCommitRequestData, ProduceRequestData, SyncGroupRequestData} import org.apache.kafka.common.network.ListenerName import org.apache.kafka.common.protocol.{ApiKeys, Errors} -import org.apache.kafka.common.record.{CompressionType, MemoryRecords, RecordBatch, Records, SimpleRecord} +import org.apache.kafka.common.record.{CompressionType, MemoryRecords, RecordBatch, SimpleRecord} import org.apache.kafka.common.requests._ import org.apache.kafka.common.resource.PatternType.{LITERAL, PREFIXED} import org.apache.kafka.common.resource.ResourceType._ @@ -159,7 +159,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest { val requestKeyToError = (topicNames: Map[Uuid, String]) => Map[ApiKeys, Nothing => Errors]( ApiKeys.METADATA -> ((resp: requests.MetadataResponse) => resp.errors.asScala.find(_._1 == topic).getOrElse(("test", Errors.NONE))._2), ApiKeys.PRODUCE -> ((resp: requests.ProduceResponse) => resp.responses.asScala.find(_._1 == tp).get._2.error), - ApiKeys.FETCH -> ((resp: requests.FetchResponse[Records]) => resp.responseData.asScala.find(_._1 == tp).get._2.error), + ApiKeys.FETCH -> ((resp: requests.FetchResponse) => Errors.forCode(resp.responseData.asScala.find(_._1 == tp).get._2.errorCode)), ApiKeys.LIST_OFFSETS -> ((resp: ListOffsetsResponse) => { Errors.forCode( resp.data @@ -169,12 +169,12 @@ class AuthorizerIntegrationTest extends BaseRequestTest { ) }), ApiKeys.OFFSET_COMMIT -> ((resp: requests.OffsetCommitResponse) => Errors.forCode( - resp.data().topics().get(0).partitions().get(0).errorCode())), + resp.data.topics().get(0).partitions().get(0).errorCode)), ApiKeys.OFFSET_FETCH -> ((resp: requests.OffsetFetchResponse) => resp.error), ApiKeys.FIND_COORDINATOR -> ((resp: FindCoordinatorResponse) => resp.error), ApiKeys.UPDATE_METADATA -> ((resp: requests.UpdateMetadataResponse) => resp.error), ApiKeys.JOIN_GROUP -> ((resp: JoinGroupResponse) => resp.error), - ApiKeys.SYNC_GROUP -> ((resp: SyncGroupResponse) => Errors.forCode(resp.data.errorCode())), + ApiKeys.SYNC_GROUP -> ((resp: SyncGroupResponse) => Errors.forCode(resp.data.errorCode)), ApiKeys.DESCRIBE_GROUPS -> ((resp: DescribeGroupsResponse) => { Errors.forCode(resp.data.groups.asScala.find(g => group == g.groupId).head.errorCode) }), @@ -187,8 +187,8 @@ class AuthorizerIntegrationTest extends BaseRequestTest { ApiKeys.STOP_REPLICA -> ((resp: requests.StopReplicaResponse) => Errors.forCode( resp.partitionErrors.asScala.find(pe => pe.topicName == tp.topic && pe.partitionIndex == tp.partition).get.errorCode)), ApiKeys.CONTROLLED_SHUTDOWN -> ((resp: requests.ControlledShutdownResponse) => resp.error), - ApiKeys.CREATE_TOPICS -> ((resp: CreateTopicsResponse) => Errors.forCode(resp.data.topics.find(topic).errorCode())), - ApiKeys.DELETE_TOPICS -> ((resp: requests.DeleteTopicsResponse) => Errors.forCode(resp.data.responses.find(topic).errorCode())), + ApiKeys.CREATE_TOPICS -> ((resp: CreateTopicsResponse) => Errors.forCode(resp.data.topics.find(topic).errorCode)), + ApiKeys.DELETE_TOPICS -> ((resp: requests.DeleteTopicsResponse) => Errors.forCode(resp.data.responses.find(topic).errorCode)), ApiKeys.DELETE_RECORDS -> ((resp: requests.DeleteRecordsResponse) => Errors.forCode( resp.data.topics.find(tp.topic).partitions.find(tp.partition).errorCode)), ApiKeys.OFFSET_FOR_LEADER_EPOCH -> ((resp: OffsetsForLeaderEpochResponse) => Errors.forCode( @@ -211,17 +211,17 @@ class AuthorizerIntegrationTest extends BaseRequestTest { .find(p => p.partitionIndex == tp.partition).get.errorCode)), ApiKeys.DESCRIBE_LOG_DIRS -> ((resp: DescribeLogDirsResponse) => if (resp.data.results.size() > 0) Errors.forCode(resp.data.results.get(0).errorCode) else Errors.CLUSTER_AUTHORIZATION_FAILED), - ApiKeys.CREATE_PARTITIONS -> ((resp: CreatePartitionsResponse) => Errors.forCode(resp.data.results.asScala.head.errorCode())), - ApiKeys.ELECT_LEADERS -> ((resp: ElectLeadersResponse) => Errors.forCode(resp.data().errorCode())), + ApiKeys.CREATE_PARTITIONS -> ((resp: CreatePartitionsResponse) => Errors.forCode(resp.data.results.asScala.head.errorCode)), + ApiKeys.ELECT_LEADERS -> ((resp: ElectLeadersResponse) => Errors.forCode(resp.data.errorCode)), ApiKeys.INCREMENTAL_ALTER_CONFIGS -> ((resp: IncrementalAlterConfigsResponse) => { - val topicResourceError = IncrementalAlterConfigsResponse.fromResponseData(resp.data()).get(new ConfigResource(ConfigResource.Type.TOPIC, tp.topic)) + val topicResourceError = IncrementalAlterConfigsResponse.fromResponseData(resp.data).get(new ConfigResource(ConfigResource.Type.TOPIC, tp.topic)) if (topicResourceError == null) - IncrementalAlterConfigsResponse.fromResponseData(resp.data()).get(new ConfigResource(ConfigResource.Type.BROKER_LOGGER, brokerId.toString)).error + IncrementalAlterConfigsResponse.fromResponseData(resp.data).get(new ConfigResource(ConfigResource.Type.BROKER_LOGGER, brokerId.toString)).error else topicResourceError.error() }), - ApiKeys.ALTER_PARTITION_REASSIGNMENTS -> ((resp: AlterPartitionReassignmentsResponse) => Errors.forCode(resp.data().errorCode())), - ApiKeys.LIST_PARTITION_REASSIGNMENTS -> ((resp: ListPartitionReassignmentsResponse) => Errors.forCode(resp.data().errorCode())), + ApiKeys.ALTER_PARTITION_REASSIGNMENTS -> ((resp: AlterPartitionReassignmentsResponse) => Errors.forCode(resp.data.errorCode)), + ApiKeys.LIST_PARTITION_REASSIGNMENTS -> ((resp: ListPartitionReassignmentsResponse) => Errors.forCode(resp.data.errorCode)), ApiKeys.OFFSET_DELETE -> ((resp: OffsetDeleteResponse) => { Errors.forCode( resp.data @@ -326,9 +326,9 @@ class AuthorizerIntegrationTest extends BaseRequestTest { requests.ProduceRequest.forCurrentMagic(new ProduceRequestData() .setTopicData(new ProduceRequestData.TopicProduceDataCollection( Collections.singletonList(new ProduceRequestData.TopicProduceData() - .setName(tp.topic()).setPartitionData(Collections.singletonList( + .setName(tp.topic).setPartitionData(Collections.singletonList( new ProduceRequestData.PartitionProduceData() - .setIndex(tp.partition()) + .setIndex(tp.partition) .setRecords(MemoryRecords.withRecords(CompressionType.NONE, new SimpleRecord("test".getBytes)))))) .iterator)) .setAcks(1.toShort) @@ -363,9 +363,9 @@ class AuthorizerIntegrationTest extends BaseRequestTest { private def offsetsForLeaderEpochRequest: OffsetsForLeaderEpochRequest = { val epochs = new OffsetForLeaderTopicCollection() epochs.add(new OffsetForLeaderTopic() - .setTopic(tp.topic()) + .setTopic(tp.topic) .setPartitions(List(new OffsetForLeaderPartition() - .setPartition(tp.partition()) + .setPartition(tp.partition) .setLeaderEpoch(7) .setCurrentLeaderEpoch(27)).asJava)) OffsetsForLeaderEpochRequest.Builder.forConsumer(epochs).build() @@ -509,9 +509,9 @@ class AuthorizerIntegrationTest extends BaseRequestTest { private def stopReplicaRequest: StopReplicaRequest = { val topicStates = Seq( new StopReplicaTopicState() - .setTopicName(tp.topic()) + .setTopicName(tp.topic) .setPartitionStates(Seq(new StopReplicaPartitionState() - .setPartitionIndex(tp.partition()) + .setPartitionIndex(tp.partition) .setLeaderEpoch(LeaderAndIsr.initialLeaderEpoch + 2) .setDeletePartition(true)).asJava) ).asJava @@ -658,7 +658,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest { List(new AlterPartitionReassignmentsRequestData.ReassignableTopic() .setName(topic) .setPartitions( - List(new AlterPartitionReassignmentsRequestData.ReassignablePartition().setPartitionIndex(tp.partition())).asJava + List(new AlterPartitionReassignmentsRequestData.ReassignablePartition().setPartitionIndex(tp.partition)).asJava )).asJava ) ).build() @@ -1625,7 +1625,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest { @Test def testUnauthorizedCreatePartitions(): Unit = { val createPartitionsResponse = connectAndReceive[CreatePartitionsResponse](createPartitionsRequest) - assertEquals(Errors.TOPIC_AUTHORIZATION_FAILED.code(), createPartitionsResponse.data.results.asScala.head.errorCode()) + assertEquals(Errors.TOPIC_AUTHORIZATION_FAILED.code, createPartitionsResponse.data.results.asScala.head.errorCode) } @Test @@ -1633,7 +1633,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest { createTopic(topic) addAndVerifyAcls(Set(new AccessControlEntry(clientPrincipalString, WildcardHost, ALTER, ALLOW)), new ResourcePattern(TOPIC, "*", LITERAL)) val createPartitionsResponse = connectAndReceive[CreatePartitionsResponse](createPartitionsRequest) - assertEquals(Errors.NONE.code(), createPartitionsResponse.data.results.asScala.head.errorCode()) + assertEquals(Errors.NONE.code, createPartitionsResponse.data.results.asScala.head.errorCode) } @Test @@ -2133,7 +2133,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest { numRecords: Int, tp: TopicPartition): Unit = { val futures = (0 until numRecords).map { i => - producer.send(new ProducerRecord(tp.topic(), tp.partition(), i.toString.getBytes, i.toString.getBytes)) + producer.send(new ProducerRecord(tp.topic, tp.partition, i.toString.getBytes, i.toString.getBytes)) } try { futures.foreach(_.get) diff --git a/core/src/test/scala/kafka/tools/ReplicaVerificationToolTest.scala b/core/src/test/scala/kafka/tools/ReplicaVerificationToolTest.scala index cb98116f0a0c0..217260403df4e 100644 --- a/core/src/test/scala/kafka/tools/ReplicaVerificationToolTest.scala +++ b/core/src/test/scala/kafka/tools/ReplicaVerificationToolTest.scala @@ -18,9 +18,8 @@ package kafka.tools import org.apache.kafka.common.TopicPartition -import org.apache.kafka.common.protocol.Errors +import org.apache.kafka.common.message.FetchResponseData import org.apache.kafka.common.record.{CompressionType, MemoryRecords, SimpleRecord} -import org.apache.kafka.common.requests.FetchResponse import org.junit.jupiter.api.Test import org.junit.jupiter.api.Assertions.assertTrue @@ -44,7 +43,12 @@ class ReplicaVerificationToolTest { } val initialOffset = 4 val memoryRecords = MemoryRecords.withRecords(initialOffset, CompressionType.NONE, records: _*) - val partitionData = new FetchResponse.PartitionData(Errors.NONE, 20, 20, 0L, null, memoryRecords) + val partitionData = new FetchResponseData.PartitionData() + .setPartitionIndex(tp.partition) + .setHighWatermark(20) + .setLastStableOffset(20) + .setLogStartOffset(0) + .setRecords(memoryRecords) replicaBuffer.addFetchedData(tp, replicaId, partitionData) } diff --git a/core/src/test/scala/unit/kafka/log/LogTest.scala b/core/src/test/scala/unit/kafka/log/LogTest.scala index a76345fd730e5..1e5257df49853 100755 --- a/core/src/test/scala/unit/kafka/log/LogTest.scala +++ b/core/src/test/scala/unit/kafka/log/LogTest.scala @@ -34,11 +34,11 @@ import kafka.server.{BrokerTopicStats, FetchDataInfo, FetchHighWatermark, FetchI import kafka.utils._ import org.apache.kafka.common.{InvalidRecordException, KafkaException, TopicPartition, Uuid} import org.apache.kafka.common.errors._ +import org.apache.kafka.common.message.FetchResponseData import org.apache.kafka.common.record.FileRecords.TimestampAndOffset import org.apache.kafka.common.record.MemoryRecords.RecordFilter import org.apache.kafka.common.record.MemoryRecords.RecordFilter.BatchRetention import org.apache.kafka.common.record._ -import org.apache.kafka.common.requests.FetchResponse.AbortedTransaction import org.apache.kafka.common.requests.{ListOffsetsRequest, ListOffsetsResponse} import org.apache.kafka.common.utils.{BufferSupplier, Time, Utils} import org.easymock.EasyMock @@ -4725,7 +4725,8 @@ class LogTest { assertEquals(1, fetchDataInfo.abortedTransactions.size) assertTrue(fetchDataInfo.abortedTransactions.isDefined) - assertEquals(new AbortedTransaction(pid, 0), fetchDataInfo.abortedTransactions.get.head) + assertEquals(new FetchResponseData.AbortedTransaction().setProducerId(pid).setFirstOffset(0), + fetchDataInfo.abortedTransactions.get.head) } @Test diff --git a/core/src/test/scala/unit/kafka/log/TransactionIndexTest.scala b/core/src/test/scala/unit/kafka/log/TransactionIndexTest.scala index 05f23cfbe3c83..790bcd88a9a09 100644 --- a/core/src/test/scala/unit/kafka/log/TransactionIndexTest.scala +++ b/core/src/test/scala/unit/kafka/log/TransactionIndexTest.scala @@ -17,7 +17,7 @@ package kafka.log import kafka.utils.TestUtils -import org.apache.kafka.common.requests.FetchResponse.AbortedTransaction +import org.apache.kafka.common.message.FetchResponseData import org.junit.jupiter.api.Assertions._ import org.junit.jupiter.api.{AfterEach, BeforeEach, Test} @@ -136,7 +136,7 @@ class TransactionIndexTest { assertEquals(abortedTransactions.take(3), index.collectAbortedTxns(0L, 100L).abortedTransactions) index.reset() - assertEquals(List.empty[AbortedTransaction], index.collectAbortedTxns(0L, 100L).abortedTransactions) + assertEquals(List.empty[FetchResponseData.AbortedTransaction], index.collectAbortedTxns(0L, 100L).abortedTransactions) } @Test diff --git a/core/src/test/scala/unit/kafka/server/AbstractFetcherThreadTest.scala b/core/src/test/scala/unit/kafka/server/AbstractFetcherThreadTest.scala index 9b9fe8e12fa43..394897f41e2f3 100644 --- a/core/src/test/scala/unit/kafka/server/AbstractFetcherThreadTest.scala +++ b/core/src/test/scala/unit/kafka/server/AbstractFetcherThreadTest.scala @@ -20,7 +20,6 @@ package kafka.server import java.nio.ByteBuffer import java.util.Optional import java.util.concurrent.atomic.AtomicInteger - import kafka.cluster.BrokerEndPoint import kafka.log.LogAppendInfo import kafka.message.NoCompressionCodec @@ -37,7 +36,7 @@ import org.apache.kafka.common.message.OffsetForLeaderEpochResponseData.EpochEnd import org.apache.kafka.common.protocol.{ApiKeys, Errors} import org.apache.kafka.common.record._ import org.apache.kafka.common.requests.OffsetsForLeaderEpochResponse.{UNDEFINED_EPOCH, UNDEFINED_EPOCH_OFFSET} -import org.apache.kafka.common.requests.FetchRequest +import org.apache.kafka.common.requests.{FetchRequest, FetchResponse} import org.apache.kafka.common.utils.Time import org.junit.jupiter.api.Assertions._ import org.junit.jupiter.api.{BeforeEach, Test} @@ -907,8 +906,8 @@ class AbstractFetcherThreadTest { partitionData: FetchData): Option[LogAppendInfo] = { val state = replicaPartitionState(topicPartition) - if (isTruncationOnFetchSupported && partitionData.divergingEpoch.isPresent) { - val divergingEpoch = partitionData.divergingEpoch.get + if (isTruncationOnFetchSupported && FetchResponse.isDivergingEpoch(partitionData)) { + val divergingEpoch = partitionData.divergingEpoch truncateOnFetchResponse(Map(topicPartition -> new EpochEndOffset() .setPartition(topicPartition.partition) .setErrorCode(Errors.NONE.code) @@ -923,7 +922,7 @@ class AbstractFetcherThreadTest { s"fetched offset = $fetchOffset, log end offset = ${state.logEndOffset}.") // Now check message's crc - val batches = partitionData.records.batches.asScala + val batches = FetchResponse.recordsOrFail(partitionData).batches.asScala var maxTimestamp = RecordBatch.NO_TIMESTAMP var offsetOfMaxTimestamp = -1L var lastOffset = state.logEndOffset @@ -955,7 +954,7 @@ class AbstractFetcherThreadTest { sourceCodec = NoCompressionCodec, targetCodec = NoCompressionCodec, shallowCount = batches.size, - validBytes = partitionData.records.sizeInBytes, + validBytes = FetchResponse.recordsSize(partitionData), offsetsMonotonic = true, lastOffsetOfFirstBatch = batches.headOption.map(_.lastOffset).getOrElse(-1))) } @@ -1143,9 +1142,16 @@ class AbstractFetcherThreadTest { (Errors.NONE, records) } + val partitionData = new FetchData() + .setPartitionIndex(partition.partition) + .setErrorCode(error.code) + .setHighWatermark(leaderState.highWatermark) + .setLastStableOffset(leaderState.highWatermark) + .setLogStartOffset(leaderState.logStartOffset) + .setRecords(records) + divergingEpoch.foreach(partitionData.setDivergingEpoch) - (partition, new FetchData(error, leaderState.highWatermark, leaderState.highWatermark, leaderState.logStartOffset, - Optional.empty[Integer], List.empty.asJava, divergingEpoch.asJava, records)) + (partition, partitionData) }.toMap } diff --git a/core/src/test/scala/unit/kafka/server/FetchRequestDownConversionConfigTest.scala b/core/src/test/scala/unit/kafka/server/FetchRequestDownConversionConfigTest.scala index 1971fbef4d4d2..03ed0181fd216 100644 --- a/core/src/test/scala/unit/kafka/server/FetchRequestDownConversionConfigTest.scala +++ b/core/src/test/scala/unit/kafka/server/FetchRequestDownConversionConfigTest.scala @@ -23,7 +23,6 @@ import kafka.utils.TestUtils import org.apache.kafka.clients.producer.{KafkaProducer, ProducerRecord} import org.apache.kafka.common.TopicPartition import org.apache.kafka.common.protocol.Errors -import org.apache.kafka.common.record.MemoryRecords import org.apache.kafka.common.requests.{FetchRequest, FetchResponse} import org.apache.kafka.common.serialization.StringSerializer import org.junit.jupiter.api.Assertions._ @@ -79,8 +78,8 @@ class FetchRequestDownConversionConfigTest extends BaseRequestTest { partitionMap } - private def sendFetchRequest(leaderId: Int, request: FetchRequest): FetchResponse[MemoryRecords] = { - connectAndReceive[FetchResponse[MemoryRecords]](request, destination = brokerSocketServer(leaderId)) + private def sendFetchRequest(leaderId: Int, request: FetchRequest): FetchResponse = { + connectAndReceive[FetchResponse](request, destination = brokerSocketServer(leaderId)) } /** @@ -90,11 +89,11 @@ class FetchRequestDownConversionConfigTest extends BaseRequestTest { def testV1FetchWithDownConversionDisabled(): Unit = { val topicMap = createTopics(numTopics = 5, numPartitions = 1) val topicPartitions = topicMap.keySet.toSeq - topicPartitions.foreach(tp => producer.send(new ProducerRecord(tp.topic(), "key", "value")).get()) + topicPartitions.foreach(tp => producer.send(new ProducerRecord(tp.topic, "key", "value")).get()) val fetchRequest = FetchRequest.Builder.forConsumer(Int.MaxValue, 0, createPartitionMap(1024, topicPartitions)).build(1) val fetchResponse = sendFetchRequest(topicMap.head._2, fetchRequest) - topicPartitions.foreach(tp => assertEquals(Errors.UNSUPPORTED_VERSION, fetchResponse.responseData().get(tp).error)) + topicPartitions.foreach(tp => assertEquals(Errors.UNSUPPORTED_VERSION, Errors.forCode(fetchResponse.responseData.get(tp).errorCode))) } /** @@ -104,11 +103,11 @@ class FetchRequestDownConversionConfigTest extends BaseRequestTest { def testLatestFetchWithDownConversionDisabled(): Unit = { val topicMap = createTopics(numTopics = 5, numPartitions = 1) val topicPartitions = topicMap.keySet.toSeq - topicPartitions.foreach(tp => producer.send(new ProducerRecord(tp.topic(), "key", "value")).get()) + topicPartitions.foreach(tp => producer.send(new ProducerRecord(tp.topic, "key", "value")).get()) val fetchRequest = FetchRequest.Builder.forConsumer(Int.MaxValue, 0, createPartitionMap(1024, topicPartitions)).build() val fetchResponse = sendFetchRequest(topicMap.head._2, fetchRequest) - topicPartitions.foreach(tp => assertEquals(Errors.NONE, fetchResponse.responseData().get(tp).error)) + topicPartitions.foreach(tp => assertEquals(Errors.NONE, Errors.forCode(fetchResponse.responseData.get(tp).errorCode))) } /** @@ -129,13 +128,13 @@ class FetchRequestDownConversionConfigTest extends BaseRequestTest { val allTopics = conversionDisabledTopicPartitions ++ conversionEnabledTopicPartitions val leaderId = conversionDisabledTopicsMap.head._2 - allTopics.foreach(tp => producer.send(new ProducerRecord(tp.topic(), "key", "value")).get()) + allTopics.foreach(tp => producer.send(new ProducerRecord(tp.topic, "key", "value")).get()) val fetchRequest = FetchRequest.Builder.forConsumer(Int.MaxValue, 0, createPartitionMap(1024, allTopics)).build(1) val fetchResponse = sendFetchRequest(leaderId, fetchRequest) - conversionDisabledTopicPartitions.foreach(tp => assertEquals(Errors.UNSUPPORTED_VERSION, fetchResponse.responseData().get(tp).error)) - conversionEnabledTopicPartitions.foreach(tp => assertEquals(Errors.NONE, fetchResponse.responseData().get(tp).error)) + conversionDisabledTopicPartitions.foreach(tp => assertEquals(Errors.UNSUPPORTED_VERSION.code, fetchResponse.responseData.get(tp).errorCode)) + conversionEnabledTopicPartitions.foreach(tp => assertEquals(Errors.NONE.code, fetchResponse.responseData.get(tp).errorCode)) } /** @@ -155,11 +154,11 @@ class FetchRequestDownConversionConfigTest extends BaseRequestTest { val allTopicPartitions = conversionDisabledTopicPartitions ++ conversionEnabledTopicPartitions val leaderId = conversionDisabledTopicsMap.head._2 - allTopicPartitions.foreach(tp => producer.send(new ProducerRecord(tp.topic(), "key", "value")).get()) + allTopicPartitions.foreach(tp => producer.send(new ProducerRecord(tp.topic, "key", "value")).get()) val fetchRequest = FetchRequest.Builder.forReplica(1, 1, Int.MaxValue, 0, createPartitionMap(1024, allTopicPartitions)).build() val fetchResponse = sendFetchRequest(leaderId, fetchRequest) - allTopicPartitions.foreach(tp => assertEquals(Errors.NONE, fetchResponse.responseData().get(tp).error)) + allTopicPartitions.foreach(tp => assertEquals(Errors.NONE.code, fetchResponse.responseData.get(tp).errorCode)) } } diff --git a/core/src/test/scala/unit/kafka/server/FetchRequestMaxBytesTest.scala b/core/src/test/scala/unit/kafka/server/FetchRequestMaxBytesTest.scala index 5889cbb33eda8..c919f7f6714cf 100644 --- a/core/src/test/scala/unit/kafka/server/FetchRequestMaxBytesTest.scala +++ b/core/src/test/scala/unit/kafka/server/FetchRequestMaxBytesTest.scala @@ -17,17 +17,16 @@ package kafka.server -import java.util.{Optional, Properties} import kafka.log.LogConfig import kafka.utils.TestUtils import org.apache.kafka.clients.producer.{KafkaProducer, ProducerRecord} import org.apache.kafka.common.TopicPartition -import org.apache.kafka.common.record.MemoryRecords import org.apache.kafka.common.requests.FetchRequest.PartitionData import org.apache.kafka.common.requests.{FetchRequest, FetchResponse} import org.junit.jupiter.api.Assertions._ import org.junit.jupiter.api.{AfterEach, BeforeEach, Test} +import java.util.{Optional, Properties} import scala.jdk.CollectionConverters._ /** @@ -92,8 +91,8 @@ class FetchRequestMaxBytesTest extends BaseRequestTest { }) } - private def sendFetchRequest(leaderId: Int, request: FetchRequest): FetchResponse[MemoryRecords] = { - connectAndReceive[FetchResponse[MemoryRecords]](request, destination = brokerSocketServer(leaderId)) + private def sendFetchRequest(leaderId: Int, request: FetchRequest): FetchResponse = { + connectAndReceive[FetchResponse](request, destination = brokerSocketServer(leaderId)) } /** @@ -117,7 +116,7 @@ class FetchRequestMaxBytesTest extends BaseRequestTest { FetchRequest.Builder.forConsumer(Int.MaxValue, 0, Map(testTopicPartition -> new PartitionData(fetchOffset, 0, Integer.MAX_VALUE, Optional.empty())).asJava).build(3)) - val records = response.responseData().get(testTopicPartition).records.records() + val records = FetchResponse.recordsOrFail(response.responseData.get(testTopicPartition)).records() assertNotNull(records) val recordsList = records.asScala.toList assertEquals(expected.size, recordsList.size) diff --git a/core/src/test/scala/unit/kafka/server/FetchRequestTest.scala b/core/src/test/scala/unit/kafka/server/FetchRequestTest.scala index c4b3a2841c150..b7dde352d6bab 100644 --- a/core/src/test/scala/unit/kafka/server/FetchRequestTest.scala +++ b/core/src/test/scala/unit/kafka/server/FetchRequestTest.scala @@ -16,24 +16,25 @@ */ package kafka.server -import java.io.DataInputStream -import java.util -import java.util.{Optional, Properties} import kafka.api.KAFKA_0_11_0_IV2 import kafka.log.LogConfig import kafka.message.{GZIPCompressionCodec, ProducerCompressionCodec, ZStdCompressionCodec} import kafka.utils.TestUtils import org.apache.kafka.clients.producer.{KafkaProducer, ProducerRecord, RecordMetadata} +import org.apache.kafka.common.message.FetchResponseData import org.apache.kafka.common.protocol.{ApiKeys, Errors} -import org.apache.kafka.common.record.{MemoryRecords, Record, RecordBatch} +import org.apache.kafka.common.record.{Record, RecordBatch} import org.apache.kafka.common.requests.{FetchRequest, FetchResponse, FetchMetadata => JFetchMetadata} import org.apache.kafka.common.serialization.{ByteArraySerializer, StringSerializer} import org.apache.kafka.common.{IsolationLevel, TopicPartition} import org.junit.jupiter.api.Assertions._ import org.junit.jupiter.api.{AfterEach, Test} -import scala.jdk.CollectionConverters._ +import java.io.DataInputStream +import java.util +import java.util.{Optional, Properties} import scala.collection.Seq +import scala.jdk.CollectionConverters._ import scala.util.Random /** @@ -70,8 +71,8 @@ class FetchRequestTest extends BaseRequestTest { partitionMap } - private def sendFetchRequest(leaderId: Int, request: FetchRequest): FetchResponse[MemoryRecords] = { - connectAndReceive[FetchResponse[MemoryRecords]](request, destination = brokerSocketServer(leaderId)) + private def sendFetchRequest(leaderId: Int, request: FetchRequest): FetchResponse = { + connectAndReceive[FetchResponse](request, destination = brokerSocketServer(leaderId)) } private def initProducer(): Unit = { @@ -133,12 +134,12 @@ class FetchRequestTest extends BaseRequestTest { }.sum assertTrue(responseSize3 <= maxResponseBytes) val partitionData3 = fetchResponse3.responseData.get(partitionWithLargeMessage1) - assertEquals(Errors.NONE, partitionData3.error) + assertEquals(Errors.NONE.code, partitionData3.errorCode) assertTrue(partitionData3.highWatermark > 0) val size3 = records(partitionData3).map(_.sizeInBytes).sum assertTrue(size3 <= maxResponseBytes, s"Expected $size3 to be smaller than $maxResponseBytes") assertTrue(size3 > maxPartitionBytes, s"Expected $size3 to be larger than $maxPartitionBytes") - assertTrue(maxPartitionBytes < partitionData3.records.sizeInBytes) + assertTrue(maxPartitionBytes < FetchResponse.recordsSize(partitionData3)) // 4. Partition with message larger than the response limit at the start of the list val shuffledTopicPartitions4 = Seq(partitionWithLargeMessage2, partitionWithLargeMessage1) ++ @@ -151,11 +152,11 @@ class FetchRequestTest extends BaseRequestTest { } assertEquals(Seq(partitionWithLargeMessage2), nonEmptyPartitions4) val partitionData4 = fetchResponse4.responseData.get(partitionWithLargeMessage2) - assertEquals(Errors.NONE, partitionData4.error) + assertEquals(Errors.NONE.code, partitionData4.errorCode) assertTrue(partitionData4.highWatermark > 0) val size4 = records(partitionData4).map(_.sizeInBytes).sum assertTrue(size4 > maxResponseBytes, s"Expected $size4 to be larger than $maxResponseBytes") - assertTrue(maxResponseBytes < partitionData4.records.sizeInBytes) + assertTrue(maxResponseBytes < FetchResponse.recordsSize(partitionData4)) } @Test @@ -169,9 +170,9 @@ class FetchRequestTest extends BaseRequestTest { Seq(topicPartition))).build(2) val fetchResponse = sendFetchRequest(leaderId, fetchRequest) val partitionData = fetchResponse.responseData.get(topicPartition) - assertEquals(Errors.NONE, partitionData.error) + assertEquals(Errors.NONE.code, partitionData.errorCode) assertTrue(partitionData.highWatermark > 0) - assertEquals(maxPartitionBytes, partitionData.records.sizeInBytes) + assertEquals(maxPartitionBytes, FetchResponse.recordsSize(partitionData)) assertEquals(0, records(partitionData).map(_.sizeInBytes).sum) } @@ -186,7 +187,7 @@ class FetchRequestTest extends BaseRequestTest { Seq(topicPartition))).isolationLevel(IsolationLevel.READ_COMMITTED).build(4) val fetchResponse = sendFetchRequest(leaderId, fetchRequest) val partitionData = fetchResponse.responseData.get(topicPartition) - assertEquals(Errors.NONE, partitionData.error) + assertEquals(Errors.NONE.code, partitionData.errorCode) assertTrue(partitionData.lastStableOffset > 0) assertTrue(records(partitionData).map(_.sizeInBytes).sum > 0) } @@ -209,7 +210,7 @@ class FetchRequestTest extends BaseRequestTest { Seq(topicPartition))).build() val fetchResponse = sendFetchRequest(nonReplicaId, fetchRequest) val partitionData = fetchResponse.responseData.get(topicPartition) - assertEquals(Errors.NOT_LEADER_OR_FOLLOWER, partitionData.error) + assertEquals(Errors.NOT_LEADER_OR_FOLLOWER.code, partitionData.errorCode) } @Test @@ -243,11 +244,11 @@ class FetchRequestTest extends BaseRequestTest { // Validate the expected truncation val fetchResponse = sendFetchRequest(secondLeaderId, fetchRequest) val partitionData = fetchResponse.responseData.get(topicPartition) - assertEquals(Errors.NONE, partitionData.error) - assertEquals(0L, partitionData.records.sizeInBytes()) - assertTrue(partitionData.divergingEpoch.isPresent) + assertEquals(Errors.NONE.code, partitionData.errorCode) + assertEquals(0L, FetchResponse.recordsSize(partitionData)) + assertTrue(FetchResponse.isDivergingEpoch(partitionData)) - val divergingEpoch = partitionData.divergingEpoch.get() + val divergingEpoch = partitionData.divergingEpoch assertEquals(firstLeaderEpoch, divergingEpoch.epoch) assertEquals(firstEpochEndOffset, divergingEpoch.endOffset) } @@ -265,7 +266,7 @@ class FetchRequestTest extends BaseRequestTest { val fetchRequest = FetchRequest.Builder.forConsumer(0, 1, partitionMap).build() val fetchResponse = sendFetchRequest(brokerId, fetchRequest) val partitionData = fetchResponse.responseData.get(topicPartition) - assertEquals(error, partitionData.error) + assertEquals(error.code, partitionData.errorCode) } // We need a leader change in order to check epoch fencing since the first epoch is 0 and @@ -329,7 +330,7 @@ class FetchRequestTest extends BaseRequestTest { .build() val fetchResponse = sendFetchRequest(destinationBrokerId, fetchRequest) val partitionData = fetchResponse.responseData.get(topicPartition) - assertEquals(expectedError, partitionData.error) + assertEquals(expectedError.code, partitionData.errorCode) } // We only check errors because we do not expect the partition in the response otherwise @@ -366,7 +367,7 @@ class FetchRequestTest extends BaseRequestTest { // batch is not complete, but sent when the producer is closed futures.foreach(_.get) - def fetch(version: Short, maxPartitionBytes: Int, closeAfterPartialResponse: Boolean): Option[FetchResponse[MemoryRecords]] = { + def fetch(version: Short, maxPartitionBytes: Int, closeAfterPartialResponse: Boolean): Option[FetchResponse] = { val fetchRequest = FetchRequest.Builder.forConsumer(Int.MaxValue, 0, createPartitionMap(maxPartitionBytes, Seq(topicPartition))).build(version) @@ -383,7 +384,7 @@ class FetchRequestTest extends BaseRequestTest { s"Fetch size too small $size, broker may have run out of memory") None } else { - Some(receive[FetchResponse[MemoryRecords]](socket, ApiKeys.FETCH, version)) + Some(receive[FetchResponse](socket, ApiKeys.FETCH, version)) } } finally { socket.close() @@ -396,8 +397,8 @@ class FetchRequestTest extends BaseRequestTest { val response = fetch(version, maxPartitionBytes = batchSize, closeAfterPartialResponse = false) val fetchResponse = response.getOrElse(throw new IllegalStateException("No fetch response")) val partitionData = fetchResponse.responseData.get(topicPartition) - assertEquals(Errors.NONE, partitionData.error) - val batches = partitionData.records.batches.asScala.toBuffer + assertEquals(Errors.NONE.code, partitionData.errorCode) + val batches = FetchResponse.recordsOrFail(partitionData).batches.asScala.toBuffer assertEquals(3, batches.size) // size is 3 (not 4) since maxPartitionBytes=msgValueSize*4, excluding key and headers } @@ -442,9 +443,9 @@ class FetchRequestTest extends BaseRequestTest { // validate response val partitionData = fetchResponse.responseData.get(topicPartition) - assertEquals(Errors.NONE, partitionData.error) + assertEquals(Errors.NONE.code, partitionData.errorCode) assertTrue(partitionData.highWatermark > 0) - val batches = partitionData.records.batches.asScala.toBuffer + val batches = FetchResponse.recordsOrFail(partitionData).batches.asScala.toBuffer val batch = batches.head assertEquals(expectedMagic, batch.magic) assertEquals(currentExpectedOffset, batch.baseOffset) @@ -504,35 +505,34 @@ class FetchRequestTest extends BaseRequestTest { assertEquals(Errors.NONE, resp1.error()) assertTrue(resp1.sessionId() > 0, "Expected the broker to create a new incremental fetch session") debug(s"Test created an incremental fetch session ${resp1.sessionId}") - assertTrue(resp1.responseData().containsKey(foo0)) - assertTrue(resp1.responseData().containsKey(foo1)) - assertTrue(resp1.responseData().containsKey(bar0)) - assertEquals(Errors.NONE, resp1.responseData().get(foo0).error) - assertEquals(Errors.NONE, resp1.responseData().get(foo1).error) - assertEquals(Errors.UNKNOWN_TOPIC_OR_PARTITION, resp1.responseData().get(bar0).error) + assertTrue(resp1.responseData.containsKey(foo0)) + assertTrue(resp1.responseData.containsKey(foo1)) + assertTrue(resp1.responseData.containsKey(bar0)) + assertEquals(Errors.NONE.code, resp1.responseData.get(foo0).errorCode) + assertEquals(Errors.NONE.code, resp1.responseData.get(foo1).errorCode) + assertEquals(Errors.UNKNOWN_TOPIC_OR_PARTITION.code, resp1.responseData.get(bar0).errorCode) val req2 = createFetchRequest(Nil, new JFetchMetadata(resp1.sessionId(), 1), Nil) val resp2 = sendFetchRequest(0, req2) assertEquals(Errors.NONE, resp2.error()) - assertEquals(resp1.sessionId(), - resp2.sessionId(), "Expected the broker to continue the incremental fetch session") + assertEquals(resp1.sessionId(), resp2.sessionId(), "Expected the broker to continue the incremental fetch session") assertFalse(resp2.responseData().containsKey(foo0)) assertFalse(resp2.responseData().containsKey(foo1)) assertTrue(resp2.responseData().containsKey(bar0)) - assertEquals(Errors.UNKNOWN_TOPIC_OR_PARTITION, resp2.responseData().get(bar0).error) + assertEquals(Errors.UNKNOWN_TOPIC_OR_PARTITION.code(), resp2.responseData().get(bar0).errorCode()) createTopic("bar", Map(0 -> List(0, 1))) val req3 = createFetchRequest(Nil, new JFetchMetadata(resp1.sessionId(), 2), Nil) val resp3 = sendFetchRequest(0, req3) assertEquals(Errors.NONE, resp3.error()) - assertFalse(resp3.responseData().containsKey(foo0)) - assertFalse(resp3.responseData().containsKey(foo1)) - assertTrue(resp3.responseData().containsKey(bar0)) - assertEquals(Errors.NONE, resp3.responseData().get(bar0).error) + assertFalse(resp3.responseData.containsKey(foo0)) + assertFalse(resp3.responseData.containsKey(foo1)) + assertTrue(resp3.responseData.containsKey(bar0)) + assertEquals(Errors.NONE.code, resp3.responseData.get(bar0).errorCode) val req4 = createFetchRequest(Nil, new JFetchMetadata(resp1.sessionId(), 3), Nil) val resp4 = sendFetchRequest(0, req4) assertEquals(Errors.NONE, resp4.error()) - assertFalse(resp4.responseData().containsKey(foo0)) - assertFalse(resp4.responseData().containsKey(foo1)) - assertFalse(resp4.responseData().containsKey(bar0)) + assertFalse(resp4.responseData.containsKey(foo0)) + assertFalse(resp4.responseData.containsKey(foo1)) + assertFalse(resp4.responseData.containsKey(bar0)) } @Test @@ -560,7 +560,7 @@ class FetchRequestTest extends BaseRequestTest { val res0 = sendFetchRequest(leaderId, req0) val data0 = res0.responseData.get(topicPartition) - assertEquals(Errors.UNSUPPORTED_COMPRESSION_TYPE, data0.error) + assertEquals(Errors.UNSUPPORTED_COMPRESSION_TYPE.code, data0.errorCode) // fetch request with version 10: works fine! val req1= new FetchRequest.Builder(0, 10, -1, Int.MaxValue, 0, @@ -568,14 +568,14 @@ class FetchRequestTest extends BaseRequestTest { .setMaxBytes(800).build() val res1 = sendFetchRequest(leaderId, req1) val data1 = res1.responseData.get(topicPartition) - assertEquals(Errors.NONE, data1.error) + assertEquals(Errors.NONE.code, data1.errorCode) assertEquals(3, records(data1).size) } @Test def testPartitionDataEquals(): Unit = { assertEquals(new FetchRequest.PartitionData(300, 0L, 300, Optional.of(300)), - new FetchRequest.PartitionData(300, 0L, 300, Optional.of(300))); + new FetchRequest.PartitionData(300, 0L, 300, Optional.of(300))) } @Test @@ -614,7 +614,7 @@ class FetchRequestTest extends BaseRequestTest { val res0 = sendFetchRequest(leaderId, req0) val data0 = res0.responseData.get(topicPartition) - assertEquals(Errors.NONE, data0.error) + assertEquals(Errors.NONE.code, data0.errorCode) assertEquals(1, records(data0).size) val req1 = new FetchRequest.Builder(0, 1, -1, Int.MaxValue, 0, @@ -623,7 +623,7 @@ class FetchRequestTest extends BaseRequestTest { val res1 = sendFetchRequest(leaderId, req1) val data1 = res1.responseData.get(topicPartition) - assertEquals(Errors.UNSUPPORTED_COMPRESSION_TYPE, data1.error) + assertEquals(Errors.UNSUPPORTED_COMPRESSION_TYPE.code, data1.errorCode) // fetch request with fetch version v3 (magic 1): // gzip compressed record is returned with down-conversion. @@ -634,7 +634,7 @@ class FetchRequestTest extends BaseRequestTest { val res2 = sendFetchRequest(leaderId, req2) val data2 = res2.responseData.get(topicPartition) - assertEquals(Errors.NONE, data2.error) + assertEquals(Errors.NONE.code, data2.errorCode) assertEquals(1, records(data2).size) val req3 = new FetchRequest.Builder(0, 1, -1, Int.MaxValue, 0, @@ -643,7 +643,7 @@ class FetchRequestTest extends BaseRequestTest { val res3 = sendFetchRequest(leaderId, req3) val data3 = res3.responseData.get(topicPartition) - assertEquals(Errors.UNSUPPORTED_COMPRESSION_TYPE, data3.error) + assertEquals(Errors.UNSUPPORTED_COMPRESSION_TYPE.code, data3.errorCode) // fetch request with version 10: works fine! val req4= new FetchRequest.Builder(0, 10, -1, Int.MaxValue, 0, @@ -651,15 +651,15 @@ class FetchRequestTest extends BaseRequestTest { .setMaxBytes(800).build() val res4 = sendFetchRequest(leaderId, req4) val data4 = res4.responseData.get(topicPartition) - assertEquals(Errors.NONE, data4.error) + assertEquals(Errors.NONE.code, data4.errorCode) assertEquals(3, records(data4).size) } - private def records(partitionData: FetchResponse.PartitionData[MemoryRecords]): Seq[Record] = { - partitionData.records.records.asScala.toBuffer + private def records(partitionData: FetchResponseData.PartitionData): Seq[Record] = { + FetchResponse.recordsOrFail(partitionData).records.asScala.toBuffer } - private def checkFetchResponse(expectedPartitions: Seq[TopicPartition], fetchResponse: FetchResponse[MemoryRecords], + private def checkFetchResponse(expectedPartitions: Seq[TopicPartition], fetchResponse: FetchResponse, maxPartitionBytes: Int, maxResponseBytes: Int, numMessagesPerPartition: Int): Unit = { assertEquals(expectedPartitions, fetchResponse.responseData.keySet.asScala.toSeq) var emptyResponseSeen = false @@ -668,10 +668,10 @@ class FetchRequestTest extends BaseRequestTest { expectedPartitions.foreach { tp => val partitionData = fetchResponse.responseData.get(tp) - assertEquals(Errors.NONE, partitionData.error) + assertEquals(Errors.NONE.code, partitionData.errorCode) assertTrue(partitionData.highWatermark > 0) - val records = partitionData.records + val records = FetchResponse.recordsOrFail(partitionData) responseBufferSize += records.sizeInBytes val batches = records.batches.asScala.toBuffer diff --git a/core/src/test/scala/unit/kafka/server/FetchSessionTest.scala b/core/src/test/scala/unit/kafka/server/FetchSessionTest.scala index e00395a98111e..81844f517d439 100755 --- a/core/src/test/scala/unit/kafka/server/FetchSessionTest.scala +++ b/core/src/test/scala/unit/kafka/server/FetchSessionTest.scala @@ -16,26 +16,26 @@ */ package kafka.server -import java.util -import java.util.{Collections, Optional} import kafka.utils.MockTime import org.apache.kafka.common.TopicPartition import org.apache.kafka.common.message.FetchResponseData import org.apache.kafka.common.protocol.Errors -import org.apache.kafka.common.record.Records import org.apache.kafka.common.requests.FetchMetadata.{FINAL_EPOCH, INVALID_SESSION_ID} -import org.apache.kafka.common.requests.{FetchRequest, FetchResponse, FetchMetadata => JFetchMetadata} +import org.apache.kafka.common.requests.{FetchRequest, FetchMetadata => JFetchMetadata} import org.apache.kafka.common.utils.Utils import org.junit.jupiter.api.Assertions._ import org.junit.jupiter.api.{Test, Timeout} +import java.util +import java.util.{Collections, Optional} + @Timeout(120) class FetchSessionTest { @Test def testNewSessionId(): Unit = { val cache = new FetchSessionCache(3, 100) - for (i <- 0 to 10000) { + for (_ <- 0 to 10000) { val id = cache.newSessionId() assertTrue(id > 0) } @@ -125,7 +125,7 @@ class FetchSessionTest { assertEquals(3, cache.totalPartitions) } - val EMPTY_PART_LIST = Collections.unmodifiableList(new util.ArrayList[TopicPartition]()) + private val EMPTY_PART_LIST = Collections.unmodifiableList(new util.ArrayList[TopicPartition]()) @Test @@ -155,13 +155,22 @@ class FetchSessionTest { assertEquals(Optional.of(1), epochs1(tp1)) assertEquals(Optional.of(2), epochs1(tp2)) - val response = new util.LinkedHashMap[TopicPartition, FetchResponse.PartitionData[Records]] - response.put(tp0, new FetchResponse.PartitionData(Errors.NONE, 100, 100, - 100, null, null)) - response.put(tp1, new FetchResponse.PartitionData( - Errors.NONE, 10, 10, 10, null, null)) - response.put(tp2, new FetchResponse.PartitionData( - Errors.NONE, 5, 5, 5, null, null)) + val response = new util.LinkedHashMap[TopicPartition, FetchResponseData.PartitionData] + response.put(tp0, new FetchResponseData.PartitionData() + .setPartitionIndex(tp0.partition) + .setHighWatermark(100) + .setLastStableOffset(100) + .setLogStartOffset(100)) + response.put(tp1, new FetchResponseData.PartitionData() + .setPartitionIndex(tp1.partition) + .setHighWatermark(10) + .setLastStableOffset(10) + .setLogStartOffset(10)) + response.put(tp2, new FetchResponseData.PartitionData() + .setPartitionIndex(tp2.partition) + .setHighWatermark(5) + .setLastStableOffset(5) + .setLogStartOffset(5)) val sessionId = context1.updateAndGenerateResponseData(response).sessionId() @@ -220,10 +229,22 @@ class FetchSessionTest { assertEquals(Map(tp0 -> Optional.empty, tp1 -> Optional.empty, tp2 -> Optional.of(1)), cachedLastFetchedEpochs(context1)) - val response = new util.LinkedHashMap[TopicPartition, FetchResponse.PartitionData[Records]] - response.put(tp0, new FetchResponse.PartitionData(Errors.NONE, 100, 100, 100, null, null)) - response.put(tp1, new FetchResponse.PartitionData(Errors.NONE, 10, 10, 10, null, null)) - response.put(tp2, new FetchResponse.PartitionData(Errors.NONE, 5, 5, 5, null, null)) + val response = new util.LinkedHashMap[TopicPartition, FetchResponseData.PartitionData] + response.put(tp0, new FetchResponseData.PartitionData() + .setPartitionIndex(tp0.partition) + .setHighWatermark(100) + .setLastStableOffset(100) + .setLogStartOffset(100)) + response.put(tp1, new FetchResponseData.PartitionData() + .setPartitionIndex(tp1.partition) + .setHighWatermark(10) + .setLastStableOffset(10) + .setLogStartOffset(10)) + response.put(tp2, new FetchResponseData.PartitionData() + .setPartitionIndex(tp2.partition) + .setHighWatermark(5) + .setLastStableOffset(5) + .setLogStartOffset(5)) val sessionId = context1.updateAndGenerateResponseData(response).sessionId() @@ -275,15 +296,23 @@ class FetchSessionTest { }) assertEquals(0, context2.getFetchOffset(new TopicPartition("foo", 0)).get) assertEquals(10, context2.getFetchOffset(new TopicPartition("foo", 1)).get) - val respData2 = new util.LinkedHashMap[TopicPartition, FetchResponse.PartitionData[Records]] - respData2.put(new TopicPartition("foo", 0), new FetchResponse.PartitionData( - Errors.NONE, 100, 100, 100, null, null)) - respData2.put(new TopicPartition("foo", 1), new FetchResponse.PartitionData( - Errors.NONE, 10, 10, 10, null, null)) + val respData2 = new util.LinkedHashMap[TopicPartition, FetchResponseData.PartitionData] + respData2.put(new TopicPartition("foo", 0), + new FetchResponseData.PartitionData() + .setPartitionIndex(0) + .setHighWatermark(100) + .setLastStableOffset(100) + .setLogStartOffset(100)) + respData2.put(new TopicPartition("foo", 1), + new FetchResponseData.PartitionData() + .setPartitionIndex(1) + .setHighWatermark(10) + .setLastStableOffset(10) + .setLogStartOffset(10)) val resp2 = context2.updateAndGenerateResponseData(respData2) assertEquals(Errors.NONE, resp2.error()) assertTrue(resp2.sessionId() != INVALID_SESSION_ID) - assertEquals(respData2, resp2.responseData()) + assertEquals(respData2, resp2.responseData) // Test trying to create a new session with an invalid epoch val context3 = fetchManager.newContext( @@ -314,7 +343,7 @@ class FetchSessionTest { val resp5 = context5.updateAndGenerateResponseData(respData2) assertEquals(Errors.NONE, resp5.error()) assertEquals(resp2.sessionId(), resp5.sessionId()) - assertEquals(0, resp5.responseData().size()) + assertEquals(0, resp5.responseData.size()) // Test setting an invalid fetch session epoch. val context6 = fetchManager.newContext( @@ -345,11 +374,19 @@ class FetchSessionTest { new JFetchMetadata(prevSessionId, FINAL_EPOCH), reqData8, EMPTY_PART_LIST, false) assertEquals(classOf[SessionlessFetchContext], context8.getClass) assertEquals(0, cache.size) - val respData8 = new util.LinkedHashMap[TopicPartition, FetchResponse.PartitionData[Records]] + val respData8 = new util.LinkedHashMap[TopicPartition, FetchResponseData.PartitionData] respData8.put(new TopicPartition("bar", 0), - new FetchResponse.PartitionData(Errors.NONE, 100, 100, 100, null, null)) + new FetchResponseData.PartitionData() + .setPartitionIndex(0) + .setHighWatermark(100) + .setLastStableOffset(100) + .setLogStartOffset(100)) respData8.put(new TopicPartition("bar", 1), - new FetchResponse.PartitionData(Errors.NONE, 100, 100, 100, null, null)) + new FetchResponseData.PartitionData() + .setPartitionIndex(1) + .setHighWatermark(100) + .setLastStableOffset(100) + .setLogStartOffset(100)) val resp8 = context8.updateAndGenerateResponseData(respData8) assertEquals(Errors.NONE, resp8.error) nextSessionId = resp8.sessionId @@ -370,15 +407,21 @@ class FetchSessionTest { Optional.empty())) val context1 = fetchManager.newContext(JFetchMetadata.INITIAL, reqData1, EMPTY_PART_LIST, false) assertEquals(classOf[FullFetchContext], context1.getClass) - val respData1 = new util.LinkedHashMap[TopicPartition, FetchResponse.PartitionData[Records]] - respData1.put(new TopicPartition("foo", 0), new FetchResponse.PartitionData( - Errors.NONE, 100, 100, 100, null, null)) - respData1.put(new TopicPartition("foo", 1), new FetchResponse.PartitionData( - Errors.NONE, 10, 10, 10, null, null)) + val respData1 = new util.LinkedHashMap[TopicPartition, FetchResponseData.PartitionData] + respData1.put(new TopicPartition("foo", 0), new FetchResponseData.PartitionData() + .setPartitionIndex(0) + .setHighWatermark(100) + .setLastStableOffset(100) + .setLogStartOffset(100)) + respData1.put(new TopicPartition("foo", 1), new FetchResponseData.PartitionData() + .setPartitionIndex(1) + .setHighWatermark(10) + .setLastStableOffset(10) + .setLogStartOffset(10)) val resp1 = context1.updateAndGenerateResponseData(respData1) assertEquals(Errors.NONE, resp1.error()) assertTrue(resp1.sessionId() != INVALID_SESSION_ID) - assertEquals(2, resp1.responseData().size()) + assertEquals(2, resp1.responseData.size()) // Create an incremental fetch request that removes foo-0 and adds bar-0 val reqData2 = new util.LinkedHashMap[TopicPartition, FetchRequest.PartitionData] @@ -391,18 +434,26 @@ class FetchSessionTest { assertEquals(classOf[IncrementalFetchContext], context2.getClass) val parts2 = Set(new TopicPartition("foo", 1), new TopicPartition("bar", 0)) val reqData2Iter = parts2.iterator - context2.foreachPartition((topicPart, data) => { + context2.foreachPartition((topicPart, _) => { assertEquals(reqData2Iter.next(), topicPart) }) assertEquals(None, context2.getFetchOffset(new TopicPartition("foo", 0))) assertEquals(10, context2.getFetchOffset(new TopicPartition("foo", 1)).get) assertEquals(15, context2.getFetchOffset(new TopicPartition("bar", 0)).get) assertEquals(None, context2.getFetchOffset(new TopicPartition("bar", 2))) - val respData2 = new util.LinkedHashMap[TopicPartition, FetchResponse.PartitionData[Records]] - respData2.put(new TopicPartition("foo", 1), new FetchResponse.PartitionData( - Errors.NONE, 10, 10, 10, null, null)) - respData2.put(new TopicPartition("bar", 0), new FetchResponse.PartitionData( - Errors.NONE, 10, 10, 10, null, null)) + val respData2 = new util.LinkedHashMap[TopicPartition, FetchResponseData.PartitionData] + respData2.put(new TopicPartition("foo", 1), + new FetchResponseData.PartitionData() + .setPartitionIndex(1) + .setHighWatermark(10) + .setLastStableOffset(10) + .setLogStartOffset(10)) + respData2.put(new TopicPartition("bar", 0), + new FetchResponseData.PartitionData() + .setPartitionIndex(0) + .setHighWatermark(10) + .setLastStableOffset(10) + .setLogStartOffset(10)) val resp2 = context2.updateAndGenerateResponseData(respData2) assertEquals(Errors.NONE, resp2.error) assertEquals(1, resp2.responseData.size) @@ -424,15 +475,21 @@ class FetchSessionTest { Optional.empty())) val session1context1 = fetchManager.newContext(JFetchMetadata.INITIAL, session1req, EMPTY_PART_LIST, false) assertEquals(classOf[FullFetchContext], session1context1.getClass) - val respData1 = new util.LinkedHashMap[TopicPartition, FetchResponse.PartitionData[Records]] - respData1.put(new TopicPartition("foo", 0), new FetchResponse.PartitionData( - Errors.NONE, 100, 100, 100, null, null)) - respData1.put(new TopicPartition("foo", 1), new FetchResponse.PartitionData( - Errors.NONE, 10, 10, 10, null, null)) + val respData1 = new util.LinkedHashMap[TopicPartition, FetchResponseData.PartitionData] + respData1.put(new TopicPartition("foo", 0), new FetchResponseData.PartitionData() + .setPartitionIndex(0) + .setHighWatermark(100) + .setLastStableOffset(100) + .setLogStartOffset(100)) + respData1.put(new TopicPartition("foo", 1), new FetchResponseData.PartitionData() + .setPartitionIndex(1) + .setHighWatermark(10) + .setLastStableOffset(10) + .setLogStartOffset(10)) val session1resp = session1context1.updateAndGenerateResponseData(respData1) assertEquals(Errors.NONE, session1resp.error()) assertTrue(session1resp.sessionId() != INVALID_SESSION_ID) - assertEquals(2, session1resp.responseData().size()) + assertEquals(2, session1resp.responseData.size) // check session entered into case assertTrue(cache.get(session1resp.sessionId()).isDefined) @@ -446,15 +503,22 @@ class FetchSessionTest { Optional.empty())) val session2context = fetchManager.newContext(JFetchMetadata.INITIAL, session1req, EMPTY_PART_LIST, false) assertEquals(classOf[FullFetchContext], session2context.getClass) - val session2RespData = new util.LinkedHashMap[TopicPartition, FetchResponse.PartitionData[Records]] - session2RespData.put(new TopicPartition("foo", 0), new FetchResponse.PartitionData( - Errors.NONE, 100, 100, 100, null, null)) - session2RespData.put(new TopicPartition("foo", 1), new FetchResponse.PartitionData( - Errors.NONE, 10, 10, 10, null, null)) + val session2RespData = new util.LinkedHashMap[TopicPartition, FetchResponseData.PartitionData] + session2RespData.put(new TopicPartition("foo", 0), + new FetchResponseData.PartitionData() + .setPartitionIndex(0) + .setHighWatermark(100) + .setLastStableOffset(100) + .setLogStartOffset(100)) + session2RespData.put(new TopicPartition("foo", 1), new FetchResponseData.PartitionData() + .setPartitionIndex(1) + .setHighWatermark(10) + .setLastStableOffset(10) + .setLogStartOffset(10)) val session2resp = session2context.updateAndGenerateResponseData(respData1) assertEquals(Errors.NONE, session2resp.error()) assertTrue(session2resp.sessionId() != INVALID_SESSION_ID) - assertEquals(2, session2resp.responseData().size()) + assertEquals(2, session2resp.responseData.size) // both newly created entries are present in cache assertTrue(cache.get(session1resp.sessionId()).isDefined) @@ -481,19 +545,25 @@ class FetchSessionTest { Optional.empty())) val session3context = fetchManager.newContext(JFetchMetadata.INITIAL, session3req, EMPTY_PART_LIST, false) assertEquals(classOf[FullFetchContext], session3context.getClass) - val respData3 = new util.LinkedHashMap[TopicPartition, FetchResponse.PartitionData[Records]] - respData3.put(new TopicPartition("foo", 0), new FetchResponse.PartitionData( - Errors.NONE, 100, 100, 100, null, null)) - respData3.put(new TopicPartition("foo", 1), new FetchResponse.PartitionData( - Errors.NONE, 10, 10, 10, null, null)) + val respData3 = new util.LinkedHashMap[TopicPartition, FetchResponseData.PartitionData] + respData3.put(new TopicPartition("foo", 0), new FetchResponseData.PartitionData() + .setPartitionIndex(0) + .setHighWatermark(100) + .setLastStableOffset(100) + .setLogStartOffset(100)) + respData3.put(new TopicPartition("foo", 1), + new FetchResponseData.PartitionData() + .setPartitionIndex(1) + .setHighWatermark(10) + .setLastStableOffset(10) + .setLogStartOffset(10)) val session3resp = session3context.updateAndGenerateResponseData(respData3) assertEquals(Errors.NONE, session3resp.error()) assertTrue(session3resp.sessionId() != INVALID_SESSION_ID) - assertEquals(2, session3resp.responseData().size()) + assertEquals(2, session3resp.responseData.size) assertTrue(cache.get(session1resp.sessionId()).isDefined) - assertFalse(cache.get(session2resp.sessionId()).isDefined, - "session 2 should have been evicted by latest session, as session 1 was used more recently") + assertFalse(cache.get(session2resp.sessionId()).isDefined, "session 2 should have been evicted by latest session, as session 1 was used more recently") assertTrue(cache.get(session3resp.sessionId()).isDefined) } @@ -512,15 +582,21 @@ class FetchSessionTest { Optional.empty())) val session1context = fetchManager.newContext(JFetchMetadata.INITIAL, session1req, EMPTY_PART_LIST, true) assertEquals(classOf[FullFetchContext], session1context.getClass) - val respData1 = new util.LinkedHashMap[TopicPartition, FetchResponse.PartitionData[Records]] - respData1.put(new TopicPartition("foo", 0), new FetchResponse.PartitionData( - Errors.NONE, 100, 100, 100, null, null)) - respData1.put(new TopicPartition("foo", 1), new FetchResponse.PartitionData( - Errors.NONE, 10, 10, 10, null, null)) + val respData1 = new util.LinkedHashMap[TopicPartition, FetchResponseData.PartitionData] + respData1.put(new TopicPartition("foo", 0), new FetchResponseData.PartitionData() + .setPartitionIndex(0) + .setHighWatermark(100) + .setLastStableOffset(100) + .setLogStartOffset(100)) + respData1.put(new TopicPartition("foo", 1), new FetchResponseData.PartitionData() + .setPartitionIndex(1) + .setHighWatermark(10) + .setLastStableOffset(10) + .setLogStartOffset(10)) val session1resp = session1context.updateAndGenerateResponseData(respData1) assertEquals(Errors.NONE, session1resp.error()) assertTrue(session1resp.sessionId() != INVALID_SESSION_ID) - assertEquals(2, session1resp.responseData().size()) + assertEquals(2, session1resp.responseData.size) assertEquals(1, cache.size) // move time forward to age session 1 a little compared to session 2 @@ -534,15 +610,23 @@ class FetchSessionTest { Optional.empty())) val session2context = fetchManager.newContext(JFetchMetadata.INITIAL, session1req, EMPTY_PART_LIST, false) assertEquals(classOf[FullFetchContext], session2context.getClass) - val session2RespData = new util.LinkedHashMap[TopicPartition, FetchResponse.PartitionData[Records]] - session2RespData.put(new TopicPartition("foo", 0), new FetchResponse.PartitionData( - Errors.NONE, 100, 100, 100, null, null)) - session2RespData.put(new TopicPartition("foo", 1), new FetchResponse.PartitionData( - Errors.NONE, 10, 10, 10, null, null)) + val session2RespData = new util.LinkedHashMap[TopicPartition, FetchResponseData.PartitionData] + session2RespData.put(new TopicPartition("foo", 0), + new FetchResponseData.PartitionData() + .setPartitionIndex(0) + .setHighWatermark(100) + .setLastStableOffset(100) + .setLogStartOffset(100)) + session2RespData.put(new TopicPartition("foo", 1), + new FetchResponseData.PartitionData() + .setPartitionIndex(1) + .setHighWatermark(10) + .setLastStableOffset(10) + .setLogStartOffset(10)) val session2resp = session2context.updateAndGenerateResponseData(respData1) assertEquals(Errors.NONE, session2resp.error()) assertTrue(session2resp.sessionId() != INVALID_SESSION_ID) - assertEquals(2, session2resp.responseData().size()) + assertEquals(2, session2resp.responseData.size) // both newly created entries are present in cache assertTrue(cache.get(session1resp.sessionId()).isDefined) @@ -558,21 +642,28 @@ class FetchSessionTest { Optional.empty())) val session3context = fetchManager.newContext(JFetchMetadata.INITIAL, session3req, EMPTY_PART_LIST, true) assertEquals(classOf[FullFetchContext], session3context.getClass) - val respData3 = new util.LinkedHashMap[TopicPartition, FetchResponse.PartitionData[Records]] - respData3.put(new TopicPartition("foo", 0), new FetchResponse.PartitionData( - Errors.NONE, 100, 100, 100, null, null)) - respData3.put(new TopicPartition("foo", 1), new FetchResponse.PartitionData( - Errors.NONE, 10, 10, 10, null, null)) + val respData3 = new util.LinkedHashMap[TopicPartition, FetchResponseData.PartitionData] + respData3.put(new TopicPartition("foo", 0), + new FetchResponseData.PartitionData() + .setPartitionIndex(0) + .setHighWatermark(100) + .setLastStableOffset(100) + .setLogStartOffset(100)) + respData3.put(new TopicPartition("foo", 1), + new FetchResponseData.PartitionData() + .setPartitionIndex(1) + .setHighWatermark(10) + .setLastStableOffset(10) + .setLogStartOffset(10)) val session3resp = session3context.updateAndGenerateResponseData(respData3) assertEquals(Errors.NONE, session3resp.error()) assertTrue(session3resp.sessionId() != INVALID_SESSION_ID) - assertEquals(2, session3resp.responseData().size()) + assertEquals(2, session3resp.responseData.size) assertTrue(cache.get(session1resp.sessionId()).isDefined) // even though session 2 is more recent than session 1, and has not reached expiry time, it is less // privileged than session 2, and thus session 3 should be entered and session 2 evicted. - assertFalse(cache.get(session2resp.sessionId()).isDefined, - "session 2 should have been evicted by session 3") + assertFalse(cache.get(session2resp.sessionId()).isDefined, "session 2 should have been evicted by session 3") assertTrue(cache.get(session3resp.sessionId()).isDefined) assertEquals(2, cache.size) @@ -586,18 +677,25 @@ class FetchSessionTest { Optional.empty())) val session4context = fetchManager.newContext(JFetchMetadata.INITIAL, session4req, EMPTY_PART_LIST, true) assertEquals(classOf[FullFetchContext], session4context.getClass) - val respData4 = new util.LinkedHashMap[TopicPartition, FetchResponse.PartitionData[Records]] - respData4.put(new TopicPartition("foo", 0), new FetchResponse.PartitionData( - Errors.NONE, 100, 100, 100, null, null)) - respData4.put(new TopicPartition("foo", 1), new FetchResponse.PartitionData( - Errors.NONE, 10, 10, 10, null, null)) + val respData4 = new util.LinkedHashMap[TopicPartition, FetchResponseData.PartitionData] + respData4.put(new TopicPartition("foo", 0), + new FetchResponseData.PartitionData() + .setPartitionIndex(0) + .setHighWatermark(100) + .setLastStableOffset(100) + .setLogStartOffset(100)) + respData4.put(new TopicPartition("foo", 1), + new FetchResponseData.PartitionData() + .setPartitionIndex(1) + .setHighWatermark(10) + .setLastStableOffset(10) + .setLogStartOffset(10)) val session4resp = session3context.updateAndGenerateResponseData(respData4) assertEquals(Errors.NONE, session4resp.error()) assertTrue(session4resp.sessionId() != INVALID_SESSION_ID) - assertEquals(2, session4resp.responseData().size()) + assertEquals(2, session4resp.responseData.size) - assertFalse(cache.get(session1resp.sessionId()).isDefined, - "session 1 should have been evicted by session 4 even though it is privileged as it has hit eviction time") + assertFalse(cache.get(session1resp.sessionId()).isDefined, "session 1 should have been evicted by session 4 even though it is privileged as it has hit eviction time") assertTrue(cache.get(session3resp.sessionId()).isDefined) assertTrue(cache.get(session4resp.sessionId()).isDefined) assertEquals(2, cache.size) @@ -617,11 +715,17 @@ class FetchSessionTest { Optional.empty())) val context1 = fetchManager.newContext(JFetchMetadata.INITIAL, reqData1, EMPTY_PART_LIST, false) assertEquals(classOf[FullFetchContext], context1.getClass) - val respData1 = new util.LinkedHashMap[TopicPartition, FetchResponse.PartitionData[Records]] - respData1.put(new TopicPartition("foo", 0), new FetchResponse.PartitionData( - Errors.NONE, 100, 100, 100, null, null)) - respData1.put(new TopicPartition("foo", 1), new FetchResponse.PartitionData( - Errors.NONE, 10, 10, 10, null, null)) + val respData1 = new util.LinkedHashMap[TopicPartition, FetchResponseData.PartitionData] + respData1.put(new TopicPartition("foo", 0), new FetchResponseData.PartitionData() + .setPartitionIndex(0) + .setHighWatermark(100) + .setLastStableOffset(100) + .setLogStartOffset(100)) + respData1.put(new TopicPartition("foo", 1), new FetchResponseData.PartitionData() + .setPartitionIndex(1) + .setHighWatermark(10) + .setLastStableOffset(10) + .setLogStartOffset(10)) val resp1 = context1.updateAndGenerateResponseData(respData1) assertEquals(Errors.NONE, resp1.error) assertTrue(resp1.sessionId() != INVALID_SESSION_ID) @@ -636,10 +740,10 @@ class FetchSessionTest { val context2 = fetchManager.newContext( new JFetchMetadata(resp1.sessionId, 1), reqData2, removed2, false) assertEquals(classOf[SessionlessFetchContext], context2.getClass) - val respData2 = new util.LinkedHashMap[TopicPartition, FetchResponse.PartitionData[Records]] + val respData2 = new util.LinkedHashMap[TopicPartition, FetchResponseData.PartitionData] val resp2 = context2.updateAndGenerateResponseData(respData2) assertEquals(INVALID_SESSION_ID, resp2.sessionId) - assertTrue(resp2.responseData().isEmpty) + assertTrue(resp2.responseData.isEmpty) assertEquals(0, cache.size) } @@ -658,12 +762,19 @@ class FetchSessionTest { // Full fetch context returns all partitions in the response val context1 = fetchManager.newContext(JFetchMetadata.INITIAL, reqData, EMPTY_PART_LIST, isFollower = false) assertEquals(classOf[FullFetchContext], context1.getClass) - val respData = new util.LinkedHashMap[TopicPartition, FetchResponse.PartitionData[Records]] - respData.put(tp1, new FetchResponse.PartitionData(Errors.NONE, - 105, 105, 0, Optional.empty(), Collections.emptyList(), Optional.empty(), null)) - val divergingEpoch = Optional.of(new FetchResponseData.EpochEndOffset().setEpoch(3).setEndOffset(90)) - respData.put(tp2, new FetchResponse.PartitionData(Errors.NONE, - 105, 105, 0, Optional.empty(), Collections.emptyList(), divergingEpoch, null)) + val respData = new util.LinkedHashMap[TopicPartition, FetchResponseData.PartitionData] + respData.put(tp1, new FetchResponseData.PartitionData() + .setPartitionIndex(tp1.partition) + .setHighWatermark(105) + .setLastStableOffset(105) + .setLogStartOffset(0)) + val divergingEpoch = new FetchResponseData.EpochEndOffset().setEpoch(3).setEndOffset(90) + respData.put(tp2, new FetchResponseData.PartitionData() + .setPartitionIndex(tp2.partition) + .setHighWatermark(105) + .setLastStableOffset(105) + .setLogStartOffset(0) + .setDivergingEpoch(divergingEpoch)) val resp1 = context1.updateAndGenerateResponseData(respData) assertEquals(Errors.NONE, resp1.error) assertNotEquals(INVALID_SESSION_ID, resp1.sessionId) @@ -679,8 +790,12 @@ class FetchSessionTest { assertEquals(Collections.singleton(tp2), resp2.responseData.keySet) // All partitions with divergent epoch should be returned. - respData.put(tp1, new FetchResponse.PartitionData(Errors.NONE, - 105, 105, 0, Optional.empty(), Collections.emptyList(), divergingEpoch, null)) + respData.put(tp1, new FetchResponseData.PartitionData() + .setPartitionIndex(tp1.partition) + .setHighWatermark(105) + .setLastStableOffset(105) + .setLogStartOffset(0) + .setDivergingEpoch(divergingEpoch)) val resp3 = context2.updateAndGenerateResponseData(respData) assertEquals(Errors.NONE, resp3.error) assertEquals(resp1.sessionId, resp3.sessionId) @@ -688,8 +803,11 @@ class FetchSessionTest { // Partitions that meet other conditions should be returned regardless of whether // divergingEpoch is set or not. - respData.put(tp1, new FetchResponse.PartitionData(Errors.NONE, - 110, 110, 0, Optional.empty(), Collections.emptyList(), Optional.empty(), null)) + respData.put(tp1, new FetchResponseData.PartitionData() + .setPartitionIndex(tp1.partition) + .setHighWatermark(110) + .setLastStableOffset(110) + .setLogStartOffset(0)) val resp4 = context2.updateAndGenerateResponseData(respData) assertEquals(Errors.NONE, resp4.error) assertEquals(resp1.sessionId, resp4.sessionId) diff --git a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala index e995c69302ce8..4d48fecf99a2b 100644 --- a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala +++ b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala @@ -1072,7 +1072,7 @@ class KafkaApisTest { val response = capturedResponse.getValue.asInstanceOf[OffsetCommitResponse] assertEquals(Errors.UNKNOWN_TOPIC_OR_PARTITION, - Errors.forCode(response.data().topics().get(0).partitions().get(0).errorCode())) + Errors.forCode(response.data.topics().get(0).partitions().get(0).errorCode)) } checkInvalidPartition(-1) @@ -1425,9 +1425,9 @@ class KafkaApisTest { val produceRequest = ProduceRequest.forCurrentMagic(new ProduceRequestData() .setTopicData(new ProduceRequestData.TopicProduceDataCollection( Collections.singletonList(new ProduceRequestData.TopicProduceData() - .setName(tp.topic()).setPartitionData(Collections.singletonList( + .setName(tp.topic).setPartitionData(Collections.singletonList( new ProduceRequestData.PartitionProduceData() - .setIndex(tp.partition()) + .setIndex(tp.partition) .setRecords(MemoryRecords.withRecords(CompressionType.NONE, new SimpleRecord("test".getBytes)))))) .iterator)) .setAcks(1.toShort) @@ -1632,21 +1632,21 @@ class KafkaApisTest { val topicStates = Seq( new StopReplicaTopicState() - .setTopicName(groupMetadataPartition.topic()) + .setTopicName(groupMetadataPartition.topic) .setPartitionStates(Seq(new StopReplicaPartitionState() - .setPartitionIndex(groupMetadataPartition.partition()) + .setPartitionIndex(groupMetadataPartition.partition) .setLeaderEpoch(leaderEpoch) .setDeletePartition(deletePartition)).asJava), new StopReplicaTopicState() - .setTopicName(txnStatePartition.topic()) + .setTopicName(txnStatePartition.topic) .setPartitionStates(Seq(new StopReplicaPartitionState() - .setPartitionIndex(txnStatePartition.partition()) + .setPartitionIndex(txnStatePartition.partition) .setLeaderEpoch(leaderEpoch) .setDeletePartition(deletePartition)).asJava), new StopReplicaTopicState() - .setTopicName(fooPartition.topic()) + .setTopicName(fooPartition.topic) .setPartitionStates(Seq(new StopReplicaPartitionState() - .setPartitionIndex(fooPartition.partition()) + .setPartitionIndex(fooPartition.partition) .setLeaderEpoch(leaderEpoch) .setDeletePartition(deletePartition)).asJava) ).asJava @@ -1806,8 +1806,8 @@ class KafkaApisTest { val response = capturedResponse.getValue.asInstanceOf[DescribeGroupsResponse] - val group = response.data().groups().get(0) - assertEquals(Errors.NONE, Errors.forCode(group.errorCode())) + val group = response.data.groups().get(0) + assertEquals(Errors.NONE, Errors.forCode(group.errorCode)) assertEquals(groupId, group.groupId()) assertEquals(groupSummary.state, group.groupState()) assertEquals(groupSummary.protocolType, group.protocolType()) @@ -1873,7 +1873,7 @@ class KafkaApisTest { val response = capturedResponse.getValue.asInstanceOf[OffsetDeleteResponse] def errorForPartition(topic: String, partition: Int): Errors = { - Errors.forCode(response.data.topics.find(topic).partitions.find(partition).errorCode()) + Errors.forCode(response.data.topics.find(topic).partitions.find(partition).errorCode) } assertEquals(2, response.data.topics.size) @@ -1914,7 +1914,7 @@ class KafkaApisTest { val response = capturedResponse.getValue.asInstanceOf[OffsetDeleteResponse] assertEquals(Errors.UNKNOWN_TOPIC_OR_PARTITION, - Errors.forCode(response.data.topics.find(topic).partitions.find(invalidPartitionId).errorCode())) + Errors.forCode(response.data.topics.find(topic).partitions.find(invalidPartitionId).errorCode)) } checkInvalidPartition(-1) @@ -1942,7 +1942,7 @@ class KafkaApisTest { val response = capturedResponse.getValue.asInstanceOf[OffsetDeleteResponse] - assertEquals(Errors.GROUP_ID_NOT_FOUND, Errors.forCode(response.data.errorCode())) + assertEquals(Errors.GROUP_ID_NOT_FOUND, Errors.forCode(response.data.errorCode)) } private def testListOffsetFailedGetLeaderReplica(error: Errors): Unit = { @@ -2130,16 +2130,15 @@ class KafkaApisTest { EasyMock.replay(replicaManager, clientQuotaManager, clientRequestQuotaManager, requestChannel, fetchManager) createKafkaApis().handleFetchRequest(request) - val response = capturedResponse.getValue.asInstanceOf[FetchResponse[BaseRecords]] + val response = capturedResponse.getValue.asInstanceOf[FetchResponse] assertTrue(response.responseData.containsKey(tp)) val partitionData = response.responseData.get(tp) - assertEquals(Errors.NONE, partitionData.error) + assertEquals(Errors.NONE.code, partitionData.errorCode) assertEquals(hw, partitionData.highWatermark) assertEquals(-1, partitionData.lastStableOffset) assertEquals(0, partitionData.logStartOffset) - assertEquals(timestamp, - partitionData.records.asInstanceOf[MemoryRecords].batches.iterator.next.maxTimestamp) + assertEquals(timestamp, FetchResponse.recordsOrFail(partitionData).batches.iterator.next.maxTimestamp) assertNull(partitionData.abortedTransactions) } @@ -2563,7 +2562,7 @@ class KafkaApisTest { .setPartitions(Collections.singletonList( new OffsetCommitResponseData.OffsetCommitResponsePartition() .setPartitionIndex(0) - .setErrorCode(Errors.UNSUPPORTED_VERSION.code()) + .setErrorCode(Errors.UNSUPPORTED_VERSION.code) )) ) val response = capturedResponse.getValue.asInstanceOf[OffsetCommitResponse] @@ -2871,9 +2870,9 @@ class KafkaApisTest { val fooPartition = new TopicPartition("foo", 0) val topicStates = Seq( new StopReplicaTopicState() - .setTopicName(fooPartition.topic()) + .setTopicName(fooPartition.topic) .setPartitionStates(Seq(new StopReplicaPartitionState() - .setPartitionIndex(fooPartition.partition()) + .setPartitionIndex(fooPartition.partition) .setLeaderEpoch(1) .setDeletePartition(false)).asJava) ).asJava @@ -3246,15 +3245,18 @@ class KafkaApisTest { @Test def testSizeOfThrottledPartitions(): Unit = { - def fetchResponse(data: Map[TopicPartition, String]): FetchResponse[Records] = { - val responseData = new util.LinkedHashMap[TopicPartition, FetchResponse.PartitionData[Records]]( + + def fetchResponse(data: Map[TopicPartition, String]): FetchResponse = { + val responseData = new util.LinkedHashMap[TopicPartition, FetchResponseData.PartitionData]( data.map { case (tp, raw) => - tp -> new FetchResponse.PartitionData(Errors.NONE, - 105, 105, 0, Optional.empty(), Collections.emptyList(), Optional.empty(), - MemoryRecords.withRecords(CompressionType.NONE, - new SimpleRecord(100, raw.getBytes(StandardCharsets.UTF_8))).asInstanceOf[Records]) + tp -> new FetchResponseData.PartitionData() + .setPartitionIndex(tp.partition) + .setHighWatermark(105) + .setLastStableOffset(105) + .setLogStartOffset(0) + .setRecords(MemoryRecords.withRecords(CompressionType.NONE, new SimpleRecord(100, raw.getBytes(StandardCharsets.UTF_8)))) }.toMap.asJava) - new FetchResponse(Errors.NONE, responseData, 100, 100) + FetchResponse.of(Errors.NONE, 100, 100, responseData) } val throttledPartition = new TopicPartition("throttledData", 0) diff --git a/core/src/test/scala/unit/kafka/server/LogOffsetTest.scala b/core/src/test/scala/unit/kafka/server/LogOffsetTest.scala index 1ce8006c63090..746374583434d 100755 --- a/core/src/test/scala/unit/kafka/server/LogOffsetTest.scala +++ b/core/src/test/scala/unit/kafka/server/LogOffsetTest.scala @@ -17,26 +17,22 @@ package kafka.server -import java.io.File -import java.util.concurrent.atomic.AtomicInteger -import java.util.{Optional, Properties, Random} - import kafka.log.{ClientRecordDeletion, Log, LogSegment} import kafka.utils.{MockTime, TestUtils} -import org.apache.kafka.common.message.ListOffsetsRequestData.ListOffsetsTopic -import org.apache.kafka.common.message.ListOffsetsRequestData.ListOffsetsPartition -import org.apache.kafka.common.message.ListOffsetsResponseData.ListOffsetsPartitionResponse -import org.apache.kafka.common.message.ListOffsetsResponseData.ListOffsetsTopicResponse +import org.apache.kafka.common.message.ListOffsetsRequestData.{ListOffsetsPartition, ListOffsetsTopic} +import org.apache.kafka.common.message.ListOffsetsResponseData.{ListOffsetsPartitionResponse, ListOffsetsTopicResponse} import org.apache.kafka.common.protocol.Errors -import org.apache.kafka.common.record.MemoryRecords import org.apache.kafka.common.requests.{FetchRequest, FetchResponse, ListOffsetsRequest, ListOffsetsResponse} import org.apache.kafka.common.{IsolationLevel, TopicPartition} import org.easymock.{EasyMock, IAnswer} import org.junit.jupiter.api.Assertions._ import org.junit.jupiter.api.Test -import scala.jdk.CollectionConverters._ +import java.io.File +import java.util.concurrent.atomic.AtomicInteger +import java.util.{Optional, Properties, Random} import scala.collection.mutable.Buffer +import scala.jdk.CollectionConverters._ class LogOffsetTest extends BaseRequestTest { @@ -127,7 +123,7 @@ class LogOffsetTest extends BaseRequestTest { Map(topicPartition -> new FetchRequest.PartitionData(consumerOffsets.head, FetchRequest.INVALID_LOG_START_OFFSET, 300 * 1024, Optional.empty())).asJava).build() val fetchResponse = sendFetchRequest(fetchRequest) - assertFalse(fetchResponse.responseData.get(topicPartition).records.batches.iterator.hasNext) + assertFalse(FetchResponse.recordsOrFail(fetchResponse.responseData.get(topicPartition)).batches.iterator.hasNext) } @Test @@ -251,8 +247,8 @@ class LogOffsetTest extends BaseRequestTest { connectAndReceive[ListOffsetsResponse](request) } - private def sendFetchRequest(request: FetchRequest): FetchResponse[MemoryRecords] = { - connectAndReceive[FetchResponse[MemoryRecords]](request) + private def sendFetchRequest(request: FetchRequest): FetchResponse = { + connectAndReceive[FetchResponse](request) } private def buildTargetTimes(tp: TopicPartition, timestamp: Long, maxNumOffsets: Int): List[ListOffsetsTopic] = { diff --git a/core/src/test/scala/unit/kafka/server/ReplicaFetcherThreadTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaFetcherThreadTest.scala index fda8b64a53d2b..af3989cd827ba 100644 --- a/core/src/test/scala/unit/kafka/server/ReplicaFetcherThreadTest.scala +++ b/core/src/test/scala/unit/kafka/server/ReplicaFetcherThreadTest.scala @@ -16,9 +16,6 @@ */ package kafka.server -import java.nio.charset.StandardCharsets -import java.util.{Collections, Optional} - import kafka.api.{ApiVersion, KAFKA_2_6_IV0} import kafka.cluster.{BrokerEndPoint, Partition} import kafka.log.{Log, LogAppendInfo, LogManager} @@ -32,17 +29,18 @@ import org.apache.kafka.common.message.OffsetForLeaderEpochResponseData.EpochEnd import org.apache.kafka.common.metrics.Metrics import org.apache.kafka.common.protocol.Errors._ import org.apache.kafka.common.protocol.{ApiKeys, Errors} -import org.apache.kafka.common.record.{CompressionType, MemoryRecords, Records, SimpleRecord} +import org.apache.kafka.common.record.{CompressionType, MemoryRecords, SimpleRecord} import org.apache.kafka.common.requests.OffsetsForLeaderEpochResponse.{UNDEFINED_EPOCH, UNDEFINED_EPOCH_OFFSET} -import org.apache.kafka.common.requests.FetchResponse import org.apache.kafka.common.utils.SystemTime import org.easymock.EasyMock._ import org.easymock.{Capture, CaptureType} import org.junit.jupiter.api.Assertions._ import org.junit.jupiter.api.{AfterEach, Test} -import scala.jdk.CollectionConverters._ +import java.nio.charset.StandardCharsets +import java.util.Collections import scala.collection.{Map, mutable} +import scala.jdk.CollectionConverters._ class ReplicaFetcherThreadTest { @@ -531,16 +529,18 @@ class ReplicaFetcherThreadTest { assertEquals(1, mockNetwork.fetchCount) partitions.foreach { tp => assertEquals(Fetching, thread.fetchState(tp).get.state) } - def partitionData(divergingEpoch: FetchResponseData.EpochEndOffset): FetchResponse.PartitionData[Records] = { - new FetchResponse.PartitionData[Records]( - Errors.NONE, 0, 0, 0, Optional.empty(), Collections.emptyList(), - Optional.of(divergingEpoch), MemoryRecords.EMPTY) + def partitionData(partition: Int, divergingEpoch: FetchResponseData.EpochEndOffset): FetchResponseData.PartitionData = { + new FetchResponseData.PartitionData() + .setPartitionIndex(partition) + .setLastStableOffset(0) + .setLogStartOffset(0) + .setDivergingEpoch(divergingEpoch) } // Loop 2 should truncate based on diverging epoch and continue to send fetch requests. mockNetwork.setFetchPartitionDataForNextResponse(Map( - t1p0 -> partitionData(new FetchResponseData.EpochEndOffset().setEpoch(4).setEndOffset(140)), - t1p1 -> partitionData(new FetchResponseData.EpochEndOffset().setEpoch(4).setEndOffset(141)) + t1p0 -> partitionData(t1p0.partition, new FetchResponseData.EpochEndOffset().setEpoch(4).setEndOffset(140)), + t1p1 -> partitionData(t1p1.partition, new FetchResponseData.EpochEndOffset().setEpoch(4).setEndOffset(141)) )) latestLogEpoch = Some(4) thread.doWork() @@ -555,8 +555,8 @@ class ReplicaFetcherThreadTest { // Loop 3 should truncate because of diverging epoch. Offset truncation is not complete // because divergent epoch is not known to follower. We truncate and stay in Fetching state. mockNetwork.setFetchPartitionDataForNextResponse(Map( - t1p0 -> partitionData(new FetchResponseData.EpochEndOffset().setEpoch(3).setEndOffset(130)), - t1p1 -> partitionData(new FetchResponseData.EpochEndOffset().setEpoch(3).setEndOffset(131)) + t1p0 -> partitionData(t1p0.partition, new FetchResponseData.EpochEndOffset().setEpoch(3).setEndOffset(130)), + t1p1 -> partitionData(t1p1.partition, new FetchResponseData.EpochEndOffset().setEpoch(3).setEndOffset(131)) )) thread.doWork() assertEquals(0, mockNetwork.epochFetchCount) @@ -569,8 +569,8 @@ class ReplicaFetcherThreadTest { // because divergent epoch is not known to follower. Last fetched epoch cannot be determined // from the log. We truncate and stay in Fetching state. mockNetwork.setFetchPartitionDataForNextResponse(Map( - t1p0 -> partitionData(new FetchResponseData.EpochEndOffset().setEpoch(2).setEndOffset(120)), - t1p1 -> partitionData(new FetchResponseData.EpochEndOffset().setEpoch(2).setEndOffset(121)) + t1p0 -> partitionData(t1p0.partition, new FetchResponseData.EpochEndOffset().setEpoch(2).setEndOffset(120)), + t1p1 -> partitionData(t1p1.partition, new FetchResponseData.EpochEndOffset().setEpoch(2).setEndOffset(121)) )) latestLogEpoch = None thread.doWork() @@ -963,9 +963,11 @@ class ReplicaFetcherThreadTest { val records = MemoryRecords.withRecords(CompressionType.NONE, new SimpleRecord(1000, "foo".getBytes(StandardCharsets.UTF_8))) - - val partitionData: thread.FetchData = new FetchResponse.PartitionData[Records]( - Errors.NONE, 0, 0, 0, Optional.empty(), Collections.emptyList(), records) + val partitionData: thread.FetchData = new FetchResponseData.PartitionData() + .setPartitionIndex(t1p0.partition) + .setLastStableOffset(0) + .setLogStartOffset(0) + .setRecords(records) thread.processPartitionData(t1p0, 0, partitionData) if (isReassigning) diff --git a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala index 9b289e578173d..76ce6c7a5f145 100644 --- a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala +++ b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala @@ -17,24 +17,17 @@ package kafka.server -import java.io.File -import java.net.InetAddress -import java.util.concurrent.atomic.{AtomicBoolean, AtomicReference} -import java.util.concurrent.{CountDownLatch, TimeUnit} -import java.util.{Collections, Optional, Properties} - import kafka.api._ -import kafka.log.{AppendOrigin, Log, LogConfig, LogManager, ProducerStateManager} import kafka.cluster.{BrokerEndPoint, Partition} -import kafka.log.LeaderOffsetIncremented +import kafka.log._ import kafka.server.QuotaFactory.{QuotaManagers, UnboundedQuota} -import kafka.server.checkpoints.LazyOffsetCheckpoints -import kafka.server.checkpoints.OffsetCheckpointFile +import kafka.server.checkpoints.{LazyOffsetCheckpoints, OffsetCheckpointFile} import kafka.server.epoch.util.ReplicaFetcherMockBlockingSend import kafka.server.metadata.CachedConfigRepository import kafka.utils.TestUtils.createBroker import kafka.utils.timer.MockTimer import kafka.utils.{MockScheduler, MockTime, TestUtils} +import org.apache.kafka.common.message.FetchResponseData import org.apache.kafka.common.message.LeaderAndIsrRequestData import org.apache.kafka.common.message.LeaderAndIsrRequestData.LeaderAndIsrPartitionState import org.apache.kafka.common.message.OffsetForLeaderEpochResponseData.EpochEndOffset @@ -45,21 +38,23 @@ import org.apache.kafka.common.record._ import org.apache.kafka.common.replica.ClientMetadata import org.apache.kafka.common.replica.ClientMetadata.DefaultClientMetadata import org.apache.kafka.common.requests.FetchRequest.PartitionData -import org.apache.kafka.common.requests.FetchResponse.AbortedTransaction import org.apache.kafka.common.requests.ProduceResponse.PartitionResponse import org.apache.kafka.common.requests._ import org.apache.kafka.common.security.auth.KafkaPrincipal -import org.apache.kafka.common.utils.Time -import org.apache.kafka.common.utils.Utils +import org.apache.kafka.common.utils.{Time, Utils} import org.apache.kafka.common.{IsolationLevel, Node, TopicPartition, Uuid} import org.easymock.EasyMock import org.junit.jupiter.api.Assertions._ import org.junit.jupiter.api.{AfterEach, BeforeEach, Test} import org.mockito.Mockito -import scala.collection.mutable +import java.io.File +import java.net.InetAddress +import java.util.concurrent.atomic.{AtomicBoolean, AtomicReference} +import java.util.concurrent.{CountDownLatch, TimeUnit} +import java.util.{Collections, Optional, Properties} +import scala.collection.{Map, Seq, mutable} import scala.jdk.CollectionConverters._ -import scala.collection.{Map, Seq} class ReplicaManagerTest { @@ -403,7 +398,7 @@ class ReplicaManagerTest { assertEquals(Errors.NONE, fetchData.error) assertTrue(fetchData.records.batches.asScala.isEmpty) assertEquals(Some(0), fetchData.lastStableOffset) - assertEquals(Some(List.empty[AbortedTransaction]), fetchData.abortedTransactions) + assertEquals(Some(List.empty[FetchResponseData.AbortedTransaction]), fetchData.abortedTransactions) // delayed fetch should timeout and return nothing consumerFetchResult = fetchAsConsumer(replicaManager, new TopicPartition(topic, 0), @@ -416,7 +411,7 @@ class ReplicaManagerTest { assertEquals(Errors.NONE, fetchData.error) assertTrue(fetchData.records.batches.asScala.isEmpty) assertEquals(Some(0), fetchData.lastStableOffset) - assertEquals(Some(List.empty[AbortedTransaction]), fetchData.abortedTransactions) + assertEquals(Some(List.empty[FetchResponseData.AbortedTransaction]), fetchData.abortedTransactions) // now commit the transaction val endTxnMarker = new EndTransactionMarker(ControlRecordType.COMMIT, 0) @@ -448,7 +443,7 @@ class ReplicaManagerTest { fetchData = consumerFetchResult.assertFired assertEquals(Errors.NONE, fetchData.error) assertEquals(Some(numRecords + 1), fetchData.lastStableOffset) - assertEquals(Some(List.empty[AbortedTransaction]), fetchData.abortedTransactions) + assertEquals(Some(List.empty[FetchResponseData.AbortedTransaction]), fetchData.abortedTransactions) assertEquals(numRecords + 1, fetchData.records.batches.asScala.size) } finally { replicaManager.shutdown(checkpointHW = false) diff --git a/core/src/test/scala/unit/kafka/server/UpdateFeaturesTest.scala b/core/src/test/scala/unit/kafka/server/UpdateFeaturesTest.scala index 3ede6af097f38..92ba0425dcb27 100644 --- a/core/src/test/scala/unit/kafka/server/UpdateFeaturesTest.scala +++ b/core/src/test/scala/unit/kafka/server/UpdateFeaturesTest.scala @@ -193,7 +193,7 @@ class UpdateFeaturesTest extends BaseRequestTest { new UpdateFeaturesRequest.Builder(new UpdateFeaturesRequestData().setFeatureUpdates(validUpdates)).build(), notControllerSocketServer) - assertEquals(Errors.NOT_CONTROLLER, Errors.forCode(response.data.errorCode())) + assertEquals(Errors.NOT_CONTROLLER, Errors.forCode(response.data.errorCode)) assertNotNull(response.data.errorMessage()) assertEquals(0, response.data.results.size) checkFeatures( diff --git a/core/src/test/scala/unit/kafka/server/epoch/util/ReplicaFetcherMockBlockingSend.scala b/core/src/test/scala/unit/kafka/server/epoch/util/ReplicaFetcherMockBlockingSend.scala index 384eacca568d1..b367887c6edb2 100644 --- a/core/src/test/scala/unit/kafka/server/epoch/util/ReplicaFetcherMockBlockingSend.scala +++ b/core/src/test/scala/unit/kafka/server/epoch/util/ReplicaFetcherMockBlockingSend.scala @@ -18,14 +18,12 @@ package kafka.server.epoch.util import java.net.SocketTimeoutException import java.util - import kafka.cluster.BrokerEndPoint import kafka.server.BlockingSend import org.apache.kafka.clients.{ClientRequest, ClientResponse, MockClient, NetworkClientUtils} -import org.apache.kafka.common.message.OffsetForLeaderEpochResponseData -import org.apache.kafka.common.message.OffsetForLeaderEpochResponseData.{OffsetForLeaderTopicResult, EpochEndOffset} +import org.apache.kafka.common.message.{FetchResponseData, OffsetForLeaderEpochResponseData} +import org.apache.kafka.common.message.OffsetForLeaderEpochResponseData.{EpochEndOffset, OffsetForLeaderTopicResult} import org.apache.kafka.common.protocol.{ApiKeys, Errors} -import org.apache.kafka.common.record.Records import org.apache.kafka.common.requests.AbstractRequest.Builder import org.apache.kafka.common.requests.{AbstractRequest, FetchResponse, OffsetsForLeaderEpochResponse, FetchMetadata => JFetchMetadata} import org.apache.kafka.common.utils.{SystemTime, Time} @@ -52,7 +50,7 @@ class ReplicaFetcherMockBlockingSend(offsets: java.util.Map[TopicPartition, Epoc var lastUsedOffsetForLeaderEpochVersion = -1 var callback: Option[() => Unit] = None var currentOffsets: util.Map[TopicPartition, EpochEndOffset] = offsets - var fetchPartitionData: Map[TopicPartition, FetchResponse.PartitionData[Records]] = Map.empty + var fetchPartitionData: Map[TopicPartition, FetchResponseData.PartitionData] = Map.empty private val sourceNode = new Node(sourceBroker.id, sourceBroker.host, sourceBroker.port) def setEpochRequestCallback(postEpochFunction: () => Unit): Unit = { @@ -63,7 +61,7 @@ class ReplicaFetcherMockBlockingSend(offsets: java.util.Map[TopicPartition, Epoc currentOffsets = newOffsets } - def setFetchPartitionDataForNextResponse(partitionData: Map[TopicPartition, FetchResponse.PartitionData[Records]]): Unit = { + def setFetchPartitionDataForNextResponse(partitionData: Map[TopicPartition, FetchResponseData.PartitionData]): Unit = { fetchPartitionData = partitionData } @@ -97,11 +95,11 @@ class ReplicaFetcherMockBlockingSend(offsets: java.util.Map[TopicPartition, Epoc case ApiKeys.FETCH => fetchCount += 1 - val partitionData = new util.LinkedHashMap[TopicPartition, FetchResponse.PartitionData[Records]] + val partitionData = new util.LinkedHashMap[TopicPartition, FetchResponseData.PartitionData] fetchPartitionData.foreach { case (tp, data) => partitionData.put(tp, data) } fetchPartitionData = Map.empty - new FetchResponse(Errors.NONE, partitionData, 0, - if (partitionData.isEmpty) JFetchMetadata.INVALID_SESSION_ID else 1) + FetchResponse.of(Errors.NONE, 0, + if (partitionData.isEmpty) JFetchMetadata.INVALID_SESSION_ID else 1, partitionData) case _ => throw new UnsupportedOperationException diff --git a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/common/FetchResponseBenchmark.java b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/common/FetchResponseBenchmark.java index 8785ecf6b0273..c2d0d7305a383 100644 --- a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/common/FetchResponseBenchmark.java +++ b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/common/FetchResponseBenchmark.java @@ -18,6 +18,7 @@ package org.apache.kafka.jmh.common; import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.message.FetchResponseData; import org.apache.kafka.common.network.Send; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; @@ -42,9 +43,7 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; -import java.util.Collections; import java.util.LinkedHashMap; -import java.util.Optional; import java.util.UUID; import java.util.concurrent.TimeUnit; @@ -61,11 +60,11 @@ public class FetchResponseBenchmark { @Param({"3", "10", "20"}) private int partitionCount; - LinkedHashMap> responseData; + LinkedHashMap responseData; ResponseHeader header; - FetchResponse fetchResponse; + FetchResponse fetchResponse; @Setup(Level.Trial) public void setup() { @@ -78,19 +77,22 @@ public void setup() { for (int topicIdx = 0; topicIdx < topicCount; topicIdx++) { String topic = UUID.randomUUID().toString(); for (int partitionId = 0; partitionId < partitionCount; partitionId++) { - FetchResponse.PartitionData partitionData = new FetchResponse.PartitionData<>( - Errors.NONE, 0, 0, 0, Optional.empty(), Collections.emptyList(), records); + FetchResponseData.PartitionData partitionData = new FetchResponseData.PartitionData() + .setPartitionIndex(partitionId) + .setLastStableOffset(0) + .setLogStartOffset(0) + .setRecords(records); responseData.put(new TopicPartition(topic, partitionId), partitionData); } } this.header = new ResponseHeader(100, ApiKeys.FETCH.responseHeaderVersion(ApiKeys.FETCH.latestVersion())); - this.fetchResponse = new FetchResponse<>(Errors.NONE, responseData, 0, 0); + this.fetchResponse = FetchResponse.of(Errors.NONE, 0, 0, responseData); } @Benchmark public int testConstructFetchResponse() { - FetchResponse fetchResponse = new FetchResponse<>(Errors.NONE, responseData, 0, 0); + FetchResponse fetchResponse = FetchResponse.of(Errors.NONE, 0, 0, responseData); return fetchResponse.responseData().size(); } diff --git a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/fetcher/ReplicaFetcherThreadBenchmark.java b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/fetcher/ReplicaFetcherThreadBenchmark.java index 424b8df7e762f..453fa40096b2f 100644 --- a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/fetcher/ReplicaFetcherThreadBenchmark.java +++ b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/fetcher/ReplicaFetcherThreadBenchmark.java @@ -44,13 +44,13 @@ import kafka.utils.KafkaScheduler; import kafka.utils.Pool; import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.message.FetchResponseData; import org.apache.kafka.common.message.LeaderAndIsrRequestData; import org.apache.kafka.common.message.OffsetForLeaderEpochRequestData.OffsetForLeaderPartition; import org.apache.kafka.common.message.OffsetForLeaderEpochResponseData.EpochEndOffset; import org.apache.kafka.common.metrics.Metrics; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.record.BaseRecords; -import org.apache.kafka.common.record.Records; import org.apache.kafka.common.record.RecordsSend; import org.apache.kafka.common.requests.FetchRequest; import org.apache.kafka.common.requests.FetchResponse; @@ -82,7 +82,6 @@ import java.util.Arrays; import java.util.Collections; import java.util.LinkedHashMap; -import java.util.LinkedList; import java.util.List; import java.util.Optional; import java.util.Properties; @@ -137,7 +136,7 @@ public void setup() throws IOException { Time.SYSTEM, true); - LinkedHashMap> initialFetched = new LinkedHashMap<>(); + LinkedHashMap initialFetched = new LinkedHashMap<>(); scala.collection.mutable.Map initialFetchStates = new scala.collection.mutable.HashMap<>(); for (int i = 0; i < partitionCount; i++) { TopicPartition tp = new TopicPartition("topic", i); @@ -174,8 +173,11 @@ public RecordsSend toSend() { return null; } }; - initialFetched.put(tp, new FetchResponse.PartitionData<>(Errors.NONE, 0, 0, 0, - new LinkedList<>(), fetched)); + initialFetched.put(tp, new FetchResponseData.PartitionData() + .setPartitionIndex(tp.partition()) + .setLastStableOffset(0) + .setLogStartOffset(0) + .setRecords(fetched)); } ReplicaManager replicaManager = Mockito.mock(ReplicaManager.class); @@ -186,7 +188,7 @@ public RecordsSend toSend() { // so that we do not measure this time as part of the steady state work fetcher.doWork(); // handle response to engage the incremental fetch session handler - fetcher.fetchSessionHandler().handleResponse(new FetchResponse<>(Errors.NONE, initialFetched, 0, 999)); + fetcher.fetchSessionHandler().handleResponse(FetchResponse.of(Errors.NONE, 0, 999, initialFetched)); } @TearDown(Level.Trial) @@ -292,7 +294,8 @@ public Option endOffsetForEpoch(TopicPartition topicPartition, i } @Override - public Option processPartitionData(TopicPartition topicPartition, long fetchOffset, FetchResponse.PartitionData partitionData) { + public Option processPartitionData(TopicPartition topicPartition, long fetchOffset, + FetchResponseData.PartitionData partitionData) { return Option.empty(); } @@ -317,7 +320,7 @@ public Map fetchEpochEndOffsets(Map> fetchFromLeader(FetchRequest.Builder fetchRequest) { + public Map fetchFromLeader(FetchRequest.Builder fetchRequest) { return new scala.collection.mutable.HashMap<>(); } } diff --git a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/fetchsession/FetchSessionBenchmark.java b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/fetchsession/FetchSessionBenchmark.java index 9fa25139909b7..01d83a7b4d04e 100644 --- a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/fetchsession/FetchSessionBenchmark.java +++ b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/fetchsession/FetchSessionBenchmark.java @@ -19,8 +19,8 @@ import org.apache.kafka.clients.FetchSessionHandler; import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.message.FetchResponseData; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.record.MemoryRecords; import org.apache.kafka.common.requests.FetchRequest; import org.apache.kafka.common.requests.FetchResponse; import org.apache.kafka.common.utils.LogContext; @@ -70,24 +70,21 @@ public void setUp() { handler = new FetchSessionHandler(LOG_CONTEXT, 1); FetchSessionHandler.Builder builder = handler.newBuilder(); - LinkedHashMap> respMap = new LinkedHashMap<>(); + LinkedHashMap respMap = new LinkedHashMap<>(); for (int i = 0; i < partitionCount; i++) { TopicPartition tp = new TopicPartition("foo", i); FetchRequest.PartitionData partitionData = new FetchRequest.PartitionData(0, 0, 200, Optional.empty()); fetches.put(tp, partitionData); builder.add(tp, partitionData); - respMap.put(tp, new FetchResponse.PartitionData<>( - Errors.NONE, - 0L, - 0L, - 0, - null, - null)); + respMap.put(tp, new FetchResponseData.PartitionData() + .setPartitionIndex(tp.partition()) + .setLastStableOffset(0) + .setLogStartOffset(0)); } builder.build(); // build and handle an initial response so that the next fetch will be incremental - handler.handleResponse(new FetchResponse<>(Errors.NONE, respMap, 0, 1)); + handler.handleResponse(FetchResponse.of(Errors.NONE, 0, 1, respMap)); int counter = 0; for (TopicPartition topicPartition: new ArrayList<>(fetches.keySet())) { diff --git a/raft/src/main/java/org/apache/kafka/raft/KafkaRaftClient.java b/raft/src/main/java/org/apache/kafka/raft/KafkaRaftClient.java index f99ffb6d02298..d252244cf5d3d 100644 --- a/raft/src/main/java/org/apache/kafka/raft/KafkaRaftClient.java +++ b/raft/src/main/java/org/apache/kafka/raft/KafkaRaftClient.java @@ -40,6 +40,7 @@ import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ApiMessage; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.FetchResponse; import org.apache.kafka.common.utils.BufferSupplier; import org.apache.kafka.common.record.CompressionType; import org.apache.kafka.common.record.MemoryRecords; @@ -906,7 +907,7 @@ private FetchResponseData buildFetchResponse( ) { return RaftUtil.singletonFetchResponse(log.topicPartition(), Errors.NONE, partitionData -> { partitionData - .setRecordSet(records) + .setRecords(records) .setErrorCode(error.code()) .setLogStartOffset(log.startOffset()) .setHighWatermark(highWatermark @@ -991,11 +992,11 @@ private CompletableFuture handleFetchRequest( } FetchResponseData response = tryCompleteFetchRequest(request.replicaId(), fetchPartition, currentTimeMs); - FetchResponseData.FetchablePartitionResponse partitionResponse = - response.responses().get(0).partitionResponses().get(0); + FetchResponseData.PartitionData partitionResponse = + response.responses().get(0).partitions().get(0); if (partitionResponse.errorCode() != Errors.NONE.code() - || partitionResponse.recordSet().sizeInBytes() > 0 + || FetchResponse.recordsSize(partitionResponse) > 0 || request.maxWaitMs() == 0) { return completedFuture(response); } @@ -1084,8 +1085,8 @@ private boolean handleFetchResponse( return false; } - FetchResponseData.FetchablePartitionResponse partitionResponse = - response.responses().get(0).partitionResponses().get(0); + FetchResponseData.PartitionData partitionResponse = + response.responses().get(0).partitions().get(0); FetchResponseData.LeaderIdAndEpoch currentLeaderIdAndEpoch = partitionResponse.currentLeader(); OptionalInt responseLeaderId = optionalLeaderId(currentLeaderIdAndEpoch.leaderId()); @@ -1143,7 +1144,7 @@ private boolean handleFetchResponse( state.setFetchingSnapshot(Optional.of(log.createSnapshot(snapshotId))); } } else { - Records records = (Records) partitionResponse.recordSet(); + Records records = FetchResponse.recordsOrFail(partitionResponse); if (records.sizeInBytes() > 0) { appendAsFollower(records); } diff --git a/raft/src/main/java/org/apache/kafka/raft/RaftUtil.java b/raft/src/main/java/org/apache/kafka/raft/RaftUtil.java index 7462fde041b50..e7acd0800ec3d 100644 --- a/raft/src/main/java/org/apache/kafka/raft/RaftUtil.java +++ b/raft/src/main/java/org/apache/kafka/raft/RaftUtil.java @@ -73,19 +73,19 @@ public static FetchRequestData singletonFetchRequest( public static FetchResponseData singletonFetchResponse( TopicPartition topicPartition, Errors topLevelError, - Consumer partitionConsumer + Consumer partitionConsumer ) { - FetchResponseData.FetchablePartitionResponse fetchablePartition = - new FetchResponseData.FetchablePartitionResponse(); + FetchResponseData.PartitionData fetchablePartition = + new FetchResponseData.PartitionData(); - fetchablePartition.setPartition(topicPartition.partition()); + fetchablePartition.setPartitionIndex(topicPartition.partition()); partitionConsumer.accept(fetchablePartition); FetchResponseData.FetchableTopicResponse fetchableTopic = new FetchResponseData.FetchableTopicResponse() .setTopic(topicPartition.topic()) - .setPartitionResponses(Collections.singletonList(fetchablePartition)); + .setPartitions(Collections.singletonList(fetchablePartition)); return new FetchResponseData() .setErrorCode(topLevelError.code()) @@ -102,8 +102,8 @@ static boolean hasValidTopicPartition(FetchRequestData data, TopicPartition topi static boolean hasValidTopicPartition(FetchResponseData data, TopicPartition topicPartition) { return data.responses().size() == 1 && data.responses().get(0).topic().equals(topicPartition.topic()) && - data.responses().get(0).partitionResponses().size() == 1 && - data.responses().get(0).partitionResponses().get(0).partition() == topicPartition.partition(); + data.responses().get(0).partitions().size() == 1 && + data.responses().get(0).partitions().get(0).partitionIndex() == topicPartition.partition(); } static boolean hasValidTopicPartition(VoteResponseData data, TopicPartition topicPartition) { diff --git a/raft/src/test/java/org/apache/kafka/raft/KafkaRaftClientSnapshotTest.java b/raft/src/test/java/org/apache/kafka/raft/KafkaRaftClientSnapshotTest.java index 2b7cea5554151..c66b9bdf31e18 100644 --- a/raft/src/test/java/org/apache/kafka/raft/KafkaRaftClientSnapshotTest.java +++ b/raft/src/test/java/org/apache/kafka/raft/KafkaRaftClientSnapshotTest.java @@ -89,7 +89,7 @@ public void testFetchRequestOffsetLessThanLogStart() throws Exception { // Send Fetch request less than start offset context.deliverRequest(context.fetchRequest(epoch, otherNodeId, 0, epoch, 0)); context.pollUntilResponse(); - FetchResponseData.FetchablePartitionResponse partitionResponse = context.assertSentFetchPartitionResponse(); + FetchResponseData.PartitionData partitionResponse = context.assertSentFetchPartitionResponse(); assertEquals(Errors.NONE, Errors.forCode(partitionResponse.errorCode())); assertEquals(epoch, partitionResponse.currentLeader().leaderEpoch()); assertEquals(localId, partitionResponse.currentLeader().leaderId()); @@ -176,7 +176,7 @@ public void testFetchRequestTruncateToLogStart() throws Exception { context.fetchRequest(epoch, otherNodeId, oldestSnapshotId.offset + 1, oldestSnapshotId.epoch + 1, 0) ); context.pollUntilResponse(); - FetchResponseData.FetchablePartitionResponse partitionResponse = context.assertSentFetchPartitionResponse(); + FetchResponseData.PartitionData partitionResponse = context.assertSentFetchPartitionResponse(); assertEquals(Errors.NONE, Errors.forCode(partitionResponse.errorCode())); assertEquals(epoch, partitionResponse.currentLeader().leaderEpoch()); assertEquals(localId, partitionResponse.currentLeader().leaderId()); @@ -265,7 +265,7 @@ public void testFetchRequestAtLogStartOffsetWithInvalidEpoch() throws Exception context.fetchRequest(epoch, otherNodeId, oldestSnapshotId.offset, oldestSnapshotId.epoch + 1, 0) ); context.pollUntilResponse(); - FetchResponseData.FetchablePartitionResponse partitionResponse = context.assertSentFetchPartitionResponse(); + FetchResponseData.PartitionData partitionResponse = context.assertSentFetchPartitionResponse(); assertEquals(Errors.NONE, Errors.forCode(partitionResponse.errorCode())); assertEquals(epoch, partitionResponse.currentLeader().leaderEpoch()); assertEquals(localId, partitionResponse.currentLeader().leaderId()); @@ -318,7 +318,7 @@ public void testFetchRequestWithLastFetchedEpochLessThanOldestSnapshot() throws ) ); context.pollUntilResponse(); - FetchResponseData.FetchablePartitionResponse partitionResponse = context.assertSentFetchPartitionResponse(); + FetchResponseData.PartitionData partitionResponse = context.assertSentFetchPartitionResponse(); assertEquals(Errors.NONE, Errors.forCode(partitionResponse.errorCode())); assertEquals(epoch, partitionResponse.currentLeader().leaderEpoch()); assertEquals(localId, partitionResponse.currentLeader().leaderId()); @@ -1329,9 +1329,7 @@ private static FetchResponseData snapshotFetchResponse( long highWatermark ) { return RaftUtil.singletonFetchResponse(topicPartition, Errors.NONE, partitionData -> { - partitionData - .setErrorCode(Errors.NONE.code()) - .setHighWatermark(highWatermark); + partitionData.setHighWatermark(highWatermark); partitionData.currentLeader() .setLeaderEpoch(epoch) diff --git a/raft/src/test/java/org/apache/kafka/raft/RaftClientTestContext.java b/raft/src/test/java/org/apache/kafka/raft/RaftClientTestContext.java index e57995c2cd303..15e550f5f954c 100644 --- a/raft/src/test/java/org/apache/kafka/raft/RaftClientTestContext.java +++ b/raft/src/test/java/org/apache/kafka/raft/RaftClientTestContext.java @@ -606,7 +606,7 @@ int assertSentFetchRequest( return raftMessage.correlationId(); } - FetchResponseData.FetchablePartitionResponse assertSentFetchPartitionResponse() { + FetchResponseData.PartitionData assertSentFetchPartitionResponse() { List sentMessages = drainSentResponses(ApiKeys.FETCH); assertEquals( 1, sentMessages.size(), "Found unexpected sent messages " + sentMessages); @@ -617,8 +617,8 @@ FetchResponseData.FetchablePartitionResponse assertSentFetchPartitionResponse() assertEquals(1, response.responses().size()); assertEquals(metadataPartition.topic(), response.responses().get(0).topic()); - assertEquals(1, response.responses().get(0).partitionResponses().size()); - return response.responses().get(0).partitionResponses().get(0); + assertEquals(1, response.responses().get(0).partitions().size()); + return response.responses().get(0).partitions().get(0); } void assertSentFetchPartitionResponse(Errors error) { @@ -637,7 +637,7 @@ MemoryRecords assertSentFetchPartitionResponse( int epoch, OptionalInt leaderId ) { - FetchResponseData.FetchablePartitionResponse partitionResponse = assertSentFetchPartitionResponse(); + FetchResponseData.PartitionData partitionResponse = assertSentFetchPartitionResponse(); assertEquals(error, Errors.forCode(partitionResponse.errorCode())); assertEquals(epoch, partitionResponse.currentLeader().leaderEpoch()); assertEquals(leaderId.orElse(-1), partitionResponse.currentLeader().leaderId()); @@ -645,14 +645,14 @@ MemoryRecords assertSentFetchPartitionResponse( assertEquals(-1, partitionResponse.divergingEpoch().epoch()); assertEquals(-1, partitionResponse.snapshotId().endOffset()); assertEquals(-1, partitionResponse.snapshotId().epoch()); - return (MemoryRecords) partitionResponse.recordSet(); + return (MemoryRecords) partitionResponse.records(); } MemoryRecords assertSentFetchPartitionResponse( long highWatermark, int leaderEpoch ) { - FetchResponseData.FetchablePartitionResponse partitionResponse = assertSentFetchPartitionResponse(); + FetchResponseData.PartitionData partitionResponse = assertSentFetchPartitionResponse(); assertEquals(Errors.NONE, Errors.forCode(partitionResponse.errorCode())); assertEquals(leaderEpoch, partitionResponse.currentLeader().leaderEpoch()); assertEquals(highWatermark, partitionResponse.highWatermark()); @@ -660,7 +660,7 @@ MemoryRecords assertSentFetchPartitionResponse( assertEquals(-1, partitionResponse.divergingEpoch().epoch()); assertEquals(-1, partitionResponse.snapshotId().endOffset()); assertEquals(-1, partitionResponse.snapshotId().epoch()); - return (MemoryRecords) partitionResponse.recordSet(); + return (MemoryRecords) partitionResponse.records(); } RaftRequest.Outbound assertSentFetchSnapshotRequest() { @@ -928,7 +928,7 @@ FetchResponseData fetchResponse( ) { return RaftUtil.singletonFetchResponse(metadataPartition, Errors.NONE, partitionData -> { partitionData - .setRecordSet(records) + .setRecords(records) .setErrorCode(error.code()) .setHighWatermark(highWatermark); @@ -946,9 +946,7 @@ FetchResponseData divergingFetchResponse( long highWatermark ) { return RaftUtil.singletonFetchResponse(metadataPartition, Errors.NONE, partitionData -> { - partitionData - .setErrorCode(Errors.NONE.code()) - .setHighWatermark(highWatermark); + partitionData.setHighWatermark(highWatermark); partitionData.currentLeader() .setLeaderEpoch(epoch) From ea005cc700b1b9e3cb001da19d73798ea8520d63 Mon Sep 17 00:00:00 2001 From: Lee Dongjin Date: Thu, 4 Mar 2021 23:24:50 +0900 Subject: [PATCH 09/59] KAFKA-12407: Document Controller Health Metrics (#10257) Reviewers: Luke Chen , Dong Lin --- docs/ops.html | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/ops.html b/docs/ops.html index 1bef09aed1d90..14f967e0b8202 100644 --- a/docs/ops.html +++ b/docs/ops.html @@ -1271,6 +1271,24 @@

Date: Thu, 4 Mar 2021 16:47:48 +0100 Subject: [PATCH 10/59] KAFKA-12393: Document multi-tenancy considerations (#334) (#10263) KAFKA-12393: Document multi-tenancy considerations Addressed review feedback by @dajac and @rajinisivaram Ported from apache/kafka-site#334 Reviewers: Bill Bejeck --- docs/ops.html | 168 ++++++++++++++++++++++++++++++++++++++++++++++++-- docs/toc.html | 21 +++++-- 2 files changed, 179 insertions(+), 10 deletions(-) diff --git a/docs/ops.html b/docs/ops.html index 14f967e0b8202..402d6f1c76392 100644 --- a/docs/ops.html +++ b/docs/ops.html @@ -1089,7 +1089,165 @@

-

6.4 Kafka Configuration

+

6.4 Multi-Tenancy

+ +

Multi-Tenancy Overview

+ +

+ As a highly scalable event streaming platform, Kafka is used by many users as their central nervous system, connecting in real-time a wide range of different systems and applications from various teams and lines of businesses. Such multi-tenant cluster environments command proper control and management to ensure the peaceful coexistence of these different needs. This section highlights features and best practices to set up such shared environments, which should help you operate clusters that meet SLAs/OLAs and that minimize potential collateral damage caused by "noisy neighbors". +

+ +

+ Multi-tenancy is a many-sided subject, including but not limited to: +

+ +
    +
  • Creating user spaces for tenants (sometimes called namespaces)
    • +
  • Configuring topics with data retention policies and more
    • +
  • Securing topics and clusters with encryption, authentication, and authorization
    • +
  • Isolating tenants with quotas and rate limits
    • +
  • Monitoring and metering
    • +
  • Inter-cluster data sharing (cf. geo-replication)
    • +
+ +

Creating User Spaces (Namespaces) For Tenants With Topic Naming

+ +

+ Kafka administrators operating a multi-tenant cluster typically need to define user spaces for each tenant. For the purpose of this section, "user spaces" are a collection of topics, which are grouped together under the management of a single entity or user. +

+ +

+ In Kafka, the main unit of data is the topic. Users can create and name each topic. They can also delete them, but it is not possible to rename a topic directly. Instead, to rename a topic, the user must create a new topic, move the messages from the original topic to the new, and then delete the original. With this in mind, it is recommended to define logical spaces, based on an hierarchical topic naming structure. This setup can then be combined with security features, such as prefixed ACLs, to isolate different spaces and tenants, while also minimizing the administrative overhead for securing the data in the cluster. +

+ +

+ These logical user spaces can be grouped in different ways, and the concrete choice depends on how your organization prefers to use your Kafka clusters. The most common groupings are as follows. +

+ +

+ By team or organizational unit: Here, the team is the main aggregator. In an organization where teams are the main user of the Kafka infrastructure, this might be the best grouping. +

+ +

+ Example topic naming structure: +

+ +
    +
  • <organization>.<team>.<dataset>.<event-name>
    (e.g., "acme.infosec.telemetry.logins")
    • +
+ +

+ By project or product: Here, a team manages more than one project. Their credentials will be different for each project, so all the controls and settings will always be project related. +

+ +

+ Example topic naming structure: +

+ +
    +
  • <project>.<product>.<event-name>
    (e.g., "mobility.payments.suspicious")
    • +
+ +

+ Certain information should normally not be put in a topic name, such as information that is likely to change over time (e.g., the name of the intended consumer) or that is a technical detail or metadata that is available elsewhere (e.g., the topic's partition count and other configuration settings). +

+ +

+ To enforce a topic naming structure, several options are available: +

+ +
    +
  • Use prefix ACLs (cf. KIP-290) to enforce a common prefix for topic names. For example, team A may only be permitted to create topics whose names start with payments.teamA..
    • +
  • Define a custom CreateTopicPolicy (cf. KIP-108 and the setting create.topic.policy.class.name) to enforce strict naming patterns. These policies provide the most flexibility and can cover complex patterns and rules to match an organization's needs.
    • +
  • Disable topic creation for normal users by denying it with an ACL, and then rely on an external process to create topics on behalf of users (e.g., scripting or your favorite automation toolkit).
    • +
  • It may also be useful to disable the Kafka feature to auto-create topics on demand by setting auto.create.topics.enable=false in the broker configuration. Note that you should not rely solely on this option.
    • +
+ + +

Configuring Topics: Data Retention And More

+ +

+ Kafka's configuration is very flexible due to its fine granularity, and it supports a plethora of per-topic configuration settings to help administrators set up multi-tenant clusters. For example, administrators often need to define data retention policies to control how much and/or for how long data will be stored in a topic, with settings such as retention.bytes (size) and retention.ms (time). This limits storage consumption within the cluster, and helps complying with legal requirements such as GDPR. +

+ +

Securing Clusters and Topics: Authentication, Authorization, Encryption

+ +

+ Because the documentation has a dedicated chapter on security that applies to any Kafka deployment, this section focuses on additional considerations for multi-tenant environments. +

+ +

+Security settings for Kafka fall into three main categories, which are similar to how administrators would secure other client-server data systems, like relational databases and traditional messaging systems. +

+ +
    +
  1. Encryption of data transferred between Kafka brokers and Kafka clients, between brokers, between brokers and ZooKeeper nodes, and between brokers and other, optional tools.
    2. +
  2. Authentication of connections from Kafka clients and applications to Kafka brokers, as well as connections from Kafka brokers to ZooKeeper nodes.
    3. +
  3. Authorization of client operations such as creating, deleting, and altering the configuration of topics; writing events to or reading events from a topic; creating and deleting ACLs. Administrators can also define custom policies to put in place additional restrictions, such as a CreateTopicPolicy and AlterConfigPolicy (see KIP-108 and the settings create.topic.policy.class.name, alter.config.policy.class.name).
    4. +
+ +

+ When securing a multi-tenant Kafka environment, the most common administrative task is the third category (authorization), i.e., managing the user/client permissions that grant or deny access to certain topics and thus to the data stored by users within a cluster. This task is performed predominantly through the setting of access control lists (ACLs). Here, administrators of multi-tenant environments in particular benefit from putting a hierarchical topic naming structure in place as described in a previous section, because they can conveniently control access to topics through prefixed ACLs (--resource-pattern-type Prefixed). This significantly minimizes the administrative overhead of securing topics in multi-tenant environments: administrators can make their own trade-offs between higher developer convenience (more lenient permissions, using fewer and broader ACLs) vs. tighter security (more stringent permissions, using more and narrower ACLs). +

+ +

+ In the following example, user Alice—a new member of ACME corporation's InfoSec team—is granted write permissions to all topics whose names start with "acme.infosec.", such as "acme.infosec.telemetry.logins" and "acme.infosec.syslogs.events". +

+ +
# Grant permissions to user Alice
+$ bin/kafka-acls.sh \
+    --bootstrap-server broker1:9092 \
+    --add --allow-principal User:Alice \
+    --producer \
+    --resource-pattern-type prefixed --topic acme.infosec.
+
+ +

+ You can similarly use this approach to isolate different customers on the same shared cluster. +

+ +

Isolating Tenants: Quotas, Rate Limiting, Throttling

+ +

+ Multi-tenant clusters should generally be configured with quotas, which protect against users (tenants) eating up too many cluster resources, such as when they attempt to write or read very high volumes of data, or create requests to brokers at an excessively high rate. This may cause network saturation, monopolize broker resources, and impact other clients—all of which you want to avoid in a shared environment. +

+ +

+ Client quotas: Kafka supports different types of (per-user principal) client quotas. Because a client's quotas apply irrespective of which topics the client is writing to or reading from, they are a convenient and effective tool to allocate resources in a multi-tenant cluster. Request rate quotas, for example, help to limit a user's impact on broker CPU usage by limiting the time a broker spends on the request handling path for that user, after which throttling kicks in. In many situations, isolating users with request rate quotas has a bigger impact in multi-tenant clusters than setting incoming/outgoing network bandwidth quotas, because excessive broker CPU usage for processing requests reduces the effective bandwidth the broker can serve. Furthermore, administrators can also define quotas on topic operations—such as create, delete, and alter—to prevent Kafka clusters from being overwhelmed by highly concurrent topic operations (see KIP-599 and the quota type controller_mutations_rate). +

+ +

+ Server quotas: Kafka also supports different types of broker-side quotas. For example, administrators can set a limit on the rate with which the broker accepts new connections, set the maximum number of connections per broker, or set the maximum number of connections allowed from a specific IP address. +

+ +

+ For more information, please refer to the quota overview and how to set quotas. +

+ +

Monitoring and Metering

+ +

+ Monitoring is a broader subject that is covered elsewhere in the documentation. Administrators of any Kafka environment, but especially multi-tenant ones, should set up monitoring according to these instructions. Kafka supports a wide range of metrics, such as the rate of failed authentication attempts, request latency, consumer lag, total number of consumer groups, metrics on the quotas described in the previous section, and many more. +

+ +

+ For example, monitoring can be configured to track the size of topic-partitions (with the JMX metric kafka.log.Log.Size.<TOPIC-NAME>), and thus the total size of data stored in a topic. You can then define alerts when tenants on shared clusters are getting close to using too much storage space. +

+ +

Multi-Tenancy and Geo-Replication

+ +

+ Kafka lets you share data across different clusters, which may be located in different geographical regions, data centers, and so on. Apart from use cases such as disaster recovery, this functionality is useful when a multi-tenant setup requires inter-cluster data sharing. See the section Geo-Replication (Cross-Cluster Data Mirroring) for more information. +

+ +

Further considerations

+ +

+ Data contracts: You may need to define data contracts between the producers and the consumers of data in a cluster, using event schemas. This ensures that events written to Kafka can always be read properly again, and prevents malformed or corrupt events being written. The best way to achieve this is to deploy a so-called schema registry alongside the cluster. (Kafka does not include a schema registry, but there are third-party implementations available.) A schema registry manages the event schemas and maps the schemas to topics, so that producers know which topics are accepting which types (schemas) of events, and consumers know how to read and parse events in a topic. Some registry implementations provide further functionality, such as schema evolution, storing a history of all schemas, and schema compatibility settings. +

+ + +

6.5 Kafka Configuration

Important Client Configurations

@@ -1122,7 +1280,7 @@

6.5 Java Version

+

6.6 Java Version

Java 8 and Java 11 are supported. Java 11 performs significantly better if TLS is enabled, so it is highly recommended (it also includes a number of other performance improvements: G1GC, CRC32C, Compact Strings, Thread-Local Handshakes and more). @@ -1145,7 +1303,7 @@

All of the brokers in that cluster have a 90% GC pause time of about 21ms with less than 1 young GC per second. -

6.6 Hardware and OS

+

6.7 Hardware and OS

We are using dual quad-core Intel Xeon machines with 24GB of memory.

You need sufficient memory to buffer active readers and writers. You can do a back-of-the-envelope estimate of memory needs by assuming you want to be able to buffer for 30 seconds and compute your memory need as write_throughput*30. @@ -1230,7 +1388,7 @@

  • delalloc: Delayed allocation means that the filesystem avoid allocating any blocks until the physical write occurs. This allows ext4 to allocate a large extent instead of smaller pages and helps ensure the data is written sequentially. This feature is great for throughput. It does seem to involve some locking in the filesystem which adds a bit of latency variance. -

    6.7 Monitoring

    +

    6.8 Monitoring

    Kafka uses Yammer Metrics for metrics reporting in the server. The Java clients use Kafka Metrics, a built-in metrics registry that minimizes transitive dependencies pulled into client applications. Both expose metrics via JMX and can be configured to report stats using pluggable stats reporters to hook up to your monitoring system.

    @@ -2866,7 +3024,7 @@

    6.8 ZooKeeper

    +

    6.9 ZooKeeper

    Stable version

    The current stable branch is 3.5. Kafka is regularly updated to include the latest release in the 3.5 series. diff --git a/docs/toc.html b/docs/toc.html index 8d15b2b9ea19c..fa11c813d3935 100644 --- a/docs/toc.html +++ b/docs/toc.html @@ -96,13 +96,24 @@
  • Applying Configuration Changes
  • Monitoring Geo-Replication
    • -
  • 6.4 Important Configs +
  • 6.4 Multi-Tenancy
    • + +
  • 6.5 Important Configs -
  • 6.5 Java Version -
  • 6.6 Hardware and OS +
  • 6.6 Java Version +
  • 6.7 Hardware and OS -
  • 6.7 Monitoring +
  • 6.8 Monitoring -
  • 6.8 ZooKeeper +
  • 6.9 ZooKeeper
    • Stable Version
    • Operationalization From be1476869fc93553b3099d387d26bfd0092a9d65 Mon Sep 17 00:00:00 2001 From: Chia-Ping Tsai Date: Fri, 5 Mar 2021 00:22:57 +0800 Subject: [PATCH 11/59] MINOR: make sure all generated data tests cover all versions (#10078) Reviewers: David Jacot --- .../apache/kafka/common/protocol/ApiKeys.java | 2 +- .../kafka/common/message/MessageTest.java | 24 +-- .../AddPartitionsToTxnRequestTest.java | 2 +- .../AddPartitionsToTxnResponseTest.java | 2 +- .../ControlledShutdownRequestTest.java | 2 +- .../common/requests/EndTxnRequestTest.java | 2 +- .../common/requests/EndTxnResponseTest.java | 2 +- .../common/requests/EnvelopeRequestTest.java | 2 +- .../common/requests/EnvelopeResponseTest.java | 2 +- .../requests/LeaderAndIsrRequestTest.java | 5 +- .../requests/LeaderAndIsrResponseTest.java | 6 +- .../requests/LeaveGroupRequestTest.java | 2 +- .../requests/LeaveGroupResponseTest.java | 10 +- .../requests/OffsetCommitRequestTest.java | 4 +- .../requests/OffsetCommitResponseTest.java | 2 +- .../requests/OffsetFetchRequestTest.java | 6 +- .../requests/OffsetFetchResponseTest.java | 4 +- .../OffsetsForLeaderEpochRequestTest.java | 6 +- .../common/requests/ProduceRequestTest.java | 32 ++-- .../common/requests/ProduceResponseTest.java | 6 +- .../common/requests/RequestResponseTest.java | 171 +++++++++--------- .../requests/StopReplicaRequestTest.java | 8 +- .../requests/StopReplicaResponseTest.java | 2 +- .../requests/TxnOffsetCommitRequestTest.java | 2 +- .../requests/TxnOffsetCommitResponseTest.java | 2 +- .../requests/UpdateMetadataRequestTest.java | 4 +- .../requests/WriteTxnMarkersRequestTest.java | 4 +- 27 files changed, 155 insertions(+), 161 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java b/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java index 3297dc5734aa6..8ec2d02ea3de6 100644 --- a/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java +++ b/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java @@ -201,7 +201,7 @@ public short oldestVersion() { public List allVersions() { List versions = new ArrayList<>(latestVersion() - oldestVersion() + 1); - for (short version = oldestVersion(); version < latestVersion(); version++) { + for (short version = oldestVersion(); version <= latestVersion(); version++) { versions.add(version); } return versions; diff --git a/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java b/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java index 5dc379e0e7fee..e191ad6526ace 100644 --- a/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java +++ b/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java @@ -186,7 +186,7 @@ public void testListOffsetsResponseVersions() throws Exception { .setPartitions(Collections.singletonList(partition))); Supplier response = () -> new ListOffsetsResponseData() .setTopics(topics); - for (short version = 0; version <= ApiKeys.LIST_OFFSETS.latestVersion(); version++) { + for (short version : ApiKeys.LIST_OFFSETS.allVersions()) { ListOffsetsResponseData responseData = response.get(); if (version > 0) { responseData.topics().get(0).partitions().get(0) @@ -459,7 +459,7 @@ public void testOffsetCommitRequestVersions() throws Exception { )))) .setRetentionTimeMs(20); - for (short version = 0; version <= ApiKeys.OFFSET_COMMIT.latestVersion(); version++) { + for (short version : ApiKeys.OFFSET_COMMIT.allVersions()) { OffsetCommitRequestData requestData = request.get(); if (version < 1) { requestData.setMemberId(""); @@ -485,7 +485,7 @@ public void testOffsetCommitRequestVersions() throws Exception { if (version == 1) { testEquivalentMessageRoundTrip(version, requestData); } else if (version >= 2 && version <= 4) { - testAllMessageRoundTripsBetweenVersions(version, (short) 4, requestData, requestData); + testAllMessageRoundTripsBetweenVersions(version, (short) 5, requestData, requestData); } else { testAllMessageRoundTripsFromVersion(version, requestData); } @@ -509,7 +509,7 @@ public void testOffsetCommitResponseVersions() throws Exception { ) .setThrottleTimeMs(20); - for (short version = 0; version <= ApiKeys.OFFSET_COMMIT.latestVersion(); version++) { + for (short version : ApiKeys.OFFSET_COMMIT.allVersions()) { OffsetCommitResponseData responseData = response.get(); if (version < 3) { responseData.setThrottleTimeMs(0); @@ -568,7 +568,7 @@ public void testTxnOffsetCommitRequestVersions() throws Exception { .setCommittedOffset(offset) )))); - for (short version = 0; version <= ApiKeys.TXN_OFFSET_COMMIT.latestVersion(); version++) { + for (short version : ApiKeys.TXN_OFFSET_COMMIT.allVersions()) { TxnOffsetCommitRequestData requestData = request.get(); if (version < 2) { requestData.topics().get(0).partitions().get(0).setCommittedLeaderEpoch(-1); @@ -632,7 +632,7 @@ public void testOffsetFetchVersions() throws Exception { .setTopics(topics) .setRequireStable(true); - for (short version = 0; version <= ApiKeys.OFFSET_FETCH.latestVersion(); version++) { + for (short version : ApiKeys.OFFSET_FETCH.allVersions()) { final short finalVersion = version; if (version < 2) { assertThrows(NullPointerException.class, () -> testAllMessageRoundTripsFromVersion(finalVersion, allPartitionData)); @@ -661,7 +661,7 @@ public void testOffsetFetchVersions() throws Exception { .setErrorCode(Errors.UNKNOWN_TOPIC_OR_PARTITION.code()))))) .setErrorCode(Errors.NOT_COORDINATOR.code()) .setThrottleTimeMs(10); - for (short version = 0; version <= ApiKeys.OFFSET_FETCH.latestVersion(); version++) { + for (short version : ApiKeys.OFFSET_FETCH.allVersions()) { OffsetFetchResponseData responseData = response.get(); if (version <= 1) { responseData.setErrorCode(Errors.NONE.code()); @@ -720,7 +720,7 @@ public void testProduceResponseVersions() throws Exception { .setErrorMessage(errorMessage)))).iterator())) .setThrottleTimeMs(throttleTimeMs); - for (short version = 0; version <= ApiKeys.PRODUCE.latestVersion(); version++) { + for (short version : ApiKeys.PRODUCE.allVersions()) { ProduceResponseData responseData = response.get(); if (version < 8) { @@ -741,9 +741,9 @@ public void testProduceResponseVersions() throws Exception { } if (version >= 3 && version <= 4) { - testAllMessageRoundTripsBetweenVersions(version, (short) 4, responseData, responseData); + testAllMessageRoundTripsBetweenVersions(version, (short) 5, responseData, responseData); } else if (version >= 6 && version <= 7) { - testAllMessageRoundTripsBetweenVersions(version, (short) 7, responseData, responseData); + testAllMessageRoundTripsBetweenVersions(version, (short) 8, responseData, responseData); } else { testEquivalentMessageRoundTrip(version, responseData); } @@ -924,8 +924,8 @@ public void testNonIgnorableFieldWithDefaultNull() { @Test public void testWriteNullForNonNullableFieldRaisesException() { CreateTopicsRequestData createTopics = new CreateTopicsRequestData().setTopics(null); - for (short i = (short) 0; i <= createTopics.highestSupportedVersion(); i++) { - verifyWriteRaisesNpe(i, createTopics); + for (short version : ApiKeys.CREATE_TOPICS.allVersions()) { + verifyWriteRaisesNpe(version, createTopics); } MetadataRequestData metadata = new MetadataRequestData().setTopics(null); verifyWriteRaisesNpe((short) 0, metadata); diff --git a/clients/src/test/java/org/apache/kafka/common/requests/AddPartitionsToTxnRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/AddPartitionsToTxnRequestTest.java index 24cb9a5d70251..acba8be966a75 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/AddPartitionsToTxnRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/AddPartitionsToTxnRequestTest.java @@ -43,7 +43,7 @@ public void testConstructor() { AddPartitionsToTxnRequest.Builder builder = new AddPartitionsToTxnRequest.Builder(transactionalId, producerId, producerEpoch, partitions); - for (short version = 0; version <= ApiKeys.ADD_PARTITIONS_TO_TXN.latestVersion(); version++) { + for (short version : ApiKeys.ADD_PARTITIONS_TO_TXN.allVersions()) { AddPartitionsToTxnRequest request = builder.build(version); assertEquals(transactionalId, request.data().transactionalId()); diff --git a/clients/src/test/java/org/apache/kafka/common/requests/AddPartitionsToTxnResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/AddPartitionsToTxnResponseTest.java index b7901880bf48f..5b67bd47a01f6 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/AddPartitionsToTxnResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/AddPartitionsToTxnResponseTest.java @@ -89,7 +89,7 @@ public void testParse() { .setThrottleTimeMs(throttleTimeMs); AddPartitionsToTxnResponse response = new AddPartitionsToTxnResponse(data); - for (short version = 0; version <= ApiKeys.ADD_PARTITIONS_TO_TXN.latestVersion(); version++) { + for (short version : ApiKeys.ADD_PARTITIONS_TO_TXN.allVersions()) { AddPartitionsToTxnResponse parsedResponse = AddPartitionsToTxnResponse.parse(response.serialize(version), version); assertEquals(expectedErrorCounts, parsedResponse.errorCounts()); assertEquals(throttleTimeMs, parsedResponse.throttleTimeMs()); diff --git a/clients/src/test/java/org/apache/kafka/common/requests/ControlledShutdownRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/ControlledShutdownRequestTest.java index 04f294c718263..867be713ba2e2 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/ControlledShutdownRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/ControlledShutdownRequestTest.java @@ -38,7 +38,7 @@ public void testUnsupportedVersion() { @Test public void testGetErrorResponse() { - for (short version = CONTROLLED_SHUTDOWN.oldestVersion(); version < CONTROLLED_SHUTDOWN.latestVersion(); version++) { + for (short version : CONTROLLED_SHUTDOWN.allVersions()) { ControlledShutdownRequest.Builder builder = new ControlledShutdownRequest.Builder( new ControlledShutdownRequestData().setBrokerId(1), version); ControlledShutdownRequest request = builder.build(); diff --git a/clients/src/test/java/org/apache/kafka/common/requests/EndTxnRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/EndTxnRequestTest.java index 722e9a5a589a9..f14bf66161980 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/EndTxnRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/EndTxnRequestTest.java @@ -41,7 +41,7 @@ public void testConstructor() { .setProducerId(producerId) .setTransactionalId(transactionId)); - for (short version = 0; version <= ApiKeys.END_TXN.latestVersion(); version++) { + for (short version : ApiKeys.END_TXN.allVersions()) { EndTxnRequest request = builder.build(version); EndTxnResponse response = request.getErrorResponse(throttleTimeMs, Errors.NOT_COORDINATOR.exception()); diff --git a/clients/src/test/java/org/apache/kafka/common/requests/EndTxnResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/EndTxnResponseTest.java index 34646dc0b4b5d..39c4bc04104ae 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/EndTxnResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/EndTxnResponseTest.java @@ -38,7 +38,7 @@ public void testConstructor() { Map expectedErrorCounts = Collections.singletonMap(Errors.NOT_COORDINATOR, 1); - for (short version = 0; version <= ApiKeys.END_TXN.latestVersion(); version++) { + for (short version : ApiKeys.END_TXN.allVersions()) { EndTxnResponse response = new EndTxnResponse(data); assertEquals(expectedErrorCounts, response.errorCounts()); assertEquals(throttleTimeMs, response.throttleTimeMs()); diff --git a/clients/src/test/java/org/apache/kafka/common/requests/EnvelopeRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/EnvelopeRequestTest.java index a7bccd56cf10f..36b8618f49df2 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/EnvelopeRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/EnvelopeRequestTest.java @@ -46,7 +46,7 @@ public void testGetPrincipal() { @Test public void testToSend() throws IOException { - for (short version = ApiKeys.ENVELOPE.oldestVersion(); version <= ApiKeys.ENVELOPE.latestVersion(); version++) { + for (short version : ApiKeys.ENVELOPE.allVersions()) { ByteBuffer requestData = ByteBuffer.wrap("foobar".getBytes()); RequestHeader header = new RequestHeader(ApiKeys.ENVELOPE, version, "clientId", 15); EnvelopeRequest request = new EnvelopeRequest.Builder( diff --git a/clients/src/test/java/org/apache/kafka/common/requests/EnvelopeResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/EnvelopeResponseTest.java index 0a384cdd1661a..e0fa2fdbcddb0 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/EnvelopeResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/EnvelopeResponseTest.java @@ -32,7 +32,7 @@ class EnvelopeResponseTest { @Test public void testToSend() { - for (short version = ApiKeys.ENVELOPE.oldestVersion(); version <= ApiKeys.ENVELOPE.latestVersion(); version++) { + for (short version : ApiKeys.ENVELOPE.allVersions()) { ByteBuffer responseData = ByteBuffer.wrap("foobar".getBytes()); EnvelopeResponse response = new EnvelopeResponse(responseData, Errors.NONE); short headerVersion = ApiKeys.ENVELOPE.responseHeaderVersion(version); diff --git a/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrRequestTest.java index 4fe51a0dccd3c..de9914c575e31 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrRequestTest.java @@ -64,8 +64,7 @@ public void testGetErrorResponse() { Uuid topicId = Uuid.randomUuid(); String topicName = "topic"; int partition = 0; - - for (short version = LEADER_AND_ISR.oldestVersion(); version <= LEADER_AND_ISR.latestVersion(); version++) { + for (short version : LEADER_AND_ISR.allVersions()) { LeaderAndIsrRequest request = new LeaderAndIsrRequest.Builder(version, 0, 0, 0, Collections.singletonList(new LeaderAndIsrPartitionState() .setTopicName(topicName) @@ -108,7 +107,7 @@ public void testGetErrorResponse() { */ @Test public void testVersionLogic() { - for (short version = LEADER_AND_ISR.oldestVersion(); version <= LEADER_AND_ISR.latestVersion(); version++) { + for (short version : LEADER_AND_ISR.allVersions()) { List partitionStates = asList( new LeaderAndIsrPartitionState() .setTopicName("topic0") diff --git a/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrResponseTest.java index 9ae2fdb4204fc..9f46304a4de31 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrResponseTest.java @@ -71,7 +71,7 @@ public void testErrorCountsFromGetErrorResponse() { @Test public void testErrorCountsWithTopLevelError() { - for (short version = LEADER_AND_ISR.oldestVersion(); version < LEADER_AND_ISR.latestVersion(); version++) { + for (short version : LEADER_AND_ISR.allVersions()) { LeaderAndIsrResponse response; if (version < 5) { List partitions = createPartitions("foo", @@ -92,7 +92,7 @@ public void testErrorCountsWithTopLevelError() { @Test public void testErrorCountsNoTopLevelError() { - for (short version = LEADER_AND_ISR.oldestVersion(); version < LEADER_AND_ISR.latestVersion(); version++) { + for (short version : LEADER_AND_ISR.allVersions()) { LeaderAndIsrResponse response; if (version < 5) { List partitions = createPartitions("foo", @@ -116,7 +116,7 @@ public void testErrorCountsNoTopLevelError() { @Test public void testToString() { - for (short version = LEADER_AND_ISR.oldestVersion(); version < LEADER_AND_ISR.latestVersion(); version++) { + for (short version : LEADER_AND_ISR.allVersions()) { LeaderAndIsrResponse response; if (version < 5) { List partitions = createPartitions("foo", diff --git a/clients/src/test/java/org/apache/kafka/common/requests/LeaveGroupRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/LeaveGroupRequestTest.java index 411efef71f546..1694ef5fdf938 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/LeaveGroupRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/LeaveGroupRequestTest.java @@ -67,7 +67,7 @@ public void testMultiLeaveConstructor() { .setGroupId(groupId) .setMembers(members); - for (short version = 0; version <= ApiKeys.LEAVE_GROUP.latestVersion(); version++) { + for (short version : ApiKeys.LEAVE_GROUP.allVersions()) { try { LeaveGroupRequest request = builder.build(version); if (version <= 2) { diff --git a/clients/src/test/java/org/apache/kafka/common/requests/LeaveGroupResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/LeaveGroupResponseTest.java index cc9819133f466..d5132182ceee2 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/LeaveGroupResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/LeaveGroupResponseTest.java @@ -67,7 +67,7 @@ public void testConstructorWithMemberResponses() { expectedErrorCounts.put(Errors.UNKNOWN_MEMBER_ID, 1); expectedErrorCounts.put(Errors.FENCED_INSTANCE_ID, 1); - for (short version = 0; version <= ApiKeys.LEAVE_GROUP.latestVersion(); version++) { + for (short version : ApiKeys.LEAVE_GROUP.allVersions()) { LeaveGroupResponse leaveGroupResponse = new LeaveGroupResponse(memberResponses, Errors.NONE, throttleTimeMs, @@ -95,7 +95,7 @@ public void testConstructorWithMemberResponses() { @Test public void testShouldThrottle() { LeaveGroupResponse response = new LeaveGroupResponse(new LeaveGroupResponseData()); - for (short version = 0; version <= ApiKeys.LEAVE_GROUP.latestVersion(); version++) { + for (short version : ApiKeys.LEAVE_GROUP.allVersions()) { if (version >= 2) { assertTrue(response.shouldClientThrottle(version)); } else { @@ -109,7 +109,7 @@ public void testEqualityWithSerialization() { LeaveGroupResponseData responseData = new LeaveGroupResponseData() .setErrorCode(Errors.NONE.code()) .setThrottleTimeMs(throttleTimeMs); - for (short version = 0; version <= ApiKeys.LEAVE_GROUP.latestVersion(); version++) { + for (short version : ApiKeys.LEAVE_GROUP.allVersions()) { LeaveGroupResponse primaryResponse = LeaveGroupResponse.parse( MessageUtil.toByteBuffer(responseData, version), version); LeaveGroupResponse secondaryResponse = LeaveGroupResponse.parse( @@ -129,7 +129,7 @@ public void testParse() { .setErrorCode(Errors.NOT_COORDINATOR.code()) .setThrottleTimeMs(throttleTimeMs); - for (short version = 0; version <= ApiKeys.LEAVE_GROUP.latestVersion(); version++) { + for (short version : ApiKeys.LEAVE_GROUP.allVersions()) { ByteBuffer buffer = MessageUtil.toByteBuffer(data, version); LeaveGroupResponse leaveGroupResponse = LeaveGroupResponse.parse(buffer, version); assertEquals(expectedErrorCounts, leaveGroupResponse.errorCounts()); @@ -146,7 +146,7 @@ public void testParse() { @Test public void testEqualityWithMemberResponses() { - for (short version = 0; version <= ApiKeys.LEAVE_GROUP.latestVersion(); version++) { + for (short version : ApiKeys.LEAVE_GROUP.allVersions()) { List localResponses = version > 2 ? memberResponses : memberResponses.subList(0, 1); LeaveGroupResponse primaryResponse = new LeaveGroupResponse(localResponses, Errors.NONE, diff --git a/clients/src/test/java/org/apache/kafka/common/requests/OffsetCommitRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/OffsetCommitRequestTest.java index cb853162209be..08ae7a3fbd572 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/OffsetCommitRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/OffsetCommitRequestTest.java @@ -91,7 +91,7 @@ public void testConstructor() { OffsetCommitRequest.Builder builder = new OffsetCommitRequest.Builder(data); - for (short version = 0; version <= ApiKeys.TXN_OFFSET_COMMIT.latestVersion(); version++) { + for (short version : ApiKeys.TXN_OFFSET_COMMIT.allVersions()) { OffsetCommitRequest request = builder.build(version); assertEquals(expectedOffsets, request.offsets()); @@ -130,7 +130,7 @@ public void testVersionSupportForGroupInstanceId() { .setGroupInstanceId(groupInstanceId) ); - for (short version = 0; version <= ApiKeys.OFFSET_COMMIT.latestVersion(); version++) { + for (short version : ApiKeys.OFFSET_COMMIT.allVersions()) { if (version >= 7) { builder.build(version); } else { diff --git a/clients/src/test/java/org/apache/kafka/common/requests/OffsetCommitResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/OffsetCommitResponseTest.java index c6791e64f4f74..b9ce03f1e3ad4 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/OffsetCommitResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/OffsetCommitResponseTest.java @@ -85,7 +85,7 @@ public void testParse() { )) .setThrottleTimeMs(throttleTimeMs); - for (short version = 0; version <= ApiKeys.OFFSET_COMMIT.latestVersion(); version++) { + for (short version : ApiKeys.OFFSET_COMMIT.allVersions()) { ByteBuffer buffer = MessageUtil.toByteBuffer(data, version); OffsetCommitResponse response = OffsetCommitResponse.parse(buffer, version); assertEquals(expectedErrorCounts, response.errorCounts()); diff --git a/clients/src/test/java/org/apache/kafka/common/requests/OffsetFetchRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/OffsetFetchRequestTest.java index 51db93d629d40..ddb2cd9a943ad 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/OffsetFetchRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/OffsetFetchRequestTest.java @@ -75,7 +75,7 @@ public void testConstructor() { )); } - for (short version = 0; version <= ApiKeys.OFFSET_FETCH.latestVersion(); version++) { + for (short version : ApiKeys.OFFSET_FETCH.allVersions()) { OffsetFetchRequest request = builder.build(version); assertFalse(request.isAllPartitions()); assertEquals(groupId, request.groupId()); @@ -101,7 +101,7 @@ public void testConstructor() { @Test public void testConstructorFailForUnsupportedRequireStable() { - for (short version = 0; version <= ApiKeys.OFFSET_FETCH.latestVersion(); version++) { + for (short version : ApiKeys.OFFSET_FETCH.allVersions()) { // The builder needs to be initialized every cycle as the internal data `requireStable` flag is flipped. builder = new OffsetFetchRequest.Builder(groupId, true, null, false); final short finalVersion = version; @@ -123,7 +123,7 @@ public void testConstructorFailForUnsupportedRequireStable() { @Test public void testBuildThrowForUnsupportedRequireStable() { - for (short version = 0; version <= ApiKeys.OFFSET_FETCH.latestVersion(); version++) { + for (short version : ApiKeys.OFFSET_FETCH.allVersions()) { builder = new OffsetFetchRequest.Builder(groupId, true, null, true); if (version < 7) { final short finalVersion = version; diff --git a/clients/src/test/java/org/apache/kafka/common/requests/OffsetFetchResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/OffsetFetchResponseTest.java index 62cf3a9535399..e202fc200f7a8 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/OffsetFetchResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/OffsetFetchResponseTest.java @@ -103,7 +103,7 @@ public void testStructBuild() { OffsetFetchResponse latestResponse = new OffsetFetchResponse(throttleTimeMs, Errors.NONE, partitionDataMap); - for (short version = 0; version <= ApiKeys.OFFSET_FETCH.latestVersion(); version++) { + for (short version : ApiKeys.OFFSET_FETCH.allVersions()) { OffsetFetchResponseData data = new OffsetFetchResponseData( new ByteBufferAccessor(latestResponse.serialize(version)), version); @@ -154,7 +154,7 @@ public void testStructBuild() { @Test public void testShouldThrottle() { OffsetFetchResponse response = new OffsetFetchResponse(throttleTimeMs, Errors.NONE, partitionDataMap); - for (short version = 0; version <= ApiKeys.OFFSET_FETCH.latestVersion(); version++) { + for (short version : ApiKeys.OFFSET_FETCH.allVersions()) { if (version >= 4) { assertTrue(response.shouldClientThrottle(version)); } else { diff --git a/clients/src/test/java/org/apache/kafka/common/requests/OffsetsForLeaderEpochRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/OffsetsForLeaderEpochRequestTest.java index 3d4b41b52e435..e5dacd5b3d799 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/OffsetsForLeaderEpochRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/OffsetsForLeaderEpochRequestTest.java @@ -34,15 +34,15 @@ public void testForConsumerRequiresVersion3() { assertThrows(UnsupportedVersionException.class, () -> builder.build(v)); } - for (short version = 3; version < ApiKeys.OFFSET_FOR_LEADER_EPOCH.latestVersion(); version++) { - OffsetsForLeaderEpochRequest request = builder.build((short) 3); + for (short version = 3; version <= ApiKeys.OFFSET_FOR_LEADER_EPOCH.latestVersion(); version++) { + OffsetsForLeaderEpochRequest request = builder.build(version); assertEquals(OffsetsForLeaderEpochRequest.CONSUMER_REPLICA_ID, request.replicaId()); } } @Test public void testDefaultReplicaId() { - for (short version = 0; version < ApiKeys.OFFSET_FOR_LEADER_EPOCH.latestVersion(); version++) { + for (short version : ApiKeys.OFFSET_FOR_LEADER_EPOCH.allVersions()) { int replicaId = 1; OffsetsForLeaderEpochRequest.Builder builder = OffsetsForLeaderEpochRequest.Builder.forFollower( version, new OffsetForLeaderTopicCollection(), replicaId); diff --git a/clients/src/test/java/org/apache/kafka/common/requests/ProduceRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/ProduceRequestTest.java index a2367e4904285..fee026e7481cf 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/ProduceRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/ProduceRequestTest.java @@ -18,6 +18,7 @@ package org.apache.kafka.common.requests; import org.apache.kafka.common.InvalidRecordException; +import org.apache.kafka.common.errors.UnsupportedCompressionTypeException; import org.apache.kafka.common.message.ProduceRequestData; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.record.CompressionType; @@ -32,11 +33,12 @@ import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collections; +import java.util.stream.IntStream; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; public class ProduceRequestTest { @@ -151,7 +153,7 @@ public void testV3AndAboveShouldContainOnlyOneRecordBatch() { .setRecords(MemoryRecords.readableRecords(buffer))))).iterator())) .setAcks((short) 1) .setTimeoutMs(5000)); - assertThrowsInvalidRecordExceptionForAllVersions(requestBuilder); + assertThrowsForAllVersions(requestBuilder, InvalidRecordException.class); } @Test @@ -166,7 +168,7 @@ public void testV3AndAboveCannotHaveNoRecordBatches() { .setRecords(MemoryRecords.EMPTY)))).iterator())) .setAcks((short) 1) .setTimeoutMs(5000)); - assertThrowsInvalidRecordExceptionForAllVersions(requestBuilder); + assertThrowsForAllVersions(requestBuilder, InvalidRecordException.class); } @Test @@ -186,7 +188,7 @@ public void testV3AndAboveCannotUseMagicV0() { .setRecords(builder.build())))).iterator())) .setAcks((short) 1) .setTimeoutMs(5000)); - assertThrowsInvalidRecordExceptionForAllVersions(requestBuilder); + assertThrowsForAllVersions(requestBuilder, InvalidRecordException.class); } @Test @@ -206,7 +208,7 @@ public void testV3AndAboveCannotUseMagicV1() { .iterator())) .setAcks((short) 1) .setTimeoutMs(5000)); - assertThrowsInvalidRecordExceptionForAllVersions(requestBuilder); + assertThrowsForAllVersions(requestBuilder, InvalidRecordException.class); } @Test @@ -230,7 +232,7 @@ public void testV6AndBelowCannotUseZStdCompression() { for (short version = 3; version < 7; version++) { ProduceRequest.Builder requestBuilder = new ProduceRequest.Builder(version, version, produceData); - assertThrowsInvalidRecordExceptionForAllVersions(requestBuilder); + assertThrowsForAllVersions(requestBuilder, UnsupportedCompressionTypeException.class); } // Works fine with current version (>= 7) @@ -291,20 +293,10 @@ public void testMixedIdempotentData() { assertTrue(RequestTestUtils.hasIdempotentRecords(request)); } - private void assertThrowsInvalidRecordExceptionForAllVersions(ProduceRequest.Builder builder) { - for (short version = builder.oldestAllowedVersion(); version < builder.latestAllowedVersion(); version++) { - assertThrowsInvalidRecordException(builder, version); - } - } - - private void assertThrowsInvalidRecordException(ProduceRequest.Builder builder, short version) { - try { - builder.build(version).serialize(); - fail("Builder did not raise " + InvalidRecordException.class.getName() + " as expected"); - } catch (RuntimeException e) { - assertTrue(InvalidRecordException.class.isAssignableFrom(e.getClass()), - "Unexpected exception type " + e.getClass().getName()); - } + private static void assertThrowsForAllVersions(ProduceRequest.Builder builder, + Class expectedType) { + IntStream.range(builder.oldestAllowedVersion(), builder.latestAllowedVersion() + 1) + .forEach(version -> assertThrows(expectedType, () -> builder.build((short) version).serialize())); } private ProduceRequest createNonIdempotentNonTransactionalRecords() { diff --git a/clients/src/test/java/org/apache/kafka/common/requests/ProduceResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/ProduceResponseTest.java index bfadb8c3bbd70..771ff0da7b44e 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/ProduceResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/ProduceResponseTest.java @@ -89,10 +89,10 @@ public void produceResponseRecordErrorsTest() { "Produce failed"); responseData.put(tp, partResponse); - for (short ver = 0; ver <= PRODUCE.latestVersion(); ver++) { + for (short version : PRODUCE.allVersions()) { ProduceResponse response = new ProduceResponse(responseData); - ProduceResponse.PartitionResponse deserialized = ProduceResponse.parse(response.serialize(ver), ver).responses().get(tp); - if (ver >= 8) { + ProduceResponse.PartitionResponse deserialized = ProduceResponse.parse(response.serialize(version), version).responses().get(tp); + if (version >= 8) { assertEquals(1, deserialized.recordErrors.size()); assertEquals(3, deserialized.recordErrors.get(0).batchIndex); assertEquals("Record error", deserialized.recordErrors.get(0).message); diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java index cebb99d17ef22..d3d965d3910ec 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java @@ -221,11 +221,16 @@ import static java.util.Arrays.asList; import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; +import static org.apache.kafka.common.protocol.ApiKeys.CREATE_PARTITIONS; +import static org.apache.kafka.common.protocol.ApiKeys.CREATE_TOPICS; +import static org.apache.kafka.common.protocol.ApiKeys.DELETE_TOPICS; import static org.apache.kafka.common.protocol.ApiKeys.DESCRIBE_CONFIGS; import static org.apache.kafka.common.protocol.ApiKeys.FETCH; import static org.apache.kafka.common.protocol.ApiKeys.JOIN_GROUP; +import static org.apache.kafka.common.protocol.ApiKeys.LEADER_AND_ISR; import static org.apache.kafka.common.protocol.ApiKeys.LIST_GROUPS; import static org.apache.kafka.common.protocol.ApiKeys.LIST_OFFSETS; +import static org.apache.kafka.common.protocol.ApiKeys.STOP_REPLICA; import static org.apache.kafka.common.protocol.ApiKeys.SYNC_GROUP; import static org.apache.kafka.common.requests.FetchMetadata.INVALID_SESSION_ID; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -267,26 +272,26 @@ public void testSerialization() throws Exception { checkErrorResponse(createHeartBeatRequest(), unknownServerException, true); checkResponse(createHeartBeatResponse(), 0, true); - for (int v = ApiKeys.JOIN_GROUP.oldestVersion(); v <= ApiKeys.JOIN_GROUP.latestVersion(); v++) { - checkRequest(createJoinGroupRequest(v), true); - checkErrorResponse(createJoinGroupRequest(v), unknownServerException, true); - checkResponse(createJoinGroupResponse(v), v, true); + for (short version : JOIN_GROUP.allVersions()) { + checkRequest(createJoinGroupRequest(version), true); + checkErrorResponse(createJoinGroupRequest(version), unknownServerException, true); + checkResponse(createJoinGroupResponse(version), version, true); } - for (int v = ApiKeys.SYNC_GROUP.oldestVersion(); v <= ApiKeys.SYNC_GROUP.latestVersion(); v++) { - checkRequest(createSyncGroupRequest(v), true); - checkErrorResponse(createSyncGroupRequest(v), unknownServerException, true); - checkResponse(createSyncGroupResponse(v), v, true); + for (short version : SYNC_GROUP.allVersions()) { + checkRequest(createSyncGroupRequest(version), true); + checkErrorResponse(createSyncGroupRequest(version), unknownServerException, true); + checkResponse(createSyncGroupResponse(version), version, true); } checkRequest(createLeaveGroupRequest(), true); checkErrorResponse(createLeaveGroupRequest(), unknownServerException, true); checkResponse(createLeaveGroupResponse(), 0, true); - for (short v = ApiKeys.LIST_GROUPS.oldestVersion(); v <= ApiKeys.LIST_GROUPS.latestVersion(); v++) { - checkRequest(createListGroupsRequest(v), false); - checkErrorResponse(createListGroupsRequest(v), unknownServerException, true); - checkResponse(createListGroupsResponse(v), v, true); + for (short version : ApiKeys.LIST_GROUPS.allVersions()) { + checkRequest(createListGroupsRequest(version), false); + checkErrorResponse(createListGroupsRequest(version), unknownServerException, true); + checkResponse(createListGroupsResponse(version), version, true); } checkRequest(createDescribeGroupRequest(), true); @@ -295,10 +300,10 @@ public void testSerialization() throws Exception { checkRequest(createDeleteGroupsRequest(), true); checkErrorResponse(createDeleteGroupsRequest(), unknownServerException, true); checkResponse(createDeleteGroupsResponse(), 0, true); - for (int i = 0; i < ApiKeys.LIST_OFFSETS.latestVersion(); i++) { - checkRequest(createListOffsetRequest(i), true); - checkErrorResponse(createListOffsetRequest(i), unknownServerException, true); - checkResponse(createListOffsetResponse(i), i, true); + for (short version : LIST_OFFSETS.allVersions()) { + checkRequest(createListOffsetRequest(version), true); + checkErrorResponse(createListOffsetRequest(version), unknownServerException, true); + checkResponse(createListOffsetResponse(version), version, true); } checkRequest(MetadataRequest.Builder.allTopics().build((short) 2), true); checkRequest(createMetadataRequest(1, Collections.singletonList("topic1")), true); @@ -331,18 +336,18 @@ public void testSerialization() throws Exception { checkResponse(createProduceResponse(), 2, true); checkResponse(createProduceResponseWithErrorMessage(), 8, true); - for (int v = ApiKeys.STOP_REPLICA.oldestVersion(); v <= ApiKeys.STOP_REPLICA.latestVersion(); v++) { - checkRequest(createStopReplicaRequest(v, true), true); - checkRequest(createStopReplicaRequest(v, false), true); - checkErrorResponse(createStopReplicaRequest(v, true), unknownServerException, true); - checkErrorResponse(createStopReplicaRequest(v, false), unknownServerException, true); - checkResponse(createStopReplicaResponse(), v, true); + for (short version : STOP_REPLICA.allVersions()) { + checkRequest(createStopReplicaRequest(version, true), true); + checkRequest(createStopReplicaRequest(version, false), true); + checkErrorResponse(createStopReplicaRequest(version, true), unknownServerException, true); + checkErrorResponse(createStopReplicaRequest(version, false), unknownServerException, true); + checkResponse(createStopReplicaResponse(), version, true); } - for (int v = ApiKeys.LEADER_AND_ISR.oldestVersion(); v <= ApiKeys.LEADER_AND_ISR.latestVersion(); v++) { - checkRequest(createLeaderAndIsrRequest(v), true); - checkErrorResponse(createLeaderAndIsrRequest(v), unknownServerException, false); - checkResponse(createLeaderAndIsrResponse(v), v, true); + for (short version : LEADER_AND_ISR.allVersions()) { + checkRequest(createLeaderAndIsrRequest(version), true); + checkErrorResponse(createLeaderAndIsrRequest(version), unknownServerException, false); + checkResponse(createLeaderAndIsrResponse(version), version, true); } checkRequest(createSaslHandshakeRequest(), true); @@ -353,23 +358,23 @@ public void testSerialization() throws Exception { checkResponse(createSaslAuthenticateResponse(), 0, true); checkResponse(createSaslAuthenticateResponse(), 1, true); - for (int v = ApiKeys.CREATE_TOPICS.oldestVersion(); v <= ApiKeys.CREATE_TOPICS.latestVersion(); v++) { - checkRequest(createCreateTopicRequest(v), true); - checkErrorResponse(createCreateTopicRequest(v), unknownServerException, true); - checkResponse(createCreateTopicResponse(), v, true); + for (short version : CREATE_TOPICS.allVersions()) { + checkRequest(createCreateTopicRequest(version), true); + checkErrorResponse(createCreateTopicRequest(version), unknownServerException, true); + checkResponse(createCreateTopicResponse(), version, true); } - for (int v = ApiKeys.DELETE_TOPICS.oldestVersion(); v <= ApiKeys.DELETE_TOPICS.latestVersion(); v++) { - checkRequest(createDeleteTopicsRequest(v), true); - checkErrorResponse(createDeleteTopicsRequest(v), unknownServerException, true); - checkResponse(createDeleteTopicsResponse(), v, true); + for (short version : DELETE_TOPICS.allVersions()) { + checkRequest(createDeleteTopicsRequest(version), true); + checkErrorResponse(createDeleteTopicsRequest(version), unknownServerException, true); + checkResponse(createDeleteTopicsResponse(), version, true); } - for (int v = ApiKeys.CREATE_PARTITIONS.oldestVersion(); v <= ApiKeys.CREATE_PARTITIONS.latestVersion(); v++) { - checkRequest(createCreatePartitionsRequest(v), true); - checkRequest(createCreatePartitionsRequestWithAssignments(v), false); - checkErrorResponse(createCreatePartitionsRequest(v), unknownServerException, true); - checkResponse(createCreatePartitionsResponse(), v, true); + for (short version : CREATE_PARTITIONS.allVersions()) { + checkRequest(createCreatePartitionsRequest(version), true); + checkRequest(createCreatePartitionsRequestWithAssignments(version), false); + checkErrorResponse(createCreatePartitionsRequest(version), unknownServerException, true); + checkResponse(createCreatePartitionsResponse(), version, true); } checkRequest(createInitPidRequest(), true); @@ -518,75 +523,75 @@ public void testSerialization() throws Exception { @Test public void testApiVersionsSerialization() { - for (short v : ApiKeys.API_VERSIONS.allVersions()) { - checkRequest(createApiVersionRequest(v), true); - checkErrorResponse(createApiVersionRequest(v), unknownServerException, true); - checkErrorResponse(createApiVersionRequest(v), new UnsupportedVersionException("Not Supported"), true); - checkResponse(createApiVersionResponse(), v, true); - checkResponse(ApiVersionsResponse.defaultApiVersionsResponse(ApiMessageType.ListenerType.ZK_BROKER), v, true); + for (short version : ApiKeys.API_VERSIONS.allVersions()) { + checkRequest(createApiVersionRequest(version), true); + checkErrorResponse(createApiVersionRequest(version), unknownServerException, true); + checkErrorResponse(createApiVersionRequest(version), new UnsupportedVersionException("Not Supported"), true); + checkResponse(createApiVersionResponse(), version, true); + checkResponse(ApiVersionsResponse.defaultApiVersionsResponse(ApiMessageType.ListenerType.ZK_BROKER), version, true); } } @Test public void testBrokerHeartbeatSerialization() { - for (short v : ApiKeys.BROKER_HEARTBEAT.allVersions()) { - checkRequest(createBrokerHeartbeatRequest(v), true); - checkErrorResponse(createBrokerHeartbeatRequest(v), unknownServerException, true); - checkResponse(createBrokerHeartbeatResponse(), v, true); + for (short version : ApiKeys.BROKER_HEARTBEAT.allVersions()) { + checkRequest(createBrokerHeartbeatRequest(version), true); + checkErrorResponse(createBrokerHeartbeatRequest(version), unknownServerException, true); + checkResponse(createBrokerHeartbeatResponse(), version, true); } } @Test public void testBrokerRegistrationSerialization() { - for (short v : ApiKeys.BROKER_REGISTRATION.allVersions()) { - checkRequest(createBrokerRegistrationRequest(v), true); - checkErrorResponse(createBrokerRegistrationRequest(v), unknownServerException, true); + for (short version : ApiKeys.BROKER_REGISTRATION.allVersions()) { + checkRequest(createBrokerRegistrationRequest(version), true); + checkErrorResponse(createBrokerRegistrationRequest(version), unknownServerException, true); checkResponse(createBrokerRegistrationResponse(), 0, true); } } @Test public void testDescribeProducersSerialization() { - for (short v : ApiKeys.DESCRIBE_PRODUCERS.allVersions()) { - checkRequest(createDescribeProducersRequest(v), true); - checkErrorResponse(createDescribeProducersRequest(v), unknownServerException, true); - checkResponse(createDescribeProducersResponse(), v, true); + for (short version : ApiKeys.DESCRIBE_PRODUCERS.allVersions()) { + checkRequest(createDescribeProducersRequest(version), true); + checkErrorResponse(createDescribeProducersRequest(version), unknownServerException, true); + checkResponse(createDescribeProducersResponse(), version, true); } } @Test public void testDescribeTransactionsSerialization() { - for (short v : ApiKeys.DESCRIBE_TRANSACTIONS.allVersions()) { - checkRequest(createDescribeTransactionsRequest(v), true); - checkErrorResponse(createDescribeTransactionsRequest(v), unknownServerException, true); - checkResponse(createDescribeTransactionsResponse(), v, true); + for (short version : ApiKeys.DESCRIBE_TRANSACTIONS.allVersions()) { + checkRequest(createDescribeTransactionsRequest(version), true); + checkErrorResponse(createDescribeTransactionsRequest(version), unknownServerException, true); + checkResponse(createDescribeTransactionsResponse(), version, true); } } @Test public void testListTransactionsSerialization() { - for (short v : ApiKeys.LIST_TRANSACTIONS.allVersions()) { - checkRequest(createListTransactionsRequest(v), true); - checkErrorResponse(createListTransactionsRequest(v), unknownServerException, true); - checkResponse(createListTransactionsResponse(), v, true); + for (short version : ApiKeys.LIST_TRANSACTIONS.allVersions()) { + checkRequest(createListTransactionsRequest(version), true); + checkErrorResponse(createListTransactionsRequest(version), unknownServerException, true); + checkResponse(createListTransactionsResponse(), version, true); } } @Test public void testDescribeClusterSerialization() { - for (short v : ApiKeys.DESCRIBE_CLUSTER.allVersions()) { - checkRequest(createDescribeClusterRequest(v), true); - checkErrorResponse(createDescribeClusterRequest(v), unknownServerException, true); - checkResponse(createDescribeClusterResponse(), v, true); + for (short version : ApiKeys.DESCRIBE_CLUSTER.allVersions()) { + checkRequest(createDescribeClusterRequest(version), true); + checkErrorResponse(createDescribeClusterRequest(version), unknownServerException, true); + checkResponse(createDescribeClusterResponse(), version, true); } } @Test public void testUnregisterBrokerSerialization() { - for (short v : ApiKeys.UNREGISTER_BROKER.allVersions()) { - checkRequest(createUnregisterBrokerRequest(v), true); - checkErrorResponse(createUnregisterBrokerRequest(v), unknownServerException, true); - checkResponse(createUnregisterBrokerResponse(), v, true); + for (short version : ApiKeys.UNREGISTER_BROKER.allVersions()) { + checkRequest(createUnregisterBrokerRequest(version), true); + checkErrorResponse(createUnregisterBrokerRequest(version), unknownServerException, true); + checkResponse(createUnregisterBrokerResponse(), version, true); } } @@ -623,13 +628,12 @@ public void testResponseHeader() { } private void checkOlderFetchVersions() { - int latestVersion = FETCH.latestVersion(); - for (int i = 0; i < latestVersion; ++i) { - if (i > 7) { - checkErrorResponse(createFetchRequest(i), unknownServerException, true); + for (short version : FETCH.allVersions()) { + if (version > 7) { + checkErrorResponse(createFetchRequest(version), unknownServerException, true); } - checkRequest(createFetchRequest(i), true); - checkResponse(createFetchResponse(i >= 4), i, true); + checkRequest(createFetchRequest(version), true); + checkResponse(createFetchResponse(version >= 4), version, true); } } @@ -668,12 +672,11 @@ private void verifyDescribeConfigsResponse(DescribeConfigsResponse expected, Des } private void checkDescribeConfigsResponseVersions() { - for (int version = ApiKeys.DESCRIBE_CONFIGS.oldestVersion(); version < ApiKeys.DESCRIBE_CONFIGS.latestVersion(); ++version) { - short apiVersion = (short) version; - DescribeConfigsResponse response = createDescribeConfigsResponse(apiVersion); + for (short version : ApiKeys.DESCRIBE_CONFIGS.allVersions()) { + DescribeConfigsResponse response = createDescribeConfigsResponse(version); DescribeConfigsResponse deserialized0 = (DescribeConfigsResponse) AbstractResponse.parseResponse(ApiKeys.DESCRIBE_CONFIGS, - response.serialize(apiVersion), apiVersion); - verifyDescribeConfigsResponse(response, deserialized0, apiVersion); + response.serialize(version), version); + verifyDescribeConfigsResponse(response, deserialized0, version); } } @@ -859,7 +862,7 @@ public void verifyFetchResponseFullWrites() throws Exception { verifyFetchResponseFullWrite(FETCH.latestVersion(), createFetchResponse(123)); verifyFetchResponseFullWrite(FETCH.latestVersion(), createFetchResponse(Errors.FETCH_SESSION_ID_NOT_FOUND, 123)); - for (short version = 0; version <= FETCH.latestVersion(); version++) { + for (short version : FETCH.allVersions()) { verifyFetchResponseFullWrite(version, createFetchResponse(version >= 4)); } } diff --git a/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaRequestTest.java index 50c6974613951..3446d498c6aac 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaRequestTest.java @@ -66,7 +66,7 @@ public void testGetErrorResponse() { } } - for (short version = STOP_REPLICA.oldestVersion(); version <= STOP_REPLICA.latestVersion(); version++) { + for (short version : STOP_REPLICA.allVersions()) { StopReplicaRequest.Builder builder = new StopReplicaRequest.Builder(version, 0, 0, 0L, false, topicStates); StopReplicaRequest request = builder.build(); @@ -93,7 +93,7 @@ private void testBuilderNormalization(boolean deletePartitions) { Map expectedPartitionStates = StopReplicaRequestTest.partitionStates(topicStates); - for (short version = STOP_REPLICA.oldestVersion(); version <= STOP_REPLICA.latestVersion(); version++) { + for (short version : STOP_REPLICA.allVersions()) { StopReplicaRequest request = new StopReplicaRequest.Builder(version, 0, 1, 0, deletePartitions, topicStates).build(version); StopReplicaRequestData data = request.data(); @@ -128,7 +128,7 @@ private void testBuilderNormalization(boolean deletePartitions) { public void testTopicStatesNormalization() { List topicStates = topicStates(true); - for (short version = STOP_REPLICA.oldestVersion(); version <= STOP_REPLICA.latestVersion(); version++) { + for (short version : STOP_REPLICA.allVersions()) { // Create a request for version to get its serialized form StopReplicaRequest baseRequest = new StopReplicaRequest.Builder(version, 0, 1, 0, true, topicStates).build(version); @@ -163,7 +163,7 @@ public void testTopicStatesNormalization() { public void testPartitionStatesNormalization() { List topicStates = topicStates(true); - for (short version = STOP_REPLICA.oldestVersion(); version <= STOP_REPLICA.latestVersion(); version++) { + for (short version : STOP_REPLICA.allVersions()) { // Create a request for version to get its serialized form StopReplicaRequest baseRequest = new StopReplicaRequest.Builder(version, 0, 1, 0, true, topicStates).build(version); diff --git a/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaResponseTest.java index c3d049d482cc9..a0a5eda01e822 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaResponseTest.java @@ -44,7 +44,7 @@ public void testErrorCountsFromGetErrorResponse() { new StopReplicaPartitionState().setPartitionIndex(0), new StopReplicaPartitionState().setPartitionIndex(1)))); - for (short version = STOP_REPLICA.oldestVersion(); version <= STOP_REPLICA.latestVersion(); version++) { + for (short version : STOP_REPLICA.allVersions()) { StopReplicaRequest request = new StopReplicaRequest.Builder(version, 15, 20, 0, false, topicStates).build(version); StopReplicaResponse response = request diff --git a/clients/src/test/java/org/apache/kafka/common/requests/TxnOffsetCommitRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/TxnOffsetCommitRequestTest.java index ec31e587c81e1..037066e9a036d 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/TxnOffsetCommitRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/TxnOffsetCommitRequestTest.java @@ -116,7 +116,7 @@ public void testConstructor() { )) ); - for (short version = 0; version <= ApiKeys.TXN_OFFSET_COMMIT.latestVersion(); version++) { + for (short version : ApiKeys.TXN_OFFSET_COMMIT.allVersions()) { final TxnOffsetCommitRequest request; if (version < 3) { request = builder.build(version); diff --git a/clients/src/test/java/org/apache/kafka/common/requests/TxnOffsetCommitResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/TxnOffsetCommitResponseTest.java index f3510f1725159..1f19ff2c93714 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/TxnOffsetCommitResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/TxnOffsetCommitResponseTest.java @@ -54,7 +54,7 @@ public void testParse() { .setErrorCode(errorTwo.code()))) )); - for (short version = 0; version <= ApiKeys.TXN_OFFSET_COMMIT.latestVersion(); version++) { + for (short version : ApiKeys.TXN_OFFSET_COMMIT.allVersions()) { TxnOffsetCommitResponse response = TxnOffsetCommitResponse.parse( MessageUtil.toByteBuffer(data, version), version); assertEquals(expectedErrorCounts, response.errorCounts()); diff --git a/clients/src/test/java/org/apache/kafka/common/requests/UpdateMetadataRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/UpdateMetadataRequestTest.java index 86178ca8521e3..6f9d5c2454606 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/UpdateMetadataRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/UpdateMetadataRequestTest.java @@ -61,7 +61,7 @@ public void testUnsupportedVersion() { @Test public void testGetErrorResponse() { - for (short version = UPDATE_METADATA.oldestVersion(); version < UPDATE_METADATA.latestVersion(); version++) { + for (short version : UPDATE_METADATA.allVersions()) { UpdateMetadataRequest.Builder builder = new UpdateMetadataRequest.Builder( version, 0, 0, 0, Collections.emptyList(), Collections.emptyList(), Collections.emptyMap()); UpdateMetadataRequest request = builder.build(); @@ -81,7 +81,7 @@ public void testGetErrorResponse() { public void testVersionLogic() { String topic0 = "topic0"; String topic1 = "topic1"; - for (short version = UPDATE_METADATA.oldestVersion(); version <= UPDATE_METADATA.latestVersion(); version++) { + for (short version : UPDATE_METADATA.allVersions()) { List partitionStates = asList( new UpdateMetadataPartitionState() .setTopicName(topic0) diff --git a/clients/src/test/java/org/apache/kafka/common/requests/WriteTxnMarkersRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/WriteTxnMarkersRequestTest.java index 6435845b766c8..13e8c8cd94035 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/WriteTxnMarkersRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/WriteTxnMarkersRequestTest.java @@ -51,7 +51,7 @@ public void setUp() { @Test public void testConstructor() { WriteTxnMarkersRequest.Builder builder = new WriteTxnMarkersRequest.Builder(ApiKeys.WRITE_TXN_MARKERS.latestVersion(), markers); - for (short version = 0; version <= ApiKeys.WRITE_TXN_MARKERS.latestVersion(); version++) { + for (short version : ApiKeys.WRITE_TXN_MARKERS.allVersions()) { WriteTxnMarkersRequest request = builder.build(version); assertEquals(1, request.markers().size()); WriteTxnMarkersRequest.TxnMarkerEntry marker = request.markers().get(0); @@ -66,7 +66,7 @@ public void testConstructor() { @Test public void testGetErrorResponse() { WriteTxnMarkersRequest.Builder builder = new WriteTxnMarkersRequest.Builder(ApiKeys.WRITE_TXN_MARKERS.latestVersion(), markers); - for (short version = 0; version <= ApiKeys.WRITE_TXN_MARKERS.latestVersion(); version++) { + for (short version : ApiKeys.WRITE_TXN_MARKERS.allVersions()) { WriteTxnMarkersRequest request = builder.build(version); WriteTxnMarkersResponse errorResponse = request.getErrorResponse(throttleTimeMs, Errors.UNKNOWN_PRODUCER_ID.exception()); From 96a2b7aac4f1c4a9e020ccc08dadc7a71b460abc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Armando=20Garc=C3=ADa=20Sancio?= Date: Thu, 4 Mar 2021 10:55:43 -0800 Subject: [PATCH 12/59] KAFKA-12376: Apply atomic append to the log (#10253) --- .../main/scala/kafka/raft/RaftManager.scala | 29 ++++++- .../controller/ClientQuotaControlManager.java | 2 +- .../controller/ClusterControlManager.java | 2 +- .../ConfigurationControlManager.java | 4 +- .../kafka/controller/ControllerResult.java | 38 ++++++--- .../controller/ControllerResultAndOffset.java | 32 +++---- .../controller/FeatureControlManager.java | 3 +- .../kafka/controller/QuorumController.java | 23 ++--- .../controller/ReplicationControlManager.java | 12 +-- .../apache/kafka/metalog/LocalLogManager.java | 17 +++- .../apache/kafka/metalog/MetaLogManager.java | 23 ++++- .../ConfigurationControlManagerTest.java | 83 ++++++++++++++----- .../controller/FeatureControlManagerTest.java | 58 +++++++++---- .../apache/kafka/metalog/LocalLogManager.java | 17 +++- .../kafka/raft/metadata/MetaLogRaftShim.java | 28 ++++++- 15 files changed, 268 insertions(+), 103 deletions(-) diff --git a/core/src/main/scala/kafka/raft/RaftManager.scala b/core/src/main/scala/kafka/raft/RaftManager.scala index 1881a1db9a70d..3b714f3786862 100644 --- a/core/src/main/scala/kafka/raft/RaftManager.scala +++ b/core/src/main/scala/kafka/raft/RaftManager.scala @@ -92,6 +92,11 @@ trait RaftManager[T] { listener: RaftClient.Listener[T] ): Unit + def scheduleAtomicAppend( + epoch: Int, + records: Seq[T] + ): Option[Long] + def scheduleAppend( epoch: Int, records: Seq[T] @@ -157,16 +162,32 @@ class KafkaRaftManager[T]( raftClient.register(listener) } + override def scheduleAtomicAppend( + epoch: Int, + records: Seq[T] + ): Option[Long] = { + append(epoch, records, true) + } + override def scheduleAppend( epoch: Int, records: Seq[T] ): Option[Long] = { - val offset: java.lang.Long = raftClient.scheduleAppend(epoch, records.asJava) - if (offset == null) { - None + append(epoch, records, false) + } + + private def append( + epoch: Int, + records: Seq[T], + isAtomic: Boolean + ): Option[Long] = { + val offset = if (isAtomic) { + raftClient.scheduleAtomicAppend(epoch, records.asJava) } else { - Some(Long.unbox(offset)) + raftClient.scheduleAppend(epoch, records.asJava) } + + Option(offset).map(Long.unbox) } override def handleRequest( diff --git a/metadata/src/main/java/org/apache/kafka/controller/ClientQuotaControlManager.java b/metadata/src/main/java/org/apache/kafka/controller/ClientQuotaControlManager.java index 4aac9e4882f46..9b8e2d683b650 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/ClientQuotaControlManager.java +++ b/metadata/src/main/java/org/apache/kafka/controller/ClientQuotaControlManager.java @@ -86,7 +86,7 @@ ControllerResult> alterClientQuotas( } }); - return new ControllerResult<>(outputRecords, outputResults); + return ControllerResult.atomicOf(outputRecords, outputResults); } /** diff --git a/metadata/src/main/java/org/apache/kafka/controller/ClusterControlManager.java b/metadata/src/main/java/org/apache/kafka/controller/ClusterControlManager.java index 6e329c72a0e3f..4748d195986ab 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/ClusterControlManager.java +++ b/metadata/src/main/java/org/apache/kafka/controller/ClusterControlManager.java @@ -213,7 +213,7 @@ public ControllerResult registerBroker( List records = new ArrayList<>(); records.add(new ApiMessageAndVersion(record, (short) 0)); - return new ControllerResult<>(records, new BrokerRegistrationReply(brokerEpoch)); + return ControllerResult.of(records, new BrokerRegistrationReply(brokerEpoch)); } public void replay(RegisterBrokerRecord record) { diff --git a/metadata/src/main/java/org/apache/kafka/controller/ConfigurationControlManager.java b/metadata/src/main/java/org/apache/kafka/controller/ConfigurationControlManager.java index 4402b3a117d83..5bc82ecafa477 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/ConfigurationControlManager.java +++ b/metadata/src/main/java/org/apache/kafka/controller/ConfigurationControlManager.java @@ -83,7 +83,7 @@ ControllerResult> incrementalAlterConfigs( outputRecords, outputResults); } - return new ControllerResult<>(outputRecords, outputResults); + return ControllerResult.atomicOf(outputRecords, outputResults); } private void incrementalAlterConfigResource(ConfigResource configResource, @@ -171,7 +171,7 @@ ControllerResult> legacyAlterConfigs( outputRecords, outputResults); } - return new ControllerResult<>(outputRecords, outputResults); + return ControllerResult.atomicOf(outputRecords, outputResults); } private void legacyAlterConfigResource(ConfigResource configResource, diff --git a/metadata/src/main/java/org/apache/kafka/controller/ControllerResult.java b/metadata/src/main/java/org/apache/kafka/controller/ControllerResult.java index 4906c8b0972a8..e6ae031b9b3b8 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/ControllerResult.java +++ b/metadata/src/main/java/org/apache/kafka/controller/ControllerResult.java @@ -19,7 +19,7 @@ import org.apache.kafka.metadata.ApiMessageAndVersion; -import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.stream.Collectors; @@ -28,15 +28,13 @@ class ControllerResult { private final List records; private final T response; + private final boolean isAtomic; - public ControllerResult(T response) { - this(new ArrayList<>(), response); - } - - public ControllerResult(List records, T response) { + protected ControllerResult(List records, T response, boolean isAtomic) { Objects.requireNonNull(records); this.records = records; this.response = response; + this.isAtomic = isAtomic; } public List records() { @@ -47,6 +45,10 @@ public T response() { return response; } + public boolean isAtomic() { + return isAtomic; + } + @Override public boolean equals(Object o) { if (o == null || (!o.getClass().equals(getClass()))) { @@ -54,22 +56,34 @@ public boolean equals(Object o) { } ControllerResult other = (ControllerResult) o; return records.equals(other.records) && - Objects.equals(response, other.response); + Objects.equals(response, other.response) && + Objects.equals(isAtomic, other.isAtomic); } @Override public int hashCode() { - return Objects.hash(records, response); + return Objects.hash(records, response, isAtomic); } @Override public String toString() { - return "ControllerResult(records=" + String.join(",", - records.stream().map(r -> r.toString()).collect(Collectors.toList())) + - ", response=" + response + ")"; + return String.format( + "ControllerResult(records=%s, response=%s, isAtomic=%s)", + String.join(",", records.stream().map(ApiMessageAndVersion::toString).collect(Collectors.toList())), + response, + isAtomic + ); } public ControllerResult withoutRecords() { - return new ControllerResult<>(new ArrayList<>(), response); + return new ControllerResult<>(Collections.emptyList(), response, false); + } + + public static ControllerResult atomicOf(List records, T response) { + return new ControllerResult<>(records, response, true); + } + + public static ControllerResult of(List records, T response) { + return new ControllerResult<>(records, response, false); } } diff --git a/metadata/src/main/java/org/apache/kafka/controller/ControllerResultAndOffset.java b/metadata/src/main/java/org/apache/kafka/controller/ControllerResultAndOffset.java index 5e483f773d5e2..8b8ca8dea80da 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/ControllerResultAndOffset.java +++ b/metadata/src/main/java/org/apache/kafka/controller/ControllerResultAndOffset.java @@ -19,24 +19,15 @@ import org.apache.kafka.metadata.ApiMessageAndVersion; -import java.util.ArrayList; -import java.util.List; import java.util.Objects; import java.util.stream.Collectors; -class ControllerResultAndOffset extends ControllerResult { +final class ControllerResultAndOffset extends ControllerResult { private final long offset; - public ControllerResultAndOffset(T response) { - super(new ArrayList<>(), response); - this.offset = -1; - } - - public ControllerResultAndOffset(long offset, - List records, - T response) { - super(records, response); + private ControllerResultAndOffset(long offset, ControllerResult result) { + super(result.records(), result.response(), result.isAtomic()); this.offset = offset; } @@ -52,18 +43,27 @@ public boolean equals(Object o) { ControllerResultAndOffset other = (ControllerResultAndOffset) o; return records().equals(other.records()) && response().equals(other.response()) && + isAtomic() == other.isAtomic() && offset == other.offset; } @Override public int hashCode() { - return Objects.hash(records(), response(), offset); + return Objects.hash(records(), response(), isAtomic(), offset); } @Override public String toString() { - return "ControllerResultAndOffset(records=" + String.join(",", - records().stream().map(r -> r.toString()).collect(Collectors.toList())) + - ", response=" + response() + ", offset=" + offset + ")"; + return String.format( + "ControllerResultAndOffset(records=%s, response=%s, isAtomic=%s, offset=%s)", + String.join(",", records().stream().map(ApiMessageAndVersion::toString).collect(Collectors.toList())), + response(), + isAtomic(), + offset + ); + } + + public static ControllerResultAndOffset of(long offset, ControllerResult result) { + return new ControllerResultAndOffset<>(offset, result); } } diff --git a/metadata/src/main/java/org/apache/kafka/controller/FeatureControlManager.java b/metadata/src/main/java/org/apache/kafka/controller/FeatureControlManager.java index 25ff3fdcdd80e..99874ac3c5ef7 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/FeatureControlManager.java +++ b/metadata/src/main/java/org/apache/kafka/controller/FeatureControlManager.java @@ -69,7 +69,8 @@ ControllerResult> updateFeatures( results.put(entry.getKey(), updateFeature(entry.getKey(), entry.getValue(), downgradeables.contains(entry.getKey()), brokerFeatures, records)); } - return new ControllerResult<>(records, results); + + return ControllerResult.atomicOf(records, results); } private ApiError updateFeature(String featureName, diff --git a/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java b/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java index 198097538985a..759db1efac790 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java +++ b/metadata/src/main/java/org/apache/kafka/controller/QuorumController.java @@ -17,7 +17,6 @@ package org.apache.kafka.controller; -import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -265,7 +264,7 @@ private Throwable handleEventException(String name, class ControlEvent implements EventQueue.Event { private final String name; private final Runnable handler; - private long eventCreatedTimeNs = time.nanoseconds(); + private final long eventCreatedTimeNs = time.nanoseconds(); private Optional startProcessingTimeNs = Optional.empty(); ControlEvent(String name, Runnable handler) { @@ -307,7 +306,7 @@ class ControllerReadEvent implements EventQueue.Event { private final String name; private final CompletableFuture future; private final Supplier handler; - private long eventCreatedTimeNs = time.nanoseconds(); + private final long eventCreatedTimeNs = time.nanoseconds(); private Optional startProcessingTimeNs = Optional.empty(); ControllerReadEvent(String name, Supplier handler) { @@ -389,7 +388,7 @@ class ControllerWriteEvent implements EventQueue.Event, DeferredEvent { private final String name; private final CompletableFuture future; private final ControllerWriteOperation op; - private long eventCreatedTimeNs = time.nanoseconds(); + private final long eventCreatedTimeNs = time.nanoseconds(); private Optional startProcessingTimeNs = Optional.empty(); private ControllerResultAndOffset resultAndOffset; @@ -423,8 +422,7 @@ public void run() throws Exception { if (!maybeOffset.isPresent()) { // If the purgatory is empty, there are no pending operations and no // uncommitted state. We can return immediately. - resultAndOffset = new ControllerResultAndOffset<>(-1, - new ArrayList<>(), result.response()); + resultAndOffset = ControllerResultAndOffset.of(-1, result); log.debug("Completing read-only operation {} immediately because " + "the purgatory is empty.", this); complete(null); @@ -432,8 +430,7 @@ public void run() throws Exception { } // If there are operations in the purgatory, we want to wait for the latest // one to complete before returning our result to the user. - resultAndOffset = new ControllerResultAndOffset<>(maybeOffset.get(), - result.records(), result.response()); + resultAndOffset = ControllerResultAndOffset.of(maybeOffset.get(), result); log.debug("Read-only operation {} will be completed when the log " + "reaches offset {}", this, resultAndOffset.offset()); } else { @@ -441,11 +438,15 @@ public void run() throws Exception { // written before we can return our result to the user. Here, we hand off // the batch of records to the metadata log manager. They will be written // out asynchronously. - long offset = logManager.scheduleWrite(controllerEpoch, result.records()); + final long offset; + if (result.isAtomic()) { + offset = logManager.scheduleAtomicWrite(controllerEpoch, result.records()); + } else { + offset = logManager.scheduleWrite(controllerEpoch, result.records()); + } op.processBatchEndOffset(offset); writeOffset = offset; - resultAndOffset = new ControllerResultAndOffset<>(offset, - result.records(), result.response()); + resultAndOffset = ControllerResultAndOffset.of(offset, result); for (ApiMessageAndVersion message : result.records()) { replay(message.message()); } diff --git a/metadata/src/main/java/org/apache/kafka/controller/ReplicationControlManager.java b/metadata/src/main/java/org/apache/kafka/controller/ReplicationControlManager.java index 9fd172f486ae6..ca571058265b7 100644 --- a/metadata/src/main/java/org/apache/kafka/controller/ReplicationControlManager.java +++ b/metadata/src/main/java/org/apache/kafka/controller/ReplicationControlManager.java @@ -407,7 +407,7 @@ public void replay(PartitionChangeRecord record) { resultsPrefix = ", "; } log.info("createTopics result(s): {}", resultsBuilder.toString()); - return new ControllerResult<>(records, data); + return ControllerResult.atomicOf(records, data); } private ApiError createTopic(CreatableTopic topic, @@ -626,7 +626,7 @@ ControllerResult alterIsr(AlterIsrRequestData request) { setIsr(partitionData.newIsr())); } } - return new ControllerResult<>(records, response); + return ControllerResult.of(records, response); } /** @@ -780,7 +780,7 @@ ControllerResult electLeaders(ElectLeadersRequestData setErrorMessage(error.message())); } } - return new ControllerResult<>(records, response); + return ControllerResult.of(records, response); } static boolean electionIsUnclean(byte electionType) { @@ -875,7 +875,7 @@ ControllerResult processBrokerHeartbeat( states.next().fenced(), states.next().inControlledShutdown(), states.next().shouldShutDown()); - return new ControllerResult<>(records, reply); + return ControllerResult.of(records, reply); } int bestLeader(int[] replicas, int[] isr, boolean unclean) { @@ -904,7 +904,7 @@ public ControllerResult unregisterBroker(int brokerId) { } List records = new ArrayList<>(); handleBrokerUnregistered(brokerId, registration.epoch(), records); - return new ControllerResult<>(records, null); + return ControllerResult.of(records, null); } ControllerResult maybeFenceStaleBrokers() { @@ -916,6 +916,6 @@ ControllerResult maybeFenceStaleBrokers() { handleBrokerFenced(brokerId, records); heartbeatManager.fence(brokerId); } - return new ControllerResult<>(records, null); + return ControllerResult.of(records, null); } } diff --git a/metadata/src/main/java/org/apache/kafka/metalog/LocalLogManager.java b/metadata/src/main/java/org/apache/kafka/metalog/LocalLogManager.java index ef85314e0ef2f..99ae3a7e9baa8 100644 --- a/metadata/src/main/java/org/apache/kafka/metalog/LocalLogManager.java +++ b/metadata/src/main/java/org/apache/kafka/metalog/LocalLogManager.java @@ -328,8 +328,21 @@ public void register(MetaLogListener listener) throws Exception { @Override public long scheduleWrite(long epoch, List batch) { - return shared.tryAppend(nodeId, leader.epoch(), new LocalRecordBatch( - batch.stream().map(r -> r.message()).collect(Collectors.toList()))); + return scheduleAtomicWrite(epoch, batch); + } + + @Override + public long scheduleAtomicWrite(long epoch, List batch) { + return shared.tryAppend( + nodeId, + leader.epoch(), + new LocalRecordBatch( + batch + .stream() + .map(ApiMessageAndVersion::message) + .collect(Collectors.toList()) + ) + ); } @Override diff --git a/metadata/src/main/java/org/apache/kafka/metalog/MetaLogManager.java b/metadata/src/main/java/org/apache/kafka/metalog/MetaLogManager.java index 67a6ca5385f75..9126245ef3855 100644 --- a/metadata/src/main/java/org/apache/kafka/metalog/MetaLogManager.java +++ b/metadata/src/main/java/org/apache/kafka/metalog/MetaLogManager.java @@ -50,13 +50,30 @@ public interface MetaLogManager { * offset before renouncing its leadership. The listener should determine this by * monitoring the committed offsets. * - * @param epoch The controller epoch. - * @param batch The batch of messages to write. + * @param epoch the controller epoch + * @param batch the batch of messages to write * - * @return The offset of the message. + * @return the offset of the last message in the batch + * @throws IllegalArgumentException if buffer allocatio failed and the client should backoff */ long scheduleWrite(long epoch, List batch); + /** + * Schedule a atomic write to the log. + * + * The write will be scheduled to happen at some time in the future. All of the messages in batch + * will be appended atomically in one batch. The listener may regard the write as successful + * if and only if the MetaLogManager reaches the given offset before renouncing its leadership. + * The listener should determine this by monitoring the committed offsets. + * + * @param epoch the controller epoch + * @param batch the batch of messages to write + * + * @return the offset of the last message in the batch + * @throws IllegalArgumentException if buffer allocatio failed and the client should backoff + */ + long scheduleAtomicWrite(long epoch, List batch); + /** * Renounce the leadership. * diff --git a/metadata/src/test/java/org/apache/kafka/controller/ConfigurationControlManagerTest.java b/metadata/src/test/java/org/apache/kafka/controller/ConfigurationControlManagerTest.java index 49a55338309b5..561a25b2d0ee2 100644 --- a/metadata/src/test/java/org/apache/kafka/controller/ConfigurationControlManagerTest.java +++ b/metadata/src/test/java/org/apache/kafka/controller/ConfigurationControlManagerTest.java @@ -135,18 +135,42 @@ public void testIncrementalAlterConfigs() { SnapshotRegistry snapshotRegistry = new SnapshotRegistry(new LogContext()); ConfigurationControlManager manager = new ConfigurationControlManager(new LogContext(), snapshotRegistry, CONFIGS); - assertEquals(new ControllerResult>(Collections.singletonList( - new ApiMessageAndVersion(new ConfigRecord(). - setResourceType(TOPIC.id()).setResourceName("mytopic"). - setName("abc").setValue("123"), (short) 0)), - toMap(entry(BROKER0, new ApiError( - Errors.INVALID_REQUEST, "A DELETE op was given with a non-null value.")), - entry(MYTOPIC, ApiError.NONE))), - manager.incrementalAlterConfigs(toMap(entry(BROKER0, toMap( - entry("foo.bar", entry(DELETE, "abc")), - entry("quux", entry(SET, "abc")))), - entry(MYTOPIC, toMap( - entry("abc", entry(APPEND, "123"))))))); + assertEquals( + ControllerResult.atomicOf( + Collections.singletonList( + new ApiMessageAndVersion( + new ConfigRecord() + .setResourceType(TOPIC.id()) + .setResourceName("mytopic") + .setName("abc") + .setValue("123"), + (short) 0 + ) + ), + toMap( + entry( + BROKER0, + new ApiError( + Errors.INVALID_REQUEST, + "A DELETE op was given with a non-null value." + ) + ), + entry(MYTOPIC, ApiError.NONE) + ) + ), + manager.incrementalAlterConfigs( + toMap( + entry( + BROKER0, + toMap( + entry("foo.bar", entry(DELETE, "abc")), + entry("quux", entry(SET, "abc")) + ) + ), + entry(MYTOPIC, toMap(entry("abc", entry(APPEND, "123")))) + ) + ) + ); } @Test @@ -184,20 +208,33 @@ public void testLegacyAlterConfigs() { new ApiMessageAndVersion(new ConfigRecord(). setResourceType(TOPIC.id()).setResourceName("mytopic"). setName("def").setValue("901"), (short) 0)); - assertEquals(new ControllerResult>( + assertEquals( + ControllerResult.atomicOf( expectedRecords1, - toMap(entry(MYTOPIC, ApiError.NONE))), - manager.legacyAlterConfigs(toMap(entry(MYTOPIC, toMap( - entry("abc", "456"), entry("def", "901")))))); + toMap(entry(MYTOPIC, ApiError.NONE)) + ), + manager.legacyAlterConfigs( + toMap(entry(MYTOPIC, toMap(entry("abc", "456"), entry("def", "901")))) + ) + ); for (ApiMessageAndVersion message : expectedRecords1) { manager.replay((ConfigRecord) message.message()); } - assertEquals(new ControllerResult>(Arrays.asList( - new ApiMessageAndVersion(new ConfigRecord(). - setResourceType(TOPIC.id()).setResourceName("mytopic"). - setName("abc").setValue(null), (short) 0)), - toMap(entry(MYTOPIC, ApiError.NONE))), - manager.legacyAlterConfigs(toMap(entry(MYTOPIC, toMap( - entry("def", "901")))))); + assertEquals( + ControllerResult.atomicOf( + Arrays.asList( + new ApiMessageAndVersion( + new ConfigRecord() + .setResourceType(TOPIC.id()) + .setResourceName("mytopic") + .setName("abc") + .setValue(null), + (short) 0 + ) + ), + toMap(entry(MYTOPIC, ApiError.NONE)) + ), + manager.legacyAlterConfigs(toMap(entry(MYTOPIC, toMap(entry("def", "901"))))) + ); } } diff --git a/metadata/src/test/java/org/apache/kafka/controller/FeatureControlManagerTest.java b/metadata/src/test/java/org/apache/kafka/controller/FeatureControlManagerTest.java index 8687cc8f562d8..0670984e52876 100644 --- a/metadata/src/test/java/org/apache/kafka/controller/FeatureControlManagerTest.java +++ b/metadata/src/test/java/org/apache/kafka/controller/FeatureControlManagerTest.java @@ -18,10 +18,8 @@ package org.apache.kafka.controller; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import org.apache.kafka.common.metadata.FeatureLevelRecord; @@ -61,11 +59,11 @@ public void testUpdateFeatures() { rangeMap("foo", 1, 2), snapshotRegistry); assertEquals(new FeatureMapAndEpoch(new FeatureMap(Collections.emptyMap()), -1), manager.finalizedFeatures(-1)); - assertEquals(new ControllerResult<>(Collections. + assertEquals(ControllerResult.atomicOf(Collections.emptyList(), Collections. singletonMap("foo", new ApiError(Errors.INVALID_UPDATE_VERSION, "The controller does not support the given feature range."))), manager.updateFeatures(rangeMap("foo", 1, 3), - new HashSet<>(Arrays.asList("foo")), + Collections.singleton("foo"), Collections.emptyMap())); ControllerResult> result = manager.updateFeatures( rangeMap("foo", 1, 2, "bar", 1, 1), Collections.emptySet(), @@ -101,12 +99,24 @@ public void testUpdateFeaturesErrorCases() { SnapshotRegistry snapshotRegistry = new SnapshotRegistry(new LogContext()); FeatureControlManager manager = new FeatureControlManager( rangeMap("foo", 1, 5, "bar", 1, 2), snapshotRegistry); - assertEquals(new ControllerResult<>(Collections. - singletonMap("foo", new ApiError(Errors.INVALID_UPDATE_VERSION, - "Broker 5 does not support the given feature range."))), - manager.updateFeatures(rangeMap("foo", 1, 3), - new HashSet<>(Arrays.asList("foo")), - Collections.singletonMap(5, rangeMap()))); + + assertEquals( + ControllerResult.atomicOf( + Collections.emptyList(), + Collections.singletonMap( + "foo", + new ApiError( + Errors.INVALID_UPDATE_VERSION, + "Broker 5 does not support the given feature range." + ) + ) + ), + manager.updateFeatures( + rangeMap("foo", 1, 3), + Collections.singleton("foo"), + Collections.singletonMap(5, rangeMap()) + ) + ); ControllerResult> result = manager.updateFeatures( rangeMap("foo", 1, 3), Collections.emptySet(), Collections.emptyMap()); @@ -114,19 +124,31 @@ public void testUpdateFeaturesErrorCases() { manager.replay((FeatureLevelRecord) result.records().get(0).message(), 3); snapshotRegistry.createSnapshot(3); - assertEquals(new ControllerResult<>(Collections. + assertEquals(ControllerResult.atomicOf(Collections.emptyList(), Collections. singletonMap("foo", new ApiError(Errors.INVALID_UPDATE_VERSION, "Can't downgrade the maximum version of this feature without " + "setting downgradable to true."))), manager.updateFeatures(rangeMap("foo", 1, 2), Collections.emptySet(), Collections.emptyMap())); - assertEquals(new ControllerResult<>( - Collections.singletonList(new ApiMessageAndVersion(new FeatureLevelRecord(). - setName("foo").setMinFeatureLevel((short) 1).setMaxFeatureLevel((short) 2), - (short) 0)), - Collections.singletonMap("foo", ApiError.NONE)), - manager.updateFeatures(rangeMap("foo", 1, 2), - new HashSet<>(Collections.singletonList("foo")), Collections.emptyMap())); + assertEquals( + ControllerResult.atomicOf( + Collections.singletonList( + new ApiMessageAndVersion( + new FeatureLevelRecord() + .setName("foo") + .setMinFeatureLevel((short) 1) + .setMaxFeatureLevel((short) 2), + (short) 0 + ) + ), + Collections.singletonMap("foo", ApiError.NONE) + ), + manager.updateFeatures( + rangeMap("foo", 1, 2), + Collections.singleton("foo"), + Collections.emptyMap() + ) + ); } } diff --git a/metadata/src/test/java/org/apache/kafka/metalog/LocalLogManager.java b/metadata/src/test/java/org/apache/kafka/metalog/LocalLogManager.java index 7b6cf06212e8e..590f89c3391b3 100644 --- a/metadata/src/test/java/org/apache/kafka/metalog/LocalLogManager.java +++ b/metadata/src/test/java/org/apache/kafka/metalog/LocalLogManager.java @@ -371,8 +371,21 @@ public void register(MetaLogListener listener) throws Exception { @Override public long scheduleWrite(long epoch, List batch) { - return shared.tryAppend(nodeId, leader.epoch(), new LocalRecordBatch( - batch.stream().map(r -> r.message()).collect(Collectors.toList()))); + return scheduleAtomicWrite(epoch, batch); + } + + @Override + public long scheduleAtomicWrite(long epoch, List batch) { + return shared.tryAppend( + nodeId, + leader.epoch(), + new LocalRecordBatch( + batch + .stream() + .map(ApiMessageAndVersion::message) + .collect(Collectors.toList()) + ) + ); } @Override diff --git a/raft/src/main/java/org/apache/kafka/raft/metadata/MetaLogRaftShim.java b/raft/src/main/java/org/apache/kafka/raft/metadata/MetaLogRaftShim.java index bf88e7d8120a1..1ca63f1b9c3cd 100644 --- a/raft/src/main/java/org/apache/kafka/raft/metadata/MetaLogRaftShim.java +++ b/raft/src/main/java/org/apache/kafka/raft/metadata/MetaLogRaftShim.java @@ -52,9 +52,35 @@ public void register(MetaLogListener listener) { client.register(new ListenerShim(listener)); } + @Override + public long scheduleAtomicWrite(long epoch, List batch) { + return write(epoch, batch, true); + } + @Override public long scheduleWrite(long epoch, List batch) { - return client.scheduleAppend((int) epoch, batch); + return write(epoch, batch, false); + } + + private long write(long epoch, List batch, boolean isAtomic) { + final Long result; + if (isAtomic) { + result = client.scheduleAtomicAppend((int) epoch, batch); + } else { + result = client.scheduleAppend((int) epoch, batch); + } + + if (result == null) { + throw new IllegalArgumentException( + String.format( + "Unable to alloate a buffer for the schedule write operation: epoch %s, batch %s)", + epoch, + batch + ) + ); + } else { + return result; + } } @Override From 7a3ebbebbc6aed2049f22c650a52f6f685546207 Mon Sep 17 00:00:00 2001 From: Ismael Juma Date: Thu, 4 Mar 2021 11:22:22 -0800 Subject: [PATCH 13/59] KAFKA-12415 Prepare for Gradle 7.0 and restrict transitive scope for non api dependencies (#10203) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Gradle 7.0 is required for Java 16 compatibility and it removes a number of deprecated APIs. Fix most issues preventing the upgrade to Gradle 7.0. The remaining ones are more complicated and should be handled in a separate PR. Details of the changes: * Release tarball no longer includes includes test, sources, javadoc and test sources jars (these are still published to the Maven Central repository). * Replace `compile` with `api` or `implementation` - note that `implementation` dependencies appear with `runtime` scope in the pom file so this is a (positive) change in behavior * Add missing dependencies that were uncovered by the usage of `implementation` * Replace `testCompile` with `testImplementation` * Replace `runtime` with `runtimeOnly` and `testRuntime` with `testRuntimeOnly` * Replace `configurations.runtime` with `configurations.runtimeClasspath` * Replace `configurations.testRuntime` with `configurations.testRuntimeClasspath` (except for the usage in the `streams` project as that causes a cyclic dependency error) * Use `java-library` plugin instead of `java` * Use `maven-publish` plugin instead of deprecated `maven` plugin - this changes the commands used to publish and to install locally, but task aliases for `install` and `uploadArchives` were added for backwards compatibility * Removed `-x signArchives` line from the readme since it was wrong (it was a no-op before and it fails now, however) * Replaces `artifacts` block with an approach that works with the `maven-publish` plugin * Don't publish `jmh-benchmark` module - the shadow jar is pretty large and not particularly useful (before this PR, we would publish the non shadow jars) * Replace `version` with `archiveVersion`, `baseName` with `archiveBaseName` and `classifier` with `archiveClassifier` * Update Gradle and plugins to the latest stable version (7.0 is not stable yet) * Use `plugin` DSL to configure plugins * Updated notable changes for 3.0 Reviewers: Chia-Ping Tsai , Randall Hauch --- Jenkinsfile | 4 +- README.md | 16 +- build.gradle | 836 ++++++++++++----------- docs/upgrade.html | 8 + gradle/dependencies.gradle | 9 +- gradle/wrapper/gradle-wrapper.properties | 2 +- gradlew | 2 +- jmh-benchmarks/README.md | 2 +- jmh-benchmarks/jmh.sh | 2 +- release.py | 2 +- 10 files changed, 460 insertions(+), 423 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 557988788cd4d..bc63364fc6a79 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -46,8 +46,8 @@ def doStreamsArchetype() { echo 'Verify that Kafka Streams archetype compiles' sh ''' - ./gradlew streams:install clients:install connect:json:install connect:api:install \ - || { echo 'Could not install kafka-streams.jar (and dependencies) locally`'; exit 1; } + ./gradlew streams:publishToMavenLocal clients:publishToMavenLocal connect:json:publishToMavenLocal connect:api:publishToMavenLocal \ + || { echo 'Could not publish kafka-streams.jar (and dependencies) locally to Maven'; exit 1; } ''' VERSION = sh(script: 'grep "^version=" gradle.properties | cut -d= -f 2', returnStdout: true).trim() diff --git a/README.md b/README.md index 3c3f013662344..649127ed4aa75 100644 --- a/README.md +++ b/README.md @@ -69,10 +69,6 @@ Generate coverage for a single module, i.e.: ### Building a binary release gzipped tar ball ### ./gradlew clean releaseTarGz -The above command will fail if you haven't set up the signing key. To bypass signing the artifact, you can run: - - ./gradlew clean releaseTarGz -x signArchives - The release file can be found inside `./core/build/distributions/`. ### Building auto generated messages ### @@ -125,6 +121,12 @@ build directory (`${project_dir}/bin`) clashes with Kafka's scripts directory an to avoid known issues with this configuration. ### Publishing the jar for all version of Scala and for all projects to maven ### +The recommended command is: + + ./gradlewAll publish + +For backwards compatibility, the following also works: + ./gradlewAll uploadArchives Please note for this to work you should create/update `${GRADLE_USER_HOME}/gradle.properties` (typically, `~/.gradle/gradle.properties`) and assign the following variables @@ -167,6 +169,12 @@ Please note for this to work you should create/update user maven settings (typic ### Installing the jars to the local Maven repository ### +The recommended command is: + + ./gradlewAll publishToMavenLocal + +For backwards compatibility, the following also works: + ./gradlewAll install ### Building the test jar ### diff --git a/build.gradle b/build.gradle index acd8a84e6377b..be6859bfddf80 100644 --- a/build.gradle +++ b/build.gradle @@ -21,9 +21,6 @@ buildscript { repositories { mavenCentral() jcenter() - maven { - url "https://plugins.gradle.org/m2/" - } } apply from: file('gradle/buildscript.gradle'), to: buildscript apply from: "$rootDir/gradle/dependencies.gradle" @@ -31,17 +28,21 @@ buildscript { dependencies { // For Apache Rat plugin to ignore non-Git files classpath "org.ajoberstar.grgit:grgit-core:$versions.grgit" - classpath "com.github.ben-manes:gradle-versions-plugin:$versions.gradleVersionsPlugin" - classpath "org.scoverage:gradle-scoverage:$versions.scoveragePlugin" - classpath "com.github.jengelman.gradle.plugins:shadow:$versions.shadowPlugin" - classpath "org.owasp:dependency-check-gradle:$versions.owaspDepCheckPlugin" - classpath "com.diffplug.spotless:spotless-plugin-gradle:$versions.spotlessPlugin" - classpath "gradle.plugin.com.github.spotbugs.snom:spotbugs-gradle-plugin:$versions.spotbugsPlugin" - classpath "org.gradle:test-retry-gradle-plugin:$versions.testRetryPlugin" } } -apply plugin: "com.diffplug.spotless" +plugins { + id 'com.diffplug.spotless' version '5.10.2' + id 'com.github.ben-manes.versions' version '0.36.0' + id 'idea' + id 'org.owasp.dependencycheck' version '6.1.1' + + id "com.github.spotbugs" version '4.6.0' apply false + id 'org.gradle.test-retry' version '1.2.0' apply false + id 'org.scoverage' version '5.0.0' apply false + id 'com.github.johnrengelman.shadow' version '6.1.0' apply false +} + spotless { scala { target 'streams/**/*.scala' @@ -49,17 +50,12 @@ spotless { } } - allprojects { repositories { mavenCentral() } - apply plugin: 'idea' - apply plugin: 'org.owasp.dependencycheck' - apply plugin: 'com.github.ben-manes.versions' - dependencyUpdates { revision="release" resolutionStrategy { @@ -119,7 +115,7 @@ ext { userMaxTestRetryFailures = project.hasProperty('maxTestRetryFailures') ? maxTestRetryFailures.toInteger() : 0 skipSigning = project.hasProperty('skipSigning') && skipSigning.toBoolean() - shouldSign = !skipSigning && !version.endsWith("SNAPSHOT") && project.gradle.startParameter.taskNames.any { it.contains("upload") } + shouldSign = !skipSigning && !version.endsWith("SNAPSHOT") mavenUrl = project.hasProperty('mavenUrl') ? project.mavenUrl : '' mavenUsername = project.hasProperty('mavenUsername') ? project.mavenUsername : '' @@ -182,17 +178,30 @@ subprojects { // eg: ./gradlew allDepInsight --configuration runtime --dependency com.fasterxml.jackson.core:jackson-databind task allDepInsight(type: DependencyInsightReportTask) doLast {} - apply plugin: 'java' + apply plugin: 'java-library' + apply plugin: 'checkstyle' + apply plugin: "com.github.spotbugs" + apply plugin: 'org.gradle.test-retry' + + // We use the shadow plugin for the jmh-benchmarks module and the `-all` jar can get pretty large, so + // don't publish it + def shouldPublish = !project.name.equals('jmh-benchmarks') + + if (shouldPublish) { + apply plugin: 'maven-publish' + apply plugin: 'signing' + + // Add aliases for the task names used by the maven plugin for backwards compatibility + // The maven plugin was replaced by the maven-publish plugin in Gradle 7.0 + tasks.register('install').configure { dependsOn(publishToMavenLocal) } + tasks.register('uploadArchives').configure { dependsOn(publish) } + } + // apply the eclipse plugin only to subprojects that hold code. 'connect' is just a folder. if (!project.name.equals('connect')) { apply plugin: 'eclipse' fineTuneEclipseClasspathFile(eclipse, project) } - apply plugin: 'maven' - apply plugin: 'signing' - apply plugin: 'checkstyle' - apply plugin: "com.github.spotbugs" - apply plugin: 'org.gradle.test-retry' sourceCompatibility = minJavaVersion targetCompatibility = minJavaVersion @@ -219,34 +228,50 @@ subprojects { options.compilerArgs << "--release" << minJavaVersion } - uploadArchives { - repositories { - signing { - required { shouldSign } - sign configurations.archives - - // To test locally, replace mavenUrl in ~/.gradle/gradle.properties to file://localhost/tmp/myRepo/ - mavenDeployer { - beforeDeployment { MavenDeployment deployment -> signing.signPom(deployment) } - repository(url: "${mavenUrl}") { - authentication(userName: "${mavenUsername}", password: "${mavenPassword}") - } - afterEvaluate { - pom.artifactId = "${archivesBaseName}" - pom.project { - name 'Apache Kafka' - packaging 'jar' - url 'https://kafka.apache.org' - licenses { - license { - name 'The Apache Software License, Version 2.0' - url 'https://www.apache.org/licenses/LICENSE-2.0.txt' - distribution 'repo' - } - } - } + if (shouldPublish) { + + publishing { + repositories { + // To test locally, invoke gradlew with `-PmavenUrl=file:///some/local/path` + maven { + url = mavenUrl + credentials { + username = mavenUsername + password = mavenPassword + } + } + } + publications { + mavenJava(MavenPublication) { + from components.java + + afterEvaluate { + ["srcJar", "javadocJar", "scaladocJar", "testJar", "testSrcJar"].forEach { taskName -> + def task = tasks.findByName(taskName) + if (task != null) + artifact task + } + + artifactId = archivesBaseName + pom { + name = 'Apache Kafka' + url = 'https://kafka.apache.org' + licenses { + license { + name = 'The Apache License, Version 2.0' + url = 'http://www.apache.org/licenses/LICENSE-2.0.txt' + distribution = 'repo' + } } + } } + } + } + } + + if (shouldSign) { + signing { + sign publishing.publications.mavenJava } } } @@ -427,14 +452,14 @@ subprojects { } task srcJar(type: Jar) { - classifier = 'sources' + archiveClassifier = 'sources' from "$rootDir/LICENSE" from "$rootDir/NOTICE" from sourceSets.main.allSource } task javadocJar(type: Jar, dependsOn: javadoc) { - classifier 'javadoc' + archiveClassifier = 'javadoc' from "$rootDir/LICENSE" from "$rootDir/NOTICE" from javadoc.destinationDir @@ -450,30 +475,21 @@ subprojects { task systemTestLibs(dependsOn: jar) - artifacts { - archives srcJar - archives javadocJar - } - - if(!sourceSets.test.allSource.isEmpty()) { + if (!sourceSets.test.allSource.isEmpty()) { task testJar(type: Jar) { - classifier = 'test' + archiveClassifier = 'test' from "$rootDir/LICENSE" from "$rootDir/NOTICE" from sourceSets.test.output } task testSrcJar(type: Jar, dependsOn: testJar) { - classifier = 'test-sources' + archiveClassifier = 'test-sources' from "$rootDir/LICENSE" from "$rootDir/NOTICE" from sourceSets.test.allSource } - artifacts { - archives testJar - archives testSrcJar - } } plugins.withType(ScalaPlugin) { @@ -483,7 +499,7 @@ subprojects { } task scaladocJar(type:Jar, dependsOn: scaladoc) { - classifier = 'scaladoc' + archiveClassifier = 'scaladoc' from "$rootDir/LICENSE" from "$rootDir/NOTICE" from scaladoc.destinationDir @@ -749,53 +765,57 @@ project(':core') { archivesBaseName = "kafka_${versions.baseScala}" dependencies { - compile project(':clients') - compile project(':metadata') - compile project(':raft') - compile libs.argparse4j - compile libs.jacksonDatabind - compile libs.jacksonModuleScala - compile libs.jacksonDataformatCsv - compile libs.jacksonJDK8Datatypes - compile libs.joptSimple - compile libs.metrics - compile libs.scalaCollectionCompat - compile libs.scalaJava8Compat - compile libs.scalaLibrary + // `core` is often used in users' tests, define the following dependencies as `api` for backwards compatibility + // even though the `core` module doesn't expose any public API + api project(':clients') + api libs.scalaLibrary + + implementation project(':metadata') + implementation project(':raft') + + implementation libs.argparse4j + implementation libs.jacksonDatabind + implementation libs.jacksonModuleScala + implementation libs.jacksonDataformatCsv + implementation libs.jacksonJDK8Datatypes + implementation libs.joptSimple + implementation libs.metrics + implementation libs.scalaCollectionCompat + implementation libs.scalaJava8Compat // only needed transitively, but set it explicitly to ensure it has the same version as scala-library - compile libs.scalaReflect - compile libs.scalaLogging - compile libs.slf4jApi - compile(libs.zookeeper) { + implementation libs.scalaReflect + implementation libs.scalaLogging + implementation libs.slf4jApi + implementation(libs.zookeeper) { exclude module: 'slf4j-log4j12' exclude module: 'log4j' } // ZooKeeperMain depends on commons-cli but declares the dependency as `provided` - compile libs.commonsCli + implementation libs.commonsCli compileOnly libs.log4j - testCompile project(':clients').sourceSets.test.output - testCompile libs.bcpkix - testCompile libs.mockitoCore - testCompile libs.easymock - testCompile(libs.apacheda) { + testImplementation project(':clients').sourceSets.test.output + testImplementation libs.bcpkix + testImplementation libs.mockitoCore + testImplementation libs.easymock + testImplementation(libs.apacheda) { exclude group: 'xml-apis', module: 'xml-apis' // `mina-core` is a transitive dependency for `apacheds` and `apacheda`. // It is safer to use from `apacheds` since that is the implementation. exclude module: 'mina-core' } - testCompile libs.apachedsCoreApi - testCompile libs.apachedsInterceptorKerberos - testCompile libs.apachedsProtocolShared - testCompile libs.apachedsProtocolKerberos - testCompile libs.apachedsProtocolLdap - testCompile libs.apachedsLdifPartition - testCompile libs.apachedsMavibotPartition - testCompile libs.apachedsJdbmPartition - testCompile libs.junitJupiter - testCompile libs.slf4jlog4j - testCompile(libs.jfreechart) { + testImplementation libs.apachedsCoreApi + testImplementation libs.apachedsInterceptorKerberos + testImplementation libs.apachedsProtocolShared + testImplementation libs.apachedsProtocolKerberos + testImplementation libs.apachedsProtocolLdap + testImplementation libs.apachedsLdifPartition + testImplementation libs.apachedsMavibotPartition + testImplementation libs.apachedsJdbmPartition + testImplementation libs.junitJupiter + testImplementation libs.slf4jlog4j + testImplementation(libs.jfreechart) { exclude group: 'junit', module: 'junit' } } @@ -823,11 +843,11 @@ project(':core') { } tasks.create(name: "copyDependantLibs", type: Copy) { - from (configurations.testRuntime) { + from (configurations.testRuntimeClasspath) { include('slf4j-log4j12*') include('log4j*jar') } - from (configurations.runtime) { + from (configurations.runtimeClasspath) { exclude('kafka-clients*') } into "$buildDir/dependant-libs-${versions.scala}" @@ -933,7 +953,7 @@ project(':core') { ':connect:runtime:genSinkConnectorConfigDocs', ':connect:runtime:genSourceConnectorConfigDocs', ':streams:genStreamsConfigDocs', 'genConsumerMetricsDocs', 'genProducerMetricsDocs', ':connect:runtime:genConnectMetricsDocs'], type: Tar) { - classifier = 'site-docs' + archiveClassifier = 'site-docs' compression = Compression.GZIP from project.file("$rootDir/docs") into 'site-docs' @@ -941,41 +961,41 @@ project(':core') { } tasks.create(name: "releaseTarGz", dependsOn: configurations.archives.artifacts, type: Tar) { - into "kafka_${versions.baseScala}-${version}" + into "kafka_${versions.baseScala}-${archiveVersion.get()}" compression = Compression.GZIP from(project.file("$rootDir/bin")) { into "bin/" } from(project.file("$rootDir/config")) { into "config/" } from "$rootDir/LICENSE" from "$rootDir/NOTICE" - from(configurations.runtime) { into("libs/") } + from(configurations.runtimeClasspath) { into("libs/") } from(configurations.archives.artifacts.files) { into("libs/") } from(project.siteDocsTar) { into("site-docs/") } from(project(':tools').jar) { into("libs/") } - from(project(':tools').configurations.runtime) { into("libs/") } + from(project(':tools').configurations.runtimeClasspath) { into("libs/") } from(project(':connect:api').jar) { into("libs/") } - from(project(':connect:api').configurations.runtime) { into("libs/") } + from(project(':connect:api').configurations.runtimeClasspath) { into("libs/") } from(project(':connect:runtime').jar) { into("libs/") } - from(project(':connect:runtime').configurations.runtime) { into("libs/") } + from(project(':connect:runtime').configurations.runtimeClasspath) { into("libs/") } from(project(':connect:transforms').jar) { into("libs/") } - from(project(':connect:transforms').configurations.runtime) { into("libs/") } + from(project(':connect:transforms').configurations.runtimeClasspath) { into("libs/") } from(project(':connect:json').jar) { into("libs/") } - from(project(':connect:json').configurations.runtime) { into("libs/") } + from(project(':connect:json').configurations.runtimeClasspath) { into("libs/") } from(project(':connect:file').jar) { into("libs/") } - from(project(':connect:file').configurations.runtime) { into("libs/") } + from(project(':connect:file').configurations.runtimeClasspath) { into("libs/") } from(project(':connect:basic-auth-extension').jar) { into("libs/") } - from(project(':connect:basic-auth-extension').configurations.runtime) { into("libs/") } + from(project(':connect:basic-auth-extension').configurations.runtimeClasspath) { into("libs/") } from(project(':connect:mirror').jar) { into("libs/") } - from(project(':connect:mirror').configurations.runtime) { into("libs/") } + from(project(':connect:mirror').configurations.runtimeClasspath) { into("libs/") } from(project(':connect:mirror-client').jar) { into("libs/") } - from(project(':connect:mirror-client').configurations.runtime) { into("libs/") } + from(project(':connect:mirror-client').configurations.runtimeClasspath) { into("libs/") } from(project(':streams').jar) { into("libs/") } - from(project(':streams').configurations.runtime) { into("libs/") } + from(project(':streams').configurations.runtimeClasspath) { into("libs/") } from(project(':streams:streams-scala').jar) { into("libs/") } - from(project(':streams:streams-scala').configurations.runtime) { into("libs/") } + from(project(':streams:streams-scala').configurations.runtimeClasspath) { into("libs/") } from(project(':streams:test-utils').jar) { into("libs/") } - from(project(':streams:test-utils').configurations.runtime) { into("libs/") } + from(project(':streams:test-utils').configurations.runtimeClasspath) { into("libs/") } from(project(':streams:examples').jar) { into("libs/") } - from(project(':streams:examples').configurations.runtime) { into("libs/") } + from(project(':streams:examples').configurations.runtimeClasspath) { into("libs/") } duplicatesStrategy 'exclude' } @@ -990,7 +1010,7 @@ project(':core') { } tasks.create(name: "copyDependantTestLibs", type: Copy) { - from (configurations.testRuntime) { + from (configurations.testRuntimeClasspath) { include('*.jar') } into "$buildDir/dependant-testlibs" @@ -1027,15 +1047,15 @@ project(':metadata') { archivesBaseName = "kafka-metadata" dependencies { - compile project(':clients') - compile libs.jacksonDatabind - compile libs.jacksonJDK8Datatypes - compile libs.metrics + implementation project(':clients') + implementation libs.jacksonDatabind + implementation libs.jacksonJDK8Datatypes + implementation libs.metrics compileOnly libs.log4j - testCompile libs.junitJupiter - testCompile libs.hamcrest - testCompile libs.slf4jlog4j - testCompile project(':clients').sourceSets.test.output + testImplementation libs.junitJupiter + testImplementation libs.hamcrest + testImplementation libs.slf4jlog4j + testImplementation project(':clients').sourceSets.test.output } task processMessages(type:JavaExec) { @@ -1075,7 +1095,8 @@ project(':examples') { archivesBaseName = "kafka-examples" dependencies { - compile project(':core') + implementation project(':clients') + implementation project(':core') } javadoc { @@ -1089,11 +1110,11 @@ project(':examples') { project(':generator') { dependencies { - compile libs.argparse4j - compile libs.jacksonDatabind - compile libs.jacksonJDK8Datatypes - compile libs.jacksonJaxrsJsonProvider - testCompile libs.junitJupiter + implementation libs.argparse4j + implementation libs.jacksonDatabind + implementation libs.jacksonJDK8Datatypes + implementation libs.jacksonJaxrsJsonProvider + testImplementation libs.junitJupiter } javadoc { @@ -1104,32 +1125,23 @@ project(':generator') { project(':clients') { archivesBaseName = "kafka-clients" - configurations { - jacksonDatabindConfig - } - - // add jacksonDatabindConfig as provided scope config with high priority (1000) - conf2ScopeMappings.addMapping(1000, configurations.jacksonDatabindConfig, "provided") - dependencies { - compile libs.zstd - compile libs.lz4 - compile libs.snappy - compile libs.slf4jApi + implementation libs.zstd + implementation libs.lz4 + implementation libs.snappy + implementation libs.slf4jApi compileOnly libs.jacksonDatabind // for SASL/OAUTHBEARER bearer token parsing compileOnly libs.jacksonJDK8Datatypes - jacksonDatabindConfig libs.jacksonDatabind // to publish as provided scope dependency. - - testCompile libs.bcpkix - testCompile libs.junitJupiter - testCompile libs.mockitoCore + testImplementation libs.bcpkix + testImplementation libs.junitJupiter + testImplementation libs.mockitoCore - testRuntime libs.slf4jlog4j - testRuntime libs.jacksonDatabind - testRuntime libs.jacksonJDK8Datatypes - testCompile libs.jacksonJaxrsJsonProvider + testRuntimeOnly libs.slf4jlog4j + testRuntimeOnly libs.jacksonDatabind + testRuntimeOnly libs.jacksonJDK8Datatypes + testImplementation libs.jacksonJaxrsJsonProvider } task createVersionFile(dependsOn: determineCommitId) { @@ -1233,17 +1245,17 @@ project(':raft') { archivesBaseName = "kafka-raft" dependencies { - compile project(':clients') - compile project(':metadata') - compile libs.slf4jApi - compile libs.jacksonDatabind + implementation project(':clients') + implementation project(':metadata') + implementation libs.slf4jApi + implementation libs.jacksonDatabind - testCompile project(':clients') - testCompile project(':clients').sourceSets.test.output - testCompile libs.junitJupiter - testCompile libs.mockitoCore + testImplementation project(':clients') + testImplementation project(':clients').sourceSets.test.output + testImplementation libs.junitJupiter + testImplementation libs.mockitoCore - testRuntime libs.slf4jlog4j + testRuntimeOnly libs.slf4jlog4j } task createVersionFile(dependsOn: determineCommitId) { @@ -1308,28 +1320,29 @@ project(':tools') { archivesBaseName = "kafka-tools" dependencies { - compile project(':clients') - compile project(':log4j-appender') - compile libs.argparse4j - compile libs.jacksonDatabind - compile libs.jacksonJDK8Datatypes - compile libs.slf4jApi - - compile libs.jacksonJaxrsJsonProvider - compile libs.jerseyContainerServlet - compile libs.jerseyHk2 - compile libs.jaxbApi // Jersey dependency that was available in the JDK before Java 9 - compile libs.activation // Jersey dependency that was available in the JDK before Java 9 - compile libs.jettyServer - compile libs.jettyServlet - compile libs.jettyServlets - - testCompile project(':clients') - testCompile libs.junitJupiter - testCompile project(':clients').sourceSets.test.output - testCompile libs.mockitoInline // supports mocking static methods, final classes, etc. - - testRuntime libs.slf4jlog4j + implementation project(':clients') + implementation project(':log4j-appender') + implementation libs.argparse4j + implementation libs.jacksonDatabind + implementation libs.jacksonJDK8Datatypes + implementation libs.slf4jApi + implementation libs.log4j + + implementation libs.jacksonJaxrsJsonProvider + implementation libs.jerseyContainerServlet + implementation libs.jerseyHk2 + implementation libs.jaxbApi // Jersey dependency that was available in the JDK before Java 9 + implementation libs.activation // Jersey dependency that was available in the JDK before Java 9 + implementation libs.jettyServer + implementation libs.jettyServlet + implementation libs.jettyServlets + + testImplementation project(':clients') + testImplementation libs.junitJupiter + testImplementation project(':clients').sourceSets.test.output + testImplementation libs.mockitoInline // supports mocking static methods, final classes, etc. + + testRuntimeOnly libs.slf4jlog4j } javadoc { @@ -1337,11 +1350,11 @@ project(':tools') { } tasks.create(name: "copyDependantLibs", type: Copy) { - from (configurations.testRuntime) { + from (configurations.testRuntimeClasspath) { include('slf4j-log4j12*') include('log4j*jar') } - from (configurations.runtime) { + from (configurations.runtimeClasspath) { exclude('kafka-clients*') } into "$buildDir/dependant-libs-${versions.scala}" @@ -1357,23 +1370,23 @@ project(':shell') { archivesBaseName = "kafka-shell" dependencies { - compile libs.argparse4j - compile libs.jacksonDatabind - compile libs.jacksonJDK8Datatypes - compile libs.jline - compile libs.slf4jApi - compile project(':clients') - compile project(':core') - compile project(':log4j-appender') - compile project(':metadata') - compile project(':raft') + implementation libs.argparse4j + implementation libs.jacksonDatabind + implementation libs.jacksonJDK8Datatypes + implementation libs.jline + implementation libs.slf4jApi + implementation project(':clients') + implementation project(':core') + implementation project(':log4j-appender') + implementation project(':metadata') + implementation project(':raft') - compile libs.jacksonJaxrsJsonProvider + implementation libs.jacksonJaxrsJsonProvider - testCompile project(':clients') - testCompile libs.junitJupiter + testImplementation project(':clients') + testImplementation libs.junitJupiter - testRuntime libs.slf4jlog4j + testRuntimeOnly libs.slf4jlog4j } javadoc { @@ -1381,10 +1394,10 @@ project(':shell') { } tasks.create(name: "copyDependantLibs", type: Copy) { - from (configurations.testRuntime) { + from (configurations.testRuntimeClasspath) { include('jline-*jar') } - from (configurations.runtime) { + from (configurations.runtimeClasspath) { include('jline-*jar') } into "$buildDir/dependant-libs-${versions.scala}" @@ -1401,33 +1414,37 @@ project(':streams') { ext.buildStreamsVersionFileName = "kafka-streams-version.properties" dependencies { - compile project(':clients') + api project(':clients') + // use `api` dependency for `connect-json` for compatibility (e.g. users who use `JsonSerializer`/`JsonDeserializer` + // at compile-time without an explicit dependency on `connect-json`) // this dependency should be removed after we unify data API - compile(project(':connect:json')) { + api(project(':connect:json')) { // this transitive dependency is not used in Streams, and it breaks SBT builds exclude module: 'javax.ws.rs-api' } - compile libs.slf4jApi - compile libs.rocksDBJni + implementation libs.slf4jApi + implementation libs.rocksDBJni + implementation libs.jacksonAnnotations + implementation libs.jacksonDatabind // testCompileOnly prevents streams from exporting a dependency on test-utils, which would cause a dependency cycle testCompileOnly project(':streams:test-utils') - testCompile project(':clients').sourceSets.test.output - testCompile project(':core') - testCompile project(':core').sourceSets.test.output - testCompile libs.log4j - testCompile libs.junitJupiterApi - testCompile libs.junitVintageEngine - testCompile libs.easymock - testCompile libs.powermockJunit4 - testCompile libs.powermockEasymock - testCompile libs.bcpkix - testCompile libs.hamcrest + testImplementation project(':clients').sourceSets.test.output + testImplementation project(':core') + testImplementation project(':core').sourceSets.test.output + testImplementation libs.log4j + testImplementation libs.junitJupiterApi + testImplementation libs.junitVintageEngine + testImplementation libs.easymock + testImplementation libs.powermockJunit4 + testImplementation libs.powermockEasymock + testImplementation libs.bcpkix + testImplementation libs.hamcrest testRuntimeOnly project(':streams:test-utils') - testRuntime libs.slf4jlog4j + testRuntimeOnly libs.slf4jlog4j } task processMessages(type:JavaExec) { @@ -1468,7 +1485,7 @@ project(':streams') { include('log4j*jar') include('*hamcrest*') } - from (configurations.runtime) { + from (configurations.runtimeClasspath) { exclude('kafka-clients*') } into "$buildDir/dependant-libs-${versions.scala}" @@ -1539,24 +1556,24 @@ project(':streams:streams-scala') { apply plugin: 'scala' archivesBaseName = "kafka-streams-scala_${versions.baseScala}" dependencies { - compile project(':streams') + api project(':streams') - compile libs.scalaLibrary - compile libs.scalaCollectionCompat + api libs.scalaLibrary + api libs.scalaCollectionCompat - testCompile project(':core') - testCompile project(':core').sourceSets.test.output - testCompile project(':streams').sourceSets.test.output - testCompile project(':clients').sourceSets.test.output - testCompile project(':streams:test-utils') + testImplementation project(':core') + testImplementation project(':core').sourceSets.test.output + testImplementation project(':streams').sourceSets.test.output + testImplementation project(':clients').sourceSets.test.output + testImplementation project(':streams:test-utils') - testCompile libs.junitJupiterApi - testCompile libs.junitVintageEngine - testCompile libs.scalatest - testCompile libs.easymock - testCompile libs.hamcrest + testImplementation libs.junitJupiterApi + testImplementation libs.junitVintageEngine + testImplementation libs.scalatest + testImplementation libs.easymock + testImplementation libs.hamcrest - testRuntime libs.slf4jlog4j + testRuntimeOnly libs.slf4jlog4j } javadoc { @@ -1564,7 +1581,7 @@ project(':streams:streams-scala') { } tasks.create(name: "copyDependantLibs", type: Copy) { - from (configurations.runtime) { + from (configurations.runtimeClasspath) { exclude('kafka-streams*') } into "$buildDir/dependant-libs-${versions.scala}" @@ -1575,10 +1592,6 @@ project(':streams:streams-scala') { dependsOn 'copyDependantLibs' } - artifacts { - archives scaladocJar - } - test.dependsOn(':spotlessScalaCheck') } @@ -1586,15 +1599,17 @@ project(':streams:test-utils') { archivesBaseName = "kafka-streams-test-utils" dependencies { - compile project(':streams') - compile project(':clients') + api project(':streams') + api project(':clients') + + implementation libs.slf4jApi - testCompile project(':clients').sourceSets.test.output - testCompile libs.junitJupiter - testCompile libs.easymock - testCompile libs.hamcrest + testImplementation project(':clients').sourceSets.test.output + testImplementation libs.junitJupiter + testImplementation libs.easymock + testImplementation libs.hamcrest - testRuntime libs.slf4jlog4j + testRuntimeOnly libs.slf4jlog4j } javadoc { @@ -1603,7 +1618,7 @@ project(':streams:test-utils') { } tasks.create(name: "copyDependantLibs", type: Copy) { - from (configurations.runtime) { + from (configurations.runtimeClasspath) { exclude('kafka-streams*') } into "$buildDir/dependant-libs-${versions.scala}" @@ -1620,14 +1635,14 @@ project(':streams:examples') { archivesBaseName = "kafka-streams-examples" dependencies { - compile project(':streams') - compile project(':connect:json') // this dependency should be removed after we unify data API - compile libs.slf4jlog4j + implementation project(':streams') + implementation project(':connect:json') // this dependency should be removed after we unify data API + implementation libs.slf4jlog4j - testCompile project(':streams:test-utils') - testCompile project(':clients').sourceSets.test.output // for org.apache.kafka.test.IntegrationTest - testCompile libs.junitJupiter - testCompile libs.hamcrest + testImplementation project(':streams:test-utils') + testImplementation project(':clients').sourceSets.test.output // for org.apache.kafka.test.IntegrationTest + testImplementation libs.junitJupiter + testImplementation libs.hamcrest } javadoc { @@ -1635,7 +1650,7 @@ project(':streams:examples') { } tasks.create(name: "copyDependantLibs", type: Copy) { - from (configurations.runtime) { + from (configurations.runtimeClasspath) { exclude('kafka-streams*') } into "$buildDir/dependant-libs-${versions.scala}" @@ -1651,8 +1666,8 @@ project(':streams:upgrade-system-tests-0100') { archivesBaseName = "kafka-streams-upgrade-system-tests-0100" dependencies { - testCompile libs.kafkaStreams_0100 - testRuntime libs.junitJupiter + testImplementation libs.kafkaStreams_0100 + testRuntimeOnly libs.junitJupiter } systemTestLibs { @@ -1664,8 +1679,8 @@ project(':streams:upgrade-system-tests-0101') { archivesBaseName = "kafka-streams-upgrade-system-tests-0101" dependencies { - testCompile libs.kafkaStreams_0101 - testRuntime libs.junitJupiter + testImplementation libs.kafkaStreams_0101 + testRuntimeOnly libs.junitJupiter } systemTestLibs { @@ -1677,8 +1692,8 @@ project(':streams:upgrade-system-tests-0102') { archivesBaseName = "kafka-streams-upgrade-system-tests-0102" dependencies { - testCompile libs.kafkaStreams_0102 - testRuntime libs.junitJupiter + testImplementation libs.kafkaStreams_0102 + testRuntimeOnly libs.junitJupiter } systemTestLibs { @@ -1690,8 +1705,8 @@ project(':streams:upgrade-system-tests-0110') { archivesBaseName = "kafka-streams-upgrade-system-tests-0110" dependencies { - testCompile libs.kafkaStreams_0110 - testRuntime libs.junitJupiter + testImplementation libs.kafkaStreams_0110 + testRuntimeOnly libs.junitJupiter } systemTestLibs { @@ -1703,8 +1718,8 @@ project(':streams:upgrade-system-tests-10') { archivesBaseName = "kafka-streams-upgrade-system-tests-10" dependencies { - testCompile libs.kafkaStreams_10 - testRuntime libs.junitJupiter + testImplementation libs.kafkaStreams_10 + testRuntimeOnly libs.junitJupiter } systemTestLibs { @@ -1716,8 +1731,8 @@ project(':streams:upgrade-system-tests-11') { archivesBaseName = "kafka-streams-upgrade-system-tests-11" dependencies { - testCompile libs.kafkaStreams_11 - testRuntime libs.junitJupiter + testImplementation libs.kafkaStreams_11 + testRuntimeOnly libs.junitJupiter } systemTestLibs { @@ -1729,8 +1744,8 @@ project(':streams:upgrade-system-tests-20') { archivesBaseName = "kafka-streams-upgrade-system-tests-20" dependencies { - testCompile libs.kafkaStreams_20 - testRuntime libs.junitJupiter + testImplementation libs.kafkaStreams_20 + testRuntimeOnly libs.junitJupiter } systemTestLibs { @@ -1742,8 +1757,8 @@ project(':streams:upgrade-system-tests-21') { archivesBaseName = "kafka-streams-upgrade-system-tests-21" dependencies { - testCompile libs.kafkaStreams_21 - testRuntime libs.junitJupiter + testImplementation libs.kafkaStreams_21 + testRuntimeOnly libs.junitJupiter } systemTestLibs { @@ -1755,8 +1770,8 @@ project(':streams:upgrade-system-tests-22') { archivesBaseName = "kafka-streams-upgrade-system-tests-22" dependencies { - testCompile libs.kafkaStreams_22 - testRuntime libs.junitJupiter + testImplementation libs.kafkaStreams_22 + testRuntimeOnly libs.junitJupiter } systemTestLibs { @@ -1768,8 +1783,8 @@ project(':streams:upgrade-system-tests-23') { archivesBaseName = "kafka-streams-upgrade-system-tests-23" dependencies { - testCompile libs.kafkaStreams_23 - testRuntime libs.junitJupiter + testImplementation libs.kafkaStreams_23 + testRuntimeOnly libs.junitJupiter } systemTestLibs { @@ -1781,8 +1796,8 @@ project(':streams:upgrade-system-tests-24') { archivesBaseName = "kafka-streams-upgrade-system-tests-24" dependencies { - testCompile libs.kafkaStreams_24 - testRuntime libs.junitJupiter + testImplementation libs.kafkaStreams_24 + testRuntimeOnly libs.junitJupiter } systemTestLibs { @@ -1794,8 +1809,8 @@ project(':streams:upgrade-system-tests-25') { archivesBaseName = "kafka-streams-upgrade-system-tests-25" dependencies { - testCompile libs.kafkaStreams_25 - testRuntime libs.junitJupiter + testImplementation libs.kafkaStreams_25 + testRuntimeOnly libs.junitJupiter } systemTestLibs { @@ -1807,8 +1822,8 @@ project(':streams:upgrade-system-tests-26') { archivesBaseName = "kafka-streams-upgrade-system-tests-26" dependencies { - testCompile libs.kafkaStreams_26 - testRuntime libs.junitJupiter + testImplementation libs.kafkaStreams_26 + testRuntimeOnly libs.junitJupiter } systemTestLibs { @@ -1820,8 +1835,8 @@ project(':streams:upgrade-system-tests-27') { archivesBaseName = "kafka-streams-upgrade-system-tests-27" dependencies { - testCompile libs.kafkaStreams_27 - testRuntime libs.junitJupiter + testImplementation libs.kafkaStreams_27 + testRuntimeOnly libs.junitJupiter } systemTestLibs { @@ -1834,27 +1849,30 @@ project(':jmh-benchmarks') { apply plugin: 'com.github.johnrengelman.shadow' shadowJar { - baseName = 'kafka-jmh-benchmarks-all' - classifier = null - version = null + archiveBaseName = 'kafka-jmh-benchmarks' } dependencies { - compile(project(':core')) { + implementation(project(':core')) { // jmh requires jopt 4.x while `core` depends on 5.0, they are not binary compatible exclude group: 'net.sf.jopt-simple', module: 'jopt-simple' } - compile project(':clients') - compile project(':metadata') - compile project(':streams') - compile project(':core') - compile project(':clients').sourceSets.test.output - compile project(':core').sourceSets.test.output - compile libs.jmhCore + implementation project(':clients') + implementation project(':metadata') + implementation project(':streams') + implementation project(':core') + implementation project(':clients').sourceSets.test.output + implementation project(':core').sourceSets.test.output + + implementation libs.jmhCore annotationProcessor libs.jmhGeneratorAnnProcess - compile libs.jmhCoreBenchmarks - compile libs.mockitoCore - compile libs.slf4jlog4j + implementation libs.jmhCoreBenchmarks + implementation libs.jacksonDatabind + implementation libs.metrics + implementation libs.mockitoCore + implementation libs.slf4jlog4j + implementation libs.scalaLibrary + implementation libs.scalaJava8Compat } tasks.withType(JavaCompile) { @@ -1893,13 +1911,13 @@ project(':log4j-appender') { archivesBaseName = "kafka-log4j-appender" dependencies { - compile project(':clients') - compile libs.slf4jlog4j + implementation project(':clients') + implementation libs.slf4jlog4j - testCompile project(':clients').sourceSets.test.output - testCompile libs.junitJupiter - testCompile libs.hamcrest - testCompile libs.easymock + testImplementation project(':clients').sourceSets.test.output + testImplementation libs.junitJupiter + testImplementation libs.hamcrest + testImplementation libs.easymock } javadoc { @@ -1912,13 +1930,13 @@ project(':connect:api') { archivesBaseName = "connect-api" dependencies { - compile project(':clients') - compile libs.slf4jApi - compile libs.jaxrsApi + api project(':clients') + implementation libs.slf4jApi + implementation libs.jaxrsApi - testCompile libs.junitJupiter - testRuntime libs.slf4jlog4j - testCompile project(':clients').sourceSets.test.output + testImplementation libs.junitJupiter + testRuntimeOnly libs.slf4jlog4j + testImplementation project(':clients').sourceSets.test.output } javadoc { @@ -1931,11 +1949,11 @@ project(':connect:api') { } tasks.create(name: "copyDependantLibs", type: Copy) { - from (configurations.testRuntime) { + from (configurations.testRuntimeClasspath) { include('slf4j-log4j12*') include('log4j*jar') } - from (configurations.runtime) { + from (configurations.runtimeClasspath) { exclude('kafka-clients*') exclude('connect-*') } @@ -1952,14 +1970,15 @@ project(':connect:transforms') { archivesBaseName = "connect-transforms" dependencies { - compile project(':connect:api') - compile libs.slf4jApi + api project(':connect:api') + + implementation libs.slf4jApi - testCompile libs.easymock - testCompile libs.junitJupiter + testImplementation libs.easymock + testImplementation libs.junitJupiter - testRuntime libs.slf4jlog4j - testCompile project(':clients').sourceSets.test.output + testRuntimeOnly libs.slf4jlog4j + testImplementation project(':clients').sourceSets.test.output } javadoc { @@ -1967,11 +1986,11 @@ project(':connect:transforms') { } tasks.create(name: "copyDependantLibs", type: Copy) { - from (configurations.testRuntime) { + from (configurations.testRuntimeClasspath) { include('slf4j-log4j12*') include('log4j*jar') } - from (configurations.runtime) { + from (configurations.runtimeClasspath) { exclude('kafka-clients*') exclude('connect-*') } @@ -1988,16 +2007,18 @@ project(':connect:json') { archivesBaseName = "connect-json" dependencies { - compile project(':connect:api') - compile libs.jacksonDatabind - compile libs.jacksonJDK8Datatypes - compile libs.slf4jApi + api project(':connect:api') - testCompile libs.easymock - testCompile libs.junitJupiter + api libs.jacksonDatabind + api libs.jacksonJDK8Datatypes - testRuntime libs.slf4jlog4j - testCompile project(':clients').sourceSets.test.output + implementation libs.slf4jApi + + testImplementation libs.easymock + testImplementation libs.junitJupiter + + testRuntimeOnly libs.slf4jlog4j + testImplementation project(':clients').sourceSets.test.output } javadoc { @@ -2005,11 +2026,11 @@ project(':connect:json') { } tasks.create(name: "copyDependantLibs", type: Copy) { - from (configurations.testRuntime) { + from (configurations.testRuntimeClasspath) { include('slf4j-log4j12*') include('log4j*jar') } - from (configurations.runtime) { + from (configurations.runtimeClasspath) { exclude('kafka-clients*') exclude('connect-*') } @@ -2026,40 +2047,44 @@ project(':connect:runtime') { archivesBaseName = "connect-runtime" dependencies { - - compile project(':connect:api') - compile project(':clients') - compile project(':tools') - compile project(':connect:json') - compile project(':connect:transforms') - - compile libs.slf4jApi - compile libs.jacksonJaxrsJsonProvider - compile libs.jerseyContainerServlet - compile libs.jerseyHk2 - compile libs.jaxbApi // Jersey dependency that was available in the JDK before Java 9 - compile libs.activation // Jersey dependency that was available in the JDK before Java 9 - compile libs.jettyServer - compile libs.jettyServlet - compile libs.jettyServlets - compile libs.jettyClient - compile(libs.reflections) - compile(libs.mavenArtifact) - - testCompile project(':clients').sourceSets.test.output - testCompile libs.easymock - testCompile libs.junitJupiterApi - testCompile libs.junitVintageEngine - testCompile libs.powermockJunit4 - testCompile libs.powermockEasymock - testCompile libs.mockitoCore - testCompile libs.httpclient - - testCompile project(':clients').sourceSets.test.output - testCompile project(':core') - testCompile project(':core').sourceSets.test.output - - testRuntime libs.slf4jlog4j + // connect-runtime is used in tests, use `api` for modules below for backwards compatibility even though + // applications should generally not depend on `connect-runtime` + api project(':connect:api') + api project(':clients') + api project(':connect:json') + api project(':connect:transforms') + + implementation project(':tools') + + implementation libs.slf4jApi + implementation libs.log4j + implementation libs.jacksonAnnotations + implementation libs.jacksonJaxrsJsonProvider + implementation libs.jerseyContainerServlet + implementation libs.jerseyHk2 + implementation libs.jaxbApi // Jersey dependency that was available in the JDK before Java 9 + implementation libs.activation // Jersey dependency that was available in the JDK before Java 9 + implementation libs.jettyServer + implementation libs.jettyServlet + implementation libs.jettyServlets + implementation libs.jettyClient + implementation libs.reflections + implementation libs.mavenArtifact + + testImplementation project(':clients').sourceSets.test.output + testImplementation project(':core') + testImplementation project(':metadata') + testImplementation project(':core').sourceSets.test.output + + testImplementation libs.easymock + testImplementation libs.junitJupiterApi + testImplementation libs.junitVintageEngine + testImplementation libs.powermockJunit4 + testImplementation libs.powermockEasymock + testImplementation libs.mockitoCore + testImplementation libs.httpclient + + testRuntimeOnly libs.slf4jlog4j } javadoc { @@ -2067,11 +2092,11 @@ project(':connect:runtime') { } tasks.create(name: "copyDependantLibs", type: Copy) { - from (configurations.testRuntime) { + from (configurations.testRuntimeClasspath) { include('slf4j-log4j12*') include('log4j*jar') } - from (configurations.runtime) { + from (configurations.runtimeClasspath) { exclude('kafka-clients*') exclude('connect-*') } @@ -2131,14 +2156,14 @@ project(':connect:file') { archivesBaseName = "connect-file" dependencies { - compile project(':connect:api') - compile libs.slf4jApi + implementation project(':connect:api') + implementation libs.slf4jApi - testCompile libs.easymock - testCompile libs.junitJupiter + testImplementation libs.easymock + testImplementation libs.junitJupiter - testRuntime libs.slf4jlog4j - testCompile project(':clients').sourceSets.test.output + testRuntimeOnly libs.slf4jlog4j + testImplementation project(':clients').sourceSets.test.output } javadoc { @@ -2146,11 +2171,11 @@ project(':connect:file') { } tasks.create(name: "copyDependantLibs", type: Copy) { - from (configurations.testRuntime) { + from (configurations.testRuntimeClasspath) { include('slf4j-log4j12*') include('log4j*jar') } - from (configurations.runtime) { + from (configurations.runtimeClasspath) { exclude('kafka-clients*') exclude('connect-*') } @@ -2167,16 +2192,17 @@ project(':connect:basic-auth-extension') { archivesBaseName = "connect-basic-auth-extension" dependencies { - compile project(':connect:api') - compile libs.slf4jApi + implementation project(':connect:api') + implementation libs.slf4jApi + implementation libs.jaxrsApi - testCompile libs.bcpkix - testCompile libs.easymock - testCompile libs.junitJupiter - testCompile project(':clients').sourceSets.test.output + testImplementation libs.bcpkix + testImplementation libs.easymock + testImplementation libs.junitJupiter + testImplementation project(':clients').sourceSets.test.output - testRuntime libs.slf4jlog4j - testRuntime libs.jerseyContainerServlet + testRuntimeOnly libs.slf4jlog4j + testRuntimeOnly libs.jerseyContainerServlet } javadoc { @@ -2184,11 +2210,11 @@ project(':connect:basic-auth-extension') { } tasks.create(name: "copyDependantLibs", type: Copy) { - from (configurations.testRuntime) { + from (configurations.testRuntimeClasspath) { include('slf4j-log4j12*') include('log4j*jar') } - from (configurations.runtime) { + from (configurations.runtimeClasspath) { exclude('kafka-clients*') exclude('connect-*') } @@ -2205,23 +2231,25 @@ project(':connect:mirror') { archivesBaseName = "connect-mirror" dependencies { - compile project(':connect:api') - compile project(':connect:runtime') - compile project(':connect:mirror-client') - compile project(':clients') - compile libs.argparse4j - compile libs.slf4jApi + implementation project(':connect:api') + implementation project(':connect:runtime') + implementation project(':connect:mirror-client') + implementation project(':clients') - testCompile libs.junitJupiter - testCompile libs.mockitoCore - testCompile project(':clients').sourceSets.test.output - testCompile project(':connect:runtime').sourceSets.test.output - testCompile project(':core') - testCompile project(':core').sourceSets.test.output + implementation libs.argparse4j + implementation libs.jacksonAnnotations + implementation libs.slf4jApi - testRuntime project(':connect:runtime') - testRuntime libs.slf4jlog4j - testRuntime libs.bcpkix + testImplementation libs.junitJupiter + testImplementation libs.mockitoCore + testImplementation project(':clients').sourceSets.test.output + testImplementation project(':connect:runtime').sourceSets.test.output + testImplementation project(':core') + testImplementation project(':core').sourceSets.test.output + + testRuntimeOnly project(':connect:runtime') + testRuntimeOnly libs.slf4jlog4j + testRuntimeOnly libs.bcpkix } javadoc { @@ -2229,11 +2257,11 @@ project(':connect:mirror') { } tasks.create(name: "copyDependantLibs", type: Copy) { - from (configurations.testRuntime) { + from (configurations.testRuntimeClasspath) { include('slf4j-log4j12*') include('log4j*jar') } - from (configurations.runtime) { + from (configurations.runtimeClasspath) { exclude('kafka-clients*') exclude('connect-*') } @@ -2250,13 +2278,13 @@ project(':connect:mirror-client') { archivesBaseName = "connect-mirror-client" dependencies { - compile project(':clients') - compile libs.slf4jApi + implementation project(':clients') + implementation libs.slf4jApi - testCompile libs.junitJupiter - testCompile project(':clients').sourceSets.test.output + testImplementation libs.junitJupiter + testImplementation project(':clients').sourceSets.test.output - testRuntime libs.slf4jlog4j + testRuntimeOnly libs.slf4jlog4j } javadoc { @@ -2264,11 +2292,11 @@ project(':connect:mirror-client') { } tasks.create(name: "copyDependantLibs", type: Copy) { - from (configurations.testRuntime) { + from (configurations.testRuntimeClasspath) { include('slf4j-log4j12*') include('log4j*jar') } - from (configurations.runtime) { + from (configurations.runtimeClasspath) { exclude('kafka-clients*') exclude('connect-*') } diff --git a/docs/upgrade.html b/docs/upgrade.html index 9ce10371799e5..30550d755b61d 100644 --- a/docs/upgrade.html +++ b/docs/upgrade.html @@ -19,6 +19,14 @@