From 7c6ada547f33b932f7942904b78907c90433d170 Mon Sep 17 00:00:00 2001 From: Ismael Juma Date: Sat, 28 Sep 2019 15:37:05 -0700 Subject: [PATCH 01/51] Skip redundant `toStruct()` step when serializing requests that. rely on the auto generated protocol --- .../common/requests/AbstractRequest.java | 44 +++++++++++++++++-- .../common/requests/AbstractResponse.java | 3 ++ .../requests/AddOffsetsToTxnRequest.java | 6 +++ .../requests/AddOffsetsToTxnResponse.java | 6 +++ .../requests/AddPartitionsToTxnRequest.java | 6 +++ .../requests/AddPartitionsToTxnResponse.java | 6 +++ .../common/requests/AlterConfigsRequest.java | 6 +++ .../common/requests/AlterConfigsResponse.java | 6 +++ .../requests/AlterReplicaLogDirsRequest.java | 6 +++ .../requests/AlterReplicaLogDirsResponse.java | 6 +++ .../common/requests/ApiVersionsRequest.java | 6 +++ .../common/requests/ApiVersionsResponse.java | 6 +++ .../common/requests/CreateAclsRequest.java | 6 +++ .../common/requests/CreateAclsResponse.java | 6 +++ .../requests/CreatePartitionsRequest.java | 6 +++ .../requests/CreatePartitionsResponse.java | 6 +++ .../common/requests/DeleteAclsRequest.java | 6 +++ .../common/requests/DeleteAclsResponse.java | 6 +++ .../common/requests/DeleteGroupsRequest.java | 6 +++ .../common/requests/DeleteGroupsResponse.java | 6 +++ .../common/requests/DeleteRecordsRequest.java | 6 +++ .../requests/DeleteRecordsResponse.java | 6 +++ .../common/requests/DescribeAclsRequest.java | 6 +++ .../common/requests/DescribeAclsResponse.java | 6 +++ .../requests/DescribeConfigsRequest.java | 6 +++ .../requests/DescribeConfigsResponse.java | 6 +++ .../DescribeDelegationTokenResponse.java | 6 +++ .../requests/DescribeLogDirsRequest.java | 6 +++ .../requests/DescribeLogDirsResponse.java | 6 +++ .../kafka/common/requests/EndTxnRequest.java | 6 +++ .../kafka/common/requests/EndTxnResponse.java | 6 +++ .../ExpireDelegationTokenRequest.java | 6 +++ .../ExpireDelegationTokenResponse.java | 6 +++ .../kafka/common/requests/FetchRequest.java | 7 +++ .../kafka/common/requests/FetchResponse.java | 6 +++ .../common/requests/HeartbeatRequest.java | 6 +++ .../common/requests/HeartbeatResponse.java | 6 +++ .../requests/InitProducerIdRequest.java | 6 +++ .../requests/InitProducerIdResponse.java | 6 +++ .../common/requests/LeaderAndIsrRequest.java | 11 ++--- .../common/requests/LeaderAndIsrResponse.java | 6 +++ .../common/requests/LeaveGroupResponse.java | 6 +++ .../common/requests/ListGroupsRequest.java | 6 +++ .../common/requests/ListOffsetRequest.java | 6 +++ .../common/requests/ListOffsetResponse.java | 6 +++ .../common/requests/MetadataResponse.java | 6 +++ .../common/requests/OffsetDeleteRequest.java | 7 +++ .../common/requests/OffsetDeleteResponse.java | 6 +++ .../common/requests/OffsetFetchRequest.java | 6 +++ .../common/requests/OffsetFetchResponse.java | 6 +++ .../OffsetsForLeaderEpochRequest.java | 6 +++ .../OffsetsForLeaderEpochResponse.java | 6 +++ .../kafka/common/requests/ProduceRequest.java | 6 +++ .../common/requests/ProduceResponse.java | 6 +++ .../RenewDelegationTokenResponse.java | 6 +++ .../requests/SaslAuthenticateResponse.java | 8 +++- .../requests/SaslHandshakeResponse.java | 8 +++- .../common/requests/StopReplicaRequest.java | 11 ++--- .../common/requests/StopReplicaResponse.java | 6 +++ .../common/requests/SyncGroupRequest.java | 6 +++ .../common/requests/SyncGroupResponse.java | 6 +++ .../requests/TxnOffsetCommitRequest.java | 7 +++ .../requests/TxnOffsetCommitResponse.java | 6 +++ .../requests/UpdateMetadataRequest.java | 5 ++- .../requests/UpdateMetadataResponse.java | 7 +++ .../requests/WriteTxnMarkersRequest.java | 6 +++ .../requests/WriteTxnMarkersResponse.java | 6 +++ .../SaslClientAuthenticator.java | 2 +- .../common/protocol/MessageTestUtil.java | 4 -- .../requests/LeaderAndIsrRequestTest.java | 7 +-- .../requests/StopReplicaRequestTest.java | 3 +- .../requests/UpdateMetadataRequestTest.java | 5 +-- .../unit/kafka/network/SocketServerTest.scala | 8 ++-- .../unit/kafka/server/BaseRequestTest.scala | 6 +-- .../kafka/server/ClientQuotaManagerTest.scala | 2 +- .../unit/kafka/server/KafkaApisTest.scala | 2 +- .../ThrottledChannelExpirationTest.scala | 2 +- 77 files changed, 453 insertions(+), 42 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java index b737b49f4a5c1..6ad7bbfbf7f90 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java @@ -20,7 +20,11 @@ import org.apache.kafka.common.network.NetworkSend; import org.apache.kafka.common.network.Send; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.Message; +import org.apache.kafka.common.protocol.ObjectSerializationCache; +import org.apache.kafka.common.protocol.Writable; import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; @@ -93,16 +97,50 @@ public short version() { } public Send toSend(String destination, RequestHeader header) { - return new NetworkSend(destination, serialize(header)); + return new NetworkSend(destination, serializeWithHeader(header)); } /** * Use with care, typically {@link #toSend(String, RequestHeader)} should be used instead. */ - public ByteBuffer serialize(RequestHeader header) { - return serialize(header.toStruct(), toStruct()); + public final ByteBuffer serializeWithHeader(RequestHeader header) { + Message data = data(); + if (data == null) + return serialize(header.toStruct(), toStruct()); + + Struct headerStruct = header.toStruct(); + ObjectSerializationCache serializationCache = new ObjectSerializationCache(); + ByteBuffer buffer = ByteBuffer.allocate(headerStruct.sizeOf() + data.size(serializationCache, version)); + headerStruct.writeTo(buffer); + data.write(new ByteBufferAccessor(buffer), serializationCache, version); + buffer.rewind(); + return buffer; + } + + // Visible for testing + ByteBuffer serializeBody() { + Message data = data(); + ByteBuffer buffer; + if (data == null) { + Struct bodyStruct = toStruct(); + buffer = ByteBuffer.allocate(bodyStruct.sizeOf()); + bodyStruct.writeTo(buffer); + } else { + ObjectSerializationCache serializationCache = new ObjectSerializationCache(); + buffer = ByteBuffer.allocate(data.size(serializationCache, version)); + data.write(new ByteBufferAccessor(buffer), serializationCache, version); + } + buffer.rewind(); + return buffer; } + // Visible for testing + int sizeInBytes() { + return data().size(new ObjectSerializationCache(), version); + } + + protected abstract Message data(); + protected abstract Struct toStruct(); public String toString(boolean verbose) { diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java index 8a6edf14cbac0..ea55c1ba274b6 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java @@ -20,6 +20,7 @@ import org.apache.kafka.common.network.Send; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; @@ -51,6 +52,8 @@ public ByteBuffer serialize(ApiKeys apiKey, short version, int correlationId) { return serialize(header.toStruct(), toStruct(version)); } + protected abstract Message data(); + public abstract Map errorCounts(); protected Map errorCounts(Errors error) { diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AddOffsetsToTxnRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AddOffsetsToTxnRequest.java index 2668ae1db5709..cd5fb78487aa6 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AddOffsetsToTxnRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AddOffsetsToTxnRequest.java @@ -18,6 +18,7 @@ import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Schema; import org.apache.kafka.common.protocol.types.Struct; @@ -127,6 +128,11 @@ protected Struct toStruct() { return struct; } + @Override + protected Message data() { + return null; + } + @Override public AddOffsetsToTxnResponse getErrorResponse(int throttleTimeMs, Throwable e) { return new AddOffsetsToTxnResponse(throttleTimeMs, Errors.forException(e)); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AddOffsetsToTxnResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AddOffsetsToTxnResponse.java index 867ca6afd0e81..9a2fe9c69950f 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AddOffsetsToTxnResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AddOffsetsToTxnResponse.java @@ -18,6 +18,7 @@ import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Schema; import org.apache.kafka.common.protocol.types.Struct; @@ -86,6 +87,11 @@ protected Struct toStruct(short version) { return struct; } + @Override + protected Message data() { + return null; + } + public static AddOffsetsToTxnResponse parse(ByteBuffer buffer, short version) { return new AddOffsetsToTxnResponse(ApiKeys.ADD_OFFSETS_TO_TXN.parseResponse(version, buffer)); } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AddPartitionsToTxnRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AddPartitionsToTxnRequest.java index 3be7c9836abd7..4339be5076abf 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AddPartitionsToTxnRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AddPartitionsToTxnRequest.java @@ -19,6 +19,7 @@ import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.ArrayOf; import org.apache.kafka.common.protocol.types.Field; import org.apache.kafka.common.protocol.types.Schema; @@ -164,6 +165,11 @@ protected Struct toStruct() { return struct; } + @Override + protected Message data() { + return null; + } + @Override public AddPartitionsToTxnResponse getErrorResponse(int throttleTimeMs, Throwable e) { final HashMap errors = new HashMap<>(); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AddPartitionsToTxnResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AddPartitionsToTxnResponse.java index 7f20f07b6a117..1133ab5411e65 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AddPartitionsToTxnResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AddPartitionsToTxnResponse.java @@ -19,6 +19,7 @@ import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.ArrayOf; import org.apache.kafka.common.protocol.types.Field; import org.apache.kafka.common.protocol.types.Schema; @@ -130,6 +131,11 @@ protected Struct toStruct(short version) { return struct; } + @Override + protected Message data() { + return null; + } + public static AddPartitionsToTxnResponse parse(ByteBuffer buffer, short version) { return new AddPartitionsToTxnResponse(ApiKeys.ADD_PARTITIONS_TO_TXN.parseResponse(version, buffer)); } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AlterConfigsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AlterConfigsRequest.java index 963ad065e4389..6f7d4a23fd5ca 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AlterConfigsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AlterConfigsRequest.java @@ -19,6 +19,7 @@ import org.apache.kafka.common.config.ConfigResource; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.ArrayOf; import org.apache.kafka.common.protocol.types.Field; import org.apache.kafka.common.protocol.types.Schema; @@ -189,6 +190,11 @@ protected Struct toStruct() { return struct; } + @Override + protected Message data() { + return null; + } + @Override public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { short version = version(); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AlterConfigsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AlterConfigsResponse.java index bf805df5158e9..03af813eb1fe3 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AlterConfigsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AlterConfigsResponse.java @@ -20,6 +20,7 @@ import org.apache.kafka.common.config.ConfigResource; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.ArrayOf; import org.apache.kafka.common.protocol.types.Field; import org.apache.kafka.common.protocol.types.Schema; @@ -115,6 +116,11 @@ protected Struct toStruct(short version) { return struct; } + @Override + protected Message data() { + return null; + } + public static AlterConfigsResponse parse(ByteBuffer buffer, short version) { return new AlterConfigsResponse(ApiKeys.ALTER_CONFIGS.parseResponse(version, buffer)); } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AlterReplicaLogDirsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AlterReplicaLogDirsRequest.java index b7cf1d8d40bee..f6a26758e94b4 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AlterReplicaLogDirsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AlterReplicaLogDirsRequest.java @@ -20,6 +20,7 @@ import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.ArrayOf; import org.apache.kafka.common.protocol.types.Field; import org.apache.kafka.common.protocol.types.Schema; @@ -141,6 +142,11 @@ protected Struct toStruct() { return struct; } + @Override + protected Message data() { + return null; + } + @Override public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { Map responseMap = new HashMap<>(); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AlterReplicaLogDirsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AlterReplicaLogDirsResponse.java index 2bbdce2122bc2..f661948f33d34 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AlterReplicaLogDirsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AlterReplicaLogDirsResponse.java @@ -20,6 +20,7 @@ import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.ArrayOf; import org.apache.kafka.common.protocol.types.Field; import org.apache.kafka.common.protocol.types.Schema; @@ -121,6 +122,11 @@ protected Struct toStruct(short version) { return struct; } + @Override + protected Message data() { + return null; + } + @Override public int throttleTimeMs() { return throttleTimeMs; diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsRequest.java index 87ffceaa26e1c..08c33d9452077 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsRequest.java @@ -23,6 +23,7 @@ import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersionsResponseKeyCollection; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.utils.AppInfoParser; @@ -100,6 +101,11 @@ protected Struct toStruct() { return data.toStruct(version()); } + @Override + protected Message data() { + return null; + } + @Override public ApiVersionsResponse getErrorResponse(int throttleTimeMs, Throwable e) { ApiVersionsResponseData data = new ApiVersionsResponseData() diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsResponse.java index 4604fedd0b0e3..709e3c4ed1c53 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsResponse.java @@ -22,6 +22,7 @@ import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.SchemaException; import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.record.RecordBatch; @@ -53,6 +54,11 @@ public ApiVersionsResponse(Struct struct, short version) { this(new ApiVersionsResponseData(struct, version)); } + @Override + protected Message data() { + return null; + } + @Override protected Struct toStruct(short version) { return this.data.toStruct(version); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/CreateAclsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/CreateAclsRequest.java index 29ecd016f4b45..adeb157e8c448 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/CreateAclsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/CreateAclsRequest.java @@ -19,6 +19,7 @@ import org.apache.kafka.common.acl.AccessControlEntry; import org.apache.kafka.common.acl.AclBinding; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.resource.ResourcePattern; import org.apache.kafka.common.errors.UnsupportedVersionException; import org.apache.kafka.common.protocol.ApiKeys; @@ -151,6 +152,11 @@ protected Struct toStruct() { return struct; } + @Override + protected Message data() { + return null; + } + public List aclCreations() { return aclCreations; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/CreateAclsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/CreateAclsResponse.java index d5f52ddfb387c..7f26ec332ff6a 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/CreateAclsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/CreateAclsResponse.java @@ -18,6 +18,7 @@ import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.ArrayOf; import org.apache.kafka.common.protocol.types.Field; import org.apache.kafka.common.protocol.types.Schema; @@ -101,6 +102,11 @@ protected Struct toStruct(short version) { return struct; } + @Override + protected Message data() { + return null; + } + @Override public int throttleTimeMs() { return throttleTimeMs; diff --git a/clients/src/main/java/org/apache/kafka/common/requests/CreatePartitionsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/CreatePartitionsRequest.java index 7872cf9f8d0d8..3599811e3882d 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/CreatePartitionsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/CreatePartitionsRequest.java @@ -18,6 +18,7 @@ package org.apache.kafka.common.requests; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.ArrayOf; import org.apache.kafka.common.protocol.types.Field; import org.apache.kafka.common.protocol.types.Schema; @@ -225,6 +226,11 @@ protected Struct toStruct() { return struct; } + @Override + protected Message data() { + return null; + } + @Override public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { Map topicErrors = new HashMap<>(); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/CreatePartitionsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/CreatePartitionsResponse.java index 744cc32e7b3f3..e682aada145f3 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/CreatePartitionsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/CreatePartitionsResponse.java @@ -19,6 +19,7 @@ import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.ArrayOf; import org.apache.kafka.common.protocol.types.Field; import org.apache.kafka.common.protocol.types.Schema; @@ -96,6 +97,11 @@ protected Struct toStruct(short version) { return struct; } + @Override + protected Message data() { + return null; + } + public Map errors() { return errors; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DeleteAclsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DeleteAclsRequest.java index 2ed7c24f68a8f..6f06513766914 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DeleteAclsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DeleteAclsRequest.java @@ -20,6 +20,7 @@ import org.apache.kafka.common.acl.AclBindingFilter; import org.apache.kafka.common.errors.UnsupportedVersionException; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.ArrayOf; import org.apache.kafka.common.protocol.types.Field; import org.apache.kafka.common.protocol.types.Schema; @@ -131,6 +132,11 @@ protected Struct toStruct() { return struct; } + @Override + protected Message data() { + return null; + } + @Override public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable throwable) { short versionId = version(); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DeleteAclsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DeleteAclsResponse.java index a3b81ccaf747b..04986b084b8c2 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DeleteAclsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DeleteAclsResponse.java @@ -18,6 +18,7 @@ import org.apache.kafka.common.acl.AccessControlEntry; import org.apache.kafka.common.acl.AclBinding; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.resource.ResourcePattern; import org.apache.kafka.common.errors.UnsupportedVersionException; import org.apache.kafka.common.protocol.ApiKeys; @@ -212,6 +213,11 @@ protected Struct toStruct(short version) { return struct; } + @Override + protected Message data() { + return null; + } + @Override public int throttleTimeMs() { return throttleTimeMs; diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsRequest.java index 2ad947c5904be..2de85f1ac23be 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsRequest.java @@ -22,6 +22,7 @@ import org.apache.kafka.common.message.DeleteGroupsResponseData.DeletableGroupResultCollection; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; @@ -83,4 +84,9 @@ public static DeleteGroupsRequest parse(ByteBuffer buffer, short version) { protected Struct toStruct() { return data.toStruct(version()); } + + @Override + protected Message data() { + return data; + } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsResponse.java index a98a7c881d96e..0a030bdb8495e 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsResponse.java @@ -20,6 +20,7 @@ import org.apache.kafka.common.message.DeleteGroupsResponseData.DeletableGroupResult; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; @@ -59,6 +60,11 @@ protected Struct toStruct(short version) { return data.toStruct(version); } + @Override + protected Message data() { + return data; + } + public Map errors() { Map errorMap = new HashMap<>(); for (DeletableGroupResult result : data.results()) { diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DeleteRecordsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DeleteRecordsRequest.java index 7ea55534f6908..62db4ae63a619 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DeleteRecordsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DeleteRecordsRequest.java @@ -20,6 +20,7 @@ import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.ArrayOf; import org.apache.kafka.common.protocol.types.Field; import org.apache.kafka.common.protocol.types.Schema; @@ -147,6 +148,11 @@ protected Struct toStruct() { return struct; } + @Override + protected Message data() { + return null; + } + @Override public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { Map responseMap = new HashMap<>(); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DeleteRecordsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DeleteRecordsResponse.java index 311be1f73e6ec..7531a05a4a7a3 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DeleteRecordsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DeleteRecordsResponse.java @@ -20,6 +20,7 @@ import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.ArrayOf; import org.apache.kafka.common.protocol.types.Field; import org.apache.kafka.common.protocol.types.Schema; @@ -157,6 +158,11 @@ protected Struct toStruct(short version) { return struct; } + @Override + protected Message data() { + return null; + } + @Override public int throttleTimeMs() { return throttleTimeMs; diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeAclsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeAclsRequest.java index ed417c577309c..d7b27b026914d 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeAclsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeAclsRequest.java @@ -20,6 +20,7 @@ import org.apache.kafka.common.acl.AclBindingFilter; import org.apache.kafka.common.errors.UnsupportedVersionException; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Schema; import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.resource.PatternType; @@ -107,6 +108,11 @@ protected Struct toStruct() { return struct; } + @Override + protected Message data() { + return null; + } + @Override public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable throwable) { short versionId = version(); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeAclsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeAclsResponse.java index 341845cf2b10a..35960f2ebfdbb 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeAclsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeAclsResponse.java @@ -19,6 +19,7 @@ import org.apache.kafka.common.acl.AccessControlEntry; import org.apache.kafka.common.acl.AclBinding; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.resource.PatternType; import org.apache.kafka.common.resource.ResourcePattern; import org.apache.kafka.common.errors.UnsupportedVersionException; @@ -155,6 +156,11 @@ protected Struct toStruct(short version) { return struct; } + @Override + protected Message data() { + return null; + } + @Override public int throttleTimeMs() { return throttleTimeMs; diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeConfigsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeConfigsRequest.java index 0ee256ff53408..503cd1c1064ed 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeConfigsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeConfigsRequest.java @@ -18,6 +18,7 @@ import org.apache.kafka.common.config.ConfigResource; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.ArrayOf; import org.apache.kafka.common.protocol.types.Field; import org.apache.kafka.common.protocol.types.Schema; @@ -162,6 +163,11 @@ protected Struct toStruct() { return struct; } + @Override + protected Message data() { + return null; + } + @Override public DescribeConfigsResponse getErrorResponse(int throttleTimeMs, Throwable e) { short version = version(); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeConfigsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeConfigsResponse.java index 6d424f2f3e378..7e58b34574d85 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeConfigsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeConfigsResponse.java @@ -20,6 +20,7 @@ import org.apache.kafka.common.config.ConfigResource; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.ArrayOf; import org.apache.kafka.common.protocol.types.Field; import org.apache.kafka.common.protocol.types.Schema; @@ -356,6 +357,11 @@ protected Struct toStruct(short version) { return struct; } + @Override + protected Message data() { + return null; + } + public static DescribeConfigsResponse parse(ByteBuffer buffer, short version) { return new DescribeConfigsResponse(ApiKeys.DESCRIBE_CONFIGS.parseResponse(version, buffer)); } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenResponse.java index 91e7db0fccb1e..eb575092080eb 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenResponse.java @@ -21,6 +21,7 @@ import org.apache.kafka.common.message.DescribeDelegationTokenResponseData.DescribedDelegationTokenRenewer; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.security.auth.KafkaPrincipal; import org.apache.kafka.common.security.token.delegation.DelegationToken; @@ -81,6 +82,11 @@ protected Struct toStruct(short version) { return data.toStruct(version); } + @Override + protected Message data() { + return data; + } + @Override public int throttleTimeMs() { return data.throttleTimeMs(); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeLogDirsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeLogDirsRequest.java index e16cc1873328f..40a66e42df6bc 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeLogDirsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeLogDirsRequest.java @@ -19,6 +19,7 @@ import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.ArrayOf; import org.apache.kafka.common.protocol.types.Field; import org.apache.kafka.common.protocol.types.Schema; @@ -137,6 +138,11 @@ protected Struct toStruct() { return struct; } + @Override + protected Message data() { + return null; + } + @Override public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { short versionId = version(); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeLogDirsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeLogDirsResponse.java index ee6d95cfff843..dd80eeca7a844 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeLogDirsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeLogDirsResponse.java @@ -20,6 +20,7 @@ import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.ArrayOf; import org.apache.kafka.common.protocol.types.Field; import org.apache.kafka.common.protocol.types.Schema; @@ -163,6 +164,11 @@ protected Struct toStruct(short version) { return struct; } + @Override + protected Message data() { + return null; + } + @Override public int throttleTimeMs() { return throttleTimeMs; diff --git a/clients/src/main/java/org/apache/kafka/common/requests/EndTxnRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/EndTxnRequest.java index 833a7fe5aedee..6f29ac38cf6ef 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/EndTxnRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/EndTxnRequest.java @@ -18,6 +18,7 @@ import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Field; import org.apache.kafka.common.protocol.types.Schema; import org.apache.kafka.common.protocol.types.Struct; @@ -130,6 +131,11 @@ protected Struct toStruct() { return struct; } + @Override + protected Message data() { + return null; + } + @Override public EndTxnResponse getErrorResponse(int throttleTimeMs, Throwable e) { return new EndTxnResponse(throttleTimeMs, Errors.forException(e)); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/EndTxnResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/EndTxnResponse.java index cf499966f80d8..8d79ff4a5e2b4 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/EndTxnResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/EndTxnResponse.java @@ -18,6 +18,7 @@ import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Schema; import org.apache.kafka.common.protocol.types.Struct; @@ -85,6 +86,11 @@ protected Struct toStruct(short version) { return struct; } + @Override + protected Message data() { + return null; + } + public static EndTxnResponse parse(ByteBuffer buffer, short version) { return new EndTxnResponse(ApiKeys.END_TXN.parseResponse(version, buffer)); } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenRequest.java index ca6d2d67d1f79..45b7df327bf5c 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenRequest.java @@ -22,6 +22,7 @@ import org.apache.kafka.common.message.ExpireDelegationTokenResponseData; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Struct; public class ExpireDelegationTokenRequest extends AbstractRequest { @@ -47,6 +48,11 @@ protected Struct toStruct() { return data.toStruct(version()); } + @Override + protected Message data() { + return data; + } + @Override public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { return new ExpireDelegationTokenResponse( diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenResponse.java index 16a6e8c5b40f5..6b8a22fa6ecee 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenResponse.java @@ -23,6 +23,7 @@ import org.apache.kafka.common.message.ExpireDelegationTokenResponseData; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Struct; public class ExpireDelegationTokenResponse extends AbstractResponse { @@ -59,6 +60,11 @@ protected Struct toStruct(short version) { return data.toStruct(version); } + @Override + protected Message data() { + return null; + } + @Override public int throttleTimeMs() { return data.throttleTimeMs(); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/FetchRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/FetchRequest.java index 2c0455ae4ae1e..5e3a90a9c6439 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/FetchRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/FetchRequest.java @@ -19,6 +19,7 @@ import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Field; import org.apache.kafka.common.protocol.types.Schema; import org.apache.kafka.common.protocol.types.Struct; @@ -561,4 +562,10 @@ protected Struct toStruct() { struct.setIfExists(RACK_ID, rackId); return struct; } + + @Override + protected Message data() { + return null; + } + } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/FetchResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/FetchResponse.java index d387e8199966c..1af771003bf0e 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/FetchResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/FetchResponse.java @@ -21,6 +21,7 @@ import org.apache.kafka.common.network.Send; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.ArrayOf; import org.apache.kafka.common.protocol.types.Field; import org.apache.kafka.common.protocol.types.Schema; @@ -419,6 +420,11 @@ public Struct toStruct(short version) { return toStruct(version, throttleTimeMs, error, responseData.entrySet().iterator(), sessionId); } + @Override + protected Message data() { + return null; + } + @Override protected Send toSend(String dest, ResponseHeader responseHeader, short apiVersion) { Struct responseHeaderStruct = responseHeader.toStruct(); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatRequest.java index e5cc6c8860a58..c9a51b7d83b6e 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatRequest.java @@ -21,6 +21,7 @@ import org.apache.kafka.common.message.HeartbeatResponseData; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; @@ -81,4 +82,9 @@ public static HeartbeatRequest parse(ByteBuffer buffer, short version) { protected Struct toStruct() { return data.toStruct(version()); } + + @Override + protected Message data() { + return data; + } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatResponse.java index cc36a2037ae08..3de921155fc98 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatResponse.java @@ -19,6 +19,7 @@ import org.apache.kafka.common.message.HeartbeatResponseData; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; @@ -67,6 +68,11 @@ protected Struct toStruct(short version) { return data.toStruct(version); } + @Override + protected Message data() { + return data; + } + public static HeartbeatResponse parse(ByteBuffer buffer, short version) { return new HeartbeatResponse(ApiKeys.HEARTBEAT.parseResponse(version, buffer), version); } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/InitProducerIdRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/InitProducerIdRequest.java index 8351c2159836a..e221889011142 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/InitProducerIdRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/InitProducerIdRequest.java @@ -20,6 +20,7 @@ import org.apache.kafka.common.message.InitProducerIdResponseData; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.record.RecordBatch; @@ -78,6 +79,11 @@ public static InitProducerIdRequest parse(ByteBuffer buffer, short version) { return new InitProducerIdRequest(ApiKeys.INIT_PRODUCER_ID.parseRequest(version, buffer), version); } + @Override + protected Message data() { + return data; + } + @Override protected Struct toStruct() { return data.toStruct(version()); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/InitProducerIdResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/InitProducerIdResponse.java index a33daf3e2d220..c167fd9c29ef8 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/InitProducerIdResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/InitProducerIdResponse.java @@ -19,6 +19,7 @@ import org.apache.kafka.common.message.InitProducerIdResponseData; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; @@ -58,6 +59,11 @@ protected Struct toStruct(short version) { return data.toStruct(version); } + @Override + protected Message data() { + return data; + } + public static InitProducerIdResponse parse(ByteBuffer buffer, short version) { return new InitProducerIdResponse(ApiKeys.INIT_PRODUCER_ID.parseResponse(version, buffer), version); } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java index 63013fff2b783..3bebd1eaaa3ef 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java @@ -25,6 +25,7 @@ import org.apache.kafka.common.message.LeaderAndIsrResponseData.LeaderAndIsrPartitionError; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.utils.FlattenedIterator; import org.apache.kafka.common.utils.Utils; @@ -132,6 +133,11 @@ protected Struct toStruct() { return data.toStruct(version()); } + @Override + protected Message data() { + return data; + } + @Override public LeaderAndIsrResponse getErrorResponse(int throttleTimeMs, Throwable e) { LeaderAndIsrResponseData responseData = new LeaderAndIsrResponseData(); @@ -186,11 +192,6 @@ public List liveLeaders() { return Collections.unmodifiableList(data.liveLeaders()); } - // Visible for testing - LeaderAndIsrRequestData data() { - return data; - } - public static LeaderAndIsrRequest parse(ByteBuffer buffer, short version) { return new LeaderAndIsrRequest(ApiKeys.LEADER_AND_ISR.parseRequest(version, buffer), version); } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrResponse.java index 0329307e66693..de98cb1ec9833 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrResponse.java @@ -20,6 +20,7 @@ import org.apache.kafka.common.message.LeaderAndIsrResponseData.LeaderAndIsrPartitionError; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; @@ -72,6 +73,11 @@ protected Struct toStruct(short version) { return data.toStruct(version); } + @Override + protected Message data() { + return data; + } + @Override public String toString() { return data.toString(); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupResponse.java index e64fde168980b..71ad453500497 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupResponse.java @@ -20,6 +20,7 @@ import org.apache.kafka.common.message.LeaveGroupResponseData.MemberResponse; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; @@ -139,6 +140,11 @@ public Struct toStruct(short version) { return data.toStruct(version); } + @Override + protected Message data() { + return data; + } + public static LeaveGroupResponse parse(ByteBuffer buffer, short versionId) { return new LeaveGroupResponse(ApiKeys.LEAVE_GROUP.parseResponse(versionId, buffer), versionId); } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsRequest.java index caa6f47906bfd..07f96aba9136c 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsRequest.java @@ -20,6 +20,7 @@ import org.apache.kafka.common.message.ListGroupsResponseData; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; @@ -85,4 +86,9 @@ public static ListGroupsRequest parse(ByteBuffer buffer, short version) { protected Struct toStruct() { return data.toStruct(version()); } + + @Override + protected Message data() { + return data; + } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetRequest.java index e9fe942e8581a..179f3b9075485 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetRequest.java @@ -19,6 +19,7 @@ import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Field; import org.apache.kafka.common.protocol.types.Schema; import org.apache.kafka.common.protocol.types.Struct; @@ -332,4 +333,9 @@ protected Struct toStruct() { struct.set(TOPICS, topicArray.toArray()); return struct; } + + @Override + protected Message data() { + return null; + } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetResponse.java index 5cf35e1d7900e..8126c46207122 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetResponse.java @@ -19,6 +19,7 @@ import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Field; import org.apache.kafka.common.protocol.types.Schema; import org.apache.kafka.common.protocol.types.Struct; @@ -284,6 +285,11 @@ protected Struct toStruct(short version) { return struct; } + @Override + protected Message data() { + return null; + } + @Override public String toString() { StringBuilder bld = new StringBuilder(); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/MetadataResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/MetadataResponse.java index ef5381b444376..fcaa0a18b660e 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/MetadataResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/MetadataResponse.java @@ -25,6 +25,7 @@ import org.apache.kafka.common.message.MetadataResponseData.MetadataResponseBroker; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.record.RecordBatch; import org.apache.kafka.common.utils.Utils; @@ -74,6 +75,11 @@ protected Struct toStruct(short version) { return data.toStruct(version); } + @Override + protected Message data() { + return data; + } + @Override public int throttleTimeMs() { return data.throttleTimeMs(); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetDeleteRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetDeleteRequest.java index 293efc2f6f2db..9dc4cbbad6e84 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/OffsetDeleteRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetDeleteRequest.java @@ -20,6 +20,8 @@ import org.apache.kafka.common.message.OffsetDeleteResponseData; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.Message; +import org.apache.kafka.common.protocol.Writable; import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; @@ -80,4 +82,9 @@ public static OffsetDeleteRequest parse(ByteBuffer buffer, short version) { protected Struct toStruct() { return data.toStruct(version()); } + + @Override + protected Message data() { + return data; + } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetDeleteResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetDeleteResponse.java index ff0c2967be890..7090000a6ee37 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/OffsetDeleteResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetDeleteResponse.java @@ -21,6 +21,7 @@ import org.apache.kafka.common.message.OffsetDeleteResponseData.OffsetDeleteResponseTopic; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; @@ -66,6 +67,11 @@ protected Struct toStruct(short version) { return data.toStruct(version); } + @Override + protected Message data() { + return data; + } + @Override public Map errorCounts() { Map counts = new HashMap<>(); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchRequest.java index 2303f89f91984..5ab412498c185 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchRequest.java @@ -22,6 +22,7 @@ import org.apache.kafka.common.message.OffsetFetchRequestData.OffsetFetchRequestTopic; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; @@ -158,4 +159,9 @@ public boolean isAllPartitions() { protected Struct toStruct() { return data.toStruct(version()); } + + @Override + protected Message data() { + return data; + } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchResponse.java index 722ef4e468a7a..dbeff6d2ac6f0 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchResponse.java @@ -22,6 +22,7 @@ import org.apache.kafka.common.message.OffsetFetchResponseData.OffsetFetchResponseTopic; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; @@ -215,6 +216,11 @@ protected Struct toStruct(short version) { return data.toStruct(version); } + @Override + protected Message data() { + return null; + } + @Override public boolean shouldClientThrottle(short version) { return version >= 4; diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetsForLeaderEpochRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetsForLeaderEpochRequest.java index d5c78da5e281f..52f94f6646852 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/OffsetsForLeaderEpochRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetsForLeaderEpochRequest.java @@ -20,6 +20,7 @@ import org.apache.kafka.common.errors.UnsupportedVersionException; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Field; import org.apache.kafka.common.protocol.types.Schema; import org.apache.kafka.common.protocol.types.Struct; @@ -205,6 +206,11 @@ protected Struct toStruct() { return requestStruct; } + @Override + protected Message data() { + return null; + } + @Override public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { Errors error = Errors.forException(e); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetsForLeaderEpochResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetsForLeaderEpochResponse.java index 3fe3cdc0da6b5..02c9e33010001 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/OffsetsForLeaderEpochResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetsForLeaderEpochResponse.java @@ -19,6 +19,7 @@ import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Field; import org.apache.kafka.common.protocol.types.Schema; import org.apache.kafka.common.protocol.types.Struct; @@ -170,6 +171,11 @@ protected Struct toStruct(short version) { return responseStruct; } + @Override + protected Message data() { + return null; + } + @Override public String toString() { StringBuilder bld = new StringBuilder(); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ProduceRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ProduceRequest.java index 932473cb869ea..dea8dead1a212 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ProduceRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ProduceRequest.java @@ -22,6 +22,7 @@ import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.CommonFields; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.ArrayOf; import org.apache.kafka.common.protocol.types.Field; import org.apache.kafka.common.protocol.types.Schema; @@ -304,6 +305,11 @@ public Struct toStruct() { return struct; } + @Override + protected Message data() { + return null; + } + @Override public String toString(boolean verbose) { // Use the same format as `Struct.toString()` diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ProduceResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ProduceResponse.java index a6df88002d582..ffc38d3ed7ea1 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ProduceResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ProduceResponse.java @@ -19,6 +19,7 @@ import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.ArrayOf; import org.apache.kafka.common.protocol.types.Field; import org.apache.kafka.common.protocol.types.Schema; @@ -292,6 +293,11 @@ protected Struct toStruct(short version) { return struct; } + @Override + protected Message data() { + return null; + } + public Map responses() { return this.responses; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/RenewDelegationTokenResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/RenewDelegationTokenResponse.java index 35cd6152632b6..b454db251eb0d 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/RenewDelegationTokenResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/RenewDelegationTokenResponse.java @@ -23,6 +23,7 @@ import org.apache.kafka.common.message.RenewDelegationTokenResponseData; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Struct; public class RenewDelegationTokenResponse extends AbstractResponse { @@ -51,6 +52,11 @@ protected Struct toStruct(short version) { return data.toStruct(version); } + @Override + protected Message data() { + return data; + } + @Override public int throttleTimeMs() { return data.throttleTimeMs(); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/SaslAuthenticateResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/SaslAuthenticateResponse.java index fc8832a2beefc..530d0170909b1 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/SaslAuthenticateResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/SaslAuthenticateResponse.java @@ -19,6 +19,7 @@ import org.apache.kafka.common.message.SaslAuthenticateResponseData; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; @@ -72,7 +73,12 @@ public Struct toStruct(short version) { return data.toStruct(version); } + @Override + protected Message data() { + return data; + } + public static SaslAuthenticateResponse parse(ByteBuffer buffer, short version) { return new SaslAuthenticateResponse(ApiKeys.SASL_AUTHENTICATE.parseResponse(version, buffer), version); } -} \ No newline at end of file +} diff --git a/clients/src/main/java/org/apache/kafka/common/requests/SaslHandshakeResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/SaslHandshakeResponse.java index 939c5e85f7bce..40364ece7ec7c 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/SaslHandshakeResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/SaslHandshakeResponse.java @@ -19,6 +19,7 @@ import org.apache.kafka.common.message.SaslHandshakeResponseData; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; @@ -61,6 +62,11 @@ public Struct toStruct(short version) { return data.toStruct(version); } + @Override + protected Message data() { + return data; + } + public List enabledMechanisms() { return data.mechanisms(); } @@ -68,4 +74,4 @@ public List enabledMechanisms() { public static SaslHandshakeResponse parse(ByteBuffer buffer, short version) { return new SaslHandshakeResponse(ApiKeys.SASL_HANDSHAKE.parseResponse(version, buffer), version); } -} \ No newline at end of file +} diff --git a/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaRequest.java index 425f66a161900..f93b001a7d9d8 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaRequest.java @@ -24,6 +24,7 @@ import org.apache.kafka.common.message.StopReplicaResponseData.StopReplicaPartitionError; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.utils.CollectionUtils; import org.apache.kafka.common.utils.FlattenedIterator; @@ -169,13 +170,13 @@ public static StopReplicaRequest parse(ByteBuffer buffer, short version) { return new StopReplicaRequest(ApiKeys.STOP_REPLICA.parseRequest(version, buffer), version); } - // Visible for testing - StopReplicaRequestData data() { - return data; - } - @Override protected Struct toStruct() { return data.toStruct(version()); } + + @Override + protected Message data() { + return data; + } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaResponse.java index 7d6c7a0742899..08d80ffc2ad94 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaResponse.java @@ -20,6 +20,7 @@ import org.apache.kafka.common.message.StopReplicaResponseData.StopReplicaPartitionError; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; @@ -71,6 +72,11 @@ protected Struct toStruct(short version) { return data.toStruct(version); } + @Override + protected Message data() { + return data; + } + @Override public String toString() { return data.toString(); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/SyncGroupRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/SyncGroupRequest.java index 48319d55cde5f..daedad7594758 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/SyncGroupRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/SyncGroupRequest.java @@ -21,6 +21,7 @@ import org.apache.kafka.common.message.SyncGroupResponseData; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; @@ -106,4 +107,9 @@ public static SyncGroupRequest parse(ByteBuffer buffer, short version) { protected Struct toStruct() { return data.toStruct(version()); } + + @Override + protected Message data() { + return data; + } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/SyncGroupResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/SyncGroupResponse.java index 8c9bad13d9d24..97b765aad186e 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/SyncGroupResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/SyncGroupResponse.java @@ -19,6 +19,7 @@ import org.apache.kafka.common.message.SyncGroupResponseData; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; @@ -61,6 +62,11 @@ protected Struct toStruct(short version) { return data.toStruct(version); } + @Override + protected Message data() { + return data; + } + public static SyncGroupResponse parse(ByteBuffer buffer, short version) { return new SyncGroupResponse(ApiKeys.SYNC_GROUP.parseResponse(version, buffer)); } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitRequest.java index 279fc9cb070a7..80bd715d5e6ab 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitRequest.java @@ -25,6 +25,8 @@ import org.apache.kafka.common.message.TxnOffsetCommitResponseData.TxnOffsetCommitResponseTopic; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.Message; +import org.apache.kafka.common.protocol.Writable; import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.record.RecordBatch; @@ -114,6 +116,11 @@ protected Struct toStruct() { return data.toStruct(version()); } + @Override + protected Message data() { + return data; + } + static List getErrorResponseTopics(List requestTopics, Errors e) { List responseTopicData = new ArrayList<>(); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitResponse.java index 256b9983671e0..80f7673e23a18 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitResponse.java @@ -22,6 +22,7 @@ import org.apache.kafka.common.message.TxnOffsetCommitResponseData.TxnOffsetCommitResponseTopic; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; @@ -81,6 +82,11 @@ protected Struct toStruct(short version) { return data.toStruct(version); } + @Override + protected Message data() { + return data; + } + @Override public int throttleTimeMs() { return data.throttleTimeMs(); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java index 90dbff3430256..5b5b514f920c4 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java @@ -26,6 +26,7 @@ import org.apache.kafka.common.network.ListenerName; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.security.auth.SecurityProtocol; import org.apache.kafka.common.utils.FlattenedIterator; @@ -212,8 +213,8 @@ protected Struct toStruct() { return data.toStruct(version()); } - // Visible for testing - UpdateMetadataRequestData data() { + @Override + protected Message data() { return data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataResponse.java index c7d803f560eb1..6a5a7f5cf13e6 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataResponse.java @@ -19,6 +19,7 @@ import org.apache.kafka.common.message.UpdateMetadataResponseData; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; @@ -53,4 +54,10 @@ public static UpdateMetadataResponse parse(ByteBuffer buffer, short version) { protected Struct toStruct(short version) { return data.toStruct(version); } + + @Override + protected Message data() { + return data; + } + } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/WriteTxnMarkersRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/WriteTxnMarkersRequest.java index 33f9bb586d9a5..a79b56484b953 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/WriteTxnMarkersRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/WriteTxnMarkersRequest.java @@ -19,6 +19,7 @@ import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.ArrayOf; import org.apache.kafka.common.protocol.types.Field; import org.apache.kafka.common.protocol.types.Schema; @@ -221,6 +222,11 @@ protected Struct toStruct() { return struct; } + @Override + protected Message data() { + return null; + } + @Override public WriteTxnMarkersResponse getErrorResponse(int throttleTimeMs, Throwable e) { Errors error = Errors.forException(e); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/WriteTxnMarkersResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/WriteTxnMarkersResponse.java index 92b5fc0b29dc2..480ca9ce83df4 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/WriteTxnMarkersResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/WriteTxnMarkersResponse.java @@ -19,6 +19,7 @@ import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.ArrayOf; import org.apache.kafka.common.protocol.types.Field; import org.apache.kafka.common.protocol.types.Schema; @@ -146,6 +147,11 @@ protected Struct toStruct(short version) { return struct; } + @Override + protected Message data() { + return null; + } + public Map errors(long producerId) { return errors.get(producerId); } diff --git a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java index 6aa68cd57dabd..c2cf9d7e6286a 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java +++ b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java @@ -385,7 +385,7 @@ private boolean sendSaslClientToken(byte[] serverToken, boolean isInitial) throw SaslAuthenticateRequestData data = new SaslAuthenticateRequestData() .setAuthBytes(tokenBuf.array()); SaslAuthenticateRequest request = new SaslAuthenticateRequest.Builder(data).build(saslAuthenticateVersion); - tokenBuf = request.serialize(nextRequestHeader(ApiKeys.SASL_AUTHENTICATE, saslAuthenticateVersion)); + tokenBuf = request.serializeWithHeader(nextRequestHeader(ApiKeys.SASL_AUTHENTICATE, saslAuthenticateVersion)); } send(new NetworkSend(node, tokenBuf)); return true; diff --git a/clients/src/test/java/org/apache/kafka/common/protocol/MessageTestUtil.java b/clients/src/test/java/org/apache/kafka/common/protocol/MessageTestUtil.java index a175ccd7c14e1..31ad94efc80d0 100644 --- a/clients/src/test/java/org/apache/kafka/common/protocol/MessageTestUtil.java +++ b/clients/src/test/java/org/apache/kafka/common/protocol/MessageTestUtil.java @@ -20,10 +20,6 @@ import java.nio.ByteBuffer; public final class MessageTestUtil { - public static int messageSize(Message message, short version) { - return message.size(new ObjectSerializationCache(), version); - } - public static ByteBuffer messageToByteBuffer(Message message, short version) { ObjectSerializationCache cache = new ObjectSerializationCache(); int size = message.size(cache, version); diff --git a/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrRequestTest.java index 6ad6197b6acae..8a7a1e9ec54e4 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrRequestTest.java @@ -105,7 +105,7 @@ public void testVersionLogic() { assertEquals(2, request.controllerEpoch()); assertEquals(3, request.brokerEpoch()); - ByteBuffer byteBuffer = MessageTestUtil.messageToByteBuffer(request.data(), request.version()); + ByteBuffer byteBuffer = request.serializeBody(); LeaderAndIsrRequest deserializedRequest = new LeaderAndIsrRequest(new LeaderAndIsrRequestData( new ByteBufferAccessor(byteBuffer), version), version); @@ -139,10 +139,7 @@ public void testTopicPartitionGroupingSizeReduction() { LeaderAndIsrRequest v2 = builder.build((short) 2); LeaderAndIsrRequest v1 = builder.build((short) 1); - int size2 = MessageTestUtil.messageSize(v2.data(), v2.version()); - int size1 = MessageTestUtil.messageSize(v1.data(), v1.version()); - - assertTrue("Expected v2 < v1: v2=" + size2 + ", v1=" + size1, size2 < size1); + assertTrue("Expected v2 < v1: v2=" + v2.sizeInBytes() + ", v1=" + v1.sizeInBytes(), v2.sizeInBytes() < v1.sizeInBytes()); } private Set iterableToSet(Iterable iterable) { diff --git a/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaRequestTest.java index a143ff3301a12..00c1973b81ea0 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaRequestTest.java @@ -31,8 +31,7 @@ public class StopReplicaRequestTest { public void testStopReplicaRequestNormalization() { Set tps = TestUtils.generateRandomTopicPartitions(10, 10); StopReplicaRequest.Builder builder = new StopReplicaRequest.Builder((short) 5, 0, 0, 0, false, tps); - assertTrue(MessageTestUtil.messageSize(builder.build((short) 1).data(), (short) 1) < - MessageTestUtil.messageSize(builder.build((short) 0).data(), (short) 0)); + assertTrue(builder.build((short) 1).sizeInBytes() < builder.build((short) 0).sizeInBytes()); } } diff --git a/clients/src/test/java/org/apache/kafka/common/requests/UpdateMetadataRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/UpdateMetadataRequestTest.java index a5d0d99d3d61d..dcfba42b1dfd5 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/UpdateMetadataRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/UpdateMetadataRequestTest.java @@ -133,7 +133,7 @@ public void testVersionLogic() { assertEquals(2, request.controllerEpoch()); assertEquals(3, request.brokerEpoch()); - ByteBuffer byteBuffer = MessageTestUtil.messageToByteBuffer(request.data(), request.version()); + ByteBuffer byteBuffer = request.serializeBody(); UpdateMetadataRequest deserializedRequest = new UpdateMetadataRequest(new UpdateMetadataRequestData( new ByteBufferAccessor(byteBuffer), version), version); @@ -179,8 +179,7 @@ public void testTopicPartitionGroupingSizeReduction() { UpdateMetadataRequest.Builder builder = new UpdateMetadataRequest.Builder((short) 5, 0, 0, 0, partitionStates, Collections.emptyList()); - assertTrue(MessageTestUtil.messageSize(builder.build((short) 5).data(), (short) 5) < - MessageTestUtil.messageSize(builder.build((short) 4).data(), (short) 4)); + assertTrue(builder.build((short) 5).sizeInBytes() < builder.build((short) 4).sizeInBytes()); } private Set iterableToSet(Iterable iterable) { diff --git a/core/src/test/scala/unit/kafka/network/SocketServerTest.scala b/core/src/test/scala/unit/kafka/network/SocketServerTest.scala index 2cf637a263605..f4978a32a6b0b 100644 --- a/core/src/test/scala/unit/kafka/network/SocketServerTest.scala +++ b/core/src/test/scala/unit/kafka/network/SocketServerTest.scala @@ -129,7 +129,7 @@ class SocketServerTest { } def processRequest(channel: RequestChannel, request: RequestChannel.Request): Unit = { - val byteBuffer = request.body[AbstractRequest].serialize(request.header) + val byteBuffer = request.body[AbstractRequest].serializeWithHeader(request.header) byteBuffer.rewind() val send = new NetworkSend(request.context.connectionId, byteBuffer) @@ -171,7 +171,7 @@ class SocketServerTest { val emptyRequest = ProduceRequest.Builder.forCurrentMagic(ack, ackTimeoutMs, new HashMap[TopicPartition, MemoryRecords]()).build() val emptyHeader = new RequestHeader(ApiKeys.PRODUCE, emptyRequest.version, clientId, correlationId) - val byteBuffer = emptyRequest.serialize(emptyHeader) + val byteBuffer = emptyRequest.serializeWithHeader(emptyHeader) byteBuffer.rewind() val serializedBytes = new Array[Byte](byteBuffer.remaining) @@ -475,7 +475,7 @@ class SocketServerTest { // Mimic a primitive request handler that fetches the request from RequestChannel and place a response with a // throttled channel. val request = receiveRequest(server.dataPlaneRequestChannel) - val byteBuffer = request.body[AbstractRequest].serialize(request.header) + val byteBuffer = request.body[AbstractRequest].serializeWithHeader(request.header) val send = new NetworkSend(request.context.connectionId, byteBuffer) def channelThrottlingCallback(response: RequestChannel.Response): Unit = { server.dataPlaneRequestChannel.sendResponse(response) @@ -692,7 +692,7 @@ class SocketServerTest { new HashMap[TopicPartition, MemoryRecords]()).build() val emptyHeader = new RequestHeader(ApiKeys.PRODUCE, emptyRequest.version, clientId, correlationId) - val byteBuffer = emptyRequest.serialize(emptyHeader) + val byteBuffer = emptyRequest.serializeWithHeader(emptyHeader) byteBuffer.rewind() val serializedBytes = new Array[Byte](byteBuffer.remaining) byteBuffer.get(serializedBytes) diff --git a/core/src/test/scala/unit/kafka/server/BaseRequestTest.scala b/core/src/test/scala/unit/kafka/server/BaseRequestTest.scala index 5d00d540b149a..0ccafd6effed8 100644 --- a/core/src/test/scala/unit/kafka/server/BaseRequestTest.scala +++ b/core/src/test/scala/unit/kafka/server/BaseRequestTest.scala @@ -129,7 +129,7 @@ abstract class BaseRequestTest extends IntegrationTestHarness { */ def send(request: AbstractRequest, apiKey: ApiKeys, socket: Socket, apiVersion: Short): Unit = { val header = nextRequestHeader(apiKey, apiVersion) - val serializedBytes = request.serialize(header).array + val serializedBytes = request.serializeWithHeader(header).array sendRequest(socket, serializedBytes) } @@ -159,8 +159,8 @@ abstract class BaseRequestTest extends IntegrationTestHarness { val apiKey = requestBuilder.apiKey val request = requestBuilder.build() val header = new RequestHeader(apiKey, request.version, clientId, correlationId) - val response = requestAndReceive(socket, request.serialize(header).array) - val responseBuffer = skipResponseHeader(response, apiKey.responseHeaderVersion(request.version())) + val response = requestAndReceive(socket, request.serializeWithHeader(header).array) + val responseBuffer = skipResponseHeader(response, apiKey.responseHeaderVersion(request.version)) apiKey.parseResponse(request.version, responseBuffer) } diff --git a/core/src/test/scala/unit/kafka/server/ClientQuotaManagerTest.scala b/core/src/test/scala/unit/kafka/server/ClientQuotaManagerTest.scala index 828fb4841a271..a8dfbded0be3e 100644 --- a/core/src/test/scala/unit/kafka/server/ClientQuotaManagerTest.scala +++ b/core/src/test/scala/unit/kafka/server/ClientQuotaManagerTest.scala @@ -59,7 +59,7 @@ class ClientQuotaManagerTest { listenerName: ListenerName = ListenerName.forSecurityProtocol(SecurityProtocol.PLAINTEXT)): (T, RequestChannel.Request) = { val request = builder.build() - val buffer = request.serialize(new RequestHeader(builder.apiKey, request.version, "", 0)) + val buffer = request.serializeWithHeader(new RequestHeader(builder.apiKey, request.version, "", 0)) val requestChannelMetrics: RequestChannel.Metrics = EasyMock.createNiceMock(classOf[RequestChannel.Metrics]) // read the header from the buffer first so that the body can be read next from the Request constructor diff --git a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala index bf1a8c487b1cc..a938c990c8dba 100644 --- a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala +++ b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala @@ -798,7 +798,7 @@ class KafkaApisTest { listenerName: ListenerName = ListenerName.forSecurityProtocol(SecurityProtocol.PLAINTEXT)): (T, RequestChannel.Request) = { val request = builder.build() - val buffer = request.serialize(new RequestHeader(builder.apiKey, request.version, "", 0)) + val buffer = request.serializeWithHeader(new RequestHeader(builder.apiKey, request.version, "", 0)) // read the header from the buffer first so that the body can be read next from the Request constructor val header = RequestHeader.parse(buffer) diff --git a/core/src/test/scala/unit/kafka/server/ThrottledChannelExpirationTest.scala b/core/src/test/scala/unit/kafka/server/ThrottledChannelExpirationTest.scala index 15bbcc6e2b906..e201cbea15fa5 100644 --- a/core/src/test/scala/unit/kafka/server/ThrottledChannelExpirationTest.scala +++ b/core/src/test/scala/unit/kafka/server/ThrottledChannelExpirationTest.scala @@ -49,7 +49,7 @@ class ThrottledChannelExpirationTest { listenerName: ListenerName = ListenerName.forSecurityProtocol(SecurityProtocol.PLAINTEXT)): (T, RequestChannel.Request) = { val request = builder.build() - val buffer = request.serialize(new RequestHeader(builder.apiKey, request.version, "", 0)) + val buffer = request.serializeWithHeader(new RequestHeader(builder.apiKey, request.version, "", 0)) val requestChannelMetrics: RequestChannel.Metrics = EasyMock.createNiceMock(classOf[RequestChannel.Metrics]) // read the header from the buffer first so that the body can be read next from the Request constructor From bc333a71976c838a57b7d2c122a4c02cb78520f8 Mon Sep 17 00:00:00 2001 From: Ismael Juma Date: Sat, 28 Sep 2019 16:06:30 -0700 Subject: [PATCH 02/51] Skip redundant `toStruct` when serializing responses that rely on auto generated protocol --- .../common/requests/AbstractRequest.java | 5 ++-- .../common/requests/AbstractResponse.java | 29 ++++++++++++------- .../common/requests/OffsetDeleteRequest.java | 1 - .../requests/TxnOffsetCommitRequest.java | 1 - .../kafka/clients/NetworkClientTest.java | 2 +- .../consumer/internals/FetcherTest.java | 4 +-- .../producer/internals/SenderTest.java | 4 +-- .../requests/LeaderAndIsrRequestTest.java | 1 - .../common/requests/RequestResponseTest.java | 2 +- .../requests/StopReplicaRequestTest.java | 1 - .../requests/UpdateMetadataRequestTest.java | 1 - 11 files changed, 26 insertions(+), 25 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java index 6ad7bbfbf7f90..deddada239eb8 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java @@ -24,7 +24,6 @@ import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.ObjectSerializationCache; -import org.apache.kafka.common.protocol.Writable; import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; @@ -105,10 +104,10 @@ public Send toSend(String destination, RequestHeader header) { */ public final ByteBuffer serializeWithHeader(RequestHeader header) { Message data = data(); + Struct headerStruct = header.toStruct(); if (data == null) - return serialize(header.toStruct(), toStruct()); + return serialize(headerStruct, toStruct()); - Struct headerStruct = header.toStruct(); ObjectSerializationCache serializationCache = new ObjectSerializationCache(); ByteBuffer buffer = ByteBuffer.allocate(headerStruct.sizeOf() + data.size(serializationCache, version)); headerStruct.writeTo(buffer); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java index ea55c1ba274b6..2ad65a6b24332 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java @@ -19,8 +19,10 @@ import org.apache.kafka.common.network.NetworkSend; import org.apache.kafka.common.network.Send; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.Message; +import org.apache.kafka.common.protocol.ObjectSerializationCache; import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; @@ -32,24 +34,29 @@ public abstract class AbstractResponse extends AbstractRequestResponse { public static final int DEFAULT_THROTTLE_TIME = 0; - protected Send toSend(String destination, ResponseHeader header, short apiVersion) { - return new NetworkSend(destination, serialize(header.toStruct(), toStruct(apiVersion))); + protected Send toSend(String destination, ResponseHeader header, short version) { + return new NetworkSend(destination, serializeWithHeader(header, version)); } /** * Visible for testing, typically {@link #toSend(String, ResponseHeader, short)} should be used instead. */ - public ByteBuffer serialize(ApiKeys apiKey, int correlationId) { - return serialize(apiKey, apiKey.latestVersion(), correlationId); + public ByteBuffer serializeWithHeader(ApiKeys apiKey, short version, int correlationId) { + return serializeWithHeader(new ResponseHeader(correlationId, apiKey.responseHeaderVersion(version)), version); } - /** - * Visible for testing, typically {@link #toSend(String, ResponseHeader, short)} should be used instead. - */ - public ByteBuffer serialize(ApiKeys apiKey, short version, int correlationId) { - ResponseHeader header = - new ResponseHeader(correlationId, apiKey.responseHeaderVersion(version)); - return serialize(header.toStruct(), toStruct(version)); + private ByteBuffer serializeWithHeader(ResponseHeader header, short version) { + Struct headerStruct = header.toStruct(); + Message data = data(); + if (data == null) + return serialize(headerStruct, toStruct(version)); + + ObjectSerializationCache serializationCache = new ObjectSerializationCache(); + ByteBuffer buffer = ByteBuffer.allocate(headerStruct.sizeOf() + data.size(serializationCache, version)); + headerStruct.writeTo(buffer); + data.write(new ByteBufferAccessor(buffer), serializationCache, version); + buffer.rewind(); + return buffer; } protected abstract Message data(); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetDeleteRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetDeleteRequest.java index 9dc4cbbad6e84..88827ac5a95e7 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/OffsetDeleteRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetDeleteRequest.java @@ -21,7 +21,6 @@ import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.Message; -import org.apache.kafka.common.protocol.Writable; import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; diff --git a/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitRequest.java index 80bd715d5e6ab..b8292946fda6f 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitRequest.java @@ -26,7 +26,6 @@ import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.Message; -import org.apache.kafka.common.protocol.Writable; import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.record.RecordBatch; diff --git a/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java b/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java index e198f6c4a6be0..261271fe9fcf1 100644 --- a/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java @@ -194,7 +194,7 @@ private void checkSimpleRequestResponse(NetworkClient networkClient) { } private void delayedApiVersionsResponse(int correlationId, short version, ApiVersionsResponse response) { - ByteBuffer buffer = response.serialize(ApiKeys.API_VERSIONS, version, correlationId); + ByteBuffer buffer = response.serializeWithHeader(ApiKeys.API_VERSIONS, version, correlationId); selector.delayedReceive(new DelayedReceive(node.idString(), new NetworkReceive(node.idString(), buffer))); } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java index ff0afe92f7106..102371efd6fc6 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java @@ -1971,7 +1971,7 @@ public void testQuotaMetrics() { ByteBuffer buffer = ApiVersionsResponse. createApiVersionsResponse(400, RecordBatch.CURRENT_MAGIC_VALUE). - serialize(ApiKeys.API_VERSIONS, 0); + serializeWithHeader(ApiKeys.API_VERSIONS, ApiKeys.API_VERSIONS.latestVersion(), 0); selector.delayedReceive(new DelayedReceive(node.idString(), new NetworkReceive(node.idString(), buffer))); while (!client.ready(node, time.milliseconds())) { client.poll(1, time.milliseconds()); @@ -1988,7 +1988,7 @@ public void testQuotaMetrics() { client.send(request, time.milliseconds()); client.poll(1, time.milliseconds()); FetchResponse response = fullFetchResponse(tp0, nextRecords, Errors.NONE, i, throttleTimeMs); - buffer = response.serialize(ApiKeys.FETCH, request.correlationId()); + buffer = response.serializeWithHeader(ApiKeys.FETCH, ApiKeys.FETCH.latestVersion(), request.correlationId()); selector.completeReceive(new NetworkReceive(node.idString(), buffer)); client.poll(1, time.milliseconds()); // If a throttled response is received, advance the time to ensure progress. diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java index 71180bdd7f7a7..6b163200d6b6c 100644 --- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java @@ -275,7 +275,7 @@ public void testQuotaMetrics() throws Exception { time, true, new ApiVersions(), throttleTimeSensor, logContext); ByteBuffer buffer = ApiVersionsResponse.createApiVersionsResponse(400, RecordBatch.CURRENT_MAGIC_VALUE). - serialize(ApiKeys.API_VERSIONS, 0); + serializeWithHeader(ApiKeys.API_VERSIONS, ApiKeys.API_VERSIONS.latestVersion(), 0); selector.delayedReceive(new DelayedReceive(node.idString(), new NetworkReceive(node.idString(), buffer))); while (!client.ready(node, time.milliseconds())) { client.poll(1, time.milliseconds()); @@ -292,7 +292,7 @@ public void testQuotaMetrics() throws Exception { client.send(request, time.milliseconds()); client.poll(1, time.milliseconds()); ProduceResponse response = produceResponse(tp0, i, Errors.NONE, throttleTimeMs); - buffer = response.serialize(ApiKeys.PRODUCE, request.correlationId()); + buffer = response.serializeWithHeader(ApiKeys.PRODUCE, ApiKeys.PRODUCE.latestVersion(), request.correlationId()); selector.completeReceive(new NetworkReceive(node.idString(), buffer)); client.poll(1, time.milliseconds()); // If a throttled response is received, advance the time to ensure progress. diff --git a/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrRequestTest.java index 8a7a1e9ec54e4..41fde0d734c4c 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrRequestTest.java @@ -22,7 +22,6 @@ import org.apache.kafka.common.message.LeaderAndIsrRequestData.LeaderAndIsrLiveLeader; import org.apache.kafka.common.message.LeaderAndIsrRequestData.LeaderAndIsrPartitionState; import org.apache.kafka.common.protocol.ByteBufferAccessor; -import org.apache.kafka.common.protocol.MessageTestUtil; import org.apache.kafka.test.TestUtils; import org.junit.Test; diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java index 32b0a00679eef..05cb16724ef60 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java @@ -574,7 +574,7 @@ public void produceResponseV5Test() { ProduceResponse v5Response = new ProduceResponse(responseData, 10); short version = 5; - ByteBuffer buffer = v5Response.serialize(ApiKeys.PRODUCE, version, 0); + ByteBuffer buffer = v5Response.serializeWithHeader(ApiKeys.PRODUCE, version, 0); buffer.rewind(); ResponseHeader.parse(buffer, ApiKeys.PRODUCE.responseHeaderVersion(version)); // throw away. diff --git a/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaRequestTest.java index 00c1973b81ea0..3c6fd782c446b 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaRequestTest.java @@ -17,7 +17,6 @@ package org.apache.kafka.common.requests; import org.apache.kafka.common.TopicPartition; -import org.apache.kafka.common.protocol.MessageTestUtil; import org.apache.kafka.test.TestUtils; import org.junit.Test; diff --git a/clients/src/test/java/org/apache/kafka/common/requests/UpdateMetadataRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/UpdateMetadataRequestTest.java index dcfba42b1dfd5..969e61e482278 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/UpdateMetadataRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/UpdateMetadataRequestTest.java @@ -23,7 +23,6 @@ import org.apache.kafka.common.message.UpdateMetadataRequestData.UpdateMetadataPartitionState; import org.apache.kafka.common.network.ListenerName; import org.apache.kafka.common.protocol.ByteBufferAccessor; -import org.apache.kafka.common.protocol.MessageTestUtil; import org.apache.kafka.common.security.auth.SecurityProtocol; import org.apache.kafka.test.TestUtils; import org.junit.Test; From 10b01b964a6ead66d9421ee96c026755f1f84fd8 Mon Sep 17 00:00:00 2001 From: Ismael Juma Date: Sun, 29 Sep 2019 20:31:02 -0700 Subject: [PATCH 03/51] Add ignorable to TopicAuthorizedOperation and ClusterAuthorizedOperations --- .../src/main/resources/common/message/MetadataResponse.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clients/src/main/resources/common/message/MetadataResponse.json b/clients/src/main/resources/common/message/MetadataResponse.json index ce232b1233110..01eb8e0e9634b 100644 --- a/clients/src/main/resources/common/message/MetadataResponse.json +++ b/clients/src/main/resources/common/message/MetadataResponse.json @@ -82,9 +82,9 @@ "about": "The set of offline replicas of this partition." } ]}, { "name": "TopicAuthorizedOperations", "type": "int32", "versions": "8+", "default": "-2147483648", - "about": "32-bit bitfield to represent authorized operations for this topic." } + "ignorable": "true", "about": "32-bit bitfield to represent authorized operations for this topic." } ]}, { "name": "ClusterAuthorizedOperations", "type": "int32", "versions": "8+", "default": "-2147483648", - "about": "32-bit bitfield to represent authorized operations for this cluster." } + "ignorable": "true", "about": "32-bit bitfield to represent authorized operations for this cluster." } ] } From 7f66cdafaff99707dafc2dec4fc2c6884609f056 Mon Sep 17 00:00:00 2001 From: Ismael Juma Date: Sun, 29 Sep 2019 20:34:34 -0700 Subject: [PATCH 04/51] More ignorable --- clients/src/main/resources/common/message/MetadataRequest.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clients/src/main/resources/common/message/MetadataRequest.json b/clients/src/main/resources/common/message/MetadataRequest.json index 6129d427b7eef..9ee2bc2541624 100644 --- a/clients/src/main/resources/common/message/MetadataRequest.json +++ b/clients/src/main/resources/common/message/MetadataRequest.json @@ -36,7 +36,7 @@ { "name": "Name", "type": "string", "versions": "0+", "entityType": "topicName", "about": "The topic name." } ]}, - { "name": "AllowAutoTopicCreation", "type": "bool", "versions": "4+", "default": "true", "ignorable": false, + { "name": "AllowAutoTopicCreation", "type": "bool", "versions": "4+", "default": "true", "ignorable": true, "about": "If this is true, the broker may auto-create topics that we requested which do not already exist, if it is configured to do so." }, { "name": "IncludeClusterAuthorizedOperations", "type": "bool", "versions": "8+", "about": "Whether to include cluster authorized operations." }, From 8be8cef44745e0ae17f33ad8f99bf17cf8ae6358 Mon Sep 17 00:00:00 2001 From: Ismael Juma Date: Sun, 29 Sep 2019 20:34:43 -0700 Subject: [PATCH 05/51] Add missing whitespace --- .../java/org/apache/kafka/common/requests/MetadataRequest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clients/src/main/java/org/apache/kafka/common/requests/MetadataRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/MetadataRequest.java index cbfca4bbdf6f2..a4fd7b6697885 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/MetadataRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/MetadataRequest.java @@ -125,7 +125,7 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { Errors error = Errors.forException(e); MetadataResponseData responseData = new MetadataResponseData(); if (topics() != null) { - for (String topic :topics()) + for (String topic : topics()) responseData.topics().add(new MetadataResponseData.MetadataResponseTopic() .setName(topic) .setErrorCode(error.code()) From a84d3d875fb630bf6e85f5d842aeb693c67082ad Mon Sep 17 00:00:00 2001 From: Ismael Juma Date: Mon, 30 Sep 2019 05:34:28 -0700 Subject: [PATCH 06/51] Move `message()` to `AbstractRequestResponse` and document it --- .../apache/kafka/common/requests/AbstractRequest.java | 2 -- .../kafka/common/requests/AbstractRequestResponse.java | 9 +++++++++ .../apache/kafka/common/requests/AbstractResponse.java | 2 -- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java index deddada239eb8..08c11ecc5b85c 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java @@ -138,8 +138,6 @@ int sizeInBytes() { return data().size(new ObjectSerializationCache(), version); } - protected abstract Message data(); - protected abstract Struct toStruct(); public String toString(boolean verbose) { diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequestResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequestResponse.java index 0ba373d6fea7a..5570bf0656add 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequestResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequestResponse.java @@ -16,11 +16,20 @@ */ package org.apache.kafka.common.requests; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; public abstract class AbstractRequestResponse { + + /** + * Return the auto-generated `Message` instance if this request/response relies on one for + * serialization/deserialization. If this class has not yet been updated to rely on the auto-generated protocol + * classes, return `null`. + */ + protected abstract Message data(); + /** * Visible for testing. */ diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java index 2ad65a6b24332..e31f3996f6039 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java @@ -59,8 +59,6 @@ private ByteBuffer serializeWithHeader(ResponseHeader header, short version) { return buffer; } - protected abstract Message data(); - public abstract Map errorCounts(); protected Map errorCounts(Errors error) { From 17af66b114afa7a948e61abd39baea26160ab1e6 Mon Sep 17 00:00:00 2001 From: Ismael Juma Date: Mon, 30 Sep 2019 05:44:51 -0700 Subject: [PATCH 07/51] Use `toSend` instead of `serializeWithHeader` in `SaslClientAuthenticator`. --- .../security/authenticator/SaslClientAuthenticator.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java index c2cf9d7e6286a..dfc4f6aa08f0c 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java +++ b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java @@ -381,13 +381,16 @@ private boolean sendSaslClientToken(byte[] serverToken, boolean isInitial) throw byte[] saslToken = createSaslToken(serverToken, isInitial); if (saslToken != null) { ByteBuffer tokenBuf = ByteBuffer.wrap(saslToken); - if (saslAuthenticateVersion != DISABLE_KAFKA_SASL_AUTHENTICATE_HEADER) { + Send send; + if (saslAuthenticateVersion == DISABLE_KAFKA_SASL_AUTHENTICATE_HEADER) { + send = new NetworkSend(node, tokenBuf); + } else { SaslAuthenticateRequestData data = new SaslAuthenticateRequestData() .setAuthBytes(tokenBuf.array()); SaslAuthenticateRequest request = new SaslAuthenticateRequest.Builder(data).build(saslAuthenticateVersion); - tokenBuf = request.serializeWithHeader(nextRequestHeader(ApiKeys.SASL_AUTHENTICATE, saslAuthenticateVersion)); + send = request.toSend(node, nextRequestHeader(ApiKeys.SASL_AUTHENTICATE, saslAuthenticateVersion)); } - send(new NetworkSend(node, tokenBuf)); + send(send); return true; } } From 9a93b0bb2e717e3a592f0ccdcd43704c939631ba Mon Sep 17 00:00:00 2001 From: Ismael Juma Date: Tue, 1 Oct 2019 08:32:24 -0700 Subject: [PATCH 08/51] Don't rely on `Struct` directly when parsing requests in the broker --- .../common/requests/AbstractRequest.java | 103 +++++++++--------- .../DescribeDelegationTokenRequest.java | 9 +- .../kafka/common/requests/RequestContext.java | 4 +- .../OffsetsForLeaderEpochRequestTest.java | 5 +- .../common/requests/RequestResponseTest.java | 3 +- 5 files changed, 66 insertions(+), 58 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java index 08c11ecc5b85c..168eb0498b6a7 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java @@ -176,104 +176,109 @@ public Map errorCounts(Throwable e) { /** * Factory method for getting a request object based on ApiKey ID and a version */ - public static AbstractRequest parseRequest(ApiKeys apiKey, short apiVersion, Struct struct) { + public static RequestAndSize parseRequest(ApiKeys apiKey, short apiVersion, ByteBuffer buffer) { + int bufferSize = buffer.position() - buffer.limit(); + return new RequestAndSize(doParseRequest(apiKey, apiVersion, buffer), bufferSize); + } + + private static AbstractRequest doParseRequest(ApiKeys apiKey, short apiVersion, ByteBuffer buffer) { switch (apiKey) { case PRODUCE: - return new ProduceRequest(struct, apiVersion); + return ProduceRequest.parse(buffer, apiVersion); case FETCH: - return new FetchRequest(struct, apiVersion); + return FetchRequest.parse(buffer, apiVersion); case LIST_OFFSETS: - return new ListOffsetRequest(struct, apiVersion); + return ListOffsetRequest.parse(buffer, apiVersion); case METADATA: - return new MetadataRequest(struct, apiVersion); + return MetadataRequest.parse(buffer, apiVersion); case OFFSET_COMMIT: - return new OffsetCommitRequest(struct, apiVersion); + return OffsetCommitRequest.parse(buffer, apiVersion); case OFFSET_FETCH: - return new OffsetFetchRequest(struct, apiVersion); + return OffsetFetchRequest.parse(buffer, apiVersion); case FIND_COORDINATOR: - return new FindCoordinatorRequest(struct, apiVersion); + return FindCoordinatorRequest.parse(buffer, apiVersion); case JOIN_GROUP: - return new JoinGroupRequest(struct, apiVersion); + return JoinGroupRequest.parse(buffer, apiVersion); case HEARTBEAT: - return new HeartbeatRequest(struct, apiVersion); + return HeartbeatRequest.parse(buffer, apiVersion); case LEAVE_GROUP: - return new LeaveGroupRequest(struct, apiVersion); + return LeaveGroupRequest.parse(buffer, apiVersion); case SYNC_GROUP: - return new SyncGroupRequest(struct, apiVersion); + return SyncGroupRequest.parse(buffer, apiVersion); case STOP_REPLICA: - return new StopReplicaRequest(struct, apiVersion); + return StopReplicaRequest.parse(buffer, apiVersion); case CONTROLLED_SHUTDOWN: - return new ControlledShutdownRequest(struct, apiVersion); + return ControlledShutdownRequest.parse(buffer, apiVersion); case UPDATE_METADATA: - return new UpdateMetadataRequest(struct, apiVersion); + return UpdateMetadataRequest.parse(buffer, apiVersion); case LEADER_AND_ISR: - return new LeaderAndIsrRequest(struct, apiVersion); + return LeaderAndIsrRequest.parse(buffer, apiVersion); case DESCRIBE_GROUPS: - return new DescribeGroupsRequest(struct, apiVersion); + return DescribeGroupsRequest.parse(buffer, apiVersion); case LIST_GROUPS: - return new ListGroupsRequest(struct, apiVersion); + return ListGroupsRequest.parse(buffer, apiVersion); case SASL_HANDSHAKE: - return new SaslHandshakeRequest(struct, apiVersion); + return SaslHandshakeRequest.parse(buffer, apiVersion); case API_VERSIONS: - return new ApiVersionsRequest(struct, apiVersion); + return ApiVersionsRequest.parse(buffer, apiVersion); case CREATE_TOPICS: - return new CreateTopicsRequest(struct, apiVersion); + return CreateTopicsRequest.parse(buffer, apiVersion); case DELETE_TOPICS: - return new DeleteTopicsRequest(struct, apiVersion); + return DeleteTopicsRequest.parse(buffer, apiVersion); case DELETE_RECORDS: - return new DeleteRecordsRequest(struct, apiVersion); + return DeleteRecordsRequest.parse(buffer, apiVersion); case INIT_PRODUCER_ID: - return new InitProducerIdRequest(struct, apiVersion); + return InitProducerIdRequest.parse(buffer, apiVersion); case OFFSET_FOR_LEADER_EPOCH: - return new OffsetsForLeaderEpochRequest(struct, apiVersion); + return OffsetsForLeaderEpochRequest.parse(buffer, apiVersion); case ADD_PARTITIONS_TO_TXN: - return new AddPartitionsToTxnRequest(struct, apiVersion); + return AddPartitionsToTxnRequest.parse(buffer, apiVersion); case ADD_OFFSETS_TO_TXN: - return new AddOffsetsToTxnRequest(struct, apiVersion); + return AddOffsetsToTxnRequest.parse(buffer, apiVersion); case END_TXN: - return new EndTxnRequest(struct, apiVersion); + return EndTxnRequest.parse(buffer, apiVersion); case WRITE_TXN_MARKERS: - return new WriteTxnMarkersRequest(struct, apiVersion); + return WriteTxnMarkersRequest.parse(buffer, apiVersion); case TXN_OFFSET_COMMIT: - return new TxnOffsetCommitRequest(struct, apiVersion); + return TxnOffsetCommitRequest.parse(buffer, apiVersion); case DESCRIBE_ACLS: - return new DescribeAclsRequest(struct, apiVersion); + return DescribeAclsRequest.parse(buffer, apiVersion); case CREATE_ACLS: - return new CreateAclsRequest(struct, apiVersion); + return CreateAclsRequest.parse(buffer, apiVersion); case DELETE_ACLS: - return new DeleteAclsRequest(struct, apiVersion); + return DeleteAclsRequest.parse(buffer, apiVersion); case DESCRIBE_CONFIGS: - return new DescribeConfigsRequest(struct, apiVersion); + return DescribeConfigsRequest.parse(buffer, apiVersion); case ALTER_CONFIGS: - return new AlterConfigsRequest(struct, apiVersion); + return AlterConfigsRequest.parse(buffer, apiVersion); case ALTER_REPLICA_LOG_DIRS: - return new AlterReplicaLogDirsRequest(struct, apiVersion); + return AlterReplicaLogDirsRequest.parse(buffer, apiVersion); case DESCRIBE_LOG_DIRS: - return new DescribeLogDirsRequest(struct, apiVersion); + return DescribeLogDirsRequest.parse(buffer, apiVersion); case SASL_AUTHENTICATE: - return new SaslAuthenticateRequest(struct, apiVersion); + return SaslAuthenticateRequest.parse(buffer, apiVersion); case CREATE_PARTITIONS: - return new CreatePartitionsRequest(struct, apiVersion); + return CreatePartitionsRequest.parse(buffer, apiVersion); case CREATE_DELEGATION_TOKEN: - return new CreateDelegationTokenRequest(struct, apiVersion); + return CreateDelegationTokenRequest.parse(buffer, apiVersion); case RENEW_DELEGATION_TOKEN: - return new RenewDelegationTokenRequest(struct, apiVersion); + return RenewDelegationTokenRequest.parse(buffer, apiVersion); case EXPIRE_DELEGATION_TOKEN: - return new ExpireDelegationTokenRequest(struct, apiVersion); + return ExpireDelegationTokenRequest.parse(buffer, apiVersion); case DESCRIBE_DELEGATION_TOKEN: - return new DescribeDelegationTokenRequest(struct, apiVersion); + return DescribeDelegationTokenRequest.parse(buffer, apiVersion); case DELETE_GROUPS: - return new DeleteGroupsRequest(struct, apiVersion); + return DeleteGroupsRequest.parse(buffer, apiVersion); case ELECT_LEADERS: - return new ElectLeadersRequest(struct, apiVersion); + return ElectLeadersRequest.parse(buffer, apiVersion); case INCREMENTAL_ALTER_CONFIGS: - return new IncrementalAlterConfigsRequest(struct, apiVersion); + return IncrementalAlterConfigsRequest.parse(buffer, apiVersion); case ALTER_PARTITION_REASSIGNMENTS: - return new AlterPartitionReassignmentsRequest(struct, apiVersion); + return AlterPartitionReassignmentsRequest.parse(buffer, apiVersion); case LIST_PARTITION_REASSIGNMENTS: - return new ListPartitionReassignmentsRequest(struct, apiVersion); + return ListPartitionReassignmentsRequest.parse(buffer, apiVersion); case OFFSET_DELETE: - return new OffsetDeleteRequest(struct, apiVersion); + return OffsetDeleteRequest.parse(buffer, apiVersion); default: throw new AssertionError(String.format("ApiKey %s is not currently handled in `parseRequest`, the " + "code should be updated to do so.", apiKey)); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenRequest.java index 7db3209d621fb..1ad5f73e93cf8 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenRequest.java @@ -22,13 +22,12 @@ import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.security.auth.KafkaPrincipal; +import java.nio.ByteBuffer; import java.util.List; import java.util.stream.Collectors; public class DescribeDelegationTokenRequest extends AbstractRequest { - private final DescribeDelegationTokenRequestData data; - public static class Builder extends AbstractRequest.Builder { private final DescribeDelegationTokenRequestData data; @@ -54,6 +53,8 @@ public String toString() { } } + private final DescribeDelegationTokenRequestData data; + public DescribeDelegationTokenRequest(Struct struct, short version) { super(ApiKeys.DESCRIBE_DELEGATION_TOKEN, version); this.data = new DescribeDelegationTokenRequestData(struct, version); @@ -81,4 +82,8 @@ public boolean ownersListEmpty() { public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { return new DescribeDelegationTokenResponse(throttleTimeMs, Errors.forException(e)); } + + public static DescribeDelegationTokenRequest parse(ByteBuffer buffer, short version) { + return new DescribeDelegationTokenRequest(ApiKeys.EXPIRE_DELEGATION_TOKEN.parseRequest(version, buffer), version); + } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/RequestContext.java b/clients/src/main/java/org/apache/kafka/common/requests/RequestContext.java index 8e21f8e40be5e..1c958ba5c848b 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/RequestContext.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/RequestContext.java @@ -62,9 +62,7 @@ public RequestAndSize parseRequest(ByteBuffer buffer) { ApiKeys apiKey = header.apiKey(); try { short apiVersion = header.apiVersion(); - Struct struct = apiKey.parseRequest(apiVersion, buffer); - AbstractRequest body = AbstractRequest.parseRequest(apiKey, apiVersion, struct); - return new RequestAndSize(body, struct.sizeOf()); + return AbstractRequest.parseRequest(apiKey, apiVersion, buffer); } catch (Throwable ex) { throw new InvalidRequestException("Error getting request for apiKey: " + apiKey + ", apiVersion: " + header.apiVersion() + diff --git a/clients/src/test/java/org/apache/kafka/common/requests/OffsetsForLeaderEpochRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/OffsetsForLeaderEpochRequestTest.java index 604f88f04aeaa..ec0da45a61c61 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/OffsetsForLeaderEpochRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/OffsetsForLeaderEpochRequestTest.java @@ -48,8 +48,7 @@ public void testDefaultReplicaId() { OffsetsForLeaderEpochRequest.Builder builder = OffsetsForLeaderEpochRequest.Builder.forFollower( version, Collections.emptyMap(), replicaId); OffsetsForLeaderEpochRequest request = builder.build(); - OffsetsForLeaderEpochRequest parsed = (OffsetsForLeaderEpochRequest) AbstractRequest.parseRequest( - ApiKeys.OFFSET_FOR_LEADER_EPOCH, version, request.toStruct()); + OffsetsForLeaderEpochRequest parsed = OffsetsForLeaderEpochRequest.parse(request.serializeBody(), version); if (version < 3) assertEquals(OffsetsForLeaderEpochRequest.DEBUGGING_REPLICA_ID, parsed.replicaId()); else @@ -57,4 +56,4 @@ public void testDefaultReplicaId() { } } -} \ No newline at end of file +} diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java index 05cb16724ef60..fe45f6a19d0f6 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java @@ -474,7 +474,8 @@ private void checkRequest(AbstractRequest req, boolean checkEqualityAndHashCode) // in the request is a HashMap with multiple elements since ordering of the elements may vary) try { Struct struct = req.toStruct(); - AbstractRequest deserialized = AbstractRequest.parseRequest(req.api, req.version(), struct); + ByteBuffer serializedBytes = req.serializeBody(); + AbstractRequest deserialized = AbstractRequest.parseRequest(req.api, req.version(), serializedBytes).request; Struct struct2 = deserialized.toStruct(); if (checkEqualityAndHashCode) { assertEquals(struct, struct2); From 859041a795d88398cb52eb133b719e9ce35fb979 Mon Sep 17 00:00:00 2001 From: Ismael Juma Date: Tue, 1 Oct 2019 13:23:53 -0700 Subject: [PATCH 09/51] Avoid intermediate structs when deserializing in the broker --- .../AlterPartitionReassignmentsRequest.java | 17 +--- .../AlterPartitionReassignmentsResponse.java | 13 +-- .../requests/ControlledShutdownRequest.java | 15 +-- .../requests/ControlledShutdownResponse.java | 6 +- .../CreateDelegationTokenRequest.java | 9 +- .../CreateDelegationTokenResponse.java | 5 +- .../common/requests/CreateTopicsRequest.java | 10 +- .../common/requests/CreateTopicsResponse.java | 5 +- .../common/requests/DeleteGroupsRequest.java | 8 +- .../common/requests/DeleteGroupsResponse.java | 9 +- .../requests/DeleteRecordsResponse.java | 3 - .../common/requests/DeleteTopicsRequest.java | 19 ++-- .../common/requests/DeleteTopicsResponse.java | 4 +- .../DescribeDelegationTokenRequest.java | 9 +- .../DescribeDelegationTokenResponse.java | 11 ++- .../requests/DescribeGroupsRequest.java | 13 +-- .../requests/DescribeGroupsResponse.java | 7 +- .../common/requests/ElectLeadersRequest.java | 8 +- .../common/requests/ElectLeadersResponse.java | 31 ++---- .../ExpireDelegationTokenRequest.java | 9 +- .../ExpireDelegationTokenResponse.java | 5 +- .../requests/FindCoordinatorRequest.java | 10 +- .../requests/FindCoordinatorResponse.java | 4 +- .../common/requests/HeartbeatRequest.java | 9 +- .../common/requests/HeartbeatResponse.java | 5 +- .../IncrementalAlterConfigsRequest.java | 10 +- .../IncrementalAlterConfigsResponse.java | 6 +- .../requests/InitProducerIdRequest.java | 9 +- .../requests/InitProducerIdResponse.java | 4 +- .../common/requests/JoinGroupRequest.java | 9 +- .../common/requests/JoinGroupResponse.java | 11 +-- .../common/requests/LeaderAndIsrRequest.java | 3 +- .../common/requests/LeaderAndIsrResponse.java | 4 +- .../common/requests/LeaveGroupRequest.java | 15 +-- .../common/requests/LeaveGroupResponse.java | 11 +-- .../common/requests/ListGroupsRequest.java | 8 +- .../common/requests/ListGroupsResponse.java | 4 +- .../common/requests/ListOffsetRequest.java | 1 - .../ListPartitionReassignmentsRequest.java | 16 +-- .../ListPartitionReassignmentsResponse.java | 13 +-- .../common/requests/MetadataRequest.java | 15 +-- .../common/requests/MetadataResponse.java | 4 +- .../common/requests/OffsetCommitRequest.java | 15 +-- .../common/requests/OffsetCommitResponse.java | 4 +- .../common/requests/OffsetDeleteRequest.java | 9 +- .../common/requests/OffsetDeleteResponse.java | 11 +-- .../common/requests/OffsetFetchRequest.java | 8 +- .../common/requests/OffsetFetchResponse.java | 28 +++--- .../requests/RenewDelegationTokenRequest.java | 9 +- .../RenewDelegationTokenResponse.java | 7 +- .../kafka/common/requests/RequestContext.java | 1 - .../requests/SaslAuthenticateRequest.java | 18 +--- .../requests/SaslAuthenticateResponse.java | 4 +- .../common/requests/SaslHandshakeRequest.java | 17 +--- .../requests/SaslHandshakeResponse.java | 4 +- .../common/requests/StopReplicaRequest.java | 7 +- .../common/requests/StopReplicaResponse.java | 4 +- .../common/requests/SyncGroupRequest.java | 8 +- .../common/requests/SyncGroupResponse.java | 9 +- .../requests/TxnOffsetCommitRequest.java | 9 +- .../requests/TxnOffsetCommitResponse.java | 4 +- .../requests/UpdateMetadataRequest.java | 7 +- .../requests/UpdateMetadataResponse.java | 4 +- .../clients/admin/KafkaAdminClientTest.java | 7 +- .../common/requests/RequestResponseTest.java | 6 +- .../AbstractCreateTopicsRequestTest.scala | 2 +- .../unit/kafka/server/RequestQuotaTest.scala | 97 +++++++++---------- .../server/SaslApiVersionsRequestTest.scala | 3 +- 68 files changed, 245 insertions(+), 444 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AlterPartitionReassignmentsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AlterPartitionReassignmentsRequest.java index 7b2f848f614f9..dd20bf43a1d5e 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AlterPartitionReassignmentsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AlterPartitionReassignmentsRequest.java @@ -23,6 +23,7 @@ import org.apache.kafka.common.message.AlterPartitionReassignmentsResponseData.ReassignablePartitionResponse; import org.apache.kafka.common.message.AlterPartitionReassignmentsResponseData.ReassignableTopicResponse; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; @@ -52,25 +53,15 @@ public String toString() { } private final AlterPartitionReassignmentsRequestData data; - private final short version; private AlterPartitionReassignmentsRequest(AlterPartitionReassignmentsRequestData data, short version) { super(ApiKeys.ALTER_PARTITION_REASSIGNMENTS, version); this.data = data; - this.version = version; - } - - AlterPartitionReassignmentsRequest(Struct struct, short version) { - super(ApiKeys.ALTER_PARTITION_REASSIGNMENTS, version); - this.data = new AlterPartitionReassignmentsRequestData(struct, version); - this.version = version; } public static AlterPartitionReassignmentsRequest parse(ByteBuffer buffer, short version) { - return new AlterPartitionReassignmentsRequest( - ApiKeys.ALTER_PARTITION_REASSIGNMENTS.parseRequest(version, buffer), - version - ); + return new AlterPartitionReassignmentsRequest(new AlterPartitionReassignmentsRequestData( + new ByteBufferAccessor(buffer), version), version); } public AlterPartitionReassignmentsRequestData data() { @@ -82,7 +73,7 @@ public AlterPartitionReassignmentsRequestData data() { */ @Override public Struct toStruct() { - return data.toStruct(version); + return data.toStruct(version()); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AlterPartitionReassignmentsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AlterPartitionReassignmentsResponse.java index ef235cda34f78..b1e1dc06052c8 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AlterPartitionReassignmentsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AlterPartitionReassignmentsResponse.java @@ -20,7 +20,7 @@ import org.apache.kafka.common.message.AlterPartitionReassignmentsResponseData; import org.apache.kafka.common.message.AlterPartitionReassignmentsResponseData.ReassignableTopicResponse; import org.apache.kafka.common.message.AlterPartitionReassignmentsResponseData.ReassignablePartitionResponse; -import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.types.Struct; @@ -32,20 +32,17 @@ public class AlterPartitionReassignmentsResponse extends AbstractResponse { private final AlterPartitionReassignmentsResponseData data; - public AlterPartitionReassignmentsResponse(Struct struct) { - this(struct, ApiKeys.ALTER_PARTITION_REASSIGNMENTS.latestVersion()); + public AlterPartitionReassignmentsResponse(Struct struct, short version) { + this(new AlterPartitionReassignmentsResponseData(struct, version)); } public AlterPartitionReassignmentsResponse(AlterPartitionReassignmentsResponseData data) { this.data = data; } - AlterPartitionReassignmentsResponse(Struct struct, short version) { - this.data = new AlterPartitionReassignmentsResponseData(struct, version); - } - public static AlterPartitionReassignmentsResponse parse(ByteBuffer buffer, short version) { - return new AlterPartitionReassignmentsResponse(ApiKeys.ALTER_PARTITION_REASSIGNMENTS.responseSchema(version).read(buffer), version); + return new AlterPartitionReassignmentsResponse( + new AlterPartitionReassignmentsResponseData(new ByteBufferAccessor(buffer), version)); } public AlterPartitionReassignmentsResponseData data() { diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownRequest.java index 83e9444da0ad1..8bf70dbfbb581 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownRequest.java @@ -19,6 +19,7 @@ import org.apache.kafka.common.message.ControlledShutdownRequestData; import org.apache.kafka.common.message.ControlledShutdownResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.types.Struct; @@ -48,18 +49,10 @@ public String toString() { } private final ControlledShutdownRequestData data; - private final short version; private ControlledShutdownRequest(ControlledShutdownRequestData data, short version) { super(ApiKeys.CONTROLLED_SHUTDOWN, version); this.data = data; - this.version = version; - } - - public ControlledShutdownRequest(Struct struct, short version) { - super(ApiKeys.CONTROLLED_SHUTDOWN, version); - this.data = new ControlledShutdownRequestData(struct, version); - this.version = version; } @Override @@ -69,13 +62,13 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { } public static ControlledShutdownRequest parse(ByteBuffer buffer, short version) { - return new ControlledShutdownRequest( - ApiKeys.CONTROLLED_SHUTDOWN.parseRequest(version, buffer), version); + return new ControlledShutdownRequest(new ControlledShutdownRequestData(new ByteBufferAccessor(buffer), version), + version); } @Override protected Struct toStruct() { - return data.toStruct(version); + return data.toStruct(version()); } public ControlledShutdownRequestData data() { diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownResponse.java index a49b84a851f41..d4b82677328f3 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownResponse.java @@ -19,7 +19,7 @@ import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.message.ControlledShutdownResponseData; import org.apache.kafka.common.message.ControlledShutdownResponseData.RemainingPartition; -import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.types.Struct; @@ -45,7 +45,7 @@ public ControlledShutdownResponse(ControlledShutdownResponseData data) { } public ControlledShutdownResponse(Struct struct, short version) { - this.data = new ControlledShutdownResponseData(struct, version); + this(new ControlledShutdownResponseData(struct, version)); } public Errors error() { @@ -58,7 +58,7 @@ public Map errorCounts() { } public static ControlledShutdownResponse parse(ByteBuffer buffer, short version) { - return new ControlledShutdownResponse(ApiKeys.CONTROLLED_SHUTDOWN.parseResponse(version, buffer), version); + return new ControlledShutdownResponse(new ControlledShutdownResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/CreateDelegationTokenRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/CreateDelegationTokenRequest.java index 61b404a7e0623..923b5ef381cbb 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/CreateDelegationTokenRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/CreateDelegationTokenRequest.java @@ -18,6 +18,7 @@ import org.apache.kafka.common.message.CreateDelegationTokenRequestData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.security.auth.KafkaPrincipal; @@ -33,13 +34,9 @@ private CreateDelegationTokenRequest(CreateDelegationTokenRequestData data, shor this.data = data; } - public CreateDelegationTokenRequest(Struct struct, short version) { - super(ApiKeys.CREATE_DELEGATION_TOKEN, version); - this.data = new CreateDelegationTokenRequestData(struct, version); - } - public static CreateDelegationTokenRequest parse(ByteBuffer buffer, short version) { - return new CreateDelegationTokenRequest(ApiKeys.CREATE_DELEGATION_TOKEN.parseRequest(version, buffer), version); + return new CreateDelegationTokenRequest(new CreateDelegationTokenRequestData(new ByteBufferAccessor(buffer), version), + version); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/CreateDelegationTokenResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/CreateDelegationTokenResponse.java index 04db17ac7a2f3..eec6d09d6e5b6 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/CreateDelegationTokenResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/CreateDelegationTokenResponse.java @@ -17,7 +17,7 @@ package org.apache.kafka.common.requests; import org.apache.kafka.common.message.CreateDelegationTokenResponseData; -import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.security.auth.KafkaPrincipal; @@ -39,7 +39,8 @@ public CreateDelegationTokenResponse(Struct struct, short version) { } public static CreateDelegationTokenResponse parse(ByteBuffer buffer, short version) { - return new CreateDelegationTokenResponse(ApiKeys.CREATE_DELEGATION_TOKEN.responseSchema(version).read(buffer), version); + return new CreateDelegationTokenResponse( + new CreateDelegationTokenResponseData(new ByteBufferAccessor(buffer), version)); } public static CreateDelegationTokenResponse prepareResponse(int throttleTimeMs, diff --git a/clients/src/main/java/org/apache/kafka/common/requests/CreateTopicsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/CreateTopicsRequest.java index dd26e5642632e..b94534d46522b 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/CreateTopicsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/CreateTopicsRequest.java @@ -25,6 +25,7 @@ import org.apache.kafka.common.message.CreateTopicsResponseData; import org.apache.kafka.common.message.CreateTopicsResponseData.CreatableTopicResult; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.types.Struct; public class CreateTopicsRequest extends AbstractRequest { @@ -72,16 +73,11 @@ public String toString() { public static final int NO_NUM_PARTITIONS = -1; public static final short NO_REPLICATION_FACTOR = -1; - private CreateTopicsRequest(CreateTopicsRequestData data, short version) { + public CreateTopicsRequest(CreateTopicsRequestData data, short version) { super(ApiKeys.CREATE_TOPICS, version); this.data = data; } - public CreateTopicsRequest(Struct struct, short version) { - super(ApiKeys.CREATE_TOPICS, version); - this.data = new CreateTopicsRequestData(struct, version); - } - public CreateTopicsRequestData data() { return data; } @@ -103,7 +99,7 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { } public static CreateTopicsRequest parse(ByteBuffer buffer, short version) { - return new CreateTopicsRequest(ApiKeys.CREATE_TOPICS.parseRequest(version, buffer), version); + return new CreateTopicsRequest(new CreateTopicsRequestData(new ByteBufferAccessor(buffer), version), version); } /** diff --git a/clients/src/main/java/org/apache/kafka/common/requests/CreateTopicsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/CreateTopicsResponse.java index 3cfc60cd38459..3b804ba8263e6 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/CreateTopicsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/CreateTopicsResponse.java @@ -19,7 +19,7 @@ import org.apache.kafka.common.message.CreateTopicsResponseData; import org.apache.kafka.common.message.CreateTopicsResponseData.CreatableTopicResult; -import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.types.Struct; @@ -79,8 +79,7 @@ public Map errorCounts() { } public static CreateTopicsResponse parse(ByteBuffer buffer, short version) { - return new CreateTopicsResponse( - ApiKeys.CREATE_TOPICS.responseSchema(version).read(buffer), version); + return new CreateTopicsResponse(new CreateTopicsResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsRequest.java index 2de85f1ac23be..547f70e9fe173 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsRequest.java @@ -21,6 +21,7 @@ import org.apache.kafka.common.message.DeleteGroupsResponseData.DeletableGroupResult; import org.apache.kafka.common.message.DeleteGroupsResponseData.DeletableGroupResultCollection; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Struct; @@ -54,11 +55,6 @@ public DeleteGroupsRequest(DeleteGroupsRequestData data, short version) { this.data = data; } - public DeleteGroupsRequest(Struct struct, short version) { - super(ApiKeys.DELETE_GROUPS, version); - this.data = new DeleteGroupsRequestData(struct, version); - } - @Override public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { Errors error = Errors.forException(e); @@ -77,7 +73,7 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { } public static DeleteGroupsRequest parse(ByteBuffer buffer, short version) { - return new DeleteGroupsRequest(ApiKeys.DELETE_GROUPS.parseRequest(version, buffer), version); + return new DeleteGroupsRequest(new DeleteGroupsRequestData(new ByteBufferAccessor(buffer), version), version); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsResponse.java index 0a030bdb8495e..f82a1dffc7337 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsResponse.java @@ -18,7 +18,7 @@ import org.apache.kafka.common.message.DeleteGroupsResponseData; import org.apache.kafka.common.message.DeleteGroupsResponseData.DeletableGroupResult; -import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Struct; @@ -46,11 +46,6 @@ public DeleteGroupsResponse(DeleteGroupsResponseData data) { this.data = data; } - public DeleteGroupsResponse(Struct struct) { - short latestVersion = (short) (DeleteGroupsResponseData.SCHEMAS.length - 1); - this.data = new DeleteGroupsResponseData(struct, latestVersion); - } - public DeleteGroupsResponse(Struct struct, short version) { this.data = new DeleteGroupsResponseData(struct, version); } @@ -92,7 +87,7 @@ public Map errorCounts() { } public static DeleteGroupsResponse parse(ByteBuffer buffer, short version) { - return new DeleteGroupsResponse(ApiKeys.DELETE_GROUPS.parseResponse(version, buffer), version); + return new DeleteGroupsResponse(new DeleteGroupsResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DeleteRecordsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DeleteRecordsResponse.java index 7531a05a4a7a3..21a6b33814bef 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DeleteRecordsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DeleteRecordsResponse.java @@ -125,9 +125,6 @@ public DeleteRecordsResponse(Struct struct) { } } - /** - * Constructor for version 0. - */ public DeleteRecordsResponse(int throttleTimeMs, Map responses) { this.throttleTimeMs = throttleTimeMs; this.responses = responses; diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DeleteTopicsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DeleteTopicsRequest.java index 440acfd2b838a..f80b227e7bc32 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DeleteTopicsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DeleteTopicsRequest.java @@ -20,15 +20,13 @@ import org.apache.kafka.common.message.DeleteTopicsResponseData; import org.apache.kafka.common.message.DeleteTopicsResponseData.DeletableTopicResult; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; public class DeleteTopicsRequest extends AbstractRequest { - private DeleteTopicsRequestData data; - private final short version; - public static class Builder extends AbstractRequest.Builder { private DeleteTopicsRequestData data; @@ -48,21 +46,16 @@ public String toString() { } } + private DeleteTopicsRequestData data; + private DeleteTopicsRequest(DeleteTopicsRequestData data, short version) { super(ApiKeys.DELETE_TOPICS, version); this.data = data; - this.version = version; - } - - public DeleteTopicsRequest(Struct struct, short version) { - super(ApiKeys.DELETE_TOPICS, version); - this.data = new DeleteTopicsRequestData(struct, version); - this.version = version; } @Override protected Struct toStruct() { - return data.toStruct(version); + return data.toStruct(version()); } public DeleteTopicsRequestData data() { @@ -72,7 +65,7 @@ public DeleteTopicsRequestData data() { @Override public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { DeleteTopicsResponseData response = new DeleteTopicsResponseData(); - if (version >= 1) { + if (version() >= 1) { response.setThrottleTimeMs(throttleTimeMs); } ApiError apiError = ApiError.fromThrowable(e); @@ -85,7 +78,7 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { } public static DeleteTopicsRequest parse(ByteBuffer buffer, short version) { - return new DeleteTopicsRequest(ApiKeys.DELETE_TOPICS.parseRequest(version, buffer), version); + return new DeleteTopicsRequest(new DeleteTopicsRequestData(new ByteBufferAccessor(buffer), version), version); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DeleteTopicsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DeleteTopicsResponse.java index aa8e552d206df..234a3394d86a9 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DeleteTopicsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DeleteTopicsResponse.java @@ -18,7 +18,7 @@ import org.apache.kafka.common.message.DeleteTopicsResponseData; import org.apache.kafka.common.message.DeleteTopicsResponseData.DeletableTopicResult; -import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.types.Struct; @@ -74,7 +74,7 @@ public Map errorCounts() { } public static DeleteTopicsResponse parse(ByteBuffer buffer, short version) { - return new DeleteTopicsResponse(ApiKeys.DELETE_TOPICS.parseResponse(version, buffer), version); + return new DeleteTopicsResponse(new DeleteTopicsResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenRequest.java index 1ad5f73e93cf8..deedaa0e52824 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenRequest.java @@ -18,6 +18,7 @@ import org.apache.kafka.common.message.DescribeDelegationTokenRequestData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.security.auth.KafkaPrincipal; @@ -55,11 +56,6 @@ public String toString() { private final DescribeDelegationTokenRequestData data; - public DescribeDelegationTokenRequest(Struct struct, short version) { - super(ApiKeys.DESCRIBE_DELEGATION_TOKEN, version); - this.data = new DescribeDelegationTokenRequestData(struct, version); - } - public DescribeDelegationTokenRequest(DescribeDelegationTokenRequestData data, short version) { super(ApiKeys.DESCRIBE_DELEGATION_TOKEN, version); this.data = data; @@ -84,6 +80,7 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { } public static DescribeDelegationTokenRequest parse(ByteBuffer buffer, short version) { - return new DescribeDelegationTokenRequest(ApiKeys.EXPIRE_DELEGATION_TOKEN.parseRequest(version, buffer), version); + return new DescribeDelegationTokenRequest(new DescribeDelegationTokenRequestData( + new ByteBufferAccessor(buffer), version), version); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenResponse.java index eb575092080eb..5500445d415e4 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenResponse.java @@ -19,7 +19,7 @@ import org.apache.kafka.common.message.DescribeDelegationTokenResponseData; import org.apache.kafka.common.message.DescribeDelegationTokenResponseData.DescribedDelegationToken; import org.apache.kafka.common.message.DescribeDelegationTokenResponseData.DescribedDelegationTokenRenewer; -import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Struct; @@ -64,12 +64,17 @@ public DescribeDelegationTokenResponse(int throttleTimeMs, Errors error) { this(throttleTimeMs, error, new ArrayList<>()); } + public DescribeDelegationTokenResponse(DescribeDelegationTokenResponseData data) { + this.data = data; + } + public DescribeDelegationTokenResponse(Struct struct, short version) { - this.data = new DescribeDelegationTokenResponseData(struct, version); + this(new DescribeDelegationTokenResponseData(struct, version)); } public static DescribeDelegationTokenResponse parse(ByteBuffer buffer, short version) { - return new DescribeDelegationTokenResponse(ApiKeys.DESCRIBE_DELEGATION_TOKEN.responseSchema(version).read(buffer), version); + return new DescribeDelegationTokenResponse(new DescribeDelegationTokenResponseData( + new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeGroupsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeGroupsRequest.java index 8f003acad5b97..d62c49d9291f2 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeGroupsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeGroupsRequest.java @@ -18,6 +18,7 @@ import org.apache.kafka.common.message.DescribeGroupsRequestData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.types.Struct; @@ -46,18 +47,14 @@ public String toString() { } private final DescribeGroupsRequestData data; - private final short version; private DescribeGroupsRequest(DescribeGroupsRequestData data, short version) { super(ApiKeys.DESCRIBE_GROUPS, version); this.data = data; - this.version = version; } public DescribeGroupsRequest(Struct struct, short version) { - super(ApiKeys.DESCRIBE_GROUPS, version); - this.data = new DescribeGroupsRequestData(struct, version); - this.version = version; + this(new DescribeGroupsRequestData(struct, version), version); } public DescribeGroupsRequestData data() { @@ -66,12 +63,12 @@ public DescribeGroupsRequestData data() { @Override protected Struct toStruct() { - return data.toStruct(version); + return data.toStruct(version()); } @Override public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { - if (version == 0) { + if (version() == 0) { return DescribeGroupsResponse.fromError(DEFAULT_THROTTLE_TIME, Errors.forException(e), data.groups()); } else { return DescribeGroupsResponse.fromError(throttleTimeMs, Errors.forException(e), data.groups()); @@ -79,6 +76,6 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { } public static DescribeGroupsRequest parse(ByteBuffer buffer, short version) { - return new DescribeGroupsRequest(ApiKeys.DESCRIBE_GROUPS.parseRequest(version, buffer), version); + return new DescribeGroupsRequest(new DescribeGroupsRequestData(new ByteBufferAccessor(buffer), version), version); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeGroupsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeGroupsResponse.java index cb369a0980867..39b1fd2c549b6 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeGroupsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeGroupsResponse.java @@ -19,7 +19,7 @@ import org.apache.kafka.common.message.DescribeGroupsResponseData; import org.apache.kafka.common.message.DescribeGroupsResponseData.DescribedGroup; import org.apache.kafka.common.message.DescribeGroupsResponseData.DescribedGroupMember; -import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.utils.Utils; @@ -49,7 +49,7 @@ public DescribeGroupsResponse(DescribeGroupsResponseData data) { } public DescribeGroupsResponse(Struct struct, short version) { - this.data = new DescribeGroupsResponseData(struct, version); + this(new DescribeGroupsResponseData(struct, version)); } public static DescribedGroupMember groupMember( @@ -147,8 +147,7 @@ public static DescribeGroupsResponse fromError(int throttleTimeMs, Errors error, } public static DescribeGroupsResponse parse(ByteBuffer buffer, short version) { - return new DescribeGroupsResponse( - ApiKeys.DESCRIBE_GROUPS.responseSchema(version).read(buffer), version); + return new DescribeGroupsResponse(new DescribeGroupsResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ElectLeadersRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ElectLeadersRequest.java index 025733e39c0bc..013fe36fe8a29 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ElectLeadersRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ElectLeadersRequest.java @@ -30,6 +30,7 @@ import org.apache.kafka.common.message.ElectLeadersResponseData.PartitionResult; import org.apache.kafka.common.message.ElectLeadersResponseData.ReplicaElectionResult; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.MessageUtil; import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.utils.CollectionUtils; @@ -90,11 +91,6 @@ private ElectLeadersRequest(ElectLeadersRequestData data, short version) { this.data = data; } - public ElectLeadersRequest(Struct struct, short version) { - super(ApiKeys.ELECT_LEADERS, version); - this.data = new ElectLeadersRequestData(struct, version); - } - public ElectLeadersRequestData data() { return data; } @@ -124,7 +120,7 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { } public static ElectLeadersRequest parse(ByteBuffer buffer, short version) { - return new ElectLeadersRequest(ApiKeys.ELECT_LEADERS.parseRequest(version, buffer), version); + return new ElectLeadersRequest(new ElectLeadersRequestData(new ByteBufferAccessor(buffer), version), version); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ElectLeadersResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ElectLeadersResponse.java index b0852040612f0..80deca1469535 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ElectLeadersResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ElectLeadersResponse.java @@ -26,29 +26,20 @@ import org.apache.kafka.common.message.ElectLeadersResponseData.PartitionResult; import org.apache.kafka.common.message.ElectLeadersResponseData.ReplicaElectionResult; import org.apache.kafka.common.message.ElectLeadersResponseData; -import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.types.Struct; public class ElectLeadersResponse extends AbstractResponse { - private final short version; private final ElectLeadersResponseData data; - public ElectLeadersResponse(Struct struct) { - this(struct, ApiKeys.ELECT_LEADERS.latestVersion()); - } - public ElectLeadersResponse(Struct struct, short version) { - this.version = version; - this.data = new ElectLeadersResponseData(struct, version); + this(new ElectLeadersResponseData(struct, version)); } - public ElectLeadersResponse( - int throttleTimeMs, - short errorCode, - List electionResults) { - this(throttleTimeMs, errorCode, electionResults, ApiKeys.ELECT_LEADERS.latestVersion()); + public ElectLeadersResponse(ElectLeadersResponseData data) { + this.data = data; } public ElectLeadersResponse( @@ -56,16 +47,10 @@ public ElectLeadersResponse( short errorCode, List electionResults, short version) { - - this.version = version; this.data = new ElectLeadersResponseData(); - data.setThrottleTimeMs(throttleTimeMs); - - if (version >= 1) { + if (version >= 1) data.setErrorCode(errorCode); - } - data.setReplicaElectionResults(electionResults); } @@ -73,10 +58,6 @@ public ElectLeadersResponseData data() { return data; } - public short version() { - return version; - } - @Override protected Struct toStruct(short version) { return data.toStruct(version); @@ -100,7 +81,7 @@ public Map errorCounts() { } public static ElectLeadersResponse parse(ByteBuffer buffer, short version) { - return new ElectLeadersResponse(ApiKeys.ELECT_LEADERS.responseSchema(version).read(buffer), version); + return new ElectLeadersResponse(new ElectLeadersResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenRequest.java index 45b7df327bf5c..c7409d34a198f 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenRequest.java @@ -21,6 +21,7 @@ import org.apache.kafka.common.message.ExpireDelegationTokenRequestData; import org.apache.kafka.common.message.ExpireDelegationTokenResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Struct; @@ -34,13 +35,9 @@ private ExpireDelegationTokenRequest(ExpireDelegationTokenRequestData data, shor this.data = data; } - public ExpireDelegationTokenRequest(Struct struct, short version) { - super(ApiKeys.EXPIRE_DELEGATION_TOKEN, version); - this.data = new ExpireDelegationTokenRequestData(struct, version); - } - public static ExpireDelegationTokenRequest parse(ByteBuffer buffer, short version) { - return new ExpireDelegationTokenRequest(ApiKeys.EXPIRE_DELEGATION_TOKEN.parseRequest(version, buffer), version); + return new ExpireDelegationTokenRequest( + new ExpireDelegationTokenRequestData(new ByteBufferAccessor(buffer), version), version); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenResponse.java index 6b8a22fa6ecee..5b4ee15e3ca5f 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenResponse.java @@ -21,7 +21,7 @@ import java.util.Map; import org.apache.kafka.common.message.ExpireDelegationTokenResponseData; -import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Struct; @@ -39,7 +39,8 @@ public ExpireDelegationTokenResponse(Struct struct, short version) { } public static ExpireDelegationTokenResponse parse(ByteBuffer buffer, short version) { - return new ExpireDelegationTokenResponse(ApiKeys.EXPIRE_DELEGATION_TOKEN.responseSchema(version).read(buffer), version); + return new ExpireDelegationTokenResponse(new ExpireDelegationTokenResponseData(new ByteBufferAccessor(buffer), + version)); } public Errors error() { diff --git a/clients/src/main/java/org/apache/kafka/common/requests/FindCoordinatorRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/FindCoordinatorRequest.java index 0e728439169d4..99a2e5636046f 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/FindCoordinatorRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/FindCoordinatorRequest.java @@ -21,12 +21,12 @@ import org.apache.kafka.common.message.FindCoordinatorRequestData; import org.apache.kafka.common.message.FindCoordinatorResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; - public class FindCoordinatorRequest extends AbstractRequest { public static class Builder extends AbstractRequest.Builder { @@ -63,11 +63,6 @@ private FindCoordinatorRequest(FindCoordinatorRequestData data, short version) { this.data = data; } - public FindCoordinatorRequest(Struct struct, short version) { - super(ApiKeys.FIND_COORDINATOR, version); - this.data = new FindCoordinatorRequestData(struct, version); - } - @Override public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { FindCoordinatorResponseData response = new FindCoordinatorResponseData(); @@ -79,7 +74,8 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { } public static FindCoordinatorRequest parse(ByteBuffer buffer, short version) { - return new FindCoordinatorRequest(ApiKeys.FIND_COORDINATOR.parseRequest(version, buffer), version); + return new FindCoordinatorRequest(new FindCoordinatorRequestData(new ByteBufferAccessor(buffer), version), + version); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/FindCoordinatorResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/FindCoordinatorResponse.java index c880408a8ed30..3fd2f0eabfbde 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/FindCoordinatorResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/FindCoordinatorResponse.java @@ -18,7 +18,7 @@ import org.apache.kafka.common.Node; import org.apache.kafka.common.message.FindCoordinatorResponseData; -import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.types.Struct; @@ -81,7 +81,7 @@ protected Struct toStruct(short version) { } public static FindCoordinatorResponse parse(ByteBuffer buffer, short version) { - return new FindCoordinatorResponse(ApiKeys.FIND_COORDINATOR.responseSchema(version).read(buffer), version); + return new FindCoordinatorResponse(new FindCoordinatorResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatRequest.java index c9a51b7d83b6e..8fb6cb503fd9c 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatRequest.java @@ -20,13 +20,13 @@ import org.apache.kafka.common.message.HeartbeatRequestData; import org.apache.kafka.common.message.HeartbeatResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; - public class HeartbeatRequest extends AbstractRequest { public static class Builder extends AbstractRequest.Builder { @@ -59,11 +59,6 @@ private HeartbeatRequest(HeartbeatRequestData data, short version) { this.data = data; } - public HeartbeatRequest(Struct struct, short version) { - super(ApiKeys.HEARTBEAT, version); - this.data = new HeartbeatRequestData(struct, version); - } - @Override public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { HeartbeatResponseData responseData = new HeartbeatResponseData(). @@ -75,7 +70,7 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { } public static HeartbeatRequest parse(ByteBuffer buffer, short version) { - return new HeartbeatRequest(ApiKeys.HEARTBEAT.parseRequest(version, buffer), version); + return new HeartbeatRequest(new HeartbeatRequestData(new ByteBufferAccessor(buffer), version), version); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatResponse.java index 3de921155fc98..04b1005c1f0c6 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatResponse.java @@ -17,7 +17,7 @@ package org.apache.kafka.common.requests; import org.apache.kafka.common.message.HeartbeatResponseData; -import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Struct; @@ -26,7 +26,6 @@ import java.util.Collections; import java.util.Map; - public class HeartbeatResponse extends AbstractResponse { /** @@ -74,7 +73,7 @@ protected Message data() { } public static HeartbeatResponse parse(ByteBuffer buffer, short version) { - return new HeartbeatResponse(ApiKeys.HEARTBEAT.parseResponse(version, buffer), version); + return new HeartbeatResponse(new HeartbeatResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/IncrementalAlterConfigsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/IncrementalAlterConfigsRequest.java index 0c6c0b2814730..038c37eab7e27 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/IncrementalAlterConfigsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/IncrementalAlterConfigsRequest.java @@ -22,6 +22,7 @@ import org.apache.kafka.common.message.IncrementalAlterConfigsResponseData; import org.apache.kafka.common.message.IncrementalAlterConfigsResponseData.AlterConfigsResourceResponse; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; @@ -56,14 +57,9 @@ private IncrementalAlterConfigsRequest(IncrementalAlterConfigsRequestData data, this.version = version; } - IncrementalAlterConfigsRequest(final Struct struct, final short version) { - super(ApiKeys.INCREMENTAL_ALTER_CONFIGS, version); - this.data = new IncrementalAlterConfigsRequestData(struct, version); - this.version = version; - } - public static IncrementalAlterConfigsRequest parse(ByteBuffer buffer, short version) { - return new IncrementalAlterConfigsRequest(ApiKeys.INCREMENTAL_ALTER_CONFIGS.parseRequest(version, buffer), version); + return new IncrementalAlterConfigsRequest(new IncrementalAlterConfigsRequestData( + new ByteBufferAccessor(buffer), version), version); } public IncrementalAlterConfigsRequestData data() { diff --git a/clients/src/main/java/org/apache/kafka/common/requests/IncrementalAlterConfigsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/IncrementalAlterConfigsResponse.java index 46b1d530f5635..bd53987b196bc 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/IncrementalAlterConfigsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/IncrementalAlterConfigsResponse.java @@ -20,7 +20,7 @@ import org.apache.kafka.common.config.ConfigResource; import org.apache.kafka.common.message.IncrementalAlterConfigsResponseData; import org.apache.kafka.common.message.IncrementalAlterConfigsResponseData.AlterConfigsResourceResponse; -import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.types.Struct; @@ -93,7 +93,7 @@ public int throttleTimeMs() { } public static IncrementalAlterConfigsResponse parse(ByteBuffer buffer, short version) { - return new IncrementalAlterConfigsResponse( - ApiKeys.INCREMENTAL_ALTER_CONFIGS.responseSchema(version).read(buffer), version); + return new IncrementalAlterConfigsResponse(new IncrementalAlterConfigsResponseData( + new ByteBufferAccessor(buffer), version)); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/InitProducerIdRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/InitProducerIdRequest.java index e221889011142..1ae680d8c3dd5 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/InitProducerIdRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/InitProducerIdRequest.java @@ -19,6 +19,7 @@ import org.apache.kafka.common.message.InitProducerIdRequestData; import org.apache.kafka.common.message.InitProducerIdResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Struct; @@ -59,12 +60,6 @@ private InitProducerIdRequest(InitProducerIdRequestData data, short version) { this.data = data; } - public InitProducerIdRequest(Struct struct, short version) { - super(ApiKeys.INIT_PRODUCER_ID, version); - this.data = new InitProducerIdRequestData(struct, version); - } - - @Override public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { InitProducerIdResponseData response = new InitProducerIdResponseData() @@ -76,7 +71,7 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { } public static InitProducerIdRequest parse(ByteBuffer buffer, short version) { - return new InitProducerIdRequest(ApiKeys.INIT_PRODUCER_ID.parseRequest(version, buffer), version); + return new InitProducerIdRequest(new InitProducerIdRequestData(new ByteBufferAccessor(buffer), version), version); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/InitProducerIdResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/InitProducerIdResponse.java index c167fd9c29ef8..93630a5af5a28 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/InitProducerIdResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/InitProducerIdResponse.java @@ -17,7 +17,7 @@ package org.apache.kafka.common.requests; import org.apache.kafka.common.message.InitProducerIdResponseData; -import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Struct; @@ -65,7 +65,7 @@ protected Message data() { } public static InitProducerIdResponse parse(ByteBuffer buffer, short version) { - return new InitProducerIdResponse(ApiKeys.INIT_PRODUCER_ID.parseResponse(version, buffer), version); + return new InitProducerIdResponse(new InitProducerIdResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/JoinGroupRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/JoinGroupRequest.java index 95d125d312a17..1136828ae6acb 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/JoinGroupRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/JoinGroupRequest.java @@ -21,6 +21,7 @@ import org.apache.kafka.common.message.JoinGroupRequestData; import org.apache.kafka.common.message.JoinGroupResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.types.Struct; @@ -97,12 +98,6 @@ public JoinGroupRequest(JoinGroupRequestData data, short version) { maybeOverrideRebalanceTimeout(version); } - public JoinGroupRequest(Struct struct, short version) { - super(ApiKeys.JOIN_GROUP, version); - this.data = new JoinGroupRequestData(struct, version); - maybeOverrideRebalanceTimeout(version); - } - private void maybeOverrideRebalanceTimeout(short version) { if (version == 0) { // Version 0 has no rebalance timeout, so we use the session timeout @@ -151,7 +146,7 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { } public static JoinGroupRequest parse(ByteBuffer buffer, short version) { - return new JoinGroupRequest(ApiKeys.JOIN_GROUP.parseRequest(version, buffer), version); + return new JoinGroupRequest(new JoinGroupRequestData(new ByteBufferAccessor(buffer), version), version); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/JoinGroupResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/JoinGroupResponse.java index 0e1644e465923..2b71659290c57 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/JoinGroupResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/JoinGroupResponse.java @@ -17,7 +17,7 @@ package org.apache.kafka.common.requests; import org.apache.kafka.common.message.JoinGroupResponseData; -import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.types.Struct; @@ -37,11 +37,6 @@ public JoinGroupResponse(JoinGroupResponseData data) { this.data = data; } - public JoinGroupResponse(Struct struct) { - short latestVersion = (short) (JoinGroupResponseData.SCHEMAS.length - 1); - this.data = new JoinGroupResponseData(struct, latestVersion); - } - public JoinGroupResponse(Struct struct, short version) { this.data = new JoinGroupResponseData(struct, version); } @@ -68,8 +63,8 @@ public Map errorCounts() { return Collections.singletonMap(Errors.forCode(data.errorCode()), 1); } - public static JoinGroupResponse parse(ByteBuffer buffer, short versionId) { - return new JoinGroupResponse(ApiKeys.JOIN_GROUP.parseResponse(versionId, buffer), versionId); + public static JoinGroupResponse parse(ByteBuffer buffer, short version) { + return new JoinGroupResponse(new JoinGroupResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java index 3bebd1eaaa3ef..688cf8115daf4 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java @@ -24,6 +24,7 @@ import org.apache.kafka.common.message.LeaderAndIsrResponseData; import org.apache.kafka.common.message.LeaderAndIsrResponseData.LeaderAndIsrPartitionError; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Struct; @@ -193,6 +194,6 @@ public List liveLeaders() { } public static LeaderAndIsrRequest parse(ByteBuffer buffer, short version) { - return new LeaderAndIsrRequest(ApiKeys.LEADER_AND_ISR.parseRequest(version, buffer), version); + return new LeaderAndIsrRequest(new LeaderAndIsrRequestData(new ByteBufferAccessor(buffer), version), version); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrResponse.java index de98cb1ec9833..62f523321739f 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrResponse.java @@ -18,7 +18,7 @@ import org.apache.kafka.common.message.LeaderAndIsrResponseData; import org.apache.kafka.common.message.LeaderAndIsrResponseData.LeaderAndIsrPartitionError; -import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Struct; @@ -65,7 +65,7 @@ public Map errorCounts() { } public static LeaderAndIsrResponse parse(ByteBuffer buffer, short version) { - return new LeaderAndIsrResponse(ApiKeys.LEADER_AND_ISR.parseResponse(version, buffer), version); + return new LeaderAndIsrResponse(new LeaderAndIsrResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupRequest.java index ac77379ae2bf8..cec03954a5d9e 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupRequest.java @@ -21,6 +21,7 @@ import org.apache.kafka.common.message.LeaveGroupRequestData.MemberIdentity; import org.apache.kafka.common.message.LeaveGroupResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.MessageUtil; import org.apache.kafka.common.protocol.types.Struct; @@ -77,18 +78,10 @@ public String toString() { } } private final LeaveGroupRequestData data; - private final short version; private LeaveGroupRequest(LeaveGroupRequestData data, short version) { super(ApiKeys.LEAVE_GROUP, version); this.data = data; - this.version = version; - } - - public LeaveGroupRequest(Struct struct, short version) { - super(ApiKeys.LEAVE_GROUP, version); - this.data = new LeaveGroupRequestData(struct, version); - this.version = version; } public LeaveGroupRequestData data() { @@ -97,7 +90,7 @@ public LeaveGroupRequestData data() { public List members() { // Before version 3, leave group request is still in single mode - return version <= 2 ? Collections.singletonList( + return version() <= 2 ? Collections.singletonList( new MemberIdentity() .setMemberId(data.memberId())) : data.members(); } @@ -114,11 +107,11 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { } public static LeaveGroupRequest parse(ByteBuffer buffer, short version) { - return new LeaveGroupRequest(ApiKeys.LEAVE_GROUP.parseRequest(version, buffer), version); + return new LeaveGroupRequest(new LeaveGroupRequestData(new ByteBufferAccessor(buffer), version), version); } @Override protected Struct toStruct() { - return data.toStruct(version); + return data.toStruct(version()); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupResponse.java index 71ad453500497..1e3c22834474d 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupResponse.java @@ -18,7 +18,7 @@ import org.apache.kafka.common.message.LeaveGroupResponseData; import org.apache.kafka.common.message.LeaveGroupResponseData.MemberResponse; -import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Struct; @@ -76,11 +76,6 @@ public LeaveGroupResponse(List memberResponses, } } - public LeaveGroupResponse(Struct struct) { - short latestVersion = (short) (LeaveGroupResponseData.SCHEMAS.length - 1); - this.data = new LeaveGroupResponseData(struct, latestVersion); - } - public LeaveGroupResponse(Struct struct, short version) { this.data = new LeaveGroupResponseData(struct, version); } @@ -145,8 +140,8 @@ protected Message data() { return data; } - public static LeaveGroupResponse parse(ByteBuffer buffer, short versionId) { - return new LeaveGroupResponse(ApiKeys.LEAVE_GROUP.parseResponse(versionId, buffer), versionId); + public static LeaveGroupResponse parse(ByteBuffer buffer, short version) { + return new LeaveGroupResponse(new LeaveGroupResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsRequest.java index 07f96aba9136c..ac11910b4315e 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsRequest.java @@ -19,6 +19,7 @@ import org.apache.kafka.common.message.ListGroupsRequestData; import org.apache.kafka.common.message.ListGroupsResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Struct; @@ -62,11 +63,6 @@ public ListGroupsRequest(ListGroupsRequestData data, short version) { this.data = data; } - public ListGroupsRequest(Struct struct, short version) { - super(ApiKeys.LIST_GROUPS, version); - this.data = new ListGroupsRequestData(struct, version); - } - @Override public ListGroupsResponse getErrorResponse(int throttleTimeMs, Throwable e) { ListGroupsResponseData listGroupsResponseData = new ListGroupsResponseData(). @@ -79,7 +75,7 @@ public ListGroupsResponse getErrorResponse(int throttleTimeMs, Throwable e) { } public static ListGroupsRequest parse(ByteBuffer buffer, short version) { - return new ListGroupsRequest(ApiKeys.LIST_GROUPS.parseRequest(version, buffer), version); + return new ListGroupsRequest(new ListGroupsRequestData(new ByteBufferAccessor(buffer), version), version); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsResponse.java index f3a67ced7e9ac..cc8a870aba7b2 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsResponse.java @@ -17,7 +17,7 @@ package org.apache.kafka.common.requests; import org.apache.kafka.common.message.ListGroupsResponseData; -import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.types.Struct; @@ -57,7 +57,7 @@ protected Struct toStruct(short version) { } public static ListGroupsResponse parse(ByteBuffer buffer, short version) { - return new ListGroupsResponse(ApiKeys.LIST_GROUPS.responseSchema(version).read(buffer), version); + return new ListGroupsResponse(new ListGroupsResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetRequest.java index 179f3b9075485..561d3a03c9c59 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetRequest.java @@ -218,7 +218,6 @@ public String toString() { /** * Private constructor with a specified version. */ - @SuppressWarnings("unchecked") private ListOffsetRequest(int replicaId, Map targetTimes, IsolationLevel isolationLevel, diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ListPartitionReassignmentsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ListPartitionReassignmentsRequest.java index 0aa5e557556ea..9c914247af60a 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ListPartitionReassignmentsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ListPartitionReassignmentsRequest.java @@ -21,6 +21,7 @@ import org.apache.kafka.common.message.ListPartitionReassignmentsResponseData.OngoingPartitionReassignment; import org.apache.kafka.common.message.ListPartitionReassignmentsResponseData.OngoingTopicReassignment; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; @@ -52,24 +53,15 @@ public String toString() { } private ListPartitionReassignmentsRequestData data; - private final short version; private ListPartitionReassignmentsRequest(ListPartitionReassignmentsRequestData data, short version) { super(ApiKeys.LIST_PARTITION_REASSIGNMENTS, version); this.data = data; - this.version = version; - } - - ListPartitionReassignmentsRequest(Struct struct, short version) { - super(ApiKeys.LIST_PARTITION_REASSIGNMENTS, version); - this.data = new ListPartitionReassignmentsRequestData(struct, version); - this.version = version; } public static ListPartitionReassignmentsRequest parse(ByteBuffer buffer, short version) { - return new ListPartitionReassignmentsRequest( - ApiKeys.LIST_PARTITION_REASSIGNMENTS.parseRequest(version, buffer), version - ); + return new ListPartitionReassignmentsRequest(new ListPartitionReassignmentsRequestData( + new ByteBufferAccessor(buffer), version), version); } public ListPartitionReassignmentsRequestData data() { @@ -81,7 +73,7 @@ public ListPartitionReassignmentsRequestData data() { */ @Override public Struct toStruct() { - return data.toStruct(version); + return data.toStruct(version()); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ListPartitionReassignmentsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ListPartitionReassignmentsResponse.java index 0d53e4dd7caee..587f6fc3854fd 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ListPartitionReassignmentsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ListPartitionReassignmentsResponse.java @@ -17,7 +17,7 @@ package org.apache.kafka.common.requests; import org.apache.kafka.common.message.ListPartitionReassignmentsResponseData; -import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.types.Struct; @@ -29,20 +29,17 @@ public class ListPartitionReassignmentsResponse extends AbstractResponse { private final ListPartitionReassignmentsResponseData data; - public ListPartitionReassignmentsResponse(Struct struct) { - this(struct, ApiKeys.LIST_PARTITION_REASSIGNMENTS.latestVersion()); - } - public ListPartitionReassignmentsResponse(ListPartitionReassignmentsResponseData responseData) { this.data = responseData; } - ListPartitionReassignmentsResponse(Struct struct, short version) { - this.data = new ListPartitionReassignmentsResponseData(struct, version); + public ListPartitionReassignmentsResponse(Struct struct, short version) { + this(new ListPartitionReassignmentsResponseData(struct, version)); } public static ListPartitionReassignmentsResponse parse(ByteBuffer buffer, short version) { - return new ListPartitionReassignmentsResponse(ApiKeys.LIST_PARTITION_REASSIGNMENTS.responseSchema(version).read(buffer), version); + return new ListPartitionReassignmentsResponse(new ListPartitionReassignmentsResponseData( + new ByteBufferAccessor(buffer), version)); } public ListPartitionReassignmentsResponseData data() { diff --git a/clients/src/main/java/org/apache/kafka/common/requests/MetadataRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/MetadataRequest.java index a4fd7b6697885..8900b87be2428 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/MetadataRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/MetadataRequest.java @@ -21,6 +21,7 @@ import org.apache.kafka.common.message.MetadataRequestData.MetadataRequestTopic; import org.apache.kafka.common.message.MetadataResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.types.Struct; @@ -102,18 +103,10 @@ public String toString() { } private final MetadataRequestData data; - private final short version; public MetadataRequest(MetadataRequestData data, short version) { super(ApiKeys.METADATA, version); this.data = data; - this.version = version; - } - - public MetadataRequest(Struct struct, short version) { - super(ApiKeys.METADATA, version); - this.data = new MetadataRequestData(struct, version); - this.version = version; } public MetadataRequestData data() { @@ -155,7 +148,7 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { public boolean isAllTopics() { return (data.topics() == null) || - (data.topics().isEmpty() && version == 0); //In version 0, an empty topic list indicates + (data.topics().isEmpty() && version() == 0); //In version 0, an empty topic list indicates // "request metadata for all topics." } @@ -174,7 +167,7 @@ public boolean allowAutoTopicCreation() { } public static MetadataRequest parse(ByteBuffer buffer, short version) { - return new MetadataRequest(ApiKeys.METADATA.parseRequest(version, buffer), version); + return new MetadataRequest(new MetadataRequestData(new ByteBufferAccessor(buffer), version), version); } public static List convertToMetadataRequestTopic(final Collection topics) { @@ -185,6 +178,6 @@ public static List convertToMetadataRequestTopic(final Col @Override protected Struct toStruct() { - return data.toStruct(version); + return data.toStruct(version()); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/MetadataResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/MetadataResponse.java index fcaa0a18b660e..b04f1793e1b44 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/MetadataResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/MetadataResponse.java @@ -23,7 +23,7 @@ import org.apache.kafka.common.message.MetadataResponseData.MetadataResponseTopic; import org.apache.kafka.common.message.MetadataResponseData.MetadataResponsePartition; import org.apache.kafka.common.message.MetadataResponseData.MetadataResponseBroker; -import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Struct; @@ -212,7 +212,7 @@ public String clusterId() { } public static MetadataResponse parse(ByteBuffer buffer, short version) { - return new MetadataResponse(ApiKeys.METADATA.responseSchema(version).read(buffer), version); + return new MetadataResponse(new MetadataResponseData(new ByteBufferAccessor(buffer), version)); } public static class TopicMetadata { diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitRequest.java index 72ebef261f4df..4b5a2064df09f 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitRequest.java @@ -24,6 +24,7 @@ import org.apache.kafka.common.message.OffsetCommitResponseData.OffsetCommitResponsePartition; import org.apache.kafka.common.message.OffsetCommitResponseData.OffsetCommitResponseTopic; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.types.Struct; @@ -70,19 +71,9 @@ public String toString() { } } - private final short version; - public OffsetCommitRequest(OffsetCommitRequestData data, short version) { super(ApiKeys.OFFSET_COMMIT, version); this.data = data; - this.version = version; - } - - - public OffsetCommitRequest(Struct struct, short version) { - super(ApiKeys.OFFSET_COMMIT, version); - this.data = new OffsetCommitRequestData(struct, version); - this.version = version; } public OffsetCommitRequestData data() { @@ -152,11 +143,11 @@ public OffsetCommitResponse getErrorResponse(int throttleTimeMs, Throwable e) { } public static OffsetCommitRequest parse(ByteBuffer buffer, short version) { - return new OffsetCommitRequest(ApiKeys.OFFSET_COMMIT.parseRequest(version, buffer), version); + return new OffsetCommitRequest(new OffsetCommitRequestData(new ByteBufferAccessor(buffer), version), version); } @Override protected Struct toStruct() { - return data.toStruct(version); + return data.toStruct(version()); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitResponse.java index 3adeb157c188c..c74c225438bfe 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitResponse.java @@ -20,7 +20,7 @@ import org.apache.kafka.common.message.OffsetCommitResponseData; import org.apache.kafka.common.message.OffsetCommitResponseData.OffsetCommitResponsePartition; import org.apache.kafka.common.message.OffsetCommitResponseData.OffsetCommitResponseTopic; -import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.types.Struct; @@ -105,7 +105,7 @@ public Map errorCounts() { } public static OffsetCommitResponse parse(ByteBuffer buffer, short version) { - return new OffsetCommitResponse(ApiKeys.OFFSET_COMMIT.parseResponse(version, buffer), version); + return new OffsetCommitResponse(new OffsetCommitResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetDeleteRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetDeleteRequest.java index 88827ac5a95e7..73a0ffce6e9e4 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/OffsetDeleteRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetDeleteRequest.java @@ -19,6 +19,7 @@ import org.apache.kafka.common.message.OffsetDeleteRequestData; import org.apache.kafka.common.message.OffsetDeleteResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Struct; @@ -54,11 +55,6 @@ public OffsetDeleteRequest(OffsetDeleteRequestData data, short version) { this.data = data; } - public OffsetDeleteRequest(Struct struct, short version) { - super(ApiKeys.OFFSET_DELETE, version); - this.data = new OffsetDeleteRequestData(struct, version); - } - public AbstractResponse getErrorResponse(int throttleTimeMs, Errors error) { return new OffsetDeleteResponse( new OffsetDeleteResponseData() @@ -73,8 +69,7 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { } public static OffsetDeleteRequest parse(ByteBuffer buffer, short version) { - return new OffsetDeleteRequest(ApiKeys.OFFSET_DELETE.parseRequest(version, buffer), - version); + return new OffsetDeleteRequest(new OffsetDeleteRequestData(new ByteBufferAccessor(buffer), version), version); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetDeleteResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetDeleteResponse.java index 7090000a6ee37..1adec4190d045 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/OffsetDeleteResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetDeleteResponse.java @@ -19,7 +19,7 @@ import org.apache.kafka.common.message.OffsetDeleteResponseData; import org.apache.kafka.common.message.OffsetDeleteResponseData.OffsetDeleteResponsePartition; import org.apache.kafka.common.message.OffsetDeleteResponseData.OffsetDeleteResponseTopic; -import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Struct; @@ -53,13 +53,8 @@ public OffsetDeleteResponse(OffsetDeleteResponseData data) { this.data = data; } - public OffsetDeleteResponse(Struct struct) { - short latestVersion = (short) (OffsetDeleteResponseData.SCHEMAS.length - 1); - this.data = new OffsetDeleteResponseData(struct, latestVersion); - } - public OffsetDeleteResponse(Struct struct, short version) { - this.data = new OffsetDeleteResponseData(struct, version); + this(new OffsetDeleteResponseData(struct, version)); } @Override @@ -86,7 +81,7 @@ public Map errorCounts() { } public static OffsetDeleteResponse parse(ByteBuffer buffer, short version) { - return new OffsetDeleteResponse(ApiKeys.OFFSET_DELETE.parseResponse(version, buffer)); + return new OffsetDeleteResponse(new OffsetDeleteResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchRequest.java index 5ab412498c185..b71b232216672 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchRequest.java @@ -21,6 +21,7 @@ import org.apache.kafka.common.message.OffsetFetchRequestData; import org.apache.kafka.common.message.OffsetFetchRequestData.OffsetFetchRequestTopic; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Struct; @@ -109,11 +110,6 @@ private OffsetFetchRequest(OffsetFetchRequestData data, short version) { this.data = data; } - public OffsetFetchRequest(Struct struct, short version) { - super(ApiKeys.OFFSET_FETCH, version); - this.data = new OffsetFetchRequestData(struct, version); - } - public OffsetFetchResponse getErrorResponse(Errors error) { return getErrorResponse(AbstractResponse.DEFAULT_THROTTLE_TIME, error); } @@ -148,7 +144,7 @@ public OffsetFetchResponse getErrorResponse(int throttleTimeMs, Throwable e) { } public static OffsetFetchRequest parse(ByteBuffer buffer, short version) { - return new OffsetFetchRequest(ApiKeys.OFFSET_FETCH.parseRequest(version, buffer), version); + return new OffsetFetchRequest(new OffsetFetchRequestData(new ByteBufferAccessor(buffer), version), version); } public boolean isAllPartitions() { diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchResponse.java index dbeff6d2ac6f0..309dc317ed075 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchResponse.java @@ -20,7 +20,7 @@ import org.apache.kafka.common.message.OffsetFetchResponseData; import org.apache.kafka.common.message.OffsetFetchResponseData.OffsetFetchResponsePartition; import org.apache.kafka.common.message.OffsetFetchResponseData.OffsetFetchResponseTopic; -import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Struct; @@ -153,25 +153,29 @@ public OffsetFetchResponse(int throttleTimeMs, Errors error, Map= 2 ? Errors.forCode(data.errorCode()) : topLevelError(data); + } - Errors topLevelError = Errors.NONE; + private static Errors topLevelError(OffsetFetchResponseData data) { for (OffsetFetchResponseTopic topic : data.topics()) { for (OffsetFetchResponsePartition partition : topic.partitions()) { Errors partitionError = Errors.forCode(partition.errorCode()); if (partitionError != Errors.NONE && !PARTITION_ERRORS.contains(partitionError)) { - topLevelError = partitionError; - break; + return partitionError; } } } + return Errors.NONE; + } - // for version 2 and later use the top-level error code (in ERROR_CODE_KEY_NAME) from the response. - // for older versions there is no top-level error in the response and all errors are partition errors, - // so if there is a group or coordinator error at the partition level use that as the top-level error. - // this way clients can depend on the top-level error regardless of the offset fetch version. - this.error = version >= 2 ? Errors.forCode(data.errorCode()) : topLevelError; + public OffsetFetchResponse(Struct struct, short version) { + this(new OffsetFetchResponseData(struct, version), version); } @Override @@ -208,7 +212,7 @@ public Map responseData() { } public static OffsetFetchResponse parse(ByteBuffer buffer, short version) { - return new OffsetFetchResponse(ApiKeys.OFFSET_FETCH.parseResponse(version, buffer), version); + return new OffsetFetchResponse(new OffsetFetchResponseData(new ByteBufferAccessor(buffer), version), version); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/RenewDelegationTokenRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/RenewDelegationTokenRequest.java index 2c83f06adda62..2000e9ef55641 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/RenewDelegationTokenRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/RenewDelegationTokenRequest.java @@ -21,6 +21,7 @@ import org.apache.kafka.common.message.RenewDelegationTokenRequestData; import org.apache.kafka.common.message.RenewDelegationTokenResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.types.Struct; @@ -33,13 +34,9 @@ public RenewDelegationTokenRequest(RenewDelegationTokenRequestData data, short v this.data = data; } - public RenewDelegationTokenRequest(Struct struct, short version) { - super(ApiKeys.RENEW_DELEGATION_TOKEN, version); - this.data = new RenewDelegationTokenRequestData(struct, version); - } - public static RenewDelegationTokenRequest parse(ByteBuffer buffer, short version) { - return new RenewDelegationTokenRequest(ApiKeys.RENEW_DELEGATION_TOKEN.parseRequest(version, buffer), version); + return new RenewDelegationTokenRequest(new RenewDelegationTokenRequestData( + new ByteBufferAccessor(buffer), version), version); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/RenewDelegationTokenResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/RenewDelegationTokenResponse.java index b454db251eb0d..468fbf33c6ce2 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/RenewDelegationTokenResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/RenewDelegationTokenResponse.java @@ -21,7 +21,7 @@ import java.util.Map; import org.apache.kafka.common.message.RenewDelegationTokenResponseData; -import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Struct; @@ -35,11 +35,12 @@ public RenewDelegationTokenResponse(RenewDelegationTokenResponseData data) { } public RenewDelegationTokenResponse(Struct struct, short version) { - data = new RenewDelegationTokenResponseData(struct, version); + this(new RenewDelegationTokenResponseData(struct, version)); } public static RenewDelegationTokenResponse parse(ByteBuffer buffer, short version) { - return new RenewDelegationTokenResponse(ApiKeys.RENEW_DELEGATION_TOKEN.responseSchema(version).read(buffer), version); + return new RenewDelegationTokenResponse(new RenewDelegationTokenResponseData( + new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/RequestContext.java b/clients/src/main/java/org/apache/kafka/common/requests/RequestContext.java index 1c958ba5c848b..0c8effe07dcaf 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/RequestContext.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/RequestContext.java @@ -21,7 +21,6 @@ import org.apache.kafka.common.network.ListenerName; import org.apache.kafka.common.network.Send; import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.security.auth.KafkaPrincipal; import org.apache.kafka.common.security.auth.SecurityProtocol; diff --git a/clients/src/main/java/org/apache/kafka/common/requests/SaslAuthenticateRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/SaslAuthenticateRequest.java index d136e78ee1659..a65e87d62ecaf 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/SaslAuthenticateRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/SaslAuthenticateRequest.java @@ -19,6 +19,7 @@ import org.apache.kafka.common.message.SaslAuthenticateRequestData; import org.apache.kafka.common.message.SaslAuthenticateResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; @@ -56,22 +57,10 @@ public String toString() { } private final SaslAuthenticateRequestData data; - private final short version; - - public SaslAuthenticateRequest(SaslAuthenticateRequestData data) { - this(data, ApiKeys.SASL_AUTHENTICATE.latestVersion()); - } public SaslAuthenticateRequest(SaslAuthenticateRequestData data, short version) { super(ApiKeys.SASL_AUTHENTICATE, version); this.data = data; - this.version = version; - } - - public SaslAuthenticateRequest(Struct struct, short version) { - super(ApiKeys.SASL_AUTHENTICATE, version); - this.data = new SaslAuthenticateRequestData(struct, version); - this.version = version; } public SaslAuthenticateRequestData data() { @@ -87,12 +76,13 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { } public static SaslAuthenticateRequest parse(ByteBuffer buffer, short version) { - return new SaslAuthenticateRequest(ApiKeys.SASL_AUTHENTICATE.parseRequest(version, buffer), version); + return new SaslAuthenticateRequest(new SaslAuthenticateRequestData(new ByteBufferAccessor(buffer), version), + version); } @Override protected Struct toStruct() { - return data.toStruct(version); + return data.toStruct(version()); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/SaslAuthenticateResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/SaslAuthenticateResponse.java index 530d0170909b1..992b7d9eef7ef 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/SaslAuthenticateResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/SaslAuthenticateResponse.java @@ -17,7 +17,7 @@ package org.apache.kafka.common.requests; import org.apache.kafka.common.message.SaslAuthenticateResponseData; -import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Struct; @@ -79,6 +79,6 @@ protected Message data() { } public static SaslAuthenticateResponse parse(ByteBuffer buffer, short version) { - return new SaslAuthenticateResponse(ApiKeys.SASL_AUTHENTICATE.parseResponse(version, buffer), version); + return new SaslAuthenticateResponse(new SaslAuthenticateResponseData(new ByteBufferAccessor(buffer), version)); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/SaslHandshakeRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/SaslHandshakeRequest.java index be7f4f81800a7..4d3fbc5363d76 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/SaslHandshakeRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/SaslHandshakeRequest.java @@ -20,6 +20,7 @@ import org.apache.kafka.common.message.SaslHandshakeRequestData; import org.apache.kafka.common.message.SaslHandshakeResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; @@ -55,22 +56,10 @@ public String toString() { } private final SaslHandshakeRequestData data; - private final short version; - - public SaslHandshakeRequest(SaslHandshakeRequestData data) { - this(data, ApiKeys.SASL_HANDSHAKE.latestVersion()); - } public SaslHandshakeRequest(SaslHandshakeRequestData data, short version) { super(ApiKeys.SASL_HANDSHAKE, version); this.data = data; - this.version = version; - } - - public SaslHandshakeRequest(Struct struct, short version) { - super(ApiKeys.SASL_HANDSHAKE, version); - this.data = new SaslHandshakeRequestData(struct, version); - this.version = version; } public SaslHandshakeRequestData data() { @@ -85,12 +74,12 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { } public static SaslHandshakeRequest parse(ByteBuffer buffer, short version) { - return new SaslHandshakeRequest(ApiKeys.SASL_HANDSHAKE.parseRequest(version, buffer), version); + return new SaslHandshakeRequest(new SaslHandshakeRequestData(new ByteBufferAccessor(buffer), version), version); } @Override protected Struct toStruct() { - return data.toStruct(version); + return data.toStruct(version()); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/SaslHandshakeResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/SaslHandshakeResponse.java index 40364ece7ec7c..1dd53762197a8 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/SaslHandshakeResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/SaslHandshakeResponse.java @@ -17,7 +17,7 @@ package org.apache.kafka.common.requests; import org.apache.kafka.common.message.SaslHandshakeResponseData; -import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Struct; @@ -72,6 +72,6 @@ public List enabledMechanisms() { } public static SaslHandshakeResponse parse(ByteBuffer buffer, short version) { - return new SaslHandshakeResponse(ApiKeys.SASL_HANDSHAKE.parseResponse(version, buffer), version); + return new SaslHandshakeResponse(new SaslHandshakeResponseData(new ByteBufferAccessor(buffer), version)); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaRequest.java index f93b001a7d9d8..250038b473406 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaRequest.java @@ -23,6 +23,7 @@ import org.apache.kafka.common.message.StopReplicaResponseData; import org.apache.kafka.common.message.StopReplicaResponseData.StopReplicaPartitionError; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Struct; @@ -99,10 +100,6 @@ private StopReplicaRequest(StopReplicaRequestData data, short version) { this.data = data; } - public StopReplicaRequest(Struct struct, short version) { - this(new StopReplicaRequestData(struct, version), version); - } - @Override public StopReplicaResponse getErrorResponse(int throttleTimeMs, Throwable e) { Errors error = Errors.forException(e); @@ -167,7 +164,7 @@ public long brokerEpoch() { } public static StopReplicaRequest parse(ByteBuffer buffer, short version) { - return new StopReplicaRequest(ApiKeys.STOP_REPLICA.parseRequest(version, buffer), version); + return new StopReplicaRequest(new StopReplicaRequestData(new ByteBufferAccessor(buffer), version), version); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaResponse.java index 08d80ffc2ad94..32f387a0ed06a 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaResponse.java @@ -18,7 +18,7 @@ import org.apache.kafka.common.message.StopReplicaResponseData; import org.apache.kafka.common.message.StopReplicaResponseData.StopReplicaPartitionError; -import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Struct; @@ -64,7 +64,7 @@ public Map errorCounts() { } public static StopReplicaResponse parse(ByteBuffer buffer, short version) { - return new StopReplicaResponse(ApiKeys.STOP_REPLICA.parseResponse(version, buffer), version); + return new StopReplicaResponse(new StopReplicaResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/SyncGroupRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/SyncGroupRequest.java index daedad7594758..9819eb49e019a 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/SyncGroupRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/SyncGroupRequest.java @@ -20,6 +20,7 @@ import org.apache.kafka.common.message.SyncGroupRequestData; import org.apache.kafka.common.message.SyncGroupResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Struct; @@ -61,11 +62,6 @@ public SyncGroupRequest(SyncGroupRequestData data, short version) { this.data = data; } - public SyncGroupRequest(Struct struct, short version) { - super(ApiKeys.SYNC_GROUP, version); - this.data = new SyncGroupRequestData(struct, version); - } - @Override public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { short versionId = version(); @@ -100,7 +96,7 @@ public Map groupAssignments() { } public static SyncGroupRequest parse(ByteBuffer buffer, short version) { - return new SyncGroupRequest(ApiKeys.SYNC_GROUP.parseRequest(version, buffer), version); + return new SyncGroupRequest(new SyncGroupRequestData(new ByteBufferAccessor(buffer), version), version); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/SyncGroupResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/SyncGroupResponse.java index 97b765aad186e..0806e58840ee7 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/SyncGroupResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/SyncGroupResponse.java @@ -17,7 +17,7 @@ package org.apache.kafka.common.requests; import org.apache.kafka.common.message.SyncGroupResponseData; -import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Struct; @@ -34,11 +34,6 @@ public SyncGroupResponse(SyncGroupResponseData data) { this.data = data; } - public SyncGroupResponse(Struct struct) { - short latestVersion = (short) (SyncGroupResponseData.SCHEMAS.length - 1); - this.data = new SyncGroupResponseData(struct, latestVersion); - } - public SyncGroupResponse(Struct struct, short version) { this.data = new SyncGroupResponseData(struct, version); } @@ -68,7 +63,7 @@ protected Message data() { } public static SyncGroupResponse parse(ByteBuffer buffer, short version) { - return new SyncGroupResponse(ApiKeys.SYNC_GROUP.parseResponse(version, buffer)); + return new SyncGroupResponse(new SyncGroupResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitRequest.java index b8292946fda6f..695684e119c10 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitRequest.java @@ -24,6 +24,7 @@ import org.apache.kafka.common.message.TxnOffsetCommitResponseData.TxnOffsetCommitResponsePartition; import org.apache.kafka.common.message.TxnOffsetCommitResponseData.TxnOffsetCommitResponseTopic; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Struct; @@ -67,11 +68,6 @@ public TxnOffsetCommitRequest(TxnOffsetCommitRequestData data, short version) { this.data = data; } - public TxnOffsetCommitRequest(Struct struct, short version) { - super(ApiKeys.TXN_OFFSET_COMMIT, version); - this.data = new TxnOffsetCommitRequestData(struct, version); - } - public Map offsets() { List topics = data.topics(); Map offsetMap = new HashMap<>(); @@ -149,7 +145,8 @@ public TxnOffsetCommitResponse getErrorResponse(int throttleTimeMs, Throwable e) } public static TxnOffsetCommitRequest parse(ByteBuffer buffer, short version) { - return new TxnOffsetCommitRequest(ApiKeys.TXN_OFFSET_COMMIT.parseRequest(version, buffer), version); + return new TxnOffsetCommitRequest(new TxnOffsetCommitRequestData( + new ByteBufferAccessor(buffer), version), version); } public static class CommittedOffset { diff --git a/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitResponse.java index 80f7673e23a18..b86029421b3ce 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitResponse.java @@ -20,7 +20,7 @@ import org.apache.kafka.common.message.TxnOffsetCommitResponseData; import org.apache.kafka.common.message.TxnOffsetCommitResponseData.TxnOffsetCommitResponsePartition; import org.apache.kafka.common.message.TxnOffsetCommitResponseData.TxnOffsetCommitResponseTopic; -import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Struct; @@ -109,7 +109,7 @@ public Map errors() { } public static TxnOffsetCommitResponse parse(ByteBuffer buffer, short version) { - return new TxnOffsetCommitResponse(ApiKeys.TXN_OFFSET_COMMIT.parseResponse(version, buffer), version); + return new TxnOffsetCommitResponse(new TxnOffsetCommitResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java index 5b5b514f920c4..23a69f5544dc8 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java @@ -25,6 +25,7 @@ import org.apache.kafka.common.message.UpdateMetadataResponseData; import org.apache.kafka.common.network.ListenerName; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Struct; @@ -167,10 +168,6 @@ private static String listenerNameFromSecurityProtocol(UpdateMetadataEndpoint en return ListenerName.forSecurityProtocol(securityProtocol).value(); } - public UpdateMetadataRequest(Struct struct, short version) { - this(new UpdateMetadataRequestData(struct, version), version); - } - @Override public int controllerId() { return data.controllerId(); @@ -219,6 +216,6 @@ protected Message data() { } public static UpdateMetadataRequest parse(ByteBuffer buffer, short version) { - return new UpdateMetadataRequest(ApiKeys.UPDATE_METADATA.parseRequest(version, buffer), version); + return new UpdateMetadataRequest(new UpdateMetadataRequestData(new ByteBufferAccessor(buffer), version), version); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataResponse.java index 6a5a7f5cf13e6..50ddcf174720f 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataResponse.java @@ -17,7 +17,7 @@ package org.apache.kafka.common.requests; import org.apache.kafka.common.message.UpdateMetadataResponseData; -import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.types.Struct; @@ -47,7 +47,7 @@ public Map errorCounts() { } public static UpdateMetadataResponse parse(ByteBuffer buffer, short version) { - return new UpdateMetadataResponse(ApiKeys.UPDATE_METADATA.parseResponse(version, buffer), version); + return new UpdateMetadataResponse(new UpdateMetadataResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java index b6fece991d247..ab34505cc6592 100644 --- a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java @@ -80,6 +80,7 @@ import org.apache.kafka.common.message.OffsetDeleteResponseData.OffsetDeleteResponsePartitionCollection; import org.apache.kafka.common.message.OffsetDeleteResponseData.OffsetDeleteResponseTopic; import org.apache.kafka.common.message.OffsetDeleteResponseData.OffsetDeleteResponseTopicCollection; +import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.requests.AlterPartitionReassignmentsResponse; import org.apache.kafka.common.requests.ApiError; @@ -756,7 +757,8 @@ public void testElectLeaders() throws Exception { electionResults.add(electionResult); - env.kafkaClient().prepareResponse(new ElectLeadersResponse(0, Errors.NONE.code(), electionResults)); + env.kafkaClient().prepareResponse(new ElectLeadersResponse(0, Errors.NONE.code(), + electionResults, ApiKeys.ELECT_LEADERS.latestVersion())); ElectLeadersResult results = env.adminClient().electLeaders( electionType, new HashSet<>(asList(topic1, topic2))); @@ -769,7 +771,8 @@ public void testElectLeaders() throws Exception { partition2Result.setErrorCode(ApiError.NONE.error().code()); partition2Result.setErrorMessage(ApiError.NONE.message()); - env.kafkaClient().prepareResponse(new ElectLeadersResponse(0, Errors.NONE.code(), electionResults)); + env.kafkaClient().prepareResponse(new ElectLeadersResponse(0, Errors.NONE.code(), electionResults, + ApiKeys.ELECT_LEADERS.latestVersion())); results = env.adminClient().electLeaders(electionType, new HashSet<>(asList(topic1, topic2))); assertFalse(results.partitions().get().get(topic1).isPresent()); assertFalse(results.partitions().get().get(topic2).isPresent()); diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java index fe45f6a19d0f6..1d6dbdf4fa97c 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java @@ -770,7 +770,7 @@ public void testFetchRequestWithMetadata() throws Exception { public void testJoinGroupRequestVersion0RebalanceTimeout() { final short version = 0; JoinGroupRequest jgr = createJoinGroupRequest(version); - JoinGroupRequest jgr2 = new JoinGroupRequest(jgr.toStruct(), version); + JoinGroupRequest jgr2 = JoinGroupRequest.parse(jgr.serializeBody(), version); assertEquals(jgr2.data().rebalanceTimeoutMs(), jgr.data().rebalanceTimeoutMs()); } @@ -1427,7 +1427,7 @@ private SaslHandshakeResponse createSaslHandshakeResponse() { private SaslAuthenticateRequest createSaslAuthenticateRequest() { SaslAuthenticateRequestData data = new SaslAuthenticateRequestData().setAuthBytes(new byte[0]); - return new SaslAuthenticateRequest(data); + return new SaslAuthenticateRequest(data, ApiKeys.SASL_AUTHENTICATE.latestVersion()); } private SaslAuthenticateResponse createSaslAuthenticateResponse() { @@ -1857,7 +1857,7 @@ private ElectLeadersResponse createElectLeadersResponse() { partitionResult.setErrorMessage(Errors.UNKNOWN_TOPIC_OR_PARTITION.message()); electionResult.partitionResult().add(partitionResult); - return new ElectLeadersResponse(200, Errors.NONE.code(), electionResults); + return new ElectLeadersResponse(200, Errors.NONE.code(), electionResults, ApiKeys.ELECT_LEADERS.latestVersion()); } private IncrementalAlterConfigsRequest createIncrementalAlterConfigsRequest() { diff --git a/core/src/test/scala/unit/kafka/server/AbstractCreateTopicsRequestTest.scala b/core/src/test/scala/unit/kafka/server/AbstractCreateTopicsRequestTest.scala index d0dbd8e5ab9c4..8451470f20bd5 100644 --- a/core/src/test/scala/unit/kafka/server/AbstractCreateTopicsRequestTest.scala +++ b/core/src/test/scala/unit/kafka/server/AbstractCreateTopicsRequestTest.scala @@ -167,7 +167,7 @@ class AbstractCreateTopicsRequestTest extends BaseRequestTest { val firstTopic = topics(0).asInstanceOf[Struct] firstTopic.set("num_partitions", 1) firstTopic.set("replication_factor", 1.toShort) - new CreateTopicsRequest(struct, request.version) + new CreateTopicsRequest(new CreateTopicsRequestData(struct, request.version), request.version) } protected def validateErrorCreateTopicsRequests(request: CreateTopicsRequest, diff --git a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala index f34eae45b1e4d..8f0012bb177c3 100644 --- a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala +++ b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala @@ -513,7 +513,7 @@ class RequestQuotaTest extends BaseRequestTest { case class Client(clientId: String, apiKey: ApiKeys) { var correlationId: Int = 0 val builder = requestBuilder(apiKey) - def runUntil(until: (Struct) => Boolean): Boolean = { + def runUntil(until: Struct => Boolean): Boolean = { val startMs = System.currentTimeMillis var done = false val socket = connect() @@ -561,56 +561,50 @@ class RequestQuotaTest extends BaseRequestTest { } } - private def responseThrottleTime(apiKey: ApiKeys, response: Struct): Int = { + private def responseThrottleTime(apiKey: ApiKeys, responseStruct: Struct): Int = { + val apiVersion = apiKey.latestVersion apiKey match { - case ApiKeys.PRODUCE => new ProduceResponse(response).throttleTimeMs - case ApiKeys.FETCH => FetchResponse.parse(response).throttleTimeMs - case ApiKeys.LIST_OFFSETS => new ListOffsetResponse(response).throttleTimeMs - case ApiKeys.METADATA => - new MetadataResponse(response, ApiKeys.DESCRIBE_GROUPS.latestVersion).throttleTimeMs - case ApiKeys.OFFSET_COMMIT => - new OffsetCommitResponse(response, ApiKeys.OFFSET_COMMIT.latestVersion).throttleTimeMs - case ApiKeys.OFFSET_FETCH => new OffsetFetchResponse(response, ApiKeys.OFFSET_FETCH.latestVersion).throttleTimeMs - case ApiKeys.FIND_COORDINATOR => - new FindCoordinatorResponse(response, ApiKeys.FIND_COORDINATOR.latestVersion).throttleTimeMs - case ApiKeys.JOIN_GROUP => new JoinGroupResponse(response).throttleTimeMs - case ApiKeys.HEARTBEAT => new HeartbeatResponse(response, ApiKeys.HEARTBEAT.latestVersion).throttleTimeMs - case ApiKeys.LEAVE_GROUP => new LeaveGroupResponse(response).throttleTimeMs - case ApiKeys.SYNC_GROUP => new SyncGroupResponse(response).throttleTimeMs - case ApiKeys.DESCRIBE_GROUPS => - new DescribeGroupsResponse(response, ApiKeys.DESCRIBE_GROUPS.latestVersion).throttleTimeMs - case ApiKeys.LIST_GROUPS => new ListGroupsResponse(response, ApiKeys.LIST_GROUPS.latestVersion).throttleTimeMs - case ApiKeys.API_VERSIONS => new ApiVersionsResponse(response).throttleTimeMs - case ApiKeys.CREATE_TOPICS => - new CreateTopicsResponse(response, ApiKeys.CREATE_TOPICS.latestVersion).throttleTimeMs - case ApiKeys.DELETE_TOPICS => - new DeleteTopicsResponse(response, ApiKeys.DELETE_TOPICS.latestVersion).throttleTimeMs - case ApiKeys.DELETE_RECORDS => new DeleteRecordsResponse(response).throttleTimeMs - case ApiKeys.INIT_PRODUCER_ID => new InitProducerIdResponse(response, ApiKeys.INIT_PRODUCER_ID.latestVersion).throttleTimeMs - case ApiKeys.ADD_PARTITIONS_TO_TXN => new AddPartitionsToTxnResponse(response).throttleTimeMs - case ApiKeys.ADD_OFFSETS_TO_TXN => new AddOffsetsToTxnResponse(response).throttleTimeMs - case ApiKeys.END_TXN => new EndTxnResponse(response).throttleTimeMs - case ApiKeys.TXN_OFFSET_COMMIT => new TxnOffsetCommitResponse(response, ApiKeys.TXN_OFFSET_COMMIT.latestVersion).throttleTimeMs - case ApiKeys.DESCRIBE_ACLS => new DescribeAclsResponse(response).throttleTimeMs - case ApiKeys.CREATE_ACLS => new CreateAclsResponse(response).throttleTimeMs - case ApiKeys.DELETE_ACLS => new DeleteAclsResponse(response).throttleTimeMs - case ApiKeys.DESCRIBE_CONFIGS => new DescribeConfigsResponse(response).throttleTimeMs - case ApiKeys.ALTER_CONFIGS => new AlterConfigsResponse(response).throttleTimeMs - case ApiKeys.ALTER_REPLICA_LOG_DIRS => new AlterReplicaLogDirsResponse(response).throttleTimeMs - case ApiKeys.DESCRIBE_LOG_DIRS => new DescribeLogDirsResponse(response).throttleTimeMs - case ApiKeys.CREATE_PARTITIONS => new CreatePartitionsResponse(response).throttleTimeMs - case ApiKeys.CREATE_DELEGATION_TOKEN => new CreateDelegationTokenResponse(response, ApiKeys.CREATE_DELEGATION_TOKEN.latestVersion).throttleTimeMs - case ApiKeys.DESCRIBE_DELEGATION_TOKEN=> new DescribeDelegationTokenResponse(response, ApiKeys.DESCRIBE_DELEGATION_TOKEN.latestVersion).throttleTimeMs - case ApiKeys.RENEW_DELEGATION_TOKEN => new RenewDelegationTokenResponse(response, ApiKeys.RENEW_DELEGATION_TOKEN.latestVersion).throttleTimeMs - case ApiKeys.EXPIRE_DELEGATION_TOKEN => new ExpireDelegationTokenResponse(response, ApiKeys.EXPIRE_DELEGATION_TOKEN.latestVersion).throttleTimeMs - case ApiKeys.DELETE_GROUPS => new DeleteGroupsResponse(response).throttleTimeMs - case ApiKeys.OFFSET_FOR_LEADER_EPOCH => new OffsetsForLeaderEpochResponse(response).throttleTimeMs - case ApiKeys.ELECT_LEADERS => new ElectLeadersResponse(response).throttleTimeMs - case ApiKeys.INCREMENTAL_ALTER_CONFIGS => - new IncrementalAlterConfigsResponse(response, ApiKeys.INCREMENTAL_ALTER_CONFIGS.latestVersion()).throttleTimeMs - case ApiKeys.ALTER_PARTITION_REASSIGNMENTS => new AlterPartitionReassignmentsResponse(response).throttleTimeMs - case ApiKeys.LIST_PARTITION_REASSIGNMENTS => new ListPartitionReassignmentsResponse(response).throttleTimeMs - case ApiKeys.OFFSET_DELETE => new OffsetDeleteResponse(response).throttleTimeMs() + case ApiKeys.PRODUCE => new ProduceResponse(responseStruct).throttleTimeMs + case ApiKeys.FETCH => FetchResponse.parse(responseStruct).throttleTimeMs + case ApiKeys.LIST_OFFSETS => new ListOffsetResponse(responseStruct).throttleTimeMs + case ApiKeys.METADATA => new MetadataResponse(responseStruct, apiVersion).throttleTimeMs + case ApiKeys.OFFSET_COMMIT => new OffsetCommitResponse(responseStruct, apiVersion).throttleTimeMs + case ApiKeys.OFFSET_FETCH => new OffsetFetchResponse(responseStruct, apiVersion).throttleTimeMs + case ApiKeys.FIND_COORDINATOR => new FindCoordinatorResponse(responseStruct, apiVersion).throttleTimeMs + case ApiKeys.JOIN_GROUP => new JoinGroupResponse(responseStruct, apiVersion).throttleTimeMs + case ApiKeys.HEARTBEAT => new HeartbeatResponse(responseStruct, apiVersion).throttleTimeMs + case ApiKeys.LEAVE_GROUP => new LeaveGroupResponse(responseStruct, apiVersion).throttleTimeMs + case ApiKeys.SYNC_GROUP => new SyncGroupResponse(responseStruct, apiVersion).throttleTimeMs + case ApiKeys.DESCRIBE_GROUPS => new DescribeGroupsResponse(responseStruct, apiVersion).throttleTimeMs + case ApiKeys.LIST_GROUPS => new ListGroupsResponse(responseStruct, apiVersion).throttleTimeMs + case ApiKeys.API_VERSIONS => new ApiVersionsResponse(responseStruct).throttleTimeMs + case ApiKeys.CREATE_TOPICS => new CreateTopicsResponse(responseStruct, apiVersion).throttleTimeMs + case ApiKeys.DELETE_TOPICS => new DeleteTopicsResponse(responseStruct, apiVersion).throttleTimeMs + case ApiKeys.DELETE_RECORDS => new DeleteRecordsResponse(responseStruct).throttleTimeMs + case ApiKeys.INIT_PRODUCER_ID => new InitProducerIdResponse(responseStruct, apiVersion).throttleTimeMs + case ApiKeys.ADD_PARTITIONS_TO_TXN => new AddPartitionsToTxnResponse(responseStruct).throttleTimeMs + case ApiKeys.ADD_OFFSETS_TO_TXN => new AddOffsetsToTxnResponse(responseStruct).throttleTimeMs + case ApiKeys.END_TXN => new EndTxnResponse(responseStruct).throttleTimeMs + case ApiKeys.TXN_OFFSET_COMMIT => new TxnOffsetCommitResponse(responseStruct, apiVersion).throttleTimeMs + case ApiKeys.DESCRIBE_ACLS => new DescribeAclsResponse(responseStruct).throttleTimeMs + case ApiKeys.CREATE_ACLS => new CreateAclsResponse(responseStruct).throttleTimeMs + case ApiKeys.DELETE_ACLS => new DeleteAclsResponse(responseStruct).throttleTimeMs + case ApiKeys.DESCRIBE_CONFIGS => new DescribeConfigsResponse(responseStruct).throttleTimeMs + case ApiKeys.ALTER_CONFIGS => new AlterConfigsResponse(responseStruct).throttleTimeMs + case ApiKeys.ALTER_REPLICA_LOG_DIRS => new AlterReplicaLogDirsResponse(responseStruct).throttleTimeMs + case ApiKeys.DESCRIBE_LOG_DIRS => new DescribeLogDirsResponse(responseStruct).throttleTimeMs + case ApiKeys.CREATE_PARTITIONS => new CreatePartitionsResponse(responseStruct).throttleTimeMs + case ApiKeys.CREATE_DELEGATION_TOKEN => new CreateDelegationTokenResponse(responseStruct, apiVersion).throttleTimeMs + case ApiKeys.DESCRIBE_DELEGATION_TOKEN=> new DescribeDelegationTokenResponse(responseStruct, apiVersion).throttleTimeMs + case ApiKeys.RENEW_DELEGATION_TOKEN => new RenewDelegationTokenResponse(responseStruct, apiVersion).throttleTimeMs + case ApiKeys.EXPIRE_DELEGATION_TOKEN => new ExpireDelegationTokenResponse(responseStruct, apiVersion).throttleTimeMs + case ApiKeys.DELETE_GROUPS => new DeleteGroupsResponse(responseStruct, apiVersion).throttleTimeMs + case ApiKeys.OFFSET_FOR_LEADER_EPOCH => new OffsetsForLeaderEpochResponse(responseStruct).throttleTimeMs + case ApiKeys.ELECT_LEADERS => new ElectLeadersResponse(responseStruct, apiVersion).throttleTimeMs + case ApiKeys.INCREMENTAL_ALTER_CONFIGS => new IncrementalAlterConfigsResponse(responseStruct, apiVersion).throttleTimeMs + case ApiKeys.ALTER_PARTITION_REASSIGNMENTS => new AlterPartitionReassignmentsResponse(responseStruct, apiVersion).throttleTimeMs + case ApiKeys.LIST_PARTITION_REASSIGNMENTS => new ListPartitionReassignmentsResponse(responseStruct, apiVersion).throttleTimeMs + case ApiKeys.OFFSET_DELETE => new OffsetDeleteResponse(responseStruct, apiVersion).throttleTimeMs() case requestId => throw new IllegalArgumentException(s"No throttle time for $requestId") } } @@ -621,10 +615,9 @@ class RequestQuotaTest extends BaseRequestTest { val clientId = apiKey.toString val client = Client(clientId, apiKey) - val throttled = client.runUntil(response => { + val throttled = client.runUntil { response => responseThrottleTime(apiKey, response) > 0 } - ) assertTrue(s"Response not throttled: $client", throttled) assertTrue(s"Throttle time metrics not updated: $client" , throttleTimeMetricValue(clientId) > 0) diff --git a/core/src/test/scala/unit/kafka/server/SaslApiVersionsRequestTest.scala b/core/src/test/scala/unit/kafka/server/SaslApiVersionsRequestTest.scala index b9510d483bd4f..9853b4051e911 100644 --- a/core/src/test/scala/unit/kafka/server/SaslApiVersionsRequestTest.scala +++ b/core/src/test/scala/unit/kafka/server/SaslApiVersionsRequestTest.scala @@ -96,7 +96,8 @@ class SaslApiVersionsRequestTest extends BaseRequestTest with SaslSetup { } private def sendSaslHandshakeRequestValidateResponse(socket: Socket): Unit = { - val request = new SaslHandshakeRequest(new SaslHandshakeRequestData().setMechanism("PLAIN")) + val request = new SaslHandshakeRequest(new SaslHandshakeRequestData().setMechanism("PLAIN"), + ApiKeys.SASL_HANDSHAKE.latestVersion) val response = sendAndReceive(request, ApiKeys.SASL_HANDSHAKE, socket) val handshakeResponse = SaslHandshakeResponse.parse(response, request.version) assertEquals(Errors.NONE, handshakeResponse.error) From 98dbf1d079e3196595ecc8cbc3c7a292927b578b Mon Sep 17 00:00:00 2001 From: Ismael Juma Date: Tue, 1 Oct 2019 17:23:11 -0700 Subject: [PATCH 10/51] Avoid intermediate Struct when deserializing responses --- .../apache/kafka/clients/NetworkClient.java | 40 +++---- .../common/requests/AbstractResponse.java | 102 +++++++++--------- .../AlterPartitionReassignmentsResponse.java | 4 - .../requests/ControlledShutdownResponse.java | 9 +- .../CreateDelegationTokenResponse.java | 4 - .../common/requests/CreateTopicsResponse.java | 4 - .../common/requests/DeleteGroupsResponse.java | 4 - .../common/requests/DeleteTopicsResponse.java | 4 - .../DescribeDelegationTokenResponse.java | 4 - .../requests/DescribeGroupsResponse.java | 4 - .../common/requests/ElectLeadersResponse.java | 4 - .../ExpireDelegationTokenResponse.java | 4 - .../requests/FindCoordinatorResponse.java | 4 - .../common/requests/HeartbeatResponse.java | 4 - .../IncrementalAlterConfigsResponse.java | 4 - .../requests/InitProducerIdResponse.java | 4 - .../common/requests/JoinGroupResponse.java | 4 - .../common/requests/LeaderAndIsrResponse.java | 9 +- .../common/requests/LeaveGroupResponse.java | 4 - .../common/requests/ListGroupsResponse.java | 4 - .../ListPartitionReassignmentsResponse.java | 4 - .../common/requests/MetadataResponse.java | 4 - .../common/requests/OffsetCommitResponse.java | 9 -- .../common/requests/OffsetDeleteResponse.java | 4 - .../common/requests/OffsetFetchResponse.java | 4 - .../RenewDelegationTokenResponse.java | 4 - .../requests/SaslAuthenticateResponse.java | 9 +- .../requests/SaslHandshakeResponse.java | 9 +- .../common/requests/StopReplicaResponse.java | 9 +- .../common/requests/SyncGroupResponse.java | 4 - .../requests/TxnOffsetCommitResponse.java | 4 - .../requests/UpdateMetadataResponse.java | 9 +- .../requests/WriteTxnMarkersResponse.java | 5 + .../SaslClientAuthenticator.java | 2 +- .../requests/LeaveGroupResponseTest.java | 29 +---- .../requests/OffsetCommitResponseTest.java | 36 ------- .../requests/OffsetFetchResponseTest.java | 7 +- .../common/requests/RequestContextTest.java | 6 +- .../common/requests/RequestResponseTest.java | 5 +- .../requests/TxnOffsetCommitResponseTest.java | 32 ------ .../authenticator/SaslAuthenticatorTest.java | 2 +- .../unit/kafka/server/BaseRequestTest.scala | 7 +- .../unit/kafka/server/KafkaApisTest.scala | 5 +- ...aMetricReporterExceptionHandlingTest.scala | 5 +- .../unit/kafka/server/RequestQuotaTest.scala | 68 ++---------- 45 files changed, 128 insertions(+), 378 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java b/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java index d782df8652e5d..a61ee65fca938 100644 --- a/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java +++ b/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java @@ -31,8 +31,6 @@ import org.apache.kafka.common.network.Send; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.CommonFields; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.requests.AbstractRequest; import org.apache.kafka.common.requests.AbstractResponse; import org.apache.kafka.common.requests.ApiVersionsRequest; @@ -698,22 +696,21 @@ public Node leastLoadedNode(long now) { } } - public static AbstractResponse parseResponse(ByteBuffer responseBuffer, RequestHeader requestHeader) { - Struct responseStruct = parseStructMaybeUpdateThrottleTimeMetrics(responseBuffer, requestHeader, null, 0); - return AbstractResponse.parseResponse(requestHeader.apiKey(), responseStruct, - requestHeader.apiVersion()); + public static AbstractResponse parseResponse(RequestHeader requestHeader, ByteBuffer responseBuffer) { + return parseResponse(requestHeader, responseBuffer, null, 0); } - private static Struct parseStructMaybeUpdateThrottleTimeMetrics(ByteBuffer responseBuffer, RequestHeader requestHeader, - Sensor throttleTimeSensor, long now) { + private static AbstractResponse parseResponse(RequestHeader requestHeader, ByteBuffer responseBuffer, + Sensor throttleTimeSensor, long now) { ResponseHeader responseHeader = ResponseHeader.parse(responseBuffer, requestHeader.apiKey().responseHeaderVersion(requestHeader.apiVersion())); // Always expect the response version id to be the same as the request version id - Struct responseBody = requestHeader.apiKey().parseResponse(requestHeader.apiVersion(), responseBuffer); + AbstractResponse response = AbstractResponse.parseResponse(requestHeader.apiKey(), responseBuffer, + requestHeader.apiVersion()); correlate(requestHeader, responseHeader); - if (throttleTimeSensor != null && responseBody.hasField(CommonFields.THROTTLE_TIME_MS)) - throttleTimeSensor.record(responseBody.get(CommonFields.THROTTLE_TIME_MS), now); - return responseBody; + if (throttleTimeSensor != null && response.throttleTimeMs() != AbstractResponse.DEFAULT_THROTTLE_TIME) + throttleTimeSensor.record(response.throttleTimeMs(), now); + return response; } /** @@ -833,22 +830,19 @@ private void handleCompletedReceives(List responses, long now) { for (NetworkReceive receive : this.selector.completedReceives()) { String source = receive.source(); InFlightRequest req = inFlightRequests.completeNext(source); - Struct responseStruct = parseStructMaybeUpdateThrottleTimeMetrics(receive.payload(), req.header, - throttleTimeSensor, now); + AbstractResponse responseBody = parseResponse(req.header, receive.payload(), throttleTimeSensor, now); if (log.isTraceEnabled()) { log.trace("Completed receive from node {} for {} with correlation id {}, received {}", req.destination, - req.header.apiKey(), req.header.correlationId(), responseStruct); + req.header.apiKey(), req.header.correlationId(), responseBody); } // If the received response includes a throttle delay, throttle the connection. - AbstractResponse body = AbstractResponse. - parseResponse(req.header.apiKey(), responseStruct, req.header.apiVersion()); - maybeThrottle(body, req.header.apiVersion(), req.destination, now); - if (req.isInternalRequest && body instanceof MetadataResponse) - metadataUpdater.handleCompletedMetadataResponse(req.header, now, (MetadataResponse) body); - else if (req.isInternalRequest && body instanceof ApiVersionsResponse) - handleApiVersionsResponse(responses, req, now, (ApiVersionsResponse) body); + maybeThrottle(responseBody, req.header.apiVersion(), req.destination, now); + if (req.isInternalRequest && responseBody instanceof MetadataResponse) + metadataUpdater.handleCompletedMetadataResponse(req.header, now, (MetadataResponse) responseBody); + else if (req.isInternalRequest && responseBody instanceof ApiVersionsResponse) + handleApiVersionsResponse(responses, req, now, (ApiVersionsResponse) responseBody); else - responses.add(req.completed(body, now)); + responses.add(req.completed(responseBody, now)); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java index e31f3996f6039..6e12b04eaf052 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java @@ -86,104 +86,104 @@ protected void updateErrorCounts(Map errorCounts, Errors error) protected abstract Struct toStruct(short version); - public static AbstractResponse parseResponse(ApiKeys apiKey, Struct struct, short version) { + public static AbstractResponse parseResponse(ApiKeys apiKey, ByteBuffer responseBuffer, short version) { switch (apiKey) { case PRODUCE: - return new ProduceResponse(struct); + return ProduceResponse.parse(responseBuffer, version); case FETCH: - return FetchResponse.parse(struct); + return FetchResponse.parse(responseBuffer, version); case LIST_OFFSETS: - return new ListOffsetResponse(struct); + return ListOffsetResponse.parse(responseBuffer, version); case METADATA: - return new MetadataResponse(struct, version); + return MetadataResponse.parse(responseBuffer, version); case OFFSET_COMMIT: - return new OffsetCommitResponse(struct, version); + return OffsetCommitResponse.parse(responseBuffer, version); case OFFSET_FETCH: - return new OffsetFetchResponse(struct, version); + return OffsetFetchResponse.parse(responseBuffer, version); case FIND_COORDINATOR: - return new FindCoordinatorResponse(struct, version); + return FindCoordinatorResponse.parse(responseBuffer, version); case JOIN_GROUP: - return new JoinGroupResponse(struct, version); + return JoinGroupResponse.parse(responseBuffer, version); case HEARTBEAT: - return new HeartbeatResponse(struct, version); + return HeartbeatResponse.parse(responseBuffer, version); case LEAVE_GROUP: - return new LeaveGroupResponse(struct, version); + return LeaveGroupResponse.parse(responseBuffer, version); case SYNC_GROUP: - return new SyncGroupResponse(struct, version); + return SyncGroupResponse.parse(responseBuffer, version); case STOP_REPLICA: - return new StopReplicaResponse(struct, version); + return StopReplicaResponse.parse(responseBuffer, version); case CONTROLLED_SHUTDOWN: - return new ControlledShutdownResponse(struct, version); + return ControlledShutdownResponse.parse(responseBuffer, version); case UPDATE_METADATA: - return new UpdateMetadataResponse(struct, version); + return UpdateMetadataResponse.parse(responseBuffer, version); case LEADER_AND_ISR: - return new LeaderAndIsrResponse(struct, version); + return LeaderAndIsrResponse.parse(responseBuffer, version); case DESCRIBE_GROUPS: - return new DescribeGroupsResponse(struct, version); + return DescribeGroupsResponse.parse(responseBuffer, version); case LIST_GROUPS: - return new ListGroupsResponse(struct, version); + return ListGroupsResponse.parse(responseBuffer, version); case SASL_HANDSHAKE: - return new SaslHandshakeResponse(struct, version); + return SaslHandshakeResponse.parse(responseBuffer, version); case API_VERSIONS: - return ApiVersionsResponse.fromStruct(struct, version); + return ApiVersionsResponse.parse(responseBuffer, version); case CREATE_TOPICS: - return new CreateTopicsResponse(struct, version); + return CreateTopicsResponse.parse(responseBuffer, version); case DELETE_TOPICS: - return new DeleteTopicsResponse(struct, version); + return DeleteTopicsResponse.parse(responseBuffer, version); case DELETE_RECORDS: - return new DeleteRecordsResponse(struct); + return DeleteRecordsResponse.parse(responseBuffer, version); case INIT_PRODUCER_ID: - return new InitProducerIdResponse(struct, version); + return InitProducerIdResponse.parse(responseBuffer, version); case OFFSET_FOR_LEADER_EPOCH: - return new OffsetsForLeaderEpochResponse(struct); + return OffsetsForLeaderEpochResponse.parse(responseBuffer, version); case ADD_PARTITIONS_TO_TXN: - return new AddPartitionsToTxnResponse(struct); + return AddPartitionsToTxnResponse.parse(responseBuffer, version); case ADD_OFFSETS_TO_TXN: - return new AddOffsetsToTxnResponse(struct); + return AddOffsetsToTxnResponse.parse(responseBuffer, version); case END_TXN: - return new EndTxnResponse(struct); + return EndTxnResponse.parse(responseBuffer, version); case WRITE_TXN_MARKERS: - return new WriteTxnMarkersResponse(struct); + return WriteTxnMarkersResponse.parse(responseBuffer, version); case TXN_OFFSET_COMMIT: - return new TxnOffsetCommitResponse(struct, version); + return TxnOffsetCommitResponse.parse(responseBuffer, version); case DESCRIBE_ACLS: - return new DescribeAclsResponse(struct); + return DescribeAclsResponse.parse(responseBuffer, version); case CREATE_ACLS: - return new CreateAclsResponse(struct); + return CreateAclsResponse.parse(responseBuffer, version); case DELETE_ACLS: - return new DeleteAclsResponse(struct); + return DeleteAclsResponse.parse(responseBuffer, version); case DESCRIBE_CONFIGS: - return new DescribeConfigsResponse(struct); + return DescribeConfigsResponse.parse(responseBuffer, version); case ALTER_CONFIGS: - return new AlterConfigsResponse(struct); + return AlterConfigsResponse.parse(responseBuffer, version); case ALTER_REPLICA_LOG_DIRS: - return new AlterReplicaLogDirsResponse(struct); + return AlterReplicaLogDirsResponse.parse(responseBuffer, version); case DESCRIBE_LOG_DIRS: - return new DescribeLogDirsResponse(struct); + return DescribeLogDirsResponse.parse(responseBuffer, version); case SASL_AUTHENTICATE: - return new SaslAuthenticateResponse(struct, version); + return SaslAuthenticateResponse.parse(responseBuffer, version); case CREATE_PARTITIONS: - return new CreatePartitionsResponse(struct); + return CreatePartitionsResponse.parse(responseBuffer, version); case CREATE_DELEGATION_TOKEN: - return new CreateDelegationTokenResponse(struct, version); + return CreateDelegationTokenResponse.parse(responseBuffer, version); case RENEW_DELEGATION_TOKEN: - return new RenewDelegationTokenResponse(struct, version); + return RenewDelegationTokenResponse.parse(responseBuffer, version); case EXPIRE_DELEGATION_TOKEN: - return new ExpireDelegationTokenResponse(struct, version); + return ExpireDelegationTokenResponse.parse(responseBuffer, version); case DESCRIBE_DELEGATION_TOKEN: - return new DescribeDelegationTokenResponse(struct, version); + return DescribeDelegationTokenResponse.parse(responseBuffer, version); case DELETE_GROUPS: - return new DeleteGroupsResponse(struct, version); + return DeleteGroupsResponse.parse(responseBuffer, version); case ELECT_LEADERS: - return new ElectLeadersResponse(struct, version); + return ElectLeadersResponse.parse(responseBuffer, version); case INCREMENTAL_ALTER_CONFIGS: - return new IncrementalAlterConfigsResponse(struct, version); + return IncrementalAlterConfigsResponse.parse(responseBuffer, version); case ALTER_PARTITION_REASSIGNMENTS: - return new AlterPartitionReassignmentsResponse(struct, version); + return AlterPartitionReassignmentsResponse.parse(responseBuffer, version); case LIST_PARTITION_REASSIGNMENTS: - return new ListPartitionReassignmentsResponse(struct, version); + return ListPartitionReassignmentsResponse.parse(responseBuffer, version); case OFFSET_DELETE: - return new OffsetDeleteResponse(struct, version); + return OffsetDeleteResponse.parse(responseBuffer, version); default: throw new AssertionError(String.format("ApiKey %s is not currently handled in `parseResponse`, the " + "code should be updated to do so.", apiKey)); @@ -199,9 +199,7 @@ public boolean shouldClientThrottle(short version) { return false; } - public int throttleTimeMs() { - return DEFAULT_THROTTLE_TIME; - } + public abstract int throttleTimeMs(); public String toString(short version) { return toStruct(version).toString(); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AlterPartitionReassignmentsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AlterPartitionReassignmentsResponse.java index b1e1dc06052c8..469016ad7b408 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AlterPartitionReassignmentsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AlterPartitionReassignmentsResponse.java @@ -32,10 +32,6 @@ public class AlterPartitionReassignmentsResponse extends AbstractResponse { private final AlterPartitionReassignmentsResponseData data; - public AlterPartitionReassignmentsResponse(Struct struct, short version) { - this(new AlterPartitionReassignmentsResponseData(struct, version)); - } - public AlterPartitionReassignmentsResponse(AlterPartitionReassignmentsResponseData data) { this.data = data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownResponse.java index d4b82677328f3..9135e4c1937bf 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownResponse.java @@ -44,10 +44,6 @@ public ControlledShutdownResponse(ControlledShutdownResponseData data) { this.data = data; } - public ControlledShutdownResponse(Struct struct, short version) { - this(new ControlledShutdownResponseData(struct, version)); - } - public Errors error() { return Errors.forCode(data.errorCode()); } @@ -57,6 +53,11 @@ public Map errorCounts() { return Collections.singletonMap(error(), 1); } + @Override + public int throttleTimeMs() { + return DEFAULT_THROTTLE_TIME; + } + public static ControlledShutdownResponse parse(ByteBuffer buffer, short version) { return new ControlledShutdownResponse(new ControlledShutdownResponseData(new ByteBufferAccessor(buffer), version)); } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/CreateDelegationTokenResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/CreateDelegationTokenResponse.java index eec6d09d6e5b6..f31615806d6a2 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/CreateDelegationTokenResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/CreateDelegationTokenResponse.java @@ -34,10 +34,6 @@ public CreateDelegationTokenResponse(CreateDelegationTokenResponseData data) { this.data = data; } - public CreateDelegationTokenResponse(Struct struct, short version) { - this.data = new CreateDelegationTokenResponseData(struct, version); - } - public static CreateDelegationTokenResponse parse(ByteBuffer buffer, short version) { return new CreateDelegationTokenResponse( new CreateDelegationTokenResponseData(new ByteBufferAccessor(buffer), version)); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/CreateTopicsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/CreateTopicsResponse.java index 3b804ba8263e6..36061172b9c8a 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/CreateTopicsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/CreateTopicsResponse.java @@ -50,10 +50,6 @@ public CreateTopicsResponse(CreateTopicsResponseData data) { this.data = data; } - public CreateTopicsResponse(Struct struct, short version) { - this.data = new CreateTopicsResponseData(struct, version); - } - public CreateTopicsResponseData data() { return data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsResponse.java index f82a1dffc7337..76176dbe06d66 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsResponse.java @@ -46,10 +46,6 @@ public DeleteGroupsResponse(DeleteGroupsResponseData data) { this.data = data; } - public DeleteGroupsResponse(Struct struct, short version) { - this.data = new DeleteGroupsResponseData(struct, version); - } - @Override protected Struct toStruct(short version) { return data.toStruct(version); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DeleteTopicsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DeleteTopicsResponse.java index 234a3394d86a9..d6057aa730575 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DeleteTopicsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DeleteTopicsResponse.java @@ -45,10 +45,6 @@ public DeleteTopicsResponse(DeleteTopicsResponseData data) { this.data = data; } - public DeleteTopicsResponse(Struct struct, short version) { - this.data = new DeleteTopicsResponseData(struct, version); - } - @Override protected Struct toStruct(short version) { return data.toStruct(version); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenResponse.java index 5500445d415e4..3e80b91fc2e2a 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenResponse.java @@ -68,10 +68,6 @@ public DescribeDelegationTokenResponse(DescribeDelegationTokenResponseData data) this.data = data; } - public DescribeDelegationTokenResponse(Struct struct, short version) { - this(new DescribeDelegationTokenResponseData(struct, version)); - } - public static DescribeDelegationTokenResponse parse(ByteBuffer buffer, short version) { return new DescribeDelegationTokenResponse(new DescribeDelegationTokenResponseData( new ByteBufferAccessor(buffer), version)); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeGroupsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeGroupsResponse.java index 39b1fd2c549b6..d0843f16db6f5 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeGroupsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeGroupsResponse.java @@ -48,10 +48,6 @@ public DescribeGroupsResponse(DescribeGroupsResponseData data) { this.data = data; } - public DescribeGroupsResponse(Struct struct, short version) { - this(new DescribeGroupsResponseData(struct, version)); - } - public static DescribedGroupMember groupMember( final String memberId, final String groupInstanceId, diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ElectLeadersResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ElectLeadersResponse.java index 80deca1469535..7f3a78250c29b 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ElectLeadersResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ElectLeadersResponse.java @@ -34,10 +34,6 @@ public class ElectLeadersResponse extends AbstractResponse { private final ElectLeadersResponseData data; - public ElectLeadersResponse(Struct struct, short version) { - this(new ElectLeadersResponseData(struct, version)); - } - public ElectLeadersResponse(ElectLeadersResponseData data) { this.data = data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenResponse.java index 5b4ee15e3ca5f..abdab38ab29b3 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenResponse.java @@ -34,10 +34,6 @@ public ExpireDelegationTokenResponse(ExpireDelegationTokenResponseData data) { this.data = data; } - public ExpireDelegationTokenResponse(Struct struct, short version) { - this.data = new ExpireDelegationTokenResponseData(struct, version); - } - public static ExpireDelegationTokenResponse parse(ByteBuffer buffer, short version) { return new ExpireDelegationTokenResponse(new ExpireDelegationTokenResponseData(new ByteBufferAccessor(buffer), version)); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/FindCoordinatorResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/FindCoordinatorResponse.java index 3fd2f0eabfbde..37a5bf4a2395b 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/FindCoordinatorResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/FindCoordinatorResponse.java @@ -45,10 +45,6 @@ public FindCoordinatorResponse(FindCoordinatorResponseData data) { this.data = data; } - public FindCoordinatorResponse(Struct struct, short version) { - this.data = new FindCoordinatorResponseData(struct, version); - } - public FindCoordinatorResponseData data() { return data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatResponse.java index 04b1005c1f0c6..a5f3127cef65d 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatResponse.java @@ -44,10 +44,6 @@ public HeartbeatResponse(HeartbeatResponseData data) { this.data = data; } - public HeartbeatResponse(Struct struct, short version) { - this.data = new HeartbeatResponseData(struct, version); - } - @Override public int throttleTimeMs() { return data.throttleTimeMs(); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/IncrementalAlterConfigsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/IncrementalAlterConfigsResponse.java index bd53987b196bc..d9ffd9487ee51 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/IncrementalAlterConfigsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/IncrementalAlterConfigsResponse.java @@ -59,10 +59,6 @@ public IncrementalAlterConfigsResponse(IncrementalAlterConfigsResponseData data) this.data = data; } - public IncrementalAlterConfigsResponse(final Struct struct, final short version) { - this.data = new IncrementalAlterConfigsResponseData(struct, version); - } - public IncrementalAlterConfigsResponseData data() { return data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/InitProducerIdResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/InitProducerIdResponse.java index 93630a5af5a28..36868bd798bd1 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/InitProducerIdResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/InitProducerIdResponse.java @@ -40,10 +40,6 @@ public InitProducerIdResponse(InitProducerIdResponseData data) { this.data = data; } - public InitProducerIdResponse(Struct struct, short version) { - this.data = new InitProducerIdResponseData(struct, version); - } - @Override public int throttleTimeMs() { return data.throttleTimeMs(); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/JoinGroupResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/JoinGroupResponse.java index 2b71659290c57..78147001a797c 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/JoinGroupResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/JoinGroupResponse.java @@ -37,10 +37,6 @@ public JoinGroupResponse(JoinGroupResponseData data) { this.data = data; } - public JoinGroupResponse(Struct struct, short version) { - this.data = new JoinGroupResponseData(struct, version); - } - public JoinGroupResponseData data() { return data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrResponse.java index 62f523321739f..270adaadbcd48 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrResponse.java @@ -43,10 +43,6 @@ public LeaderAndIsrResponse(LeaderAndIsrResponseData data) { this.data = data; } - public LeaderAndIsrResponse(Struct struct, short version) { - this.data = new LeaderAndIsrResponseData(struct, version); - } - public List partitions() { return data.partitionErrors(); } @@ -64,6 +60,11 @@ public Map errorCounts() { return errorCounts(data.partitionErrors().stream().map(l -> Errors.forCode(l.errorCode())).collect(Collectors.toList())); } + @Override + public int throttleTimeMs() { + return DEFAULT_THROTTLE_TIME; + } + public static LeaderAndIsrResponse parse(ByteBuffer buffer, short version) { return new LeaderAndIsrResponse(new LeaderAndIsrResponseData(new ByteBufferAccessor(buffer), version)); } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupResponse.java index 1e3c22834474d..40bdf20ca5922 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupResponse.java @@ -76,10 +76,6 @@ public LeaveGroupResponse(List memberResponses, } } - public LeaveGroupResponse(Struct struct, short version) { - this.data = new LeaveGroupResponseData(struct, version); - } - @Override public int throttleTimeMs() { return data.throttleTimeMs(); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsResponse.java index cc8a870aba7b2..e99fb59043d46 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsResponse.java @@ -33,10 +33,6 @@ public ListGroupsResponse(ListGroupsResponseData data) { this.data = data; } - public ListGroupsResponse(Struct struct, short version) { - this.data = new ListGroupsResponseData(struct, version); - } - public ListGroupsResponseData data() { return data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ListPartitionReassignmentsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ListPartitionReassignmentsResponse.java index 587f6fc3854fd..91d22dc013432 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ListPartitionReassignmentsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ListPartitionReassignmentsResponse.java @@ -33,10 +33,6 @@ public ListPartitionReassignmentsResponse(ListPartitionReassignmentsResponseData this.data = responseData; } - public ListPartitionReassignmentsResponse(Struct struct, short version) { - this(new ListPartitionReassignmentsResponseData(struct, version)); - } - public static ListPartitionReassignmentsResponse parse(ByteBuffer buffer, short version) { return new ListPartitionReassignmentsResponse(new ListPartitionReassignmentsResponseData( new ByteBufferAccessor(buffer), version)); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/MetadataResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/MetadataResponse.java index b04f1793e1b44..433a7fac90c16 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/MetadataResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/MetadataResponse.java @@ -66,10 +66,6 @@ public MetadataResponse(MetadataResponseData data) { this.data = data; } - public MetadataResponse(Struct struct, short version) { - this(new MetadataResponseData(struct, version)); - } - @Override protected Struct toStruct(short version) { return data.toStruct(version); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitResponse.java index c74c225438bfe..4d505b1b7c655 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitResponse.java @@ -80,15 +80,6 @@ public OffsetCommitResponse(Map responseData) { this(DEFAULT_THROTTLE_TIME, responseData); } - public OffsetCommitResponse(Struct struct) { - short latestVersion = (short) (OffsetCommitResponseData.SCHEMAS.length - 1); - this.data = new OffsetCommitResponseData(struct, latestVersion); - } - - public OffsetCommitResponse(Struct struct, short version) { - this.data = new OffsetCommitResponseData(struct, version); - } - public OffsetCommitResponseData data() { return data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetDeleteResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetDeleteResponse.java index 1adec4190d045..d8359d77b8b7e 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/OffsetDeleteResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetDeleteResponse.java @@ -53,10 +53,6 @@ public OffsetDeleteResponse(OffsetDeleteResponseData data) { this.data = data; } - public OffsetDeleteResponse(Struct struct, short version) { - this(new OffsetDeleteResponseData(struct, version)); - } - @Override protected Struct toStruct(short version) { return data.toStruct(version); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchResponse.java index 309dc317ed075..31e46d6a04806 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchResponse.java @@ -174,10 +174,6 @@ private static Errors topLevelError(OffsetFetchResponseData data) { return Errors.NONE; } - public OffsetFetchResponse(Struct struct, short version) { - this(new OffsetFetchResponseData(struct, version), version); - } - @Override public int throttleTimeMs() { return data.throttleTimeMs(); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/RenewDelegationTokenResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/RenewDelegationTokenResponse.java index 468fbf33c6ce2..64fc118ed5c6a 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/RenewDelegationTokenResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/RenewDelegationTokenResponse.java @@ -34,10 +34,6 @@ public RenewDelegationTokenResponse(RenewDelegationTokenResponseData data) { this.data = data; } - public RenewDelegationTokenResponse(Struct struct, short version) { - this(new RenewDelegationTokenResponseData(struct, version)); - } - public static RenewDelegationTokenResponse parse(ByteBuffer buffer, short version) { return new RenewDelegationTokenResponse(new RenewDelegationTokenResponseData( new ByteBufferAccessor(buffer), version)); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/SaslAuthenticateResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/SaslAuthenticateResponse.java index 992b7d9eef7ef..07baaf01f3b48 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/SaslAuthenticateResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/SaslAuthenticateResponse.java @@ -39,10 +39,6 @@ public SaslAuthenticateResponse(SaslAuthenticateResponseData data) { this.data = data; } - public SaslAuthenticateResponse(Struct struct, short version) { - this.data = new SaslAuthenticateResponseData(struct, version); - } - /** * Possible error codes: * SASL_AUTHENTICATION_FAILED(57) : Authentication failed @@ -73,6 +69,11 @@ public Struct toStruct(short version) { return data.toStruct(version); } + @Override + public int throttleTimeMs() { + return DEFAULT_THROTTLE_TIME; + } + @Override protected Message data() { return data; diff --git a/clients/src/main/java/org/apache/kafka/common/requests/SaslHandshakeResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/SaslHandshakeResponse.java index 1dd53762197a8..558c22398e3b0 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/SaslHandshakeResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/SaslHandshakeResponse.java @@ -39,10 +39,6 @@ public SaslHandshakeResponse(SaslHandshakeResponseData data) { this.data = data; } - public SaslHandshakeResponse(Struct struct, short version) { - this.data = new SaslHandshakeResponseData(struct, version); - } - /* * Possible error codes: * UNSUPPORTED_SASL_MECHANISM(33): Client mechanism not enabled in server @@ -62,6 +58,11 @@ public Struct toStruct(short version) { return data.toStruct(version); } + @Override + public int throttleTimeMs() { + return DEFAULT_THROTTLE_TIME; + } + @Override protected Message data() { return data; diff --git a/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaResponse.java index 32f387a0ed06a..8f48274a0dad3 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaResponse.java @@ -43,10 +43,6 @@ public StopReplicaResponse(StopReplicaResponseData data) { this.data = data; } - public StopReplicaResponse(Struct struct, short version) { - data = new StopReplicaResponseData(struct, version); - } - public List partitionErrors() { return data.partitionErrors(); } @@ -72,6 +68,11 @@ protected Struct toStruct(short version) { return data.toStruct(version); } + @Override + public int throttleTimeMs() { + return DEFAULT_THROTTLE_TIME; + } + @Override protected Message data() { return data; diff --git a/clients/src/main/java/org/apache/kafka/common/requests/SyncGroupResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/SyncGroupResponse.java index 0806e58840ee7..2d6ab61b04035 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/SyncGroupResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/SyncGroupResponse.java @@ -34,10 +34,6 @@ public SyncGroupResponse(SyncGroupResponseData data) { this.data = data; } - public SyncGroupResponse(Struct struct, short version) { - this.data = new SyncGroupResponseData(struct, version); - } - @Override public int throttleTimeMs() { return data.throttleTimeMs(); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitResponse.java index b86029421b3ce..56b20a78d8c56 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitResponse.java @@ -51,10 +51,6 @@ public TxnOffsetCommitResponse(TxnOffsetCommitResponseData data) { this.data = data; } - public TxnOffsetCommitResponse(Struct struct, short version) { - this.data = new TxnOffsetCommitResponseData(struct, version); - } - public TxnOffsetCommitResponse(int requestThrottleMs, Map responseData) { Map responseTopicDataMap = new HashMap<>(); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataResponse.java index 50ddcf174720f..8e25f045c7c7d 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataResponse.java @@ -33,10 +33,6 @@ public UpdateMetadataResponse(UpdateMetadataResponseData data) { this.data = data; } - public UpdateMetadataResponse(Struct struct, short version) { - this(new UpdateMetadataResponseData(struct, version)); - } - public Errors error() { return Errors.forCode(data.errorCode()); } @@ -46,6 +42,11 @@ public Map errorCounts() { return errorCounts(error()); } + @Override + public int throttleTimeMs() { + return DEFAULT_THROTTLE_TIME; + } + public static UpdateMetadataResponse parse(ByteBuffer buffer, short version) { return new UpdateMetadataResponse(new UpdateMetadataResponseData(new ByteBufferAccessor(buffer), version)); } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/WriteTxnMarkersResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/WriteTxnMarkersResponse.java index 480ca9ce83df4..18f225298fb6d 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/WriteTxnMarkersResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/WriteTxnMarkersResponse.java @@ -156,6 +156,11 @@ public Map errors(long producerId) { return errors.get(producerId); } + @Override + public int throttleTimeMs() { + return DEFAULT_THROTTLE_TIME; + } + @Override public Map errorCounts() { Map errorCounts = new HashMap<>(); diff --git a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java index dfc4f6aa08f0c..5226670fc7579 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java +++ b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java @@ -514,7 +514,7 @@ private AbstractResponse receiveKafkaResponse() throws IOException { if (responseBytes == null) return null; else { - AbstractResponse response = NetworkClient.parseResponse(ByteBuffer.wrap(responseBytes), currentRequestHeader); + AbstractResponse response = NetworkClient.parseResponse(currentRequestHeader, ByteBuffer.wrap(responseBytes)); currentRequestHeader = null; return response; } diff --git a/clients/src/test/java/org/apache/kafka/common/requests/LeaveGroupResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/LeaveGroupResponseTest.java index 187640bfdeb71..2add31446ccdc 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/LeaveGroupResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/LeaveGroupResponseTest.java @@ -58,29 +58,6 @@ public void setUp() { ); } - @Test - public void testConstructorWithStruct() { - Map expectedErrorCounts = Collections.singletonMap(Errors.NOT_COORDINATOR, 1); - - LeaveGroupResponseData responseData = new LeaveGroupResponseData() - .setErrorCode(Errors.NOT_COORDINATOR.code()) - .setThrottleTimeMs(throttleTimeMs); - for (short version = 0; version <= ApiKeys.LEAVE_GROUP.latestVersion(); version++) { - LeaveGroupResponse leaveGroupResponse = new LeaveGroupResponse(responseData.toStruct(version), version); - - assertEquals(expectedErrorCounts, leaveGroupResponse.errorCounts()); - - if (version >= 1) { - assertEquals(throttleTimeMs, leaveGroupResponse.throttleTimeMs()); - } else { - assertEquals(DEFAULT_THROTTLE_TIME, leaveGroupResponse.throttleTimeMs()); - } - - assertEquals(Errors.NOT_COORDINATOR, leaveGroupResponse.error()); - } - } - - @Test public void testConstructorWithMemberResponses() { Map expectedErrorCounts = new HashMap<>(); @@ -130,9 +107,11 @@ public void testEqualityWithStruct() { .setErrorCode(Errors.NONE.code()) .setThrottleTimeMs(throttleTimeMs); for (short version = 0; version <= ApiKeys.LEAVE_GROUP.latestVersion(); version++) { - LeaveGroupResponse primaryResponse = new LeaveGroupResponse(responseData.toStruct(version), version); + LeaveGroupResponse primaryResponse = new LeaveGroupResponse( + new LeaveGroupResponseData(responseData.toStruct(version), version)); - LeaveGroupResponse secondaryResponse = new LeaveGroupResponse(responseData.toStruct(version), version); + LeaveGroupResponse secondaryResponse = new LeaveGroupResponse( + new LeaveGroupResponseData(responseData.toStruct(version), version)); assertEquals(primaryResponse, primaryResponse); assertEquals(primaryResponse, secondaryResponse); diff --git a/clients/src/test/java/org/apache/kafka/common/requests/OffsetCommitResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/OffsetCommitResponseTest.java index 62229168b2405..bd163e02f4993 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/OffsetCommitResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/OffsetCommitResponseTest.java @@ -17,20 +17,13 @@ package org.apache.kafka.common.requests; import org.apache.kafka.common.TopicPartition; -import org.apache.kafka.common.message.OffsetCommitResponseData; -import org.apache.kafka.common.message.OffsetCommitResponseData.OffsetCommitResponsePartition; -import org.apache.kafka.common.message.OffsetCommitResponseData.OffsetCommitResponseTopic; -import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; import org.junit.Before; import org.junit.Test; -import java.util.Arrays; -import java.util.Collections; import java.util.HashMap; import java.util.Map; -import static org.apache.kafka.common.requests.AbstractResponse.DEFAULT_THROTTLE_TIME; import static org.junit.Assert.assertEquals; public class OffsetCommitResponseTest { @@ -67,33 +60,4 @@ public void testConstructorWithErrorResponse() { assertEquals(expectedErrorCounts, response.errorCounts()); assertEquals(throttleTimeMs, response.throttleTimeMs()); } - - @Test - public void testConstructorWithStruct() { - OffsetCommitResponseData data = new OffsetCommitResponseData() - .setTopics(Arrays.asList( - new OffsetCommitResponseTopic().setPartitions( - Collections.singletonList(new OffsetCommitResponsePartition() - .setPartitionIndex(partitionOne) - .setErrorCode(errorOne.code()))), - new OffsetCommitResponseTopic().setPartitions( - Collections.singletonList(new OffsetCommitResponsePartition() - .setPartitionIndex(partitionTwo) - .setErrorCode(errorTwo.code()))) - )) - .setThrottleTimeMs(throttleTimeMs); - - for (short version = 0; version <= ApiKeys.OFFSET_COMMIT.latestVersion(); version++) { - OffsetCommitResponse response = new OffsetCommitResponse(data.toStruct(version), version); - assertEquals(expectedErrorCounts, response.errorCounts()); - - if (version >= 3) { - assertEquals(throttleTimeMs, response.throttleTimeMs()); - } else { - assertEquals(DEFAULT_THROTTLE_TIME, response.throttleTimeMs()); - } - - assertEquals(version >= 4, response.shouldClientThrottle(version)); - } - } } diff --git a/clients/src/test/java/org/apache/kafka/common/requests/OffsetFetchResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/OffsetFetchResponseTest.java index d3ff161ac0b07..e28918f380b4e 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/OffsetFetchResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/OffsetFetchResponseTest.java @@ -103,7 +103,8 @@ public void testStructBuild() { for (short version = 0; version <= ApiKeys.OFFSET_FETCH.latestVersion(); version++) { Struct struct = latestResponse.data.toStruct(version); - OffsetFetchResponse oldResponse = new OffsetFetchResponse(struct, version); + OffsetFetchResponse oldResponse = new OffsetFetchResponse(new OffsetFetchResponseData(struct, version), + version); if (version <= 1) { assertFalse(struct.hasField(ERROR_CODE)); @@ -139,9 +140,7 @@ public void testStructBuild() { Map responseData = oldResponse.responseData(); assertEquals(expectedDataMap, responseData); - responseData.forEach( - (tp, data) -> assertTrue(data.hasError()) - ); + responseData.forEach((tp, data) -> assertTrue(data.hasError())); } } diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestContextTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestContextTest.java index 7d066250d2f2b..4410cc65d8bfc 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/RequestContextTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestContextTest.java @@ -23,7 +23,6 @@ import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.security.auth.SecurityProtocol; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.security.auth.KafkaPrincipal; import org.junit.Test; @@ -71,9 +70,8 @@ public void testSerdeUnsupportedApiVersionRequest() throws Exception { ApiKeys.API_VERSIONS.responseHeaderVersion(header.apiVersion())); assertEquals(correlationId, responseHeader.correlationId()); - Struct struct = ApiKeys.API_VERSIONS.parseResponse((short) 0, responseBuffer); - ApiVersionsResponse response = (ApiVersionsResponse) - AbstractResponse.parseResponse(ApiKeys.API_VERSIONS, struct, (short) 0); + ApiVersionsResponse response = (ApiVersionsResponse) AbstractResponse.parseResponse(ApiKeys.API_VERSIONS, + responseBuffer, (short) 0); assertEquals(Errors.UNSUPPORTED_VERSION.code(), response.data.errorCode()); assertTrue(response.data.apiKeys().isEmpty()); } diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java index 1d6dbdf4fa97c..d1817babcee19 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java @@ -579,11 +579,8 @@ public void produceResponseV5Test() { buffer.rewind(); ResponseHeader.parse(buffer, ApiKeys.PRODUCE.responseHeaderVersion(version)); // throw away. - - Struct deserializedStruct = ApiKeys.PRODUCE.parseResponse(version, buffer); - ProduceResponse v5FromBytes = (ProduceResponse) AbstractResponse.parseResponse(ApiKeys.PRODUCE, - deserializedStruct, version); + buffer, version); assertEquals(1, v5FromBytes.responses().size()); assertTrue(v5FromBytes.responses().containsKey(tp0)); diff --git a/clients/src/test/java/org/apache/kafka/common/requests/TxnOffsetCommitResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/TxnOffsetCommitResponseTest.java index be4f5df359a05..7e35465f39bd2 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/TxnOffsetCommitResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/TxnOffsetCommitResponseTest.java @@ -16,16 +16,8 @@ */ package org.apache.kafka.common.requests; -import org.apache.kafka.common.message.TxnOffsetCommitResponseData; -import org.apache.kafka.common.message.TxnOffsetCommitResponseData.TxnOffsetCommitResponsePartition; -import org.apache.kafka.common.message.TxnOffsetCommitResponseData.TxnOffsetCommitResponseTopic; -import org.apache.kafka.common.protocol.ApiKeys; - import org.junit.Test; -import java.util.Arrays; -import java.util.Collections; - import static org.junit.Assert.assertEquals; public class TxnOffsetCommitResponseTest extends OffsetCommitResponseTest { @@ -40,28 +32,4 @@ public void testConstructorWithErrorResponse() { assertEquals(throttleTimeMs, response.throttleTimeMs()); } - @Test - @Override - public void testConstructorWithStruct() { - TxnOffsetCommitResponseData data = new TxnOffsetCommitResponseData() - .setThrottleTimeMs(throttleTimeMs) - .setTopics(Arrays.asList( - new TxnOffsetCommitResponseTopic().setPartitions( - Collections.singletonList(new TxnOffsetCommitResponsePartition() - .setPartitionIndex(partitionOne) - .setErrorCode(errorOne.code()))), - new TxnOffsetCommitResponseTopic().setPartitions( - Collections.singletonList(new TxnOffsetCommitResponsePartition() - .setPartitionIndex(partitionTwo) - .setErrorCode(errorTwo.code())) - ) - )); - - for (short version = 0; version <= ApiKeys.TXN_OFFSET_COMMIT.latestVersion(); version++) { - TxnOffsetCommitResponse response = new TxnOffsetCommitResponse(data.toStruct(version), version); - assertEquals(expectedErrorCounts, response.errorCounts()); - assertEquals(throttleTimeMs, response.throttleTimeMs()); - assertEquals(version >= 1, response.shouldClientThrottle(version)); - } - } } diff --git a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java index 6d95d2e91d857..4ea054aeee68e 100644 --- a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java +++ b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java @@ -2020,7 +2020,7 @@ private AbstractResponse sendKafkaRequestReceiveResponse(String node, ApiKeys ap Send send = request.toSend(node, header); selector.send(send); ByteBuffer responseBuffer = waitForResponse(); - return NetworkClient.parseResponse(responseBuffer, header); + return NetworkClient.parseResponse(header, responseBuffer); } private SaslHandshakeResponse sendHandshakeRequestReceiveResponse(String node, short version) throws Exception { diff --git a/core/src/test/scala/unit/kafka/server/BaseRequestTest.scala b/core/src/test/scala/unit/kafka/server/BaseRequestTest.scala index 0ccafd6effed8..8b86d39f8ed31 100644 --- a/core/src/test/scala/unit/kafka/server/BaseRequestTest.scala +++ b/core/src/test/scala/unit/kafka/server/BaseRequestTest.scala @@ -29,7 +29,7 @@ import kafka.network.SocketServer import org.apache.kafka.common.network.ListenerName import org.apache.kafka.common.protocol.types.Struct import org.apache.kafka.common.protocol.ApiKeys -import org.apache.kafka.common.requests.{AbstractRequest, AbstractRequestResponse, RequestHeader, ResponseHeader} +import org.apache.kafka.common.requests.{AbstractRequest, AbstractRequestResponse, AbstractResponse, RequestHeader, ResponseHeader} import org.apache.kafka.common.security.auth.SecurityProtocol abstract class BaseRequestTest extends IntegrationTestHarness { @@ -155,13 +155,14 @@ abstract class BaseRequestTest extends IntegrationTestHarness { /** * Sends a request built by the builder, waits for the response and parses it */ - def requestResponse(socket: Socket, clientId: String, correlationId: Int, requestBuilder: AbstractRequest.Builder[_ <: AbstractRequest]): Struct = { + def requestResponse(socket: Socket, clientId: String, correlationId: Int, + requestBuilder: AbstractRequest.Builder[_ <: AbstractRequest]): AbstractResponse = { val apiKey = requestBuilder.apiKey val request = requestBuilder.build() val header = new RequestHeader(apiKey, request.version, clientId, correlationId) val response = requestAndReceive(socket, request.serializeWithHeader(header).array) val responseBuffer = skipResponseHeader(response, apiKey.responseHeaderVersion(request.version)) - apiKey.parseResponse(request.version, responseBuffer) + AbstractResponse.parseResponse(apiKey, responseBuffer, request.version) } /** diff --git a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala index a938c990c8dba..af817b75a915d 100644 --- a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala +++ b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala @@ -817,9 +817,8 @@ class KafkaApisTest { send.writeTo(channel) channel.close() channel.buffer.getInt() // read the size - ResponseHeader.parse(channel.buffer, api.responseHeaderVersion(request.version())) - val struct = api.responseSchema(request.version).read(channel.buffer) - AbstractResponse.parseResponse(api, struct, request.version) + ResponseHeader.parse(channel.buffer, api.responseHeaderVersion(request.version)) + AbstractResponse.parseResponse(api, channel.buffer, request.version) } private def expectNoThrottling(): Capture[RequestChannel.Response] = { diff --git a/core/src/test/scala/unit/kafka/server/KafkaMetricReporterExceptionHandlingTest.scala b/core/src/test/scala/unit/kafka/server/KafkaMetricReporterExceptionHandlingTest.scala index d6f37cd4f05bb..4f5de1ed79301 100644 --- a/core/src/test/scala/unit/kafka/server/KafkaMetricReporterExceptionHandlingTest.scala +++ b/core/src/test/scala/unit/kafka/server/KafkaMetricReporterExceptionHandlingTest.scala @@ -69,9 +69,8 @@ class KafkaMetricReporterExceptionHandlingTest extends BaseRequestTest { try { TestUtils.retry(10000) { - val error = new ListGroupsResponse( - requestResponse(socket, "clientId", 0, new ListGroupsRequest.Builder(new ListGroupsRequestData)), ApiKeys.LIST_GROUPS.latestVersion) - .errorCounts() + val error = requestResponse(socket, "clientId", 0, + new ListGroupsRequest.Builder(new ListGroupsRequestData)).errorCounts() assertEquals(Collections.singletonMap(Errors.NONE, 1), error) assertEquals(KafkaMetricReporterExceptionHandlingTest.goodReporterRegistered.get, KafkaMetricReporterExceptionHandlingTest.badReporterRegistered.get) assertTrue(KafkaMetricReporterExceptionHandlingTest.goodReporterRegistered.get > 0) diff --git a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala index 8f0012bb177c3..76c2ba4902548 100644 --- a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala +++ b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala @@ -34,7 +34,6 @@ import org.apache.kafka.common.message._ import org.apache.kafka.common.metrics.{KafkaMetric, Quota, Sensor} import org.apache.kafka.common.network.ListenerName import org.apache.kafka.common.protocol.ApiKeys -import org.apache.kafka.common.protocol.types.Struct import org.apache.kafka.common.record._ import org.apache.kafka.common.requests.CreateAclsRequest.AclCreation import org.apache.kafka.common.requests._ @@ -513,7 +512,7 @@ class RequestQuotaTest extends BaseRequestTest { case class Client(clientId: String, apiKey: ApiKeys) { var correlationId: Int = 0 val builder = requestBuilder(apiKey) - def runUntil(until: Struct => Boolean): Boolean = { + def runUntil(until: AbstractResponse => Boolean): Boolean = { val startMs = System.currentTimeMillis var done = false val socket = connect() @@ -521,7 +520,7 @@ class RequestQuotaTest extends BaseRequestTest { while (!done && System.currentTimeMillis < startMs + 10000) { correlationId += 1 val response = requestResponse(socket, clientId, correlationId, builder) - done = until.apply(response) + done = until(response) } } finally { socket.close() @@ -561,73 +560,20 @@ class RequestQuotaTest extends BaseRequestTest { } } - private def responseThrottleTime(apiKey: ApiKeys, responseStruct: Struct): Int = { - val apiVersion = apiKey.latestVersion - apiKey match { - case ApiKeys.PRODUCE => new ProduceResponse(responseStruct).throttleTimeMs - case ApiKeys.FETCH => FetchResponse.parse(responseStruct).throttleTimeMs - case ApiKeys.LIST_OFFSETS => new ListOffsetResponse(responseStruct).throttleTimeMs - case ApiKeys.METADATA => new MetadataResponse(responseStruct, apiVersion).throttleTimeMs - case ApiKeys.OFFSET_COMMIT => new OffsetCommitResponse(responseStruct, apiVersion).throttleTimeMs - case ApiKeys.OFFSET_FETCH => new OffsetFetchResponse(responseStruct, apiVersion).throttleTimeMs - case ApiKeys.FIND_COORDINATOR => new FindCoordinatorResponse(responseStruct, apiVersion).throttleTimeMs - case ApiKeys.JOIN_GROUP => new JoinGroupResponse(responseStruct, apiVersion).throttleTimeMs - case ApiKeys.HEARTBEAT => new HeartbeatResponse(responseStruct, apiVersion).throttleTimeMs - case ApiKeys.LEAVE_GROUP => new LeaveGroupResponse(responseStruct, apiVersion).throttleTimeMs - case ApiKeys.SYNC_GROUP => new SyncGroupResponse(responseStruct, apiVersion).throttleTimeMs - case ApiKeys.DESCRIBE_GROUPS => new DescribeGroupsResponse(responseStruct, apiVersion).throttleTimeMs - case ApiKeys.LIST_GROUPS => new ListGroupsResponse(responseStruct, apiVersion).throttleTimeMs - case ApiKeys.API_VERSIONS => new ApiVersionsResponse(responseStruct).throttleTimeMs - case ApiKeys.CREATE_TOPICS => new CreateTopicsResponse(responseStruct, apiVersion).throttleTimeMs - case ApiKeys.DELETE_TOPICS => new DeleteTopicsResponse(responseStruct, apiVersion).throttleTimeMs - case ApiKeys.DELETE_RECORDS => new DeleteRecordsResponse(responseStruct).throttleTimeMs - case ApiKeys.INIT_PRODUCER_ID => new InitProducerIdResponse(responseStruct, apiVersion).throttleTimeMs - case ApiKeys.ADD_PARTITIONS_TO_TXN => new AddPartitionsToTxnResponse(responseStruct).throttleTimeMs - case ApiKeys.ADD_OFFSETS_TO_TXN => new AddOffsetsToTxnResponse(responseStruct).throttleTimeMs - case ApiKeys.END_TXN => new EndTxnResponse(responseStruct).throttleTimeMs - case ApiKeys.TXN_OFFSET_COMMIT => new TxnOffsetCommitResponse(responseStruct, apiVersion).throttleTimeMs - case ApiKeys.DESCRIBE_ACLS => new DescribeAclsResponse(responseStruct).throttleTimeMs - case ApiKeys.CREATE_ACLS => new CreateAclsResponse(responseStruct).throttleTimeMs - case ApiKeys.DELETE_ACLS => new DeleteAclsResponse(responseStruct).throttleTimeMs - case ApiKeys.DESCRIBE_CONFIGS => new DescribeConfigsResponse(responseStruct).throttleTimeMs - case ApiKeys.ALTER_CONFIGS => new AlterConfigsResponse(responseStruct).throttleTimeMs - case ApiKeys.ALTER_REPLICA_LOG_DIRS => new AlterReplicaLogDirsResponse(responseStruct).throttleTimeMs - case ApiKeys.DESCRIBE_LOG_DIRS => new DescribeLogDirsResponse(responseStruct).throttleTimeMs - case ApiKeys.CREATE_PARTITIONS => new CreatePartitionsResponse(responseStruct).throttleTimeMs - case ApiKeys.CREATE_DELEGATION_TOKEN => new CreateDelegationTokenResponse(responseStruct, apiVersion).throttleTimeMs - case ApiKeys.DESCRIBE_DELEGATION_TOKEN=> new DescribeDelegationTokenResponse(responseStruct, apiVersion).throttleTimeMs - case ApiKeys.RENEW_DELEGATION_TOKEN => new RenewDelegationTokenResponse(responseStruct, apiVersion).throttleTimeMs - case ApiKeys.EXPIRE_DELEGATION_TOKEN => new ExpireDelegationTokenResponse(responseStruct, apiVersion).throttleTimeMs - case ApiKeys.DELETE_GROUPS => new DeleteGroupsResponse(responseStruct, apiVersion).throttleTimeMs - case ApiKeys.OFFSET_FOR_LEADER_EPOCH => new OffsetsForLeaderEpochResponse(responseStruct).throttleTimeMs - case ApiKeys.ELECT_LEADERS => new ElectLeadersResponse(responseStruct, apiVersion).throttleTimeMs - case ApiKeys.INCREMENTAL_ALTER_CONFIGS => new IncrementalAlterConfigsResponse(responseStruct, apiVersion).throttleTimeMs - case ApiKeys.ALTER_PARTITION_REASSIGNMENTS => new AlterPartitionReassignmentsResponse(responseStruct, apiVersion).throttleTimeMs - case ApiKeys.LIST_PARTITION_REASSIGNMENTS => new ListPartitionReassignmentsResponse(responseStruct, apiVersion).throttleTimeMs - case ApiKeys.OFFSET_DELETE => new OffsetDeleteResponse(responseStruct, apiVersion).throttleTimeMs() - case requestId => throw new IllegalArgumentException(s"No throttle time for $requestId") - } - } - private def checkRequestThrottleTime(apiKey: ApiKeys): Unit = { - // Request until throttled using client-id with default small quota val clientId = apiKey.toString val client = Client(clientId, apiKey) - - val throttled = client.runUntil { response => - responseThrottleTime(apiKey, response) > 0 - } + val throttled = client.runUntil(_.throttleTimeMs > 0) assertTrue(s"Response not throttled: $client", throttled) assertTrue(s"Throttle time metrics not updated: $client" , throttleTimeMetricValue(clientId) > 0) } private def checkSmallQuotaProducerRequestThrottleTime(apiKey: ApiKeys): Unit = { - // Request until throttled using client-id with default small producer quota val smallQuotaProducerClient = Client(smallQuotaProducerClientId, apiKey) - val throttled = smallQuotaProducerClient.runUntil(response => responseThrottleTime(apiKey, response) > 0) + val throttled = smallQuotaProducerClient.runUntil(_.throttleTimeMs > 0) assertTrue(s"Response not throttled: $smallQuotaProducerClient", throttled) assertTrue(s"Throttle time metrics for produce quota not updated: $smallQuotaProducerClient", @@ -637,10 +583,9 @@ class RequestQuotaTest extends BaseRequestTest { } private def checkSmallQuotaConsumerRequestThrottleTime(apiKey: ApiKeys): Unit = { - // Request until throttled using client-id with default small consumer quota val smallQuotaConsumerClient = Client(smallQuotaConsumerClientId, apiKey) - val throttled = smallQuotaConsumerClient.runUntil(response => responseThrottleTime(apiKey, response) > 0) + val throttled = smallQuotaConsumerClient.runUntil(_.throttleTimeMs > 0) assertTrue(s"Response not throttled: $smallQuotaConsumerClientId", throttled) assertTrue(s"Throttle time metrics for consumer quota not updated: $smallQuotaConsumerClient", @@ -650,10 +595,9 @@ class RequestQuotaTest extends BaseRequestTest { } private def checkUnthrottledClient(apiKey: ApiKeys): Unit = { - // Test that request from client with large quota is not throttled val unthrottledClient = Client(unthrottledClientId, apiKey) - unthrottledClient.runUntil(response => responseThrottleTime(apiKey, response) <= 0.0) + unthrottledClient.runUntil(_.throttleTimeMs <= 0) assertEquals(1, unthrottledClient.correlationId) assertTrue(s"Client should not have been throttled: $unthrottledClient", throttleTimeMetricValue(unthrottledClientId).isNaN) } From daa6dbbb6933673edb7be79c7ac4af0c63bf5681 Mon Sep 17 00:00:00 2001 From: Ismael Juma Date: Tue, 1 Oct 2019 22:26:25 -0700 Subject: [PATCH 11/51] Add back removed tests, modify them to test parsing from buffers instead of structs --- .../requests/LeaveGroupResponseTest.java | 28 +++++++++++++ .../requests/OffsetCommitResponseTest.java | 42 +++++++++++++++++++ 2 files changed, 70 insertions(+) diff --git a/clients/src/test/java/org/apache/kafka/common/requests/LeaveGroupResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/LeaveGroupResponseTest.java index 2add31446ccdc..0a9c8edae21e0 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/LeaveGroupResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/LeaveGroupResponseTest.java @@ -19,10 +19,12 @@ import org.apache.kafka.common.message.LeaveGroupResponseData; import org.apache.kafka.common.message.LeaveGroupResponseData.MemberResponse; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.junit.Before; import org.junit.Test; +import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -120,6 +122,32 @@ public void testEqualityWithStruct() { } } + @Test + public void testParse() { + Map expectedErrorCounts = Collections.singletonMap(Errors.NOT_COORDINATOR, 1); + + LeaveGroupResponseData data = new LeaveGroupResponseData() + .setErrorCode(Errors.NOT_COORDINATOR.code()) + .setThrottleTimeMs(throttleTimeMs); + + for (short version = 0; version <= ApiKeys.LEAVE_GROUP.latestVersion(); version++) { + ByteBuffer buffer = ByteBuffer.allocate(data.size(version)); + data.write(new ByteBufferAccessor(buffer), version); + buffer.rewind(); + LeaveGroupResponse leaveGroupResponse = LeaveGroupResponse.parse(buffer, version); + + assertEquals(expectedErrorCounts, leaveGroupResponse.errorCounts()); + + if (version >= 1) { + assertEquals(throttleTimeMs, leaveGroupResponse.throttleTimeMs()); + } else { + assertEquals(DEFAULT_THROTTLE_TIME, leaveGroupResponse.throttleTimeMs()); + } + + assertEquals(Errors.NOT_COORDINATOR, leaveGroupResponse.error()); + } + } + @Test public void testEqualityWithMemberResponses() { for (short version = 0; version <= ApiKeys.LEAVE_GROUP.latestVersion(); version++) { diff --git a/clients/src/test/java/org/apache/kafka/common/requests/OffsetCommitResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/OffsetCommitResponseTest.java index bd163e02f4993..260885fcd1614 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/OffsetCommitResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/OffsetCommitResponseTest.java @@ -17,13 +17,22 @@ package org.apache.kafka.common.requests; import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.message.OffsetCommitResponseData; +import org.apache.kafka.common.message.OffsetCommitResponseData.OffsetCommitResponsePartition; +import org.apache.kafka.common.message.OffsetCommitResponseData.OffsetCommitResponseTopic; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.junit.Before; import org.junit.Test; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.Map; +import static org.apache.kafka.common.requests.AbstractResponse.DEFAULT_THROTTLE_TIME; import static org.junit.Assert.assertEquals; public class OffsetCommitResponseTest { @@ -60,4 +69,37 @@ public void testConstructorWithErrorResponse() { assertEquals(expectedErrorCounts, response.errorCounts()); assertEquals(throttleTimeMs, response.throttleTimeMs()); } + + @Test + public void testParse() { + OffsetCommitResponseData data = new OffsetCommitResponseData() + .setTopics(Arrays.asList( + new OffsetCommitResponseTopic().setPartitions( + Collections.singletonList(new OffsetCommitResponsePartition() + .setPartitionIndex(partitionOne) + .setErrorCode(errorOne.code()))), + new OffsetCommitResponseTopic().setPartitions( + Collections.singletonList(new OffsetCommitResponsePartition() + .setPartitionIndex(partitionTwo) + .setErrorCode(errorTwo.code()))) + )) + .setThrottleTimeMs(throttleTimeMs); + + for (short version = 0; version <= ApiKeys.OFFSET_COMMIT.latestVersion(); version++) { + ByteBuffer buffer = ByteBuffer.allocate(data.size(version)); + data.write(new ByteBufferAccessor(buffer), version); + buffer.rewind(); + OffsetCommitResponse response = OffsetCommitResponse.parse(buffer, version); + assertEquals(expectedErrorCounts, response.errorCounts()); + + if (version >= 3) { + assertEquals(throttleTimeMs, response.throttleTimeMs()); + } else { + assertEquals(DEFAULT_THROTTLE_TIME, response.throttleTimeMs()); + } + + assertEquals(version >= 4, response.shouldClientThrottle(version)); + } + } + } From a36bbb3dbe5cb18910d31b71fe8d01f9b9b240ff Mon Sep 17 00:00:00 2001 From: Ismael Juma Date: Thu, 3 Oct 2019 05:44:11 -0700 Subject: [PATCH 12/51] Avoid `toStruct` for request/response header if the request/response uses the auto generated protocol --- .../kafka/common/requests/AbstractRequest.java | 13 ++++++++----- .../kafka/common/requests/AbstractResponse.java | 13 ++++++++----- .../kafka/common/protocol/MessageTestUtil.java | 6 +----- .../common/requests/LeaveGroupResponseTest.java | 7 ++----- .../common/requests/OffsetCommitResponseTest.java | 6 ++---- 5 files changed, 21 insertions(+), 24 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java index 168eb0498b6a7..aad2841489033 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java @@ -104,14 +104,17 @@ public Send toSend(String destination, RequestHeader header) { */ public final ByteBuffer serializeWithHeader(RequestHeader header) { Message data = data(); - Struct headerStruct = header.toStruct(); - if (data == null) + if (data == null) { + Struct headerStruct = header.toStruct(); return serialize(headerStruct, toStruct()); + } ObjectSerializationCache serializationCache = new ObjectSerializationCache(); - ByteBuffer buffer = ByteBuffer.allocate(headerStruct.sizeOf() + data.size(serializationCache, version)); - headerStruct.writeTo(buffer); - data.write(new ByteBufferAccessor(buffer), serializationCache, version); + ByteBuffer buffer = ByteBuffer.allocate(header.data().size(serializationCache, version) + + data.size(serializationCache, version)); + ByteBufferAccessor bufferAccessor = new ByteBufferAccessor(buffer); + header.data().write(bufferAccessor, serializationCache, version); + data.write(bufferAccessor, serializationCache, version); buffer.rewind(); return buffer; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java index 6e12b04eaf052..75419e6f38593 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java @@ -46,15 +46,18 @@ public ByteBuffer serializeWithHeader(ApiKeys apiKey, short version, int correla } private ByteBuffer serializeWithHeader(ResponseHeader header, short version) { - Struct headerStruct = header.toStruct(); Message data = data(); - if (data == null) + if (data == null) { + Struct headerStruct = header.toStruct(); return serialize(headerStruct, toStruct(version)); + } ObjectSerializationCache serializationCache = new ObjectSerializationCache(); - ByteBuffer buffer = ByteBuffer.allocate(headerStruct.sizeOf() + data.size(serializationCache, version)); - headerStruct.writeTo(buffer); - data.write(new ByteBufferAccessor(buffer), serializationCache, version); + ByteBuffer buffer = ByteBuffer.allocate(header.data().size(serializationCache, version) + + data.size(serializationCache, version)); + ByteBufferAccessor bufferAccessor = new ByteBufferAccessor(buffer); + header.data().write(bufferAccessor, serializationCache, version); + data.write(bufferAccessor, serializationCache, version); buffer.rewind(); return buffer; } diff --git a/clients/src/test/java/org/apache/kafka/common/protocol/MessageTestUtil.java b/clients/src/test/java/org/apache/kafka/common/protocol/MessageTestUtil.java index 31ad94efc80d0..a093148b891aa 100644 --- a/clients/src/test/java/org/apache/kafka/common/protocol/MessageTestUtil.java +++ b/clients/src/test/java/org/apache/kafka/common/protocol/MessageTestUtil.java @@ -25,11 +25,7 @@ public static ByteBuffer messageToByteBuffer(Message message, short version) { int size = message.size(cache, version); ByteBuffer bytes = ByteBuffer.allocate(size); message.write(new ByteBufferAccessor(bytes), cache, version); - bytes.flip(); + bytes.rewind(); return bytes; } - - public static void messageFromByteBuffer(ByteBuffer bytes, Message message, short version) { - message.read(new ByteBufferAccessor(bytes.duplicate()), version); - } } diff --git a/clients/src/test/java/org/apache/kafka/common/requests/LeaveGroupResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/LeaveGroupResponseTest.java index 0a9c8edae21e0..9af931f6a9f7f 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/LeaveGroupResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/LeaveGroupResponseTest.java @@ -19,8 +19,8 @@ import org.apache.kafka.common.message.LeaveGroupResponseData; import org.apache.kafka.common.message.LeaveGroupResponseData.MemberResponse; import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.MessageTestUtil; import org.junit.Before; import org.junit.Test; @@ -131,11 +131,8 @@ public void testParse() { .setThrottleTimeMs(throttleTimeMs); for (short version = 0; version <= ApiKeys.LEAVE_GROUP.latestVersion(); version++) { - ByteBuffer buffer = ByteBuffer.allocate(data.size(version)); - data.write(new ByteBufferAccessor(buffer), version); - buffer.rewind(); + ByteBuffer buffer = MessageTestUtil.messageToByteBuffer(data, version); LeaveGroupResponse leaveGroupResponse = LeaveGroupResponse.parse(buffer, version); - assertEquals(expectedErrorCounts, leaveGroupResponse.errorCounts()); if (version >= 1) { diff --git a/clients/src/test/java/org/apache/kafka/common/requests/OffsetCommitResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/OffsetCommitResponseTest.java index 260885fcd1614..5e3a85739e50d 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/OffsetCommitResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/OffsetCommitResponseTest.java @@ -21,8 +21,8 @@ import org.apache.kafka.common.message.OffsetCommitResponseData.OffsetCommitResponsePartition; import org.apache.kafka.common.message.OffsetCommitResponseData.OffsetCommitResponseTopic; import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.MessageTestUtil; import org.junit.Before; import org.junit.Test; @@ -86,9 +86,7 @@ public void testParse() { .setThrottleTimeMs(throttleTimeMs); for (short version = 0; version <= ApiKeys.OFFSET_COMMIT.latestVersion(); version++) { - ByteBuffer buffer = ByteBuffer.allocate(data.size(version)); - data.write(new ByteBufferAccessor(buffer), version); - buffer.rewind(); + ByteBuffer buffer = MessageTestUtil.messageToByteBuffer(data, version); OffsetCommitResponse response = OffsetCommitResponse.parse(buffer, version); assertEquals(expectedErrorCounts, response.errorCounts()); From a1bf993adf0190313703e145840de97038cd623e Mon Sep 17 00:00:00 2001 From: Ismael Juma Date: Thu, 3 Oct 2019 05:44:29 -0700 Subject: [PATCH 13/51] Add `toString` to `ResponseHeader` --- .../org/apache/kafka/common/requests/ResponseHeader.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ResponseHeader.java b/clients/src/main/java/org/apache/kafka/common/requests/ResponseHeader.java index 249b5d021081f..17f2212fd7a06 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ResponseHeader.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ResponseHeader.java @@ -62,6 +62,14 @@ public ResponseHeaderData data() { return data; } + @Override + public String toString() { + return "ResponseHeader(" + + "correlationId=" + data.correlationId() + + ", headerVersion=" + headerVersion + + ")"; + } + public static ResponseHeader parse(ByteBuffer buffer, short headerVersion) { return new ResponseHeader( new ResponseHeaderData(new ByteBufferAccessor(buffer), headerVersion), From e00cacef3bc69f768fb7f7ba91d342c836d80081 Mon Sep 17 00:00:00 2001 From: Ismael Juma Date: Thu, 3 Oct 2019 05:44:55 -0700 Subject: [PATCH 14/51] Use diamond --- .../common/security/authenticator/SaslServerAuthenticator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticator.java b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticator.java index 3953a10cde131..45f98d98027e8 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticator.java +++ b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticator.java @@ -169,7 +169,7 @@ public SaslServerAuthenticator(Map configs, List enabledMechanisms = (List) this.configs.get(BrokerSecurityConfigs.SASL_ENABLED_MECHANISMS_CONFIG); if (enabledMechanisms == null || enabledMechanisms.isEmpty()) throw new IllegalArgumentException("No SASL mechanisms are enabled"); - this.enabledMechanisms = new ArrayList(new HashSet(enabledMechanisms)); + this.enabledMechanisms = new ArrayList<>(new HashSet<>(enabledMechanisms)); for (String mechanism : this.enabledMechanisms) { if (!callbackHandlers.containsKey(mechanism)) throw new IllegalArgumentException("Callback handler not specified for SASL mechanism " + mechanism); From 56cba5f9008da3b43ce13d2f09a125bc6885dde1 Mon Sep 17 00:00:00 2001 From: Ismael Juma Date: Thu, 3 Oct 2019 07:58:55 -0700 Subject: [PATCH 15/51] Remove unnecessary boxing/unboxing --- .../authenticator/SaslServerAuthenticator.java | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticator.java b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticator.java index 45f98d98027e8..34261dc08e650 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticator.java +++ b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticator.java @@ -678,30 +678,28 @@ private long calcCompletionTimesAndReturnSessionLifetimeMs() { Long connectionsMaxReauthMs = connectionsMaxReauthMsByMechanism.get(saslMechanism); if (credentialExpirationMs != null || connectionsMaxReauthMs != null) { if (credentialExpirationMs == null) - retvalSessionLifetimeMs = zeroIfNegative(connectionsMaxReauthMs.longValue()); + retvalSessionLifetimeMs = zeroIfNegative(connectionsMaxReauthMs); else if (connectionsMaxReauthMs == null) - retvalSessionLifetimeMs = zeroIfNegative(credentialExpirationMs.longValue() - authenticationEndMs); + retvalSessionLifetimeMs = zeroIfNegative(credentialExpirationMs - authenticationEndMs); else retvalSessionLifetimeMs = zeroIfNegative( - Math.min(credentialExpirationMs.longValue() - authenticationEndMs, - connectionsMaxReauthMs.longValue())); + Math.min(credentialExpirationMs - authenticationEndMs, connectionsMaxReauthMs)); if (retvalSessionLifetimeMs > 0L) - sessionExpirationTimeNanos = Long - .valueOf(authenticationEndNanos + 1000 * 1000 * retvalSessionLifetimeMs); + sessionExpirationTimeNanos = authenticationEndNanos + 1000 * 1000 * retvalSessionLifetimeMs; } if (credentialExpirationMs != null) { if (sessionExpirationTimeNanos != null) LOG.debug( "Authentication complete; session max lifetime from broker config={} ms, credential expiration={} ({} ms); session expiration = {} ({} ms), sending {} ms to client", connectionsMaxReauthMs, new Date(credentialExpirationMs), - Long.valueOf(credentialExpirationMs.longValue() - authenticationEndMs), + credentialExpirationMs - authenticationEndMs, new Date(authenticationEndMs + retvalSessionLifetimeMs), retvalSessionLifetimeMs, retvalSessionLifetimeMs); else LOG.debug( "Authentication complete; session max lifetime from broker config={} ms, credential expiration={} ({} ms); no session expiration, sending 0 ms to client", connectionsMaxReauthMs, new Date(credentialExpirationMs), - Long.valueOf(credentialExpirationMs.longValue() - authenticationEndMs)); + credentialExpirationMs - authenticationEndMs); } else { if (sessionExpirationTimeNanos != null) LOG.debug( @@ -721,7 +719,7 @@ public Long reauthenticationLatencyMs() { return null; // record at least 1 ms if there is some latency long latencyNanos = authenticationEndNanos - reauthenticationBeginNanos; - return latencyNanos == 0L ? 0L : Math.max(1L, Long.valueOf(Math.round(latencyNanos / 1000.0 / 1000.0))); + return latencyNanos == 0L ? 0L : Math.max(1L, Math.round(latencyNanos / 1000.0 / 1000.0)); } private long zeroIfNegative(long value) { From 5e0f689c3c3e3a8c406d0d2402038525aa42493c Mon Sep 17 00:00:00 2001 From: Ismael Juma Date: Thu, 3 Oct 2019 07:59:08 -0700 Subject: [PATCH 16/51] Use assertThrows to simplify code --- .../authenticator/SaslAuthenticatorTest.java | 69 +++++++++---------- 1 file changed, 32 insertions(+), 37 deletions(-) diff --git a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java index 4ea054aeee68e..fa23de4837401 100644 --- a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java +++ b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java @@ -119,6 +119,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -1558,43 +1559,37 @@ public void testCannotReauthenticateAgainFasterThanOneSecond() throws Exception configureMechanisms(OAuthBearerLoginModule.OAUTHBEARER_MECHANISM, Arrays.asList(OAuthBearerLoginModule.OAUTHBEARER_MECHANISM)); server = createEchoServer(securityProtocol); - try { - createClientConnection(securityProtocol, node); - checkClientConnection(node); - server.verifyAuthenticationMetrics(1, 0); - server.verifyReauthenticationMetrics(0, 0); - /* - * Now sleep long enough so that the next write will cause re-authentication, - * which we expect to succeed. - */ - time.sleep((long) (CONNECTIONS_MAX_REAUTH_MS_VALUE * 1.1)); - checkClientConnection(node); - server.verifyAuthenticationMetrics(1, 0); - server.verifyReauthenticationMetrics(1, 0); - /* - * Now sleep long enough so that the next write will cause re-authentication, - * but this time we expect re-authentication to not occur since it has been too - * soon. The checkClientConnection() call should return an error saying it - * expected the one byte-plus-node response but got the SaslHandshakeRequest - * instead - */ - time.sleep((long) (CONNECTIONS_MAX_REAUTH_MS_VALUE * 1.1)); - NetworkTestUtils.checkClientConnection(selector, node, 1, 1); - fail("Expected a failure when trying to re-authenticate to quickly, but that did not occur"); - } catch (AssertionError e) { - String expectedResponseTextRegex = "\\w-" + node; - String receivedResponseTextRegex = ".*" + OAuthBearerLoginModule.OAUTHBEARER_MECHANISM; - assertTrue( - "Should have received the SaslHandshakeRequest bytes back since we re-authenticated too quickly, " + - "but instead we got our generated message echoed back, implying re-auth succeeded when it " + - "should not have: " + e, - e.getMessage().matches( - ".*\\<\\[" + expectedResponseTextRegex + "]>.*\\<\\[" + receivedResponseTextRegex + "]>")); - server.verifyReauthenticationMetrics(1, 0); // unchanged - } finally { - selector.close(); - selector = null; - } + createClientConnection(securityProtocol, node); + checkClientConnection(node); + server.verifyAuthenticationMetrics(1, 0); + server.verifyReauthenticationMetrics(0, 0); + /* + * Now sleep long enough so that the next write will cause re-authentication, + * which we expect to succeed. + */ + time.sleep((long) (CONNECTIONS_MAX_REAUTH_MS_VALUE * 1.1)); + checkClientConnection(node); + server.verifyAuthenticationMetrics(1, 0); + server.verifyReauthenticationMetrics(1, 0); + /* + * Now sleep long enough so that the next write will cause re-authentication, + * but this time we expect re-authentication to not occur since it has been too + * soon. The checkClientConnection() call should return an error saying it + * expected the one byte-plus-node response but got the SaslHandshakeRequest + * instead + */ + time.sleep((long) (CONNECTIONS_MAX_REAUTH_MS_VALUE * 1.1)); + AssertionError exception = assertThrows(AssertionError.class, + () -> NetworkTestUtils.checkClientConnection(selector, node, 1, 1)); + String expectedResponseTextRegex = "\\w-" + node; + String receivedResponseTextRegex = ".*" + OAuthBearerLoginModule.OAUTHBEARER_MECHANISM; + System.out.println(exception.getMessage()); + assertTrue("Should have received the SaslHandshakeRequest bytes back since we re-authenticated too quickly, " + + "but instead we got our generated message echoed back, implying re-auth succeeded when it " + + "should not have: " + exception, + exception.getMessage().matches( + ".*\\<\\[" + expectedResponseTextRegex + "]>.*\\<\\[" + receivedResponseTextRegex + "]>")); + server.verifyReauthenticationMetrics(1, 0); // unchanged } /** From d6cb0974cd9a1b759da334e04f1211323e7abe20 Mon Sep 17 00:00:00 2001 From: Ismael Juma Date: Thu, 3 Oct 2019 08:04:06 -0700 Subject: [PATCH 17/51] Hacky test fix --- .../common/security/authenticator/SaslClientAuthenticator.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java index 5226670fc7579..7540474b3fc1d 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java +++ b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java @@ -57,6 +57,7 @@ import javax.security.sasl.SaslClient; import javax.security.sasl.SaslException; import java.io.IOException; +import java.nio.BufferUnderflowException; import java.nio.ByteBuffer; import java.nio.channels.SelectionKey; import java.security.Principal; @@ -518,7 +519,7 @@ private AbstractResponse receiveKafkaResponse() throws IOException { currentRequestHeader = null; return response; } - } catch (SchemaException | IllegalArgumentException e) { + } catch (BufferUnderflowException | SchemaException | IllegalArgumentException e) { /* * Account for the fact that during re-authentication there may be responses * arriving for requests that were sent in the past. From 9317347d598fe827a224e7de92abfc67600a179d Mon Sep 17 00:00:00 2001 From: Ismael Juma Date: Thu, 3 Oct 2019 08:06:55 -0700 Subject: [PATCH 18/51] Add comment --- .../src/main/java/org/apache/kafka/clients/NetworkClient.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java b/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java index a61ee65fca938..b41c41038689f 100644 --- a/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java +++ b/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java @@ -707,6 +707,8 @@ private static AbstractResponse parseResponse(RequestHeader requestHeader, ByteB // Always expect the response version id to be the same as the request version id AbstractResponse response = AbstractResponse.parseResponse(requestHeader.apiKey(), responseBuffer, requestHeader.apiVersion()); + // We correlate after parsing the response to avoid spurious correlation errors when receiving malformed + // responses correlate(requestHeader, responseHeader); if (throttleTimeSensor != null && response.throttleTimeMs() != AbstractResponse.DEFAULT_THROTTLE_TIME) throttleTimeSensor.record(response.throttleTimeMs(), now); From f57d04d67d8078ff6bc473a779b19c6f7930ea69 Mon Sep 17 00:00:00 2001 From: Ismael Juma Date: Thu, 10 Oct 2019 05:45:29 -0700 Subject: [PATCH 19/51] WIP --- .../main/java/org/apache/kafka/clients/NetworkClient.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java b/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java index b41c41038689f..9c3eae8b1c6ba 100644 --- a/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java +++ b/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java @@ -702,11 +702,12 @@ public static AbstractResponse parseResponse(RequestHeader requestHeader, ByteBu private static AbstractResponse parseResponse(RequestHeader requestHeader, ByteBuffer responseBuffer, Sensor throttleTimeSensor, long now) { + // Always expect the response version to be the same as the request version + short responseVersion = requestHeader.apiVersion(); ResponseHeader responseHeader = ResponseHeader.parse(responseBuffer, - requestHeader.apiKey().responseHeaderVersion(requestHeader.apiVersion())); - // Always expect the response version id to be the same as the request version id + requestHeader.apiKey().responseHeaderVersion(responseVersion)); AbstractResponse response = AbstractResponse.parseResponse(requestHeader.apiKey(), responseBuffer, - requestHeader.apiVersion()); + responseVersion); // We correlate after parsing the response to avoid spurious correlation errors when receiving malformed // responses correlate(requestHeader, responseHeader); From 4e4f66527da4b030a063e260d2bf05bfa2308e32 Mon Sep 17 00:00:00 2001 From: Ismael Juma Date: Fri, 12 Jun 2020 09:03:32 -0700 Subject: [PATCH 20/51] WIP --- .../apache/kafka/clients/NetworkClient.java | 1 + .../requests/AddOffsetsToTxnRequest.java | 6 ++ .../requests/AlterClientQuotasRequest.java | 23 +++-- .../requests/AlterClientQuotasResponse.java | 84 +++++++++-------- .../common/requests/AlterConfigsResponse.java | 3 +- .../requests/AlterReplicaLogDirsResponse.java | 2 + .../common/requests/ApiVersionsRequest.java | 4 +- .../common/requests/ApiVersionsResponse.java | 1 + .../requests/ControlledShutdownResponse.java | 2 + .../common/requests/CreateAclsResponse.java | 1 + .../CreateDelegationTokenResponse.java | 3 +- .../common/requests/DeleteGroupsResponse.java | 2 + .../common/requests/DescribeAclsResponse.java | 3 + .../requests/DescribeClientQuotasRequest.java | 8 +- .../DescribeClientQuotasResponse.java | 93 ++++++++++--------- .../requests/DescribeConfigsResponse.java | 2 + .../DescribeDelegationTokenResponse.java | 3 + .../requests/DescribeGroupsResponse.java | 2 + .../kafka/common/requests/EndTxnResponse.java | 2 + .../ExpireDelegationTokenResponse.java | 2 + .../kafka/common/requests/FetchResponse.java | 1 + .../requests/FindCoordinatorResponse.java | 2 + .../common/requests/HeartbeatResponse.java | 2 + .../IncrementalAlterConfigsResponse.java | 2 + .../requests/InitProducerIdResponse.java | 2 + .../common/requests/JoinGroupResponse.java | 2 + .../common/requests/LeaderAndIsrResponse.java | 2 + .../requests/LegacyAbstractResponse.java | 5 + .../common/requests/ListGroupsResponse.java | 2 + .../common/requests/ListOffsetResponse.java | 2 + .../ListPartitionReassignmentsResponse.java | 2 + .../common/requests/MetadataResponse.java | 7 +- .../common/requests/OffsetCommitResponse.java | 3 + .../common/requests/OffsetFetchResponse.java | 3 + .../OffsetsForLeaderEpochResponse.java | 2 + .../common/requests/ProduceResponse.java | 2 + .../RenewDelegationTokenResponse.java | 2 + .../requests/SaslAuthenticateResponse.java | 2 + .../requests/SaslHandshakeResponse.java | 2 + .../common/requests/SyncGroupResponse.java | 2 + .../requests/TxnOffsetCommitResponse.java | 3 + .../requests/UpdateMetadataResponse.java | 2 + .../requests/WriteTxnMarkersResponse.java | 27 +++--- .../SaslServerAuthenticator.java | 4 +- .../clients/admin/KafkaAdminClientTest.java | 23 +++-- .../common/requests/RequestHeaderTest.java | 6 +- .../authenticator/SaslAuthenticatorTest.java | 6 +- .../SaslServerAuthenticatorTest.java | 28 +++--- .../java/org/apache/kafka/test/TestUtils.java | 17 ++++ 49 files changed, 267 insertions(+), 145 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java b/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java index 43d1fe38bca46..06ce28a87a2ad 100644 --- a/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java +++ b/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java @@ -31,6 +31,7 @@ import org.apache.kafka.common.network.Send; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.types.SchemaException; import org.apache.kafka.common.requests.AbstractRequest; import org.apache.kafka.common.requests.AbstractResponse; import org.apache.kafka.common.requests.ApiVersionsRequest; diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AddOffsetsToTxnRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AddOffsetsToTxnRequest.java index 42d18aeea3feb..99d762d03fd11 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AddOffsetsToTxnRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AddOffsetsToTxnRequest.java @@ -21,6 +21,7 @@ import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.Message; import java.nio.ByteBuffer; @@ -52,6 +53,11 @@ public AddOffsetsToTxnRequest(AddOffsetsToTxnRequestData data, short version) { this.data = data; } + @Override + protected Message data() { + return data; + } + @Override public AddOffsetsToTxnResponse getErrorResponse(int throttleTimeMs, Throwable e) { return new AddOffsetsToTxnResponse(new AddOffsetsToTxnResponseData() diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasRequest.java index 6d4c2a1f357c3..c1aafd041c967 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasRequest.java @@ -21,10 +21,12 @@ import org.apache.kafka.common.message.AlterClientQuotasRequestData.EntryData; import org.apache.kafka.common.message.AlterClientQuotasRequestData.OpData; import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.protocol.ByteBufferAccessor; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.quota.ClientQuotaAlteration; import org.apache.kafka.common.quota.ClientQuotaEntity; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -85,11 +87,6 @@ public AlterClientQuotasRequest(AlterClientQuotasRequestData data, short version this.data = data; } - public AlterClientQuotasRequest(Struct struct, short version) { - super(ApiKeys.ALTER_CLIENT_QUOTAS, version); - this.data = new AlterClientQuotasRequestData(struct, version); - } - public Collection entries() { List entries = new ArrayList<>(data.entries().size()); for (EntryData entryData : data.entries()) { @@ -113,6 +110,11 @@ public boolean validateOnly() { return data.validateOnly(); } + @Override + protected Message data() { + return data; + } + @Override public AlterClientQuotasResponse getErrorResponse(int throttleTimeMs, Throwable e) { ArrayList entities = new ArrayList<>(data.entries().size()); @@ -123,11 +125,12 @@ public AlterClientQuotasResponse getErrorResponse(int throttleTimeMs, Throwable } entities.add(new ClientQuotaEntity(entity)); } - return new AlterClientQuotasResponse(entities, throttleTimeMs, e); + //FIXME + return null; +// return new AlterClientQuotasResponse(entities, throttleTimeMs, e); } - @Override - protected Struct toStruct() { - return data.toStruct(version()); + public static AlterClientQuotasRequest parse(ByteBuffer buffer, short version) { + return new AlterClientQuotasRequest(new AlterClientQuotasRequestData(new ByteBufferAccessor(buffer), version), version); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasResponse.java index a3b01bf3c1b2b..6eb7746e193b5 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasResponse.java @@ -20,12 +20,14 @@ import org.apache.kafka.common.message.AlterClientQuotasResponseData; import org.apache.kafka.common.message.AlterClientQuotasResponseData.EntityData; import org.apache.kafka.common.message.AlterClientQuotasResponseData.EntryData; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.quota.ClientQuotaEntity; +import java.nio.ByteBuffer; import java.util.ArrayList; -import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -34,40 +36,9 @@ public class AlterClientQuotasResponse extends AbstractResponse { private final AlterClientQuotasResponseData data; - public AlterClientQuotasResponse(Map result, int throttleTimeMs) { - List entries = new ArrayList<>(result.size()); - for (Map.Entry entry : result.entrySet()) { - ApiError e = entry.getValue(); - entries.add(new EntryData() - .setErrorCode(e.error().code()) - .setErrorMessage(e.message()) - .setEntity(toEntityData(entry.getKey()))); - } - - this.data = new AlterClientQuotasResponseData() - .setThrottleTimeMs(throttleTimeMs) - .setEntries(entries); - } - - public AlterClientQuotasResponse(Collection entities, int throttleTimeMs, Throwable e) { - short errorCode = Errors.forException(e).code(); - String errorMessage = e.getMessage(); - - List entries = new ArrayList<>(entities.size()); - for (ClientQuotaEntity entity : entities) { - entries.add(new EntryData() - .setErrorCode(errorCode) - .setErrorMessage(errorMessage) - .setEntity(toEntityData(entity))); - } - - this.data = new AlterClientQuotasResponseData() - .setThrottleTimeMs(throttleTimeMs) - .setEntries(entries); - } - - public AlterClientQuotasResponse(Struct struct, short version) { - this.data = new AlterClientQuotasResponseData(struct, version); + public AlterClientQuotasResponse(AlterClientQuotasResponseData data) { + super(ApiKeys.ALTER_CLIENT_QUOTAS); + this.data = data; } public void complete(Map> futures) { @@ -107,8 +78,8 @@ public Map errorCounts() { } @Override - protected Struct toStruct(short version) { - return data.toStruct(version); + protected Message data() { + return data; } private static List toEntityData(ClientQuotaEntity entity) { @@ -120,4 +91,41 @@ private static List toEntityData(ClientQuotaEntity entity) { } return entityData; } + + public static AlterClientQuotasResponse parse(ByteBuffer buffer, short version) { + return new AlterClientQuotasResponse(new AlterClientQuotasResponseData(new ByteBufferAccessor(buffer), version)); + } + + public static AlterClientQuotasResponse fromQuotaEntities(Map result, int throttleTimeMs) { + List entries = new ArrayList<>(result.size()); + for (Map.Entry entry : result.entrySet()) { + ApiError e = entry.getValue(); + entries.add(new EntryData() + .setErrorCode(e.error().code()) + .setErrorMessage(e.message()) + .setEntity(toEntityData(entry.getKey()))); + } + + return new AlterClientQuotasResponse(new AlterClientQuotasResponseData() + .setThrottleTimeMs(throttleTimeMs) + .setEntries(entries)); + } + +// public AlterClientQuotasResponse(Collection entities, int throttleTimeMs, Throwable e) { +// short errorCode = Errors.forException(e).code(); +// String errorMessage = e.getMessage(); +// +// List entries = new ArrayList<>(entities.size()); +// for (ClientQuotaEntity entity : entities) { +// entries.add(new EntryData() +// .setErrorCode(errorCode) +// .setErrorMessage(errorMessage) +// .setEntity(toEntityData(entity))); +// } +// +// this.data = new AlterClientQuotasResponseData() +// .setThrottleTimeMs(throttleTimeMs) +// .setEntries(entries); +// } +// } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AlterConfigsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AlterConfigsResponse.java index c1c406363a790..ab2bcdae5fb37 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AlterConfigsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AlterConfigsResponse.java @@ -33,6 +33,7 @@ public class AlterConfigsResponse extends AbstractResponse { private final AlterConfigsResponseData data; public AlterConfigsResponse(AlterConfigsResponseData data) { + super(ApiKeys.ALTER_CONFIGS); this.data = data; } @@ -61,7 +62,7 @@ protected Message data() { } public static AlterConfigsResponse parse(ByteBuffer buffer, short version) { - return new AlterConfigsResponse(new AlterConfigsResponseData(new ByteBufferAccessor(buffer), version), version); + return new AlterConfigsResponse(new AlterConfigsResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AlterReplicaLogDirsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AlterReplicaLogDirsResponse.java index cf6c60f7bcf4b..df605332ea900 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AlterReplicaLogDirsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AlterReplicaLogDirsResponse.java @@ -76,6 +76,7 @@ public static Schema[] schemaVersions() { private final int throttleTimeMs; public AlterReplicaLogDirsResponse(Struct struct) { + super(ApiKeys.ALTER_REPLICA_LOG_DIRS); throttleTimeMs = struct.get(THROTTLE_TIME_MS); responses = new HashMap<>(); for (Object topicStructObj : struct.getArray(TOPICS_KEY_NAME)) { @@ -94,6 +95,7 @@ public AlterReplicaLogDirsResponse(Struct struct) { * Constructor for version 0. */ public AlterReplicaLogDirsResponse(int throttleTimeMs, Map responses) { + super(ApiKeys.ALTER_REPLICA_LOG_DIRS); this.throttleTimeMs = throttleTimeMs; this.responses = responses; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsRequest.java index 478048b057d25..6b4d6b396dbe8 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsRequest.java @@ -61,7 +61,7 @@ public String toString() { private final Short unsupportedRequestVersion; - public final ApiVersionsRequestData data; + private final ApiVersionsRequestData data; public ApiVersionsRequest(ApiVersionsRequestData data, short version) { this(data, version, null); @@ -97,7 +97,7 @@ public boolean isValid() { } @Override - protected Message data() { + public ApiVersionsRequestData data() { return data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsResponse.java index ea11b70362408..b260daefc78fd 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsResponse.java @@ -43,6 +43,7 @@ public class ApiVersionsResponse extends AbstractResponse { public final ApiVersionsResponseData data; public ApiVersionsResponse(ApiVersionsResponseData data) { + super(ApiKeys.API_VERSIONS); this.data = data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownResponse.java index 86ed658f20fd6..6bdd0464b2719 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownResponse.java @@ -19,6 +19,7 @@ import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.message.ControlledShutdownResponseData; import org.apache.kafka.common.message.ControlledShutdownResponseData.RemainingPartition; +import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.types.Struct; @@ -40,6 +41,7 @@ public class ControlledShutdownResponse extends AbstractResponse { private final ControlledShutdownResponseData data; public ControlledShutdownResponse(ControlledShutdownResponseData data) { + super(ApiKeys.CONTROLLED_SHUTDOWN); this.data = data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/CreateAclsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/CreateAclsResponse.java index dd0652d01b722..6ad3e339e50dd 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/CreateAclsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/CreateAclsResponse.java @@ -30,6 +30,7 @@ public class CreateAclsResponse extends AbstractResponse { private final CreateAclsResponseData data; public CreateAclsResponse(CreateAclsResponseData data) { + super(ApiKeys.CREATE_ACLS); this.data = data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/CreateDelegationTokenResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/CreateDelegationTokenResponse.java index 214d43f019c57..9d39b6b641345 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/CreateDelegationTokenResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/CreateDelegationTokenResponse.java @@ -17,9 +17,9 @@ package org.apache.kafka.common.requests; import org.apache.kafka.common.message.CreateDelegationTokenResponseData; +import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.security.auth.KafkaPrincipal; import java.nio.ByteBuffer; @@ -30,6 +30,7 @@ public class CreateDelegationTokenResponse extends AbstractResponse { private final CreateDelegationTokenResponseData data; public CreateDelegationTokenResponse(CreateDelegationTokenResponseData data) { + super(ApiKeys.CREATE_DELEGATION_TOKEN); this.data = data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsResponse.java index f453ea8161596..6c50cb0683592 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsResponse.java @@ -18,6 +18,7 @@ import org.apache.kafka.common.message.DeleteGroupsResponseData; import org.apache.kafka.common.message.DeleteGroupsResponseData.DeletableGroupResult; +import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.Message; @@ -42,6 +43,7 @@ public class DeleteGroupsResponse extends AbstractResponse { public final DeleteGroupsResponseData data; public DeleteGroupsResponse(DeleteGroupsResponseData data) { + super(ApiKeys.DELETE_GROUPS); this.data = data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeAclsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeAclsResponse.java index 99f38d33d99ab..82c1ca1d34c08 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeAclsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeAclsResponse.java @@ -19,6 +19,7 @@ import org.apache.kafka.common.acl.AccessControlEntry; import org.apache.kafka.common.acl.AclBinding; +import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.resource.PatternType; @@ -49,12 +50,14 @@ public class DescribeAclsResponse extends AbstractResponse { private final DescribeAclsResponseData data; public DescribeAclsResponse(DescribeAclsResponseData data, short version) { + super(ApiKeys.DESCRIBE_ACLS); validate(Optional.of(version)); this.data = data; } // Skips version validation, should only be called when `version` is not available and the input is known to be valid DescribeAclsResponse(DescribeAclsResponseData data) { + super(ApiKeys.DESCRIBE_ACLS); validate(Optional.empty()); this.data = data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasRequest.java index 680dd267b518d..59b1245bdabfb 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasRequest.java @@ -18,8 +18,10 @@ import org.apache.kafka.common.message.DescribeClientQuotasRequestData; import org.apache.kafka.common.message.DescribeClientQuotasRequestData.ComponentData; +import org.apache.kafka.common.message.DescribeClientQuotasResponseData; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; +import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.quota.ClientQuotaFilter; import org.apache.kafka.common.quota.ClientQuotaFilterComponent; @@ -112,7 +114,11 @@ protected Message data() { @Override public DescribeClientQuotasResponse getErrorResponse(int throttleTimeMs, Throwable e) { - return new DescribeClientQuotasResponse(throttleTimeMs, e); + return new DescribeClientQuotasResponse(new DescribeClientQuotasResponseData() + .setThrottleTimeMs(throttleTimeMs) + .setErrorCode(Errors.forException(e).code()) + .setErrorMessage(e.getMessage()) + .setEntries(null)); } public static DescribeClientQuotasRequest parse(ByteBuffer buffer, short version) { diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasResponse.java index cb54b019dba88..cae42b6e2d977 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasResponse.java @@ -17,14 +17,18 @@ package org.apache.kafka.common.requests; import org.apache.kafka.common.internals.KafkaFutureImpl; +import org.apache.kafka.common.message.AddPartitionsToTxnResponseData; import org.apache.kafka.common.message.DescribeClientQuotasResponseData; import org.apache.kafka.common.message.DescribeClientQuotasResponseData.EntityData; import org.apache.kafka.common.message.DescribeClientQuotasResponseData.EntryData; import org.apache.kafka.common.message.DescribeClientQuotasResponseData.ValueData; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.quota.ClientQuotaEntity; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -34,47 +38,9 @@ public class DescribeClientQuotasResponse extends AbstractResponse { private final DescribeClientQuotasResponseData data; - public DescribeClientQuotasResponse(Map> entities, int throttleTimeMs) { - List entries = new ArrayList<>(entities.size()); - for (Map.Entry> entry : entities.entrySet()) { - ClientQuotaEntity quotaEntity = entry.getKey(); - List entityData = new ArrayList<>(quotaEntity.entries().size()); - for (Map.Entry entityEntry : quotaEntity.entries().entrySet()) { - entityData.add(new EntityData() - .setEntityType(entityEntry.getKey()) - .setEntityName(entityEntry.getValue())); - } - - Map quotaValues = entry.getValue(); - List valueData = new ArrayList<>(quotaValues.size()); - for (Map.Entry valuesEntry : entry.getValue().entrySet()) { - valueData.add(new ValueData() - .setKey(valuesEntry.getKey()) - .setValue(valuesEntry.getValue())); - } - - entries.add(new EntryData() - .setEntity(entityData) - .setValues(valueData)); - } - - this.data = new DescribeClientQuotasResponseData() - .setThrottleTimeMs(throttleTimeMs) - .setErrorCode((short) 0) - .setErrorMessage(null) - .setEntries(entries); - } - - public DescribeClientQuotasResponse(int throttleTimeMs, Throwable e) { - this.data = new DescribeClientQuotasResponseData() - .setThrottleTimeMs(throttleTimeMs) - .setErrorCode(Errors.forException(e).code()) - .setErrorMessage(e.getMessage()) - .setEntries(null); - } - - public DescribeClientQuotasResponse(Struct struct, short version) { - this.data = new DescribeClientQuotasResponseData(struct, version); + public DescribeClientQuotasResponse(DescribeClientQuotasResponseData data) { + super(ApiKeys.DESCRIBE_CLIENT_QUOTAS); + this.data = data; } public void complete(KafkaFutureImpl>> future) { @@ -106,13 +72,50 @@ public int throttleTimeMs() { return data.throttleTimeMs(); } + @Override + protected Message data() { + return data; + } + @Override public Map errorCounts() { return errorCounts(Errors.forCode(data.errorCode())); } - @Override - protected Struct toStruct(short version) { - return data.toStruct(version); + public static DescribeClientQuotasResponse parse(ByteBuffer buffer, short version) { + return new DescribeClientQuotasResponse(new DescribeClientQuotasResponseData(new ByteBufferAccessor(buffer), version)); + } + + public static DescribeClientQuotasResponse fromQuotaEntities(Map> entities, + int throttleTimeMs) { + List entries = new ArrayList<>(entities.size()); + for (Map.Entry> entry : entities.entrySet()) { + ClientQuotaEntity quotaEntity = entry.getKey(); + List entityData = new ArrayList<>(quotaEntity.entries().size()); + for (Map.Entry entityEntry : quotaEntity.entries().entrySet()) { + entityData.add(new EntityData() + .setEntityType(entityEntry.getKey()) + .setEntityName(entityEntry.getValue())); + } + + Map quotaValues = entry.getValue(); + List valueData = new ArrayList<>(quotaValues.size()); + for (Map.Entry valuesEntry : entry.getValue().entrySet()) { + valueData.add(new ValueData() + .setKey(valuesEntry.getKey()) + .setValue(valuesEntry.getValue())); + } + + entries.add(new EntryData() + .setEntity(entityData) + .setValues(valueData)); + } + + return new DescribeClientQuotasResponse(new DescribeClientQuotasResponseData() + .setThrottleTimeMs(throttleTimeMs) + .setErrorCode((short) 0) + .setErrorMessage(null) + .setEntries(entries)); } + } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeConfigsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeConfigsResponse.java index 057b4cef7c16a..93df70b08edf6 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeConfigsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeConfigsResponse.java @@ -226,11 +226,13 @@ public ConfigSource source() { private final Map configs; public DescribeConfigsResponse(int throttleTimeMs, Map configs) { + super(ApiKeys.DESCRIBE_CONFIGS); this.throttleTimeMs = throttleTimeMs; this.configs = Objects.requireNonNull(configs, "configs"); } public DescribeConfigsResponse(Struct struct) { + super(ApiKeys.DESCRIBE_CONFIGS); throttleTimeMs = struct.get(THROTTLE_TIME_MS); Object[] resourcesArray = struct.getArray(RESOURCES_KEY_NAME); configs = new HashMap<>(resourcesArray.length); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenResponse.java index a4160d2ac3c66..3882688b9c7a1 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenResponse.java @@ -19,6 +19,7 @@ import org.apache.kafka.common.message.DescribeDelegationTokenResponseData; import org.apache.kafka.common.message.DescribeDelegationTokenResponseData.DescribedDelegationToken; import org.apache.kafka.common.message.DescribeDelegationTokenResponseData.DescribedDelegationTokenRenewer; +import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.Message; @@ -37,6 +38,7 @@ public class DescribeDelegationTokenResponse extends AbstractResponse { private final DescribeDelegationTokenResponseData data; public DescribeDelegationTokenResponse(int throttleTimeMs, Errors error, List tokens) { + super(ApiKeys.DESCRIBE_DELEGATION_TOKEN); List describedDelegationTokenList = tokens .stream() .map(dt -> new DescribedDelegationToken() @@ -64,6 +66,7 @@ public DescribeDelegationTokenResponse(int throttleTimeMs, Errors error) { } public DescribeDelegationTokenResponse(DescribeDelegationTokenResponseData data) { + super(ApiKeys.DESCRIBE_DELEGATION_TOKEN); this.data = data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeGroupsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeGroupsResponse.java index aa638407e0625..27491daa8248c 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeGroupsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeGroupsResponse.java @@ -19,6 +19,7 @@ import org.apache.kafka.common.message.DescribeGroupsResponseData; import org.apache.kafka.common.message.DescribeGroupsResponseData.DescribedGroup; import org.apache.kafka.common.message.DescribeGroupsResponseData.DescribedGroupMember; +import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.utils.Utils; @@ -46,6 +47,7 @@ public class DescribeGroupsResponse extends AbstractResponse { private DescribeGroupsResponseData data; public DescribeGroupsResponse(DescribeGroupsResponseData data) { + super(ApiKeys.DESCRIBE_GROUPS); this.data = data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/EndTxnResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/EndTxnResponse.java index e06347736acff..f526ac82d8411 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/EndTxnResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/EndTxnResponse.java @@ -17,6 +17,7 @@ package org.apache.kafka.common.requests; import org.apache.kafka.common.message.EndTxnResponseData; +import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.Message; @@ -40,6 +41,7 @@ public class EndTxnResponse extends AbstractResponse { public final EndTxnResponseData data; public EndTxnResponse(EndTxnResponseData data) { + super(ApiKeys.END_TXN); this.data = data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenResponse.java index 5c786e1dec695..2398e1bcc22ba 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenResponse.java @@ -20,6 +20,7 @@ import java.util.Map; import org.apache.kafka.common.message.ExpireDelegationTokenResponseData; +import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.Message; @@ -29,6 +30,7 @@ public class ExpireDelegationTokenResponse extends AbstractResponse { private final ExpireDelegationTokenResponseData data; public ExpireDelegationTokenResponse(ExpireDelegationTokenResponseData data) { + super(ApiKeys.EXPIRE_DELEGATION_TOKEN); this.data = data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/FetchResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/FetchResponse.java index 1842c5ad73ba2..8a1732ba1c73f 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/FetchResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/FetchResponse.java @@ -367,6 +367,7 @@ public FetchResponse(Errors error, LinkedHashMap> responseData, int throttleTimeMs, int sessionId) { + super(ApiKeys.FETCH); this.error = error; this.responseData = responseData; this.throttleTimeMs = throttleTimeMs; diff --git a/clients/src/main/java/org/apache/kafka/common/requests/FindCoordinatorResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/FindCoordinatorResponse.java index 8d1b4436099d4..917b087fbab35 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/FindCoordinatorResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/FindCoordinatorResponse.java @@ -18,6 +18,7 @@ import org.apache.kafka.common.Node; import org.apache.kafka.common.message.FindCoordinatorResponseData; +import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; @@ -40,6 +41,7 @@ public class FindCoordinatorResponse extends AbstractResponse { private final FindCoordinatorResponseData data; public FindCoordinatorResponse(FindCoordinatorResponseData data) { + super(ApiKeys.FIND_COORDINATOR); this.data = data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatResponse.java index aa30e2071b8d6..9e37ecf30d016 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatResponse.java @@ -17,6 +17,7 @@ package org.apache.kafka.common.requests; import org.apache.kafka.common.message.HeartbeatResponseData; +import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.Message; @@ -39,6 +40,7 @@ public class HeartbeatResponse extends AbstractResponse { private final HeartbeatResponseData data; public HeartbeatResponse(HeartbeatResponseData data) { + super(ApiKeys.HEARTBEAT); this.data = data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/IncrementalAlterConfigsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/IncrementalAlterConfigsResponse.java index b051cb69b5f9b..7411017fb3a97 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/IncrementalAlterConfigsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/IncrementalAlterConfigsResponse.java @@ -20,6 +20,7 @@ import org.apache.kafka.common.config.ConfigResource; import org.apache.kafka.common.message.IncrementalAlterConfigsResponseData; import org.apache.kafka.common.message.IncrementalAlterConfigsResponseData.AlterConfigsResourceResponse; +import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; @@ -55,6 +56,7 @@ public static Map fromResponseData(final IncrementalAl private final IncrementalAlterConfigsResponseData data; public IncrementalAlterConfigsResponse(IncrementalAlterConfigsResponseData data) { + super(ApiKeys.INCREMENTAL_ALTER_CONFIGS); this.data = data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/InitProducerIdResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/InitProducerIdResponse.java index 5b42a0805fa3b..84079d2f25cdf 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/InitProducerIdResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/InitProducerIdResponse.java @@ -17,6 +17,7 @@ package org.apache.kafka.common.requests; import org.apache.kafka.common.message.InitProducerIdResponseData; +import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.Message; @@ -36,6 +37,7 @@ public class InitProducerIdResponse extends AbstractResponse { public final InitProducerIdResponseData data; public InitProducerIdResponse(InitProducerIdResponseData data) { + super(ApiKeys.INIT_PRODUCER_ID); this.data = data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/JoinGroupResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/JoinGroupResponse.java index f5865a3683c0f..8472175edab4a 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/JoinGroupResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/JoinGroupResponse.java @@ -17,6 +17,7 @@ package org.apache.kafka.common.requests; import org.apache.kafka.common.message.JoinGroupResponseData; +import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; @@ -28,6 +29,7 @@ public class JoinGroupResponse extends AbstractResponse { private final JoinGroupResponseData data; public JoinGroupResponse(JoinGroupResponseData data) { + super(ApiKeys.JOIN_GROUP); this.data = data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrResponse.java index 06f871d6d7925..a26b63f1f3c07 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrResponse.java @@ -18,6 +18,7 @@ import org.apache.kafka.common.message.LeaderAndIsrResponseData; import org.apache.kafka.common.message.LeaderAndIsrResponseData.LeaderAndIsrPartitionError; +import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.Message; @@ -38,6 +39,7 @@ public class LeaderAndIsrResponse extends AbstractResponse { private final LeaderAndIsrResponseData data; public LeaderAndIsrResponse(LeaderAndIsrResponseData data) { + super(ApiKeys.LEADER_AND_ISR); this.data = data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/LegacyAbstractResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/LegacyAbstractResponse.java index 67adb418d71aa..56634a39d2f49 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/LegacyAbstractResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/LegacyAbstractResponse.java @@ -1,11 +1,16 @@ package org.apache.kafka.common.requests; +import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; public abstract class LegacyAbstractResponse extends AbstractResponse { + protected LegacyAbstractResponse(ApiKeys apiKey) { + super(apiKey); + } + protected abstract Struct toStruct(short version); // Once this method is removed, we should make the superclass method private diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsResponse.java index e0ac65727347e..336368395b76f 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsResponse.java @@ -17,6 +17,7 @@ package org.apache.kafka.common.requests; import org.apache.kafka.common.message.ListGroupsResponseData; +import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; @@ -28,6 +29,7 @@ public class ListGroupsResponse extends AbstractResponse { private final ListGroupsResponseData data; public ListGroupsResponse(ListGroupsResponseData data) { + super(ApiKeys.LIST_GROUPS); this.data = data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetResponse.java index 531ab41ba8840..0d4013f636565 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetResponse.java @@ -197,11 +197,13 @@ public ListOffsetResponse(Map responseData) { } public ListOffsetResponse(int throttleTimeMs, Map responseData) { + super(ApiKeys.LIST_OFFSETS); this.throttleTimeMs = throttleTimeMs; this.responseData = responseData; } public ListOffsetResponse(Struct struct) { + super(ApiKeys.LIST_OFFSETS); this.throttleTimeMs = struct.getOrElse(THROTTLE_TIME_MS, DEFAULT_THROTTLE_TIME); responseData = new HashMap<>(); for (Object topicResponseObj : struct.get(TOPICS)) { diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ListPartitionReassignmentsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ListPartitionReassignmentsResponse.java index 4e50aa3e6a2f0..3449f1cabe9ba 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ListPartitionReassignmentsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ListPartitionReassignmentsResponse.java @@ -17,6 +17,7 @@ package org.apache.kafka.common.requests; import org.apache.kafka.common.message.ListPartitionReassignmentsResponseData; +import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; @@ -28,6 +29,7 @@ public class ListPartitionReassignmentsResponse extends AbstractResponse { private final ListPartitionReassignmentsResponseData data; public ListPartitionReassignmentsResponse(ListPartitionReassignmentsResponseData responseData) { + super(ApiKeys.LIST_PARTITION_REASSIGNMENTS); this.data = responseData; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/MetadataResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/MetadataResponse.java index b51d7ecc82325..aa2d03b8494f8 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/MetadataResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/MetadataResponse.java @@ -24,6 +24,7 @@ import org.apache.kafka.common.message.MetadataResponseData.MetadataResponseBroker; import org.apache.kafka.common.message.MetadataResponseData.MetadataResponsePartition; import org.apache.kafka.common.message.MetadataResponseData.MetadataResponseTopic; +import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.Message; @@ -69,6 +70,7 @@ public MetadataResponse(MetadataResponseData data, short version) { } MetadataResponse(MetadataResponseData data, boolean hasReliableLeaderEpochs) { + super(ApiKeys.METADATA); this.data = data; this.hasReliableLeaderEpochs = hasReliableLeaderEpochs; } @@ -478,7 +480,8 @@ public static MetadataResponse prepareResponse(boolean hasReliableLeaderEpochs, return new MetadataResponse(responseData, hasReliableLeaderEpochs); } - public static MetadataResponse prepareResponse(int throttleTimeMs, List topicMetadataList, + public static MetadataResponse prepareResponse(int throttleTimeMs, + List topicMetadataList, Collection brokers, String clusterId, int controllerId, int clusterAuthorizedOperations) { MetadataResponseData responseData = new MetadataResponseData(); @@ -496,7 +499,7 @@ public static MetadataResponse prepareResponse(int throttleTimeMs, List responseData.topics().add(topicMetadata)); - return new MetadataResponse(responseData); + return new MetadataResponse(responseData, true); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitResponse.java index dfc751d0877be..b2851a68deb84 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitResponse.java @@ -20,6 +20,7 @@ import org.apache.kafka.common.message.OffsetCommitResponseData; import org.apache.kafka.common.message.OffsetCommitResponseData.OffsetCommitResponsePartition; import org.apache.kafka.common.message.OffsetCommitResponseData.OffsetCommitResponseTopic; +import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; @@ -49,10 +50,12 @@ public class OffsetCommitResponse extends AbstractResponse { private final OffsetCommitResponseData data; public OffsetCommitResponse(OffsetCommitResponseData data) { + super(ApiKeys.OFFSET_COMMIT); this.data = data; } public OffsetCommitResponse(int requestThrottleMs, Map responseData) { + super(ApiKeys.OFFSET_COMMIT); Map responseTopicDataMap = new HashMap<>(); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchResponse.java index e016f8d9315df..0e87f6916a3fc 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchResponse.java @@ -20,6 +20,7 @@ import org.apache.kafka.common.message.OffsetFetchResponseData; import org.apache.kafka.common.message.OffsetFetchResponseData.OffsetFetchResponsePartition; import org.apache.kafka.common.message.OffsetFetchResponseData.OffsetFetchResponseTopic; +import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.Message; @@ -129,6 +130,7 @@ public OffsetFetchResponse(Errors error, Map resp * @param responseData Fetched offset information grouped by topic-partition */ public OffsetFetchResponse(int throttleTimeMs, Errors error, Map responseData) { + super(ApiKeys.OFFSET_FETCH); Map offsetFetchResponseTopicMap = new HashMap<>(); for (Map.Entry entry : responseData.entrySet()) { String topicName = entry.getKey().topic(); @@ -154,6 +156,7 @@ public OffsetFetchResponse(int throttleTimeMs, Errors error, Map epochEndOffsetsByPartition; public OffsetsForLeaderEpochResponse(Struct struct) { + super(ApiKeys.OFFSET_FOR_LEADER_EPOCH); this.throttleTimeMs = struct.getOrElse(THROTTLE_TIME_MS, DEFAULT_THROTTLE_TIME); this.epochEndOffsetsByPartition = new HashMap<>(); for (Object topicAndEpocsObj : struct.get(TOPICS)) { @@ -120,6 +121,7 @@ public OffsetsForLeaderEpochResponse(Map epochsB } public OffsetsForLeaderEpochResponse(int throttleTimeMs, Map epochsByTopic) { + super(ApiKeys.OFFSET_FOR_LEADER_EPOCH); this.throttleTimeMs = throttleTimeMs; this.epochEndOffsetsByPartition = epochsByTopic; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ProduceResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ProduceResponse.java index f8e029b0f4c50..37fc0a86cf048 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ProduceResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ProduceResponse.java @@ -205,6 +205,7 @@ public ProduceResponse(Map responses) { * @param throttleTimeMs Time in milliseconds the response was throttled */ public ProduceResponse(Map responses, int throttleTimeMs) { + super(ApiKeys.PRODUCE); this.responses = responses; this.throttleTimeMs = throttleTimeMs; } @@ -213,6 +214,7 @@ public ProduceResponse(Map responses, int thr * Constructor from a {@link Struct}. */ public ProduceResponse(Struct struct) { + super(ApiKeys.PRODUCE); responses = new HashMap<>(); for (Object topicResponse : struct.getArray(RESPONSES_KEY_NAME)) { Struct topicRespStruct = (Struct) topicResponse; diff --git a/clients/src/main/java/org/apache/kafka/common/requests/RenewDelegationTokenResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/RenewDelegationTokenResponse.java index e5d39607220db..8f33f7d7be692 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/RenewDelegationTokenResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/RenewDelegationTokenResponse.java @@ -20,6 +20,7 @@ import java.util.Map; import org.apache.kafka.common.message.RenewDelegationTokenResponseData; +import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.Message; @@ -29,6 +30,7 @@ public class RenewDelegationTokenResponse extends AbstractResponse { private final RenewDelegationTokenResponseData data; public RenewDelegationTokenResponse(RenewDelegationTokenResponseData data) { + super(ApiKeys.RENEW_DELEGATION_TOKEN); this.data = data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/SaslAuthenticateResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/SaslAuthenticateResponse.java index ba3d9c2d2891e..eb0fd42c1eada 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/SaslAuthenticateResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/SaslAuthenticateResponse.java @@ -17,6 +17,7 @@ package org.apache.kafka.common.requests; import org.apache.kafka.common.message.SaslAuthenticateResponseData; +import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.Message; @@ -33,6 +34,7 @@ public class SaslAuthenticateResponse extends AbstractResponse { private final SaslAuthenticateResponseData data; public SaslAuthenticateResponse(SaslAuthenticateResponseData data) { + super(ApiKeys.SASL_AUTHENTICATE); this.data = data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/SaslHandshakeResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/SaslHandshakeResponse.java index 4e3d28ef4deed..3863c186f5089 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/SaslHandshakeResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/SaslHandshakeResponse.java @@ -17,6 +17,7 @@ package org.apache.kafka.common.requests; import org.apache.kafka.common.message.SaslHandshakeResponseData; +import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.Message; @@ -34,6 +35,7 @@ public class SaslHandshakeResponse extends AbstractResponse { private final SaslHandshakeResponseData data; public SaslHandshakeResponse(SaslHandshakeResponseData data) { + super(ApiKeys.SASL_HANDSHAKE); this.data = data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/SyncGroupResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/SyncGroupResponse.java index abbfc947af4c9..c35f9178bf17d 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/SyncGroupResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/SyncGroupResponse.java @@ -17,6 +17,7 @@ package org.apache.kafka.common.requests; import org.apache.kafka.common.message.SyncGroupResponseData; +import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.Message; @@ -29,6 +30,7 @@ public class SyncGroupResponse extends AbstractResponse { public final SyncGroupResponseData data; public SyncGroupResponse(SyncGroupResponseData data) { + super(ApiKeys.SYNC_GROUP); this.data = data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitResponse.java index e5d9b550790a2..85cd609d963f6 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitResponse.java @@ -20,6 +20,7 @@ import org.apache.kafka.common.message.TxnOffsetCommitResponseData; import org.apache.kafka.common.message.TxnOffsetCommitResponseData.TxnOffsetCommitResponsePartition; import org.apache.kafka.common.message.TxnOffsetCommitResponseData.TxnOffsetCommitResponseTopic; +import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.Message; @@ -50,10 +51,12 @@ public class TxnOffsetCommitResponse extends AbstractResponse { public final TxnOffsetCommitResponseData data; public TxnOffsetCommitResponse(TxnOffsetCommitResponseData data) { + super(ApiKeys.TXN_OFFSET_COMMIT); this.data = data; } public TxnOffsetCommitResponse(int requestThrottleMs, Map responseData) { + super(ApiKeys.TXN_OFFSET_COMMIT); Map responseTopicDataMap = new HashMap<>(); for (Map.Entry entry : responseData.entrySet()) { diff --git a/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataResponse.java index 47aa8cf63ea5f..567bb9b6ad566 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataResponse.java @@ -17,6 +17,7 @@ package org.apache.kafka.common.requests; import org.apache.kafka.common.message.UpdateMetadataResponseData; +import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.Message; @@ -29,6 +30,7 @@ public class UpdateMetadataResponse extends AbstractResponse { private final UpdateMetadataResponseData data; public UpdateMetadataResponse(UpdateMetadataResponseData data) { + super(ApiKeys.UPDATE_METADATA); this.data = data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/WriteTxnMarkersResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/WriteTxnMarkersResponse.java index 5333dcc12d709..fb7ed00d3bdca 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/WriteTxnMarkersResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/WriteTxnMarkersResponse.java @@ -54,8 +54,7 @@ */ public class WriteTxnMarkersResponse extends AbstractResponse { - private final Map> errors; - public final WriteTxnMarkersResponseData data; + private final WriteTxnMarkersResponseData data; public WriteTxnMarkersResponse(Map> errors) { super(ApiKeys.WRITE_TXN_MARKERS); @@ -80,8 +79,6 @@ public WriteTxnMarkersResponse(Map> errors) { .setTopics(new ArrayList<>(responseTopicDataMap.values())) ); } - - this.errors = errors; this.data = new WriteTxnMarkersResponseData() .setMarkers(markers); } @@ -89,16 +86,16 @@ public WriteTxnMarkersResponse(Map> errors) { public WriteTxnMarkersResponse(WriteTxnMarkersResponseData data) { super(ApiKeys.WRITE_TXN_MARKERS); this.data = data; - //FIXME } @Override - protected Message data() { + public WriteTxnMarkersResponseData data() { return data; } public Map errors(long producerId) { - return errors.get(producerId); + return null; + //FIXME } @Override @@ -108,13 +105,15 @@ public int throttleTimeMs() { @Override public Map errorCounts() { - Map errorCounts = new HashMap<>(); - errors.values().forEach(allErrors -> - allErrors.values().forEach(error -> - updateErrorCounts(errorCounts, error) - ) - ); - return errorCounts; + //FIXME +// Map errorCounts = new HashMap<>(); +// errors.values().forEach(allErrors -> +// allErrors.values().forEach(error -> +// updateErrorCounts(errorCounts, error) +// ) +// ); +// return errorCounts; + return null; } public static WriteTxnMarkersResponse parse(ByteBuffer buffer, short version) { diff --git a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticator.java b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticator.java index 4032189bbc74e..beca99f707e33 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticator.java +++ b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticator.java @@ -566,8 +566,8 @@ private void handleApiVersionsRequest(RequestContext context, ApiVersionsRequest else if (!apiVersionsRequest.isValid()) sendKafkaResponse(context, apiVersionsRequest.getErrorResponse(0, Errors.INVALID_REQUEST.exception())); else { - metadataRegistry.registerClientInformation(new ClientInformation(apiVersionsRequest.data.clientSoftwareName(), - apiVersionsRequest.data.clientSoftwareVersion())); + metadataRegistry.registerClientInformation(new ClientInformation(apiVersionsRequest.data().clientSoftwareName(), + apiVersionsRequest.data().clientSoftwareVersion())); sendKafkaResponse(context, apiVersionsResponse()); setSaslState(SaslState.HANDSHAKE_REQUEST); } diff --git a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java index 3d3765a8048cb..076689bd5a50e 100644 --- a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java @@ -749,17 +749,18 @@ public void testDescribeAcls() throws Exception { // Test a call where we get back ACL1 and ACL2. env.kafkaClient().prepareResponse(new DescribeAclsResponse(new DescribeAclsResponseData() - .setResources(DescribeAclsResponse.aclsResources(asList(ACL1, ACL2))))); + .setResources(DescribeAclsResponse.aclsResources(asList(ACL1, ACL2))), ApiKeys.DESCRIBE_ACLS.latestVersion())); assertCollectionIs(env.adminClient().describeAcls(FILTER1).values().get(), ACL1, ACL2); // Test a call where we get back no results. - env.kafkaClient().prepareResponse(new DescribeAclsResponse(new DescribeAclsResponseData())); + env.kafkaClient().prepareResponse(new DescribeAclsResponse(new DescribeAclsResponseData(), + ApiKeys.DESCRIBE_ACLS.latestVersion())); assertTrue(env.adminClient().describeAcls(FILTER2).values().get().isEmpty()); // Test a call where we get back an error. env.kafkaClient().prepareResponse(new DescribeAclsResponse(new DescribeAclsResponseData() .setErrorCode(Errors.SECURITY_DISABLED.code()) - .setErrorMessage("Security is disabled"))); + .setErrorMessage("Security is disabled"), ApiKeys.DESCRIBE_ACLS.latestVersion())); TestUtils.assertFutureError(env.adminClient().describeAcls(FILTER2).values(), SecurityDisabledException.class); // Test a call where we supply an invalid filter. @@ -812,7 +813,8 @@ public void testDeleteAcls() throws Exception { DeleteAclsResponse.matchingAcl(ACL2, ApiError.NONE))), new DeleteAclsResponseData.DeleteAclsFilterResult() .setErrorCode(Errors.SECURITY_DISABLED.code()) - .setErrorMessage("No security"))))); + .setErrorMessage("No security"))), + ApiKeys.DELETE_ACLS.latestVersion())); DeleteAclsResult results = env.adminClient().deleteAcls(asList(FILTER1, FILTER2)); Map> filterResults = results.values(); FilterResults filter1Results = filterResults.get(FILTER1).get(); @@ -833,7 +835,8 @@ public void testDeleteAcls() throws Exception { new DeleteAclsResponseData.DeleteAclsMatchingAcl() .setErrorCode(Errors.SECURITY_DISABLED.code()) .setErrorMessage("No security"))), - new DeleteAclsResponseData.DeleteAclsFilterResult())))); + new DeleteAclsResponseData.DeleteAclsFilterResult())), + ApiKeys.DELETE_ACLS.latestVersion())); results = env.adminClient().deleteAcls(asList(FILTER1, FILTER2)); assertTrue(results.values().get(FILTER2).get().values().isEmpty()); TestUtils.assertFutureError(results.all(), SecurityDisabledException.class); @@ -845,7 +848,8 @@ public void testDeleteAcls() throws Exception { new DeleteAclsResponseData.DeleteAclsFilterResult() .setMatchingAcls(asList(DeleteAclsResponse.matchingAcl(ACL1, ApiError.NONE))), new DeleteAclsResponseData.DeleteAclsFilterResult() - .setMatchingAcls(asList(DeleteAclsResponse.matchingAcl(ACL2, ApiError.NONE))))))); + .setMatchingAcls(asList(DeleteAclsResponse.matchingAcl(ACL2, ApiError.NONE))))), + ApiKeys.DELETE_ACLS.latestVersion())); results = env.adminClient().deleteAcls(asList(FILTER1, FILTER2)); Collection deleted = results.all().get(); assertCollectionIs(deleted, ACL1, ACL2); @@ -1162,6 +1166,7 @@ public void testDescribeCluster() throws Exception { env.cluster().nodes(), env.cluster().clusterResource().clusterId(), 2, + Collections.emptyList(), MetadataResponse.AUTHORIZED_OPERATIONS_OMITTED); env.kafkaClient().prepareResponse(response); @@ -1170,6 +1175,7 @@ public void testDescribeCluster() throws Exception { env.cluster().nodes(), env.cluster().clusterResource().clusterId(), 3, + Collections.emptyList(), 1 << AclOperation.DESCRIBE.code() | 1 << AclOperation.ALTER.code()); env.kafkaClient().prepareResponse(response2); @@ -3316,7 +3322,7 @@ public void testDescribeClientQuotas() throws Exception { responseData.put(entity1, Collections.singletonMap("consumer_byte_rate", 10000.0)); responseData.put(entity2, Collections.singletonMap("producer_byte_rate", 20000.0)); - env.kafkaClient().prepareResponse(new DescribeClientQuotasResponse(responseData, 0)); + env.kafkaClient().prepareResponse(DescribeClientQuotasResponse.fromQuotaEntities(responseData, 0)); ClientQuotaFilter filter = ClientQuotaFilter.contains(asList(ClientQuotaFilterComponent.ofEntity(ClientQuotaEntity.USER, value))); @@ -3356,6 +3362,7 @@ public void testEqualsOfClientQuotaFilterComponent() { ClientQuotaFilterComponent.ofEntityType(ClientQuotaEntity.USER)); } + @Test public void testAlterClientQuotas() throws Exception { try (AdminClientUnitTestEnv env = mockClientEnv()) { env.kafkaClient().setNodeApiVersions(NodeApiVersions.create()); @@ -3369,7 +3376,7 @@ public void testAlterClientQuotas() throws Exception { responseData.put(unauthorizedEntity, new ApiError(Errors.CLUSTER_AUTHORIZATION_FAILED, "Authorization failed")); responseData.put(invalidEntity, new ApiError(Errors.INVALID_REQUEST, "Invalid quota entity")); - env.kafkaClient().prepareResponse(new AlterClientQuotasResponse(responseData, 0)); + env.kafkaClient().prepareResponse(AlterClientQuotasResponse.from(responseData, 0)); List entries = new ArrayList<>(3); entries.add(new ClientQuotaAlteration(goodEntity, Collections.singleton(new ClientQuotaAlteration.Op("consumer_byte_rate", 10000.0)))); diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestHeaderTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestHeaderTest.java index 0c3ab08a1b57c..cc883c569ed0e 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/RequestHeaderTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestHeaderTest.java @@ -17,7 +17,6 @@ package org.apache.kafka.common.requests; import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.ObjectSerializationCache; import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.test.TestUtils; import org.junit.Test; @@ -97,9 +96,6 @@ public void testRequestHeaderV2() { } private ByteBuffer toBuffer(RequestHeader header) { - ObjectSerializationCache serializationCache = new ObjectSerializationCache(); - ByteBuffer buffer = ByteBuffer.allocate(header.size(serializationCache)); - header.write(buffer, serializationCache); - return buffer; + return TestUtils.serializeRequestHeader(header); } } diff --git a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java index dee1af58a6df7..cbe852960d7cf 100644 --- a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java +++ b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java @@ -1584,11 +1584,11 @@ SaslClient createSaslClient() { public void testConvertListOffsetResponseToSaslHandshakeResponse() { ListOffsetResponse response = new ListOffsetResponse(0, Collections.singletonMap(new TopicPartition("topic", 0), new ListOffsetResponse.PartitionData(Errors.NONE, 0, 0, Optional.empty()))); - ByteBuffer buffer = response.serialize(ApiKeys.LIST_OFFSETS, LIST_OFFSETS.latestVersion(), 0); + ByteBuffer buffer = response.serializeWithHeader(LIST_OFFSETS.latestVersion(), 0); final RequestHeader header0 = new RequestHeader(LIST_OFFSETS, LIST_OFFSETS.latestVersion(), "id", SaslClientAuthenticator.MIN_RESERVED_CORRELATION_ID); - Assert.assertThrows(SchemaException.class, () -> NetworkClient.parseResponse(buffer.duplicate(), header0)); + Assert.assertThrows(SchemaException.class, () -> NetworkClient.parseResponse(header0, buffer.duplicate())); final RequestHeader header1 = new RequestHeader(LIST_OFFSETS, LIST_OFFSETS.latestVersion(), "id", 1); - Assert.assertThrows(IllegalStateException.class, () -> NetworkClient.parseResponse(buffer.duplicate(), header1)); + Assert.assertThrows(IllegalStateException.class, () -> NetworkClient.parseResponse(header1, buffer.duplicate())); } /** diff --git a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticatorTest.java b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticatorTest.java index 0b4c63bc42cb1..75f96544720a2 100644 --- a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticatorTest.java +++ b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticatorTest.java @@ -34,6 +34,7 @@ import org.apache.kafka.common.security.plain.PlainLoginModule; import org.apache.kafka.common.utils.AppInfoParser; import org.apache.kafka.common.utils.Time; +import org.apache.kafka.test.TestUtils; import org.junit.Test; import javax.security.auth.Subject; @@ -79,16 +80,16 @@ public void testUnexpectedRequestType() throws IOException { SaslServerAuthenticator authenticator = setupAuthenticator(configs, transportLayer, SCRAM_SHA_256.mechanismName(), new DefaultChannelMetadataRegistry()); - final RequestHeader header = new RequestHeader(ApiKeys.METADATA, (short) 0, "clientId", 13243); - final Struct headerStruct = header.toStruct(); + RequestHeader header = new RequestHeader(ApiKeys.METADATA, (short) 0, "clientId", 13243); + ByteBuffer headerBuffer = TestUtils.serializeRequestHeader(header); when(transportLayer.read(any(ByteBuffer.class))).then(invocation -> { - invocation.getArgument(0).putInt(headerStruct.sizeOf()); + invocation.getArgument(0).putInt(headerBuffer.remaining()); return 4; }).then(invocation -> { // serialize only the request header. the authenticator should not parse beyond this - headerStruct.writeTo(invocation.getArgument(0)); - return headerStruct.sizeOf(); + invocation.getArgument(0).put(headerBuffer.duplicate()); + return headerBuffer.remaining(); }); try { @@ -122,21 +123,22 @@ public void testApiVersionsRequest(short version, String expectedSoftwareName, SaslServerAuthenticator authenticator = setupAuthenticator(configs, transportLayer, SCRAM_SHA_256.mechanismName(), metadataRegistry); - final RequestHeader header = new RequestHeader(ApiKeys.API_VERSIONS, version, "clientId", 0); - final Struct headerStruct = header.toStruct(); + RequestHeader header = new RequestHeader(ApiKeys.API_VERSIONS, version, "clientId", 0); + ByteBuffer headerBuffer = TestUtils.serializeRequestHeader(header); - final ApiVersionsRequest request = new ApiVersionsRequest.Builder().build(version); - final Struct requestStruct = request.data.toStruct(version); + ApiVersionsRequest request = new ApiVersionsRequest.Builder().build(version); + ByteBuffer requestBuffer = TestUtils.serializeMessage(request.data(), version); when(transportLayer.socketChannel().socket().getInetAddress()).thenReturn(InetAddress.getLoopbackAddress()); when(transportLayer.read(any(ByteBuffer.class))).then(invocation -> { - invocation.getArgument(0).putInt(headerStruct.sizeOf() + requestStruct.sizeOf()); + invocation.getArgument(0).putInt(headerBuffer.remaining() + requestBuffer.remaining()); return 4; }).then(invocation -> { - headerStruct.writeTo(invocation.getArgument(0)); - requestStruct.writeTo(invocation.getArgument(0)); - return headerStruct.sizeOf() + requestStruct.sizeOf(); + ByteBuffer buffer = invocation.getArgument(0).put(headerBuffer.duplicate()); + buffer.put(headerBuffer.duplicate()); + buffer.put(requestBuffer.duplicate()); + return headerBuffer.remaining() + requestBuffer.remaining(); }); authenticator.authenticate(); diff --git a/clients/src/test/java/org/apache/kafka/test/TestUtils.java b/clients/src/test/java/org/apache/kafka/test/TestUtils.java index a5491a50cf7b6..980775d08bf36 100644 --- a/clients/src/test/java/org/apache/kafka/test/TestUtils.java +++ b/clients/src/test/java/org/apache/kafka/test/TestUtils.java @@ -26,7 +26,10 @@ import org.apache.kafka.common.internals.Topic; import org.apache.kafka.common.network.NetworkReceive; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.Message; +import org.apache.kafka.common.protocol.ObjectSerializationCache; import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.requests.MetadataResponse; import org.apache.kafka.common.requests.RequestHeader; @@ -194,6 +197,20 @@ public static MetadataResponse metadataUpdateWith(final String clusterId, return metadataResponse(nodes, clusterId, 0, topicMetadata); } + public static ByteBuffer serializeRequestHeader(RequestHeader header) { + ObjectSerializationCache serializationCache = new ObjectSerializationCache(); + ByteBuffer buffer = ByteBuffer.allocate(header.size(serializationCache)); + header.write(buffer, serializationCache); + return buffer; + } + + public static ByteBuffer serializeMessage(Message message, short version) { + ObjectSerializationCache serializationCache = new ObjectSerializationCache(); + ByteBuffer buffer = ByteBuffer.allocate(message.size(serializationCache, version)); + message.write(new ByteBufferAccessor(buffer), serializationCache, version); + return buffer; + } + @FunctionalInterface public interface PartitionMetadataSupplier { MetadataResponse.PartitionMetadata supply(Errors error, From 00204bc31ffb2b86b58e3bda34daad3bccc0890d Mon Sep 17 00:00:00 2001 From: Ismael Juma Date: Mon, 27 Jul 2020 16:35:24 -0700 Subject: [PATCH 21/51] WIP --- .../kafka/common/requests/AlterReplicaLogDirsResponse.java | 4 +++- .../org/apache/kafka/common/requests/MetadataResponse.java | 4 ++-- .../org/apache/kafka/clients/admin/KafkaAdminClientTest.java | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AlterReplicaLogDirsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AlterReplicaLogDirsResponse.java index 1987772d105e8..afa658d1e150f 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AlterReplicaLogDirsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AlterReplicaLogDirsResponse.java @@ -17,7 +17,8 @@ package org.apache.kafka.common.requests; -import org.apache.kafka.common.message.AlterReplicaLogDirsRequestData; +import org.apache.kafka.common.message.AlterReplicaLogDirsResponseData; +import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; @@ -38,6 +39,7 @@ public class AlterReplicaLogDirsResponse extends AbstractResponse { private final AlterReplicaLogDirsResponseData data; public AlterReplicaLogDirsResponse(AlterReplicaLogDirsResponseData data) { + super(ApiKeys.ALTER_REPLICA_LOG_DIRS); this.data = data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/MetadataResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/MetadataResponse.java index e80c8f43e3108..ae61793d7c3e9 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/MetadataResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/MetadataResponse.java @@ -436,7 +436,7 @@ public static MetadataResponse prepareResponse(short version, int throttleTimeMs List topicMetadataList, int clusterAuthorizedOperations) { return prepareResponse(hasReliableLeaderEpochs(version), throttleTimeMs, brokers, clusterId, controllerId, - topicMetadataList, clusterAuthorizedOperations); + topicMetadataList, clusterAuthorizedOperations, version); } // Visible for testing @@ -479,7 +479,7 @@ public static MetadataResponse prepareResponse(boolean hasReliableLeaderEpochs, } responseData.topics().add(metadataResponseTopic); }); - return new MetadataResponse(responseData, hasReliableLeaderEpochs, responseVersion); + return new MetadataResponse(responseData, hasReliableLeaderEpochs); } public static MetadataResponse prepareResponse(int throttleTimeMs, diff --git a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java index 4da77785943de..55f51ade71c71 100644 --- a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java @@ -3552,7 +3552,7 @@ public void testAlterClientQuotas() throws Exception { responseData.put(unauthorizedEntity, new ApiError(Errors.CLUSTER_AUTHORIZATION_FAILED, "Authorization failed")); responseData.put(invalidEntity, new ApiError(Errors.INVALID_REQUEST, "Invalid quota entity")); - env.kafkaClient().prepareResponse(AlterClientQuotasResponse.from(responseData, 0)); + env.kafkaClient().prepareResponse(AlterClientQuotasResponse.fromQuotaEntities(responseData, 0)); List entries = new ArrayList<>(3); entries.add(new ClientQuotaAlteration(goodEntity, Collections.singleton(new ClientQuotaAlteration.Op("consumer_byte_rate", 10000.0)))); From 5d24335ceeebf219dc6d9c7b30b5c699fb5d835b Mon Sep 17 00:00:00 2001 From: Ismael Juma Date: Tue, 1 Dec 2020 13:52:46 -0800 Subject: [PATCH 22/51] Fix merge --- .../common/requests/AbstractRequest.java | 22 ++- .../common/requests/AbstractResponse.java | 23 ++- .../requests/AlterClientQuotasRequest.java | 2 +- .../common/requests/ApiVersionsResponse.java | 16 -- .../requests/LegacyAbstractRequest.java | 1 + .../requests/LegacyAbstractResponse.java | 1 + .../common/requests/MetadataResponse.java | 63 ++------ .../kafka/common/requests/RequestHeader.java | 6 +- .../kafka/common/requests/ResponseHeader.java | 8 +- .../common/requests/StopReplicaRequest.java | 2 +- .../apache/kafka/clients/MetadataTest.java | 9 +- .../kafka/clients/NetworkClientTest.java | 59 +++---- .../clients/admin/KafkaAdminClientTest.java | 5 +- .../AddPartitionsToTxnResponseTest.java | 13 +- .../requests/CreateAclsRequestTest.java | 9 +- .../requests/DeleteAclsRequestTest.java | 11 +- .../requests/DeleteAclsResponseTest.java | 53 ++++--- .../requests/DescribeAclsRequestTest.java | 24 +-- .../requests/DescribeAclsResponseTest.java | 13 +- .../common/requests/ProduceResponseTest.java | 2 +- .../common/requests/RequestHeaderTest.java | 19 --- .../common/requests/RequestResponseTest.java | 146 +++++++++--------- .../requests/StopReplicaRequestTest.java | 26 +++- .../java/org/apache/kafka/test/TestUtils.java | 37 +++-- .../kafka/connect/util/TopicAdminTest.java | 3 +- .../scala/kafka/network/RequestChannel.scala | 6 +- .../main/scala/kafka/server/KafkaApis.scala | 73 ++++++--- .../kafka/network/RequestChannelTest.scala | 4 +- .../unit/kafka/network/SocketServerTest.scala | 4 +- .../AbstractCreateTopicsRequestTest.scala | 10 -- .../kafka/server/EdgeCaseRequestTest.scala | 2 +- .../unit/kafka/server/KafkaApisTest.scala | 12 +- .../unit/kafka/server/RequestQuotaTest.scala | 30 +--- .../server/SaslApiVersionsRequestTest.scala | 11 +- .../metadata/MetadataRequestBenchmark.java | 2 +- 35 files changed, 328 insertions(+), 399 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java index 1aa28d7bb5563..6f453b9db12a5 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java @@ -98,17 +98,9 @@ public Send toSend(String destination, RequestHeader header) { return new NetworkSend(destination, serializeWithHeader(header)); } - /** - * Use with care, typically {@link #toSend(String, RequestHeader)} should be used instead. - */ + // Visible for testing public ByteBuffer serializeWithHeader(RequestHeader header) { - Message data = data(); - ObjectSerializationCache serializationCache = new ObjectSerializationCache(); - ByteBuffer buffer = ByteBuffer.allocate(header.size(serializationCache) + data.size(serializationCache, version)); - header.write(buffer, serializationCache); - data.write(new ByteBufferAccessor(buffer), serializationCache, version); - buffer.rewind(); - return buffer; + return serialize(header, data(), version); } /** @@ -120,9 +112,15 @@ public ByteBuffer serializeWithHeader(RequestHeader header) { // Visible for testing ByteBuffer serializeBody() { - Message data = data(); + return serialize(null, data(), version); + } + + private static ByteBuffer serialize(RequestHeader header, Message data, short version) { ObjectSerializationCache serializationCache = new ObjectSerializationCache(); - ByteBuffer buffer = ByteBuffer.allocate(data.size(serializationCache, version)); + int headerSize = header == null ? 0 : header.size(serializationCache); + ByteBuffer buffer = ByteBuffer.allocate(headerSize + data.size(serializationCache, version)); + if (header != null) + header.write(buffer, serializationCache); data.write(new ByteBufferAccessor(buffer), serializationCache, version); buffer.rewind(); return buffer; diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java index b042aeb98a26f..1b74ee2073366 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java @@ -29,6 +29,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.Objects; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -53,11 +54,21 @@ public ByteBuffer serializeWithHeader(short version, int correlationId) { } protected ByteBuffer serializeWithHeader(ResponseHeader header, short version) { - Message data = data(); + Objects.requireNonNull(header, "header should not be null"); + return serialize(header, data(), version); + } + + // Visible for testing + ByteBuffer serializeBody(short version) { + return serialize(null, data(), version); + } + + private static ByteBuffer serialize(ResponseHeader header, Message data, short version) { ObjectSerializationCache serializationCache = new ObjectSerializationCache(); - ByteBuffer buffer = ByteBuffer.allocate(header.size(serializationCache, version) + - data.size(serializationCache, version)); - header.write(buffer, serializationCache, version); + int headerSize = header == null ? 0 : header.size(serializationCache); + ByteBuffer buffer = ByteBuffer.allocate(headerSize + data.size(serializationCache, version)); + if (header != null) + header.write(buffer, serializationCache); data.write(new ByteBufferAccessor(buffer), serializationCache, version); buffer.rewind(); return buffer; @@ -216,6 +227,10 @@ public boolean shouldClientThrottle(short version) { return false; } + public ApiKeys apiKey() { + return apiKey; + } + public abstract int throttleTimeMs(); public String toString(short version) { diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasRequest.java index c1aafd041c967..38d6faf24e581 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasRequest.java @@ -87,7 +87,7 @@ public AlterClientQuotasRequest(AlterClientQuotasRequestData data, short version this.data = data; } - public Collection entries() { + public List entries() { List entries = new ArrayList<>(data.entries().size()); for (EntryData entryData : data.entries()) { Map entity = new HashMap<>(entryData.entity().size()); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsResponse.java index 97fe59153d84a..7903e6a34e0b0 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsResponse.java @@ -114,22 +114,6 @@ public static ApiVersionsResponse parse(ByteBuffer buffer, short version) { } } - public static ApiVersionsResponse fromStruct(Struct struct, short version) { - // Fallback to version 0 for ApiVersions response. If a client sends an ApiVersionsRequest - // using a version higher than that supported by the broker, a version 0 response is sent - // to the client indicating UNSUPPORTED_VERSION. When the client receives the response, it - // falls back while parsing it into a Struct which means that the version received by this - // method is not necessary the real one. It may be version 0 as well. - try { - return new ApiVersionsResponse(struct, version); - } catch (SchemaException e) { - if (version != 0) - return new ApiVersionsResponse(struct, (short) 0); - else - throw e; - } - } - public static ApiVersionsResponse apiVersionsResponse( int throttleTimeMs, byte maxMagic, diff --git a/clients/src/main/java/org/apache/kafka/common/requests/LegacyAbstractRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/LegacyAbstractRequest.java index b5f54eb183c6d..40c5e721e73e9 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/LegacyAbstractRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/LegacyAbstractRequest.java @@ -5,6 +5,7 @@ import java.nio.ByteBuffer; +// This should be removed once all requests are converted to use the generated request classes public abstract class LegacyAbstractRequest extends AbstractRequest { protected LegacyAbstractRequest(ApiKeys apiKeys, short version) { diff --git a/clients/src/main/java/org/apache/kafka/common/requests/LegacyAbstractResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/LegacyAbstractResponse.java index 56634a39d2f49..6934fe35492ea 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/LegacyAbstractResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/LegacyAbstractResponse.java @@ -5,6 +5,7 @@ import java.nio.ByteBuffer; +// This should be removed once all responses are converted to use the generated response classes public abstract class LegacyAbstractResponse extends AbstractResponse { protected LegacyAbstractResponse(ApiKeys apiKey) { diff --git a/clients/src/main/java/org/apache/kafka/common/requests/MetadataResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/MetadataResponse.java index ae61793d7c3e9..b5648c28c4ec3 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/MetadataResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/MetadataResponse.java @@ -430,63 +430,24 @@ private Collection createTopicMetadata(MetadataResponseData data) } - - public static MetadataResponse prepareResponse(short version, int throttleTimeMs, - Collection brokers, String clusterId, int controllerId, - List topicMetadataList, + public static MetadataResponse prepareResponse(short version, + int throttleTimeMs, + Collection brokers, + String clusterId, + int controllerId, + List topics, int clusterAuthorizedOperations) { return prepareResponse(hasReliableLeaderEpochs(version), throttleTimeMs, brokers, clusterId, controllerId, - topicMetadataList, clusterAuthorizedOperations, version); + topics, clusterAuthorizedOperations); } // Visible for testing - public static MetadataResponse prepareResponse(boolean hasReliableLeaderEpochs, int throttleTimeMs, - Collection brokers, String clusterId, int controllerId, - List topicMetadataList, - int clusterAuthorizedOperations, - short responseVersion) { - MetadataResponseData responseData = new MetadataResponseData(); - responseData.setThrottleTimeMs(throttleTimeMs); - brokers.forEach(broker -> - responseData.brokers().add(new MetadataResponseBroker() - .setNodeId(broker.id()) - .setHost(broker.host()) - .setPort(broker.port()) - .setRack(broker.rack())) - ); - - responseData.setClusterId(clusterId); - responseData.setControllerId(controllerId); - responseData.setClusterAuthorizedOperations(clusterAuthorizedOperations); - - topicMetadataList.forEach(topicMetadata -> { - MetadataResponseTopic metadataResponseTopic = new MetadataResponseTopic(); - metadataResponseTopic - .setErrorCode(topicMetadata.error.code()) - .setName(topicMetadata.topic) - .setIsInternal(topicMetadata.isInternal) - .setTopicAuthorizedOperations(topicMetadata.authorizedOperations); - - for (PartitionMetadata partitionMetadata : topicMetadata.partitionMetadata) { - metadataResponseTopic.partitions().add(new MetadataResponsePartition() - .setErrorCode(partitionMetadata.error.code()) - .setPartitionIndex(partitionMetadata.partition()) - .setLeaderId(partitionMetadata.leaderId.orElse(NO_LEADER_ID)) - .setLeaderEpoch(partitionMetadata.leaderEpoch.orElse(RecordBatch.NO_PARTITION_LEADER_EPOCH)) - .setReplicaNodes(partitionMetadata.replicaIds) - .setIsrNodes(partitionMetadata.inSyncReplicaIds) - .setOfflineReplicas(partitionMetadata.offlineReplicaIds)); - } - responseData.topics().add(metadataResponseTopic); - }); - return new MetadataResponse(responseData, hasReliableLeaderEpochs); - } - - public static MetadataResponse prepareResponse(int throttleTimeMs, - List topicMetadataList, + public static MetadataResponse prepareResponse(boolean hasReliableEpoch, + int throttleTimeMs, Collection brokers, String clusterId, int controllerId, + List topics, int clusterAuthorizedOperations) { MetadataResponseData responseData = new MetadataResponseData(); responseData.setThrottleTimeMs(throttleTimeMs); @@ -502,8 +463,8 @@ public static MetadataResponse prepareResponse(int throttleTimeMs, responseData.setControllerId(controllerId); responseData.setClusterAuthorizedOperations(clusterAuthorizedOperations); - topicMetadataList.forEach(topicMetadata -> responseData.topics().add(topicMetadata)); - return new MetadataResponse(responseData, true); + topics.forEach(topicMetadata -> responseData.topics().add(topicMetadata)); + return new MetadataResponse(responseData, hasReliableEpoch); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/RequestHeader.java b/clients/src/main/java/org/apache/kafka/common/requests/RequestHeader.java index 2a7a4b29b8309..2ca7c4ae614a6 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/RequestHeader.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/RequestHeader.java @@ -71,11 +71,11 @@ public RequestHeaderData data() { } public void write(ByteBuffer buffer, ObjectSerializationCache serializationCache) { - data.write(new ByteBufferAccessor(buffer), serializationCache, apiVersion()); + data.write(new ByteBufferAccessor(buffer), serializationCache, headerVersion); } public int size(ObjectSerializationCache serializationCache) { - return data.size(serializationCache, apiVersion()); + return data.size(serializationCache, headerVersion); } public ResponseHeader toResponseHeader() { @@ -85,6 +85,8 @@ public ResponseHeader toResponseHeader() { public static RequestHeader parse(ByteBuffer buffer) { short apiKey = -1; try { + // We derive the header version from the request api version, so we read that first. + // The request api version is part of `RequestHeaderData`, so we rewind the buffer after the read. apiKey = buffer.getShort(); short apiVersion = buffer.getShort(); short headerVersion = ApiKeys.forId(apiKey).requestHeaderVersion(apiVersion); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ResponseHeader.java b/clients/src/main/java/org/apache/kafka/common/requests/ResponseHeader.java index e51c5e3236168..87efeb032b5ec 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ResponseHeader.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ResponseHeader.java @@ -38,8 +38,8 @@ public ResponseHeader(ResponseHeaderData data, short headerVersion) { this.headerVersion = headerVersion; } - public int size(ObjectSerializationCache serializationCache, short apiVersion) { - return data().size(serializationCache, apiVersion); + public int size(ObjectSerializationCache serializationCache) { + return data().size(serializationCache, headerVersion); } public int correlationId() { @@ -54,8 +54,8 @@ public ResponseHeaderData data() { return data; } - public void write(ByteBuffer buffer, ObjectSerializationCache serializationCache, short apiVersion) { - data.write(new ByteBufferAccessor(buffer), serializationCache, apiVersion); + public void write(ByteBuffer buffer, ObjectSerializationCache serializationCache) { + data.write(new ByteBufferAccessor(buffer), serializationCache, headerVersion); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaRequest.java index 306bfcdc6f8c5..95a08b5c57046 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaRequest.java @@ -212,7 +212,7 @@ public static StopReplicaRequest parse(ByteBuffer buffer, short version) { } @Override - protected Message data() { + protected StopReplicaRequestData data() { return data; } } diff --git a/clients/src/test/java/org/apache/kafka/clients/MetadataTest.java b/clients/src/test/java/org/apache/kafka/clients/MetadataTest.java index 4de6ec6199031..fc5b22524d8ea 100644 --- a/clients/src/test/java/org/apache/kafka/clients/MetadataTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/MetadataTest.java @@ -41,6 +41,7 @@ import org.junit.Test; import java.net.InetSocketAddress; +import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -196,8 +197,8 @@ public void testIgnoreLeaderEpochInOlderMetadataResponse() { .setBrokers(new MetadataResponseBrokerCollection()); for (short version = ApiKeys.METADATA.oldestVersion(); version < 9; version++) { - Struct struct = data.toStruct(version); - MetadataResponse response = new MetadataResponse(struct, version); + ByteBuffer buffer = TestUtils.serializeMessage(data, version); + MetadataResponse response = MetadataResponse.parse(buffer, version); assertFalse(response.hasReliableLeaderEpochs()); metadata.updateWithCurrentRequestVersion(response, false, 100); assertTrue(metadata.partitionMetadataIfCurrent(tp).isPresent()); @@ -206,8 +207,8 @@ public void testIgnoreLeaderEpochInOlderMetadataResponse() { } for (short version = 9; version <= ApiKeys.METADATA.latestVersion(); version++) { - Struct struct = data.toStruct(version); - MetadataResponse response = new MetadataResponse(struct, version); + ByteBuffer buffer = TestUtils.serializeMessage(data, version); + MetadataResponse response = MetadataResponse.parse(buffer, version); assertTrue(response.hasReliableLeaderEpochs()); metadata.updateWithCurrentRequestVersion(response, false, 100); assertTrue(metadata.partitionMetadataIfCurrent(tp).isPresent()); diff --git a/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java b/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java index e12b62d607b74..60856d3b4ee11 100644 --- a/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java @@ -32,10 +32,12 @@ import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.record.MemoryRecords; +import org.apache.kafka.common.requests.AbstractResponse; import org.apache.kafka.common.requests.ApiVersionsResponse; import org.apache.kafka.common.requests.MetadataRequest; import org.apache.kafka.common.requests.MetadataResponse; import org.apache.kafka.common.requests.ProduceRequest; +import org.apache.kafka.common.requests.ProduceResponse; import org.apache.kafka.common.requests.RequestHeader; import org.apache.kafka.common.requests.ResponseHeader; import org.apache.kafka.common.security.authenticator.SaslClientAuthenticator; @@ -186,9 +188,10 @@ public void testUnsupportedVersionDuringInternalMetadataRequest() { private void checkSimpleRequestResponse(NetworkClient networkClient) { awaitReady(networkClient, node); // has to be before creating any request, as it may send ApiVersionsRequest and its response is mocked with correlation id 0 + short requestVersion = PRODUCE.latestVersion(); ProduceRequest.Builder builder = new ProduceRequest.Builder( - PRODUCE.latestVersion(), - PRODUCE.latestVersion(), + requestVersion, + requestVersion, (short) 1, 1000, Collections.emptyMap(), @@ -199,16 +202,8 @@ private void checkSimpleRequestResponse(NetworkClient networkClient) { networkClient.send(request, time.milliseconds()); networkClient.poll(1, time.milliseconds()); assertEquals(1, networkClient.inFlightRequestCount()); - ResponseHeader respHeader = - new ResponseHeader(request.correlationId(), - request.apiKey().responseHeaderVersion(PRODUCE.latestVersion())); - Struct resp = new Struct(PRODUCE.responseSchema(PRODUCE.latestVersion())); - resp.set("responses", new Object[0]); - Struct responseHeaderStruct = respHeader.toStruct(); - int size = responseHeaderStruct.sizeOf() + resp.sizeOf(); - ByteBuffer buffer = ByteBuffer.allocate(size); - responseHeaderStruct.writeTo(buffer); - resp.writeTo(buffer); + ProduceResponse produceResponse = new ProduceResponse(Collections.emptyMap()); + ByteBuffer buffer = produceResponse.serializeWithHeader(requestVersion, request.correlationId()); buffer.flip(); selector.completeReceive(new NetworkReceive(node.idString(), buffer)); List responses = networkClient.poll(1, time.milliseconds()); @@ -220,7 +215,7 @@ private void checkSimpleRequestResponse(NetworkClient networkClient) { } private void delayedApiVersionsResponse(int correlationId, short version, ApiVersionsResponse response) { - ByteBuffer buffer = response.serializeWithHeader(ApiKeys.API_VERSIONS, version, correlationId); + ByteBuffer buffer = response.serializeWithHeader(version, correlationId); selector.delayedReceive(new DelayedReceive(node.idString(), new NetworkReceive(node.idString(), buffer))); } @@ -499,9 +494,10 @@ public void testConnectionSetupTimeout() { public void testConnectionThrottling() { // Instrument the test to return a response with a 100ms throttle delay. awaitReady(client, node); + short requestVersion = PRODUCE.latestVersion(); ProduceRequest.Builder builder = new ProduceRequest.Builder( - PRODUCE.latestVersion(), - PRODUCE.latestVersion(), + requestVersion, + requestVersion, (short) 1, 1000, Collections.emptyMap(), @@ -511,17 +507,9 @@ public void testConnectionThrottling() { defaultRequestTimeoutMs, handler); client.send(request, time.milliseconds()); client.poll(1, time.milliseconds()); - ResponseHeader respHeader = - new ResponseHeader(request.correlationId(), - request.apiKey().responseHeaderVersion(PRODUCE.latestVersion())); - Struct resp = new Struct(PRODUCE.responseSchema(PRODUCE.latestVersion())); - resp.set("responses", new Object[0]); - resp.set(CommonFields.THROTTLE_TIME_MS, 100); - Struct responseHeaderStruct = respHeader.toStruct(); - int size = responseHeaderStruct.sizeOf() + resp.sizeOf(); - ByteBuffer buffer = ByteBuffer.allocate(size); - responseHeaderStruct.writeTo(buffer); - resp.writeTo(buffer); + int throttleTime = 100; + ProduceResponse produceResponse = new ProduceResponse(Collections.emptyMap(), throttleTime); + ByteBuffer buffer = produceResponse.serializeWithHeader(requestVersion, request.correlationId()); buffer.flip(); selector.completeReceive(new NetworkReceive(node.idString(), buffer)); client.poll(1, time.milliseconds()); @@ -599,23 +587,16 @@ private int sendEmptyProduceRequest(String nodeId) { return request.correlationId(); } - private void sendResponse(ResponseHeader respHeader, Struct response) { - Struct responseHeaderStruct = respHeader.toStruct(); - int size = responseHeaderStruct.sizeOf() + response.sizeOf(); - ByteBuffer buffer = ByteBuffer.allocate(size); - responseHeaderStruct.writeTo(buffer); - response.writeTo(buffer); + private void sendResponse(AbstractResponse response, short version, int correlationId) { + ByteBuffer buffer = response.serializeWithHeader(version, correlationId); buffer.flip(); selector.completeReceive(new NetworkReceive(node.idString(), buffer)); } private void sendThrottledProduceResponse(int correlationId, int throttleMs) { - Struct resp = new Struct(PRODUCE.responseSchema(PRODUCE.latestVersion())); - resp.set("responses", new Object[0]); - resp.set(CommonFields.THROTTLE_TIME_MS, throttleMs); - sendResponse(new ResponseHeader(correlationId, - PRODUCE.responseHeaderVersion(PRODUCE.latestVersion())), - resp); + short requestVersion = PRODUCE.latestVersion(); + ProduceResponse response = new ProduceResponse(Collections.emptyMap(), throttleMs); + sendResponse(response, requestVersion, correlationId); } @Test @@ -702,7 +683,7 @@ public void testAuthenticationFailureWithInFlightMetadataRequest() { RequestHeader header = parseHeader(requestBuffer); assertEquals(ApiKeys.METADATA, header.apiKey()); - ByteBuffer responseBuffer = metadataResponse.serialize(ApiKeys.METADATA, header.apiVersion(), header.correlationId()); + ByteBuffer responseBuffer = metadataResponse.serializeWithHeader(header.apiVersion(), header.correlationId()); selector.delayedReceive(new DelayedReceive(node1.idString(), new NetworkReceive(node1.idString(), responseBuffer))); int initialUpdateVersion = metadata.updateVersion(); diff --git a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java index 9f9164105d12e..cc885e6fb2f14 100644 --- a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java @@ -441,11 +441,12 @@ private static MetadataResponse prepareMetadataResponse(Cluster cluster, Errors .setPartitions(pms); metadata.add(tm); } - return MetadataResponse.prepareResponse(0, - metadata, + return MetadataResponse.prepareResponse(true, + 0, cluster.nodes(), cluster.clusterResource().clusterId(), cluster.controller().id(), + metadata, MetadataResponse.AUTHORIZED_OPERATIONS_OMITTED); } diff --git a/clients/src/test/java/org/apache/kafka/common/requests/AddPartitionsToTxnResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/AddPartitionsToTxnResponseTest.java index 7d49fb5509f04..92c38e557b93d 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/AddPartitionsToTxnResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/AddPartitionsToTxnResponseTest.java @@ -67,7 +67,7 @@ public void testConstructorWithErrorResponse() { } @Test - public void testConstructorWithStruct() { + public void testParse() { AddPartitionsToTxnTopicResultCollection topicCollection = new AddPartitionsToTxnTopicResultCollection(); @@ -87,14 +87,13 @@ public void testConstructorWithStruct() { AddPartitionsToTxnResponseData data = new AddPartitionsToTxnResponseData() .setResults(topicCollection) .setThrottleTimeMs(throttleTimeMs); + AddPartitionsToTxnResponse response = new AddPartitionsToTxnResponse(data); for (short version = 0; version <= ApiKeys.ADD_PARTITIONS_TO_TXN.latestVersion(); version++) { - AddPartitionsToTxnResponse response = new AddPartitionsToTxnResponse(data.toStruct(version), version); - assertEquals(expectedErrorCounts, response.errorCounts()); - - assertEquals(throttleTimeMs, response.throttleTimeMs()); - - assertEquals(version >= 1, response.shouldClientThrottle(version)); + AddPartitionsToTxnResponse parsedResponse = AddPartitionsToTxnResponse.parse(response.serializeBody(version), version); + assertEquals(expectedErrorCounts, parsedResponse.errorCounts()); + assertEquals(throttleTimeMs, parsedResponse.throttleTimeMs()); + assertEquals(version >= 1, parsedResponse.shouldClientThrottle(version)); } } } diff --git a/clients/src/test/java/org/apache/kafka/common/requests/CreateAclsRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/CreateAclsRequestTest.java index 06b4b9f4bb2b2..15ffa93c751e5 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/CreateAclsRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/CreateAclsRequestTest.java @@ -29,6 +29,7 @@ import org.apache.kafka.common.resource.ResourceType; import org.junit.Test; +import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; @@ -64,9 +65,9 @@ public void shouldThrowOnIfUnknown() { @Test public void shouldRoundTripV0() { final CreateAclsRequest original = new CreateAclsRequest(V0, data(LITERAL_ACL1, LITERAL_ACL2)); - final Struct struct = original.toStruct(); + final ByteBuffer buffer = original.serializeBody(); - final CreateAclsRequest result = new CreateAclsRequest(struct, V0); + final CreateAclsRequest result = CreateAclsRequest.parse(buffer, V0); assertRequestEquals(original, result); } @@ -74,9 +75,9 @@ public void shouldRoundTripV0() { @Test public void shouldRoundTripV1() { final CreateAclsRequest original = new CreateAclsRequest(V1, data(LITERAL_ACL1, PREFIXED_ACL1)); - final Struct struct = original.toStruct(); + final ByteBuffer buffer = original.serializeBody(); - final CreateAclsRequest result = new CreateAclsRequest(struct, V1); + final CreateAclsRequest result = CreateAclsRequest.parse(buffer, V1); assertRequestEquals(original, result); } diff --git a/clients/src/test/java/org/apache/kafka/common/requests/DeleteAclsRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/DeleteAclsRequestTest.java index 19b65ed088e1c..d85e433652630 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/DeleteAclsRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/DeleteAclsRequestTest.java @@ -29,6 +29,7 @@ import org.apache.kafka.common.resource.ResourceType; import org.junit.Test; +import java.nio.ByteBuffer; import java.util.stream.Collectors; import static java.util.Arrays.asList; @@ -64,9 +65,9 @@ public void shouldThrowOnUnknownElements() { @Test public void shouldRoundTripLiteralV0() { final DeleteAclsRequest original = new DeleteAclsRequest.Builder(requestData(LITERAL_FILTER)).build(V0); - final Struct struct = original.toStruct(); + final ByteBuffer buffer = original.serializeBody(); - final DeleteAclsRequest result = new DeleteAclsRequest(struct, V0); + final DeleteAclsRequest result = DeleteAclsRequest.parse(buffer, V0); assertRequestEquals(original, result); } @@ -82,7 +83,7 @@ public void shouldRoundTripAnyV0AsLiteral() { ANY_FILTER.entryFilter())) ).build(V0); - final DeleteAclsRequest result = new DeleteAclsRequest(original.toStruct(), V0); + final DeleteAclsRequest result = DeleteAclsRequest.parse(original.serializeBody(), V0); assertRequestEquals(expected, result); } @@ -92,9 +93,9 @@ public void shouldRoundTripV1() { final DeleteAclsRequest original = new DeleteAclsRequest.Builder( requestData(LITERAL_FILTER, PREFIXED_FILTER, ANY_FILTER) ).build(V1); - final Struct struct = original.toStruct(); + final ByteBuffer buffer = original.serializeBody(); - final DeleteAclsRequest result = new DeleteAclsRequest(struct, V1); + final DeleteAclsRequest result = DeleteAclsRequest.parse(buffer, V1); assertRequestEquals(original, result); } diff --git a/clients/src/test/java/org/apache/kafka/common/requests/DeleteAclsResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/DeleteAclsResponseTest.java index 0ca1fbca6b4e8..2f390b00a61d3 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/DeleteAclsResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/DeleteAclsResponseTest.java @@ -28,9 +28,12 @@ import org.apache.kafka.common.resource.ResourceType; import org.junit.Test; +import java.nio.ByteBuffer; + import static java.util.Arrays.asList; import static java.util.Collections.singletonList; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; public class DeleteAclsResponseTest { private static final short V0 = 0; @@ -81,41 +84,47 @@ public class DeleteAclsResponseTest { private static final DeleteAclsFilterResult UNKNOWN_RESPONSE = new DeleteAclsFilterResult().setMatchingAcls(asList( UNKNOWN_ACL)); - @Test(expected = UnsupportedVersionException.class) public void shouldThrowOnV0IfNotLiteral() { - new DeleteAclsResponse(new DeleteAclsResponseData() - .setThrottleTimeMs(10) - .setFilterResults(singletonList(PREFIXED_RESPONSE)) - ).serializeWithHeader(V0, 0); + DeleteAclsResponse response = new DeleteAclsResponse( + new DeleteAclsResponseData() + .setThrottleTimeMs(10) + .setFilterResults(singletonList(PREFIXED_RESPONSE)), + V0); + assertThrows(IllegalArgumentException.class, () -> response.serializeBody(V0)); } - @Test(expected = IllegalArgumentException.class) public void shouldThrowOnIfUnknown() { - new DeleteAclsResponse(new DeleteAclsResponseData() - .setThrottleTimeMs(10) - .setFilterResults(singletonList(UNKNOWN_RESPONSE)) - ).toStruct(V1); + DeleteAclsResponse response = new DeleteAclsResponse( + new DeleteAclsResponseData() + .setThrottleTimeMs(10) + .setFilterResults(singletonList(UNKNOWN_RESPONSE)), + V1); + assertThrows(IllegalArgumentException.class, () -> response.serializeBody(V1)); } @Test public void shouldRoundTripV0() { - final DeleteAclsResponse original = new DeleteAclsResponse(new DeleteAclsResponseData() - .setThrottleTimeMs(10) - .setFilterResults(singletonList(LITERAL_RESPONSE))); - final Struct struct = original.toStruct(V0); - - final DeleteAclsResponse result = new DeleteAclsResponse(struct, V0); + final DeleteAclsResponse original = new DeleteAclsResponse( + new DeleteAclsResponseData() + .setThrottleTimeMs(10) + .setFilterResults(singletonList(LITERAL_RESPONSE)), + V0); + final ByteBuffer buffer = original.serializeBody(V0); + + final DeleteAclsResponse result = DeleteAclsResponse.parse(buffer, V0); assertEquals(original.filterResults(), result.filterResults()); } @Test public void shouldRoundTripV1() { - final DeleteAclsResponse original = new DeleteAclsResponse(new DeleteAclsResponseData() - .setThrottleTimeMs(10) - .setFilterResults(asList(LITERAL_RESPONSE, PREFIXED_RESPONSE))); - final Struct struct = original.toStruct(V1); - - final DeleteAclsResponse result = new DeleteAclsResponse(struct, V1); + final DeleteAclsResponse original = new DeleteAclsResponse( + new DeleteAclsResponseData() + .setThrottleTimeMs(10) + .setFilterResults(asList(LITERAL_RESPONSE, PREFIXED_RESPONSE)), + V1); + final ByteBuffer buffer = original.serializeBody(V1); + + final DeleteAclsResponse result = DeleteAclsResponse.parse(buffer, V1); assertEquals(original.filterResults(), result.filterResults()); } diff --git a/clients/src/test/java/org/apache/kafka/common/requests/DescribeAclsRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/DescribeAclsRequestTest.java index e9203aebe2e64..e346ba6c395de 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/DescribeAclsRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/DescribeAclsRequestTest.java @@ -22,7 +22,6 @@ import org.apache.kafka.common.acl.AclOperation; import org.apache.kafka.common.acl.AclPermissionType; import org.apache.kafka.common.errors.UnsupportedVersionException; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.resource.PatternType; import org.apache.kafka.common.resource.ResourcePatternFilter; import org.apache.kafka.common.resource.ResourceType; @@ -60,9 +59,7 @@ public void shouldThrowIfUnknown() { @Test public void shouldRoundTripLiteralV0() { final DescribeAclsRequest original = new DescribeAclsRequest.Builder(LITERAL_FILTER).build(V0); - final Struct struct = original.toStruct(); - - final DescribeAclsRequest result = new DescribeAclsRequest(struct, V0); + final DescribeAclsRequest result = DescribeAclsRequest.parse(original.serializeBody(), V0); assertRequestEquals(original, result); } @@ -77,39 +74,28 @@ public void shouldRoundTripAnyV0AsLiteral() { PatternType.LITERAL), ANY_FILTER.entryFilter())).build(V0); - final Struct struct = original.toStruct(); - final DescribeAclsRequest result = new DescribeAclsRequest(struct, V0); - + final DescribeAclsRequest result = DescribeAclsRequest.parse(original.serializeBody(), V0); assertRequestEquals(expected, result); } @Test public void shouldRoundTripLiteralV1() { final DescribeAclsRequest original = new DescribeAclsRequest.Builder(LITERAL_FILTER).build(V1); - final Struct struct = original.toStruct(); - - final DescribeAclsRequest result = new DescribeAclsRequest(struct, V1); - + final DescribeAclsRequest result = DescribeAclsRequest.parse(original.serializeBody(), V1); assertRequestEquals(original, result); } @Test public void shouldRoundTripPrefixedV1() { final DescribeAclsRequest original = new DescribeAclsRequest.Builder(PREFIXED_FILTER).build(V1); - final Struct struct = original.toStruct(); - - final DescribeAclsRequest result = new DescribeAclsRequest(struct, V1); - + final DescribeAclsRequest result = DescribeAclsRequest.parse(original.serializeBody(), V1); assertRequestEquals(original, result); } @Test public void shouldRoundTripAnyV1() { final DescribeAclsRequest original = new DescribeAclsRequest.Builder(ANY_FILTER).build(V1); - final Struct struct = original.toStruct(); - - final DescribeAclsRequest result = new DescribeAclsRequest(struct, V1); - + final DescribeAclsRequest result = DescribeAclsRequest.parse(original.serializeBody(), V1); assertRequestEquals(original, result); } diff --git a/clients/src/test/java/org/apache/kafka/common/requests/DescribeAclsResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/DescribeAclsResponseTest.java index 84500650d6cf6..36d49f0e34e21 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/DescribeAclsResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/DescribeAclsResponseTest.java @@ -32,6 +32,7 @@ import org.apache.kafka.common.resource.ResourceType; import org.junit.Test; +import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collections; import java.util.HashSet; @@ -82,21 +83,21 @@ public class DescribeAclsResponseTest { @Test(expected = UnsupportedVersionException.class) public void shouldThrowOnV0IfNotLiteral() { - buildResponse(10, Errors.NONE, Collections.singletonList(PREFIXED_ACL1)).toStruct(V0); + buildResponse(10, Errors.NONE, Collections.singletonList(PREFIXED_ACL1)).serializeBody(V0); } @Test(expected = IllegalArgumentException.class) public void shouldThrowIfUnknown() { - buildResponse(10, Errors.NONE, Collections.singletonList(UNKNOWN_ACL)).toStruct(V0); + buildResponse(10, Errors.NONE, Collections.singletonList(UNKNOWN_ACL)).serializeBody(V0); } @Test public void shouldRoundTripV0() { List resources = Arrays.asList(LITERAL_ACL1, LITERAL_ACL2); final DescribeAclsResponse original = buildResponse(10, Errors.NONE, resources); - final Struct struct = original.toStruct(V0); + final ByteBuffer buffer = original.serializeBody(V0); - final DescribeAclsResponse result = new DescribeAclsResponse(struct, V0); + final DescribeAclsResponse result = DescribeAclsResponse.parse(buffer, V0); assertResponseEquals(original, result); final DescribeAclsResponse result2 = buildResponse(10, Errors.NONE, DescribeAclsResponse.aclsResources( @@ -108,9 +109,9 @@ public void shouldRoundTripV0() { public void shouldRoundTripV1() { List resources = Arrays.asList(LITERAL_ACL1, PREFIXED_ACL1); final DescribeAclsResponse original = buildResponse(100, Errors.NONE, resources); - final Struct struct = original.toStruct(V1); + final ByteBuffer buffer = original.serializeBody(V1); - final DescribeAclsResponse result = new DescribeAclsResponse(struct, V1); + final DescribeAclsResponse result = DescribeAclsResponse.parse(buffer, V1); assertResponseEquals(original, result); final DescribeAclsResponse result2 = buildResponse(100, Errors.NONE, DescribeAclsResponse.aclsResources( diff --git a/clients/src/test/java/org/apache/kafka/common/requests/ProduceResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/ProduceResponseTest.java index 963e3219e2235..1bc0c83db7955 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/ProduceResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/ProduceResponseTest.java @@ -45,7 +45,7 @@ public void produceResponseV5Test() { ProduceResponse v5Response = new ProduceResponse(responseData, 10); short version = 5; - ByteBuffer buffer = v5Response.serializeWithHeader(ApiKeys.PRODUCE, version, 0); + ByteBuffer buffer = v5Response.serializeWithHeader(version, 0); buffer.rewind(); ResponseHeader.parse(buffer, ApiKeys.PRODUCE.responseHeaderVersion(version)); // throw away. diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestHeaderTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestHeaderTest.java index cc883c569ed0e..80e742d8ae221 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/RequestHeaderTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestHeaderTest.java @@ -17,7 +17,6 @@ package org.apache.kafka.common.requests; import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.test.TestUtils; import org.junit.Test; @@ -51,14 +50,6 @@ public void testSerdeControlledShutdownV0() { assertEquals(0, serializedBuffer.getShort(2)); assertEquals(correlationId, serializedBuffer.getInt(4)); assertEquals(8, serializedBuffer.limit()); - - Struct serialized = deserialized.data().toStruct(apiVersion); - serializedBuffer = TestUtils.toBuffer(serialized); - - assertEquals(ApiKeys.CONTROLLED_SHUTDOWN.id, serializedBuffer.getShort(0)); - assertEquals(0, serializedBuffer.getShort(2)); - assertEquals(correlationId, serializedBuffer.getInt(4)); - assertEquals(8, serializedBuffer.limit()); } @Test @@ -71,11 +62,6 @@ public void testRequestHeaderV1() { assertEquals(10, buffer.remaining()); RequestHeader deserialized = RequestHeader.parse(buffer); assertEquals(header, deserialized); - - buffer = TestUtils.toBuffer(header.data().toStruct(apiVersion)); - assertEquals(10, buffer.remaining()); - deserialized = RequestHeader.parse(buffer); - assertEquals(header, deserialized); } @Test @@ -88,11 +74,6 @@ public void testRequestHeaderV2() { assertEquals(11, buffer.remaining()); RequestHeader deserialized = RequestHeader.parse(buffer); assertEquals(header, deserialized); - - buffer = TestUtils.toBuffer(header.data().toStruct(apiVersion)); - assertEquals(11, buffer.remaining()); - deserialized = RequestHeader.parse(buffer); - assertEquals(header, deserialized); } private ByteBuffer toBuffer(RequestHeader header) { diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java index d04ccc17ab86f..cfd78935a9edd 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java @@ -35,6 +35,7 @@ import org.apache.kafka.common.errors.UnsupportedVersionException; import org.apache.kafka.common.message.AddOffsetsToTxnRequestData; import org.apache.kafka.common.message.AddOffsetsToTxnResponseData; +import org.apache.kafka.common.message.AlterClientQuotasResponseData; import org.apache.kafka.common.message.AlterConfigsResponseData; import org.apache.kafka.common.message.AlterPartitionReassignmentsRequestData; import org.apache.kafka.common.message.AlterPartitionReassignmentsResponseData; @@ -80,6 +81,7 @@ import org.apache.kafka.common.message.DescribeAclsResponseData; import org.apache.kafka.common.message.DescribeAclsResponseData.AclDescription; import org.apache.kafka.common.message.DescribeAclsResponseData.DescribeAclsResource; +import org.apache.kafka.common.message.DescribeClientQuotasResponseData; import org.apache.kafka.common.message.DescribeConfigsRequestData; import org.apache.kafka.common.message.DescribeConfigsResponseData; import org.apache.kafka.common.message.DescribeConfigsResponseData.DescribeConfigsResourceResult; @@ -144,7 +146,9 @@ import org.apache.kafka.common.network.ListenerName; import org.apache.kafka.common.network.Send; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.ObjectSerializationCache; import org.apache.kafka.common.protocol.types.SchemaException; import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.quota.ClientQuotaAlteration; @@ -167,6 +171,7 @@ import org.apache.kafka.common.security.token.delegation.TokenInformation; import org.apache.kafka.common.utils.SecurityUtils; import org.apache.kafka.common.utils.Utils; +import org.apache.kafka.test.TestUtils; import org.junit.Test; import java.lang.reflect.InvocationTargetException; @@ -190,7 +195,6 @@ import static java.util.Collections.singletonList; import static org.apache.kafka.common.protocol.ApiKeys.FETCH; import static org.apache.kafka.common.requests.FetchMetadata.INVALID_SESSION_ID; -import static org.apache.kafka.test.TestUtils.toBuffer; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -434,7 +438,7 @@ public void testSerialization() throws Exception { checkResponse(createCreateAclsResponse(), ApiKeys.CREATE_ACLS.latestVersion(), true); checkRequest(createDeleteAclsRequest(), true); checkErrorResponse(createDeleteAclsRequest(), new SecurityDisabledException("Security is not enabled."), true); - checkResponse(createDeleteAclsResponse(), ApiKeys.DELETE_ACLS.latestVersion(), true); + checkResponse(createDeleteAclsResponse(ApiKeys.DELETE_ACLS.latestVersion()), ApiKeys.DELETE_ACLS.latestVersion(), true); checkRequest(createAlterConfigsRequest(), false); checkErrorResponse(createAlterConfigsRequest(), unknownServerException, true); checkResponse(createAlterConfigsResponse(), 0, false); @@ -493,12 +497,14 @@ public void testSerialization() throws Exception { @Test public void testResponseHeader() { ResponseHeader header = createResponseHeader((short) 1); - ByteBuffer buffer = toBuffer(header.toStruct()); + ObjectSerializationCache serializationCache = new ObjectSerializationCache(); + ByteBuffer buffer = ByteBuffer.allocate(header.size(serializationCache)); + header.write(buffer, serializationCache); ResponseHeader deserialized = ResponseHeader.parse(buffer, header.headerVersion()); assertEquals(header.correlationId(), deserialized.correlationId()); } - private void checkOlderFetchVersions() throws Exception { + private void checkOlderFetchVersions() { int latestVersion = FETCH.latestVersion(); for (int i = 0; i < latestVersion; ++i) { checkErrorResponse(createFetchRequest(i), unknownServerException, true); @@ -539,73 +545,57 @@ private void verifyDescribeConfigsResponse(DescribeConfigsResponse expected, Des } private void checkDescribeConfigsResponseVersions() throws Exception { - DescribeConfigsResponse response = createDescribeConfigsResponse((short) 0); - DescribeConfigsResponse deserialized0 = (DescribeConfigsResponse) deserialize(response, - response.toStruct((short) 0), (short) 0); - verifyDescribeConfigsResponse(response, deserialized0, 0); + for (int version = ApiKeys.DESCRIBE_CONFIGS.oldestVersion(); version < ApiKeys.DESCRIBE_CONFIGS.latestVersion(); ++version) { + short apiVersion = (short) version; + DescribeConfigsResponse response = createDescribeConfigsResponse(apiVersion); + DescribeConfigsResponse deserialized0 = (DescribeConfigsResponse) AbstractResponse.parseResponse(ApiKeys.DESCRIBE_CONFIGS, + response.serializeBody(apiVersion), apiVersion); + verifyDescribeConfigsResponse(response, deserialized0, apiVersion); - response = createDescribeConfigsResponse((short) 1); - DescribeConfigsResponse deserialized1 = (DescribeConfigsResponse) deserialize(response, - response.toStruct((short) 1), (short) 1); - verifyDescribeConfigsResponse(response, deserialized1, 1); - - response = createDescribeConfigsResponse((short) 3); - DescribeConfigsResponse deserialized3 = (DescribeConfigsResponse) deserialize(response, - response.toStruct((short) 3), (short) 3); - verifyDescribeConfigsResponse(response, deserialized3, 3); + } } private void checkErrorResponse(AbstractRequest req, Throwable e, boolean checkEqualityAndHashCode) { AbstractResponse response = req.getErrorResponse(e); checkResponse(response, req.version(), checkEqualityAndHashCode); if (e instanceof UnknownServerException) { - String responseStr = response.toStruct(req.version()).toString(); - assertFalse(String.format("Unknown message included in response for %s: %s ", req.api, responseStr), + String responseStr = response.toString(req.version()); + assertFalse(String.format("Unknown message included in response for %s: %s ", req.apiKey, responseStr), responseStr.contains(e.getMessage())); } } - private void checkRequest(AbstractRequest req, boolean checkEqualityAndHashCode) { + private void checkRequest(AbstractRequest req, boolean checkEquality) { // Check that we can serialize, deserialize and serialize again - // Check for equality and hashCode of the Struct only if indicated (it is likely to fail if any of the fields + // Check for equality of the ByteBuffer only if indicated (it is likely to fail if any of the fields // in the request is a HashMap with multiple elements since ordering of the elements may vary) try { - Struct struct = req.toStruct(); ByteBuffer serializedBytes = req.serializeBody(); AbstractRequest deserialized = AbstractRequest.parseRequest(req.apiKey, req.version(), serializedBytes).request; - Struct struct2 = deserialized.toStruct(); - if (checkEqualityAndHashCode) { - assertEquals(struct, struct2); - assertEquals(struct.hashCode(), struct2.hashCode()); - } + ByteBuffer serializedBytes2 = deserialized.serializeBody(); + assertEquals(req.data(), deserialized.data()); //FIXME + if (checkEquality) + assertEquals(serializedBytes, serializedBytes2); } catch (Exception e) { throw new RuntimeException("Failed to deserialize request " + req + " with type " + req.getClass(), e); } } - private void checkResponse(AbstractResponse response, int version, boolean checkEqualityAndHashCode) { + private void checkResponse(AbstractResponse response, int version, boolean checkEquality) { // Check that we can serialize, deserialize and serialize again // Check for equality and hashCode of the Struct only if indicated (it is likely to fail if any of the fields // in the response is a HashMap with multiple elements since ordering of the elements may vary) try { - Struct struct = response.toStruct((short) version); - AbstractResponse deserialized = (AbstractResponse) deserialize(response, struct, (short) version); - Struct struct2 = deserialized.toStruct((short) version); - if (checkEqualityAndHashCode) { - assertEquals(struct, struct2); - assertEquals(struct.hashCode(), struct2.hashCode()); - } + ByteBuffer serializedBytes = response.serializeBody((short) version); + AbstractResponse deserialized = AbstractResponse.parseResponse(response.apiKey(), serializedBytes, (short) version); + ByteBuffer serializedBytes2 = deserialized.serializeBody((short) version); + if (checkEquality) + assertEquals(serializedBytes, serializedBytes2); } catch (Exception e) { throw new RuntimeException("Failed to deserialize response " + response + " with type " + response.getClass(), e); } } - private AbstractRequestResponse deserialize(AbstractRequestResponse req, Struct struct, short version) throws NoSuchMethodException, IllegalAccessException, InvocationTargetException { - ByteBuffer buffer = toBuffer(struct); - Method deserializer = req.getClass().getDeclaredMethod("parse", ByteBuffer.class, Short.TYPE); - return (AbstractRequestResponse) deserializer.invoke(null, buffer, version); - } - @Test(expected = UnsupportedVersionException.class) public void cannotUseFindCoordinatorV0ToFindTransactionCoordinator() { FindCoordinatorRequest.Builder builder = new FindCoordinatorRequest.Builder( @@ -700,7 +690,7 @@ public void testFetchResponseV4() { 6, FetchResponse.INVALID_LOG_START_OFFSET, Optional.empty(), emptyList(), records)); FetchResponse response = new FetchResponse<>(Errors.NONE, responseData, 10, INVALID_SESSION_ID); - FetchResponse deserialized = FetchResponse.parse(toBuffer(response.toStruct((short) 4)), (short) 4); + FetchResponse deserialized = FetchResponse.parse(response.serializeBody((short) 4), (short) 4); assertEquals(responseData, deserialized.responseData()); } @@ -735,17 +725,16 @@ private void verifyFetchResponseFullWrite(short apiVersion, FetchResponse ApiVersionsResponse.parse(ByteBuffer.allocate(0), version)); } @Test - public void testApiVersionResponseStructParsing() { - Struct struct = ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE.toStruct(ApiKeys.API_VERSIONS.latestVersion()); - ApiVersionsResponse response = ApiVersionsResponse. - fromStruct(struct, ApiKeys.API_VERSIONS.latestVersion()); + public void testApiVersionResponseParsing() { + ByteBuffer buffer = ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE.serializeBody(ApiKeys.API_VERSIONS.latestVersion()); + ApiVersionsResponse response = ApiVersionsResponse.parse(buffer, ApiKeys.API_VERSIONS.latestVersion()); assertEquals(Errors.NONE.code(), response.data.errorCode()); } @@ -937,7 +924,7 @@ public void testInitProducerIdRequestVersions() { setTransactionalId("abracadabra"). setProducerId(123)); final UnsupportedVersionException exception = assertThrows( - UnsupportedVersionException.class, () -> bld.build((short) 2).toStruct()); + UnsupportedVersionException.class, () -> bld.build((short) 2).serializeBody()); assertTrue(exception.getMessage().contains("Attempted to write a non-default producerId at version 2")); bld.build((short) 3); } @@ -1261,7 +1248,7 @@ private MetadataResponse createMetadataResponse() { new TopicPartition("topic3", 0), Optional.empty(), Optional.empty(), replicas, isr, offlineReplicas)))); - return MetadataResponse.prepareResponse(asList(node), null, MetadataResponse.NO_CONTROLLER_ID, allTopicMetadata); + return TestUtils.metadataResponse(asList(node), null, MetadataResponse.NO_CONTROLLER_ID, allTopicMetadata); } private OffsetCommitRequest createOffsetCommitRequest(int version) { @@ -1877,7 +1864,7 @@ private DeleteAclsRequest createDeleteAclsRequest() { return new DeleteAclsRequest.Builder(data).build(); } - private DeleteAclsResponse createDeleteAclsResponse() { + private DeleteAclsResponse createDeleteAclsResponse(int version) { List filterResults = new ArrayList<>(); filterResults.add(new DeleteAclsResponseData.DeleteAclsFilterResult().setMatchingAcls(asList( new DeleteAclsResponseData.DeleteAclsMatchingAcl() @@ -1901,7 +1888,7 @@ private DeleteAclsResponse createDeleteAclsResponse() { .setErrorMessage("No security")); return new DeleteAclsResponse(new DeleteAclsResponseData() .setThrottleTimeMs(0) - .setFilterResults(filterResults)); + .setFilterResults(filterResults), (short) version); } private DescribeConfigsRequest createDescribeConfigsRequest(int version) { @@ -2347,8 +2334,15 @@ private DescribeClientQuotasRequest createDescribeClientQuotasRequest() { } private DescribeClientQuotasResponse createDescribeClientQuotasResponse() { - ClientQuotaEntity entity = new ClientQuotaEntity(Collections.singletonMap(ClientQuotaEntity.USER, "user")); - return new DescribeClientQuotasResponse(Collections.singletonMap(entity, Collections.singletonMap("request_percentage", 1.0)), 0); + DescribeClientQuotasResponseData data = new DescribeClientQuotasResponseData().setEntries(asList( + new DescribeClientQuotasResponseData.EntryData() + .setEntity(asList(new DescribeClientQuotasResponseData.EntityData() + .setEntityType(ClientQuotaEntity.USER) + .setEntityName("user"))) + .setValues(asList(new DescribeClientQuotasResponseData.ValueData() + .setKey("request_percentage") + .setValue(1.0))))); + return new DescribeClientQuotasResponse(data); } private AlterClientQuotasRequest createAlterClientQuotasRequest() { @@ -2359,7 +2353,11 @@ private AlterClientQuotasRequest createAlterClientQuotasRequest() { } private AlterClientQuotasResponse createAlterClientQuotasResponse() { - ClientQuotaEntity entity = new ClientQuotaEntity(Collections.singletonMap(ClientQuotaEntity.USER, "user")); - return new AlterClientQuotasResponse(Collections.singletonMap(entity, ApiError.NONE), 0); + AlterClientQuotasResponseData data = new AlterClientQuotasResponseData() + .setEntries(asList(new AlterClientQuotasResponseData.EntryData() + .setEntity(asList(new AlterClientQuotasResponseData.EntityData() + .setEntityType(ClientQuotaEntity.USER) + .setEntityName("user"))))); + return new AlterClientQuotasResponse(data); } } diff --git a/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaRequestTest.java index 24a9c4509f4eb..31a32ef82f052 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaRequestTest.java @@ -36,6 +36,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; import static org.apache.kafka.common.protocol.ApiKeys.STOP_REPLICA; import static org.junit.Assert.assertEquals; @@ -80,8 +81,17 @@ public void testGetErrorResponse() { @Test public void testStopReplicaRequestNormalization() { - Set tps = TestUtils.generateRandomTopicPartitions(10, 10); - StopReplicaRequest.Builder builder = new StopReplicaRequest.Builder((short) 5, 0, 0, 0, false, tps); + List topicStates = TestUtils.generateRandomTopicPartitions(10, 10).stream() + .collect(Collectors.groupingBy(tp -> tp.topic())) + .entrySet() + .stream() + .map(entry -> new StopReplicaTopicState() + .setTopicName(entry.getKey()) + .setPartitionStates(entry.getValue().stream() + .map(tp -> new StopReplicaPartitionState().setPartitionIndex(tp.partition())) + .collect(Collectors.toList()))) + .collect(Collectors.toList()); + StopReplicaRequest.Builder builder = new StopReplicaRequest.Builder((short) 5, 0, 0, 0, false, topicStates); assertTrue(builder.build((short) 1).sizeInBytes() < builder.build((short) 0).sizeInBytes()); } @@ -136,12 +146,12 @@ public void testTopicStatesNormalization() { List topicStates = topicStates(true); for (short version = STOP_REPLICA.oldestVersion(); version < STOP_REPLICA.latestVersion(); version++) { - // Create a request for version to get its struct + // Create a request for version to get its serialized form StopReplicaRequest baseRequest = new StopReplicaRequest.Builder(version, 0, 1, 0, true, topicStates).build(version); - // Construct the request from the struct - StopReplicaRequest request = new StopReplicaRequest(baseRequest.toStruct(), version); + // Construct the request from the buffer + StopReplicaRequest request = StopReplicaRequest.parse(baseRequest.serializeBody(), version); Map partitionStates = StopReplicaRequestTest.partitionStates(request.topicStates()); @@ -171,12 +181,12 @@ public void testPartitionStatesNormalization() { List topicStates = topicStates(true); for (short version = STOP_REPLICA.oldestVersion(); version < STOP_REPLICA.latestVersion(); version++) { - // Create a request for version to get its struct + // Create a request for version to get its serialized form StopReplicaRequest baseRequest = new StopReplicaRequest.Builder(version, 0, 1, 0, true, topicStates).build(version); - // Construct the request from the struct - StopReplicaRequest request = new StopReplicaRequest(baseRequest.toStruct(), version); + // Construct the request from the buffer + StopReplicaRequest request = StopReplicaRequest.parse(baseRequest.serializeBody(), version); Map partitionStates = request.partitionStates(); assertEquals(6, partitionStates.size()); diff --git a/clients/src/test/java/org/apache/kafka/test/TestUtils.java b/clients/src/test/java/org/apache/kafka/test/TestUtils.java index 02cbd992036eb..a51bcc89e4f39 100644 --- a/clients/src/test/java/org/apache/kafka/test/TestUtils.java +++ b/clients/src/test/java/org/apache/kafka/test/TestUtils.java @@ -24,6 +24,7 @@ import org.apache.kafka.common.PartitionInfo; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.internals.Topic; +import org.apache.kafka.common.message.MetadataResponseData; import org.apache.kafka.common.network.NetworkReceive; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; @@ -31,6 +32,7 @@ import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.ObjectSerializationCache; import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.record.RecordBatch; import org.apache.kafka.common.requests.MetadataResponse; import org.apache.kafka.common.requests.RequestHeader; import org.apache.kafka.common.utils.Exit; @@ -127,11 +129,31 @@ public static MetadataResponse metadataResponse(Collection brokers, public static MetadataResponse metadataResponse(int throttleTimeMs, Collection brokers, String clusterId, int controllerId, - List topicMetadataList, + List topicMetadatas, int clusterAuthorizedOperations) { - return MetadataResponse.prepareResponse(true, throttleTimeMs, brokers, clusterId, - controllerId, topicMetadataList, clusterAuthorizedOperations); - } + List topics = new ArrayList<>(); + topicMetadatas.forEach(topicMetadata -> { + MetadataResponseData.MetadataResponseTopic metadataResponseTopic = new MetadataResponseData.MetadataResponseTopic(); + metadataResponseTopic + .setErrorCode(topicMetadata.error().code()) + .setName(topicMetadata.topic()) + .setIsInternal(topicMetadata.isInternal()) + .setTopicAuthorizedOperations(topicMetadata.authorizedOperations()); + + for (MetadataResponse.PartitionMetadata partitionMetadata : topicMetadata.partitionMetadata()) { + metadataResponseTopic.partitions().add(new MetadataResponseData.MetadataResponsePartition() + .setErrorCode(partitionMetadata.error.code()) + .setPartitionIndex(partitionMetadata.partition()) + .setLeaderId(partitionMetadata.leaderId.orElse(MetadataResponse.NO_LEADER_ID)) + .setLeaderEpoch(partitionMetadata.leaderEpoch.orElse(RecordBatch.NO_PARTITION_LEADER_EPOCH)) + .setReplicaNodes(partitionMetadata.replicaIds) + .setIsrNodes(partitionMetadata.inSyncReplicaIds) + .setOfflineReplicas(partitionMetadata.offlineReplicaIds)); + } + topics.add(metadataResponseTopic); + }); + return MetadataResponse.prepareResponse(true, throttleTimeMs, brokers, clusterId, controllerId, + topics, clusterAuthorizedOperations); } public static MetadataResponse metadataUpdateWith(final int numNodes, final Map topicPartitionCounts) { @@ -557,13 +579,6 @@ public static Set toSet(Collection collection) { return new HashSet<>(collection); } - public static ByteBuffer toBuffer(Struct struct) { - ByteBuffer buffer = ByteBuffer.allocate(struct.sizeOf()); - struct.writeTo(buffer); - buffer.rewind(); - return buffer; - } - public static Set generateRandomTopicPartitions(int numTopic, int numPartitionPerTopic) { Set tps = new HashSet<>(); for (int i = 0; i < numTopic; i++) { diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/util/TopicAdminTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/util/TopicAdminTest.java index dd3062e4159bc..e4b456df61663 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/util/TopicAdminTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/util/TopicAdminTest.java @@ -38,6 +38,7 @@ import org.apache.kafka.common.message.CreateTopicsResponseData.CreatableTopicResult; import org.apache.kafka.common.message.DescribeConfigsResponseData; import org.apache.kafka.common.message.MetadataResponseData; +import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.requests.ApiError; import org.apache.kafka.common.requests.CreateTopicsResponse; @@ -553,7 +554,7 @@ private MetadataResponse describeTopicResponse(ApiError error, NewTopic... topic .setName(topic.name()) .setErrorCode(error.error().code())); } - return new MetadataResponse(response); + return new MetadataResponse(response, ApiKeys.METADATA.latestVersion()); } private DescribeConfigsResponse describeConfigsResponseWithUnsupportedVersion(NewTopic... topics) { diff --git a/core/src/main/scala/kafka/network/RequestChannel.scala b/core/src/main/scala/kafka/network/RequestChannel.scala index 0a29e2a95f540..b93f2b348f96a 100644 --- a/core/src/main/scala/kafka/network/RequestChannel.scala +++ b/core/src/main/scala/kafka/network/RequestChannel.scala @@ -20,7 +20,6 @@ package kafka.network import java.net.InetAddress import java.nio.ByteBuffer import java.util.concurrent._ - import com.typesafe.scalalogging.Logger import com.yammer.metrics.core.Meter import kafka.log.LogConfig @@ -33,7 +32,7 @@ import org.apache.kafka.common.memory.MemoryPool import org.apache.kafka.common.message.IncrementalAlterConfigsRequestData import org.apache.kafka.common.message.IncrementalAlterConfigsRequestData._ import org.apache.kafka.common.network.Send -import org.apache.kafka.common.protocol.{ApiKeys, Errors} +import org.apache.kafka.common.protocol.{ApiKeys, Errors, ObjectSerializationCache} import org.apache.kafka.common.requests._ import org.apache.kafka.common.security.auth.KafkaPrincipal import org.apache.kafka.common.utils.{Sanitizer, Time} @@ -96,8 +95,11 @@ object RequestChannel extends Logging { private val bodyAndSize: RequestAndSize = context.parseRequest(buffer) def header: RequestHeader = context.header + def sizeOfBodyInBytes: Int = bodyAndSize.size + def sizeInBytes: Int = header.size(new ObjectSerializationCache) + sizeOfBodyInBytes + //most request types are parsed entirely into objects at this point. for those we can release the underlying buffer. //some (like produce, or any time the schema contains fields of types BYTES or NULLABLE_BYTES) retain a reference //to the buffer. for those requests we cannot release the buffer early, but only when request processing is done. diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala index 9934bfc9bb7e2..e51dbed63422e 100644 --- a/core/src/main/scala/kafka/server/KafkaApis.scala +++ b/core/src/main/scala/kafka/server/KafkaApis.scala @@ -24,7 +24,6 @@ import java.util import java.util.{Collections, Optional} import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.atomic.AtomicInteger - import kafka.admin.{AdminUtils, RackAwareMode} import kafka.api.ElectLeadersRequestOps import kafka.api.{ApiVersion, KAFKA_0_11_0_IV0, KAFKA_2_3_IV0} @@ -51,7 +50,7 @@ import org.apache.kafka.common.internals.Topic.{GROUP_METADATA_TOPIC_NAME, TRANS import org.apache.kafka.common.message.AlterConfigsResponseData.AlterConfigsResourceResponse import org.apache.kafka.common.message.CreateTopicsRequestData.CreatableTopic import org.apache.kafka.common.message.CreatePartitionsResponseData.CreatePartitionsTopicResult -import org.apache.kafka.common.message.{AddOffsetsToTxnResponseData, AlterConfigsResponseData, AlterPartitionReassignmentsResponseData, AlterReplicaLogDirsResponseData, CreateAclsResponseData, CreatePartitionsResponseData, CreateTopicsResponseData, DeleteAclsResponseData, DeleteGroupsResponseData, DeleteRecordsResponseData, DeleteTopicsResponseData, DescribeAclsResponseData, DescribeConfigsResponseData, DescribeGroupsResponseData, DescribeLogDirsResponseData, EndTxnResponseData, ExpireDelegationTokenResponseData, FindCoordinatorResponseData, HeartbeatResponseData, InitProducerIdResponseData, JoinGroupResponseData, LeaveGroupResponseData, ListGroupsResponseData, ListPartitionReassignmentsResponseData, OffsetCommitRequestData, OffsetCommitResponseData, OffsetDeleteResponseData, RenewDelegationTokenResponseData, SaslAuthenticateResponseData, SaslHandshakeResponseData, StopReplicaResponseData, SyncGroupResponseData, UpdateMetadataResponseData} +import org.apache.kafka.common.message.{AddOffsetsToTxnResponseData, AlterClientQuotasResponseData, AlterConfigsResponseData, AlterPartitionReassignmentsResponseData, AlterReplicaLogDirsResponseData, CreateAclsResponseData, CreatePartitionsResponseData, CreateTopicsResponseData, DeleteAclsResponseData, DeleteGroupsResponseData, DeleteRecordsResponseData, DeleteTopicsResponseData, DescribeAclsResponseData, DescribeClientQuotasResponseData, DescribeConfigsResponseData, DescribeGroupsResponseData, DescribeLogDirsResponseData, EndTxnResponseData, ExpireDelegationTokenResponseData, FindCoordinatorResponseData, HeartbeatResponseData, InitProducerIdResponseData, JoinGroupResponseData, LeaveGroupResponseData, ListGroupsResponseData, ListPartitionReassignmentsResponseData, MetadataResponseData, OffsetCommitRequestData, OffsetCommitResponseData, OffsetDeleteResponseData, RenewDelegationTokenResponseData, SaslAuthenticateResponseData, SaslHandshakeResponseData, StopReplicaResponseData, SyncGroupResponseData, UpdateMetadataResponseData} import org.apache.kafka.common.message.CreateTopicsResponseData.{CreatableTopicResult, CreatableTopicResultCollection} import org.apache.kafka.common.message.DeleteGroupsResponseData.{DeletableGroupResult, DeletableGroupResultCollection} import org.apache.kafka.common.message.AlterPartitionReassignmentsResponseData.{ReassignablePartitionResponse, ReassignableTopicResponse} @@ -77,7 +76,6 @@ import org.apache.kafka.common.security.auth.{KafkaPrincipal, SecurityProtocol} import org.apache.kafka.common.security.token.delegation.{DelegationToken, TokenInformation} import org.apache.kafka.common.utils.{ProducerIdAndEpoch, Time, Utils} import org.apache.kafka.common.{Node, TopicPartition} -import org.apache.kafka.common.message.MetadataResponseData import org.apache.kafka.common.message.MetadataResponseData.{MetadataResponsePartition, MetadataResponseTopic} import org.apache.kafka.server.authorizer._ @@ -487,7 +485,7 @@ class KafkaApis(val requestChannel: RequestChannel, */ def handleProduceRequest(request: RequestChannel.Request): Unit = { val produceRequest = request.body[ProduceRequest] - val numBytesAppended = request.header.toStruct.sizeOf + request.sizeOfBodyInBytes + val requestSize = request.sizeOfBodyInBytes if (produceRequest.hasTransactionalRecords) { val isAuthorizedTransactional = produceRequest.transactionalId != null && @@ -545,7 +543,7 @@ class KafkaApis(val requestChannel: RequestChannel, // have been violated. If both quotas have been violated, use the max throttle time between the two quotas. Note // that the request quota is not enforced if acks == 0. val timeMs = time.milliseconds() - val bandwidthThrottleTimeMs = quotas.produce.maybeRecordAndGetThrottleTimeMs(request, numBytesAppended, timeMs) + val bandwidthThrottleTimeMs = quotas.produce.maybeRecordAndGetThrottleTimeMs(request, requestSize, timeMs) val requestThrottleTimeMs = if (produceRequest.acks == 0) 0 else quotas.request.maybeRecordAndGetThrottleTimeMs(request, timeMs) @@ -1199,10 +1197,10 @@ class KafkaApis(val requestChannel: RequestChannel, MetadataResponse.prepareResponse( requestVersion, requestThrottleMs, - completeTopicMetadata.asJava, brokers.flatMap(_.getNode(request.context.listenerName)).asJava, clusterId, metadataCache.getControllerId.getOrElse(MetadataResponse.NO_CONTROLLER_ID), + completeTopicMetadata.asJava, clusterAuthorizedOperations )) } @@ -2325,7 +2323,8 @@ class KafkaApis(val requestChannel: RequestChannel, new DescribeAclsResponse(new DescribeAclsResponseData() .setErrorCode(Errors.SECURITY_DISABLED.code) .setErrorMessage("No Authorizer is configured on the broker") - .setThrottleTimeMs(requestThrottleMs))) + .setThrottleTimeMs(requestThrottleMs), + describeAclsRequest.version)) case Some(auth) => val filter = describeAclsRequest.filter val returnedAcls = new util.HashSet[AclBinding]() @@ -2333,7 +2332,8 @@ class KafkaApis(val requestChannel: RequestChannel, sendResponseMaybeThrottle(request, requestThrottleMs => new DescribeAclsResponse(new DescribeAclsResponseData() .setThrottleTimeMs(requestThrottleMs) - .setResources(DescribeAclsResponse.aclsResources(returnedAcls)))) + .setResources(DescribeAclsResponse.aclsResources(returnedAcls)), + describeAclsRequest.version)) } } @@ -2404,9 +2404,11 @@ class KafkaApis(val requestChannel: RequestChannel, def sendResponseCallback(): Unit = { val filterResults = deleteResults.map(_.get).map(DeleteAclsResponse.filterResult).asJava sendResponseMaybeThrottle(request, requestThrottleMs => - new DeleteAclsResponse(new DeleteAclsResponseData() - .setThrottleTimeMs(requestThrottleMs) - .setFilterResults(filterResults))) + new DeleteAclsResponse( + new DeleteAclsResponseData() + .setThrottleTimeMs(requestThrottleMs) + .setFilterResults(filterResults), + deleteAclsRequest.version)) } alterAclsPurgatory.tryCompleteElseWatch(config.connectionsMaxIdleMs, deleteResults, sendResponseCallback) } @@ -2932,11 +2934,30 @@ class KafkaApis(val requestChannel: RequestChannel, val describeClientQuotasRequest = request.body[DescribeClientQuotasRequest] if (authorize(request.context, DESCRIBE_CONFIGS, CLUSTER, CLUSTER_NAME)) { - val result = adminManager.describeClientQuotas(describeClientQuotasRequest.filter).map { case (quotaEntity, quotaConfigs) => - quotaEntity -> quotaConfigs.map { case (key, value) => key -> Double.box(value) }.asJava - }.asJava + val result = adminManager.describeClientQuotas(describeClientQuotasRequest.filter) + + val entriesData = result.iterator.map { case (quotaEntity, quotaValues) => + val entityData = quotaEntity.entries.asScala.iterator.map { case (entityType, entityName) => + new DescribeClientQuotasResponseData.EntityData() + .setEntityType(entityType) + .setEntityName(entityName) + }.toBuffer + + val valueData = quotaValues.iterator.map { case (key, value) => + new DescribeClientQuotasResponseData.ValueData() + .setKey(key) + .setValue(value) + }.toBuffer + + new DescribeClientQuotasResponseData.EntryData() + .setEntity(entityData.asJava) + .setValues(valueData.asJava) + }.toBuffer + sendResponseMaybeThrottle(request, requestThrottleMs => - new DescribeClientQuotasResponse(result, requestThrottleMs)) + new DescribeClientQuotasResponse(new DescribeClientQuotasResponseData() + .setThrottleTimeMs(requestThrottleMs) + .setEntries(entriesData.asJava))) } else { sendResponseMaybeThrottle(request, requestThrottleMs => describeClientQuotasRequest.getErrorResponse(requestThrottleMs, Errors.CLUSTER_AUTHORIZATION_FAILED.exception)) @@ -2947,10 +2968,26 @@ class KafkaApis(val requestChannel: RequestChannel, val alterClientQuotasRequest = request.body[AlterClientQuotasRequest] if (authorize(request.context, ALTER_CONFIGS, CLUSTER, CLUSTER_NAME)) { - val result = adminManager.alterClientQuotas(alterClientQuotasRequest.entries().asScala.toSeq, - alterClientQuotasRequest.validateOnly()).asJava + val result = adminManager.alterClientQuotas(alterClientQuotasRequest.entries.asScala, + alterClientQuotasRequest.validateOnly) + + val entriesData = result.iterator.map { case (quotaEntity, apiError) => + val entityData = quotaEntity.entries.asScala.iterator.map { case (key, value) => + new AlterClientQuotasResponseData.EntityData() + .setEntityType(key) + .setEntityName(value) + }.toBuffer + + new AlterClientQuotasResponseData.EntryData() + .setErrorCode(apiError.error.code) + .setErrorMessage(apiError.message) + .setEntity(entityData.asJava) + }.toBuffer + sendResponseMaybeThrottle(request, requestThrottleMs => - new AlterClientQuotasResponse(result, requestThrottleMs)) + new AlterClientQuotasResponse(new AlterClientQuotasResponseData() + .setThrottleTimeMs(requestThrottleMs) + .setEntries(entriesData.asJava))) } else { sendResponseMaybeThrottle(request, requestThrottleMs => alterClientQuotasRequest.getErrorResponse(requestThrottleMs, Errors.CLUSTER_AUTHORIZATION_FAILED.exception)) diff --git a/core/src/test/scala/unit/kafka/network/RequestChannelTest.scala b/core/src/test/scala/unit/kafka/network/RequestChannelTest.scala index ce19491bbe5af..270de377925d8 100644 --- a/core/src/test/scala/unit/kafka/network/RequestChannelTest.scala +++ b/core/src/test/scala/unit/kafka/network/RequestChannelTest.scala @@ -163,7 +163,7 @@ class RequestChannelTest { } def request(req: AbstractRequest): RequestChannel.Request = { - val buffer = req.serialize(new RequestHeader(req.api, req.version, "client-id", 1)) + val buffer = req.serializeWithHeader(new RequestHeader(req.apiKey, req.version, "client-id", 1)) val requestContext = newRequestContext(buffer) new network.RequestChannel.Request(processor = 1, requestContext, @@ -192,4 +192,4 @@ class RequestChannelTest { private def toMap(config: IncrementalAlterConfigsRequestData.AlterableConfigCollection): Map[String, String] = { config.asScala.map(e => e.name -> e.value).toMap } -} \ No newline at end of file +} diff --git a/core/src/test/scala/unit/kafka/network/SocketServerTest.scala b/core/src/test/scala/unit/kafka/network/SocketServerTest.scala index ed01e474e4653..44ad423d0260b 100644 --- a/core/src/test/scala/unit/kafka/network/SocketServerTest.scala +++ b/core/src/test/scala/unit/kafka/network/SocketServerTest.scala @@ -114,7 +114,7 @@ class SocketServerTest { } def sendApiRequest(socket: Socket, request: AbstractRequest, header: RequestHeader) = { - val byteBuffer = request.serialize(header) + val byteBuffer = request.serializeWithHeader(header) byteBuffer.rewind() val serializedBytes = new Array[Byte](byteBuffer.remaining) byteBuffer.get(serializedBytes) @@ -223,7 +223,7 @@ class SocketServerTest { private def apiVersionRequestBytes(clientId: String, version: Short): Array[Byte] = { val request = new ApiVersionsRequest.Builder().build(version) val header = new RequestHeader(ApiKeys.API_VERSIONS, request.version(), clientId, -1) - val buffer = request.serialize(header) + val buffer = request.serializeWithHeader(header) buffer.rewind() val bytes = new Array[Byte](buffer.remaining()) buffer.get(bytes) diff --git a/core/src/test/scala/unit/kafka/server/AbstractCreateTopicsRequestTest.scala b/core/src/test/scala/unit/kafka/server/AbstractCreateTopicsRequestTest.scala index 1f92a90d224ad..10f73a88ae40d 100644 --- a/core/src/test/scala/unit/kafka/server/AbstractCreateTopicsRequestTest.scala +++ b/core/src/test/scala/unit/kafka/server/AbstractCreateTopicsRequestTest.scala @@ -24,7 +24,6 @@ import kafka.network.SocketServer import kafka.utils.TestUtils import org.apache.kafka.common.message.CreateTopicsRequestData import org.apache.kafka.common.message.CreateTopicsRequestData._ -import org.apache.kafka.common.protocol.types.Struct import org.apache.kafka.common.protocol.Errors import org.apache.kafka.common.requests._ import org.junit.Assert.{assertEquals, assertFalse, assertNotNull, assertTrue} @@ -151,15 +150,6 @@ abstract class AbstractCreateTopicsRequestTest extends BaseRequestTest { protected def error(error: Errors, errorMessage: Option[String] = None): ApiError = new ApiError(error, errorMessage.orNull) - protected def addPartitionsAndReplicationFactorToFirstTopic(request: CreateTopicsRequest) = { - val struct = request.toStruct - val topics = struct.getArray("create_topic_requests") - val firstTopic = topics(0).asInstanceOf[Struct] - firstTopic.set("num_partitions", 1) - firstTopic.set("replication_factor", 1.toShort) - new CreateTopicsRequest(new CreateTopicsRequestData(struct, request.version), request.version) - } - protected def validateErrorCreateTopicsRequests(request: CreateTopicsRequest, expectedResponse: Map[String, ApiError], checkErrorMessage: Boolean = true): Unit = { diff --git a/core/src/test/scala/unit/kafka/server/EdgeCaseRequestTest.scala b/core/src/test/scala/unit/kafka/server/EdgeCaseRequestTest.scala index 73527a67b210e..9f9af4a05429e 100755 --- a/core/src/test/scala/unit/kafka/server/EdgeCaseRequestTest.scala +++ b/core/src/test/scala/unit/kafka/server/EdgeCaseRequestTest.scala @@ -124,7 +124,7 @@ class EdgeCaseRequestTest extends KafkaServerTestHarness { val byteBuffer = ByteBuffer.allocate(headerBytes.length + request.toStruct.sizeOf) byteBuffer.put(headerBytes) request.toStruct.writeTo(byteBuffer) - (byteBuffer.array(), request.api.responseHeaderVersion(version)) + (byteBuffer.array(), request.apiKey.responseHeaderVersion(version)) } val response = requestAndReceive(serializedBytes) diff --git a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala index 4cd33756f94ae..fe781182d7ea7 100644 --- a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala +++ b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala @@ -2001,12 +2001,7 @@ class KafkaApisTest { private def buildRequest[T <: AbstractRequest](request: AbstractRequest, listenerName: ListenerName = ListenerName.forSecurityProtocol(SecurityProtocol.PLAINTEXT)): RequestChannel.Request = { -<<<<<<< HEAD - val request = builder.build() - val buffer = request.serializeWithHeader(new RequestHeader(builder.apiKey, request.version, "", 0)) -======= - val buffer = request.serialize(new RequestHeader(request.api, request.version, clientId, 0)) ->>>>>>> apache-github/trunk + val buffer = request.serializeWithHeader(new RequestHeader(request.apiKey, request.version, clientId, 0)) // read the header from the buffer first so that the body can be read next from the Request constructor val header = RequestHeader.parse(buffer) @@ -2026,12 +2021,7 @@ class KafkaApisTest { channel.close() channel.buffer.getInt() // read the size ResponseHeader.parse(channel.buffer, api.responseHeaderVersion(request.version)) -<<<<<<< HEAD AbstractResponse.parseResponse(api, channel.buffer, request.version) -======= - val struct = api.responseSchema(request.version).read(channel.buffer) - AbstractResponse.parseResponse(api, struct, request.version) ->>>>>>> apache-github/trunk } private def expectNoThrottling(): Capture[RequestChannel.Response] = { diff --git a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala index a94501df0b852..e10ef81179790 100644 --- a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala +++ b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala @@ -561,10 +561,6 @@ class RequestQuotaTest extends BaseRequestTest { case class Client(clientId: String, apiKey: ApiKeys) { var correlationId: Int = 0 -<<<<<<< HEAD - val builder = requestBuilder(apiKey) -======= ->>>>>>> apache-github/trunk def runUntil(until: AbstractResponse => Boolean): Boolean = { val startMs = System.currentTimeMillis var done = false @@ -572,14 +568,9 @@ class RequestQuotaTest extends BaseRequestTest { try { while (!done && System.currentTimeMillis < startMs + 10000) { correlationId += 1 -<<<<<<< HEAD - val response = requestResponse(socket, clientId, correlationId, builder) - done = until(response) -======= val request = requestBuilder(apiKey).build() val response = sendAndReceive[AbstractResponse](request, socket, clientId, Some(correlationId)) done = until.apply(response) ->>>>>>> apache-github/trunk } } finally { socket.close() @@ -622,26 +613,16 @@ class RequestQuotaTest extends BaseRequestTest { // Request until throttled using client-id with default small quota val clientId = apiKey.toString val client = Client(clientId, apiKey) -<<<<<<< HEAD -======= - ->>>>>>> apache-github/trunk val throttled = client.runUntil(_.throttleTimeMs > 0) assertTrue(s"Response not throttled: $client", throttled) assertTrue(s"Throttle time metrics not updated: $client" , throttleTimeMetricValue(clientId) > 0) } -<<<<<<< HEAD - private def checkSmallQuotaProducerRequestThrottleTime(apiKey: ApiKeys): Unit = { - // Request until throttled using client-id with default small producer quota - val smallQuotaProducerClient = Client(smallQuotaProducerClientId, apiKey) -======= private def checkSmallQuotaProducerRequestThrottleTime(): Unit = { // Request until throttled using client-id with default small producer quota val smallQuotaProducerClient = Client(smallQuotaProducerClientId, ApiKeys.PRODUCE) ->>>>>>> apache-github/trunk val throttled = smallQuotaProducerClient.runUntil(_.throttleTimeMs > 0) assertTrue(s"Response not throttled: $smallQuotaProducerClient", throttled) @@ -651,16 +632,10 @@ class RequestQuotaTest extends BaseRequestTest { throttleTimeMetricValueForQuotaType(smallQuotaProducerClientId, QuotaType.Request).isNaN) } -<<<<<<< HEAD - private def checkSmallQuotaConsumerRequestThrottleTime(apiKey: ApiKeys): Unit = { - // Request until throttled using client-id with default small consumer quota - val smallQuotaConsumerClient = Client(smallQuotaConsumerClientId, apiKey) -======= private def checkSmallQuotaConsumerRequestThrottleTime(): Unit = { // Request until throttled using client-id with default small consumer quota val smallQuotaConsumerClient = Client(smallQuotaConsumerClientId, ApiKeys.FETCH) ->>>>>>> apache-github/trunk val throttled = smallQuotaConsumerClient.runUntil(_.throttleTimeMs > 0) assertTrue(s"Response not throttled: $smallQuotaConsumerClientId", throttled) @@ -671,13 +646,10 @@ class RequestQuotaTest extends BaseRequestTest { } private def checkUnthrottledClient(apiKey: ApiKeys): Unit = { + // Test that request from client with large quota is not throttled val unthrottledClient = Client(unthrottledClientId, apiKey) -<<<<<<< HEAD - unthrottledClient.runUntil(_.throttleTimeMs <= 0) -======= unthrottledClient.runUntil(_.throttleTimeMs <= 0.0) ->>>>>>> apache-github/trunk assertEquals(1, unthrottledClient.correlationId) assertTrue(s"Client should not have been throttled: $unthrottledClient", throttleTimeMetricValue(unthrottledClientId).isNaN) } diff --git a/core/src/test/scala/unit/kafka/server/SaslApiVersionsRequestTest.scala b/core/src/test/scala/unit/kafka/server/SaslApiVersionsRequestTest.scala index 92d8263234761..ca276d982d538 100644 --- a/core/src/test/scala/unit/kafka/server/SaslApiVersionsRequestTest.scala +++ b/core/src/test/scala/unit/kafka/server/SaslApiVersionsRequestTest.scala @@ -18,11 +18,10 @@ package kafka.server import java.net.Socket import java.util.Collections - import kafka.api.{KafkaSasl, SaslSetup} import kafka.utils.JaasTestUtils import org.apache.kafka.common.message.SaslHandshakeRequestData -import org.apache.kafka.common.protocol.Errors +import org.apache.kafka.common.protocol.{ApiKeys, Errors} import org.apache.kafka.common.requests.{ApiVersionsRequest, ApiVersionsResponse, SaslHandshakeRequest, SaslHandshakeResponse} import org.apache.kafka.common.security.auth.SecurityProtocol import org.junit.Assert._ @@ -91,18 +90,10 @@ class SaslApiVersionsRequestTest extends AbstractApiVersionsRequestTest with Sas } private def sendSaslHandshakeRequestValidateResponse(socket: Socket): Unit = { -<<<<<<< HEAD val request = new SaslHandshakeRequest(new SaslHandshakeRequestData().setMechanism("PLAIN"), ApiKeys.SASL_HANDSHAKE.latestVersion) - val response = sendAndReceive(request, ApiKeys.SASL_HANDSHAKE, socket) - val handshakeResponse = SaslHandshakeResponse.parse(response, request.version) - assertEquals(Errors.NONE, handshakeResponse.error) - assertEquals(Collections.singletonList("PLAIN"), handshakeResponse.enabledMechanisms) -======= - val request = new SaslHandshakeRequest(new SaslHandshakeRequestData().setMechanism("PLAIN")) val response = sendAndReceive[SaslHandshakeResponse](request, socket) assertEquals(Errors.NONE, response.error) assertEquals(Collections.singletonList("PLAIN"), response.enabledMechanisms) ->>>>>>> apache-github/trunk } } diff --git a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/metadata/MetadataRequestBenchmark.java b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/metadata/MetadataRequestBenchmark.java index 63d5de459a760..e9ca0aac10d0a 100644 --- a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/metadata/MetadataRequestBenchmark.java +++ b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/metadata/MetadataRequestBenchmark.java @@ -192,7 +192,7 @@ public void tearDown() { private RequestChannel.Request buildAllTopicMetadataRequest() { MetadataRequest metadataRequest = MetadataRequest.Builder.allTopics().build(); - ByteBuffer buffer = metadataRequest.serialize(new RequestHeader(metadataRequest.api, + ByteBuffer buffer = metadataRequest.serializeWithHeader(new RequestHeader(metadataRequest.apiKey, metadataRequest.version(), "", 0)); RequestHeader header = RequestHeader.parse(buffer); From fb876a11be30b1d5185525546b58eece2e7517de Mon Sep 17 00:00:00 2001 From: Ismael Juma Date: Wed, 2 Dec 2020 12:33:16 -0800 Subject: [PATCH 23/51] Minor style fixes --- .../apache/kafka/clients/ClientRequest.java | 10 +++---- .../apache/kafka/clients/NetworkClient.java | 26 +++++++++---------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/ClientRequest.java b/clients/src/main/java/org/apache/kafka/clients/ClientRequest.java index 0aebeaee77845..abba79567eb1e 100644 --- a/clients/src/main/java/org/apache/kafka/clients/ClientRequest.java +++ b/clients/src/main/java/org/apache/kafka/clients/ClientRequest.java @@ -85,11 +85,11 @@ public ApiKeys apiKey() { public RequestHeader makeHeader(short version) { ApiKeys requestApiKey = apiKey(); return new RequestHeader( - new RequestHeaderData(). - setRequestApiKey(requestApiKey.id). - setRequestApiVersion(version). - setClientId(clientId). - setCorrelationId(correlationId), + new RequestHeaderData() + .setRequestApiKey(requestApiKey.id) + .setRequestApiVersion(version) + .setClientId(clientId) + .setCorrelationId(correlationId), requestApiKey.requestHeaderVersion(version)); } diff --git a/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java b/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java index 1f5f1b5e8c8ca..99bfd85d98530 100644 --- a/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java +++ b/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java @@ -729,11 +729,11 @@ public static AbstractResponse parseResponse(ByteBuffer responseBuffer, RequestH throw new SchemaException("Buffer underflow while parsing response for request with header " + requestHeader, e); } catch (CorrelationIdMismatchException e) { if (SaslClientAuthenticator.isReserved(requestHeader.correlationId()) - && !SaslClientAuthenticator.isReserved(e.responseCorrelationId())) + && !SaslClientAuthenticator.isReserved(e.responseCorrelationId())) throw new SchemaException("The response is unrelated to Sasl request since its correlation id is " - + e.responseCorrelationId() + " and the reserved range for Sasl request is [ " - + SaslClientAuthenticator.MIN_RESERVED_CORRELATION_ID + "," - + SaslClientAuthenticator.MAX_RESERVED_CORRELATION_ID + "]"); + + e.responseCorrelationId() + " and the reserved range for Sasl request is [ " + + SaslClientAuthenticator.MIN_RESERVED_CORRELATION_ID + "," + + SaslClientAuthenticator.MAX_RESERVED_CORRELATION_ID + "]"); else { throw e; } @@ -868,24 +868,24 @@ private void handleCompletedReceives(List responses, long now) { for (NetworkReceive receive : this.selector.completedReceives()) { String source = receive.source(); InFlightRequest req = inFlightRequests.completeNext(source); - AbstractResponse responseBody = parseResponse(receive.payload(), req.header); + AbstractResponse response = parseResponse(receive.payload(), req.header); if (throttleTimeSensor != null) - throttleTimeSensor.record(responseBody.throttleTimeMs(), now); + throttleTimeSensor.record(response.throttleTimeMs(), now); if (log.isDebugEnabled()) { log.debug("Received {} response from node {} for request with header {}: {}", - req.header.apiKey(), req.destination, req.header, responseBody); + req.header.apiKey(), req.destination, req.header, response); } // If the received response includes a throttle delay, throttle the connection. - maybeThrottle(responseBody, req.header.apiVersion(), req.destination, now); - if (req.isInternalRequest && responseBody instanceof MetadataResponse) - metadataUpdater.handleSuccessfulResponse(req.header, now, (MetadataResponse) responseBody); - else if (req.isInternalRequest && responseBody instanceof ApiVersionsResponse) - handleApiVersionsResponse(responses, req, now, (ApiVersionsResponse) responseBody); + maybeThrottle(response, req.header.apiVersion(), req.destination, now); + if (req.isInternalRequest && response instanceof MetadataResponse) + metadataUpdater.handleSuccessfulResponse(req.header, now, (MetadataResponse) response); + else if (req.isInternalRequest && response instanceof ApiVersionsResponse) + handleApiVersionsResponse(responses, req, now, (ApiVersionsResponse) response); else - responses.add(req.completed(responseBody, now)); + responses.add(req.completed(response, now)); } } From 18d90966deb2e650741b2007d4707e9ada4ef522 Mon Sep 17 00:00:00 2001 From: Ismael Juma Date: Thu, 3 Dec 2020 05:38:01 -0800 Subject: [PATCH 24/51] Complete reimplementation of WriteTxnMarkersResponse --- .../requests/WriteTxnMarkersResponse.java | 33 ++++++++++++------- .../requests/WriteTxnMarkersRequestTest.java | 2 +- .../requests/WriteTxnMarkersResponseTest.java | 8 ++--- ...actionMarkerRequestCompletionHandler.scala | 3 +- .../kafka/api/AuthorizerIntegrationTest.scala | 2 +- .../unit/kafka/server/KafkaApisTest.scala | 8 ++--- 6 files changed, 33 insertions(+), 23 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/common/requests/WriteTxnMarkersResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/WriteTxnMarkersResponse.java index e0ea251e0dab1..7fef4ac6f0f80 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/WriteTxnMarkersResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/WriteTxnMarkersResponse.java @@ -88,9 +88,19 @@ public WriteTxnMarkersResponseData data() { return data; } - public Map errors(long producerId) { - return null; - //FIXME + public Map> errors() { + Map> errors = new HashMap<>(); + for (WritableTxnMarkerResult marker : data.markers()) { + Map topicPartitionErrorsMap = new HashMap<>(); + for (WritableTxnMarkerTopicResult topic : marker.topics()) { + for (WritableTxnMarkerPartitionResult partitionResult : topic.partitions()) { + topicPartitionErrorsMap.put(new TopicPartition(topic.name(), partitionResult.partitionIndex()), + Errors.forCode(partitionResult.errorCode())); + } + } + errors.put(marker.producerId(), topicPartitionErrorsMap); + } + return errors; } @Override @@ -100,15 +110,14 @@ public int throttleTimeMs() { @Override public Map errorCounts() { - //FIXME -// Map errorCounts = new HashMap<>(); -// errors.values().forEach(allErrors -> -// allErrors.values().forEach(error -> -// updateErrorCounts(errorCounts, error) -// ) -// ); -// return errorCounts; - return null; + Map errorCounts = new HashMap<>(); + for (WritableTxnMarkerResult marker : data.markers()) { + for (WritableTxnMarkerTopicResult topic : marker.topics()) { + for (WritableTxnMarkerPartitionResult partitionResult : topic.partitions()) + updateErrorCounts(errorCounts, Errors.forCode(partitionResult.errorCode())); + } + } + return errorCounts; } public static WriteTxnMarkersResponse parse(ByteBuffer buffer, short version) { diff --git a/clients/src/test/java/org/apache/kafka/common/requests/WriteTxnMarkersRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/WriteTxnMarkersRequestTest.java index d869201b52662..7e76ed72b8cce 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/WriteTxnMarkersRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/WriteTxnMarkersRequestTest.java @@ -72,7 +72,7 @@ public void testGetErrorResponse() { request.getErrorResponse(throttleTimeMs, Errors.UNKNOWN_PRODUCER_ID.exception()); assertEquals(Collections.singletonMap( - topicPartition, Errors.UNKNOWN_PRODUCER_ID), errorResponse.errors(producerId)); + topicPartition, Errors.UNKNOWN_PRODUCER_ID), errorResponse.errors().get(producerId)); assertEquals(Collections.singletonMap(Errors.UNKNOWN_PRODUCER_ID, 1), errorResponse.errorCounts()); // Write txn marker has no throttle time defined in response. assertEquals(0, errorResponse.throttleTimeMs()); diff --git a/clients/src/test/java/org/apache/kafka/common/requests/WriteTxnMarkersResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/WriteTxnMarkersResponseTest.java index 4cec88c22e923..c24c39ff04ece 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/WriteTxnMarkersResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/WriteTxnMarkersResponseTest.java @@ -46,15 +46,15 @@ public void setUp() { errorMap.put(producerIdOne, Collections.singletonMap(tp1, pidOneError)); errorMap.put(producerIdTwo, Collections.singletonMap(tp2, pidTwoError)); } - @Test - public void testConstructorWithStruct() { + @Test + public void testConstructor() { Map expectedErrorCounts = new HashMap<>(); expectedErrorCounts.put(Errors.UNKNOWN_PRODUCER_ID, 1); expectedErrorCounts.put(Errors.INVALID_PRODUCER_EPOCH, 1); WriteTxnMarkersResponse response = new WriteTxnMarkersResponse(errorMap); assertEquals(expectedErrorCounts, response.errorCounts()); - assertEquals(Collections.singletonMap(tp1, pidOneError), response.errors(producerIdOne)); - assertEquals(Collections.singletonMap(tp2, pidTwoError), response.errors(producerIdTwo)); + assertEquals(Collections.singletonMap(tp1, pidOneError), response.errors().get(producerIdOne)); + assertEquals(Collections.singletonMap(tp2, pidTwoError), response.errors().get(producerIdTwo)); } } diff --git a/core/src/main/scala/kafka/coordinator/transaction/TransactionMarkerRequestCompletionHandler.scala b/core/src/main/scala/kafka/coordinator/transaction/TransactionMarkerRequestCompletionHandler.scala index c2efdbd934672..2a20008ab5a3e 100644 --- a/core/src/main/scala/kafka/coordinator/transaction/TransactionMarkerRequestCompletionHandler.scala +++ b/core/src/main/scala/kafka/coordinator/transaction/TransactionMarkerRequestCompletionHandler.scala @@ -89,10 +89,11 @@ class TransactionMarkerRequestCompletionHandler(brokerId: Int, val writeTxnMarkerResponse = response.responseBody.asInstanceOf[WriteTxnMarkersResponse] + val responseErrors = writeTxnMarkerResponse.errors; for (txnIdAndMarker <- txnIdAndMarkerEntries.asScala) { val transactionalId = txnIdAndMarker.txnId val txnMarker = txnIdAndMarker.txnMarkerEntry - val errors = writeTxnMarkerResponse.errors(txnMarker.producerId) + val errors = responseErrors.get(txnMarker.producerId) if (errors == null) throw new IllegalStateException(s"WriteTxnMarkerResponse does not contain expected error map for producer id ${txnMarker.producerId}") diff --git a/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala index a3df972520e49..2e2a03c2eda77 100644 --- a/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala +++ b/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala @@ -195,7 +195,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest { ApiKeys.ALTER_CONFIGS -> ((resp: AlterConfigsResponse) => resp.errors.get(new ConfigResource(ConfigResource.Type.TOPIC, tp.topic)).error), ApiKeys.INIT_PRODUCER_ID -> ((resp: InitProducerIdResponse) => resp.error), - ApiKeys.WRITE_TXN_MARKERS -> ((resp: WriteTxnMarkersResponse) => resp.errors(producerId).get(tp)), + ApiKeys.WRITE_TXN_MARKERS -> ((resp: WriteTxnMarkersResponse) => resp.errors.get(producerId).get(tp)), ApiKeys.ADD_PARTITIONS_TO_TXN -> ((resp: AddPartitionsToTxnResponse) => resp.errors.get(tp)), ApiKeys.ADD_OFFSETS_TO_TXN -> ((resp: AddOffsetsToTxnResponse) => Errors.forCode(resp.data.errorCode)), ApiKeys.END_TXN -> ((resp: EndTxnResponse) => resp.error), diff --git a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala index 5e2083392e880..b3bd2c536e120 100644 --- a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala +++ b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala @@ -1358,7 +1358,7 @@ class KafkaApisTest { val markersResponse = readResponse(writeTxnMarkersRequest, capturedResponse) .asInstanceOf[WriteTxnMarkersResponse] - assertEquals(expectedErrors, markersResponse.errors(1)) + assertEquals(expectedErrors, markersResponse.errors.get(1)) } @Test @@ -1377,7 +1377,7 @@ class KafkaApisTest { val markersResponse = readResponse(writeTxnMarkersRequest, capturedResponse) .asInstanceOf[WriteTxnMarkersResponse] - assertEquals(expectedErrors, markersResponse.errors(1)) + assertEquals(expectedErrors, markersResponse.errors.get(1)) } @Test @@ -1412,7 +1412,7 @@ class KafkaApisTest { val markersResponse = readResponse(writeTxnMarkersRequest, capturedResponse) .asInstanceOf[WriteTxnMarkersResponse] - assertEquals(expectedErrors, markersResponse.errors(1)) + assertEquals(expectedErrors, markersResponse.errors.get(1)) EasyMock.verify(replicaManager) } @@ -1549,7 +1549,7 @@ class KafkaApisTest { val markersResponse = readResponse(writeTxnMarkersRequest, capturedResponse) .asInstanceOf[WriteTxnMarkersResponse] - assertEquals(expectedErrors, markersResponse.errors(1)) + assertEquals(expectedErrors, markersResponse.errors.get(1)) EasyMock.verify(replicaManager) } From 0b8ea1546ac220011a39c61ec8c915b54a9c02b9 Mon Sep 17 00:00:00 2001 From: Ismael Juma Date: Thu, 3 Dec 2020 05:38:54 -0800 Subject: [PATCH 25/51] AbstractResponse toString no longer requires version --- .../apache/kafka/clients/admin/KafkaAdminClient.java | 3 +-- .../apache/kafka/common/requests/AbstractRequest.java | 6 +++++- .../apache/kafka/common/requests/AbstractResponse.java | 9 ++------- .../kafka/common/requests/RequestResponseTest.java | 10 +++++----- .../kafka/controller/ControllerChannelManager.scala | 2 +- core/src/main/scala/kafka/network/RequestChannel.scala | 2 +- .../scala/kafka/tools/TestRaftRequestHandler.scala | 2 +- .../kafka/jmh/metadata/MetadataRequestBenchmark.java | 2 +- 8 files changed, 17 insertions(+), 19 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java index 9c931cd7556f7..cf0a6f6949f90 100644 --- a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java +++ b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java @@ -1182,8 +1182,7 @@ private void handleResponses(long now, List responses) { try { call.handleResponse(response.responseBody()); if (log.isTraceEnabled()) - log.trace("{} got response {}", call, - response.responseBody().toString(response.requestHeader().apiVersion())); + log.trace("{} got response {}", call, response.responseBody()); } catch (Throwable t) { if (log.isTraceEnabled()) log.trace("{} handleResponse failed with {}", call, prettyPrintException(t)); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java index 533ba8982e392..f2e2d4c6c2236 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java @@ -78,7 +78,7 @@ public T build() { } private final short version; - public final ApiKeys apiKey; + private final ApiKeys apiKey; public AbstractRequest(ApiKeys apiKey, short version) { if (!apiKey.isVersionSupported(version)) @@ -94,6 +94,10 @@ public short version() { return version; } + public ApiKeys apiKey() { + return apiKey; + } + public Send toSend(String destination, RequestHeader header) { return new NetworkSend(destination, serializeWithHeader(header)); } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java index 428a2a0c046d6..b71eb4c118674 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java @@ -53,7 +53,7 @@ public ByteBuffer serializeWithHeader(short version, int correlationId) { return serializeWithHeader(new ResponseHeader(correlationId, apiKey.responseHeaderVersion(version)), version); } - protected ByteBuffer serializeWithHeader(ResponseHeader header, short version) { + ByteBuffer serializeWithHeader(ResponseHeader header, short version) { Objects.requireNonNull(header, "header should not be null"); return serialize(header, data(), version); } @@ -108,11 +108,6 @@ protected void updateErrorCounts(Map errorCounts, Errors error) errorCounts.put(error, count + 1); } - /** - * Return the auto-generated `Message` instance if this request/response relies on one for - * serialization/deserialization. If this class has not yet been updated to rely on the auto-generated protocol - * classes, return `null`. - */ protected abstract Message data(); /** @@ -279,7 +274,7 @@ public ApiKeys apiKey() { public abstract int throttleTimeMs(); - public String toString(short version) { + public String toString() { return data().toString(); } } diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java index d3ad12713b436..4bd6686f77391 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java @@ -575,8 +575,8 @@ private void checkErrorResponse(AbstractRequest req, Throwable e, boolean checkE AbstractResponse response = req.getErrorResponse(e); checkResponse(response, req.version(), checkEqualityAndHashCode); if (e instanceof UnknownServerException) { - String responseStr = response.toString(req.version()); - assertFalse(String.format("Unknown message included in response for %s: %s ", req.apiKey, responseStr), + String responseStr = response.toString(); + assertFalse(String.format("Unknown message included in response for %s: %s ", req.apiKey(), responseStr), responseStr.contains(e.getMessage())); } } @@ -587,7 +587,7 @@ private void checkRequest(AbstractRequest req, boolean checkEquality) { // in the request is a HashMap with multiple elements since ordering of the elements may vary) try { ByteBuffer serializedBytes = req.serializeBody(); - AbstractRequest deserialized = AbstractRequest.parseRequest(req.apiKey, req.version(), serializedBytes).request; + AbstractRequest deserialized = AbstractRequest.parseRequest(req.apiKey(), req.version(), serializedBytes).request; ByteBuffer serializedBytes2 = deserialized.serializeBody(); assertEquals(req.data(), deserialized.data()); //FIXME if (checkEquality) @@ -816,12 +816,12 @@ public void testFetchRequestMaxBytesOldVersions() { @Test public void testFetchRequestIsolationLevel() throws Exception { FetchRequest request = createFetchRequest(4, IsolationLevel.READ_COMMITTED); - FetchRequest deserialized = (FetchRequest) AbstractRequest.parseRequest(request.apiKey, request.version(), + FetchRequest deserialized = (FetchRequest) AbstractRequest.parseRequest(request.apiKey(), request.version(), request.serializeBody()).request; assertEquals(request.isolationLevel(), deserialized.isolationLevel()); request = createFetchRequest(4, IsolationLevel.READ_UNCOMMITTED); - deserialized = (FetchRequest) AbstractRequest.parseRequest(request.apiKey, request.version(), + deserialized = (FetchRequest) AbstractRequest.parseRequest(request.apiKey(), request.version(), request.serializeBody()).request; assertEquals(request.isolationLevel(), deserialized.isolationLevel()); } diff --git a/core/src/main/scala/kafka/controller/ControllerChannelManager.scala b/core/src/main/scala/kafka/controller/ControllerChannelManager.scala index 1391d345c685f..f217776109cad 100755 --- a/core/src/main/scala/kafka/controller/ControllerChannelManager.scala +++ b/core/src/main/scala/kafka/controller/ControllerChannelManager.scala @@ -271,7 +271,7 @@ class RequestSendThread(val controllerId: Int, val response = clientResponse.responseBody stateChangeLogger.withControllerEpoch(controllerContext.epoch).trace(s"Received response " + - s"${response.toString(requestHeader.apiVersion)} for request $api with correlation id " + + s"$response for request $api with correlation id " + s"${requestHeader.correlationId} sent to broker $brokerNode") if (callback != null) { diff --git a/core/src/main/scala/kafka/network/RequestChannel.scala b/core/src/main/scala/kafka/network/RequestChannel.scala index 06b83b65b6a5d..39ca69f1f1893 100644 --- a/core/src/main/scala/kafka/network/RequestChannel.scala +++ b/core/src/main/scala/kafka/network/RequestChannel.scala @@ -125,7 +125,7 @@ object RequestChannel extends Logging { def responseString(response: AbstractResponse): Option[String] = { if (RequestChannel.isRequestLoggingEnabled) - Some(response.toString(context.apiVersion)) + Some(response.toString) else None } diff --git a/core/src/main/scala/kafka/tools/TestRaftRequestHandler.scala b/core/src/main/scala/kafka/tools/TestRaftRequestHandler.scala index a8f7cd93838c3..fefd274927df2 100644 --- a/core/src/main/scala/kafka/tools/TestRaftRequestHandler.scala +++ b/core/src/main/scala/kafka/tools/TestRaftRequestHandler.scala @@ -94,7 +94,7 @@ class TestRaftRequestHandler( case Some(response) => val responseSend = request.context.buildResponseSend(response) val responseString = - if (RequestChannel.isRequestLoggingEnabled) Some(response.toString(request.context.apiVersion)) + if (RequestChannel.isRequestLoggingEnabled) Some(response.toString) else None new RequestChannel.SendResponse(request, responseSend, responseString, None) case None => diff --git a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/metadata/MetadataRequestBenchmark.java b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/metadata/MetadataRequestBenchmark.java index 699735f6b036b..4ac4c56d4c18f 100644 --- a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/metadata/MetadataRequestBenchmark.java +++ b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/metadata/MetadataRequestBenchmark.java @@ -200,7 +200,7 @@ public void tearDown() { private RequestChannel.Request buildAllTopicMetadataRequest() { MetadataRequest metadataRequest = MetadataRequest.Builder.allTopics().build(); - ByteBuffer buffer = metadataRequest.serializeWithHeader(new RequestHeader(metadataRequest.apiKey, + ByteBuffer buffer = metadataRequest.serializeWithHeader(new RequestHeader(metadataRequest.apiKey(), metadataRequest.version(), "", 0)); RequestHeader header = RequestHeader.parse(buffer); From aa1735f2deab05ce155775748ff6707137932cbe Mon Sep 17 00:00:00 2001 From: Ismael Juma Date: Thu, 3 Dec 2020 05:51:11 -0800 Subject: [PATCH 26/51] Various minor fixes --- .../requests/AddOffsetsToTxnRequest.java | 3 +- .../requests/AddOffsetsToTxnResponse.java | 3 +- .../requests/AddPartitionsToTxnRequest.java | 3 +- .../requests/AddPartitionsToTxnResponse.java | 3 +- .../requests/AlterClientQuotasRequest.java | 3 +- .../common/requests/AlterConfigsRequest.java | 3 +- .../requests/AlterReplicaLogDirsRequest.java | 3 +- .../common/requests/ApiVersionsResponse.java | 50 +++---------------- .../requests/BeginQuorumEpochResponse.java | 2 +- .../common/requests/CreateAclsRequest.java | 3 +- .../common/requests/CreateAclsResponse.java | 3 +- .../common/requests/DeleteAclsRequest.java | 2 +- 12 files changed, 18 insertions(+), 63 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AddOffsetsToTxnRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AddOffsetsToTxnRequest.java index 99d762d03fd11..4ad56aa94268f 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AddOffsetsToTxnRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AddOffsetsToTxnRequest.java @@ -21,7 +21,6 @@ import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.Message; import java.nio.ByteBuffer; @@ -54,7 +53,7 @@ public AddOffsetsToTxnRequest(AddOffsetsToTxnRequestData data, short version) { } @Override - protected Message data() { + protected AddOffsetsToTxnRequestData data() { return data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AddOffsetsToTxnResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AddOffsetsToTxnResponse.java index 4f1bd55b4799c..6908e51960609 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AddOffsetsToTxnResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AddOffsetsToTxnResponse.java @@ -20,7 +20,6 @@ import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.Message; import java.nio.ByteBuffer; import java.util.Map; @@ -58,7 +57,7 @@ public int throttleTimeMs() { } @Override - protected Message data() { + protected AddOffsetsToTxnResponseData data() { return data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AddPartitionsToTxnRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AddPartitionsToTxnRequest.java index 25722c80510c8..57645789ca000 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AddPartitionsToTxnRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AddPartitionsToTxnRequest.java @@ -23,7 +23,6 @@ import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.Message; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -113,7 +112,7 @@ public List partitions() { } @Override - protected Message data() { + protected AddPartitionsToTxnRequestData data() { return data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AddPartitionsToTxnResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AddPartitionsToTxnResponse.java index aab642fb68a3d..c6da3af0a9319 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AddPartitionsToTxnResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AddPartitionsToTxnResponse.java @@ -25,7 +25,6 @@ import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.Message; import java.nio.ByteBuffer; import java.util.HashMap; @@ -118,7 +117,7 @@ public Map errorCounts() { } @Override - protected Message data() { + protected AddPartitionsToTxnResponseData data() { return data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasRequest.java index 38d6faf24e581..9b5fdc0df720c 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasRequest.java @@ -22,7 +22,6 @@ import org.apache.kafka.common.message.AlterClientQuotasRequestData.OpData; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; -import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.quota.ClientQuotaAlteration; import org.apache.kafka.common.quota.ClientQuotaEntity; @@ -111,7 +110,7 @@ public boolean validateOnly() { } @Override - protected Message data() { + protected AlterClientQuotasRequestData data() { return data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AlterConfigsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AlterConfigsRequest.java index 73357cb90a288..f30e8b9c1ffe9 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AlterConfigsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AlterConfigsRequest.java @@ -22,7 +22,6 @@ import org.apache.kafka.common.message.AlterConfigsResponseData; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; -import org.apache.kafka.common.protocol.Message; import java.nio.ByteBuffer; import java.util.Collection; @@ -113,7 +112,7 @@ public boolean validateOnly() { } @Override - protected Message data() { + protected AlterConfigsRequestData data() { return data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AlterReplicaLogDirsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AlterReplicaLogDirsRequest.java index b96f716b74be4..5eba039fa556b 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AlterReplicaLogDirsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AlterReplicaLogDirsRequest.java @@ -22,7 +22,6 @@ import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.Message; import java.nio.ByteBuffer; import java.util.HashMap; @@ -62,7 +61,7 @@ public AlterReplicaLogDirsRequest(AlterReplicaLogDirsRequestData data, short ver } @Override - protected Message data() { + protected AlterReplicaLogDirsRequestData data() { return data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsResponse.java index 1d0f401e26676..eaf8113fcd0b7 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsResponse.java @@ -43,14 +43,8 @@ public class ApiVersionsResponse extends AbstractResponse { public static final long UNKNOWN_FINALIZED_FEATURES_EPOCH = -1L; - public static final ApiVersionsResponse DEFAULT_API_VERSIONS_RESPONSE = - createApiVersionsResponse( - DEFAULT_THROTTLE_TIME, - RecordBatch.CURRENT_MAGIC_VALUE, - Features.emptySupportedFeatures(), - Features.emptyFinalizedFeatures(), - UNKNOWN_FINALIZED_FEATURES_EPOCH - ); + public static final ApiVersionsResponse DEFAULT_API_VERSIONS_RESPONSE = createApiVersionsResponse( + DEFAULT_THROTTLE_TIME, RecordBatch.CURRENT_MAGIC_VALUE); public final ApiVersionsResponseData data; @@ -91,49 +85,19 @@ public static ApiVersionsResponse parse(ByteBuffer buffer, short version) { // method is not necessarily the real one. It may be version 0 as well. int prev = buffer.position(); try { - return new ApiVersionsResponse( - new ApiVersionsResponseData(new ByteBufferAccessor(buffer), version)); + return new ApiVersionsResponse(new ApiVersionsResponseData(new ByteBufferAccessor(buffer), version)); } catch (RuntimeException e) { buffer.position(prev); if (version != 0) - return new ApiVersionsResponse( - new ApiVersionsResponseData(new ByteBufferAccessor(buffer), (short) 0)); + return new ApiVersionsResponse(new ApiVersionsResponseData(new ByteBufferAccessor(buffer), (short) 0)); else throw e; } } - public static ApiVersionsResponse apiVersionsResponse( - int throttleTimeMs, - byte maxMagic, - Features latestSupportedFeatures) { - return apiVersionsResponse( - throttleTimeMs, maxMagic, latestSupportedFeatures, Features.emptyFinalizedFeatures(), UNKNOWN_FINALIZED_FEATURES_EPOCH); - } - - public static ApiVersionsResponse apiVersionsResponse( - int throttleTimeMs, - byte maxMagic, - Features latestSupportedFeatures, - Features finalizedFeatures, - long finalizedFeaturesEpoch) { - if (maxMagic == RecordBatch.CURRENT_MAGIC_VALUE && throttleTimeMs == DEFAULT_THROTTLE_TIME) { - return DEFAULT_API_VERSIONS_RESPONSE; - } - return createApiVersionsResponse( - throttleTimeMs, maxMagic, latestSupportedFeatures, finalizedFeatures, finalizedFeaturesEpoch); - } - - public static ApiVersionsResponse createApiVersionsResponse( - final int throttleTimeMs, - final byte minMagic) { - return createApiVersionsResponse( - throttleTimeMs, - minMagic, - Features.emptySupportedFeatures(), - Features.emptyFinalizedFeatures(), - UNKNOWN_FINALIZED_FEATURES_EPOCH - ); + public static ApiVersionsResponse createApiVersionsResponse(final int throttleTimeMs, final byte minMagic) { + return createApiVersionsResponse(throttleTimeMs, minMagic, Features.emptySupportedFeatures(), + Features.emptyFinalizedFeatures(), UNKNOWN_FINALIZED_FEATURES_EPOCH); } private static ApiVersionsResponse createApiVersionsResponse( diff --git a/clients/src/main/java/org/apache/kafka/common/requests/BeginQuorumEpochResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/BeginQuorumEpochResponse.java index 4f68201ae41b2..c3e80eccaa82c 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/BeginQuorumEpochResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/BeginQuorumEpochResponse.java @@ -92,7 +92,7 @@ protected BeginQuorumEpochResponseData data() { @Override public int throttleTimeMs() { - return 0; + return DEFAULT_THROTTLE_TIME; } public static BeginQuorumEpochResponse parse(ByteBuffer buffer, short version) { diff --git a/clients/src/main/java/org/apache/kafka/common/requests/CreateAclsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/CreateAclsRequest.java index 103dca0d3eee1..2ce651565b745 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/CreateAclsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/CreateAclsRequest.java @@ -28,7 +28,6 @@ import org.apache.kafka.common.message.CreateAclsResponseData.AclCreationResult; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; -import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.resource.PatternType; import org.apache.kafka.common.resource.ResourcePattern; import org.apache.kafka.common.resource.ResourceType; @@ -71,7 +70,7 @@ public List aclCreations() { } @Override - protected Message data() { + protected CreateAclsRequestData data() { return data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/CreateAclsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/CreateAclsResponse.java index 6ad3e339e50dd..d0149b9a2a0d7 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/CreateAclsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/CreateAclsResponse.java @@ -20,7 +20,6 @@ import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.Message; import java.nio.ByteBuffer; import java.util.List; @@ -35,7 +34,7 @@ public CreateAclsResponse(CreateAclsResponseData data) { } @Override - protected Message data() { + protected CreateAclsResponseData data() { return data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DeleteAclsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DeleteAclsRequest.java index 1a8c2b6194dc8..e21bda31a77c3 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DeleteAclsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DeleteAclsRequest.java @@ -101,7 +101,7 @@ public List filters() { } @Override - protected Message data() { + protected DeleteAclsRequestData data() { return data; } From fe9eccbb6679ea6f39ebbd88e41560cad28689e3 Mon Sep 17 00:00:00 2001 From: Ismael Juma Date: Thu, 3 Dec 2020 05:51:34 -0800 Subject: [PATCH 27/51] Complete implementation of AlterClientQuotasRequest/Response --- .../requests/AlterClientQuotasRequest.java | 18 +++++++++++------- .../requests/AlterClientQuotasResponse.java | 18 ------------------ 2 files changed, 11 insertions(+), 25 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasRequest.java index 9b5fdc0df720c..7178475123633 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasRequest.java @@ -20,6 +20,7 @@ import org.apache.kafka.common.message.AlterClientQuotasRequestData.EntityData; import org.apache.kafka.common.message.AlterClientQuotasRequestData.EntryData; import org.apache.kafka.common.message.AlterClientQuotasRequestData.OpData; +import org.apache.kafka.common.message.AlterClientQuotasResponseData; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.quota.ClientQuotaAlteration; @@ -116,17 +117,20 @@ protected AlterClientQuotasRequestData data() { @Override public AlterClientQuotasResponse getErrorResponse(int throttleTimeMs, Throwable e) { - ArrayList entities = new ArrayList<>(data.entries().size()); + List responseEntries = new ArrayList<>(); for (EntryData entryData : data.entries()) { - Map entity = new HashMap<>(entryData.entity().size()); + List responseEntities = new ArrayList<>(); for (EntityData entityData : entryData.entity()) { - entity.put(entityData.entityType(), entityData.entityName()); + responseEntities.add(new AlterClientQuotasResponseData.EntityData() + .setEntityType(entityData.entityType()) + .setEntityName(entityData.entityName())); } - entities.add(new ClientQuotaEntity(entity)); + responseEntries.add(new AlterClientQuotasResponseData.EntryData().setEntity(responseEntities)); } - //FIXME - return null; -// return new AlterClientQuotasResponse(entities, throttleTimeMs, e); + AlterClientQuotasResponseData responseData = new AlterClientQuotasResponseData() + .setThrottleTimeMs(throttleTimeMs) + .setEntries(responseEntries); + return new AlterClientQuotasResponse(responseData); } public static AlterClientQuotasRequest parse(ByteBuffer buffer, short version) { diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasResponse.java index da37bc6e68994..9127e26080c8b 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasResponse.java @@ -111,22 +111,4 @@ public static AlterClientQuotasResponse fromQuotaEntities(Map entities, int throttleTimeMs, Throwable e) { -// short errorCode = Errors.forException(e).code(); -// String errorMessage = e.getMessage(); -// -// List entries = new ArrayList<>(entities.size()); -// for (ClientQuotaEntity entity : entities) { -// entries.add(new EntryData() -// .setErrorCode(errorCode) -// .setErrorMessage(errorMessage) -// .setEntity(toEntityData(entity))); -// } -// -// this.data = new AlterClientQuotasResponseData() -// .setThrottleTimeMs(throttleTimeMs) -// .setEntries(entries); -// } -// } - } From 5b7695b5c75412b23f129c64b656d4c40721685a Mon Sep 17 00:00:00 2001 From: Ismael Juma Date: Thu, 3 Dec 2020 06:14:01 -0800 Subject: [PATCH 28/51] Minor style fixes --- .../apache/kafka/common/requests/DescribeQuorumRequest.java | 2 -- .../apache/kafka/common/requests/DescribeQuorumResponse.java | 2 +- .../apache/kafka/common/requests/EndQuorumEpochResponse.java | 4 ++-- .../org/apache/kafka/common/requests/EnvelopeResponse.java | 2 +- .../java/org/apache/kafka/common/requests/VoteResponse.java | 2 +- 5 files changed, 5 insertions(+), 7 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeQuorumRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeQuorumRequest.java index 62d1900a02cb1..c318c214b6adf 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeQuorumRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeQuorumRequest.java @@ -81,8 +81,6 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { return new DescribeQuorumResponse(getTopLevelErrorResponse(Errors.forException(e))); } - - public static DescribeQuorumResponseData getPartitionLevelErrorResponse(DescribeQuorumRequestData data, Errors error) { short errorCode = error.code(); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeQuorumResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeQuorumResponse.java index de30616cc7e07..cb1a5ec277c08 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeQuorumResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeQuorumResponse.java @@ -69,7 +69,7 @@ protected DescribeQuorumResponseData data() { @Override public int throttleTimeMs() { - return 0; + return DEFAULT_THROTTLE_TIME; } public static DescribeQuorumResponseData singletonResponse(TopicPartition topicPartition, diff --git a/clients/src/main/java/org/apache/kafka/common/requests/EndQuorumEpochResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/EndQuorumEpochResponse.java index c8e850423fac6..15c0fa04fd74c 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/EndQuorumEpochResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/EndQuorumEpochResponse.java @@ -65,13 +65,13 @@ public Map errorCounts() { } @Override - protected Message data() { + protected EndQuorumEpochResponseData data() { return data; } @Override public int throttleTimeMs() { - return 0; + return DEFAULT_THROTTLE_TIME; } public static EndQuorumEpochResponseData singletonResponse( diff --git a/clients/src/main/java/org/apache/kafka/common/requests/EnvelopeResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/EnvelopeResponse.java index f026b41abf446..6064bc3400630 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/EnvelopeResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/EnvelopeResponse.java @@ -65,7 +65,7 @@ public EnvelopeResponseData data() { @Override public int throttleTimeMs() { - return 0; + return DEFAULT_THROTTLE_TIME; } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/VoteResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/VoteResponse.java index 159621971af7f..5956fe0077e99 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/VoteResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/VoteResponse.java @@ -89,7 +89,7 @@ protected VoteResponseData data() { @Override public int throttleTimeMs() { - return 0; + return DEFAULT_THROTTLE_TIME; } public static VoteResponse parse(ByteBuffer buffer, short version) { From d5cbad3aa52a50e2e96f1d47d936517d68450b47 Mon Sep 17 00:00:00 2001 From: Ismael Juma Date: Thu, 3 Dec 2020 06:14:21 -0800 Subject: [PATCH 29/51] Revert SaslAuthenticateRequest.Builder.toString change as it could leak authentication data --- .../apache/kafka/common/requests/SaslAuthenticateRequest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clients/src/main/java/org/apache/kafka/common/requests/SaslAuthenticateRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/SaslAuthenticateRequest.java index defcfcafdca56..35da4caf9c5c1 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/SaslAuthenticateRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/SaslAuthenticateRequest.java @@ -49,7 +49,7 @@ public SaslAuthenticateRequest build(short version) { @Override public String toString() { - return data.toString(); + return "(type=SaslAuthenticateRequest)"; } } From 02a98d8adaf23f27692fc43e3e38727a4d0ec69d Mon Sep 17 00:00:00 2001 From: Ismael Juma Date: Thu, 3 Dec 2020 06:14:52 -0800 Subject: [PATCH 30/51] Fix bug introduced in previous commit where handleProduceRequest did not account for header bytes --- core/src/main/scala/kafka/server/KafkaApis.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala index 76379315ad72f..f514d07569d31 100644 --- a/core/src/main/scala/kafka/server/KafkaApis.scala +++ b/core/src/main/scala/kafka/server/KafkaApis.scala @@ -565,7 +565,7 @@ class KafkaApis(val requestChannel: RequestChannel, */ def handleProduceRequest(request: RequestChannel.Request): Unit = { val produceRequest = request.body[ProduceRequest] - val requestSize = request.sizeOfBodyInBytes + val requestSize = request.sizeInBytes val (hasIdempotentRecords, hasTransactionalRecords) = { val flags = RequestUtils.flags(produceRequest) From 5023f3e68364536996f871fa620448a81a758d91 Mon Sep 17 00:00:00 2001 From: Ismael Juma Date: Thu, 3 Dec 2020 06:26:19 -0800 Subject: [PATCH 31/51] Minor style fixes --- .../kafka/common/requests/AlterClientQuotasResponse.java | 3 +-- .../org/apache/kafka/common/requests/DeleteAclsRequest.java | 1 - .../org/apache/kafka/common/requests/DeleteAclsResponse.java | 3 +-- .../org/apache/kafka/common/requests/DeleteGroupsRequest.java | 3 +-- .../org/apache/kafka/common/requests/DeleteGroupsResponse.java | 3 +-- .../org/apache/kafka/common/requests/DescribeAclsResponse.java | 3 +-- .../kafka/common/requests/DescribeClientQuotasRequest.java | 3 +-- .../kafka/common/requests/DescribeClientQuotasResponse.java | 3 +-- .../kafka/common/requests/DescribeDelegationTokenResponse.java | 3 +-- .../apache/kafka/common/requests/EndQuorumEpochResponse.java | 1 - .../java/org/apache/kafka/common/requests/EndTxnRequest.java | 3 +-- .../java/org/apache/kafka/common/requests/EndTxnResponse.java | 3 +-- .../kafka/common/requests/ExpireDelegationTokenRequest.java | 3 +-- .../kafka/common/requests/ExpireDelegationTokenResponse.java | 3 +-- .../org/apache/kafka/common/requests/HeartbeatRequest.java | 3 +-- .../org/apache/kafka/common/requests/HeartbeatResponse.java | 3 +-- .../apache/kafka/common/requests/InitProducerIdRequest.java | 3 +-- .../apache/kafka/common/requests/InitProducerIdResponse.java | 3 +-- .../org/apache/kafka/common/requests/LeaderAndIsrResponse.java | 3 +-- .../org/apache/kafka/common/requests/LeaveGroupResponse.java | 3 +-- .../org/apache/kafka/common/requests/MetadataResponse.java | 3 +-- .../org/apache/kafka/common/requests/OffsetDeleteRequest.java | 3 +-- .../org/apache/kafka/common/requests/OffsetDeleteResponse.java | 3 +-- .../org/apache/kafka/common/requests/OffsetFetchRequest.java | 3 +-- .../org/apache/kafka/common/requests/OffsetFetchResponse.java | 3 +-- .../java/org/apache/kafka/common/requests/ProduceRequest.java | 3 +-- .../kafka/common/requests/RenewDelegationTokenResponse.java | 3 +-- .../apache/kafka/common/requests/SaslAuthenticateResponse.java | 3 +-- .../apache/kafka/common/requests/SaslHandshakeResponse.java | 3 +-- .../org/apache/kafka/common/requests/StopReplicaResponse.java | 3 +-- .../org/apache/kafka/common/requests/SyncGroupRequest.java | 3 +-- .../org/apache/kafka/common/requests/SyncGroupResponse.java | 3 +-- .../apache/kafka/common/requests/TxnOffsetCommitRequest.java | 3 +-- .../apache/kafka/common/requests/TxnOffsetCommitResponse.java | 3 +-- .../apache/kafka/common/requests/UpdateMetadataResponse.java | 3 +-- .../apache/kafka/common/requests/WriteTxnMarkersRequest.java | 3 +-- 36 files changed, 34 insertions(+), 70 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasResponse.java index 9127e26080c8b..bce548822d895 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasResponse.java @@ -23,7 +23,6 @@ import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.quota.ClientQuotaEntity; import java.nio.ByteBuffer; @@ -78,7 +77,7 @@ public Map errorCounts() { } @Override - protected Message data() { + protected AlterClientQuotasResponseData data() { return data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DeleteAclsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DeleteAclsRequest.java index e21bda31a77c3..6face08f5675d 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DeleteAclsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DeleteAclsRequest.java @@ -29,7 +29,6 @@ import org.apache.kafka.common.message.DeleteAclsResponseData.DeleteAclsFilterResult; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; -import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.resource.PatternType; import org.apache.kafka.common.resource.ResourcePatternFilter; import org.apache.kafka.common.resource.ResourceType; diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DeleteAclsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DeleteAclsResponse.java index 6ab814ad4e98c..fed627a62d42c 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DeleteAclsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DeleteAclsResponse.java @@ -20,7 +20,6 @@ import org.apache.kafka.common.acl.AclBinding; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; -import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.resource.ResourcePattern; import org.apache.kafka.common.acl.AclOperation; import org.apache.kafka.common.acl.AclPermissionType; @@ -52,7 +51,7 @@ public DeleteAclsResponse(DeleteAclsResponseData data, short version) { } @Override - protected Message data() { + protected DeleteAclsResponseData data() { return data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsRequest.java index 2c93fdd25c307..d09a4d4c02d77 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsRequest.java @@ -23,7 +23,6 @@ import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.Message; import java.nio.ByteBuffer; @@ -76,7 +75,7 @@ public static DeleteGroupsRequest parse(ByteBuffer buffer, short version) { } @Override - protected Message data() { + protected DeleteGroupsRequestData data() { return data; } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsResponse.java index 6c50cb0683592..a8e8d482de1d1 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsResponse.java @@ -21,7 +21,6 @@ import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.Message; import java.nio.ByteBuffer; import java.util.HashMap; @@ -48,7 +47,7 @@ public DeleteGroupsResponse(DeleteGroupsResponseData data) { } @Override - protected Message data() { + protected DeleteGroupsResponseData data() { return data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeAclsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeAclsResponse.java index 82c1ca1d34c08..1562b76e83d5b 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeAclsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeAclsResponse.java @@ -21,7 +21,6 @@ import org.apache.kafka.common.acl.AclBinding; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; -import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.resource.PatternType; import org.apache.kafka.common.resource.ResourcePattern; import org.apache.kafka.common.errors.UnsupportedVersionException; @@ -63,7 +62,7 @@ public DescribeAclsResponse(DescribeAclsResponseData data, short version) { } @Override - protected Message data() { + protected DescribeAclsResponseData data() { return data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasRequest.java index 59b1245bdabfb..dc8064652f9b2 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasRequest.java @@ -22,7 +22,6 @@ import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.quota.ClientQuotaFilter; import org.apache.kafka.common.quota.ClientQuotaFilterComponent; @@ -108,7 +107,7 @@ public ClientQuotaFilter filter() { } @Override - protected Message data() { + protected DescribeClientQuotasRequestData data() { return data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasResponse.java index 304a89ecc83ae..94fca64221d0f 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasResponse.java @@ -24,7 +24,6 @@ import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.quota.ClientQuotaEntity; import java.nio.ByteBuffer; @@ -72,7 +71,7 @@ public int throttleTimeMs() { } @Override - protected Message data() { + protected DescribeClientQuotasResponseData data() { return data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenResponse.java index 3882688b9c7a1..f6579a06b72da 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenResponse.java @@ -22,7 +22,6 @@ import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.security.auth.KafkaPrincipal; import org.apache.kafka.common.security.token.delegation.DelegationToken; import org.apache.kafka.common.security.token.delegation.TokenInformation; @@ -81,7 +80,7 @@ public Map errorCounts() { } @Override - protected Message data() { + protected DescribeDelegationTokenResponseData data() { return data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/EndQuorumEpochResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/EndQuorumEpochResponse.java index 15c0fa04fd74c..9b446663ea540 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/EndQuorumEpochResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/EndQuorumEpochResponse.java @@ -22,7 +22,6 @@ import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.Message; import java.nio.ByteBuffer; import java.util.Collections; diff --git a/clients/src/main/java/org/apache/kafka/common/requests/EndTxnRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/EndTxnRequest.java index 700d5e54a5a9f..fdfafd005ca66 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/EndTxnRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/EndTxnRequest.java @@ -21,7 +21,6 @@ import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.Message; import java.nio.ByteBuffer; @@ -61,7 +60,7 @@ public TransactionResult result() { } @Override - protected Message data() { + protected EndTxnRequestData data() { return data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/EndTxnResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/EndTxnResponse.java index 049d66e5f1d77..b782cefd50717 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/EndTxnResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/EndTxnResponse.java @@ -20,7 +20,6 @@ import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.Message; import java.nio.ByteBuffer; import java.util.Map; @@ -62,7 +61,7 @@ public Map errorCounts() { } @Override - protected Message data() { + protected EndTxnResponseData data() { return data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenRequest.java index cc3c0de894347..ac03944ab029e 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenRequest.java @@ -23,7 +23,6 @@ import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.Message; public class ExpireDelegationTokenRequest extends AbstractRequest { @@ -40,7 +39,7 @@ public static ExpireDelegationTokenRequest parse(ByteBuffer buffer, short versio } @Override - protected Message data() { + protected ExpireDelegationTokenRequestData data() { return data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenResponse.java index 2398e1bcc22ba..451c736ffb7e6 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenResponse.java @@ -23,7 +23,6 @@ import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.Message; public class ExpireDelegationTokenResponse extends AbstractResponse { @@ -53,7 +52,7 @@ public Map errorCounts() { } @Override - protected Message data() { + protected ExpireDelegationTokenResponseData data() { return data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatRequest.java index bffdf8e13dd19..754566db7a425 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatRequest.java @@ -22,7 +22,6 @@ import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.Message; import java.nio.ByteBuffer; @@ -73,7 +72,7 @@ public static HeartbeatRequest parse(ByteBuffer buffer, short version) { } @Override - protected Message data() { + protected HeartbeatRequestData data() { return data; } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatResponse.java index 9e37ecf30d016..c8fdae2507c6b 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatResponse.java @@ -20,7 +20,6 @@ import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.Message; import java.nio.ByteBuffer; import java.util.Map; @@ -59,7 +58,7 @@ public Map errorCounts() { } @Override - protected Message data() { + protected HeartbeatResponseData data() { return data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/InitProducerIdRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/InitProducerIdRequest.java index c78cd31c4e567..08e75cb536c63 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/InitProducerIdRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/InitProducerIdRequest.java @@ -21,7 +21,6 @@ import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.record.RecordBatch; import java.nio.ByteBuffer; @@ -74,7 +73,7 @@ public static InitProducerIdRequest parse(ByteBuffer buffer, short version) { } @Override - protected Message data() { + protected InitProducerIdRequestData data() { return data; } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/InitProducerIdResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/InitProducerIdResponse.java index 2782c509684ed..6066c13ef5ab6 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/InitProducerIdResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/InitProducerIdResponse.java @@ -20,7 +20,6 @@ import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.Message; import java.nio.ByteBuffer; import java.util.Map; @@ -55,7 +54,7 @@ public Map errorCounts() { } @Override - protected Message data() { + protected InitProducerIdResponseData data() { return data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrResponse.java index e9587dcac712d..1e1069edea3a1 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrResponse.java @@ -21,7 +21,6 @@ import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.Message; import java.nio.ByteBuffer; import java.util.Collections; @@ -73,7 +72,7 @@ public static LeaderAndIsrResponse parse(ByteBuffer buffer, short version) { } @Override - protected Message data() { + protected LeaderAndIsrResponseData data() { return data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupResponse.java index e12a6d21ac659..3bc70674f8a75 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupResponse.java @@ -21,7 +21,6 @@ import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.Message; import java.nio.ByteBuffer; import java.util.HashMap; @@ -123,7 +122,7 @@ public Map errorCounts() { } @Override - protected Message data() { + protected LeaveGroupResponseData data() { return data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/MetadataResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/MetadataResponse.java index 7ae20a164ea86..ba06439bad1ec 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/MetadataResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/MetadataResponse.java @@ -27,7 +27,6 @@ import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.utils.Utils; import java.nio.ByteBuffer; @@ -75,7 +74,7 @@ public MetadataResponse(MetadataResponseData data, short version) { } @Override - protected Message data() { + protected MetadataResponseData data() { return data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetDeleteRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetDeleteRequest.java index 44ed5f014d492..2f9d0203a9d7a 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/OffsetDeleteRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetDeleteRequest.java @@ -21,7 +21,6 @@ import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.Message; import java.nio.ByteBuffer; @@ -72,7 +71,7 @@ public static OffsetDeleteRequest parse(ByteBuffer buffer, short version) { } @Override - protected Message data() { + protected OffsetDeleteRequestData data() { return data; } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetDeleteResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetDeleteResponse.java index 9d91bbc138d60..ceb932fb21d1e 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/OffsetDeleteResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetDeleteResponse.java @@ -20,7 +20,6 @@ import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.Message; import java.nio.ByteBuffer; import java.util.HashMap; @@ -53,7 +52,7 @@ public OffsetDeleteResponse(OffsetDeleteResponseData data) { } @Override - protected Message data() { + protected OffsetDeleteResponseData data() { return data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchRequest.java index a65892d2fb550..fd08c725399cb 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchRequest.java @@ -23,7 +23,6 @@ import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.Message; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -176,7 +175,7 @@ public boolean isAllPartitions() { } @Override - protected Message data() { + protected OffsetFetchRequestData data() { return data; } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchResponse.java index 6ed2155392a7e..3e30968715f8d 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchResponse.java @@ -23,7 +23,6 @@ import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.Message; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -220,7 +219,7 @@ public static OffsetFetchResponse parse(ByteBuffer buffer, short version) { } @Override - protected Message data() { + protected OffsetFetchResponseData data() { return data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ProduceRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ProduceRequest.java index 615e3d5514828..8b96b82005212 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ProduceRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ProduceRequest.java @@ -25,7 +25,6 @@ import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.SendBuilder; import org.apache.kafka.common.record.BaseRecords; import org.apache.kafka.common.record.CompressionType; @@ -167,7 +166,7 @@ public ProduceRequestData dataOrException() { } @Override - protected Message data() { + protected ProduceRequestData data() { return data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/RenewDelegationTokenResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/RenewDelegationTokenResponse.java index 8f33f7d7be692..2ef840378acb3 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/RenewDelegationTokenResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/RenewDelegationTokenResponse.java @@ -23,7 +23,6 @@ import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.Message; public class RenewDelegationTokenResponse extends AbstractResponse { @@ -45,7 +44,7 @@ public Map errorCounts() { } @Override - protected Message data() { + protected RenewDelegationTokenResponseData data() { return data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/SaslAuthenticateResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/SaslAuthenticateResponse.java index eb0fd42c1eada..2e1a2189a39be 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/SaslAuthenticateResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/SaslAuthenticateResponse.java @@ -20,7 +20,6 @@ import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.Message; import java.nio.ByteBuffer; import java.util.Map; @@ -69,7 +68,7 @@ public int throttleTimeMs() { } @Override - protected Message data() { + protected SaslAuthenticateResponseData data() { return data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/SaslHandshakeResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/SaslHandshakeResponse.java index 3863c186f5089..b0ac2efcdec5f 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/SaslHandshakeResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/SaslHandshakeResponse.java @@ -20,7 +20,6 @@ import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.Message; import java.nio.ByteBuffer; import java.util.List; @@ -59,7 +58,7 @@ public int throttleTimeMs() { } @Override - protected Message data() { + protected SaslHandshakeResponseData data() { return data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaResponse.java index 5336d296ecfed..27a2502e75103 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaResponse.java @@ -21,7 +21,6 @@ import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.Message; import java.nio.ByteBuffer; import java.util.Collections; @@ -72,7 +71,7 @@ public int throttleTimeMs() { } @Override - protected Message data() { + protected StopReplicaResponseData data() { return data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/SyncGroupRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/SyncGroupRequest.java index ceecc376bf026..18e5eb8b80964 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/SyncGroupRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/SyncGroupRequest.java @@ -22,7 +22,6 @@ import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.Message; import java.nio.ByteBuffer; import java.util.HashMap; @@ -93,7 +92,7 @@ public static SyncGroupRequest parse(ByteBuffer buffer, short version) { } @Override - protected Message data() { + protected SyncGroupRequestData data() { return data; } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/SyncGroupResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/SyncGroupResponse.java index c35f9178bf17d..a26b6c900bbdf 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/SyncGroupResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/SyncGroupResponse.java @@ -20,7 +20,6 @@ import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.Message; import java.nio.ByteBuffer; import java.util.Map; @@ -49,7 +48,7 @@ public Map errorCounts() { } @Override - protected Message data() { + protected SyncGroupResponseData data() { return data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitRequest.java index b1a13fcf72dfa..055793cf54b15 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitRequest.java @@ -27,7 +27,6 @@ import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.record.RecordBatch; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -169,7 +168,7 @@ static List getTopics(Map Date: Thu, 3 Dec 2020 06:48:30 -0800 Subject: [PATCH 32/51] Complete implementation of DescribeConfigsResponse --- .../requests/DescribeConfigsResponse.java | 45 ++++++++++--------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeConfigsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeConfigsResponse.java index b9a6717195c1a..aa7a713e8ab4e 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeConfigsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeConfigsResponse.java @@ -220,29 +220,30 @@ public Map re public DescribeConfigsResponse(DescribeConfigsResponseData data) { super(ApiKeys.DESCRIBE_CONFIGS); this.data = data; - //FIXME Post-process version 0 } -// public DescribeConfigsResponse(Struct struct, short version) { -// this.data = new DescribeConfigsResponseData(struct, version); -// if (version == 0) { -// for (DescribeConfigsResult result : data.results()) { -// for (DescribeConfigsResponseData.DescribeConfigsResourceResult config : result.configs()) { -// if (config.isDefault()) { -// config.setConfigSource(ConfigSource.DEFAULT_CONFIG.id); -// } else { -// if (result.resourceType() == ConfigResource.Type.BROKER.id()) { -// config.setConfigSource(ConfigSource.STATIC_BROKER_CONFIG.id); -// } else if (result.resourceType() == ConfigResource.Type.TOPIC.id()) { -// config.setConfigSource(ConfigSource.TOPIC_CONFIG.id); -// } else { -// config.setConfigSource(ConfigSource.UNKNOWN.id); -// } -// } -// } -// } -// } -// } + // This constructor should only be used after deserialization, it has special handling for version 0 + private DescribeConfigsResponse(DescribeConfigsResponseData data, short version) { + super(ApiKeys.DESCRIBE_CONFIGS); + this.data = data; + if (version == 0) { + for (DescribeConfigsResponseData.DescribeConfigsResult result : data.results()) { + for (DescribeConfigsResponseData.DescribeConfigsResourceResult config : result.configs()) { + if (config.isDefault()) { + config.setConfigSource(ConfigSource.DEFAULT_CONFIG.id); + } else { + if (result.resourceType() == ConfigResource.Type.BROKER.id()) { + config.setConfigSource(ConfigSource.STATIC_BROKER_CONFIG.id); + } else if (result.resourceType() == ConfigResource.Type.TOPIC.id()) { + config.setConfigSource(ConfigSource.TOPIC_CONFIG.id); + } else { + config.setConfigSource(ConfigSource.UNKNOWN.id); + } + } + } + } + } + } @Override public DescribeConfigsResponseData data() { @@ -264,7 +265,7 @@ public Map errorCounts() { } public static DescribeConfigsResponse parse(ByteBuffer buffer, short version) { - return new DescribeConfigsResponse(new DescribeConfigsResponseData(new ByteBufferAccessor(buffer), version)); + return new DescribeConfigsResponse(new DescribeConfigsResponseData(new ByteBufferAccessor(buffer), version), version); } @Override From ff108fa54b279eff97dfb1cf23c66c6d1b8631dd Mon Sep 17 00:00:00 2001 From: Ismael Juma Date: Thu, 3 Dec 2020 08:35:22 -0800 Subject: [PATCH 33/51] Fix existing bug in ProduceRequest uncovered by AbstractResponse.toString The test for this functionality was broken before the version parameter was removed from AbstractResponse.toString. --- .../org/apache/kafka/common/requests/ProduceRequest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ProduceRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ProduceRequest.java index 8b96b82005212..8eabd9c3b85ff 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ProduceRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ProduceRequest.java @@ -190,7 +190,7 @@ public String toString(boolean verbose) { public ProduceResponse getErrorResponse(int throttleTimeMs, Throwable e) { /* In case the producer doesn't actually want any response */ if (acks == 0) return null; - Errors error = Errors.forException(e); + ApiError apiError = ApiError.fromThrowable(e); ProduceResponseData data = new ProduceResponseData().setThrottleTimeMs(throttleTimeMs); partitionSizes().forEach((tp, ignored) -> { ProduceResponseData.TopicProduceResponse tpr = data.responses().find(tp.topic()); @@ -204,8 +204,8 @@ public ProduceResponse getErrorResponse(int throttleTimeMs, Throwable e) { .setBaseOffset(INVALID_OFFSET) .setLogAppendTimeMs(RecordBatch.NO_TIMESTAMP) .setLogStartOffset(INVALID_OFFSET) - .setErrorMessage(e.getMessage()) - .setErrorCode(error.code())); + .setErrorMessage(apiError.message()) + .setErrorCode(apiError.error().code())); }); return new ProduceResponse(data); } From bf7f76d0e7cefbf9595bc6dc202e35841fcdabac Mon Sep 17 00:00:00 2001 From: Ismael Juma Date: Thu, 3 Dec 2020 08:35:39 -0800 Subject: [PATCH 34/51] Style fix --- .../java/org/apache/kafka/common/requests/FetchResponse.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/common/requests/FetchResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/FetchResponse.java index c658e97b41204..62631173715db 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/FetchResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/FetchResponse.java @@ -317,10 +317,7 @@ public Map errorCounts() { } public static FetchResponse parse(ByteBuffer buffer, short version) { - FetchResponseData fetchResponseData = new FetchResponseData(); - ByteBufferAccessor reader = new ByteBufferAccessor(buffer); - fetchResponseData.read(reader, version); - return new FetchResponse<>(fetchResponseData); + return new FetchResponse<>(new FetchResponseData(new ByteBufferAccessor(buffer), version)); } @SuppressWarnings("unchecked") From 0ad34fb67c33537d26dda2ddc1c061b610538473 Mon Sep 17 00:00:00 2001 From: Ismael Juma Date: Thu, 3 Dec 2020 08:37:07 -0800 Subject: [PATCH 35/51] Fix NPEs in DeleteAclsResponse and DescribeAclsResponse --- .../org/apache/kafka/common/requests/DeleteAclsResponse.java | 2 +- .../apache/kafka/common/requests/DescribeAclsResponse.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DeleteAclsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DeleteAclsResponse.java index fed627a62d42c..3ff8a9834f888 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DeleteAclsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DeleteAclsResponse.java @@ -46,8 +46,8 @@ public class DeleteAclsResponse extends AbstractResponse { public DeleteAclsResponse(DeleteAclsResponseData data, short version) { super(ApiKeys.DELETE_ACLS); - validate(version); this.data = data; + validate(version); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeAclsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeAclsResponse.java index 1562b76e83d5b..43439a429eaeb 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeAclsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeAclsResponse.java @@ -50,15 +50,15 @@ public class DescribeAclsResponse extends AbstractResponse { public DescribeAclsResponse(DescribeAclsResponseData data, short version) { super(ApiKeys.DESCRIBE_ACLS); - validate(Optional.of(version)); this.data = data; + validate(Optional.of(version)); } // Skips version validation, should only be called when `version` is not available and the input is known to be valid DescribeAclsResponse(DescribeAclsResponseData data) { super(ApiKeys.DESCRIBE_ACLS); - validate(Optional.empty()); this.data = data; + validate(Optional.empty()); } @Override From dee68746b7b95547d4c56345fabdfa7c9cddd52c Mon Sep 17 00:00:00 2001 From: Ismael Juma Date: Thu, 3 Dec 2020 08:45:26 -0800 Subject: [PATCH 36/51] RequestResponseTest fixes --- .../common/requests/RequestResponseTest.java | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java index 4bd6686f77391..54a0a440357ec 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java @@ -96,6 +96,7 @@ import org.apache.kafka.common.message.ExpireDelegationTokenRequestData; import org.apache.kafka.common.message.ExpireDelegationTokenResponseData; import org.apache.kafka.common.message.FetchRequestData; +import org.apache.kafka.common.message.FetchResponseData; import org.apache.kafka.common.message.FindCoordinatorRequestData; import org.apache.kafka.common.message.HeartbeatRequestData; import org.apache.kafka.common.message.HeartbeatResponseData; @@ -159,8 +160,6 @@ import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.ObjectSerializationCache; -import org.apache.kafka.common.protocol.types.SchemaException; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.quota.ClientQuotaAlteration; import org.apache.kafka.common.quota.ClientQuotaEntity; import org.apache.kafka.common.quota.ClientQuotaFilter; @@ -184,6 +183,7 @@ import org.apache.kafka.test.TestUtils; import org.junit.Test; +import java.nio.BufferUnderflowException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; @@ -514,6 +514,7 @@ public void testResponseHeader() { ObjectSerializationCache serializationCache = new ObjectSerializationCache(); ByteBuffer buffer = ByteBuffer.allocate(header.size(serializationCache)); header.write(buffer, serializationCache); + buffer.flip(); ResponseHeader deserialized = ResponseHeader.parse(buffer, header.headerVersion()); assertEquals(header.correlationId(), deserialized.correlationId()); } @@ -560,14 +561,13 @@ private void verifyDescribeConfigsResponse(DescribeConfigsResponse expected, Des } } - private void checkDescribeConfigsResponseVersions() throws Exception { + private void checkDescribeConfigsResponseVersions() { for (int version = ApiKeys.DESCRIBE_CONFIGS.oldestVersion(); version < ApiKeys.DESCRIBE_CONFIGS.latestVersion(); ++version) { short apiVersion = (short) version; DescribeConfigsResponse response = createDescribeConfigsResponse(apiVersion); DescribeConfigsResponse deserialized0 = (DescribeConfigsResponse) AbstractResponse.parseResponse(ApiKeys.DESCRIBE_CONFIGS, response.serializeBody(apiVersion), apiVersion); verifyDescribeConfigsResponse(response, deserialized0, apiVersion); - } } @@ -589,9 +589,9 @@ private void checkRequest(AbstractRequest req, boolean checkEquality) { ByteBuffer serializedBytes = req.serializeBody(); AbstractRequest deserialized = AbstractRequest.parseRequest(req.apiKey(), req.version(), serializedBytes).request; ByteBuffer serializedBytes2 = deserialized.serializeBody(); - assertEquals(req.data(), deserialized.data()); //FIXME + serializedBytes.rewind(); if (checkEquality) - assertEquals(serializedBytes, serializedBytes2); + assertEquals("Request " + req + "failed equality test", serializedBytes, serializedBytes2); } catch (Exception e) { throw new RuntimeException("Failed to deserialize request " + req + " with type " + req.getClass(), e); } @@ -605,8 +605,9 @@ private void checkResponse(AbstractResponse response, int version, boolean check ByteBuffer serializedBytes = response.serializeBody((short) version); AbstractResponse deserialized = AbstractResponse.parseResponse(response.apiKey(), serializedBytes, (short) version); ByteBuffer serializedBytes2 = deserialized.serializeBody((short) version); + serializedBytes.rewind(); if (checkEquality) - assertEquals(serializedBytes, serializedBytes2); + assertEquals("Response " + response + "failed equality test", serializedBytes, serializedBytes2); } catch (Exception e) { throw new RuntimeException("Failed to deserialize response " + response + " with type " + response.getClass(), e); } @@ -761,11 +762,10 @@ private void verifyFetchResponseFullWrite(short apiVersion, FetchResponse ApiVersionsResponse.parse(ByteBuffer.allocate(0), version)); + assertThrows(BufferUnderflowException.class, () -> ApiVersionsResponse.parse(ByteBuffer.allocate(0), version)); } @Test From b17d1bf41f3d94eb5667c2ef6f512218e149bef3 Mon Sep 17 00:00:00 2001 From: Ismael Juma Date: Thu, 3 Dec 2020 09:53:28 -0800 Subject: [PATCH 37/51] Fix NetworkClientTest failures --- .../java/org/apache/kafka/clients/NetworkClientTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java b/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java index 48c1ed6919503..dd7ca94c45f08 100644 --- a/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java @@ -200,7 +200,7 @@ private void checkSimpleRequestResponse(NetworkClient networkClient) { assertEquals(1, networkClient.inFlightRequestCount()); ProduceResponse produceResponse = new ProduceResponse(new ProduceResponseData()); ByteBuffer buffer = produceResponse.serializeWithHeader(requestVersion, request.correlationId()); - buffer.flip(); + buffer.rewind(); selector.completeReceive(new NetworkReceive(node.idString(), buffer)); List responses = networkClient.poll(1, time.milliseconds()); assertEquals(1, responses.size()); @@ -509,7 +509,7 @@ public void testConnectionThrottling() { int throttleTime = 100; ProduceResponse produceResponse = new ProduceResponse(new ProduceResponseData().setThrottleTimeMs(throttleTime)); ByteBuffer buffer = produceResponse.serializeWithHeader(requestVersion, request.correlationId()); - buffer.flip(); + buffer.rewind(); selector.completeReceive(new NetworkReceive(node.idString(), buffer)); client.poll(1, time.milliseconds()); @@ -590,7 +590,7 @@ private int sendEmptyProduceRequest(String nodeId) { private void sendResponse(AbstractResponse response, short version, int correlationId) { ByteBuffer buffer = response.serializeWithHeader(version, correlationId); - buffer.flip(); + buffer.rewind(); selector.completeReceive(new NetworkReceive(node.idString(), buffer)); } From 830d46c9aa635f7d9681a1b9ef46fedfd935647d Mon Sep 17 00:00:00 2001 From: Ismael Juma Date: Thu, 3 Dec 2020 09:58:52 -0800 Subject: [PATCH 38/51] Fix KafkaApisTest failures and rename errors to errorsByProducerId for clarity --- .../kafka/common/requests/WriteTxnMarkersResponse.java | 2 +- .../kafka/common/requests/WriteTxnMarkersRequestTest.java | 2 +- .../common/requests/WriteTxnMarkersResponseTest.java | 4 ++-- .../TransactionMarkerRequestCompletionHandler.scala | 2 +- .../integration/kafka/api/AuthorizerIntegrationTest.scala | 2 +- core/src/test/scala/unit/kafka/server/KafkaApisTest.scala | 8 ++++---- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/common/requests/WriteTxnMarkersResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/WriteTxnMarkersResponse.java index 7fef4ac6f0f80..fd2a834d24569 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/WriteTxnMarkersResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/WriteTxnMarkersResponse.java @@ -88,7 +88,7 @@ public WriteTxnMarkersResponseData data() { return data; } - public Map> errors() { + public Map> errorsByProducerId() { Map> errors = new HashMap<>(); for (WritableTxnMarkerResult marker : data.markers()) { Map topicPartitionErrorsMap = new HashMap<>(); diff --git a/clients/src/test/java/org/apache/kafka/common/requests/WriteTxnMarkersRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/WriteTxnMarkersRequestTest.java index 7e76ed72b8cce..12fa71b92e015 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/WriteTxnMarkersRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/WriteTxnMarkersRequestTest.java @@ -72,7 +72,7 @@ public void testGetErrorResponse() { request.getErrorResponse(throttleTimeMs, Errors.UNKNOWN_PRODUCER_ID.exception()); assertEquals(Collections.singletonMap( - topicPartition, Errors.UNKNOWN_PRODUCER_ID), errorResponse.errors().get(producerId)); + topicPartition, Errors.UNKNOWN_PRODUCER_ID), errorResponse.errorsByProducerId().get(producerId)); assertEquals(Collections.singletonMap(Errors.UNKNOWN_PRODUCER_ID, 1), errorResponse.errorCounts()); // Write txn marker has no throttle time defined in response. assertEquals(0, errorResponse.throttleTimeMs()); diff --git a/clients/src/test/java/org/apache/kafka/common/requests/WriteTxnMarkersResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/WriteTxnMarkersResponseTest.java index c24c39ff04ece..3860b186f072e 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/WriteTxnMarkersResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/WriteTxnMarkersResponseTest.java @@ -54,7 +54,7 @@ public void testConstructor() { expectedErrorCounts.put(Errors.INVALID_PRODUCER_EPOCH, 1); WriteTxnMarkersResponse response = new WriteTxnMarkersResponse(errorMap); assertEquals(expectedErrorCounts, response.errorCounts()); - assertEquals(Collections.singletonMap(tp1, pidOneError), response.errors().get(producerIdOne)); - assertEquals(Collections.singletonMap(tp2, pidTwoError), response.errors().get(producerIdTwo)); + assertEquals(Collections.singletonMap(tp1, pidOneError), response.errorsByProducerId().get(producerIdOne)); + assertEquals(Collections.singletonMap(tp2, pidTwoError), response.errorsByProducerId().get(producerIdTwo)); } } diff --git a/core/src/main/scala/kafka/coordinator/transaction/TransactionMarkerRequestCompletionHandler.scala b/core/src/main/scala/kafka/coordinator/transaction/TransactionMarkerRequestCompletionHandler.scala index 2a20008ab5a3e..848e0fa65ceeb 100644 --- a/core/src/main/scala/kafka/coordinator/transaction/TransactionMarkerRequestCompletionHandler.scala +++ b/core/src/main/scala/kafka/coordinator/transaction/TransactionMarkerRequestCompletionHandler.scala @@ -89,7 +89,7 @@ class TransactionMarkerRequestCompletionHandler(brokerId: Int, val writeTxnMarkerResponse = response.responseBody.asInstanceOf[WriteTxnMarkersResponse] - val responseErrors = writeTxnMarkerResponse.errors; + val responseErrors = writeTxnMarkerResponse.errorsByProducerId; for (txnIdAndMarker <- txnIdAndMarkerEntries.asScala) { val transactionalId = txnIdAndMarker.txnId val txnMarker = txnIdAndMarker.txnMarkerEntry diff --git a/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala index 2e2a03c2eda77..21e59e3bfbc8b 100644 --- a/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala +++ b/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala @@ -195,7 +195,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest { ApiKeys.ALTER_CONFIGS -> ((resp: AlterConfigsResponse) => resp.errors.get(new ConfigResource(ConfigResource.Type.TOPIC, tp.topic)).error), ApiKeys.INIT_PRODUCER_ID -> ((resp: InitProducerIdResponse) => resp.error), - ApiKeys.WRITE_TXN_MARKERS -> ((resp: WriteTxnMarkersResponse) => resp.errors.get(producerId).get(tp)), + ApiKeys.WRITE_TXN_MARKERS -> ((resp: WriteTxnMarkersResponse) => resp.errorsByProducerId.get(producerId).get(tp)), ApiKeys.ADD_PARTITIONS_TO_TXN -> ((resp: AddPartitionsToTxnResponse) => resp.errors.get(tp)), ApiKeys.ADD_OFFSETS_TO_TXN -> ((resp: AddOffsetsToTxnResponse) => Errors.forCode(resp.data.errorCode)), ApiKeys.END_TXN -> ((resp: EndTxnResponse) => resp.error), diff --git a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala index b3bd2c536e120..b89fb2efdb1b3 100644 --- a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala +++ b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala @@ -1358,7 +1358,7 @@ class KafkaApisTest { val markersResponse = readResponse(writeTxnMarkersRequest, capturedResponse) .asInstanceOf[WriteTxnMarkersResponse] - assertEquals(expectedErrors, markersResponse.errors.get(1)) + assertEquals(expectedErrors, markersResponse.errorsByProducerId.get(1L)) } @Test @@ -1377,7 +1377,7 @@ class KafkaApisTest { val markersResponse = readResponse(writeTxnMarkersRequest, capturedResponse) .asInstanceOf[WriteTxnMarkersResponse] - assertEquals(expectedErrors, markersResponse.errors.get(1)) + assertEquals(expectedErrors, markersResponse.errorsByProducerId.get(1L)) } @Test @@ -1412,7 +1412,7 @@ class KafkaApisTest { val markersResponse = readResponse(writeTxnMarkersRequest, capturedResponse) .asInstanceOf[WriteTxnMarkersResponse] - assertEquals(expectedErrors, markersResponse.errors.get(1)) + assertEquals(expectedErrors, markersResponse.errorsByProducerId.get(1L)) EasyMock.verify(replicaManager) } @@ -1549,7 +1549,7 @@ class KafkaApisTest { val markersResponse = readResponse(writeTxnMarkersRequest, capturedResponse) .asInstanceOf[WriteTxnMarkersResponse] - assertEquals(expectedErrors, markersResponse.errors.get(1)) + assertEquals(expectedErrors, markersResponse.errorsByProducerId.get(1L)) EasyMock.verify(replicaManager) } From 8e719a47cde7a4ab6110760b26eb3692a4cbd668 Mon Sep 17 00:00:00 2001 From: Ismael Juma Date: Thu, 3 Dec 2020 09:59:02 -0800 Subject: [PATCH 39/51] Minor style fix --- .../org/apache/kafka/clients/admin/KafkaAdminClientTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java index 3508a291d3f7f..c380477ab98e0 100644 --- a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java @@ -4935,7 +4935,7 @@ protected Message data() { @Override public int throttleTimeMs() { - return 0; + return DEFAULT_THROTTLE_TIME; } }; From de995be3fb26a6e1b0bec05f4a3eda3474006bb7 Mon Sep 17 00:00:00 2001 From: Ismael Juma Date: Thu, 3 Dec 2020 10:34:42 -0800 Subject: [PATCH 40/51] Fix RequestResponseTest and error message in DescribeClientQuotasRequest --- .../requests/DescribeClientQuotasRequest.java | 5 +++-- .../common/requests/RequestResponseTest.java | 22 ++++++++++--------- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasRequest.java index dc8064652f9b2..d0aeff881242a 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasRequest.java @@ -113,10 +113,11 @@ protected DescribeClientQuotasRequestData data() { @Override public DescribeClientQuotasResponse getErrorResponse(int throttleTimeMs, Throwable e) { + ApiError error = ApiError.fromThrowable(e); return new DescribeClientQuotasResponse(new DescribeClientQuotasResponseData() .setThrottleTimeMs(throttleTimeMs) - .setErrorCode(Errors.forException(e).code()) - .setErrorMessage(e.getMessage()) + .setErrorCode(error.error().code()) + .setErrorMessage(error.message()) .setEntries(null)); } diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java index 54a0a440357ec..17df012aa9f7f 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java @@ -190,7 +190,6 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; @@ -530,12 +529,15 @@ private void checkOlderFetchVersions() { } } - private void verifyDescribeConfigsResponse(DescribeConfigsResponse expected, DescribeConfigsResponse actual, int version) throws Exception { + private void verifyDescribeConfigsResponse(DescribeConfigsResponse expected, DescribeConfigsResponse actual, + int version) { for (Map.Entry resource : expected.resultMap().entrySet()) { List actualEntries = actual.resultMap().get(resource.getKey()).configs(); - Iterator expectedEntries = expected.resultMap().get(resource.getKey()).configs().iterator(); - for (DescribeConfigsResourceResult actualEntry : actualEntries) { - DescribeConfigsResourceResult expectedEntry = expectedEntries.next(); + List expectedEntries = expected.resultMap().get(resource.getKey()).configs(); + assertEquals(expectedEntries.size(), actualEntries.size()); + for (int i = 0; i < actualEntries.size(); ++i) { + DescribeConfigsResourceResult actualEntry = actualEntries.get(i); + DescribeConfigsResourceResult expectedEntry = expectedEntries.get(i); assertEquals(expectedEntry.name(), actualEntry.name()); assertEquals("Non-matching values for " + actualEntry.name() + " in version " + version, expectedEntry.value(), actualEntry.value()); @@ -550,13 +552,13 @@ private void verifyDescribeConfigsResponse(DescribeConfigsResponse expected, Des assertEquals("Non-matching configType for " + actualEntry.name() + " in version " + version, expectedEntry.configType(), actualEntry.configType()); } - if (version == 1 || version == 3 || (expectedEntry.configSource() != DescribeConfigsResponse.ConfigSource.DYNAMIC_BROKER_CONFIG.id() && - expectedEntry.configSource() != DescribeConfigsResponse.ConfigSource.DYNAMIC_DEFAULT_BROKER_CONFIG.id())) - assertEquals("Non-matching configSource for " + actualEntry.name() + " in version " + version, - expectedEntry.configSource(), actualEntry.configSource()); - else + if (version == 0) { assertEquals("Non matching configSource for " + actualEntry.name() + " in version " + version, DescribeConfigsResponse.ConfigSource.STATIC_BROKER_CONFIG.id(), actualEntry.configSource()); + } else { + assertEquals("Non-matching configSource for " + actualEntry.name() + " in version " + version, + expectedEntry.configSource(), actualEntry.configSource()); + } } } } From ce9bd0b549b086db38eade56660bc5787b0b67b8 Mon Sep 17 00:00:00 2001 From: Ismael Juma Date: Thu, 3 Dec 2020 10:41:06 -0800 Subject: [PATCH 41/51] Fix DeleteAclsResponseTest --- .../requests/DeleteAclsResponseTest.java | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/clients/src/test/java/org/apache/kafka/common/requests/DeleteAclsResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/DeleteAclsResponseTest.java index 0013e01a87131..96f04ebf29e61 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/DeleteAclsResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/DeleteAclsResponseTest.java @@ -19,6 +19,7 @@ import org.apache.kafka.common.acl.AclOperation; import org.apache.kafka.common.acl.AclPermissionType; +import org.apache.kafka.common.errors.UnsupportedVersionException; import org.apache.kafka.common.message.DeleteAclsResponseData; import org.apache.kafka.common.message.DeleteAclsResponseData.DeleteAclsFilterResult; import org.apache.kafka.common.message.DeleteAclsResponseData.DeleteAclsMatchingAcl; @@ -82,22 +83,22 @@ public class DeleteAclsResponseTest { private static final DeleteAclsFilterResult UNKNOWN_RESPONSE = new DeleteAclsFilterResult().setMatchingAcls(asList( UNKNOWN_ACL)); + @Test public void shouldThrowOnV0IfNotLiteral() { - DeleteAclsResponse response = new DeleteAclsResponse( - new DeleteAclsResponseData() - .setThrottleTimeMs(10) - .setFilterResults(singletonList(PREFIXED_RESPONSE)), - V0); - assertThrows(IllegalArgumentException.class, () -> response.serializeBody(V0)); + assertThrows(UnsupportedVersionException.class, () -> new DeleteAclsResponse( + new DeleteAclsResponseData() + .setThrottleTimeMs(10) + .setFilterResults(singletonList(PREFIXED_RESPONSE)), + V0)); } + @Test public void shouldThrowOnIfUnknown() { - DeleteAclsResponse response = new DeleteAclsResponse( - new DeleteAclsResponseData() - .setThrottleTimeMs(10) - .setFilterResults(singletonList(UNKNOWN_RESPONSE)), - V1); - assertThrows(IllegalArgumentException.class, () -> response.serializeBody(V1)); + assertThrows(IllegalArgumentException.class, () -> new DeleteAclsResponse( + new DeleteAclsResponseData() + .setThrottleTimeMs(10) + .setFilterResults(singletonList(UNKNOWN_RESPONSE)), + V1)); } @Test From d3c8fa6e7cd0b043b2bc59bd08d3409f733b1dc6 Mon Sep 17 00:00:00 2001 From: Ismael Juma Date: Thu, 3 Dec 2020 10:47:04 -0800 Subject: [PATCH 42/51] Fix EndTxnResponseTest --- .../kafka/common/requests/EndTxnResponseTest.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/clients/src/test/java/org/apache/kafka/common/requests/EndTxnResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/EndTxnResponseTest.java index 431424aef7f45..c1d4367c929e0 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/EndTxnResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/EndTxnResponseTest.java @@ -28,14 +28,13 @@ public class EndTxnResponseTest { - //FIXME Do we need to serialize to buffer and back? Also change the name of the test @Test - public void testConstructorWithStruct() { + public void testConstructor() { int throttleTimeMs = 10; EndTxnResponseData data = new EndTxnResponseData() - .setErrorCode(Errors.NOT_COORDINATOR.code()) - .setThrottleTimeMs(throttleTimeMs); + .setErrorCode(Errors.NOT_COORDINATOR.code()) + .setThrottleTimeMs(throttleTimeMs); Map expectedErrorCounts = Collections.singletonMap(Errors.NOT_COORDINATOR, 1); @@ -44,6 +43,11 @@ public void testConstructorWithStruct() { assertEquals(expectedErrorCounts, response.errorCounts()); assertEquals(throttleTimeMs, response.throttleTimeMs()); assertEquals(version >= 1, response.shouldClientThrottle(version)); + + response = EndTxnResponse.parse(response.serializeBody(version), version); + assertEquals(expectedErrorCounts, response.errorCounts()); + assertEquals(throttleTimeMs, response.throttleTimeMs()); + assertEquals(version >= 1, response.shouldClientThrottle(version)); } } } From b7490aff8c90d1302b3b03bf239d906099353fa4 Mon Sep 17 00:00:00 2001 From: Ismael Juma Date: Thu, 3 Dec 2020 10:52:40 -0800 Subject: [PATCH 43/51] Remove TestUtils.serializeMessage in favor of MessageTestUtil.messageToByteBuffer --- .../test/java/org/apache/kafka/clients/MetadataTest.java | 5 +++-- .../kafka/common/requests/ListOffsetRequestTest.java | 4 ++-- .../authenticator/SaslServerAuthenticatorTest.java | 4 ++-- clients/src/test/java/org/apache/kafka/test/TestUtils.java | 7 ------- 4 files changed, 7 insertions(+), 13 deletions(-) diff --git a/clients/src/test/java/org/apache/kafka/clients/MetadataTest.java b/clients/src/test/java/org/apache/kafka/clients/MetadataTest.java index 5e4086bfeeefe..58180008d668d 100644 --- a/clients/src/test/java/org/apache/kafka/clients/MetadataTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/MetadataTest.java @@ -30,6 +30,7 @@ import org.apache.kafka.common.message.MetadataResponseData.MetadataResponseTopicCollection; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.MessageTestUtil; import org.apache.kafka.common.requests.MetadataRequest; import org.apache.kafka.common.requests.MetadataResponse; import org.apache.kafka.common.utils.LogContext; @@ -196,7 +197,7 @@ public void testIgnoreLeaderEpochInOlderMetadataResponse() { .setBrokers(new MetadataResponseBrokerCollection()); for (short version = ApiKeys.METADATA.oldestVersion(); version < 9; version++) { - ByteBuffer buffer = TestUtils.serializeMessage(data, version); + ByteBuffer buffer = MessageTestUtil.messageToByteBuffer(data, version); MetadataResponse response = MetadataResponse.parse(buffer, version); assertFalse(response.hasReliableLeaderEpochs()); metadata.updateWithCurrentRequestVersion(response, false, 100); @@ -206,7 +207,7 @@ public void testIgnoreLeaderEpochInOlderMetadataResponse() { } for (short version = 9; version <= ApiKeys.METADATA.latestVersion(); version++) { - ByteBuffer buffer = TestUtils.serializeMessage(data, version); + ByteBuffer buffer = MessageTestUtil.messageToByteBuffer(data, version); MetadataResponse response = MetadataResponse.parse(buffer, version); assertTrue(response.hasReliableLeaderEpochs()); metadata.updateWithCurrentRequestVersion(response, false, 100); diff --git a/clients/src/test/java/org/apache/kafka/common/requests/ListOffsetRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/ListOffsetRequestTest.java index 837b0e4b162a4..22e7c5736286e 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/ListOffsetRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/ListOffsetRequestTest.java @@ -35,7 +35,7 @@ import org.apache.kafka.common.message.ListOffsetResponseData.ListOffsetTopicResponse; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.test.TestUtils; +import org.apache.kafka.common.protocol.MessageTestUtil; import org.junit.Test; public class ListOffsetRequestTest { @@ -53,7 +53,7 @@ public void testDuplicatePartitions() { ListOffsetRequestData data = new ListOffsetRequestData() .setTopics(topics) .setReplicaId(-1); - ListOffsetRequest request = ListOffsetRequest.parse(TestUtils.serializeMessage(data, (short) 0), (short) 0); + ListOffsetRequest request = ListOffsetRequest.parse(MessageTestUtil.messageToByteBuffer(data, (short) 0), (short) 0); assertEquals(Collections.singleton(new TopicPartition("topic", 0)), request.duplicatePartitions()); } diff --git a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticatorTest.java b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticatorTest.java index c378484a47dad..eccf782db2196 100644 --- a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticatorTest.java +++ b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticatorTest.java @@ -113,7 +113,7 @@ public void testLatestApiVersionsRequest() throws IOException { "apache-kafka-java", AppInfoParser.getVersion()); } - public void testApiVersionsRequest(short version, String expectedSoftwareName, + private void testApiVersionsRequest(short version, String expectedSoftwareName, String expectedSoftwareVersion) throws IOException { TransportLayer transportLayer = mock(TransportLayer.class, Answers.RETURNS_DEEP_STUBS); Map configs = Collections.singletonMap(BrokerSecurityConfigs.SASL_ENABLED_MECHANISMS_CONFIG, @@ -126,7 +126,7 @@ public void testApiVersionsRequest(short version, String expectedSoftwareName, ByteBuffer headerBuffer = TestUtils.serializeRequestHeader(header); ApiVersionsRequest request = new ApiVersionsRequest.Builder().build(version); - ByteBuffer requestBuffer = TestUtils.serializeMessage(request.data(), version); + ByteBuffer requestBuffer = request.serializeBody(); when(transportLayer.socketChannel().socket().getInetAddress()).thenReturn(InetAddress.getLoopbackAddress()); diff --git a/clients/src/test/java/org/apache/kafka/test/TestUtils.java b/clients/src/test/java/org/apache/kafka/test/TestUtils.java index b1c4751e0ec93..045d3c142339d 100644 --- a/clients/src/test/java/org/apache/kafka/test/TestUtils.java +++ b/clients/src/test/java/org/apache/kafka/test/TestUtils.java @@ -246,13 +246,6 @@ public static ByteBuffer serializeRequestHeader(RequestHeader header) { return buffer; } - public static ByteBuffer serializeMessage(Message message, short version) { - ObjectSerializationCache serializationCache = new ObjectSerializationCache(); - ByteBuffer buffer = ByteBuffer.allocate(message.size(serializationCache, version)); - message.write(new ByteBufferAccessor(buffer), serializationCache, version); - return buffer; - } - @FunctionalInterface public interface PartitionMetadataSupplier { MetadataResponse.PartitionMetadata supply(Errors error, From 55a6110d10c6d0e8f86cc0c026623857ff759c0b Mon Sep 17 00:00:00 2001 From: Ismael Juma Date: Thu, 3 Dec 2020 10:57:59 -0800 Subject: [PATCH 44/51] Fix LeaveGroupResponseTest --- .../common/requests/LeaveGroupResponseTest.java | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/clients/src/test/java/org/apache/kafka/common/requests/LeaveGroupResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/LeaveGroupResponseTest.java index 6a49944291f8a..99c863627b9f2 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/LeaveGroupResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/LeaveGroupResponseTest.java @@ -105,21 +105,19 @@ public void testShouldThrottle() { } @Test - public void testEqualityWithStruct() { + public void testEqualityWithSerialization() { LeaveGroupResponseData responseData = new LeaveGroupResponseData() - .setErrorCode(Errors.NONE.code()) - .setThrottleTimeMs(throttleTimeMs); + .setErrorCode(Errors.NONE.code()) + .setThrottleTimeMs(throttleTimeMs); for (short version = 0; version <= ApiKeys.LEAVE_GROUP.latestVersion(); version++) { - LeaveGroupResponse primaryResponse = new LeaveGroupResponse( - new LeaveGroupResponseData(responseData.toStruct(version), version)); - - LeaveGroupResponse secondaryResponse = new LeaveGroupResponse( - new LeaveGroupResponseData(responseData.toStruct(version), version)); + LeaveGroupResponse primaryResponse = LeaveGroupResponse.parse( + MessageTestUtil.messageToByteBuffer(responseData, version), version); + LeaveGroupResponse secondaryResponse = LeaveGroupResponse.parse( + MessageTestUtil.messageToByteBuffer(responseData, version), version); assertEquals(primaryResponse, primaryResponse); assertEquals(primaryResponse, secondaryResponse); assertEquals(primaryResponse.hashCode(), secondaryResponse.hashCode()); - } } From 7ad198f22c823838efb05164c14d386e1beb973d Mon Sep 17 00:00:00 2001 From: Ismael Juma Date: Thu, 3 Dec 2020 11:02:40 -0800 Subject: [PATCH 45/51] Minor fixes --- .../kafka/common/requests/DescribeClientQuotasRequest.java | 1 - .../apache/kafka/common/requests/OffsetFetchResponseTest.java | 3 +-- clients/src/test/java/org/apache/kafka/test/TestUtils.java | 2 -- 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasRequest.java index d0aeff881242a..1f167b0ee3c2b 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasRequest.java @@ -21,7 +21,6 @@ import org.apache.kafka.common.message.DescribeClientQuotasResponseData; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; -import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.quota.ClientQuotaFilter; import org.apache.kafka.common.quota.ClientQuotaFilterComponent; diff --git a/clients/src/test/java/org/apache/kafka/common/requests/OffsetFetchResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/OffsetFetchResponseTest.java index 61451837148f0..f20a74412880a 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/OffsetFetchResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/OffsetFetchResponseTest.java @@ -108,8 +108,7 @@ public void testStructBuild() { for (short version = 0; version <= ApiKeys.OFFSET_FETCH.latestVersion(); version++) { Struct struct = latestResponse.data.toStruct(version); - OffsetFetchResponse oldResponse = new OffsetFetchResponse(new OffsetFetchResponseData(struct, version), - version); + OffsetFetchResponse oldResponse = OffsetFetchResponse.parse(latestResponse.serializeBody(version), version); if (version <= 1) { assertFalse(struct.hasField(ERROR_CODE)); diff --git a/clients/src/test/java/org/apache/kafka/test/TestUtils.java b/clients/src/test/java/org/apache/kafka/test/TestUtils.java index 045d3c142339d..f751617eff305 100644 --- a/clients/src/test/java/org/apache/kafka/test/TestUtils.java +++ b/clients/src/test/java/org/apache/kafka/test/TestUtils.java @@ -28,9 +28,7 @@ import org.apache.kafka.common.network.NetworkReceive; import org.apache.kafka.common.network.Send; import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.ObjectSerializationCache; import org.apache.kafka.common.record.RecordBatch; import org.apache.kafka.common.requests.ByteBufferChannel; From cfbbede920fbb3c5f521ecc972dd56e0ba82416a Mon Sep 17 00:00:00 2001 From: Ismael Juma Date: Thu, 3 Dec 2020 11:50:12 -0800 Subject: [PATCH 46/51] Test fixes/improvements --- .../common/requests/DescribeAclsRequest.java | 2 +- .../common/requests/DescribeAclsResponse.java | 2 +- .../common/requests/ListOffsetRequest.java | 12 ++++++- .../clients/admin/KafkaAdminClientTest.java | 6 +++- .../common/requests/RequestHeaderTest.java | 16 +++++----- .../requests/StopReplicaRequestTest.java | 17 ---------- .../requests/TxnOffsetCommitResponseTest.java | 31 +++++++++++++++++++ .../SaslServerAuthenticatorTest.java | 7 +++-- .../java/org/apache/kafka/test/TestUtils.java | 1 + 9 files changed, 61 insertions(+), 33 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeAclsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeAclsRequest.java index 3f058ebcb6419..e6f9c38fd598b 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeAclsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeAclsRequest.java @@ -100,7 +100,7 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable throwable .setThrottleTimeMs(throttleTimeMs) .setErrorCode(error.error().code()) .setErrorMessage(error.message()); - return new DescribeAclsResponse(response); + return new DescribeAclsResponse(response, version()); } public static DescribeAclsRequest parse(ByteBuffer buffer, short version) { diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeAclsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeAclsResponse.java index 43439a429eaeb..4308c9ea45bf7 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeAclsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeAclsResponse.java @@ -54,7 +54,7 @@ public DescribeAclsResponse(DescribeAclsResponseData data, short version) { validate(Optional.of(version)); } - // Skips version validation, should only be called when `version` is not available and the input is known to be valid + // Skips version validation, visible for testing DescribeAclsResponse(DescribeAclsResponseData data) { super(ApiKeys.DESCRIBE_ACLS); this.data = data; diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetRequest.java index f3e1da48afcad..9ebb06bdd6511 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetRequest.java @@ -20,6 +20,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -94,7 +95,16 @@ public String toString() { private ListOffsetRequest(ListOffsetRequestData data, short version) { super(ApiKeys.LIST_OFFSETS, version); this.data = data; - this.duplicatePartitions = Collections.emptySet(); + duplicatePartitions = new HashSet<>(); + Set partitions = new HashSet<>(); + for (ListOffsetTopic topic : data.topics()) { + for (ListOffsetPartition partition : topic.partitions()) { + TopicPartition tp = new TopicPartition(topic.name(), partition.partitionIndex()); + if (!partitions.add(tp)) { + duplicatePartitions.add(tp); + } + } + } } @Override diff --git a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java index c380477ab98e0..071120e94015b 100644 --- a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java @@ -1261,7 +1261,11 @@ public void testDeleteAcls() throws Exception { DeleteAclsResponse.matchingAcl(ACL1, ApiError.NONE), new DeleteAclsResponseData.DeleteAclsMatchingAcl() .setErrorCode(Errors.SECURITY_DISABLED.code()) - .setErrorMessage("No security"))), + .setErrorMessage("No security") + .setPermissionType(AclPermissionType.ALLOW.code()) + .setOperation(AclOperation.ALTER.code()) + .setResourceType(ResourceType.CLUSTER.code()) + .setPatternType(FILTER2.patternFilter().patternType().code()))), new DeleteAclsResponseData.DeleteAclsFilterResult())), ApiKeys.DELETE_ACLS.latestVersion())); results = env.adminClient().deleteAcls(asList(FILTER1, FILTER2)); diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestHeaderTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestHeaderTest.java index c6dbbab8585f3..8cf5fb2c0b72b 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/RequestHeaderTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestHeaderTest.java @@ -45,7 +45,7 @@ public void testSerdeControlledShutdownV0() { assertEquals("", deserialized.clientId()); assertEquals(0, deserialized.headerVersion()); - ByteBuffer serializedBuffer = toBuffer(deserialized); + ByteBuffer serializedBuffer = TestUtils.serializeRequestHeader(deserialized); assertEquals(ApiKeys.CONTROLLED_SHUTDOWN.id, serializedBuffer.getShort(0)); assertEquals(0, serializedBuffer.getShort(2)); @@ -59,7 +59,7 @@ public void testRequestHeaderV1() { RequestHeader header = new RequestHeader(ApiKeys.FIND_COORDINATOR, apiVersion, "", 10); assertEquals(1, header.headerVersion()); - ByteBuffer buffer = toBuffer(header); + ByteBuffer buffer = TestUtils.serializeRequestHeader(header); assertEquals(10, buffer.remaining()); RequestHeader deserialized = RequestHeader.parse(buffer); assertEquals(header, deserialized); @@ -71,7 +71,7 @@ public void testRequestHeaderV2() { RequestHeader header = new RequestHeader(ApiKeys.CREATE_DELEGATION_TOKEN, apiVersion, "", 10); assertEquals(2, header.headerVersion()); - ByteBuffer buffer = toBuffer(header); + ByteBuffer buffer = TestUtils.serializeRequestHeader(header); assertEquals(11, buffer.remaining()); RequestHeader deserialized = RequestHeader.parse(buffer); assertEquals(header, deserialized); @@ -83,7 +83,10 @@ public void parseHeaderFromBufferWithNonZeroPosition() { buffer.position(10); RequestHeader header = new RequestHeader(ApiKeys.FIND_COORDINATOR, (short) 1, "", 10); - header.write(buffer, new ObjectSerializationCache()); + ObjectSerializationCache serializationCache = new ObjectSerializationCache(); + // size must be called before write to avoid an NPE with the current implementation + header.size(serializationCache); + header.write(buffer, serializationCache); int limit = buffer.position(); buffer.position(10); buffer.limit(limit); @@ -91,9 +94,4 @@ public void parseHeaderFromBufferWithNonZeroPosition() { RequestHeader parsed = RequestHeader.parse(buffer); assertEquals(header, parsed); } - - private ByteBuffer toBuffer(RequestHeader header) { - return TestUtils.serializeRequestHeader(header); - - } } diff --git a/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaRequestTest.java index 31a32ef82f052..243efa9bbf4b9 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaRequestTest.java @@ -26,7 +26,6 @@ import org.apache.kafka.common.message.StopReplicaRequestData.StopReplicaTopicState; import org.apache.kafka.common.message.StopReplicaResponseData.StopReplicaPartitionError; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.test.TestUtils; import org.junit.Test; import java.util.ArrayList; @@ -36,7 +35,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.stream.Collectors; import static org.apache.kafka.common.protocol.ApiKeys.STOP_REPLICA; import static org.junit.Assert.assertEquals; @@ -80,21 +78,6 @@ public void testGetErrorResponse() { } @Test - public void testStopReplicaRequestNormalization() { - List topicStates = TestUtils.generateRandomTopicPartitions(10, 10).stream() - .collect(Collectors.groupingBy(tp -> tp.topic())) - .entrySet() - .stream() - .map(entry -> new StopReplicaTopicState() - .setTopicName(entry.getKey()) - .setPartitionStates(entry.getValue().stream() - .map(tp -> new StopReplicaPartitionState().setPartitionIndex(tp.partition())) - .collect(Collectors.toList()))) - .collect(Collectors.toList()); - StopReplicaRequest.Builder builder = new StopReplicaRequest.Builder((short) 5, 0, 0, 0, false, topicStates); - assertTrue(builder.build((short) 1).sizeInBytes() < builder.build((short) 0).sizeInBytes()); - } - public void testBuilderNormalizationWithAllDeletePartitionEqualToTrue() { testBuilderNormalization(true); } diff --git a/clients/src/test/java/org/apache/kafka/common/requests/TxnOffsetCommitResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/TxnOffsetCommitResponseTest.java index 7e35465f39bd2..bf3e9c8d2115e 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/TxnOffsetCommitResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/TxnOffsetCommitResponseTest.java @@ -16,8 +16,14 @@ */ package org.apache.kafka.common.requests; +import org.apache.kafka.common.message.TxnOffsetCommitResponseData; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.MessageTestUtil; import org.junit.Test; +import java.util.Arrays; +import java.util.Collections; + import static org.junit.Assert.assertEquals; public class TxnOffsetCommitResponseTest extends OffsetCommitResponseTest { @@ -32,4 +38,29 @@ public void testConstructorWithErrorResponse() { assertEquals(throttleTimeMs, response.throttleTimeMs()); } + @Test + @Override + public void testParse() { + TxnOffsetCommitResponseData data = new TxnOffsetCommitResponseData() + .setThrottleTimeMs(throttleTimeMs) + .setTopics(Arrays.asList( + new TxnOffsetCommitResponseData.TxnOffsetCommitResponseTopic().setPartitions( + Collections.singletonList(new TxnOffsetCommitResponseData.TxnOffsetCommitResponsePartition() + .setPartitionIndex(partitionOne) + .setErrorCode(errorOne.code()))), + new TxnOffsetCommitResponseData.TxnOffsetCommitResponseTopic().setPartitions( + Collections.singletonList(new TxnOffsetCommitResponseData.TxnOffsetCommitResponsePartition() + .setPartitionIndex(partitionTwo) + .setErrorCode(errorTwo.code()))) + )); + + for (short version = 0; version <= ApiKeys.TXN_OFFSET_COMMIT.latestVersion(); version++) { + TxnOffsetCommitResponse response = TxnOffsetCommitResponse.parse( + MessageTestUtil.messageToByteBuffer(data, version), version); + assertEquals(expectedErrorCounts, response.errorCounts()); + assertEquals(throttleTimeMs, response.throttleTimeMs()); + assertEquals(version >= 1, response.shouldClientThrottle(version)); + } + } + } diff --git a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticatorTest.java b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticatorTest.java index eccf782db2196..a4d2d771f5c17 100644 --- a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticatorTest.java +++ b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticatorTest.java @@ -127,6 +127,7 @@ private void testApiVersionsRequest(short version, String expectedSoftwareName, ApiVersionsRequest request = new ApiVersionsRequest.Builder().build(version); ByteBuffer requestBuffer = request.serializeBody(); + requestBuffer.rewind(); when(transportLayer.socketChannel().socket().getInetAddress()).thenReturn(InetAddress.getLoopbackAddress()); @@ -134,9 +135,9 @@ private void testApiVersionsRequest(short version, String expectedSoftwareName, invocation.getArgument(0).putInt(headerBuffer.remaining() + requestBuffer.remaining()); return 4; }).then(invocation -> { - ByteBuffer buffer = invocation.getArgument(0).put(headerBuffer.duplicate()); - buffer.put(headerBuffer.duplicate()); - buffer.put(requestBuffer.duplicate()); + invocation.getArgument(0) + .put(headerBuffer.duplicate()) + .put(requestBuffer.duplicate()); return headerBuffer.remaining() + requestBuffer.remaining(); }); diff --git a/clients/src/test/java/org/apache/kafka/test/TestUtils.java b/clients/src/test/java/org/apache/kafka/test/TestUtils.java index f751617eff305..50a79c7dbb19e 100644 --- a/clients/src/test/java/org/apache/kafka/test/TestUtils.java +++ b/clients/src/test/java/org/apache/kafka/test/TestUtils.java @@ -241,6 +241,7 @@ public static ByteBuffer serializeRequestHeader(RequestHeader header) { ObjectSerializationCache serializationCache = new ObjectSerializationCache(); ByteBuffer buffer = ByteBuffer.allocate(header.size(serializationCache)); header.write(buffer, serializationCache); + buffer.rewind(); return buffer; } From 4829f7cb7e55bab44f4686851bad84f36a8c2cc2 Mon Sep 17 00:00:00 2001 From: Ismael Juma Date: Thu, 3 Dec 2020 21:10:31 -0800 Subject: [PATCH 47/51] Fix MetadataRequestTest failure --- clients/src/main/resources/common/message/MetadataRequest.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clients/src/main/resources/common/message/MetadataRequest.json b/clients/src/main/resources/common/message/MetadataRequest.json index 9ee2bc2541624..6129d427b7eef 100644 --- a/clients/src/main/resources/common/message/MetadataRequest.json +++ b/clients/src/main/resources/common/message/MetadataRequest.json @@ -36,7 +36,7 @@ { "name": "Name", "type": "string", "versions": "0+", "entityType": "topicName", "about": "The topic name." } ]}, - { "name": "AllowAutoTopicCreation", "type": "bool", "versions": "4+", "default": "true", "ignorable": true, + { "name": "AllowAutoTopicCreation", "type": "bool", "versions": "4+", "default": "true", "ignorable": false, "about": "If this is true, the broker may auto-create topics that we requested which do not already exist, if it is configured to do so." }, { "name": "IncludeClusterAuthorizedOperations", "type": "bool", "versions": "8+", "about": "Whether to include cluster authorized operations." }, From 420a74dce8c10c76f61430917d7a050f47e542d6 Mon Sep 17 00:00:00 2001 From: Ismael Juma Date: Thu, 3 Dec 2020 22:31:31 -0800 Subject: [PATCH 48/51] Use SendBuilder in AbstractRequest/Response Also consolidate and clean-up. --- .../kafka/common/protocol/SendBuilder.java | 14 +++---- .../common/requests/AbstractRequest.java | 28 ++++--------- .../common/requests/AbstractResponse.java | 29 ++++---------- .../common/requests/EnvelopeRequest.java | 8 ---- .../common/requests/EnvelopeResponse.java | 7 ---- .../kafka/common/requests/FetchResponse.java | 7 ---- .../kafka/common/requests/ProduceRequest.java | 15 +------ .../common/requests/ProduceResponse.java | 7 ---- .../kafka/common/requests/RequestUtils.java | 40 ++++++++++++++++++- .../producer/internals/SenderTest.java | 2 +- .../internals/TransactionManagerTest.java | 2 +- .../common/requests/RequestResponseTest.java | 4 +- .../main/scala/kafka/server/KafkaApis.scala | 4 +- 13 files changed, 69 insertions(+), 98 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/SendBuilder.java b/clients/src/main/java/org/apache/kafka/common/protocol/SendBuilder.java index 19d5600da89b8..6ed4c19a5719b 100644 --- a/clients/src/main/java/org/apache/kafka/common/protocol/SendBuilder.java +++ b/clients/src/main/java/org/apache/kafka/common/protocol/SendBuilder.java @@ -22,6 +22,7 @@ import org.apache.kafka.common.record.MemoryRecords; import org.apache.kafka.common.record.MultiRecordsSend; import org.apache.kafka.common.requests.RequestHeader; +import org.apache.kafka.common.requests.RequestUtils; import org.apache.kafka.common.requests.ResponseHeader; import org.apache.kafka.common.utils.ByteUtils; @@ -181,7 +182,7 @@ public Send build() { public static Send buildRequestSend( String destination, RequestHeader header, - ApiMessage apiRequest + Message apiRequest ) { return buildSend( destination, @@ -195,7 +196,7 @@ public static Send buildRequestSend( public static Send buildResponseSend( String destination, ResponseHeader header, - ApiMessage apiResponse, + Message apiResponse, short apiVersion ) { return buildSend( @@ -209,16 +210,13 @@ public static Send buildResponseSend( private static Send buildSend( String destination, - ApiMessage header, + Message header, short headerVersion, - ApiMessage apiMessage, + Message apiMessage, short apiVersion ) { ObjectSerializationCache serializationCache = new ObjectSerializationCache(); - MessageSizeAccumulator messageSize = new MessageSizeAccumulator(); - - header.addSize(messageSize, serializationCache, headerVersion); - apiMessage.addSize(messageSize, serializationCache, apiVersion); + MessageSizeAccumulator messageSize = RequestUtils.size(serializationCache, header, headerVersion, apiMessage, apiVersion); int totalSize = messageSize.totalSize(); int sizeExcludingZeroCopyFields = totalSize - messageSize.zeroCopySize(); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java index f2e2d4c6c2236..f85d88d08900d 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java @@ -17,13 +17,12 @@ package org.apache.kafka.common.requests; import org.apache.kafka.common.errors.UnsupportedVersionException; -import org.apache.kafka.common.network.NetworkSend; import org.apache.kafka.common.network.Send; import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.protocol.ObjectSerializationCache; +import org.apache.kafka.common.protocol.SendBuilder; import java.nio.ByteBuffer; import java.util.Map; @@ -98,35 +97,24 @@ public ApiKeys apiKey() { return apiKey; } - public Send toSend(String destination, RequestHeader header) { - return new NetworkSend(destination, serializeWithHeader(header)); + public final Send toSend(String destination, RequestHeader header) { + return SendBuilder.buildRequestSend(destination, header, data()); } // Visible for testing - public ByteBuffer serializeWithHeader(RequestHeader header) { - return serialize(header, data(), version); + public final ByteBuffer serializeWithHeader(RequestHeader header) { + return RequestUtils.serialize(header.data(), header.headerVersion(), data(), version); } protected abstract Message data(); // Visible for testing - public ByteBuffer serializeBody() { - return serialize(null, data(), version); - } - - private static ByteBuffer serialize(RequestHeader header, Message data, short version) { - ObjectSerializationCache serializationCache = new ObjectSerializationCache(); - int headerSize = header == null ? 0 : header.size(serializationCache); - ByteBuffer buffer = ByteBuffer.allocate(headerSize + data.size(serializationCache, version)); - if (header != null) - header.write(buffer, serializationCache); - data.write(new ByteBufferAccessor(buffer), serializationCache, version); - buffer.rewind(); - return buffer; + public final ByteBuffer serializeBody() { + return RequestUtils.serialize(null, (short) 0, data(), version); } // Visible for testing - int sizeInBytes() { + final int sizeInBytes() { return data().size(new ObjectSerializationCache(), version); } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java index b71eb4c118674..e3caf3e1b64f8 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java @@ -16,13 +16,11 @@ */ package org.apache.kafka.common.requests; -import org.apache.kafka.common.network.NetworkSend; import org.apache.kafka.common.network.Send; import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.Message; -import org.apache.kafka.common.protocol.ObjectSerializationCache; +import org.apache.kafka.common.protocol.SendBuilder; import java.nio.ByteBuffer; import java.util.Collection; @@ -42,36 +40,25 @@ protected AbstractResponse(ApiKeys apiKey) { this.apiKey = apiKey; } - protected Send toSend(String destination, ResponseHeader header, short version) { - return new NetworkSend(destination, serializeWithHeader(header, version)); + public final Send toSend(String destination, ResponseHeader header, short version) { + return SendBuilder.buildResponseSend(destination, header, data(), version); } /** * Visible for testing, typically {@link #toSend(String, ResponseHeader, short)} should be used instead. */ - public ByteBuffer serializeWithHeader(short version, int correlationId) { + public final ByteBuffer serializeWithHeader(short version, int correlationId) { return serializeWithHeader(new ResponseHeader(correlationId, apiKey.responseHeaderVersion(version)), version); } - ByteBuffer serializeWithHeader(ResponseHeader header, short version) { + final ByteBuffer serializeWithHeader(ResponseHeader header, short version) { Objects.requireNonNull(header, "header should not be null"); - return serialize(header, data(), version); + return RequestUtils.serialize(header.data(), header.headerVersion(), data(), version); } // Visible for testing - ByteBuffer serializeBody(short version) { - return serialize(null, data(), version); - } - - private static ByteBuffer serialize(ResponseHeader header, Message data, short version) { - ObjectSerializationCache serializationCache = new ObjectSerializationCache(); - int headerSize = header == null ? 0 : header.size(serializationCache); - ByteBuffer buffer = ByteBuffer.allocate(headerSize + data.size(serializationCache, version)); - if (header != null) - header.write(buffer, serializationCache); - data.write(new ByteBufferAccessor(buffer), serializationCache, version); - buffer.rewind(); - return buffer; + final ByteBuffer serializeBody(short version) { + return RequestUtils.serialize(null, (short) 0, data(), version); } /** diff --git a/clients/src/main/java/org/apache/kafka/common/requests/EnvelopeRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/EnvelopeRequest.java index b9e98d40aa610..4be5259458b6d 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/EnvelopeRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/EnvelopeRequest.java @@ -18,9 +18,7 @@ import org.apache.kafka.common.message.EnvelopeRequestData; import org.apache.kafka.common.message.EnvelopeResponseData; -import org.apache.kafka.common.network.Send; import org.apache.kafka.common.protocol.ByteBufferAccessor; -import org.apache.kafka.common.protocol.SendBuilder; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; @@ -85,10 +83,4 @@ public static EnvelopeRequest parse(ByteBuffer buffer, short version) { public EnvelopeRequestData data() { return data; } - - @Override - public Send toSend(String destination, RequestHeader header) { - return SendBuilder.buildRequestSend(destination, header, this.data); - } - } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/EnvelopeResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/EnvelopeResponse.java index 6064bc3400630..c7c8d24e4a338 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/EnvelopeResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/EnvelopeResponse.java @@ -17,10 +17,8 @@ package org.apache.kafka.common.requests; import org.apache.kafka.common.message.EnvelopeResponseData; -import org.apache.kafka.common.network.Send; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; -import org.apache.kafka.common.protocol.SendBuilder; import org.apache.kafka.common.protocol.Errors; import java.nio.ByteBuffer; @@ -68,11 +66,6 @@ public int throttleTimeMs() { return DEFAULT_THROTTLE_TIME; } - @Override - protected Send toSend(String destination, ResponseHeader header, short apiVersion) { - return SendBuilder.buildResponseSend(destination, header, this.data, apiVersion); - } - public static EnvelopeResponse parse(ByteBuffer buffer, short version) { return new EnvelopeResponse(new EnvelopeResponseData(new ByteBufferAccessor(buffer), version)); } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/FetchResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/FetchResponse.java index 62631173715db..d65a55e6c2b29 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/FetchResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/FetchResponse.java @@ -18,12 +18,10 @@ import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.message.FetchResponseData; -import org.apache.kafka.common.network.Send; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.ObjectSerializationCache; -import org.apache.kafka.common.protocol.SendBuilder; import org.apache.kafka.common.record.BaseRecords; import org.apache.kafka.common.record.MemoryRecords; @@ -284,11 +282,6 @@ public FetchResponse(FetchResponseData fetchResponseData) { this.responseDataMap = toResponseDataMap(fetchResponseData); } - @Override - public Send toSend(String dest, ResponseHeader responseHeader, short apiVersion) { - return SendBuilder.buildResponseSend(dest, responseHeader, this.data, apiVersion); - } - public Errors error() { return Errors.forCode(data.errorCode()); } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ProduceRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ProduceRequest.java index 8eabd9c3b85ff..758631a1d87aa 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ProduceRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ProduceRequest.java @@ -21,11 +21,9 @@ import org.apache.kafka.common.errors.UnsupportedCompressionTypeException; import org.apache.kafka.common.message.ProduceRequestData; import org.apache.kafka.common.message.ProduceResponseData; -import org.apache.kafka.common.network.Send; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.SendBuilder; import org.apache.kafka.common.record.BaseRecords; import org.apache.kafka.common.record.CompressionType; import org.apache.kafka.common.record.RecordBatch; @@ -129,11 +127,6 @@ public ProduceRequest(ProduceRequestData produceRequestData, short version) { this.transactionalId = data.transactionalId(); } - @Override - public Send toSend(String destination, RequestHeader header) { - return SendBuilder.buildRequestSend(destination, header, dataOrException()); - } - // visible for testing Map partitionSizes() { if (partitionSizes == null) { @@ -157,7 +150,8 @@ Map partitionSizes() { /** * @return data or IllegalStateException if the data is removed (to prevent unnecessary memory retention). */ - public ProduceRequestData dataOrException() { + @Override + public ProduceRequestData data() { // Store it in a local variable to protect against concurrent updates ProduceRequestData tmp = data; if (tmp == null) @@ -165,11 +159,6 @@ public ProduceRequestData dataOrException() { return tmp; } - @Override - protected ProduceRequestData data() { - return data; - } - @Override public String toString(boolean verbose) { // Use the same format as `Struct.toString()` diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ProduceResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ProduceResponse.java index 13d2c818220c4..2860576d53972 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ProduceResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ProduceResponse.java @@ -18,11 +18,9 @@ import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.message.ProduceResponseData; -import org.apache.kafka.common.network.Send; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.SendBuilder; import org.apache.kafka.common.record.RecordBatch; import java.nio.ByteBuffer; @@ -83,11 +81,6 @@ public ProduceResponse(Map responses, int thr this(toData(responses, throttleTimeMs)); } - @Override - protected Send toSend(String destination, ResponseHeader header, short apiVersion) { - return SendBuilder.buildResponseSend(destination, header, this.data, apiVersion); - } - private static ProduceResponseData toData(Map responses, int throttleTimeMs) { ProduceResponseData data = new ProduceResponseData().setThrottleTimeMs(throttleTimeMs); responses.forEach((tp, response) -> { diff --git a/clients/src/main/java/org/apache/kafka/common/requests/RequestUtils.java b/clients/src/main/java/org/apache/kafka/common/requests/RequestUtils.java index b8c920e639c80..4fa701f76476d 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/RequestUtils.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/RequestUtils.java @@ -17,10 +17,15 @@ package org.apache.kafka.common.requests; import org.apache.kafka.common.message.ProduceRequestData; +import org.apache.kafka.common.protocol.ByteBufferAccessor; +import org.apache.kafka.common.protocol.Message; +import org.apache.kafka.common.protocol.MessageSizeAccumulator; +import org.apache.kafka.common.protocol.ObjectSerializationCache; import org.apache.kafka.common.record.BaseRecords; import org.apache.kafka.common.record.RecordBatch; import org.apache.kafka.common.record.Records; +import java.nio.ByteBuffer; import java.util.AbstractMap; import java.util.Iterator; import java.util.Optional; @@ -52,7 +57,7 @@ public static boolean hasTransactionalRecords(ProduceRequest request) { public static AbstractMap.SimpleEntry flags(ProduceRequest request) { boolean hasIdempotentRecords = false; boolean hasTransactionalRecords = false; - for (ProduceRequestData.TopicProduceData tpd : request.dataOrException().topicData()) { + for (ProduceRequestData.TopicProduceData tpd : request.data().topicData()) { for (ProduceRequestData.PartitionProduceData ppd : tpd.partitionData()) { BaseRecords records = ppd.records(); if (records instanceof Records) { @@ -70,4 +75,37 @@ public static AbstractMap.SimpleEntry flags(ProduceRequest req } return new AbstractMap.SimpleEntry<>(hasIdempotentRecords, hasTransactionalRecords); } + + public static MessageSizeAccumulator size( + ObjectSerializationCache serializationCache, + Message header, + short headerVersion, + Message apiMessage, + short apiVersion + ) { + MessageSizeAccumulator messageSize = new MessageSizeAccumulator(); + if (header != null) + header.addSize(messageSize, serializationCache, headerVersion); + apiMessage.addSize(messageSize, serializationCache, apiVersion); + return messageSize; + } + + public static ByteBuffer serialize( + Message header, + short headerVersion, + Message apiMessage, + short apiVersion + ) { + ObjectSerializationCache serializationCache = new ObjectSerializationCache(); + MessageSizeAccumulator messageSize = RequestUtils.size(serializationCache, header, headerVersion, apiMessage, apiVersion); + + ByteBuffer buffer = ByteBuffer.allocate(messageSize.totalSize()); + ByteBufferAccessor bufferWritable = new ByteBufferAccessor(buffer); + if (header != null) + header.write(bufferWritable, serializationCache, headerVersion); + apiMessage.write(bufferWritable, serializationCache, apiVersion); + + buffer.rewind(); + return buffer; + } } diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java index 0618a71dc4635..6ded3f72de291 100644 --- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java @@ -160,7 +160,7 @@ public void tearDown() { private static Map partitionRecords(ProduceRequest request) { Map partitionRecords = new HashMap<>(); - request.dataOrException().topicData().forEach(tpData -> tpData.partitionData().forEach(p -> { + request.data().topicData().forEach(tpData -> tpData.partitionData().forEach(p -> { TopicPartition tp = new TopicPartition(tpData.name(), p.index()); partitionRecords.put(tp, (MemoryRecords) p.records()); })); diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java index cd71aabb3dfd0..70b52c5df05c2 100644 --- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java @@ -3356,7 +3356,7 @@ private void prepareProduceResponse(Errors error, final long producerId, final s private MockClient.RequestMatcher produceRequestMatcher(final long producerId, final short epoch, TopicPartition tp) { return body -> { ProduceRequest produceRequest = (ProduceRequest) body; - MemoryRecords records = produceRequest.dataOrException().topicData() + MemoryRecords records = produceRequest.data().topicData() .stream() .filter(t -> t.name().equals(tp.topic())) .findAny() diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java index 8bce23cbdef93..b62376c8881d6 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java @@ -655,7 +655,7 @@ public void testPartitionSize() { @Test public void produceRequestToStringTest() { ProduceRequest request = createProduceRequest(ApiKeys.PRODUCE.latestVersion()); - assertEquals(1, request.dataOrException().topicData().size()); + assertEquals(1, request.data().topicData().size()); assertFalse(request.toString(false).contains("partitionSizes")); assertTrue(request.toString(false).contains("numPartitions=1")); assertTrue(request.toString(true).contains("partitionSizes")); @@ -663,7 +663,7 @@ public void produceRequestToStringTest() { request.clearPartitionRecords(); try { - request.dataOrException(); + request.data(); fail("dataOrException should fail after clearPartitionRecords()"); } catch (IllegalStateException e) { // OK diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala index 8d9261e40d255..6aa71505acf4f 100644 --- a/core/src/main/scala/kafka/server/KafkaApis.scala +++ b/core/src/main/scala/kafka/server/KafkaApis.scala @@ -594,9 +594,9 @@ class KafkaApis(val requestChannel: RequestChannel, val authorizedRequestInfo = mutable.Map[TopicPartition, MemoryRecords]() // cache the result to avoid redundant authorization calls val authorizedTopics = filterByAuthorized(request.context, WRITE, TOPIC, - produceRequest.dataOrException().topicData().asScala)(_.name()) + produceRequest.data().topicData().asScala)(_.name()) - produceRequest.dataOrException.topicData.forEach(topic => topic.partitionData.forEach { partition => + produceRequest.data.topicData.forEach(topic => topic.partitionData.forEach { partition => val topicPartition = new TopicPartition(topic.name, partition.index) // This caller assumes the type is MemoryRecords and that is true on current serialization // We cast the type to avoid causing big change to code base. From cfd0c6602c241082e80e59ee7c9c0ee55f193328 Mon Sep 17 00:00:00 2001 From: Ismael Juma Date: Thu, 3 Dec 2020 22:53:00 -0800 Subject: [PATCH 49/51] Fix request size returned by parseRequest --- .../java/org/apache/kafka/common/requests/AbstractRequest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java index f85d88d08900d..e86cd54224ad7 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java @@ -155,7 +155,7 @@ public Map errorCounts(Throwable e) { * Factory method for getting a request object based on ApiKey ID and a version */ public static RequestAndSize parseRequest(ApiKeys apiKey, short apiVersion, ByteBuffer buffer) { - int bufferSize = buffer.position() - buffer.limit(); + int bufferSize = buffer.remaining(); return new RequestAndSize(doParseRequest(apiKey, apiVersion, buffer), bufferSize); } From de6dca14966338db718ebe6da7054bd703121dae Mon Sep 17 00:00:00 2001 From: Ismael Juma Date: Thu, 3 Dec 2020 23:05:46 -0800 Subject: [PATCH 50/51] Revert correlation id verification order change --- .../org/apache/kafka/common/requests/AbstractResponse.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java index e3caf3e1b64f8..747698166359c 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java @@ -106,10 +106,7 @@ public static AbstractResponse parseResponse(ByteBuffer buffer, RequestHeader re short apiVersion = requestHeader.apiVersion(); ResponseHeader responseHeader = ResponseHeader.parse(buffer, apiKey.responseHeaderVersion(apiVersion)); - AbstractResponse response = AbstractResponse.parseResponse(apiKey, buffer, apiVersion); - // We correlate after parsing the response to avoid spurious correlation errors when receiving malformed - // responses if (requestHeader.correlationId() != responseHeader.correlationId()) { throw new CorrelationIdMismatchException("Correlation id for response (" + responseHeader.correlationId() + ") does not match request (" @@ -117,7 +114,7 @@ public static AbstractResponse parseResponse(ByteBuffer buffer, RequestHeader re requestHeader.correlationId(), responseHeader.correlationId()); } - return response; + return AbstractResponse.parseResponse(apiKey, buffer, apiVersion); } public static AbstractResponse parseResponse(ApiKeys apiKey, ByteBuffer responseBuffer, short version) { From ad0e0d534090ffe2bec563592f556f4e2c7ed06a Mon Sep 17 00:00:00 2001 From: Ismael Juma Date: Thu, 3 Dec 2020 23:05:59 -0800 Subject: [PATCH 51/51] Revert MetadataResponse.json change --- .../src/main/resources/common/message/MetadataResponse.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clients/src/main/resources/common/message/MetadataResponse.json b/clients/src/main/resources/common/message/MetadataResponse.json index bf753e3f1d068..19faecff80d22 100644 --- a/clients/src/main/resources/common/message/MetadataResponse.json +++ b/clients/src/main/resources/common/message/MetadataResponse.json @@ -82,9 +82,9 @@ "about": "The set of offline replicas of this partition." } ]}, { "name": "TopicAuthorizedOperations", "type": "int32", "versions": "8+", "default": "-2147483648", - "ignorable": "true", "about": "32-bit bitfield to represent authorized operations for this topic." } + "about": "32-bit bitfield to represent authorized operations for this topic." } ]}, { "name": "ClusterAuthorizedOperations", "type": "int32", "versions": "8+", "default": "-2147483648", - "ignorable": "true", "about": "32-bit bitfield to represent authorized operations for this cluster." } + "about": "32-bit bitfield to represent authorized operations for this cluster." } ] }