diff --git a/checkstyle/import-control.xml b/checkstyle/import-control.xml index ceee02adffa50..5368ef217bfbf 100644 --- a/checkstyle/import-control.xml +++ b/checkstyle/import-control.xml @@ -318,6 +318,7 @@ + @@ -329,6 +330,12 @@ + + + + + + diff --git a/clients/src/main/java/org/apache/kafka/clients/ClientRequest.java b/clients/src/main/java/org/apache/kafka/clients/ClientRequest.java index cf3b911495cb5..abba79567eb1e 100644 --- a/clients/src/main/java/org/apache/kafka/clients/ClientRequest.java +++ b/clients/src/main/java/org/apache/kafka/clients/ClientRequest.java @@ -83,14 +83,14 @@ public ApiKeys apiKey() { } public RequestHeader makeHeader(short version) { - short requestApiKey = requestBuilder.apiKey().id; + ApiKeys requestApiKey = apiKey(); return new RequestHeader( new RequestHeaderData() - .setRequestApiKey(requestApiKey) + .setRequestApiKey(requestApiKey.id) .setRequestApiVersion(version) .setClientId(clientId) .setCorrelationId(correlationId), - ApiKeys.forId(requestApiKey).requestHeaderVersion(version)); + requestApiKey.requestHeaderVersion(version)); } public AbstractRequest.Builder requestBuilder() { diff --git a/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java b/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java index 7ec3b93be4b2e..99bfd85d98530 100644 --- a/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java +++ b/clients/src/main/java/org/apache/kafka/clients/NetworkClient.java @@ -870,9 +870,8 @@ private void handleCompletedReceives(List responses, long now) { InFlightRequest req = inFlightRequests.completeNext(source); AbstractResponse response = parseResponse(receive.payload(), req.header); - if (throttleTimeSensor != null) { - throttleTimeSensor.record(response.throttleTimeMs()); - } + if (throttleTimeSensor != null) + throttleTimeSensor.record(response.throttleTimeMs(), now); if (log.isDebugEnabled()) { log.debug("Received {} response from node {} for request with header {}: {}", diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java index 9c931cd7556f7..cf0a6f6949f90 100644 --- a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java +++ b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java @@ -1182,8 +1182,7 @@ private void handleResponses(long now, List responses) { try { call.handleResponse(response.responseBody()); if (log.isTraceEnabled()) - log.trace("{} got response {}", call, - response.responseBody().toString(response.requestHeader().apiVersion())); + log.trace("{} got response {}", call, response.responseBody()); } catch (Throwable t) { if (log.isTraceEnabled()) log.trace("{} handleResponse failed with {}", call, prettyPrintException(t)); diff --git a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java index af6e262078d7a..4ca71bb626a06 100644 --- a/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java +++ b/clients/src/main/java/org/apache/kafka/clients/consumer/internals/AbstractCoordinator.java @@ -465,7 +465,13 @@ boolean joinGroupIfNeeded(final Timer timer) { } } else { final RuntimeException exception = future.exception(); - log.info("Rebalance failed.", exception); + + // we do not need to log error for memberId required, + // since it is not really an error and is transient + if (!(exception instanceof MemberIdRequiredException)) { + log.info("Rebalance failed.", exception); + } + resetJoinGroupFuture(); if (exception instanceof UnknownMemberIdException || exception instanceof RebalanceInProgressException || diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/SendBuilder.java b/clients/src/main/java/org/apache/kafka/common/protocol/SendBuilder.java index 19d5600da89b8..6ed4c19a5719b 100644 --- a/clients/src/main/java/org/apache/kafka/common/protocol/SendBuilder.java +++ b/clients/src/main/java/org/apache/kafka/common/protocol/SendBuilder.java @@ -22,6 +22,7 @@ import org.apache.kafka.common.record.MemoryRecords; import org.apache.kafka.common.record.MultiRecordsSend; import org.apache.kafka.common.requests.RequestHeader; +import org.apache.kafka.common.requests.RequestUtils; import org.apache.kafka.common.requests.ResponseHeader; import org.apache.kafka.common.utils.ByteUtils; @@ -181,7 +182,7 @@ public Send build() { public static Send buildRequestSend( String destination, RequestHeader header, - ApiMessage apiRequest + Message apiRequest ) { return buildSend( destination, @@ -195,7 +196,7 @@ public static Send buildRequestSend( public static Send buildResponseSend( String destination, ResponseHeader header, - ApiMessage apiResponse, + Message apiResponse, short apiVersion ) { return buildSend( @@ -209,16 +210,13 @@ public static Send buildResponseSend( private static Send buildSend( String destination, - ApiMessage header, + Message header, short headerVersion, - ApiMessage apiMessage, + Message apiMessage, short apiVersion ) { ObjectSerializationCache serializationCache = new ObjectSerializationCache(); - MessageSizeAccumulator messageSize = new MessageSizeAccumulator(); - - header.addSize(messageSize, serializationCache, headerVersion); - apiMessage.addSize(messageSize, serializationCache, apiVersion); + MessageSizeAccumulator messageSize = RequestUtils.size(serializationCache, header, headerVersion, apiMessage, apiVersion); int totalSize = messageSize.totalSize(); int sizeExcludingZeroCopyFields = totalSize - messageSize.zeroCopySize(); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java index a0121666dfcf2..e86cd54224ad7 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java @@ -17,14 +17,12 @@ package org.apache.kafka.common.requests; import org.apache.kafka.common.errors.UnsupportedVersionException; -import org.apache.kafka.common.message.FetchRequestData; -import org.apache.kafka.common.message.AlterIsrRequestData; -import org.apache.kafka.common.message.ProduceRequestData; -import org.apache.kafka.common.network.NetworkSend; import org.apache.kafka.common.network.Send; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.protocol.Message; +import org.apache.kafka.common.protocol.ObjectSerializationCache; +import org.apache.kafka.common.protocol.SendBuilder; import java.nio.ByteBuffer; import java.util.Map; @@ -79,13 +77,13 @@ public T build() { } private final short version; - public final ApiKeys api; + private final ApiKeys apiKey; - public AbstractRequest(ApiKeys api, short version) { - if (!api.isVersionSupported(version)) - throw new UnsupportedVersionException("The " + api + " protocol does not support version " + version); + public AbstractRequest(ApiKeys apiKey, short version) { + if (!apiKey.isVersionSupported(version)) + throw new UnsupportedVersionException("The " + apiKey + " protocol does not support version " + version); this.version = version; - this.api = api; + this.apiKey = apiKey; } /** @@ -95,21 +93,33 @@ public short version() { return version; } - public Send toSend(String destination, RequestHeader header) { - return new NetworkSend(destination, serialize(header)); + public ApiKeys apiKey() { + return apiKey; } - /** - * Use with care, typically {@link #toSend(String, RequestHeader)} should be used instead. - */ - public ByteBuffer serialize(RequestHeader header) { - return RequestUtils.serialize(header.toStruct(), toStruct()); + public final Send toSend(String destination, RequestHeader header) { + return SendBuilder.buildRequestSend(destination, header, data()); + } + + // Visible for testing + public final ByteBuffer serializeWithHeader(RequestHeader header) { + return RequestUtils.serialize(header.data(), header.headerVersion(), data(), version); + } + + protected abstract Message data(); + + // Visible for testing + public final ByteBuffer serializeBody() { + return RequestUtils.serialize(null, (short) 0, data(), version); } - protected abstract Struct toStruct(); + // Visible for testing + final int sizeInBytes() { + return data().size(new ObjectSerializationCache(), version); + } public String toString(boolean verbose) { - return toStruct().toString(); + return data().toString(); } @Override @@ -144,126 +154,131 @@ public Map errorCounts(Throwable e) { /** * Factory method for getting a request object based on ApiKey ID and a version */ - public static AbstractRequest parseRequest(ApiKeys apiKey, short apiVersion, Struct struct) { + public static RequestAndSize parseRequest(ApiKeys apiKey, short apiVersion, ByteBuffer buffer) { + int bufferSize = buffer.remaining(); + return new RequestAndSize(doParseRequest(apiKey, apiVersion, buffer), bufferSize); + } + + private static AbstractRequest doParseRequest(ApiKeys apiKey, short apiVersion, ByteBuffer buffer) { switch (apiKey) { case PRODUCE: - return new ProduceRequest(new ProduceRequestData(struct, apiVersion), apiVersion); + return ProduceRequest.parse(buffer, apiVersion); case FETCH: - return new FetchRequest(new FetchRequestData(struct, apiVersion), apiVersion); + return FetchRequest.parse(buffer, apiVersion); case LIST_OFFSETS: - return new ListOffsetRequest(struct, apiVersion); + return ListOffsetRequest.parse(buffer, apiVersion); case METADATA: - return new MetadataRequest(struct, apiVersion); + return MetadataRequest.parse(buffer, apiVersion); case OFFSET_COMMIT: - return new OffsetCommitRequest(struct, apiVersion); + return OffsetCommitRequest.parse(buffer, apiVersion); case OFFSET_FETCH: - return new OffsetFetchRequest(struct, apiVersion); + return OffsetFetchRequest.parse(buffer, apiVersion); case FIND_COORDINATOR: - return new FindCoordinatorRequest(struct, apiVersion); + return FindCoordinatorRequest.parse(buffer, apiVersion); case JOIN_GROUP: - return new JoinGroupRequest(struct, apiVersion); + return JoinGroupRequest.parse(buffer, apiVersion); case HEARTBEAT: - return new HeartbeatRequest(struct, apiVersion); + return HeartbeatRequest.parse(buffer, apiVersion); case LEAVE_GROUP: - return new LeaveGroupRequest(struct, apiVersion); + return LeaveGroupRequest.parse(buffer, apiVersion); case SYNC_GROUP: - return new SyncGroupRequest(struct, apiVersion); + return SyncGroupRequest.parse(buffer, apiVersion); case STOP_REPLICA: - return new StopReplicaRequest(struct, apiVersion); + return StopReplicaRequest.parse(buffer, apiVersion); case CONTROLLED_SHUTDOWN: - return new ControlledShutdownRequest(struct, apiVersion); + return ControlledShutdownRequest.parse(buffer, apiVersion); case UPDATE_METADATA: - return new UpdateMetadataRequest(struct, apiVersion); + return UpdateMetadataRequest.parse(buffer, apiVersion); case LEADER_AND_ISR: - return new LeaderAndIsrRequest(struct, apiVersion); + return LeaderAndIsrRequest.parse(buffer, apiVersion); case DESCRIBE_GROUPS: - return new DescribeGroupsRequest(struct, apiVersion); + return DescribeGroupsRequest.parse(buffer, apiVersion); case LIST_GROUPS: - return new ListGroupsRequest(struct, apiVersion); + return ListGroupsRequest.parse(buffer, apiVersion); case SASL_HANDSHAKE: - return new SaslHandshakeRequest(struct, apiVersion); + return SaslHandshakeRequest.parse(buffer, apiVersion); case API_VERSIONS: - return new ApiVersionsRequest(struct, apiVersion); + return ApiVersionsRequest.parse(buffer, apiVersion); case CREATE_TOPICS: - return new CreateTopicsRequest(struct, apiVersion); + return CreateTopicsRequest.parse(buffer, apiVersion); case DELETE_TOPICS: - return new DeleteTopicsRequest(struct, apiVersion); + return DeleteTopicsRequest.parse(buffer, apiVersion); case DELETE_RECORDS: - return new DeleteRecordsRequest(struct, apiVersion); + return DeleteRecordsRequest.parse(buffer, apiVersion); case INIT_PRODUCER_ID: - return new InitProducerIdRequest(struct, apiVersion); + return InitProducerIdRequest.parse(buffer, apiVersion); case OFFSET_FOR_LEADER_EPOCH: - return new OffsetsForLeaderEpochRequest(struct, apiVersion); + return OffsetsForLeaderEpochRequest.parse(buffer, apiVersion); case ADD_PARTITIONS_TO_TXN: - return new AddPartitionsToTxnRequest(struct, apiVersion); + return AddPartitionsToTxnRequest.parse(buffer, apiVersion); case ADD_OFFSETS_TO_TXN: - return new AddOffsetsToTxnRequest(struct, apiVersion); + return AddOffsetsToTxnRequest.parse(buffer, apiVersion); case END_TXN: - return new EndTxnRequest(struct, apiVersion); + return EndTxnRequest.parse(buffer, apiVersion); case WRITE_TXN_MARKERS: - return new WriteTxnMarkersRequest(struct, apiVersion); + return WriteTxnMarkersRequest.parse(buffer, apiVersion); case TXN_OFFSET_COMMIT: - return new TxnOffsetCommitRequest(struct, apiVersion); + return TxnOffsetCommitRequest.parse(buffer, apiVersion); case DESCRIBE_ACLS: - return new DescribeAclsRequest(struct, apiVersion); + return DescribeAclsRequest.parse(buffer, apiVersion); case CREATE_ACLS: - return new CreateAclsRequest(struct, apiVersion); + return CreateAclsRequest.parse(buffer, apiVersion); case DELETE_ACLS: - return new DeleteAclsRequest(struct, apiVersion); + return DeleteAclsRequest.parse(buffer, apiVersion); case DESCRIBE_CONFIGS: - return new DescribeConfigsRequest(struct, apiVersion); + return DescribeConfigsRequest.parse(buffer, apiVersion); case ALTER_CONFIGS: - return new AlterConfigsRequest(struct, apiVersion); + return AlterConfigsRequest.parse(buffer, apiVersion); case ALTER_REPLICA_LOG_DIRS: - return new AlterReplicaLogDirsRequest(struct, apiVersion); + return AlterReplicaLogDirsRequest.parse(buffer, apiVersion); case DESCRIBE_LOG_DIRS: - return new DescribeLogDirsRequest(struct, apiVersion); + return DescribeLogDirsRequest.parse(buffer, apiVersion); case SASL_AUTHENTICATE: - return new SaslAuthenticateRequest(struct, apiVersion); + return SaslAuthenticateRequest.parse(buffer, apiVersion); case CREATE_PARTITIONS: - return new CreatePartitionsRequest(struct, apiVersion); + return CreatePartitionsRequest.parse(buffer, apiVersion); case CREATE_DELEGATION_TOKEN: - return new CreateDelegationTokenRequest(struct, apiVersion); + return CreateDelegationTokenRequest.parse(buffer, apiVersion); case RENEW_DELEGATION_TOKEN: - return new RenewDelegationTokenRequest(struct, apiVersion); + return RenewDelegationTokenRequest.parse(buffer, apiVersion); case EXPIRE_DELEGATION_TOKEN: - return new ExpireDelegationTokenRequest(struct, apiVersion); + return ExpireDelegationTokenRequest.parse(buffer, apiVersion); case DESCRIBE_DELEGATION_TOKEN: - return new DescribeDelegationTokenRequest(struct, apiVersion); + return DescribeDelegationTokenRequest.parse(buffer, apiVersion); case DELETE_GROUPS: - return new DeleteGroupsRequest(struct, apiVersion); + return DeleteGroupsRequest.parse(buffer, apiVersion); case ELECT_LEADERS: - return new ElectLeadersRequest(struct, apiVersion); + return ElectLeadersRequest.parse(buffer, apiVersion); case INCREMENTAL_ALTER_CONFIGS: - return new IncrementalAlterConfigsRequest(struct, apiVersion); + return IncrementalAlterConfigsRequest.parse(buffer, apiVersion); case ALTER_PARTITION_REASSIGNMENTS: - return new AlterPartitionReassignmentsRequest(struct, apiVersion); + return AlterPartitionReassignmentsRequest.parse(buffer, apiVersion); case LIST_PARTITION_REASSIGNMENTS: - return new ListPartitionReassignmentsRequest(struct, apiVersion); + return ListPartitionReassignmentsRequest.parse(buffer, apiVersion); case OFFSET_DELETE: - return new OffsetDeleteRequest(struct, apiVersion); + return OffsetDeleteRequest.parse(buffer, apiVersion); case DESCRIBE_CLIENT_QUOTAS: - return new DescribeClientQuotasRequest(struct, apiVersion); + return DescribeClientQuotasRequest.parse(buffer, apiVersion); case ALTER_CLIENT_QUOTAS: - return new AlterClientQuotasRequest(struct, apiVersion); + return AlterClientQuotasRequest.parse(buffer, apiVersion); case DESCRIBE_USER_SCRAM_CREDENTIALS: - return new DescribeUserScramCredentialsRequest(struct, apiVersion); + return DescribeUserScramCredentialsRequest.parse(buffer, apiVersion); case ALTER_USER_SCRAM_CREDENTIALS: - return new AlterUserScramCredentialsRequest(struct, apiVersion); + return AlterUserScramCredentialsRequest.parse(buffer, apiVersion); case VOTE: - return new VoteRequest(struct, apiVersion); + return VoteRequest.parse(buffer, apiVersion); case BEGIN_QUORUM_EPOCH: - return new BeginQuorumEpochRequest(struct, apiVersion); + return BeginQuorumEpochRequest.parse(buffer, apiVersion); case END_QUORUM_EPOCH: - return new EndQuorumEpochRequest(struct, apiVersion); + return EndQuorumEpochRequest.parse(buffer, apiVersion); case DESCRIBE_QUORUM: - return new DescribeQuorumRequest(struct, apiVersion); + return DescribeQuorumRequest.parse(buffer, apiVersion); case ALTER_ISR: - return new AlterIsrRequest(new AlterIsrRequestData(struct, apiVersion), apiVersion); + return AlterIsrRequest.parse(buffer, apiVersion); case UPDATE_FEATURES: - return new UpdateFeaturesRequest(struct, apiVersion); + return UpdateFeaturesRequest.parse(buffer, apiVersion); case ENVELOPE: - return new EnvelopeRequest(struct, apiVersion); + return EnvelopeRequest.parse(buffer, apiVersion); default: throw new AssertionError(String.format("ApiKey %s is not currently handled in `parseRequest`, the " + "code should be updated to do so.", apiKey)); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequestResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequestResponse.java index 39698a1d20b74..5d655bd0075dd 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequestResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequestResponse.java @@ -16,6 +16,4 @@ */ package org.apache.kafka.common.requests; -public interface AbstractRequestResponse { - -} +public interface AbstractRequestResponse { } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java index 11566073374cf..747698166359c 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java @@ -16,38 +16,49 @@ */ package org.apache.kafka.common.requests; -import org.apache.kafka.common.message.EnvelopeResponseData; -import org.apache.kafka.common.message.FetchResponseData; -import org.apache.kafka.common.message.AlterIsrResponseData; -import org.apache.kafka.common.message.ProduceResponseData; -import org.apache.kafka.common.network.NetworkSend; import org.apache.kafka.common.network.Send; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.protocol.Message; +import org.apache.kafka.common.protocol.SendBuilder; import java.nio.ByteBuffer; import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.Objects; import java.util.stream.Collectors; import java.util.stream.Stream; public abstract class AbstractResponse implements AbstractRequestResponse { public static final int DEFAULT_THROTTLE_TIME = 0; - protected Send toSend(String destination, ResponseHeader header, short apiVersion) { - return new NetworkSend(destination, RequestUtils.serialize(header.toStruct(), toStruct(apiVersion))); + private final ApiKeys apiKey; + + protected AbstractResponse(ApiKeys apiKey) { + this.apiKey = apiKey; + } + + public final Send toSend(String destination, ResponseHeader header, short version) { + return SendBuilder.buildResponseSend(destination, header, data(), version); } /** * Visible for testing, typically {@link #toSend(String, ResponseHeader, short)} should be used instead. */ - public ByteBuffer serialize(ApiKeys apiKey, short version, int correlationId) { - ResponseHeader header = - new ResponseHeader(correlationId, apiKey.responseHeaderVersion(version)); - return RequestUtils.serialize(header.toStruct(), toStruct(version)); + public final ByteBuffer serializeWithHeader(short version, int correlationId) { + return serializeWithHeader(new ResponseHeader(correlationId, apiKey.responseHeaderVersion(version)), version); + } + + final ByteBuffer serializeWithHeader(ResponseHeader header, short version) { + Objects.requireNonNull(header, "header should not be null"); + return RequestUtils.serialize(header.data(), header.headerVersion(), data(), version); + } + + // Visible for testing + final ByteBuffer serializeBody(short version) { + return RequestUtils.serialize(null, (short) 0, data(), version); } /** @@ -84,17 +95,18 @@ protected void updateErrorCounts(Map errorCounts, Errors error) errorCounts.put(error, count + 1); } - protected abstract Struct toStruct(short version); + protected abstract Message data(); /** * Parse a response from the provided buffer. The buffer is expected to hold both * the {@link ResponseHeader} as well as the response payload. */ - public static AbstractResponse parseResponse(ByteBuffer byteBuffer, RequestHeader requestHeader) { + public static AbstractResponse parseResponse(ByteBuffer buffer, RequestHeader requestHeader) { ApiKeys apiKey = requestHeader.apiKey(); short apiVersion = requestHeader.apiVersion(); - ResponseHeader responseHeader = ResponseHeader.parse(byteBuffer, apiKey.responseHeaderVersion(apiVersion)); + ResponseHeader responseHeader = ResponseHeader.parse(buffer, apiKey.responseHeaderVersion(apiVersion)); + if (requestHeader.correlationId() != responseHeader.correlationId()) { throw new CorrelationIdMismatchException("Correlation id for response (" + responseHeader.correlationId() + ") does not match request (" @@ -102,130 +114,129 @@ public static AbstractResponse parseResponse(ByteBuffer byteBuffer, RequestHeade requestHeader.correlationId(), responseHeader.correlationId()); } - Struct struct = apiKey.parseResponse(apiVersion, byteBuffer); - return AbstractResponse.parseResponse(apiKey, struct, apiVersion); + return AbstractResponse.parseResponse(apiKey, buffer, apiVersion); } - public static AbstractResponse parseResponse(ApiKeys apiKey, Struct struct, short version) { + public static AbstractResponse parseResponse(ApiKeys apiKey, ByteBuffer responseBuffer, short version) { switch (apiKey) { case PRODUCE: - return new ProduceResponse(new ProduceResponseData(struct, version)); + return ProduceResponse.parse(responseBuffer, version); case FETCH: - return new FetchResponse<>(new FetchResponseData(struct, version)); + return FetchResponse.parse(responseBuffer, version); case LIST_OFFSETS: - return new ListOffsetResponse(struct, version); + return ListOffsetResponse.parse(responseBuffer, version); case METADATA: - return new MetadataResponse(struct, version); + return MetadataResponse.parse(responseBuffer, version); case OFFSET_COMMIT: - return new OffsetCommitResponse(struct, version); + return OffsetCommitResponse.parse(responseBuffer, version); case OFFSET_FETCH: - return new OffsetFetchResponse(struct, version); + return OffsetFetchResponse.parse(responseBuffer, version); case FIND_COORDINATOR: - return new FindCoordinatorResponse(struct, version); + return FindCoordinatorResponse.parse(responseBuffer, version); case JOIN_GROUP: - return new JoinGroupResponse(struct, version); + return JoinGroupResponse.parse(responseBuffer, version); case HEARTBEAT: - return new HeartbeatResponse(struct, version); + return HeartbeatResponse.parse(responseBuffer, version); case LEAVE_GROUP: - return new LeaveGroupResponse(struct, version); + return LeaveGroupResponse.parse(responseBuffer, version); case SYNC_GROUP: - return new SyncGroupResponse(struct, version); + return SyncGroupResponse.parse(responseBuffer, version); case STOP_REPLICA: - return new StopReplicaResponse(struct, version); + return StopReplicaResponse.parse(responseBuffer, version); case CONTROLLED_SHUTDOWN: - return new ControlledShutdownResponse(struct, version); + return ControlledShutdownResponse.parse(responseBuffer, version); case UPDATE_METADATA: - return new UpdateMetadataResponse(struct, version); + return UpdateMetadataResponse.parse(responseBuffer, version); case LEADER_AND_ISR: - return new LeaderAndIsrResponse(struct, version); + return LeaderAndIsrResponse.parse(responseBuffer, version); case DESCRIBE_GROUPS: - return new DescribeGroupsResponse(struct, version); + return DescribeGroupsResponse.parse(responseBuffer, version); case LIST_GROUPS: - return new ListGroupsResponse(struct, version); + return ListGroupsResponse.parse(responseBuffer, version); case SASL_HANDSHAKE: - return new SaslHandshakeResponse(struct, version); + return SaslHandshakeResponse.parse(responseBuffer, version); case API_VERSIONS: - return ApiVersionsResponse.fromStruct(struct, version); + return ApiVersionsResponse.parse(responseBuffer, version); case CREATE_TOPICS: - return new CreateTopicsResponse(struct, version); + return CreateTopicsResponse.parse(responseBuffer, version); case DELETE_TOPICS: - return new DeleteTopicsResponse(struct, version); + return DeleteTopicsResponse.parse(responseBuffer, version); case DELETE_RECORDS: - return new DeleteRecordsResponse(struct, version); + return DeleteRecordsResponse.parse(responseBuffer, version); case INIT_PRODUCER_ID: - return new InitProducerIdResponse(struct, version); + return InitProducerIdResponse.parse(responseBuffer, version); case OFFSET_FOR_LEADER_EPOCH: - return new OffsetsForLeaderEpochResponse(struct, version); + return OffsetsForLeaderEpochResponse.parse(responseBuffer, version); case ADD_PARTITIONS_TO_TXN: - return new AddPartitionsToTxnResponse(struct, version); + return AddPartitionsToTxnResponse.parse(responseBuffer, version); case ADD_OFFSETS_TO_TXN: - return new AddOffsetsToTxnResponse(struct, version); + return AddOffsetsToTxnResponse.parse(responseBuffer, version); case END_TXN: - return new EndTxnResponse(struct, version); + return EndTxnResponse.parse(responseBuffer, version); case WRITE_TXN_MARKERS: - return new WriteTxnMarkersResponse(struct, version); + return WriteTxnMarkersResponse.parse(responseBuffer, version); case TXN_OFFSET_COMMIT: - return new TxnOffsetCommitResponse(struct, version); + return TxnOffsetCommitResponse.parse(responseBuffer, version); case DESCRIBE_ACLS: - return new DescribeAclsResponse(struct, version); + return DescribeAclsResponse.parse(responseBuffer, version); case CREATE_ACLS: - return new CreateAclsResponse(struct, version); + return CreateAclsResponse.parse(responseBuffer, version); case DELETE_ACLS: - return new DeleteAclsResponse(struct, version); + return DeleteAclsResponse.parse(responseBuffer, version); case DESCRIBE_CONFIGS: - return new DescribeConfigsResponse(struct, version); + return DescribeConfigsResponse.parse(responseBuffer, version); case ALTER_CONFIGS: - return new AlterConfigsResponse(struct, version); + return AlterConfigsResponse.parse(responseBuffer, version); case ALTER_REPLICA_LOG_DIRS: - return new AlterReplicaLogDirsResponse(struct, version); + return AlterReplicaLogDirsResponse.parse(responseBuffer, version); case DESCRIBE_LOG_DIRS: - return new DescribeLogDirsResponse(struct, version); + return DescribeLogDirsResponse.parse(responseBuffer, version); case SASL_AUTHENTICATE: - return new SaslAuthenticateResponse(struct, version); + return SaslAuthenticateResponse.parse(responseBuffer, version); case CREATE_PARTITIONS: - return new CreatePartitionsResponse(struct, version); + return CreatePartitionsResponse.parse(responseBuffer, version); case CREATE_DELEGATION_TOKEN: - return new CreateDelegationTokenResponse(struct, version); + return CreateDelegationTokenResponse.parse(responseBuffer, version); case RENEW_DELEGATION_TOKEN: - return new RenewDelegationTokenResponse(struct, version); + return RenewDelegationTokenResponse.parse(responseBuffer, version); case EXPIRE_DELEGATION_TOKEN: - return new ExpireDelegationTokenResponse(struct, version); + return ExpireDelegationTokenResponse.parse(responseBuffer, version); case DESCRIBE_DELEGATION_TOKEN: - return new DescribeDelegationTokenResponse(struct, version); + return DescribeDelegationTokenResponse.parse(responseBuffer, version); case DELETE_GROUPS: - return new DeleteGroupsResponse(struct, version); + return DeleteGroupsResponse.parse(responseBuffer, version); case ELECT_LEADERS: - return new ElectLeadersResponse(struct, version); + return ElectLeadersResponse.parse(responseBuffer, version); case INCREMENTAL_ALTER_CONFIGS: - return new IncrementalAlterConfigsResponse(struct, version); + return IncrementalAlterConfigsResponse.parse(responseBuffer, version); case ALTER_PARTITION_REASSIGNMENTS: - return new AlterPartitionReassignmentsResponse(struct, version); + return AlterPartitionReassignmentsResponse.parse(responseBuffer, version); case LIST_PARTITION_REASSIGNMENTS: - return new ListPartitionReassignmentsResponse(struct, version); + return ListPartitionReassignmentsResponse.parse(responseBuffer, version); case OFFSET_DELETE: - return new OffsetDeleteResponse(struct, version); + return OffsetDeleteResponse.parse(responseBuffer, version); case DESCRIBE_CLIENT_QUOTAS: - return new DescribeClientQuotasResponse(struct, version); + return DescribeClientQuotasResponse.parse(responseBuffer, version); case ALTER_CLIENT_QUOTAS: - return new AlterClientQuotasResponse(struct, version); + return AlterClientQuotasResponse.parse(responseBuffer, version); case DESCRIBE_USER_SCRAM_CREDENTIALS: - return new DescribeUserScramCredentialsResponse(struct, version); + return DescribeUserScramCredentialsResponse.parse(responseBuffer, version); case ALTER_USER_SCRAM_CREDENTIALS: - return new AlterUserScramCredentialsResponse(struct, version); + return AlterUserScramCredentialsResponse.parse(responseBuffer, version); case VOTE: - return new VoteResponse(struct, version); + return VoteResponse.parse(responseBuffer, version); case BEGIN_QUORUM_EPOCH: - return new BeginQuorumEpochResponse(struct, version); + return BeginQuorumEpochResponse.parse(responseBuffer, version); case END_QUORUM_EPOCH: - return new EndQuorumEpochResponse(struct, version); + return EndQuorumEpochResponse.parse(responseBuffer, version); case DESCRIBE_QUORUM: - return new DescribeQuorumResponse(struct, version); + return DescribeQuorumResponse.parse(responseBuffer, version); case ALTER_ISR: - return new AlterIsrResponse(new AlterIsrResponseData(struct, version)); + return AlterIsrResponse.parse(responseBuffer, version); case UPDATE_FEATURES: - return new UpdateFeaturesResponse(struct, version); + return UpdateFeaturesResponse.parse(responseBuffer, version); case ENVELOPE: - return new EnvelopeResponse(new EnvelopeResponseData(struct, version)); + return EnvelopeResponse.parse(responseBuffer, version); default: throw new AssertionError(String.format("ApiKey %s is not currently handled in `parseResponse`, the " + "code should be updated to do so.", apiKey)); @@ -241,11 +252,13 @@ public boolean shouldClientThrottle(short version) { return false; } - public int throttleTimeMs() { - return DEFAULT_THROTTLE_TIME; + public ApiKeys apiKey() { + return apiKey; } - public String toString(short version) { - return toStruct(version).toString(); + public abstract int throttleTimeMs(); + + public String toString() { + return data().toString(); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AddOffsetsToTxnRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AddOffsetsToTxnRequest.java index 3b1c746c5b5c9..4ad56aa94268f 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AddOffsetsToTxnRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AddOffsetsToTxnRequest.java @@ -19,8 +19,8 @@ import org.apache.kafka.common.message.AddOffsetsToTxnRequestData; import org.apache.kafka.common.message.AddOffsetsToTxnResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; @@ -52,14 +52,9 @@ public AddOffsetsToTxnRequest(AddOffsetsToTxnRequestData data, short version) { this.data = data; } - public AddOffsetsToTxnRequest(Struct struct, short version) { - super(ApiKeys.ADD_OFFSETS_TO_TXN, version); - this.data = new AddOffsetsToTxnRequestData(struct, version); - } - @Override - protected Struct toStruct() { - return data.toStruct(version()); + protected AddOffsetsToTxnRequestData data() { + return data; } @Override @@ -70,6 +65,6 @@ public AddOffsetsToTxnResponse getErrorResponse(int throttleTimeMs, Throwable e) } public static AddOffsetsToTxnRequest parse(ByteBuffer buffer, short version) { - return new AddOffsetsToTxnRequest(ApiKeys.ADD_OFFSETS_TO_TXN.parseRequest(version, buffer), version); + return new AddOffsetsToTxnRequest(new AddOffsetsToTxnRequestData(new ByteBufferAccessor(buffer), version), version); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AddOffsetsToTxnResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AddOffsetsToTxnResponse.java index 99864f4bffd4a..6908e51960609 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AddOffsetsToTxnResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AddOffsetsToTxnResponse.java @@ -18,8 +18,8 @@ import org.apache.kafka.common.message.AddOffsetsToTxnResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.Map; @@ -42,30 +42,27 @@ public class AddOffsetsToTxnResponse extends AbstractResponse { public AddOffsetsToTxnResponseData data; public AddOffsetsToTxnResponse(AddOffsetsToTxnResponseData data) { + super(ApiKeys.ADD_OFFSETS_TO_TXN); this.data = data; } - public AddOffsetsToTxnResponse(Struct struct, short version) { - this.data = new AddOffsetsToTxnResponseData(struct, version); - } - @Override public Map errorCounts() { return errorCounts(Errors.forCode(data.errorCode())); } @Override - protected Struct toStruct(short version) { - return data.toStruct(version); + public int throttleTimeMs() { + return data.throttleTimeMs(); } @Override - public int throttleTimeMs() { - return data.throttleTimeMs(); + protected AddOffsetsToTxnResponseData data() { + return data; } public static AddOffsetsToTxnResponse parse(ByteBuffer buffer, short version) { - return new AddOffsetsToTxnResponse(ApiKeys.ADD_OFFSETS_TO_TXN.parseResponse(version, buffer), version); + return new AddOffsetsToTxnResponse(new AddOffsetsToTxnResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AddPartitionsToTxnRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AddPartitionsToTxnRequest.java index 13be2127cdd4d..57645789ca000 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AddPartitionsToTxnRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AddPartitionsToTxnRequest.java @@ -21,8 +21,8 @@ import org.apache.kafka.common.message.AddPartitionsToTxnRequestData.AddPartitionsToTxnTopic; import org.apache.kafka.common.message.AddPartitionsToTxnRequestData.AddPartitionsToTxnTopicCollection; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -103,11 +103,6 @@ public AddPartitionsToTxnRequest(final AddPartitionsToTxnRequestData data, short this.data = data; } - public AddPartitionsToTxnRequest(Struct struct, short version) { - super(ApiKeys.ADD_PARTITIONS_TO_TXN, version); - this.data = new AddPartitionsToTxnRequestData(struct, version); - } - public List partitions() { if (cachedPartitions != null) { return cachedPartitions; @@ -117,8 +112,8 @@ public List partitions() { } @Override - protected Struct toStruct() { - return data.toStruct(version()); + protected AddPartitionsToTxnRequestData data() { + return data; } @Override @@ -131,6 +126,6 @@ public AddPartitionsToTxnResponse getErrorResponse(int throttleTimeMs, Throwable } public static AddPartitionsToTxnRequest parse(ByteBuffer buffer, short version) { - return new AddPartitionsToTxnRequest(ApiKeys.ADD_PARTITIONS_TO_TXN.parseRequest(version, buffer), version); + return new AddPartitionsToTxnRequest(new AddPartitionsToTxnRequestData(new ByteBufferAccessor(buffer), version), version); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AddPartitionsToTxnResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AddPartitionsToTxnResponse.java index cc79ce8ca4200..c6da3af0a9319 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AddPartitionsToTxnResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AddPartitionsToTxnResponse.java @@ -23,8 +23,8 @@ import org.apache.kafka.common.message.AddPartitionsToTxnResponseData.AddPartitionsToTxnTopicResult; import org.apache.kafka.common.message.AddPartitionsToTxnResponseData.AddPartitionsToTxnTopicResultCollection; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.HashMap; @@ -50,11 +50,13 @@ public class AddPartitionsToTxnResponse extends AbstractResponse { private Map cachedErrorsMap = null; - public AddPartitionsToTxnResponse(Struct struct, short version) { - this.data = new AddPartitionsToTxnResponseData(struct, version); + public AddPartitionsToTxnResponse(AddPartitionsToTxnResponseData data) { + super(ApiKeys.ADD_PARTITIONS_TO_TXN); + this.data = data; } public AddPartitionsToTxnResponse(int throttleTimeMs, Map errors) { + super(ApiKeys.ADD_PARTITIONS_TO_TXN); Map resultMap = new HashMap<>(); @@ -115,12 +117,12 @@ public Map errorCounts() { } @Override - protected Struct toStruct(short version) { - return data.toStruct(version); + protected AddPartitionsToTxnResponseData data() { + return data; } public static AddPartitionsToTxnResponse parse(ByteBuffer buffer, short version) { - return new AddPartitionsToTxnResponse(ApiKeys.ADD_PARTITIONS_TO_TXN.parseResponse(version, buffer), version); + return new AddPartitionsToTxnResponse(new AddPartitionsToTxnResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasRequest.java index 6d4c2a1f357c3..7178475123633 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasRequest.java @@ -20,11 +20,13 @@ import org.apache.kafka.common.message.AlterClientQuotasRequestData.EntityData; import org.apache.kafka.common.message.AlterClientQuotasRequestData.EntryData; import org.apache.kafka.common.message.AlterClientQuotasRequestData.OpData; +import org.apache.kafka.common.message.AlterClientQuotasResponseData; import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.quota.ClientQuotaAlteration; import org.apache.kafka.common.quota.ClientQuotaEntity; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -85,12 +87,7 @@ public AlterClientQuotasRequest(AlterClientQuotasRequestData data, short version this.data = data; } - public AlterClientQuotasRequest(Struct struct, short version) { - super(ApiKeys.ALTER_CLIENT_QUOTAS, version); - this.data = new AlterClientQuotasRequestData(struct, version); - } - - public Collection entries() { + public List entries() { List entries = new ArrayList<>(data.entries().size()); for (EntryData entryData : data.entries()) { Map entity = new HashMap<>(entryData.entity().size()); @@ -113,21 +110,30 @@ public boolean validateOnly() { return data.validateOnly(); } + @Override + protected AlterClientQuotasRequestData data() { + return data; + } + @Override public AlterClientQuotasResponse getErrorResponse(int throttleTimeMs, Throwable e) { - ArrayList entities = new ArrayList<>(data.entries().size()); + List responseEntries = new ArrayList<>(); for (EntryData entryData : data.entries()) { - Map entity = new HashMap<>(entryData.entity().size()); + List responseEntities = new ArrayList<>(); for (EntityData entityData : entryData.entity()) { - entity.put(entityData.entityType(), entityData.entityName()); + responseEntities.add(new AlterClientQuotasResponseData.EntityData() + .setEntityType(entityData.entityType()) + .setEntityName(entityData.entityName())); } - entities.add(new ClientQuotaEntity(entity)); + responseEntries.add(new AlterClientQuotasResponseData.EntryData().setEntity(responseEntities)); } - return new AlterClientQuotasResponse(entities, throttleTimeMs, e); + AlterClientQuotasResponseData responseData = new AlterClientQuotasResponseData() + .setThrottleTimeMs(throttleTimeMs) + .setEntries(responseEntries); + return new AlterClientQuotasResponse(responseData); } - @Override - protected Struct toStruct() { - return data.toStruct(version()); + public static AlterClientQuotasRequest parse(ByteBuffer buffer, short version) { + return new AlterClientQuotasRequest(new AlterClientQuotasRequestData(new ByteBufferAccessor(buffer), version), version); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasResponse.java index 3e95671a6f939..bce548822d895 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasResponse.java @@ -21,13 +21,12 @@ import org.apache.kafka.common.message.AlterClientQuotasResponseData.EntityData; import org.apache.kafka.common.message.AlterClientQuotasResponseData.EntryData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.quota.ClientQuotaEntity; import java.nio.ByteBuffer; import java.util.ArrayList; -import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -36,39 +35,9 @@ public class AlterClientQuotasResponse extends AbstractResponse { private final AlterClientQuotasResponseData data; - public AlterClientQuotasResponse(Map result, int throttleTimeMs) { - List entries = new ArrayList<>(result.size()); - for (Map.Entry entry : result.entrySet()) { - ApiError e = entry.getValue(); - entries.add(new EntryData() - .setErrorCode(e.error().code()) - .setErrorMessage(e.message()) - .setEntity(toEntityData(entry.getKey()))); - } - - this.data = new AlterClientQuotasResponseData() - .setThrottleTimeMs(throttleTimeMs) - .setEntries(entries); - } - - public AlterClientQuotasResponse(Collection entities, int throttleTimeMs, Throwable e) { - ApiError apiError = ApiError.fromThrowable(e); - - List entries = new ArrayList<>(entities.size()); - for (ClientQuotaEntity entity : entities) { - entries.add(new EntryData() - .setErrorCode(apiError.error().code()) - .setErrorMessage(apiError.message()) - .setEntity(toEntityData(entity))); - } - - this.data = new AlterClientQuotasResponseData() - .setThrottleTimeMs(throttleTimeMs) - .setEntries(entries); - } - - public AlterClientQuotasResponse(Struct struct, short version) { - this.data = new AlterClientQuotasResponseData(struct, version); + public AlterClientQuotasResponse(AlterClientQuotasResponseData data) { + super(ApiKeys.ALTER_CLIENT_QUOTAS); + this.data = data; } public void complete(Map> futures) { @@ -108,8 +77,8 @@ public Map errorCounts() { } @Override - protected Struct toStruct(short version) { - return data.toStruct(version); + protected AlterClientQuotasResponseData data() { + return data; } private static List toEntityData(ClientQuotaEntity entity) { @@ -123,6 +92,22 @@ private static List toEntityData(ClientQuotaEntity entity) { } public static AlterClientQuotasResponse parse(ByteBuffer buffer, short version) { - return new AlterClientQuotasResponse(ApiKeys.ALTER_CLIENT_QUOTAS.parseResponse(version, buffer), version); + return new AlterClientQuotasResponse(new AlterClientQuotasResponseData(new ByteBufferAccessor(buffer), version)); } + + public static AlterClientQuotasResponse fromQuotaEntities(Map result, int throttleTimeMs) { + List entries = new ArrayList<>(result.size()); + for (Map.Entry entry : result.entrySet()) { + ApiError e = entry.getValue(); + entries.add(new EntryData() + .setErrorCode(e.error().code()) + .setErrorMessage(e.message()) + .setEntity(toEntityData(entry.getKey()))); + } + + return new AlterClientQuotasResponse(new AlterClientQuotasResponseData() + .setThrottleTimeMs(throttleTimeMs) + .setEntries(entries)); + } + } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AlterConfigsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AlterConfigsRequest.java index 1ee3330a71ee5..f30e8b9c1ffe9 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AlterConfigsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AlterConfigsRequest.java @@ -21,7 +21,7 @@ import org.apache.kafka.common.message.AlterConfigsRequestData; import org.apache.kafka.common.message.AlterConfigsResponseData; import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import java.nio.ByteBuffer; import java.util.Collection; @@ -97,11 +97,6 @@ public AlterConfigsRequest(AlterConfigsRequestData data, short version) { this.data = data; } - public AlterConfigsRequest(Struct struct, short version) { - super(ApiKeys.ALTER_CONFIGS, version); - this.data = new AlterConfigsRequestData(struct, version); - } - public Map configs() { return data.resources().stream().collect(Collectors.toMap( resource -> new ConfigResource( @@ -117,8 +112,8 @@ public boolean validateOnly() { } @Override - protected Struct toStruct() { - return data.toStruct(version()); + protected AlterConfigsRequestData data() { + return data; } @Override @@ -138,6 +133,6 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { } public static AlterConfigsRequest parse(ByteBuffer buffer, short version) { - return new AlterConfigsRequest(ApiKeys.ALTER_CONFIGS.parseRequest(version, buffer), version); + return new AlterConfigsRequest(new AlterConfigsRequestData(new ByteBufferAccessor(buffer), version), version); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AlterConfigsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AlterConfigsResponse.java index 7122a132e5393..1115f06ee80a9 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AlterConfigsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AlterConfigsResponse.java @@ -20,8 +20,8 @@ import org.apache.kafka.common.config.ConfigResource; import org.apache.kafka.common.message.AlterConfigsResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.Map; @@ -32,17 +32,10 @@ public class AlterConfigsResponse extends AbstractResponse { private final AlterConfigsResponseData data; public AlterConfigsResponse(AlterConfigsResponseData data) { + super(ApiKeys.ALTER_CONFIGS); this.data = data; } - public AlterConfigsResponse(Struct struct, short version) { - this.data = new AlterConfigsResponseData(struct, version); - } - - public AlterConfigsResponseData data() { - return data; - } - public Map errors() { return data.responses().stream().collect(Collectors.toMap( response -> new ConfigResource( @@ -63,12 +56,12 @@ public int throttleTimeMs() { } @Override - protected Struct toStruct(short version) { - return data.toStruct(version); + public AlterConfigsResponseData data() { + return data; } public static AlterConfigsResponse parse(ByteBuffer buffer, short version) { - return new AlterConfigsResponse(ApiKeys.ALTER_CONFIGS.parseResponse(version, buffer), version); + return new AlterConfigsResponse(new AlterConfigsResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AlterIsrRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AlterIsrRequest.java index 61662721f19b6..7ce86aedf432e 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AlterIsrRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AlterIsrRequest.java @@ -20,8 +20,10 @@ import org.apache.kafka.common.message.AlterIsrRequestData; import org.apache.kafka.common.message.AlterIsrResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; + +import java.nio.ByteBuffer; public class AlterIsrRequest extends AbstractRequest { @@ -36,11 +38,6 @@ public AlterIsrRequestData data() { return data; } - @Override - protected Struct toStruct() { - return data.toStruct(version()); - } - /** * Get an error response for a request with specified throttle time in the response if applicable */ @@ -51,6 +48,10 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { .setErrorCode(Errors.forException(e).code())); } + public static AlterIsrRequest parse(ByteBuffer buffer, short version) { + return new AlterIsrRequest(new AlterIsrRequestData(new ByteBufferAccessor(buffer), version), version); + } + public static class Builder extends AbstractRequest.Builder { private final AlterIsrRequestData data; diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AlterIsrResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AlterIsrResponse.java index b475bfaecf705..433ba66bdadd3 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AlterIsrResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AlterIsrResponse.java @@ -18,9 +18,11 @@ package org.apache.kafka.common.requests; import org.apache.kafka.common.message.AlterIsrResponseData; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; +import java.nio.ByteBuffer; import java.util.HashMap; import java.util.Map; @@ -29,6 +31,7 @@ public class AlterIsrResponse extends AbstractResponse { private final AlterIsrResponseData data; public AlterIsrResponse(AlterIsrResponseData data) { + super(ApiKeys.ALTER_ISR); this.data = data; } @@ -46,13 +49,12 @@ public Map errorCounts() { return counts; } - @Override - protected Struct toStruct(short version) { - return data.toStruct(version); - } - @Override public int throttleTimeMs() { return data.throttleTimeMs(); } + + public static AlterIsrResponse parse(ByteBuffer buffer, short version) { + return new AlterIsrResponse(new AlterIsrResponseData(new ByteBufferAccessor(buffer), version)); + } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AlterPartitionReassignmentsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AlterPartitionReassignmentsRequest.java index 7b2f848f614f9..2d289cc1497e1 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AlterPartitionReassignmentsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AlterPartitionReassignmentsRequest.java @@ -23,7 +23,7 @@ import org.apache.kafka.common.message.AlterPartitionReassignmentsResponseData.ReassignablePartitionResponse; import org.apache.kafka.common.message.AlterPartitionReassignmentsResponseData.ReassignableTopicResponse; import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -52,39 +52,21 @@ public String toString() { } private final AlterPartitionReassignmentsRequestData data; - private final short version; private AlterPartitionReassignmentsRequest(AlterPartitionReassignmentsRequestData data, short version) { super(ApiKeys.ALTER_PARTITION_REASSIGNMENTS, version); this.data = data; - this.version = version; - } - - AlterPartitionReassignmentsRequest(Struct struct, short version) { - super(ApiKeys.ALTER_PARTITION_REASSIGNMENTS, version); - this.data = new AlterPartitionReassignmentsRequestData(struct, version); - this.version = version; } public static AlterPartitionReassignmentsRequest parse(ByteBuffer buffer, short version) { - return new AlterPartitionReassignmentsRequest( - ApiKeys.ALTER_PARTITION_REASSIGNMENTS.parseRequest(version, buffer), - version - ); + return new AlterPartitionReassignmentsRequest(new AlterPartitionReassignmentsRequestData( + new ByteBufferAccessor(buffer), version), version); } public AlterPartitionReassignmentsRequestData data() { return data; } - /** - * Visible for testing. - */ - @Override - public Struct toStruct() { - return data.toStruct(version); - } - @Override public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { ApiError apiError = ApiError.fromThrowable(e); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AlterPartitionReassignmentsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AlterPartitionReassignmentsResponse.java index 495258655e9bb..6aea8b1edcc87 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AlterPartitionReassignmentsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AlterPartitionReassignmentsResponse.java @@ -19,8 +19,8 @@ import org.apache.kafka.common.message.AlterPartitionReassignmentsResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.HashMap; @@ -30,20 +30,14 @@ public class AlterPartitionReassignmentsResponse extends AbstractResponse { private final AlterPartitionReassignmentsResponseData data; - public AlterPartitionReassignmentsResponse(Struct struct) { - this(struct, ApiKeys.ALTER_PARTITION_REASSIGNMENTS.latestVersion()); - } - public AlterPartitionReassignmentsResponse(AlterPartitionReassignmentsResponseData data) { + super(ApiKeys.ALTER_PARTITION_REASSIGNMENTS); this.data = data; } - AlterPartitionReassignmentsResponse(Struct struct, short version) { - this.data = new AlterPartitionReassignmentsResponseData(struct, version); - } - public static AlterPartitionReassignmentsResponse parse(ByteBuffer buffer, short version) { - return new AlterPartitionReassignmentsResponse(ApiKeys.ALTER_PARTITION_REASSIGNMENTS.responseSchema(version).read(buffer), version); + return new AlterPartitionReassignmentsResponse( + new AlterPartitionReassignmentsResponseData(new ByteBufferAccessor(buffer), version)); } public AlterPartitionReassignmentsResponseData data() { @@ -71,9 +65,4 @@ public Map errorCounts() { )); return counts; } - - @Override - protected Struct toStruct(short version) { - return data.toStruct(version); - } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AlterReplicaLogDirsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AlterReplicaLogDirsRequest.java index 8cde2c0a1bde2..5eba039fa556b 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AlterReplicaLogDirsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AlterReplicaLogDirsRequest.java @@ -17,18 +17,20 @@ package org.apache.kafka.common.requests; + +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; +import org.apache.kafka.common.protocol.Errors; + import java.nio.ByteBuffer; import java.util.HashMap; import java.util.Map; import java.util.stream.Collectors; -import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.message.AlterReplicaLogDirsRequestData; import org.apache.kafka.common.message.AlterReplicaLogDirsResponseData; import org.apache.kafka.common.message.AlterReplicaLogDirsResponseData.AlterReplicaLogDirTopicResult; -import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; public class AlterReplicaLogDirsRequest extends AbstractRequest { @@ -53,22 +55,16 @@ public String toString() { } } - public AlterReplicaLogDirsRequest(Struct struct, short version) { - super(ApiKeys.ALTER_REPLICA_LOG_DIRS, version); - this.data = new AlterReplicaLogDirsRequestData(struct, version); - } - public AlterReplicaLogDirsRequest(AlterReplicaLogDirsRequestData data, short version) { super(ApiKeys.ALTER_REPLICA_LOG_DIRS, version); this.data = data; } @Override - protected Struct toStruct() { - return data.toStruct(version()); + protected AlterReplicaLogDirsRequestData data() { + return data; } - @Override public AlterReplicaLogDirsResponse getErrorResponse(int throttleTimeMs, Throwable e) { AlterReplicaLogDirsResponseData data = new AlterReplicaLogDirsResponseData(); data.setResults(this.data.dirs().stream().flatMap(alterDir -> @@ -93,6 +89,6 @@ public Map partitionDirs() { } public static AlterReplicaLogDirsRequest parse(ByteBuffer buffer, short version) { - return new AlterReplicaLogDirsRequest(ApiKeys.ALTER_REPLICA_LOG_DIRS.parseRequest(version, buffer), version); + return new AlterReplicaLogDirsRequest(new AlterReplicaLogDirsRequestData(new ByteBufferAccessor(buffer), version), version); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AlterReplicaLogDirsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AlterReplicaLogDirsResponse.java index bdc1ed0003b9e..afa658d1e150f 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AlterReplicaLogDirsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AlterReplicaLogDirsResponse.java @@ -17,14 +17,14 @@ package org.apache.kafka.common.requests; -import java.nio.ByteBuffer; -import java.util.HashMap; -import java.util.Map; - import org.apache.kafka.common.message.AlterReplicaLogDirsResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; + +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.Map; /** * Possible error codes: @@ -38,23 +38,16 @@ public class AlterReplicaLogDirsResponse extends AbstractResponse { private final AlterReplicaLogDirsResponseData data; - public AlterReplicaLogDirsResponse(Struct struct, short version) { - this.data = new AlterReplicaLogDirsResponseData(struct, version); - } - public AlterReplicaLogDirsResponse(AlterReplicaLogDirsResponseData data) { + super(ApiKeys.ALTER_REPLICA_LOG_DIRS); this.data = data; } + @Override public AlterReplicaLogDirsResponseData data() { return data; } - @Override - protected Struct toStruct(short version) { - return data.toStruct(version); - } - @Override public int throttleTimeMs() { return data.throttleTimeMs(); @@ -70,7 +63,7 @@ public Map errorCounts() { } public static AlterReplicaLogDirsResponse parse(ByteBuffer buffer, short version) { - return new AlterReplicaLogDirsResponse(ApiKeys.ALTER_REPLICA_LOG_DIRS.responseSchema(version).read(buffer), version); + return new AlterReplicaLogDirsResponse(new AlterReplicaLogDirsResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AlterUserScramCredentialsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AlterUserScramCredentialsRequest.java index 8d3a18846021c..c319ec344525a 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AlterUserScramCredentialsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AlterUserScramCredentialsRequest.java @@ -19,7 +19,7 @@ import org.apache.kafka.common.message.AlterUserScramCredentialsRequestData; import org.apache.kafka.common.message.AlterUserScramCredentialsResponseData; import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import java.nio.ByteBuffer; import java.util.List; @@ -48,39 +48,21 @@ public String toString() { } } - private AlterUserScramCredentialsRequestData data; - private final short version; + private final AlterUserScramCredentialsRequestData data; private AlterUserScramCredentialsRequest(AlterUserScramCredentialsRequestData data, short version) { super(ApiKeys.ALTER_USER_SCRAM_CREDENTIALS, version); this.data = data; - this.version = version; - } - - AlterUserScramCredentialsRequest(Struct struct, short version) { - super(ApiKeys.ALTER_USER_SCRAM_CREDENTIALS, version); - this.data = new AlterUserScramCredentialsRequestData(struct, version); - this.version = version; } public static AlterUserScramCredentialsRequest parse(ByteBuffer buffer, short version) { - return new AlterUserScramCredentialsRequest( - ApiKeys.ALTER_USER_SCRAM_CREDENTIALS.parseRequest(version, buffer), version - ); + return new AlterUserScramCredentialsRequest(new AlterUserScramCredentialsRequestData(new ByteBufferAccessor(buffer), version), version); } public AlterUserScramCredentialsRequestData data() { return data; } - /** - * Visible for testing. - */ - @Override - public Struct toStruct() { - return data.toStruct(version); - } - @Override public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { ApiError apiError = ApiError.fromThrowable(e); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AlterUserScramCredentialsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AlterUserScramCredentialsResponse.java index 88ff920d3f132..2fa4937242742 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AlterUserScramCredentialsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AlterUserScramCredentialsResponse.java @@ -18,8 +18,8 @@ import org.apache.kafka.common.message.AlterUserScramCredentialsResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.Map; @@ -28,22 +28,11 @@ public class AlterUserScramCredentialsResponse extends AbstractResponse { private final AlterUserScramCredentialsResponseData data; - public AlterUserScramCredentialsResponse(Struct struct) { - this(struct, ApiKeys.ALTER_USER_SCRAM_CREDENTIALS.latestVersion()); - } - public AlterUserScramCredentialsResponse(AlterUserScramCredentialsResponseData responseData) { + super(ApiKeys.ALTER_USER_SCRAM_CREDENTIALS); this.data = responseData; } - AlterUserScramCredentialsResponse(Struct struct, short version) { - this.data = new AlterUserScramCredentialsResponseData(struct, version); - } - - public static AlterUserScramCredentialsResponse parse(ByteBuffer buffer, short version) { - return new AlterUserScramCredentialsResponse(ApiKeys.ALTER_USER_SCRAM_CREDENTIALS.responseSchema(version).read(buffer), version); - } - public AlterUserScramCredentialsResponseData data() { return data; } @@ -63,8 +52,7 @@ public Map errorCounts() { return errorCounts(data.results().stream().map(r -> Errors.forCode(r.errorCode()))); } - @Override - protected Struct toStruct(short version) { - return data.toStruct(version); + public static AlterUserScramCredentialsResponse parse(ByteBuffer buffer, short version) { + return new AlterUserScramCredentialsResponse(new AlterUserScramCredentialsResponseData(new ByteBufferAccessor(buffer), version)); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ApiError.java b/clients/src/main/java/org/apache/kafka/common/requests/ApiError.java index a790bc8b161c6..38712adbd55f8 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ApiError.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ApiError.java @@ -19,13 +19,9 @@ import org.apache.kafka.common.errors.ApiException; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.util.Objects; -import static org.apache.kafka.common.protocol.CommonFields.ERROR_CODE; -import static org.apache.kafka.common.protocol.CommonFields.ERROR_MESSAGE; - /** * Encapsulates an error code (via the Errors enum) and an optional message. Generally, the optional message is only * defined if it adds information over the default message associated with the error code. @@ -47,12 +43,6 @@ public static ApiError fromThrowable(Throwable t) { return new ApiError(error, message); } - public ApiError(Struct struct) { - error = Errors.forCode(struct.get(ERROR_CODE)); - // In some cases, the error message field was introduced in newer version - message = struct.getOrElse(ERROR_MESSAGE, null); - } - public ApiError(Errors error) { this(error, error.message()); } @@ -67,12 +57,6 @@ public ApiError(short code, String message) { this.message = message; } - public void write(Struct struct) { - struct.set(ERROR_CODE, error.code()); - if (error != Errors.NONE) - struct.setIfExists(ERROR_MESSAGE, message); - } - public boolean is(Errors error) { return this.error == error; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsRequest.java index 87ffceaa26e1c..946a9d9da7bed 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsRequest.java @@ -22,8 +22,8 @@ import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersionsResponseKey; import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersionsResponseKeyCollection; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.utils.AppInfoParser; import java.nio.ByteBuffer; @@ -60,7 +60,7 @@ public String toString() { private final Short unsupportedRequestVersion; - public final ApiVersionsRequestData data; + private final ApiVersionsRequestData data; public ApiVersionsRequest(ApiVersionsRequestData data, short version) { this(data, version, null); @@ -78,10 +78,6 @@ public ApiVersionsRequest(ApiVersionsRequestData data, short version, Short unsu this.unsupportedRequestVersion = unsupportedRequestVersion; } - public ApiVersionsRequest(Struct struct, short version) { - this(new ApiVersionsRequestData(struct, version), version); - } - public boolean hasUnsupportedRequestVersion() { return unsupportedRequestVersion != null; } @@ -96,8 +92,8 @@ public boolean isValid() { } @Override - protected Struct toStruct() { - return data.toStruct(version()); + public ApiVersionsRequestData data() { + return data; } @Override @@ -124,7 +120,7 @@ public ApiVersionsResponse getErrorResponse(int throttleTimeMs, Throwable e) { } public static ApiVersionsRequest parse(ByteBuffer buffer, short version) { - return new ApiVersionsRequest(ApiKeys.API_VERSIONS.parseRequest(version, buffer), version); + return new ApiVersionsRequest(new ApiVersionsRequestData(new ByteBufferAccessor(buffer), version), version); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsResponse.java index 1e4ad17b47dc0..eaf8113fcd0b7 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsResponse.java @@ -29,8 +29,6 @@ import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.SchemaException; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.record.RecordBatch; import java.nio.ByteBuffer; @@ -45,38 +43,21 @@ public class ApiVersionsResponse extends AbstractResponse { public static final long UNKNOWN_FINALIZED_FEATURES_EPOCH = -1L; - public static final ApiVersionsResponse DEFAULT_API_VERSIONS_RESPONSE = - createApiVersionsResponse( - DEFAULT_THROTTLE_TIME, - RecordBatch.CURRENT_MAGIC_VALUE, - Features.emptySupportedFeatures(), - Features.emptyFinalizedFeatures(), - UNKNOWN_FINALIZED_FEATURES_EPOCH - ); + public static final ApiVersionsResponse DEFAULT_API_VERSIONS_RESPONSE = createApiVersionsResponse( + DEFAULT_THROTTLE_TIME, RecordBatch.CURRENT_MAGIC_VALUE); public final ApiVersionsResponseData data; public ApiVersionsResponse(ApiVersionsResponseData data) { + super(ApiKeys.API_VERSIONS); this.data = data; } - public ApiVersionsResponse(Struct struct) { - this(new ApiVersionsResponseData(struct, (short) (ApiVersionsResponseData.SCHEMAS.length - 1))); - } - - public ApiVersionsResponse(Struct struct, short version) { - this(new ApiVersionsResponseData(struct, version)); - } - + @Override public ApiVersionsResponseData data() { return data; } - @Override - protected Struct toStruct(short version) { - return this.data.toStruct(version); - } - public ApiVersionsResponseKey apiVersion(short apiKey) { return data.apiKeys().find(apiKey); } @@ -100,48 +81,23 @@ public static ApiVersionsResponse parse(ByteBuffer buffer, short version) { // Fallback to version 0 for ApiVersions response. If a client sends an ApiVersionsRequest // using a version higher than that supported by the broker, a version 0 response is sent // to the client indicating UNSUPPORTED_VERSION. When the client receives the response, it - // falls back while parsing it into a Struct which means that the version received by this - // method is not necessary the real one. It may be version 0 as well. + // falls back while parsing it which means that the version received by this + // method is not necessarily the real one. It may be version 0 as well. int prev = buffer.position(); try { - return new ApiVersionsResponse( - new ApiVersionsResponseData(new ByteBufferAccessor(buffer), version)); + return new ApiVersionsResponse(new ApiVersionsResponseData(new ByteBufferAccessor(buffer), version)); } catch (RuntimeException e) { buffer.position(prev); if (version != 0) - return new ApiVersionsResponse( - new ApiVersionsResponseData(new ByteBufferAccessor(buffer), (short) 0)); + return new ApiVersionsResponse(new ApiVersionsResponseData(new ByteBufferAccessor(buffer), (short) 0)); else throw e; } } - public static ApiVersionsResponse fromStruct(Struct struct, short version) { - // Fallback to version 0 for ApiVersions response. If a client sends an ApiVersionsRequest - // using a version higher than that supported by the broker, a version 0 response is sent - // to the client indicating UNSUPPORTED_VERSION. When the client receives the response, it - // falls back while parsing it into a Struct which means that the version received by this - // method is not necessary the real one. It may be version 0 as well. - try { - return new ApiVersionsResponse(struct, version); - } catch (SchemaException e) { - if (version != 0) - return new ApiVersionsResponse(struct, (short) 0); - else - throw e; - } - } - - public static ApiVersionsResponse createApiVersionsResponse( - final int throttleTimeMs, - final byte minMagic) { - return createApiVersionsResponse( - throttleTimeMs, - minMagic, - Features.emptySupportedFeatures(), - Features.emptyFinalizedFeatures(), - UNKNOWN_FINALIZED_FEATURES_EPOCH - ); + public static ApiVersionsResponse createApiVersionsResponse(final int throttleTimeMs, final byte minMagic) { + return createApiVersionsResponse(throttleTimeMs, minMagic, Features.emptySupportedFeatures(), + Features.emptyFinalizedFeatures(), UNKNOWN_FINALIZED_FEATURES_EPOCH); } private static ApiVersionsResponse createApiVersionsResponse( diff --git a/clients/src/main/java/org/apache/kafka/common/requests/BeginQuorumEpochRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/BeginQuorumEpochRequest.java index a3e492b72e233..c83e29dd3d52a 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/BeginQuorumEpochRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/BeginQuorumEpochRequest.java @@ -20,9 +20,10 @@ import org.apache.kafka.common.message.BeginQuorumEpochRequestData; import org.apache.kafka.common.message.BeginQuorumEpochResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; +import java.nio.ByteBuffer; import java.util.Collections; public class BeginQuorumEpochRequest extends AbstractRequest { @@ -52,14 +53,9 @@ private BeginQuorumEpochRequest(BeginQuorumEpochRequestData data, short version) this.data = data; } - public BeginQuorumEpochRequest(Struct struct, short version) { - super(ApiKeys.BEGIN_QUORUM_EPOCH, version); - this.data = new BeginQuorumEpochRequestData(struct, version); - } - @Override - protected Struct toStruct() { - return data.toStruct(version()); + protected BeginQuorumEpochRequestData data() { + return data; } @Override @@ -68,6 +64,10 @@ public BeginQuorumEpochResponse getErrorResponse(int throttleTimeMs, Throwable e .setErrorCode(Errors.forException(e).code())); } + public static BeginQuorumEpochRequest parse(ByteBuffer buffer, short version) { + return new BeginQuorumEpochRequest(new BeginQuorumEpochRequestData(new ByteBufferAccessor(buffer), version), version); + } + public static BeginQuorumEpochRequestData singletonRequest(TopicPartition topicPartition, int leaderEpoch, int leaderId) { diff --git a/clients/src/main/java/org/apache/kafka/common/requests/BeginQuorumEpochResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/BeginQuorumEpochResponse.java index 24eedb2bd82c6..c3e80eccaa82c 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/BeginQuorumEpochResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/BeginQuorumEpochResponse.java @@ -20,8 +20,8 @@ import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.message.BeginQuorumEpochResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.Collections; @@ -45,18 +45,10 @@ public class BeginQuorumEpochResponse extends AbstractResponse { public final BeginQuorumEpochResponseData data; public BeginQuorumEpochResponse(BeginQuorumEpochResponseData data) { + super(ApiKeys.BEGIN_QUORUM_EPOCH); this.data = data; } - public BeginQuorumEpochResponse(Struct struct, short version) { - this.data = new BeginQuorumEpochResponseData(struct, version); - } - - public BeginQuorumEpochResponse(Struct struct) { - short latestVersion = (short) (BeginQuorumEpochResponseData.SCHEMAS.length - 1); - this.data = new BeginQuorumEpochResponseData(struct, latestVersion); - } - public static BeginQuorumEpochResponseData singletonResponse( Errors topLevelError, TopicPartition topicPartition, @@ -78,11 +70,6 @@ public static BeginQuorumEpochResponseData singletonResponse( ); } - @Override - protected Struct toStruct(short version) { - return data.toStruct(version); - } - @Override public Map errorCounts() { Map errors = new HashMap<>(); @@ -98,8 +85,18 @@ public Map errorCounts() { return errors; } + @Override + protected BeginQuorumEpochResponseData data() { + return data; + } + + @Override + public int throttleTimeMs() { + return DEFAULT_THROTTLE_TIME; + } + public static BeginQuorumEpochResponse parse(ByteBuffer buffer, short version) { - return new BeginQuorumEpochResponse(ApiKeys.BEGIN_QUORUM_EPOCH.responseSchema(version).read(buffer), version); + return new BeginQuorumEpochResponse(new BeginQuorumEpochResponseData(new ByteBufferAccessor(buffer), version)); } -} \ No newline at end of file +} diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownRequest.java index 71238452cf782..f3c063a7a060d 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownRequest.java @@ -19,8 +19,8 @@ import org.apache.kafka.common.message.ControlledShutdownRequestData; import org.apache.kafka.common.message.ControlledShutdownResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; @@ -47,18 +47,10 @@ public String toString() { } private final ControlledShutdownRequestData data; - private final short version; private ControlledShutdownRequest(ControlledShutdownRequestData data, short version) { super(ApiKeys.CONTROLLED_SHUTDOWN, version); this.data = data; - this.version = version; - } - - public ControlledShutdownRequest(Struct struct, short version) { - super(ApiKeys.CONTROLLED_SHUTDOWN, version); - this.data = new ControlledShutdownRequestData(struct, version); - this.version = version; } @Override @@ -69,13 +61,8 @@ public ControlledShutdownResponse getErrorResponse(int throttleTimeMs, Throwable } public static ControlledShutdownRequest parse(ByteBuffer buffer, short version) { - return new ControlledShutdownRequest( - ApiKeys.CONTROLLED_SHUTDOWN.parseRequest(version, buffer), version); - } - - @Override - protected Struct toStruct() { - return data.toStruct(version); + return new ControlledShutdownRequest(new ControlledShutdownRequestData(new ByteBufferAccessor(buffer), version), + version); } public ControlledShutdownRequestData data() { diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownResponse.java index 53742c21a49a6..1add8000364e0 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ControlledShutdownResponse.java @@ -20,8 +20,8 @@ import org.apache.kafka.common.message.ControlledShutdownResponseData; import org.apache.kafka.common.message.ControlledShutdownResponseData.RemainingPartition; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.Map; @@ -40,13 +40,10 @@ public class ControlledShutdownResponse extends AbstractResponse { private final ControlledShutdownResponseData data; public ControlledShutdownResponse(ControlledShutdownResponseData data) { + super(ApiKeys.CONTROLLED_SHUTDOWN); this.data = data; } - public ControlledShutdownResponse(Struct struct, short version) { - this.data = new ControlledShutdownResponseData(struct, version); - } - public Errors error() { return Errors.forCode(data.errorCode()); } @@ -56,13 +53,13 @@ public Map errorCounts() { return errorCounts(error()); } - public static ControlledShutdownResponse parse(ByteBuffer buffer, short version) { - return new ControlledShutdownResponse(ApiKeys.CONTROLLED_SHUTDOWN.parseResponse(version, buffer), version); + @Override + public int throttleTimeMs() { + return DEFAULT_THROTTLE_TIME; } - @Override - protected Struct toStruct(short version) { - return data.toStruct(version); + public static ControlledShutdownResponse parse(ByteBuffer buffer, short version) { + return new ControlledShutdownResponse(new ControlledShutdownResponseData(new ByteBufferAccessor(buffer), version)); } public ControlledShutdownResponseData data() { diff --git a/clients/src/main/java/org/apache/kafka/common/requests/CreateAclsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/CreateAclsRequest.java index 3eb88a9b9144b..2ce651565b745 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/CreateAclsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/CreateAclsRequest.java @@ -21,15 +21,15 @@ import org.apache.kafka.common.acl.AclBinding; import org.apache.kafka.common.acl.AclOperation; import org.apache.kafka.common.acl.AclPermissionType; +import org.apache.kafka.common.errors.UnsupportedVersionException; import org.apache.kafka.common.message.CreateAclsRequestData; import org.apache.kafka.common.message.CreateAclsRequestData.AclCreation; import org.apache.kafka.common.message.CreateAclsResponseData; import org.apache.kafka.common.message.CreateAclsResponseData.AclCreationResult; -import org.apache.kafka.common.resource.ResourcePattern; -import org.apache.kafka.common.errors.UnsupportedVersionException; import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.resource.PatternType; +import org.apache.kafka.common.resource.ResourcePattern; import org.apache.kafka.common.resource.ResourceType; import java.nio.ByteBuffer; @@ -48,7 +48,7 @@ public Builder(CreateAclsRequestData data) { @Override public CreateAclsRequest build(short version) { - return new CreateAclsRequest(version, data); + return new CreateAclsRequest(data, version); } @Override @@ -59,23 +59,19 @@ public String toString() { private final CreateAclsRequestData data; - CreateAclsRequest(short version, CreateAclsRequestData data) { + CreateAclsRequest(CreateAclsRequestData data, short version) { super(ApiKeys.CREATE_ACLS, version); validate(data); this.data = data; } - public CreateAclsRequest(Struct struct, short version) { - this(version, new CreateAclsRequestData(struct, version)); - } - public List aclCreations() { return data.creations(); } @Override - protected Struct toStruct() { - return data.toStruct(version()); + protected CreateAclsRequestData data() { + return data; } @Override @@ -88,7 +84,7 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable throwable } public static CreateAclsRequest parse(ByteBuffer buffer, short version) { - return new CreateAclsRequest(ApiKeys.CREATE_ACLS.parseRequest(version, buffer), version); + return new CreateAclsRequest(new CreateAclsRequestData(new ByteBufferAccessor(buffer), version), version); } private void validate(CreateAclsRequestData data) { diff --git a/clients/src/main/java/org/apache/kafka/common/requests/CreateAclsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/CreateAclsResponse.java index 5afce9eba217e..d0149b9a2a0d7 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/CreateAclsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/CreateAclsResponse.java @@ -18,8 +18,8 @@ import org.apache.kafka.common.message.CreateAclsResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.List; @@ -29,16 +29,13 @@ public class CreateAclsResponse extends AbstractResponse { private final CreateAclsResponseData data; public CreateAclsResponse(CreateAclsResponseData data) { + super(ApiKeys.CREATE_ACLS); this.data = data; } - public CreateAclsResponse(Struct struct, short version) { - this.data = new CreateAclsResponseData(struct, version); - } - @Override - protected Struct toStruct(short version) { - return data.toStruct(version); + protected CreateAclsResponseData data() { + return data; } @Override @@ -56,7 +53,7 @@ public Map errorCounts() { } public static CreateAclsResponse parse(ByteBuffer buffer, short version) { - return new CreateAclsResponse(ApiKeys.CREATE_ACLS.responseSchema(version).read(buffer), version); + return new CreateAclsResponse(new CreateAclsResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/CreateDelegationTokenRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/CreateDelegationTokenRequest.java index 61b404a7e0623..e1d4cfaab80bd 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/CreateDelegationTokenRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/CreateDelegationTokenRequest.java @@ -18,8 +18,8 @@ import org.apache.kafka.common.message.CreateDelegationTokenRequestData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.security.auth.KafkaPrincipal; import java.nio.ByteBuffer; @@ -33,18 +33,9 @@ private CreateDelegationTokenRequest(CreateDelegationTokenRequestData data, shor this.data = data; } - public CreateDelegationTokenRequest(Struct struct, short version) { - super(ApiKeys.CREATE_DELEGATION_TOKEN, version); - this.data = new CreateDelegationTokenRequestData(struct, version); - } - public static CreateDelegationTokenRequest parse(ByteBuffer buffer, short version) { - return new CreateDelegationTokenRequest(ApiKeys.CREATE_DELEGATION_TOKEN.parseRequest(version, buffer), version); - } - - @Override - protected Struct toStruct() { - return data.toStruct(version()); + return new CreateDelegationTokenRequest(new CreateDelegationTokenRequestData(new ByteBufferAccessor(buffer), version), + version); } public CreateDelegationTokenRequestData data() { diff --git a/clients/src/main/java/org/apache/kafka/common/requests/CreateDelegationTokenResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/CreateDelegationTokenResponse.java index 9796d68a82b99..9d39b6b641345 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/CreateDelegationTokenResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/CreateDelegationTokenResponse.java @@ -18,8 +18,8 @@ import org.apache.kafka.common.message.CreateDelegationTokenResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.security.auth.KafkaPrincipal; import java.nio.ByteBuffer; @@ -30,15 +30,13 @@ public class CreateDelegationTokenResponse extends AbstractResponse { private final CreateDelegationTokenResponseData data; public CreateDelegationTokenResponse(CreateDelegationTokenResponseData data) { + super(ApiKeys.CREATE_DELEGATION_TOKEN); this.data = data; } - public CreateDelegationTokenResponse(Struct struct, short version) { - this.data = new CreateDelegationTokenResponseData(struct, version); - } - public static CreateDelegationTokenResponse parse(ByteBuffer buffer, short version) { - return new CreateDelegationTokenResponse(ApiKeys.CREATE_DELEGATION_TOKEN.responseSchema(version).read(buffer), version); + return new CreateDelegationTokenResponse( + new CreateDelegationTokenResponseData(new ByteBufferAccessor(buffer), version)); } public static CreateDelegationTokenResponse prepareResponse(int throttleTimeMs, @@ -75,11 +73,6 @@ public Map errorCounts() { return errorCounts(error()); } - @Override - protected Struct toStruct(short version) { - return data.toStruct(version); - } - @Override public int throttleTimeMs() { return data.throttleTimeMs(); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/CreatePartitionsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/CreatePartitionsRequest.java index 77d4175c339de..8b0b9f81cc92c 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/CreatePartitionsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/CreatePartitionsRequest.java @@ -22,7 +22,7 @@ import org.apache.kafka.common.message.CreatePartitionsResponseData; import org.apache.kafka.common.message.CreatePartitionsResponseData.CreatePartitionsTopicResult; import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import java.nio.ByteBuffer; @@ -55,15 +55,6 @@ public String toString() { this.data = data; } - public CreatePartitionsRequest(Struct struct, short apiVersion) { - this(new CreatePartitionsRequestData(struct, apiVersion), apiVersion); - } - - @Override - protected Struct toStruct() { - return data.toStruct(version()); - } - public CreatePartitionsRequestData data() { return data; } @@ -85,6 +76,6 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { } public static CreatePartitionsRequest parse(ByteBuffer buffer, short version) { - return new CreatePartitionsRequest(ApiKeys.CREATE_PARTITIONS.parseRequest(version, buffer), version); + return new CreatePartitionsRequest(new CreatePartitionsRequestData(new ByteBufferAccessor(buffer), version), version); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/CreatePartitionsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/CreatePartitionsResponse.java index d658b0e4c230d..e0af04b07ae6a 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/CreatePartitionsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/CreatePartitionsResponse.java @@ -19,35 +19,26 @@ import org.apache.kafka.common.message.CreatePartitionsResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.HashMap; import java.util.Map; - public class CreatePartitionsResponse extends AbstractResponse { private final CreatePartitionsResponseData data; public CreatePartitionsResponse(CreatePartitionsResponseData data) { + super(ApiKeys.CREATE_PARTITIONS); this.data = data; } - public CreatePartitionsResponse(Struct struct, short version) { - this.data = new CreatePartitionsResponseData(struct, version); - } - public CreatePartitionsResponseData data() { return data; } - @Override - protected Struct toStruct(short version) { - return data.toStruct(version); - } - @Override public Map errorCounts() { Map counts = new HashMap<>(); @@ -58,7 +49,7 @@ public Map errorCounts() { } public static CreatePartitionsResponse parse(ByteBuffer buffer, short version) { - return new CreatePartitionsResponse(ApiKeys.CREATE_PARTITIONS.parseResponse(version, buffer), version); + return new CreatePartitionsResponse(new CreatePartitionsResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/CreateTopicsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/CreateTopicsRequest.java index dd26e5642632e..7ec2c9adde3c4 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/CreateTopicsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/CreateTopicsRequest.java @@ -25,7 +25,7 @@ import org.apache.kafka.common.message.CreateTopicsResponseData; import org.apache.kafka.common.message.CreateTopicsResponseData.CreatableTopicResult; import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.protocol.ByteBufferAccessor; public class CreateTopicsRequest extends AbstractRequest { public static class Builder extends AbstractRequest.Builder { @@ -72,16 +72,11 @@ public String toString() { public static final int NO_NUM_PARTITIONS = -1; public static final short NO_REPLICATION_FACTOR = -1; - private CreateTopicsRequest(CreateTopicsRequestData data, short version) { + public CreateTopicsRequest(CreateTopicsRequestData data, short version) { super(ApiKeys.CREATE_TOPICS, version); this.data = data; } - public CreateTopicsRequest(Struct struct, short version) { - super(ApiKeys.CREATE_TOPICS, version); - this.data = new CreateTopicsRequestData(struct, version); - } - public CreateTopicsRequestData data() { return data; } @@ -103,14 +98,6 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { } public static CreateTopicsRequest parse(ByteBuffer buffer, short version) { - return new CreateTopicsRequest(ApiKeys.CREATE_TOPICS.parseRequest(version, buffer), version); - } - - /** - * Visible for testing. - */ - @Override - public Struct toStruct() { - return data.toStruct(version()); + return new CreateTopicsRequest(new CreateTopicsRequestData(new ByteBufferAccessor(buffer), version), version); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/CreateTopicsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/CreateTopicsResponse.java index 7b64684d4dd4a..c15da20dfc79a 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/CreateTopicsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/CreateTopicsResponse.java @@ -19,8 +19,8 @@ import org.apache.kafka.common.message.CreateTopicsResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.HashMap; @@ -46,22 +46,14 @@ public class CreateTopicsResponse extends AbstractResponse { private final CreateTopicsResponseData data; public CreateTopicsResponse(CreateTopicsResponseData data) { + super(ApiKeys.CREATE_TOPICS); this.data = data; } - public CreateTopicsResponse(Struct struct, short version) { - this.data = new CreateTopicsResponseData(struct, version); - } - public CreateTopicsResponseData data() { return data; } - @Override - protected Struct toStruct(short version) { - return data.toStruct(version); - } - @Override public int throttleTimeMs() { return data.throttleTimeMs(); @@ -77,8 +69,7 @@ public Map errorCounts() { } public static CreateTopicsResponse parse(ByteBuffer buffer, short version) { - return new CreateTopicsResponse( - ApiKeys.CREATE_TOPICS.responseSchema(version).read(buffer), version); + return new CreateTopicsResponse(new CreateTopicsResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DeleteAclsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DeleteAclsRequest.java index b020b2b0ae3d3..6face08f5675d 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DeleteAclsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DeleteAclsRequest.java @@ -28,7 +28,7 @@ import org.apache.kafka.common.message.DeleteAclsResponseData; import org.apache.kafka.common.message.DeleteAclsResponseData.DeleteAclsFilterResult; import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.resource.PatternType; import org.apache.kafka.common.resource.ResourcePatternFilter; import org.apache.kafka.common.resource.ResourceType; @@ -49,7 +49,7 @@ public Builder(DeleteAclsRequestData data) { @Override public DeleteAclsRequest build(short version) { - return new DeleteAclsRequest(version, data); + return new DeleteAclsRequest(data, version); } @Override @@ -61,7 +61,7 @@ public String toString() { private final DeleteAclsRequestData data; - private DeleteAclsRequest(short version, DeleteAclsRequestData data) { + private DeleteAclsRequest(DeleteAclsRequestData data, short version) { super(ApiKeys.DELETE_ACLS, version); this.data = data; normalizeAndValidate(); @@ -95,18 +95,13 @@ else if (patternType != PatternType.LITERAL) } } - public DeleteAclsRequest(Struct struct, short version) { - super(ApiKeys.DELETE_ACLS, version); - this.data = new DeleteAclsRequestData(struct, version); - } - public List filters() { return data.filters().stream().map(DeleteAclsRequest::aclBindingFilter).collect(Collectors.toList()); } @Override - protected Struct toStruct() { - return data.toStruct(version()); + protected DeleteAclsRequestData data() { + return data; } @Override @@ -118,11 +113,11 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable throwable .setErrorMessage(apiError.message())); return new DeleteAclsResponse(new DeleteAclsResponseData() .setThrottleTimeMs(throttleTimeMs) - .setFilterResults(filterResults)); + .setFilterResults(filterResults), version()); } public static DeleteAclsRequest parse(ByteBuffer buffer, short version) { - return new DeleteAclsRequest(DELETE_ACLS.parseRequest(version, buffer), version); + return new DeleteAclsRequest(new DeleteAclsRequestData(new ByteBufferAccessor(buffer), version), version); } public static DeleteAclsFilter deleteAclsFilter(AclBindingFilter filter) { diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DeleteAclsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DeleteAclsResponse.java index 33d72a1df77fb..3ff8a9834f888 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DeleteAclsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DeleteAclsResponse.java @@ -18,17 +18,17 @@ import org.apache.kafka.common.acl.AccessControlEntry; import org.apache.kafka.common.acl.AclBinding; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; +import org.apache.kafka.common.resource.ResourcePattern; import org.apache.kafka.common.acl.AclOperation; import org.apache.kafka.common.acl.AclPermissionType; import org.apache.kafka.common.errors.UnsupportedVersionException; import org.apache.kafka.common.message.DeleteAclsResponseData; import org.apache.kafka.common.message.DeleteAclsResponseData.DeleteAclsFilterResult; import org.apache.kafka.common.message.DeleteAclsResponseData.DeleteAclsMatchingAcl; -import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.resource.PatternType; -import org.apache.kafka.common.resource.ResourcePattern; import org.apache.kafka.common.resource.ResourceType; import org.apache.kafka.server.authorizer.AclDeleteResult; import org.slf4j.Logger; @@ -44,18 +44,15 @@ public class DeleteAclsResponse extends AbstractResponse { private final DeleteAclsResponseData data; - public DeleteAclsResponse(DeleteAclsResponseData data) { + public DeleteAclsResponse(DeleteAclsResponseData data, short version) { + super(ApiKeys.DELETE_ACLS); this.data = data; - } - - public DeleteAclsResponse(Struct struct, short version) { - data = new DeleteAclsResponseData(struct, version); + validate(version); } @Override - protected Struct toStruct(short version) { - validate(version); - return data.toStruct(version); + protected DeleteAclsResponseData data() { + return data; } @Override @@ -73,7 +70,7 @@ public Map errorCounts() { } public static DeleteAclsResponse parse(ByteBuffer buffer, short version) { - return new DeleteAclsResponse(ApiKeys.DELETE_ACLS.parseResponse(version, buffer), version); + return new DeleteAclsResponse(new DeleteAclsResponseData(new ByteBufferAccessor(buffer), version), version); } public String toString() { diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsRequest.java index 2ad947c5904be..d09a4d4c02d77 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsRequest.java @@ -21,8 +21,8 @@ import org.apache.kafka.common.message.DeleteGroupsResponseData.DeletableGroupResult; import org.apache.kafka.common.message.DeleteGroupsResponseData.DeletableGroupResultCollection; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; @@ -53,11 +53,6 @@ public DeleteGroupsRequest(DeleteGroupsRequestData data, short version) { this.data = data; } - public DeleteGroupsRequest(Struct struct, short version) { - super(ApiKeys.DELETE_GROUPS, version); - this.data = new DeleteGroupsRequestData(struct, version); - } - @Override public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { Errors error = Errors.forException(e); @@ -76,11 +71,11 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { } public static DeleteGroupsRequest parse(ByteBuffer buffer, short version) { - return new DeleteGroupsRequest(ApiKeys.DELETE_GROUPS.parseRequest(version, buffer), version); + return new DeleteGroupsRequest(new DeleteGroupsRequestData(new ByteBufferAccessor(buffer), version), version); } @Override - protected Struct toStruct() { - return data.toStruct(version()); + protected DeleteGroupsRequestData data() { + return data; } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsResponse.java index ad171c8f9d03b..a8e8d482de1d1 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DeleteGroupsResponse.java @@ -19,8 +19,8 @@ import org.apache.kafka.common.message.DeleteGroupsResponseData; import org.apache.kafka.common.message.DeleteGroupsResponseData.DeletableGroupResult; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.HashMap; @@ -42,21 +42,13 @@ public class DeleteGroupsResponse extends AbstractResponse { public final DeleteGroupsResponseData data; public DeleteGroupsResponse(DeleteGroupsResponseData data) { + super(ApiKeys.DELETE_GROUPS); this.data = data; } - public DeleteGroupsResponse(Struct struct) { - short latestVersion = (short) (DeleteGroupsResponseData.SCHEMAS.length - 1); - this.data = new DeleteGroupsResponseData(struct, latestVersion); - } - - public DeleteGroupsResponse(Struct struct, short version) { - this.data = new DeleteGroupsResponseData(struct, version); - } - @Override - protected Struct toStruct(short version) { - return data.toStruct(version); + protected DeleteGroupsResponseData data() { + return data; } public Map errors() { @@ -85,7 +77,7 @@ public Map errorCounts() { } public static DeleteGroupsResponse parse(ByteBuffer buffer, short version) { - return new DeleteGroupsResponse(ApiKeys.DELETE_GROUPS.parseResponse(version, buffer), version); + return new DeleteGroupsResponse(new DeleteGroupsResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DeleteRecordsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DeleteRecordsRequest.java index 1939923c554e4..a4f62e18f675a 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DeleteRecordsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DeleteRecordsRequest.java @@ -22,8 +22,8 @@ import org.apache.kafka.common.message.DeleteRecordsResponseData; import org.apache.kafka.common.message.DeleteRecordsResponseData.DeleteRecordsTopicResult; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; @@ -57,16 +57,7 @@ private DeleteRecordsRequest(DeleteRecordsRequestData data, short version) { this.data = data; } - public DeleteRecordsRequest(Struct struct, short version) { - super(ApiKeys.DELETE_RECORDS, version); - this.data = new DeleteRecordsRequestData(struct, version); - } - @Override - protected Struct toStruct() { - return data.toStruct(version()); - } - public DeleteRecordsRequestData data() { return data; } @@ -89,6 +80,6 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { } public static DeleteRecordsRequest parse(ByteBuffer buffer, short version) { - return new DeleteRecordsRequest(ApiKeys.DELETE_RECORDS.parseRequest(version, buffer), version); + return new DeleteRecordsRequest(new DeleteRecordsRequestData(new ByteBufferAccessor(buffer), version), version); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DeleteRecordsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DeleteRecordsResponse.java index 968a35bd32c59..ef34102f8a8fd 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DeleteRecordsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DeleteRecordsResponse.java @@ -19,8 +19,8 @@ import org.apache.kafka.common.message.DeleteRecordsResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.HashMap; @@ -42,18 +42,10 @@ public class DeleteRecordsResponse extends AbstractResponse { */ public DeleteRecordsResponse(DeleteRecordsResponseData data) { + super(ApiKeys.DELETE_RECORDS); this.data = data; } - public DeleteRecordsResponse(Struct struct, short version) { - this.data = new DeleteRecordsResponseData(struct, version); - } - - @Override - protected Struct toStruct(short version) { - return data.toStruct(version); - } - public DeleteRecordsResponseData data() { return data; } @@ -75,7 +67,7 @@ public Map errorCounts() { } public static DeleteRecordsResponse parse(ByteBuffer buffer, short version) { - return new DeleteRecordsResponse(ApiKeys.DELETE_RECORDS.parseResponse(version, buffer), version); + return new DeleteRecordsResponse(new DeleteRecordsResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DeleteTopicsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DeleteTopicsRequest.java index 440acfd2b838a..5ab64184aeab0 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DeleteTopicsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DeleteTopicsRequest.java @@ -20,15 +20,12 @@ import org.apache.kafka.common.message.DeleteTopicsResponseData; import org.apache.kafka.common.message.DeleteTopicsResponseData.DeletableTopicResult; import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import java.nio.ByteBuffer; public class DeleteTopicsRequest extends AbstractRequest { - private DeleteTopicsRequestData data; - private final short version; - public static class Builder extends AbstractRequest.Builder { private DeleteTopicsRequestData data; @@ -48,21 +45,11 @@ public String toString() { } } + private DeleteTopicsRequestData data; + private DeleteTopicsRequest(DeleteTopicsRequestData data, short version) { super(ApiKeys.DELETE_TOPICS, version); this.data = data; - this.version = version; - } - - public DeleteTopicsRequest(Struct struct, short version) { - super(ApiKeys.DELETE_TOPICS, version); - this.data = new DeleteTopicsRequestData(struct, version); - this.version = version; - } - - @Override - protected Struct toStruct() { - return data.toStruct(version); } public DeleteTopicsRequestData data() { @@ -72,7 +59,7 @@ public DeleteTopicsRequestData data() { @Override public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { DeleteTopicsResponseData response = new DeleteTopicsResponseData(); - if (version >= 1) { + if (version() >= 1) { response.setThrottleTimeMs(throttleTimeMs); } ApiError apiError = ApiError.fromThrowable(e); @@ -85,7 +72,7 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { } public static DeleteTopicsRequest parse(ByteBuffer buffer, short version) { - return new DeleteTopicsRequest(ApiKeys.DELETE_TOPICS.parseRequest(version, buffer), version); + return new DeleteTopicsRequest(new DeleteTopicsRequestData(new ByteBufferAccessor(buffer), version), version); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DeleteTopicsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DeleteTopicsResponse.java index bad23f6a2bd0b..3584ddfbfc497 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DeleteTopicsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DeleteTopicsResponse.java @@ -18,8 +18,8 @@ import org.apache.kafka.common.message.DeleteTopicsResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.HashMap; @@ -41,18 +41,10 @@ public class DeleteTopicsResponse extends AbstractResponse { private DeleteTopicsResponseData data; public DeleteTopicsResponse(DeleteTopicsResponseData data) { + super(ApiKeys.DELETE_TOPICS); this.data = data; } - public DeleteTopicsResponse(Struct struct, short version) { - this.data = new DeleteTopicsResponseData(struct, version); - } - - @Override - protected Struct toStruct(short version) { - return data.toStruct(version); - } - @Override public int throttleTimeMs() { return data.throttleTimeMs(); @@ -72,7 +64,7 @@ public Map errorCounts() { } public static DeleteTopicsResponse parse(ByteBuffer buffer, short version) { - return new DeleteTopicsResponse(ApiKeys.DELETE_TOPICS.parseResponse(version, buffer), version); + return new DeleteTopicsResponse(new DeleteTopicsResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeAclsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeAclsRequest.java index 98059cfddbb8d..e6f9c38fd598b 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeAclsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeAclsRequest.java @@ -24,7 +24,7 @@ import org.apache.kafka.common.message.DescribeAclsRequestData; import org.apache.kafka.common.message.DescribeAclsResponseData; import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.resource.PatternType; import org.apache.kafka.common.resource.ResourcePatternFilter; import org.apache.kafka.common.resource.ResourceType; @@ -89,20 +89,10 @@ else if (patternType != PatternType.LITERAL) } } - public DescribeAclsRequest(Struct struct, short version) { - super(ApiKeys.DESCRIBE_ACLS, version); - this.data = new DescribeAclsRequestData(struct, version); - } - public DescribeAclsRequestData data() { return data; } - @Override - protected Struct toStruct() { - return data.toStruct(version()); - } - @Override public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable throwable) { ApiError error = ApiError.fromThrowable(throwable); @@ -110,11 +100,11 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable throwable .setThrottleTimeMs(throttleTimeMs) .setErrorCode(error.error().code()) .setErrorMessage(error.message()); - return new DescribeAclsResponse(response); + return new DescribeAclsResponse(response, version()); } public static DescribeAclsRequest parse(ByteBuffer buffer, short version) { - return new DescribeAclsRequest(ApiKeys.DESCRIBE_ACLS.parseRequest(version, buffer), version); + return new DescribeAclsRequest(new DescribeAclsRequestData(new ByteBufferAccessor(buffer), version), version); } public AclBindingFilter filter() { diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeAclsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeAclsResponse.java index 3a41cb3de85b1..4308c9ea45bf7 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeAclsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeAclsResponse.java @@ -17,6 +17,15 @@ package org.apache.kafka.common.requests; +import org.apache.kafka.common.acl.AccessControlEntry; +import org.apache.kafka.common.acl.AclBinding; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; +import org.apache.kafka.common.resource.PatternType; +import org.apache.kafka.common.resource.ResourcePattern; +import org.apache.kafka.common.errors.UnsupportedVersionException; +import org.apache.kafka.common.protocol.Errors; + import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collection; @@ -24,40 +33,37 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.Stream; -import org.apache.kafka.common.acl.AccessControlEntry; -import org.apache.kafka.common.acl.AclBinding; import org.apache.kafka.common.acl.AclOperation; import org.apache.kafka.common.acl.AclPermissionType; -import org.apache.kafka.common.errors.UnsupportedVersionException; import org.apache.kafka.common.message.DescribeAclsResponseData; import org.apache.kafka.common.message.DescribeAclsResponseData.AclDescription; import org.apache.kafka.common.message.DescribeAclsResponseData.DescribeAclsResource; -import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; -import org.apache.kafka.common.resource.PatternType; -import org.apache.kafka.common.resource.ResourcePattern; import org.apache.kafka.common.resource.ResourceType; public class DescribeAclsResponse extends AbstractResponse { private final DescribeAclsResponseData data; - public DescribeAclsResponse(DescribeAclsResponseData data) { + public DescribeAclsResponse(DescribeAclsResponseData data, short version) { + super(ApiKeys.DESCRIBE_ACLS); this.data = data; + validate(Optional.of(version)); } - public DescribeAclsResponse(Struct struct, short version) { - this.data = new DescribeAclsResponseData(struct, version); + // Skips version validation, visible for testing + DescribeAclsResponse(DescribeAclsResponseData data) { + super(ApiKeys.DESCRIBE_ACLS); + this.data = data; + validate(Optional.empty()); } @Override - protected Struct toStruct(short version) { - validate(version); - return data.toStruct(version); + protected DescribeAclsResponseData data() { + return data; } @Override @@ -79,7 +85,7 @@ public List acls() { } public static DescribeAclsResponse parse(ByteBuffer buffer, short version) { - return new DescribeAclsResponse(ApiKeys.DESCRIBE_ACLS.responseSchema(version).read(buffer), version); + return new DescribeAclsResponse(new DescribeAclsResponseData(new ByteBufferAccessor(buffer), version), version); } @Override @@ -87,8 +93,8 @@ public boolean shouldClientThrottle(short version) { return version >= 1; } - private void validate(short version) { - if (version == 0) { + private void validate(Optional version) { + if (version.isPresent() && version.get() == 0) { final boolean unsupported = acls().stream() .anyMatch(acl -> acl.patternType() != PatternType.LITERAL.code()); if (unsupported) { diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasRequest.java index 593cc6a9aace0..1f167b0ee3c2b 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasRequest.java @@ -18,11 +18,13 @@ import org.apache.kafka.common.message.DescribeClientQuotasRequestData; import org.apache.kafka.common.message.DescribeClientQuotasRequestData.ComponentData; +import org.apache.kafka.common.message.DescribeClientQuotasResponseData; import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.quota.ClientQuotaFilter; import org.apache.kafka.common.quota.ClientQuotaFilterComponent; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; @@ -77,11 +79,6 @@ public DescribeClientQuotasRequest(DescribeClientQuotasRequestData data, short v this.data = data; } - public DescribeClientQuotasRequest(Struct struct, short version) { - super(ApiKeys.DESCRIBE_CLIENT_QUOTAS, version); - this.data = new DescribeClientQuotasRequestData(struct, version); - } - public ClientQuotaFilter filter() { List components = new ArrayList<>(data.components().size()); for (ComponentData componentData : data.components()) { @@ -109,12 +106,23 @@ public ClientQuotaFilter filter() { } @Override - public DescribeClientQuotasResponse getErrorResponse(int throttleTimeMs, Throwable e) { - return new DescribeClientQuotasResponse(throttleTimeMs, e); + protected DescribeClientQuotasRequestData data() { + return data; } @Override - protected Struct toStruct() { - return data.toStruct(version()); + public DescribeClientQuotasResponse getErrorResponse(int throttleTimeMs, Throwable e) { + ApiError error = ApiError.fromThrowable(e); + return new DescribeClientQuotasResponse(new DescribeClientQuotasResponseData() + .setThrottleTimeMs(throttleTimeMs) + .setErrorCode(error.error().code()) + .setErrorMessage(error.message()) + .setEntries(null)); + } + + public static DescribeClientQuotasRequest parse(ByteBuffer buffer, short version) { + return new DescribeClientQuotasRequest(new DescribeClientQuotasRequestData(new ByteBufferAccessor(buffer), version), + version); } + } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasResponse.java index bda3673c27f2c..94fca64221d0f 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasResponse.java @@ -22,8 +22,8 @@ import org.apache.kafka.common.message.DescribeClientQuotasResponseData.EntryData; import org.apache.kafka.common.message.DescribeClientQuotasResponseData.ValueData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.quota.ClientQuotaEntity; import java.nio.ByteBuffer; @@ -36,48 +36,9 @@ public class DescribeClientQuotasResponse extends AbstractResponse { private final DescribeClientQuotasResponseData data; - public DescribeClientQuotasResponse(Map> entities, int throttleTimeMs) { - List entries = new ArrayList<>(entities.size()); - for (Map.Entry> entry : entities.entrySet()) { - ClientQuotaEntity quotaEntity = entry.getKey(); - List entityData = new ArrayList<>(quotaEntity.entries().size()); - for (Map.Entry entityEntry : quotaEntity.entries().entrySet()) { - entityData.add(new EntityData() - .setEntityType(entityEntry.getKey()) - .setEntityName(entityEntry.getValue())); - } - - Map quotaValues = entry.getValue(); - List valueData = new ArrayList<>(quotaValues.size()); - for (Map.Entry valuesEntry : entry.getValue().entrySet()) { - valueData.add(new ValueData() - .setKey(valuesEntry.getKey()) - .setValue(valuesEntry.getValue())); - } - - entries.add(new EntryData() - .setEntity(entityData) - .setValues(valueData)); - } - - this.data = new DescribeClientQuotasResponseData() - .setThrottleTimeMs(throttleTimeMs) - .setErrorCode((short) 0) - .setErrorMessage(null) - .setEntries(entries); - } - - public DescribeClientQuotasResponse(int throttleTimeMs, Throwable e) { - ApiError apiError = ApiError.fromThrowable(e); - this.data = new DescribeClientQuotasResponseData() - .setThrottleTimeMs(throttleTimeMs) - .setErrorCode(apiError.error().code()) - .setErrorMessage(apiError.message()) - .setEntries(null); - } - - public DescribeClientQuotasResponse(Struct struct, short version) { - this.data = new DescribeClientQuotasResponseData(struct, version); + public DescribeClientQuotasResponse(DescribeClientQuotasResponseData data) { + super(ApiKeys.DESCRIBE_CLIENT_QUOTAS); + this.data = data; } public void complete(KafkaFutureImpl>> future) { @@ -110,16 +71,49 @@ public int throttleTimeMs() { } @Override - public Map errorCounts() { - return errorCounts(Errors.forCode(data.errorCode())); + protected DescribeClientQuotasResponseData data() { + return data; } @Override - protected Struct toStruct(short version) { - return data.toStruct(version); + public Map errorCounts() { + return errorCounts(Errors.forCode(data.errorCode())); } public static DescribeClientQuotasResponse parse(ByteBuffer buffer, short version) { - return new DescribeClientQuotasResponse(ApiKeys.DESCRIBE_CLIENT_QUOTAS.parseResponse(version, buffer), version); + return new DescribeClientQuotasResponse(new DescribeClientQuotasResponseData(new ByteBufferAccessor(buffer), version)); + } + + public static DescribeClientQuotasResponse fromQuotaEntities(Map> entities, + int throttleTimeMs) { + List entries = new ArrayList<>(entities.size()); + for (Map.Entry> entry : entities.entrySet()) { + ClientQuotaEntity quotaEntity = entry.getKey(); + List entityData = new ArrayList<>(quotaEntity.entries().size()); + for (Map.Entry entityEntry : quotaEntity.entries().entrySet()) { + entityData.add(new EntityData() + .setEntityType(entityEntry.getKey()) + .setEntityName(entityEntry.getValue())); + } + + Map quotaValues = entry.getValue(); + List valueData = new ArrayList<>(quotaValues.size()); + for (Map.Entry valuesEntry : entry.getValue().entrySet()) { + valueData.add(new ValueData() + .setKey(valuesEntry.getKey()) + .setValue(valuesEntry.getValue())); + } + + entries.add(new EntryData() + .setEntity(entityData) + .setValues(valueData)); + } + + return new DescribeClientQuotasResponse(new DescribeClientQuotasResponseData() + .setThrottleTimeMs(throttleTimeMs) + .setErrorCode((short) 0) + .setErrorMessage(null) + .setEntries(entries)); } + } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeConfigsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeConfigsRequest.java index f17896e66e0ca..d612ca80949a0 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeConfigsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeConfigsRequest.java @@ -19,8 +19,8 @@ import org.apache.kafka.common.message.DescribeConfigsRequestData; import org.apache.kafka.common.message.DescribeConfigsResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.stream.Collectors; @@ -48,20 +48,11 @@ public DescribeConfigsRequest(DescribeConfigsRequestData data, short version) { this.data = data; } - public DescribeConfigsRequest(Struct struct, short version) { - super(ApiKeys.DESCRIBE_CONFIGS, version); - this.data = new DescribeConfigsRequestData(struct, version); - } - + @Override public DescribeConfigsRequestData data() { return data; } - @Override - protected Struct toStruct() { - return data.toStruct(version()); - } - @Override public DescribeConfigsResponse getErrorResponse(int throttleTimeMs, Throwable e) { Errors error = Errors.forException(e); @@ -77,6 +68,6 @@ public DescribeConfigsResponse getErrorResponse(int throttleTimeMs, Throwable e) } public static DescribeConfigsRequest parse(ByteBuffer buffer, short version) { - return new DescribeConfigsRequest(ApiKeys.DESCRIBE_CONFIGS.parseRequest(version, buffer), version); + return new DescribeConfigsRequest(new DescribeConfigsRequestData(new ByteBufferAccessor(buffer), version), version); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeConfigsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeConfigsResponse.java index 3767226395e81..aa7a713e8ab4e 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeConfigsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeConfigsResponse.java @@ -19,10 +19,9 @@ import org.apache.kafka.common.config.ConfigResource; import org.apache.kafka.common.message.DescribeConfigsResponseData; -import org.apache.kafka.common.message.DescribeConfigsResponseData.DescribeConfigsResult; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.Collection; @@ -219,13 +218,16 @@ public Map re private final DescribeConfigsResponseData data; public DescribeConfigsResponse(DescribeConfigsResponseData data) { + super(ApiKeys.DESCRIBE_CONFIGS); this.data = data; } - public DescribeConfigsResponse(Struct struct, short version) { - this.data = new DescribeConfigsResponseData(struct, version); + // This constructor should only be used after deserialization, it has special handling for version 0 + private DescribeConfigsResponse(DescribeConfigsResponseData data, short version) { + super(ApiKeys.DESCRIBE_CONFIGS); + this.data = data; if (version == 0) { - for (DescribeConfigsResult result : data.results()) { + for (DescribeConfigsResponseData.DescribeConfigsResult result : data.results()) { for (DescribeConfigsResponseData.DescribeConfigsResourceResult config : result.configs()) { if (config.isDefault()) { config.setConfigSource(ConfigSource.DEFAULT_CONFIG.id); @@ -243,6 +245,7 @@ public DescribeConfigsResponse(Struct struct, short version) { } } + @Override public DescribeConfigsResponseData data() { return data; } @@ -261,13 +264,8 @@ public Map errorCounts() { return errorCounts; } - @Override - protected Struct toStruct(short version) { - return data.toStruct(version); - } - public static DescribeConfigsResponse parse(ByteBuffer buffer, short version) { - return new DescribeConfigsResponse(ApiKeys.DESCRIBE_CONFIGS.parseResponse(version, buffer), version); + return new DescribeConfigsResponse(new DescribeConfigsResponseData(new ByteBufferAccessor(buffer), version), version); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenRequest.java index 7db3209d621fb..3e67140d375d0 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenRequest.java @@ -18,17 +18,16 @@ import org.apache.kafka.common.message.DescribeDelegationTokenRequestData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.security.auth.KafkaPrincipal; +import java.nio.ByteBuffer; import java.util.List; import java.util.stream.Collectors; public class DescribeDelegationTokenRequest extends AbstractRequest { - private final DescribeDelegationTokenRequestData data; - public static class Builder extends AbstractRequest.Builder { private final DescribeDelegationTokenRequestData data; @@ -54,21 +53,13 @@ public String toString() { } } - public DescribeDelegationTokenRequest(Struct struct, short version) { - super(ApiKeys.DESCRIBE_DELEGATION_TOKEN, version); - this.data = new DescribeDelegationTokenRequestData(struct, version); - } + private final DescribeDelegationTokenRequestData data; public DescribeDelegationTokenRequest(DescribeDelegationTokenRequestData data, short version) { super(ApiKeys.DESCRIBE_DELEGATION_TOKEN, version); this.data = data; } - @Override - protected Struct toStruct() { - return data.toStruct(version()); - } - public DescribeDelegationTokenRequestData data() { return data; } @@ -81,4 +72,9 @@ public boolean ownersListEmpty() { public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { return new DescribeDelegationTokenResponse(throttleTimeMs, Errors.forException(e)); } + + public static DescribeDelegationTokenRequest parse(ByteBuffer buffer, short version) { + return new DescribeDelegationTokenRequest(new DescribeDelegationTokenRequestData( + new ByteBufferAccessor(buffer), version), version); + } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenResponse.java index 91e7db0fccb1e..f6579a06b72da 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenResponse.java @@ -20,8 +20,8 @@ import org.apache.kafka.common.message.DescribeDelegationTokenResponseData.DescribedDelegationToken; import org.apache.kafka.common.message.DescribeDelegationTokenResponseData.DescribedDelegationTokenRenewer; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.security.auth.KafkaPrincipal; import org.apache.kafka.common.security.token.delegation.DelegationToken; import org.apache.kafka.common.security.token.delegation.TokenInformation; @@ -37,6 +37,7 @@ public class DescribeDelegationTokenResponse extends AbstractResponse { private final DescribeDelegationTokenResponseData data; public DescribeDelegationTokenResponse(int throttleTimeMs, Errors error, List tokens) { + super(ApiKeys.DESCRIBE_DELEGATION_TOKEN); List describedDelegationTokenList = tokens .stream() .map(dt -> new DescribedDelegationToken() @@ -63,12 +64,14 @@ public DescribeDelegationTokenResponse(int throttleTimeMs, Errors error) { this(throttleTimeMs, error, new ArrayList<>()); } - public DescribeDelegationTokenResponse(Struct struct, short version) { - this.data = new DescribeDelegationTokenResponseData(struct, version); + public DescribeDelegationTokenResponse(DescribeDelegationTokenResponseData data) { + super(ApiKeys.DESCRIBE_DELEGATION_TOKEN); + this.data = data; } public static DescribeDelegationTokenResponse parse(ByteBuffer buffer, short version) { - return new DescribeDelegationTokenResponse(ApiKeys.DESCRIBE_DELEGATION_TOKEN.responseSchema(version).read(buffer), version); + return new DescribeDelegationTokenResponse(new DescribeDelegationTokenResponseData( + new ByteBufferAccessor(buffer), version)); } @Override @@ -77,8 +80,8 @@ public Map errorCounts() { } @Override - protected Struct toStruct(short version) { - return data.toStruct(version); + protected DescribeDelegationTokenResponseData data() { + return data; } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeGroupsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeGroupsRequest.java index 8f003acad5b97..db0490dc1b498 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeGroupsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeGroupsRequest.java @@ -18,8 +18,8 @@ import org.apache.kafka.common.message.DescribeGroupsRequestData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; @@ -46,32 +46,19 @@ public String toString() { } private final DescribeGroupsRequestData data; - private final short version; private DescribeGroupsRequest(DescribeGroupsRequestData data, short version) { super(ApiKeys.DESCRIBE_GROUPS, version); this.data = data; - this.version = version; - } - - public DescribeGroupsRequest(Struct struct, short version) { - super(ApiKeys.DESCRIBE_GROUPS, version); - this.data = new DescribeGroupsRequestData(struct, version); - this.version = version; } public DescribeGroupsRequestData data() { return data; } - @Override - protected Struct toStruct() { - return data.toStruct(version); - } - @Override public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { - if (version == 0) { + if (version() == 0) { return DescribeGroupsResponse.fromError(DEFAULT_THROTTLE_TIME, Errors.forException(e), data.groups()); } else { return DescribeGroupsResponse.fromError(throttleTimeMs, Errors.forException(e), data.groups()); @@ -79,6 +66,6 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { } public static DescribeGroupsRequest parse(ByteBuffer buffer, short version) { - return new DescribeGroupsRequest(ApiKeys.DESCRIBE_GROUPS.parseRequest(version, buffer), version); + return new DescribeGroupsRequest(new DescribeGroupsRequestData(new ByteBufferAccessor(buffer), version), version); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeGroupsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeGroupsResponse.java index 5836c7f3cd0cc..67c43d87ea092 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeGroupsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeGroupsResponse.java @@ -20,8 +20,8 @@ import org.apache.kafka.common.message.DescribeGroupsResponseData.DescribedGroup; import org.apache.kafka.common.message.DescribeGroupsResponseData.DescribedGroupMember; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.utils.Utils; import java.nio.ByteBuffer; @@ -47,13 +47,10 @@ public class DescribeGroupsResponse extends AbstractResponse { private DescribeGroupsResponseData data; public DescribeGroupsResponse(DescribeGroupsResponseData data) { + super(ApiKeys.DESCRIBE_GROUPS); this.data = data; } - public DescribeGroupsResponse(Struct struct, short version) { - this.data = new DescribeGroupsResponseData(struct, version); - } - public static DescribedGroupMember groupMember( final String memberId, final String groupInstanceId, @@ -112,11 +109,6 @@ public DescribeGroupsResponseData data() { return data; } - @Override - protected Struct toStruct(short version) { - return data.toStruct(version); - } - @Override public int throttleTimeMs() { return data.throttleTimeMs(); @@ -129,9 +121,8 @@ public int throttleTimeMs() { @Override public Map errorCounts() { Map errorCounts = new HashMap<>(); - data.groups().forEach(describedGroup -> { - updateErrorCounts(errorCounts, Errors.forCode(describedGroup.errorCode())); - }); + data.groups().forEach(describedGroup -> + updateErrorCounts(errorCounts, Errors.forCode(describedGroup.errorCode()))); return errorCounts; } @@ -149,8 +140,7 @@ public static DescribeGroupsResponse fromError(int throttleTimeMs, Errors error, } public static DescribeGroupsResponse parse(ByteBuffer buffer, short version) { - return new DescribeGroupsResponse( - ApiKeys.DESCRIBE_GROUPS.responseSchema(version).read(buffer), version); + return new DescribeGroupsResponse(new DescribeGroupsResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeLogDirsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeLogDirsRequest.java index a58ef1fedb532..05ca4f0d59391 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeLogDirsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeLogDirsRequest.java @@ -20,7 +20,7 @@ import org.apache.kafka.common.message.DescribeLogDirsRequestData; import org.apache.kafka.common.message.DescribeLogDirsResponseData; import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import java.nio.ByteBuffer; @@ -47,25 +47,16 @@ public String toString() { } } - public DescribeLogDirsRequest(Struct struct, short version) { - super(ApiKeys.DESCRIBE_LOG_DIRS, version); - this.data = new DescribeLogDirsRequestData(struct, version); - } - public DescribeLogDirsRequest(DescribeLogDirsRequestData data, short version) { super(ApiKeys.DESCRIBE_LOG_DIRS, version); this.data = data; } + @Override public DescribeLogDirsRequestData data() { return data; } - @Override - protected Struct toStruct() { - return data.toStruct(version()); - } - @Override public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { return new DescribeLogDirsResponse(new DescribeLogDirsResponseData().setThrottleTimeMs(throttleTimeMs)); @@ -76,6 +67,6 @@ public boolean isAllTopicPartitions() { } public static DescribeLogDirsRequest parse(ByteBuffer buffer, short version) { - return new DescribeLogDirsRequest(ApiKeys.DESCRIBE_LOG_DIRS.parseRequest(version, buffer), version); + return new DescribeLogDirsRequest(new DescribeLogDirsRequestData(new ByteBufferAccessor(buffer), version), version); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeLogDirsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeLogDirsResponse.java index e26fc554dccc1..61149253d5982 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeLogDirsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeLogDirsResponse.java @@ -20,8 +20,8 @@ import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.message.DescribeLogDirsResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.HashMap; @@ -34,11 +34,8 @@ public class DescribeLogDirsResponse extends AbstractResponse { private final DescribeLogDirsResponseData data; - public DescribeLogDirsResponse(Struct struct, short version) { - this.data = new DescribeLogDirsResponseData(struct, version); - } - public DescribeLogDirsResponse(DescribeLogDirsResponseData data) { + super(ApiKeys.DESCRIBE_LOG_DIRS); this.data = data; } @@ -46,11 +43,6 @@ public DescribeLogDirsResponseData data() { return data; } - @Override - protected Struct toStruct(short version) { - return data.toStruct(version); - } - @Override public int throttleTimeMs() { return data.throttleTimeMs(); @@ -66,7 +58,7 @@ public Map errorCounts() { } public static DescribeLogDirsResponse parse(ByteBuffer buffer, short version) { - return new DescribeLogDirsResponse(ApiKeys.DESCRIBE_LOG_DIRS.responseSchema(version).read(buffer), version); + return new DescribeLogDirsResponse(new DescribeLogDirsResponseData(new ByteBufferAccessor(buffer), version)); } // Note this class is part of the public API, reachable from Admin.describeLogDirs() diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeQuorumRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeQuorumRequest.java index b2f5fd1d4a8a7..c318c214b6adf 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeQuorumRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeQuorumRequest.java @@ -20,9 +20,10 @@ import org.apache.kafka.common.message.DescribeQuorumRequestData; import org.apache.kafka.common.message.DescribeQuorumResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -55,13 +56,8 @@ private DescribeQuorumRequest(DescribeQuorumRequestData data, short version) { this.data = data; } - public DescribeQuorumRequest(Struct struct, short version) { - super(ApiKeys.DESCRIBE_QUORUM, version); - this.data = new DescribeQuorumRequestData(struct, version); - } - - public DescribeQuorumRequest(DescribeQuorumRequestData data) { - this(data, (short) (DescribeQuorumRequestData.SCHEMAS.length - 1)); + public static DescribeQuorumRequest parse(ByteBuffer buffer, short version) { + return new DescribeQuorumRequest(new DescribeQuorumRequestData(new ByteBufferAccessor(buffer), version), version); } public static DescribeQuorumRequestData singletonRequest(TopicPartition topicPartition) { @@ -76,8 +72,8 @@ public static DescribeQuorumRequestData singletonRequest(TopicPartition topicPar } @Override - protected Struct toStruct() { - return data.toStruct(version()); + protected DescribeQuorumRequestData data() { + return data; } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeQuorumResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeQuorumResponse.java index ad0ce8aee406c..cb1a5ec277c08 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeQuorumResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeQuorumResponse.java @@ -20,8 +20,8 @@ import org.apache.kafka.common.message.DescribeQuorumResponseData; import org.apache.kafka.common.message.DescribeQuorumResponseData.ReplicaState; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.Collections; @@ -44,18 +44,10 @@ public class DescribeQuorumResponse extends AbstractResponse { public final DescribeQuorumResponseData data; public DescribeQuorumResponse(DescribeQuorumResponseData data) { + super(ApiKeys.DESCRIBE_QUORUM); this.data = data; } - public DescribeQuorumResponse(Struct struct, short version) { - this.data = new DescribeQuorumResponseData(struct, version); - } - - @Override - protected Struct toStruct(short version) { - return data.toStruct(version); - } - @Override public Map errorCounts() { Map errors = new HashMap<>(); @@ -70,6 +62,16 @@ public Map errorCounts() { return errors; } + @Override + protected DescribeQuorumResponseData data() { + return data; + } + + @Override + public int throttleTimeMs() { + return DEFAULT_THROTTLE_TIME; + } + public static DescribeQuorumResponseData singletonResponse(TopicPartition topicPartition, int leaderId, int leaderEpoch, @@ -89,6 +91,6 @@ public static DescribeQuorumResponseData singletonResponse(TopicPartition topicP } public static DescribeQuorumResponse parse(ByteBuffer buffer, short version) { - return new DescribeQuorumResponse(ApiKeys.DESCRIBE_QUORUM.responseSchema(version).read(buffer), version); + return new DescribeQuorumResponse(new DescribeQuorumResponseData(new ByteBufferAccessor(buffer), version)); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeUserScramCredentialsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeUserScramCredentialsRequest.java index e0d6bbac3b074..8b28d6b18628b 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeUserScramCredentialsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeUserScramCredentialsRequest.java @@ -19,7 +19,7 @@ import org.apache.kafka.common.message.DescribeUserScramCredentialsRequestData; import org.apache.kafka.common.message.DescribeUserScramCredentialsResponseData; import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import java.nio.ByteBuffer; @@ -53,30 +53,15 @@ private DescribeUserScramCredentialsRequest(DescribeUserScramCredentialsRequestD this.version = version; } - DescribeUserScramCredentialsRequest(Struct struct, short version) { - super(ApiKeys.DESCRIBE_USER_SCRAM_CREDENTIALS, version); - this.data = new DescribeUserScramCredentialsRequestData(struct, version); - this.version = version; - } - public static DescribeUserScramCredentialsRequest parse(ByteBuffer buffer, short version) { - return new DescribeUserScramCredentialsRequest( - ApiKeys.DESCRIBE_USER_SCRAM_CREDENTIALS.parseRequest(version, buffer), version - ); + return new DescribeUserScramCredentialsRequest(new DescribeUserScramCredentialsRequestData( + new ByteBufferAccessor(buffer), version), version); } public DescribeUserScramCredentialsRequestData data() { return data; } - /** - * Visible for testing. - */ - @Override - public Struct toStruct() { - return data.toStruct(version); - } - @Override public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { ApiError apiError = ApiError.fromThrowable(e); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeUserScramCredentialsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeUserScramCredentialsResponse.java index 3043e023af4e1..a736c2c6876e5 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeUserScramCredentialsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeUserScramCredentialsResponse.java @@ -18,8 +18,8 @@ import org.apache.kafka.common.message.DescribeUserScramCredentialsResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.Map; @@ -28,22 +28,11 @@ public class DescribeUserScramCredentialsResponse extends AbstractResponse { private final DescribeUserScramCredentialsResponseData data; - public DescribeUserScramCredentialsResponse(Struct struct) { - this(struct, ApiKeys.DESCRIBE_USER_SCRAM_CREDENTIALS.latestVersion()); - } - public DescribeUserScramCredentialsResponse(DescribeUserScramCredentialsResponseData responseData) { + super(ApiKeys.DESCRIBE_USER_SCRAM_CREDENTIALS); this.data = responseData; } - DescribeUserScramCredentialsResponse(Struct struct, short version) { - this.data = new DescribeUserScramCredentialsResponseData(struct, version); - } - - public static DescribeUserScramCredentialsResponse parse(ByteBuffer buffer, short version) { - return new DescribeUserScramCredentialsResponse(ApiKeys.DESCRIBE_USER_SCRAM_CREDENTIALS.responseSchema(version).read(buffer), version); - } - public DescribeUserScramCredentialsResponseData data() { return data; } @@ -63,8 +52,7 @@ public Map errorCounts() { return errorCounts(data.results().stream().map(r -> Errors.forCode(r.errorCode()))); } - @Override - protected Struct toStruct(short version) { - return data.toStruct(version); + public static DescribeUserScramCredentialsResponse parse(ByteBuffer buffer, short version) { + return new DescribeUserScramCredentialsResponse(new DescribeUserScramCredentialsResponseData(new ByteBufferAccessor(buffer), version)); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ElectLeadersRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ElectLeadersRequest.java index 025733e39c0bc..89600a9072465 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ElectLeadersRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ElectLeadersRequest.java @@ -30,8 +30,8 @@ import org.apache.kafka.common.message.ElectLeadersResponseData.PartitionResult; import org.apache.kafka.common.message.ElectLeadersResponseData.ReplicaElectionResult; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.MessageUtil; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.utils.CollectionUtils; public class ElectLeadersRequest extends AbstractRequest { @@ -90,11 +90,6 @@ private ElectLeadersRequest(ElectLeadersRequestData data, short version) { this.data = data; } - public ElectLeadersRequest(Struct struct, short version) { - super(ApiKeys.ELECT_LEADERS, version); - this.data = new ElectLeadersRequestData(struct, version); - } - public ElectLeadersRequestData data() { return data; } @@ -124,11 +119,6 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { } public static ElectLeadersRequest parse(ByteBuffer buffer, short version) { - return new ElectLeadersRequest(ApiKeys.ELECT_LEADERS.parseRequest(version, buffer), version); - } - - @Override - protected Struct toStruct() { - return data.toStruct(version()); + return new ElectLeadersRequest(new ElectLeadersRequestData(new ByteBufferAccessor(buffer), version), version); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ElectLeadersResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ElectLeadersResponse.java index 787ed4e939149..65eb09751b12d 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ElectLeadersResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ElectLeadersResponse.java @@ -25,30 +25,18 @@ import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.message.ElectLeadersResponseData; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.message.ElectLeadersResponseData.ReplicaElectionResult; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; public class ElectLeadersResponse extends AbstractResponse { - private final short version; private final ElectLeadersResponseData data; - public ElectLeadersResponse(Struct struct) { - this(struct, ApiKeys.ELECT_LEADERS.latestVersion()); - } - - public ElectLeadersResponse(Struct struct, short version) { - this.version = version; - this.data = new ElectLeadersResponseData(struct, version); - } - - public ElectLeadersResponse( - int throttleTimeMs, - short errorCode, - List electionResults) { - this(throttleTimeMs, errorCode, electionResults, ApiKeys.ELECT_LEADERS.latestVersion()); + public ElectLeadersResponse(ElectLeadersResponseData data) { + super(ApiKeys.ELECT_LEADERS); + this.data = data; } public ElectLeadersResponse( @@ -56,16 +44,11 @@ public ElectLeadersResponse( short errorCode, List electionResults, short version) { - - this.version = version; + super(ApiKeys.ELECT_LEADERS); this.data = new ElectLeadersResponseData(); - data.setThrottleTimeMs(throttleTimeMs); - - if (version >= 1) { + if (version >= 1) data.setErrorCode(errorCode); - } - data.setReplicaElectionResults(electionResults); } @@ -73,15 +56,6 @@ public ElectLeadersResponseData data() { return data; } - public short version() { - return version; - } - - @Override - protected Struct toStruct(short version) { - return data.toStruct(version); - } - @Override public int throttleTimeMs() { return data.throttleTimeMs(); @@ -100,7 +74,7 @@ public Map errorCounts() { } public static ElectLeadersResponse parse(ByteBuffer buffer, short version) { - return new ElectLeadersResponse(ApiKeys.ELECT_LEADERS.responseSchema(version).read(buffer), version); + return new ElectLeadersResponse(new ElectLeadersResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/EndQuorumEpochRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/EndQuorumEpochRequest.java index 017897f5aa227..b9f87d73fbd24 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/EndQuorumEpochRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/EndQuorumEpochRequest.java @@ -20,9 +20,10 @@ import org.apache.kafka.common.message.EndQuorumEpochRequestData; import org.apache.kafka.common.message.EndQuorumEpochResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; +import java.nio.ByteBuffer; import java.util.Collections; import java.util.List; @@ -53,14 +54,9 @@ private EndQuorumEpochRequest(EndQuorumEpochRequestData data, short version) { this.data = data; } - public EndQuorumEpochRequest(Struct struct, short version) { - super(ApiKeys.END_QUORUM_EPOCH, version); - this.data = new EndQuorumEpochRequestData(struct, version); - } - @Override - protected Struct toStruct() { - return data.toStruct(version()); + protected EndQuorumEpochRequestData data() { + return data; } @Override @@ -69,6 +65,10 @@ public EndQuorumEpochResponse getErrorResponse(int throttleTimeMs, Throwable e) .setErrorCode(Errors.forException(e).code())); } + public static EndQuorumEpochRequest parse(ByteBuffer buffer, short version) { + return new EndQuorumEpochRequest(new EndQuorumEpochRequestData(new ByteBufferAccessor(buffer), version), version); + } + public static EndQuorumEpochRequestData singletonRequest(TopicPartition topicPartition, int leaderEpoch, int leaderId, diff --git a/clients/src/main/java/org/apache/kafka/common/requests/EndQuorumEpochResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/EndQuorumEpochResponse.java index b6243365c54a6..9b446663ea540 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/EndQuorumEpochResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/EndQuorumEpochResponse.java @@ -20,8 +20,8 @@ import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.message.EndQuorumEpochResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.Collections; @@ -45,23 +45,10 @@ public class EndQuorumEpochResponse extends AbstractResponse { public final EndQuorumEpochResponseData data; public EndQuorumEpochResponse(EndQuorumEpochResponseData data) { + super(ApiKeys.END_QUORUM_EPOCH); this.data = data; } - public EndQuorumEpochResponse(Struct struct, short version) { - this.data = new EndQuorumEpochResponseData(struct, version); - } - - public EndQuorumEpochResponse(Struct struct) { - short latestVersion = (short) (EndQuorumEpochResponseData.SCHEMAS.length - 1); - this.data = new EndQuorumEpochResponseData(struct, latestVersion); - } - - @Override - protected Struct toStruct(short version) { - return data.toStruct(version); - } - @Override public Map errorCounts() { Map errors = new HashMap<>(); @@ -76,6 +63,16 @@ public Map errorCounts() { return errors; } + @Override + protected EndQuorumEpochResponseData data() { + return data; + } + + @Override + public int throttleTimeMs() { + return DEFAULT_THROTTLE_TIME; + } + public static EndQuorumEpochResponseData singletonResponse( Errors topLevelError, TopicPartition topicPartition, @@ -98,6 +95,6 @@ public static EndQuorumEpochResponseData singletonResponse( } public static EndQuorumEpochResponse parse(ByteBuffer buffer, short version) { - return new EndQuorumEpochResponse(ApiKeys.END_QUORUM_EPOCH.responseSchema(version).read(buffer), version); + return new EndQuorumEpochResponse(new EndQuorumEpochResponseData(new ByteBufferAccessor(buffer), version)); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/EndTxnRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/EndTxnRequest.java index 95a08275f2797..fdfafd005ca66 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/EndTxnRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/EndTxnRequest.java @@ -19,8 +19,8 @@ import org.apache.kafka.common.message.EndTxnRequestData; import org.apache.kafka.common.message.EndTxnResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; @@ -52,11 +52,6 @@ private EndTxnRequest(EndTxnRequestData data, short version) { this.data = data; } - public EndTxnRequest(Struct struct, short version) { - super(ApiKeys.END_TXN, version); - this.data = new EndTxnRequestData(struct, version); - } - public TransactionResult result() { if (data.committed()) return TransactionResult.COMMIT; @@ -65,8 +60,8 @@ public TransactionResult result() { } @Override - protected Struct toStruct() { - return data.toStruct(version()); + protected EndTxnRequestData data() { + return data; } @Override @@ -78,6 +73,6 @@ public EndTxnResponse getErrorResponse(int throttleTimeMs, Throwable e) { } public static EndTxnRequest parse(ByteBuffer buffer, short version) { - return new EndTxnRequest(ApiKeys.END_TXN.parseRequest(version, buffer), version); + return new EndTxnRequest(new EndTxnRequestData(new ByteBufferAccessor(buffer), version), version); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/EndTxnResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/EndTxnResponse.java index 562b7a35a26b6..b782cefd50717 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/EndTxnResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/EndTxnResponse.java @@ -18,8 +18,8 @@ import org.apache.kafka.common.message.EndTxnResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.Map; @@ -41,18 +41,10 @@ public class EndTxnResponse extends AbstractResponse { public final EndTxnResponseData data; public EndTxnResponse(EndTxnResponseData data) { + super(ApiKeys.END_TXN); this.data = data; } - public EndTxnResponse(Struct struct) { - this(struct, (short) (EndTxnResponseData.SCHEMAS.length - 1)); - } - - - public EndTxnResponse(Struct struct, short version) { - this.data = new EndTxnResponseData(struct, version); - } - @Override public int throttleTimeMs() { return data.throttleTimeMs(); @@ -69,12 +61,12 @@ public Map errorCounts() { } @Override - protected Struct toStruct(short version) { - return data.toStruct(version); + protected EndTxnResponseData data() { + return data; } public static EndTxnResponse parse(ByteBuffer buffer, short version) { - return new EndTxnResponse(ApiKeys.END_TXN.parseResponse(version, buffer), version); + return new EndTxnResponse(new EndTxnResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/EnvelopeRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/EnvelopeRequest.java index 58f2d245c530d..4be5259458b6d 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/EnvelopeRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/EnvelopeRequest.java @@ -18,11 +18,9 @@ import org.apache.kafka.common.message.EnvelopeRequestData; import org.apache.kafka.common.message.EnvelopeResponseData; -import org.apache.kafka.common.network.Send; -import org.apache.kafka.common.protocol.SendBuilder; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; @@ -60,11 +58,6 @@ public EnvelopeRequest(EnvelopeRequestData data, short version) { this.data = data; } - public EnvelopeRequest(Struct struct, short version) { - super(ApiKeys.ENVELOPE, version); - this.data = new EnvelopeRequestData(struct, version); - } - public ByteBuffer requestData() { return data.requestData(); } @@ -77,11 +70,6 @@ public byte[] requestPrincipal() { return data.requestPrincipal(); } - @Override - protected Struct toStruct() { - return data.toStruct(version()); - } - @Override public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { return new EnvelopeResponse(new EnvelopeResponseData() @@ -89,16 +77,10 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { } public static EnvelopeRequest parse(ByteBuffer buffer, short version) { - return new EnvelopeRequest(ApiKeys.ENVELOPE.parseRequest(version, buffer), version); + return new EnvelopeRequest(new EnvelopeRequestData(new ByteBufferAccessor(buffer), version), version); } public EnvelopeRequestData data() { return data; } - - @Override - public Send toSend(String destination, RequestHeader header) { - return SendBuilder.buildRequestSend(destination, header, this.data); - } - } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/EnvelopeResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/EnvelopeResponse.java index d95325b2214c3..c7c8d24e4a338 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/EnvelopeResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/EnvelopeResponse.java @@ -17,10 +17,9 @@ package org.apache.kafka.common.requests; import org.apache.kafka.common.message.EnvelopeResponseData; -import org.apache.kafka.common.network.Send; -import org.apache.kafka.common.protocol.SendBuilder; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.Map; @@ -29,8 +28,8 @@ public class EnvelopeResponse extends AbstractResponse { private final EnvelopeResponseData data; - public EnvelopeResponse(ByteBuffer responseData, - Errors error) { + public EnvelopeResponse(ByteBuffer responseData, Errors error) { + super(ApiKeys.ENVELOPE); this.data = new EnvelopeResponseData() .setResponseData(responseData) .setErrorCode(error.code()); @@ -41,6 +40,7 @@ public EnvelopeResponse(Errors error) { } public EnvelopeResponse(EnvelopeResponseData data) { + super(ApiKeys.ENVELOPE); this.data = data; } @@ -53,11 +53,6 @@ public Map errorCounts() { return errorCounts(error()); } - @Override - protected Struct toStruct(short version) { - return data.toStruct(version); - } - public Errors error() { return Errors.forCode(data.errorCode()); } @@ -67,8 +62,12 @@ public EnvelopeResponseData data() { } @Override - protected Send toSend(String destination, ResponseHeader header, short apiVersion) { - return SendBuilder.buildResponseSend(destination, header, this.data, apiVersion); + public int throttleTimeMs() { + return DEFAULT_THROTTLE_TIME; + } + + public static EnvelopeResponse parse(ByteBuffer buffer, short version) { + return new EnvelopeResponse(new EnvelopeResponseData(new ByteBufferAccessor(buffer), version)); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenRequest.java index ca6d2d67d1f79..ac03944ab029e 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenRequest.java @@ -21,8 +21,8 @@ import org.apache.kafka.common.message.ExpireDelegationTokenRequestData; import org.apache.kafka.common.message.ExpireDelegationTokenResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; public class ExpireDelegationTokenRequest extends AbstractRequest { @@ -33,18 +33,14 @@ private ExpireDelegationTokenRequest(ExpireDelegationTokenRequestData data, shor this.data = data; } - public ExpireDelegationTokenRequest(Struct struct, short version) { - super(ApiKeys.EXPIRE_DELEGATION_TOKEN, version); - this.data = new ExpireDelegationTokenRequestData(struct, version); - } - public static ExpireDelegationTokenRequest parse(ByteBuffer buffer, short version) { - return new ExpireDelegationTokenRequest(ApiKeys.EXPIRE_DELEGATION_TOKEN.parseRequest(version, buffer), version); + return new ExpireDelegationTokenRequest( + new ExpireDelegationTokenRequestData(new ByteBufferAccessor(buffer), version), version); } @Override - protected Struct toStruct() { - return data.toStruct(version()); + protected ExpireDelegationTokenRequestData data() { + return data; } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenResponse.java index 75b6c5328da5f..451c736ffb7e6 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenResponse.java @@ -21,23 +21,21 @@ import org.apache.kafka.common.message.ExpireDelegationTokenResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; public class ExpireDelegationTokenResponse extends AbstractResponse { private final ExpireDelegationTokenResponseData data; public ExpireDelegationTokenResponse(ExpireDelegationTokenResponseData data) { + super(ApiKeys.EXPIRE_DELEGATION_TOKEN); this.data = data; } - public ExpireDelegationTokenResponse(Struct struct, short version) { - this.data = new ExpireDelegationTokenResponseData(struct, version); - } - public static ExpireDelegationTokenResponse parse(ByteBuffer buffer, short version) { - return new ExpireDelegationTokenResponse(ApiKeys.EXPIRE_DELEGATION_TOKEN.responseSchema(version).read(buffer), version); + return new ExpireDelegationTokenResponse(new ExpireDelegationTokenResponseData(new ByteBufferAccessor(buffer), + version)); } public Errors error() { @@ -54,8 +52,8 @@ public Map errorCounts() { } @Override - protected Struct toStruct(short version) { - return data.toStruct(version); + protected ExpireDelegationTokenResponseData data() { + return data; } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/FetchRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/FetchRequest.java index 6c88a37e30b9c..b62c844f63b5a 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/FetchRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/FetchRequest.java @@ -19,12 +19,8 @@ import org.apache.kafka.common.IsolationLevel; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.message.FetchRequestData; -import org.apache.kafka.common.message.RequestHeaderData; import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.ObjectSerializationCache; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.record.MemoryRecords; import org.apache.kafka.common.record.RecordBatch; import org.apache.kafka.common.utils.Utils; @@ -55,10 +51,6 @@ public class FetchRequest extends AbstractRequest { private final List toForget; private final FetchMetadata metadata; - public FetchRequestData data() { - return data; - } - public static final class PartitionData { public final long fetchOffset; public final long logStartOffset; @@ -378,34 +370,12 @@ public String rackId() { return data.rackId(); } - @Override - public ByteBuffer serialize(RequestHeader header) { - // Unlike the custom FetchResponse#toSend, we don't include the buffer size here. This buffer is passed - // to a NetworkSend which adds the length value in the eventual serialization - - ObjectSerializationCache cache = new ObjectSerializationCache(); - RequestHeaderData requestHeaderData = header.data(); - - int headerSize = requestHeaderData.size(cache, header.headerVersion()); - int bodySize = data.size(cache, header.apiVersion()); - - ByteBuffer buffer = ByteBuffer.allocate(headerSize + bodySize); - ByteBufferAccessor writer = new ByteBufferAccessor(buffer); - - requestHeaderData.write(writer, cache, header.headerVersion()); - data.write(writer, cache, header.apiVersion()); - - buffer.rewind(); - return buffer; - } - - // For testing public static FetchRequest parse(ByteBuffer buffer, short version) { return new FetchRequest(new FetchRequestData(ApiKeys.FETCH.parseRequest(version, buffer), version), version); } @Override - protected Struct toStruct() { - return data.toStruct(version()); + public FetchRequestData data() { + return data; } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/FetchResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/FetchResponse.java index db3b12a5258d8..d65a55e6c2b29 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/FetchResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/FetchResponse.java @@ -18,12 +18,10 @@ import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.message.FetchResponseData; -import org.apache.kafka.common.network.Send; +import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.ObjectSerializationCache; -import org.apache.kafka.common.protocol.SendBuilder; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.record.BaseRecords; import org.apache.kafka.common.record.MemoryRecords; @@ -273,25 +271,17 @@ public FetchResponse(Errors error, LinkedHashMap> responseData, int throttleTimeMs, int sessionId) { + super(ApiKeys.FETCH); this.data = toMessage(throttleTimeMs, error, responseData.entrySet().iterator(), sessionId); this.responseDataMap = responseData; } public FetchResponse(FetchResponseData fetchResponseData) { + super(ApiKeys.FETCH); this.data = fetchResponseData; this.responseDataMap = toResponseDataMap(fetchResponseData); } - @Override - public Struct toStruct(short version) { - return data.toStruct(version); - } - - @Override - public Send toSend(String dest, ResponseHeader responseHeader, short apiVersion) { - return SendBuilder.buildResponseSend(dest, responseHeader, this.data, apiVersion); - } - public Errors error() { return Errors.forCode(data.errorCode()); } @@ -320,10 +310,7 @@ public Map errorCounts() { } public static FetchResponse parse(ByteBuffer buffer, short version) { - FetchResponseData fetchResponseData = new FetchResponseData(); - ByteBufferAccessor reader = new ByteBufferAccessor(buffer); - fetchResponseData.read(reader, version); - return new FetchResponse<>(fetchResponseData); + return new FetchResponse<>(new FetchResponseData(new ByteBufferAccessor(buffer), version)); } @SuppressWarnings("unchecked") diff --git a/clients/src/main/java/org/apache/kafka/common/requests/FindCoordinatorRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/FindCoordinatorRequest.java index 0e728439169d4..b6fdbfc1e63a5 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/FindCoordinatorRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/FindCoordinatorRequest.java @@ -21,12 +21,11 @@ import org.apache.kafka.common.message.FindCoordinatorRequestData; import org.apache.kafka.common.message.FindCoordinatorResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; - public class FindCoordinatorRequest extends AbstractRequest { public static class Builder extends AbstractRequest.Builder { @@ -63,11 +62,6 @@ private FindCoordinatorRequest(FindCoordinatorRequestData data, short version) { this.data = data; } - public FindCoordinatorRequest(Struct struct, short version) { - super(ApiKeys.FIND_COORDINATOR, version); - this.data = new FindCoordinatorRequestData(struct, version); - } - @Override public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { FindCoordinatorResponseData response = new FindCoordinatorResponseData(); @@ -79,12 +73,8 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { } public static FindCoordinatorRequest parse(ByteBuffer buffer, short version) { - return new FindCoordinatorRequest(ApiKeys.FIND_COORDINATOR.parseRequest(version, buffer), version); - } - - @Override - protected Struct toStruct() { - return data.toStruct(version()); + return new FindCoordinatorRequest(new FindCoordinatorRequestData(new ByteBufferAccessor(buffer), version), + version); } public FindCoordinatorRequestData data() { diff --git a/clients/src/main/java/org/apache/kafka/common/requests/FindCoordinatorResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/FindCoordinatorResponse.java index 2c6819e1f1421..917b087fbab35 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/FindCoordinatorResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/FindCoordinatorResponse.java @@ -19,8 +19,8 @@ import org.apache.kafka.common.Node; import org.apache.kafka.common.message.FindCoordinatorResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.Map; @@ -41,13 +41,10 @@ public class FindCoordinatorResponse extends AbstractResponse { private final FindCoordinatorResponseData data; public FindCoordinatorResponse(FindCoordinatorResponseData data) { + super(ApiKeys.FIND_COORDINATOR); this.data = data; } - public FindCoordinatorResponse(Struct struct, short version) { - this.data = new FindCoordinatorResponseData(struct, version); - } - public FindCoordinatorResponseData data() { return data; } @@ -74,13 +71,8 @@ public Map errorCounts() { return errorCounts(error()); } - @Override - protected Struct toStruct(short version) { - return data.toStruct(version); - } - public static FindCoordinatorResponse parse(ByteBuffer buffer, short version) { - return new FindCoordinatorResponse(ApiKeys.FIND_COORDINATOR.responseSchema(version).read(buffer), version); + return new FindCoordinatorResponse(new FindCoordinatorResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatRequest.java index e5cc6c8860a58..754566db7a425 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatRequest.java @@ -20,12 +20,11 @@ import org.apache.kafka.common.message.HeartbeatRequestData; import org.apache.kafka.common.message.HeartbeatResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; - public class HeartbeatRequest extends AbstractRequest { public static class Builder extends AbstractRequest.Builder { @@ -58,11 +57,6 @@ private HeartbeatRequest(HeartbeatRequestData data, short version) { this.data = data; } - public HeartbeatRequest(Struct struct, short version) { - super(ApiKeys.HEARTBEAT, version); - this.data = new HeartbeatRequestData(struct, version); - } - @Override public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { HeartbeatResponseData responseData = new HeartbeatResponseData(). @@ -74,11 +68,11 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { } public static HeartbeatRequest parse(ByteBuffer buffer, short version) { - return new HeartbeatRequest(ApiKeys.HEARTBEAT.parseRequest(version, buffer), version); + return new HeartbeatRequest(new HeartbeatRequestData(new ByteBufferAccessor(buffer), version), version); } @Override - protected Struct toStruct() { - return data.toStruct(version()); + protected HeartbeatRequestData data() { + return data; } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatResponse.java index e135f30089101..c8fdae2507c6b 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/HeartbeatResponse.java @@ -18,13 +18,12 @@ import org.apache.kafka.common.message.HeartbeatResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.Map; - public class HeartbeatResponse extends AbstractResponse { /** @@ -40,13 +39,10 @@ public class HeartbeatResponse extends AbstractResponse { private final HeartbeatResponseData data; public HeartbeatResponse(HeartbeatResponseData data) { + super(ApiKeys.HEARTBEAT); this.data = data; } - public HeartbeatResponse(Struct struct, short version) { - this.data = new HeartbeatResponseData(struct, version); - } - @Override public int throttleTimeMs() { return data.throttleTimeMs(); @@ -62,12 +58,12 @@ public Map errorCounts() { } @Override - protected Struct toStruct(short version) { - return data.toStruct(version); + protected HeartbeatResponseData data() { + return data; } public static HeartbeatResponse parse(ByteBuffer buffer, short version) { - return new HeartbeatResponse(ApiKeys.HEARTBEAT.parseResponse(version, buffer), version); + return new HeartbeatResponse(new HeartbeatResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/IncrementalAlterConfigsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/IncrementalAlterConfigsRequest.java index a01351feeba57..210e18b37accb 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/IncrementalAlterConfigsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/IncrementalAlterConfigsRequest.java @@ -24,7 +24,7 @@ import org.apache.kafka.common.message.IncrementalAlterConfigsResponseData; import org.apache.kafka.common.message.IncrementalAlterConfigsResponseData.AlterConfigsResourceResponse; import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import java.nio.ByteBuffer; import java.util.Collection; @@ -86,25 +86,15 @@ private IncrementalAlterConfigsRequest(IncrementalAlterConfigsRequestData data, this.version = version; } - IncrementalAlterConfigsRequest(final Struct struct, final short version) { - super(ApiKeys.INCREMENTAL_ALTER_CONFIGS, version); - this.data = new IncrementalAlterConfigsRequestData(struct, version); - this.version = version; - } - public static IncrementalAlterConfigsRequest parse(ByteBuffer buffer, short version) { - return new IncrementalAlterConfigsRequest(ApiKeys.INCREMENTAL_ALTER_CONFIGS.parseRequest(version, buffer), version); + return new IncrementalAlterConfigsRequest(new IncrementalAlterConfigsRequestData( + new ByteBufferAccessor(buffer), version), version); } public IncrementalAlterConfigsRequestData data() { return data; } - @Override - protected Struct toStruct() { - return data.toStruct(version); - } - @Override public AbstractResponse getErrorResponse(final int throttleTimeMs, final Throwable e) { IncrementalAlterConfigsResponseData response = new IncrementalAlterConfigsResponseData(); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/IncrementalAlterConfigsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/IncrementalAlterConfigsResponse.java index 32913ebe5df11..99427c7ad9f3e 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/IncrementalAlterConfigsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/IncrementalAlterConfigsResponse.java @@ -21,8 +21,8 @@ import org.apache.kafka.common.message.IncrementalAlterConfigsResponseData; import org.apache.kafka.common.message.IncrementalAlterConfigsResponseData.AlterConfigsResourceResponse; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -34,6 +34,7 @@ public class IncrementalAlterConfigsResponse extends AbstractResponse { public IncrementalAlterConfigsResponse(final int requestThrottleMs, final Map results) { + super(ApiKeys.INCREMENTAL_ALTER_CONFIGS); final List newResults = new ArrayList<>(results.size()); results.forEach( (resource, error) -> newResults.add( @@ -61,13 +62,10 @@ public static Map fromResponseData(final IncrementalAl private final IncrementalAlterConfigsResponseData data; public IncrementalAlterConfigsResponse(IncrementalAlterConfigsResponseData data) { + super(ApiKeys.INCREMENTAL_ALTER_CONFIGS); this.data = data; } - public IncrementalAlterConfigsResponse(final Struct struct, final short version) { - this.data = new IncrementalAlterConfigsResponseData(struct, version); - } - public IncrementalAlterConfigsResponseData data() { return data; } @@ -81,11 +79,6 @@ public Map errorCounts() { return counts; } - @Override - protected Struct toStruct(final short version) { - return data.toStruct(version); - } - @Override public boolean shouldClientThrottle(short version) { return version >= 0; @@ -97,7 +90,7 @@ public int throttleTimeMs() { } public static IncrementalAlterConfigsResponse parse(ByteBuffer buffer, short version) { - return new IncrementalAlterConfigsResponse( - ApiKeys.INCREMENTAL_ALTER_CONFIGS.responseSchema(version).read(buffer), version); + return new IncrementalAlterConfigsResponse(new IncrementalAlterConfigsResponseData( + new ByteBufferAccessor(buffer), version)); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/InitProducerIdRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/InitProducerIdRequest.java index 8351c2159836a..08e75cb536c63 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/InitProducerIdRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/InitProducerIdRequest.java @@ -19,8 +19,8 @@ import org.apache.kafka.common.message.InitProducerIdRequestData; import org.apache.kafka.common.message.InitProducerIdResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.record.RecordBatch; import java.nio.ByteBuffer; @@ -58,12 +58,6 @@ private InitProducerIdRequest(InitProducerIdRequestData data, short version) { this.data = data; } - public InitProducerIdRequest(Struct struct, short version) { - super(ApiKeys.INIT_PRODUCER_ID, version); - this.data = new InitProducerIdRequestData(struct, version); - } - - @Override public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { InitProducerIdResponseData response = new InitProducerIdResponseData() @@ -75,12 +69,11 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { } public static InitProducerIdRequest parse(ByteBuffer buffer, short version) { - return new InitProducerIdRequest(ApiKeys.INIT_PRODUCER_ID.parseRequest(version, buffer), version); + return new InitProducerIdRequest(new InitProducerIdRequestData(new ByteBufferAccessor(buffer), version), version); } @Override - protected Struct toStruct() { - return data.toStruct(version()); + protected InitProducerIdRequestData data() { + return data; } - } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/InitProducerIdResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/InitProducerIdResponse.java index 822a4b6ce7cd6..6066c13ef5ab6 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/InitProducerIdResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/InitProducerIdResponse.java @@ -18,8 +18,8 @@ import org.apache.kafka.common.message.InitProducerIdResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.Map; @@ -39,13 +39,10 @@ public class InitProducerIdResponse extends AbstractResponse { public final InitProducerIdResponseData data; public InitProducerIdResponse(InitProducerIdResponseData data) { + super(ApiKeys.INIT_PRODUCER_ID); this.data = data; } - public InitProducerIdResponse(Struct struct, short version) { - this.data = new InitProducerIdResponseData(struct, version); - } - @Override public int throttleTimeMs() { return data.throttleTimeMs(); @@ -57,12 +54,12 @@ public Map errorCounts() { } @Override - protected Struct toStruct(short version) { - return data.toStruct(version); + protected InitProducerIdResponseData data() { + return data; } public static InitProducerIdResponse parse(ByteBuffer buffer, short version) { - return new InitProducerIdResponse(ApiKeys.INIT_PRODUCER_ID.parseResponse(version, buffer), version); + return new InitProducerIdResponse(new InitProducerIdResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/JoinGroupRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/JoinGroupRequest.java index 61486abc1be74..7be16b6410c3b 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/JoinGroupRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/JoinGroupRequest.java @@ -21,8 +21,8 @@ import org.apache.kafka.common.message.JoinGroupRequestData; import org.apache.kafka.common.message.JoinGroupResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.Collections; @@ -99,12 +99,6 @@ public JoinGroupRequest(JoinGroupRequestData data, short version) { maybeOverrideRebalanceTimeout(version); } - public JoinGroupRequest(Struct struct, short version) { - super(ApiKeys.JOIN_GROUP, version); - this.data = new JoinGroupRequestData(struct, version); - maybeOverrideRebalanceTimeout(version); - } - private void maybeOverrideRebalanceTimeout(short version) { if (version == 0) { // Version 0 has no rebalance timeout, so we use the session timeout @@ -137,11 +131,6 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { } public static JoinGroupRequest parse(ByteBuffer buffer, short version) { - return new JoinGroupRequest(ApiKeys.JOIN_GROUP.parseRequest(version, buffer), version); - } - - @Override - protected Struct toStruct() { - return data.toStruct(version()); + return new JoinGroupRequest(new JoinGroupRequestData(new ByteBufferAccessor(buffer), version), version); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/JoinGroupResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/JoinGroupResponse.java index 8e585d660aa76..8472175edab4a 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/JoinGroupResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/JoinGroupResponse.java @@ -18,8 +18,8 @@ import org.apache.kafka.common.message.JoinGroupResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.Map; @@ -29,18 +29,10 @@ public class JoinGroupResponse extends AbstractResponse { private final JoinGroupResponseData data; public JoinGroupResponse(JoinGroupResponseData data) { + super(ApiKeys.JOIN_GROUP); this.data = data; } - public JoinGroupResponse(Struct struct) { - short latestVersion = (short) (JoinGroupResponseData.SCHEMAS.length - 1); - this.data = new JoinGroupResponseData(struct, latestVersion); - } - - public JoinGroupResponse(Struct struct, short version) { - this.data = new JoinGroupResponseData(struct, version); - } - public JoinGroupResponseData data() { return data; } @@ -63,13 +55,8 @@ public Map errorCounts() { return errorCounts(Errors.forCode(data.errorCode())); } - public static JoinGroupResponse parse(ByteBuffer buffer, short versionId) { - return new JoinGroupResponse(ApiKeys.JOIN_GROUP.parseResponse(versionId, buffer), versionId); - } - - @Override - protected Struct toStruct(short version) { - return data.toStruct(version); + public static JoinGroupResponse parse(ByteBuffer buffer, short version) { + return new JoinGroupResponse(new JoinGroupResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java index 2e6b9560baeea..9927dda1de6e6 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrRequest.java @@ -24,8 +24,8 @@ import org.apache.kafka.common.message.LeaderAndIsrResponseData; import org.apache.kafka.common.message.LeaderAndIsrResponseData.LeaderAndIsrPartitionError; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.utils.FlattenedIterator; import org.apache.kafka.common.utils.Utils; @@ -123,15 +123,6 @@ private void normalize() { } } - public LeaderAndIsrRequest(Struct struct, short version) { - this(new LeaderAndIsrRequestData(struct, version), version); - } - - @Override - protected Struct toStruct() { - return data.toStruct(version()); - } - @Override public LeaderAndIsrResponse getErrorResponse(int throttleTimeMs, Throwable e) { LeaderAndIsrResponseData responseData = new LeaderAndIsrResponseData(); @@ -175,11 +166,11 @@ public List liveLeaders() { return Collections.unmodifiableList(data.liveLeaders()); } - public LeaderAndIsrRequestData data() { + protected LeaderAndIsrRequestData data() { return data; } public static LeaderAndIsrRequest parse(ByteBuffer buffer, short version) { - return new LeaderAndIsrRequest(ApiKeys.LEADER_AND_ISR.parseRequest(version, buffer), version); + return new LeaderAndIsrRequest(new LeaderAndIsrRequestData(new ByteBufferAccessor(buffer), version), version); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrResponse.java index d2ded841224bd..1e1069edea3a1 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/LeaderAndIsrResponse.java @@ -19,8 +19,8 @@ import org.apache.kafka.common.message.LeaderAndIsrResponseData; import org.apache.kafka.common.message.LeaderAndIsrResponseData.LeaderAndIsrPartitionError; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.Collections; @@ -38,13 +38,10 @@ public class LeaderAndIsrResponse extends AbstractResponse { private final LeaderAndIsrResponseData data; public LeaderAndIsrResponse(LeaderAndIsrResponseData data) { + super(ApiKeys.LEADER_AND_ISR); this.data = data; } - public LeaderAndIsrResponse(Struct struct, short version) { - this.data = new LeaderAndIsrResponseData(struct, version); - } - public List partitions() { return data.partitionErrors(); } @@ -65,13 +62,18 @@ public Map errorCounts() { return errors; } + @Override + public int throttleTimeMs() { + return DEFAULT_THROTTLE_TIME; + } + public static LeaderAndIsrResponse parse(ByteBuffer buffer, short version) { - return new LeaderAndIsrResponse(ApiKeys.LEADER_AND_ISR.parseResponse(version, buffer), version); + return new LeaderAndIsrResponse(new LeaderAndIsrResponseData(new ByteBufferAccessor(buffer), version)); } @Override - protected Struct toStruct(short version) { - return data.toStruct(version); + protected LeaderAndIsrResponseData data() { + return data; } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupRequest.java index 4cd72a6cf44ca..ad7547c648ee7 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupRequest.java @@ -21,9 +21,9 @@ import org.apache.kafka.common.message.LeaveGroupRequestData.MemberIdentity; import org.apache.kafka.common.message.LeaveGroupResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.MessageUtil; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.Collections; @@ -81,18 +81,10 @@ public String toString() { } } private final LeaveGroupRequestData data; - private final short version; private LeaveGroupRequest(LeaveGroupRequestData data, short version) { super(ApiKeys.LEAVE_GROUP, version); this.data = data; - this.version = version; - } - - public LeaveGroupRequest(Struct struct, short version) { - super(ApiKeys.LEAVE_GROUP, version); - this.data = new LeaveGroupRequestData(struct, version); - this.version = version; } public LeaveGroupRequestData data() { @@ -101,7 +93,7 @@ public LeaveGroupRequestData data() { public List members() { // Before version 3, leave group request is still in single mode - return version <= 2 ? Collections.singletonList( + return version() <= 2 ? Collections.singletonList( new MemberIdentity() .setMemberId(data.memberId())) : data.members(); } @@ -118,11 +110,6 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { } public static LeaveGroupRequest parse(ByteBuffer buffer, short version) { - return new LeaveGroupRequest(ApiKeys.LEAVE_GROUP.parseRequest(version, buffer), version); - } - - @Override - protected Struct toStruct() { - return data.toStruct(version); + return new LeaveGroupRequest(new LeaveGroupRequestData(new ByteBufferAccessor(buffer), version), version); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupResponse.java index 717730f25d04b..3bc70674f8a75 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/LeaveGroupResponse.java @@ -19,8 +19,8 @@ import org.apache.kafka.common.message.LeaveGroupResponseData; import org.apache.kafka.common.message.LeaveGroupResponseData.MemberResponse; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.HashMap; @@ -51,6 +51,7 @@ public class LeaveGroupResponse extends AbstractResponse { public final LeaveGroupResponseData data; public LeaveGroupResponse(LeaveGroupResponseData data) { + super(ApiKeys.LEAVE_GROUP); this.data = data; } @@ -58,6 +59,7 @@ public LeaveGroupResponse(List memberResponses, Errors topLevelError, final int throttleTimeMs, final short version) { + super(ApiKeys.LEAVE_GROUP); if (version <= 2) { // Populate member level error. final short errorCode = getError(topLevelError, memberResponses).code(); @@ -75,15 +77,6 @@ public LeaveGroupResponse(List memberResponses, } } - public LeaveGroupResponse(Struct struct) { - short latestVersion = (short) (LeaveGroupResponseData.SCHEMAS.length - 1); - this.data = new LeaveGroupResponseData(struct, latestVersion); - } - - public LeaveGroupResponse(Struct struct, short version) { - this.data = new LeaveGroupResponseData(struct, version); - } - @Override public int throttleTimeMs() { return data.throttleTimeMs(); @@ -129,17 +122,12 @@ public Map errorCounts() { } @Override - public String toString() { - return data.toString(); - } - - @Override - public Struct toStruct(short version) { - return data.toStruct(version); + protected LeaveGroupResponseData data() { + return data; } - public static LeaveGroupResponse parse(ByteBuffer buffer, short versionId) { - return new LeaveGroupResponse(ApiKeys.LEAVE_GROUP.parseResponse(versionId, buffer), versionId); + public static LeaveGroupResponse parse(ByteBuffer buffer, short version) { + return new LeaveGroupResponse(new LeaveGroupResponseData(new ByteBufferAccessor(buffer), version)); } @Override @@ -157,4 +145,9 @@ public boolean equals(Object other) { public int hashCode() { return Objects.hashCode(data); } + + @Override + public String toString() { + return data.toString(); + } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsRequest.java index ce3938530b615..ab7ec61f6c343 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsRequest.java @@ -20,8 +20,8 @@ import org.apache.kafka.common.message.ListGroupsRequestData; import org.apache.kafka.common.message.ListGroupsResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.Collections; @@ -66,15 +66,6 @@ public ListGroupsRequest(ListGroupsRequestData data, short version) { this.data = data; } - public ListGroupsRequest(Struct struct, short version) { - super(ApiKeys.LIST_GROUPS, version); - this.data = new ListGroupsRequestData(struct, version); - } - - public ListGroupsRequestData data() { - return data; - } - @Override public ListGroupsResponse getErrorResponse(int throttleTimeMs, Throwable e) { ListGroupsResponseData listGroupsResponseData = new ListGroupsResponseData(). @@ -87,11 +78,11 @@ public ListGroupsResponse getErrorResponse(int throttleTimeMs, Throwable e) { } public static ListGroupsRequest parse(ByteBuffer buffer, short version) { - return new ListGroupsRequest(ApiKeys.LIST_GROUPS.parseRequest(version, buffer), version); + return new ListGroupsRequest(new ListGroupsRequestData(new ByteBufferAccessor(buffer), version), version); } @Override - protected Struct toStruct() { - return data.toStruct(version()); + public ListGroupsRequestData data() { + return data; } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsResponse.java index f3b3bb663595e..336368395b76f 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ListGroupsResponse.java @@ -18,8 +18,8 @@ import org.apache.kafka.common.message.ListGroupsResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.Map; @@ -29,13 +29,10 @@ public class ListGroupsResponse extends AbstractResponse { private final ListGroupsResponseData data; public ListGroupsResponse(ListGroupsResponseData data) { + super(ApiKeys.LIST_GROUPS); this.data = data; } - public ListGroupsResponse(Struct struct, short version) { - this.data = new ListGroupsResponseData(struct, version); - } - public ListGroupsResponseData data() { return data; } @@ -50,13 +47,8 @@ public Map errorCounts() { return errorCounts(Errors.forCode(data.errorCode())); } - @Override - protected Struct toStruct(short version) { - return data.toStruct(version); - } - public static ListGroupsResponse parse(ByteBuffer buffer, short version) { - return new ListGroupsResponse(ApiKeys.LIST_GROUPS.responseSchema(version).read(buffer), version); + return new ListGroupsResponse(new ListGroupsResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetRequest.java index 07f2998a4b4a5..9ebb06bdd6511 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetRequest.java @@ -34,8 +34,8 @@ import org.apache.kafka.common.message.ListOffsetResponseData.ListOffsetPartitionResponse; import org.apache.kafka.common.message.ListOffsetResponseData.ListOffsetTopicResponse; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; public class ListOffsetRequest extends AbstractRequest { public static final long EARLIEST_TIMESTAMP = -2L; @@ -80,7 +80,7 @@ public Builder setTargetTimes(List topics) { @Override public ListOffsetRequest build(short version) { - return new ListOffsetRequest(version, data); + return new ListOffsetRequest(data, version); } @Override @@ -92,15 +92,9 @@ public String toString() { /** * Private constructor with a specified version. */ - private ListOffsetRequest(short version, ListOffsetRequestData data) { + private ListOffsetRequest(ListOffsetRequestData data, short version) { super(ApiKeys.LIST_OFFSETS, version); this.data = data; - this.duplicatePartitions = Collections.emptySet(); - } - - public ListOffsetRequest(Struct struct, short version) { - super(ApiKeys.LIST_OFFSETS, version); - data = new ListOffsetRequestData(struct, version); duplicatePartitions = new HashSet<>(); Set partitions = new HashSet<>(); for (ListOffsetTopic topic : data.topics()) { @@ -143,6 +137,7 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { return new ListOffsetResponse(responseData); } + @Override public ListOffsetRequestData data() { return data; } @@ -164,12 +159,7 @@ public Set duplicatePartitions() { } public static ListOffsetRequest parse(ByteBuffer buffer, short version) { - return new ListOffsetRequest(ApiKeys.LIST_OFFSETS.parseRequest(version, buffer), version); - } - - @Override - protected Struct toStruct() { - return data.toStruct(version()); + return new ListOffsetRequest(new ListOffsetRequestData(new ByteBufferAccessor(buffer), version), version); } public static List toListOffsetTopics(Map timestampsToSearch) { diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetResponse.java index dd941d742ab90..ca57382d9c0ca 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ListOffsetResponse.java @@ -27,8 +27,8 @@ import org.apache.kafka.common.message.ListOffsetResponseData.ListOffsetPartitionResponse; import org.apache.kafka.common.message.ListOffsetResponseData.ListOffsetTopicResponse; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.record.RecordBatch; /** @@ -55,18 +55,16 @@ public class ListOffsetResponse extends AbstractResponse { private final ListOffsetResponseData data; public ListOffsetResponse(ListOffsetResponseData data) { + super(ApiKeys.LIST_OFFSETS); this.data = data; } - public ListOffsetResponse(Struct struct, short version) { - data = new ListOffsetResponseData(struct, version); - } - @Override public int throttleTimeMs() { return data.throttleTimeMs(); } + @Override public ListOffsetResponseData data() { return data; } @@ -87,12 +85,7 @@ public Map errorCounts() { } public static ListOffsetResponse parse(ByteBuffer buffer, short version) { - return new ListOffsetResponse(ApiKeys.LIST_OFFSETS.parseResponse(version, buffer), version); - } - - @Override - protected Struct toStruct(short version) { - return data.toStruct(version); + return new ListOffsetResponse(new ListOffsetResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ListPartitionReassignmentsRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ListPartitionReassignmentsRequest.java index 0aa5e557556ea..86cfc6632b7a0 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ListPartitionReassignmentsRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ListPartitionReassignmentsRequest.java @@ -21,7 +21,7 @@ import org.apache.kafka.common.message.ListPartitionReassignmentsResponseData.OngoingPartitionReassignment; import org.apache.kafka.common.message.ListPartitionReassignmentsResponseData.OngoingTopicReassignment; import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -52,38 +52,21 @@ public String toString() { } private ListPartitionReassignmentsRequestData data; - private final short version; private ListPartitionReassignmentsRequest(ListPartitionReassignmentsRequestData data, short version) { super(ApiKeys.LIST_PARTITION_REASSIGNMENTS, version); this.data = data; - this.version = version; - } - - ListPartitionReassignmentsRequest(Struct struct, short version) { - super(ApiKeys.LIST_PARTITION_REASSIGNMENTS, version); - this.data = new ListPartitionReassignmentsRequestData(struct, version); - this.version = version; } public static ListPartitionReassignmentsRequest parse(ByteBuffer buffer, short version) { - return new ListPartitionReassignmentsRequest( - ApiKeys.LIST_PARTITION_REASSIGNMENTS.parseRequest(version, buffer), version - ); + return new ListPartitionReassignmentsRequest(new ListPartitionReassignmentsRequestData( + new ByteBufferAccessor(buffer), version), version); } public ListPartitionReassignmentsRequestData data() { return data; } - /** - * Visible for testing. - */ - @Override - public Struct toStruct() { - return data.toStruct(version); - } - @Override public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { ApiError apiError = ApiError.fromThrowable(e); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ListPartitionReassignmentsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ListPartitionReassignmentsResponse.java index dd4e6989d606b..3449f1cabe9ba 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ListPartitionReassignmentsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ListPartitionReassignmentsResponse.java @@ -18,8 +18,8 @@ import org.apache.kafka.common.message.ListPartitionReassignmentsResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.Map; @@ -28,20 +28,14 @@ public class ListPartitionReassignmentsResponse extends AbstractResponse { private final ListPartitionReassignmentsResponseData data; - public ListPartitionReassignmentsResponse(Struct struct) { - this(struct, ApiKeys.LIST_PARTITION_REASSIGNMENTS.latestVersion()); - } - public ListPartitionReassignmentsResponse(ListPartitionReassignmentsResponseData responseData) { + super(ApiKeys.LIST_PARTITION_REASSIGNMENTS); this.data = responseData; } - ListPartitionReassignmentsResponse(Struct struct, short version) { - this.data = new ListPartitionReassignmentsResponseData(struct, version); - } - public static ListPartitionReassignmentsResponse parse(ByteBuffer buffer, short version) { - return new ListPartitionReassignmentsResponse(ApiKeys.LIST_PARTITION_REASSIGNMENTS.responseSchema(version).read(buffer), version); + return new ListPartitionReassignmentsResponse(new ListPartitionReassignmentsResponseData( + new ByteBufferAccessor(buffer), version)); } public ListPartitionReassignmentsResponseData data() { @@ -62,9 +56,4 @@ public int throttleTimeMs() { public Map errorCounts() { return errorCounts(Errors.forCode(data.errorCode())); } - - @Override - protected Struct toStruct(short version) { - return data.toStruct(version); - } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/MetadataRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/MetadataRequest.java index c2533e4f3779c..5aceb2291ff45 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/MetadataRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/MetadataRequest.java @@ -21,8 +21,8 @@ import org.apache.kafka.common.message.MetadataRequestData.MetadataRequestTopic; import org.apache.kafka.common.message.MetadataResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.Collection; @@ -102,18 +102,10 @@ public String toString() { } private final MetadataRequestData data; - private final short version; public MetadataRequest(MetadataRequestData data, short version) { super(ApiKeys.METADATA, version); this.data = data; - this.version = version; - } - - public MetadataRequest(Struct struct, short version) { - super(ApiKeys.METADATA, version); - this.data = new MetadataRequestData(struct, version); - this.version = version; } public MetadataRequestData data() { @@ -125,7 +117,7 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { Errors error = Errors.forException(e); MetadataResponseData responseData = new MetadataResponseData(); if (topics() != null) { - for (String topic :topics()) + for (String topic : topics()) responseData.topics().add(new MetadataResponseData.MetadataResponseTopic() .setName(topic) .setErrorCode(error.code()) @@ -134,12 +126,12 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { } responseData.setThrottleTimeMs(throttleTimeMs); - return new MetadataResponse(responseData); + return new MetadataResponse(responseData, true); } public boolean isAllTopics() { return (data.topics() == null) || - (data.topics().isEmpty() && version == 0); //In version 0, an empty topic list indicates + (data.topics().isEmpty() && version() == 0); //In version 0, an empty topic list indicates // "request metadata for all topics." } @@ -158,7 +150,7 @@ public boolean allowAutoTopicCreation() { } public static MetadataRequest parse(ByteBuffer buffer, short version) { - return new MetadataRequest(ApiKeys.METADATA.parseRequest(version, buffer), version); + return new MetadataRequest(new MetadataRequestData(new ByteBufferAccessor(buffer), version), version); } public static List convertToMetadataRequestTopic(final Collection topics) { @@ -166,9 +158,4 @@ public static List convertToMetadataRequestTopic(final Col .setName(topic)) .collect(Collectors.toList()); } - - @Override - protected Struct toStruct() { - return data.toStruct(version); - } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/MetadataResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/MetadataResponse.java index 2014a05c7b0f9..ba06439bad1ec 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/MetadataResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/MetadataResponse.java @@ -25,9 +25,8 @@ import org.apache.kafka.common.message.MetadataResponseData.MetadataResponsePartition; import org.apache.kafka.common.message.MetadataResponseData.MetadataResponseTopic; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; -import org.apache.kafka.common.record.RecordBatch; import org.apache.kafka.common.utils.Utils; import java.nio.ByteBuffer; @@ -64,28 +63,19 @@ public class MetadataResponse extends AbstractResponse { private volatile Holder holder; private final boolean hasReliableLeaderEpochs; - public MetadataResponse(MetadataResponseData data) { - this(data, true); + public MetadataResponse(MetadataResponseData data, short version) { + this(data, hasReliableLeaderEpochs(version)); } - public MetadataResponse(Struct struct, short version) { - // Prior to Kafka version 2.4 (which coincides with Metadata version 9), the broker - // does not propagate leader epoch information accurately while a reassignment is in - // progress. Relying on a stale epoch can lead to FENCED_LEADER_EPOCH errors which - // can prevent consumption throughout the course of a reassignment. It is safer in - // this case to revert to the behavior in previous protocol versions which checks - // leader status only. - this(new MetadataResponseData(struct, version), version >= 9); - } - - private MetadataResponse(MetadataResponseData data, boolean hasReliableLeaderEpochs) { + MetadataResponse(MetadataResponseData data, boolean hasReliableLeaderEpochs) { + super(ApiKeys.METADATA); this.data = data; this.hasReliableLeaderEpochs = hasReliableLeaderEpochs; } @Override - protected Struct toStruct(short version) { - return data.toStruct(version); + protected MetadataResponseData data() { + return data; } @Override @@ -243,8 +233,19 @@ public boolean hasReliableLeaderEpochs() { return hasReliableLeaderEpochs; } + // Prior to Kafka version 2.4 (which coincides with Metadata version 9), the broker + // does not propagate leader epoch information accurately while a reassignment is in + // progress. Relying on a stale epoch can lead to FENCED_LEADER_EPOCH errors which + // can prevent consumption throughout the course of a reassignment. It is safer in + // this case to revert to the behavior in previous protocol versions which checks + // leader status only. + private static boolean hasReliableLeaderEpochs(short version) { + return version >= 9; + } + public static MetadataResponse parse(ByteBuffer buffer, short version) { - return new MetadataResponse(ApiKeys.METADATA.responseSchema(version).read(buffer), version); + return new MetadataResponse(new MetadataResponseData(new ByteBufferAccessor(buffer), version), + hasReliableLeaderEpochs(version)); } public static class TopicMetadata { @@ -429,79 +430,24 @@ private Collection createTopicMetadata(MetadataResponseData data) } - public static MetadataResponse prepareResponse(int throttleTimeMs, Collection brokers, String clusterId, - int controllerId, List topicMetadataList, - int clusterAuthorizedOperations, - short responseVersion) { - MetadataResponseData responseData = new MetadataResponseData(); - responseData.setThrottleTimeMs(throttleTimeMs); - brokers.forEach(broker -> - responseData.brokers().add(new MetadataResponseBroker() - .setNodeId(broker.id()) - .setHost(broker.host()) - .setPort(broker.port()) - .setRack(broker.rack())) - ); - - responseData.setClusterId(clusterId); - responseData.setControllerId(controllerId); - responseData.setClusterAuthorizedOperations(clusterAuthorizedOperations); - - topicMetadataList.forEach(topicMetadata -> { - MetadataResponseTopic metadataResponseTopic = new MetadataResponseTopic(); - metadataResponseTopic - .setErrorCode(topicMetadata.error.code()) - .setName(topicMetadata.topic) - .setIsInternal(topicMetadata.isInternal) - .setTopicAuthorizedOperations(topicMetadata.authorizedOperations); - - for (PartitionMetadata partitionMetadata : topicMetadata.partitionMetadata) { - metadataResponseTopic.partitions().add(new MetadataResponsePartition() - .setErrorCode(partitionMetadata.error.code()) - .setPartitionIndex(partitionMetadata.partition()) - .setLeaderId(partitionMetadata.leaderId.orElse(NO_LEADER_ID)) - .setLeaderEpoch(partitionMetadata.leaderEpoch.orElse(RecordBatch.NO_PARTITION_LEADER_EPOCH)) - .setReplicaNodes(partitionMetadata.replicaIds) - .setIsrNodes(partitionMetadata.inSyncReplicaIds) - .setOfflineReplicas(partitionMetadata.offlineReplicaIds)); - } - responseData.topics().add(metadataResponseTopic); - }); - return new MetadataResponse(responseData.toStruct(responseVersion), responseVersion); - } - - public static MetadataResponse prepareResponse(int throttleTimeMs, + public static MetadataResponse prepareResponse(short version, + int throttleTimeMs, Collection brokers, String clusterId, int controllerId, - List topicMetadataList, - short responseVersion) { - return prepareResponse(throttleTimeMs, brokers, clusterId, controllerId, topicMetadataList, - MetadataResponse.AUTHORIZED_OPERATIONS_OMITTED, responseVersion); - } - - public static MetadataResponse prepareResponse(Collection brokers, - String clusterId, - int controllerId, - List topicMetadata, - short responseVersion) { - return prepareResponse(AbstractResponse.DEFAULT_THROTTLE_TIME, brokers, clusterId, controllerId, - topicMetadata, responseVersion); - } - - public static MetadataResponse prepareResponse(Collection brokers, - String clusterId, - int controllerId, - List topicMetadata) { - return prepareResponse(AbstractResponse.DEFAULT_THROTTLE_TIME, brokers, clusterId, controllerId, - topicMetadata, ApiKeys.METADATA.latestVersion()); + List topics, + int clusterAuthorizedOperations) { + return prepareResponse(hasReliableLeaderEpochs(version), throttleTimeMs, brokers, clusterId, controllerId, + topics, clusterAuthorizedOperations); } - public static MetadataResponse prepareResponse(int throttleTimeMs, - List topicMetadataList, + // Visible for testing + public static MetadataResponse prepareResponse(boolean hasReliableEpoch, + int throttleTimeMs, Collection brokers, String clusterId, int controllerId, + List topics, int clusterAuthorizedOperations) { MetadataResponseData responseData = new MetadataResponseData(); responseData.setThrottleTimeMs(throttleTimeMs); @@ -517,8 +463,8 @@ public static MetadataResponse prepareResponse(int throttleTimeMs, responseData.setControllerId(controllerId); responseData.setClusterAuthorizedOperations(clusterAuthorizedOperations); - topicMetadataList.forEach(topicMetadata -> responseData.topics().add(topicMetadata)); - return new MetadataResponse(responseData); + topics.forEach(topicMetadata -> responseData.topics().add(topicMetadata)); + return new MetadataResponse(responseData, hasReliableEpoch); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitRequest.java index ba6d182af6950..6542ed425746a 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitRequest.java @@ -24,8 +24,8 @@ import org.apache.kafka.common.message.OffsetCommitResponseData.OffsetCommitResponsePartition; import org.apache.kafka.common.message.OffsetCommitResponseData.OffsetCommitResponseTopic; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -68,19 +68,9 @@ public String toString() { } } - private final short version; - public OffsetCommitRequest(OffsetCommitRequestData data, short version) { super(ApiKeys.OFFSET_COMMIT, version); this.data = data; - this.version = version; - } - - - public OffsetCommitRequest(Struct struct, short version) { - super(ApiKeys.OFFSET_COMMIT, version); - this.data = new OffsetCommitRequestData(struct, version); - this.version = version; } public OffsetCommitRequestData data() { @@ -128,11 +118,6 @@ public OffsetCommitResponse getErrorResponse(int throttleTimeMs, Throwable e) { } public static OffsetCommitRequest parse(ByteBuffer buffer, short version) { - return new OffsetCommitRequest(ApiKeys.OFFSET_COMMIT.parseRequest(version, buffer), version); - } - - @Override - protected Struct toStruct() { - return data.toStruct(version); + return new OffsetCommitRequest(new OffsetCommitRequestData(new ByteBufferAccessor(buffer), version), version); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitResponse.java index bacc9ad30fe02..b2851a68deb84 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetCommitResponse.java @@ -21,8 +21,8 @@ import org.apache.kafka.common.message.OffsetCommitResponseData.OffsetCommitResponsePartition; import org.apache.kafka.common.message.OffsetCommitResponseData.OffsetCommitResponseTopic; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -50,10 +50,12 @@ public class OffsetCommitResponse extends AbstractResponse { private final OffsetCommitResponseData data; public OffsetCommitResponse(OffsetCommitResponseData data) { + super(ApiKeys.OFFSET_COMMIT); this.data = data; } public OffsetCommitResponse(int requestThrottleMs, Map responseData) { + super(ApiKeys.OFFSET_COMMIT); Map responseTopicDataMap = new HashMap<>(); @@ -79,15 +81,6 @@ public OffsetCommitResponse(Map responseData) { this(DEFAULT_THROTTLE_TIME, responseData); } - public OffsetCommitResponse(Struct struct) { - short latestVersion = (short) (OffsetCommitResponseData.SCHEMAS.length - 1); - this.data = new OffsetCommitResponseData(struct, latestVersion); - } - - public OffsetCommitResponse(Struct struct, short version) { - this.data = new OffsetCommitResponseData(struct, version); - } - public OffsetCommitResponseData data() { return data; } @@ -100,12 +93,7 @@ public Map errorCounts() { } public static OffsetCommitResponse parse(ByteBuffer buffer, short version) { - return new OffsetCommitResponse(ApiKeys.OFFSET_COMMIT.parseResponse(version, buffer), version); - } - - @Override - public Struct toStruct(short version) { - return data.toStruct(version); + return new OffsetCommitResponse(new OffsetCommitResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetDeleteRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetDeleteRequest.java index 293efc2f6f2db..2f9d0203a9d7a 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/OffsetDeleteRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetDeleteRequest.java @@ -19,8 +19,8 @@ import org.apache.kafka.common.message.OffsetDeleteRequestData; import org.apache.kafka.common.message.OffsetDeleteResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; @@ -53,11 +53,6 @@ public OffsetDeleteRequest(OffsetDeleteRequestData data, short version) { this.data = data; } - public OffsetDeleteRequest(Struct struct, short version) { - super(ApiKeys.OFFSET_DELETE, version); - this.data = new OffsetDeleteRequestData(struct, version); - } - public AbstractResponse getErrorResponse(int throttleTimeMs, Errors error) { return new OffsetDeleteResponse( new OffsetDeleteResponseData() @@ -72,12 +67,11 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { } public static OffsetDeleteRequest parse(ByteBuffer buffer, short version) { - return new OffsetDeleteRequest(ApiKeys.OFFSET_DELETE.parseRequest(version, buffer), - version); + return new OffsetDeleteRequest(new OffsetDeleteRequestData(new ByteBufferAccessor(buffer), version), version); } @Override - protected Struct toStruct() { - return data.toStruct(version()); + protected OffsetDeleteRequestData data() { + return data; } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetDeleteResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetDeleteResponse.java index d52ab4dd49472..ceb932fb21d1e 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/OffsetDeleteResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetDeleteResponse.java @@ -18,8 +18,8 @@ import org.apache.kafka.common.message.OffsetDeleteResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.HashMap; @@ -47,21 +47,13 @@ public class OffsetDeleteResponse extends AbstractResponse { public final OffsetDeleteResponseData data; public OffsetDeleteResponse(OffsetDeleteResponseData data) { + super(ApiKeys.OFFSET_DELETE); this.data = data; } - public OffsetDeleteResponse(Struct struct) { - short latestVersion = (short) (OffsetDeleteResponseData.SCHEMAS.length - 1); - this.data = new OffsetDeleteResponseData(struct, latestVersion); - } - - public OffsetDeleteResponse(Struct struct, short version) { - this.data = new OffsetDeleteResponseData(struct, version); - } - @Override - protected Struct toStruct(short version) { - return data.toStruct(version); + protected OffsetDeleteResponseData data() { + return data; } @Override @@ -77,7 +69,7 @@ public Map errorCounts() { } public static OffsetDeleteResponse parse(ByteBuffer buffer, short version) { - return new OffsetDeleteResponse(ApiKeys.OFFSET_DELETE.parseResponse(version, buffer)); + return new OffsetDeleteResponse(new OffsetDeleteResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchRequest.java index 382ca4ad93614..fd08c725399cb 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchRequest.java @@ -21,8 +21,8 @@ import org.apache.kafka.common.message.OffsetFetchRequestData; import org.apache.kafka.common.message.OffsetFetchRequestData.OffsetFetchRequestTopic; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -133,11 +133,6 @@ private OffsetFetchRequest(OffsetFetchRequestData data, short version) { this.data = data; } - public OffsetFetchRequest(Struct struct, short version) { - super(ApiKeys.OFFSET_FETCH, version); - this.data = new OffsetFetchRequestData(struct, version); - } - public OffsetFetchResponse getErrorResponse(Errors error) { return getErrorResponse(AbstractResponse.DEFAULT_THROTTLE_TIME, error); } @@ -172,7 +167,7 @@ public OffsetFetchResponse getErrorResponse(int throttleTimeMs, Throwable e) { } public static OffsetFetchRequest parse(ByteBuffer buffer, short version) { - return new OffsetFetchRequest(ApiKeys.OFFSET_FETCH.parseRequest(version, buffer), version); + return new OffsetFetchRequest(new OffsetFetchRequestData(new ByteBufferAccessor(buffer), version), version); } public boolean isAllPartitions() { @@ -180,7 +175,7 @@ public boolean isAllPartitions() { } @Override - protected Struct toStruct() { - return data.toStruct(version()); + protected OffsetFetchRequestData data() { + return data; } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchResponse.java index 8b550bef7aed5..3e30968715f8d 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetFetchResponse.java @@ -21,8 +21,8 @@ import org.apache.kafka.common.message.OffsetFetchResponseData.OffsetFetchResponsePartition; import org.apache.kafka.common.message.OffsetFetchResponseData.OffsetFetchResponseTopic; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -129,6 +129,7 @@ public OffsetFetchResponse(Errors error, Map resp * @param responseData Fetched offset information grouped by topic-partition */ public OffsetFetchResponse(int throttleTimeMs, Errors error, Map responseData) { + super(ApiKeys.OFFSET_FETCH); Map offsetFetchResponseTopicMap = new HashMap<>(); for (Map.Entry entry : responseData.entrySet()) { String topicName = entry.getKey().topic(); @@ -153,25 +154,26 @@ public OffsetFetchResponse(int throttleTimeMs, Errors error, Map= 2 ? Errors.forCode(data.errorCode()) : topLevelError(data); + } - Errors topLevelError = Errors.NONE; + private static Errors topLevelError(OffsetFetchResponseData data) { for (OffsetFetchResponseTopic topic : data.topics()) { for (OffsetFetchResponsePartition partition : topic.partitions()) { Errors partitionError = Errors.forCode(partition.errorCode()); if (partitionError != Errors.NONE && !PARTITION_ERRORS.contains(partitionError)) { - topLevelError = partitionError; - break; + return partitionError; } } } - - // for version 2 and later use the top-level error code (in ERROR_CODE_KEY_NAME) from the response. - // for older versions there is no top-level error in the response and all errors are partition errors, - // so if there is a group or coordinator error at the partition level use that as the top-level error. - // this way clients can depend on the top-level error regardless of the offset fetch version. - this.error = version >= 2 ? Errors.forCode(data.errorCode()) : topLevelError; + return Errors.NONE; } @Override @@ -213,12 +215,12 @@ public Map responseData() { } public static OffsetFetchResponse parse(ByteBuffer buffer, short version) { - return new OffsetFetchResponse(ApiKeys.OFFSET_FETCH.parseResponse(version, buffer), version); + return new OffsetFetchResponse(new OffsetFetchResponseData(new ByteBufferAccessor(buffer), version), version); } @Override - protected Struct toStruct(short version) { - return data.toStruct(version); + protected OffsetFetchResponseData data() { + return data; } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetsForLeaderEpochRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetsForLeaderEpochRequest.java index 6d377b257be3e..727c7087c439a 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/OffsetsForLeaderEpochRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetsForLeaderEpochRequest.java @@ -26,8 +26,8 @@ import org.apache.kafka.common.message.OffsetForLeaderEpochResponseData.EpochEndOffset; import org.apache.kafka.common.message.OffsetForLeaderEpochResponseData.OffsetForLeaderTopicResult; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.record.RecordBatch; import java.nio.ByteBuffer; @@ -38,6 +38,7 @@ import static org.apache.kafka.common.requests.OffsetsForLeaderEpochResponse.UNDEFINED_EPOCH_OFFSET; public class OffsetsForLeaderEpochRequest extends AbstractRequest { + /** * Sentinel replica_id value to indicate a regular consumer rather than another broker */ @@ -109,11 +110,7 @@ public OffsetsForLeaderEpochRequest(OffsetForLeaderEpochRequestData data, short this.data = data; } - public OffsetsForLeaderEpochRequest(Struct struct, short version) { - super(ApiKeys.OFFSET_FOR_LEADER_EPOCH, version); - this.data = new OffsetForLeaderEpochRequestData(struct, version); - } - + @Override public OffsetForLeaderEpochRequestData data() { return data; } @@ -123,12 +120,7 @@ public int replicaId() { } public static OffsetsForLeaderEpochRequest parse(ByteBuffer buffer, short version) { - return new OffsetsForLeaderEpochRequest(ApiKeys.OFFSET_FOR_LEADER_EPOCH.parseRequest(version, buffer), version); - } - - @Override - protected Struct toStruct() { - return data.toStruct(version()); + return new OffsetsForLeaderEpochRequest(new OffsetForLeaderEpochRequestData(new ByteBufferAccessor(buffer), version), version); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/OffsetsForLeaderEpochResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/OffsetsForLeaderEpochResponse.java index d737051724f17..893d5a2af20a0 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/OffsetsForLeaderEpochResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/OffsetsForLeaderEpochResponse.java @@ -18,8 +18,8 @@ import org.apache.kafka.common.message.OffsetForLeaderEpochResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.HashMap; @@ -46,13 +46,11 @@ public class OffsetsForLeaderEpochResponse extends AbstractResponse { private final OffsetForLeaderEpochResponseData data; public OffsetsForLeaderEpochResponse(OffsetForLeaderEpochResponseData data) { + super(ApiKeys.OFFSET_FOR_LEADER_EPOCH); this.data = data; } - public OffsetsForLeaderEpochResponse(Struct struct, short version) { - data = new OffsetForLeaderEpochResponseData(struct, version); - } - + @Override public OffsetForLeaderEpochResponseData data() { return data; } @@ -71,12 +69,7 @@ public int throttleTimeMs() { } public static OffsetsForLeaderEpochResponse parse(ByteBuffer buffer, short version) { - return new OffsetsForLeaderEpochResponse(ApiKeys.OFFSET_FOR_LEADER_EPOCH.responseSchema(version).read(buffer), version); - } - - @Override - protected Struct toStruct(short version) { - return data.toStruct(version); + return new OffsetsForLeaderEpochResponse(new OffsetForLeaderEpochResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ProduceRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ProduceRequest.java index bf1a38f9dfab2..758631a1d87aa 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ProduceRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ProduceRequest.java @@ -21,12 +21,9 @@ import org.apache.kafka.common.errors.UnsupportedCompressionTypeException; import org.apache.kafka.common.message.ProduceRequestData; import org.apache.kafka.common.message.ProduceResponseData; -import org.apache.kafka.common.network.Send; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.SendBuilder; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.record.BaseRecords; import org.apache.kafka.common.record.CompressionType; import org.apache.kafka.common.record.RecordBatch; @@ -130,11 +127,6 @@ public ProduceRequest(ProduceRequestData produceRequestData, short version) { this.transactionalId = data.transactionalId(); } - @Override - public Send toSend(String destination, RequestHeader header) { - return SendBuilder.buildRequestSend(destination, header, dataOrException()); - } - // visible for testing Map partitionSizes() { if (partitionSizes == null) { @@ -158,7 +150,8 @@ Map partitionSizes() { /** * @return data or IllegalStateException if the data is removed (to prevent unnecessary memory retention). */ - public ProduceRequestData dataOrException() { + @Override + public ProduceRequestData data() { // Store it in a local variable to protect against concurrent updates ProduceRequestData tmp = data; if (tmp == null) @@ -166,14 +159,6 @@ public ProduceRequestData dataOrException() { return tmp; } - /** - * Visible for testing. - */ - @Override - public Struct toStruct() { - return dataOrException().toStruct(version()); - } - @Override public String toString(boolean verbose) { // Use the same format as `Struct.toString()` @@ -194,7 +179,7 @@ public String toString(boolean verbose) { public ProduceResponse getErrorResponse(int throttleTimeMs, Throwable e) { /* In case the producer doesn't actually want any response */ if (acks == 0) return null; - Errors error = Errors.forException(e); + ApiError apiError = ApiError.fromThrowable(e); ProduceResponseData data = new ProduceResponseData().setThrottleTimeMs(throttleTimeMs); partitionSizes().forEach((tp, ignored) -> { ProduceResponseData.TopicProduceResponse tpr = data.responses().find(tp.topic()); @@ -208,8 +193,8 @@ public ProduceResponse getErrorResponse(int throttleTimeMs, Throwable e) { .setBaseOffset(INVALID_OFFSET) .setLogAppendTimeMs(RecordBatch.NO_TIMESTAMP) .setLogStartOffset(INVALID_OFFSET) - .setErrorMessage(e.getMessage()) - .setErrorCode(error.code())); + .setErrorMessage(apiError.message()) + .setErrorCode(apiError.error().code())); }); return new ProduceResponse(data); } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ProduceResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ProduceResponse.java index d2b72408078d9..2860576d53972 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ProduceResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ProduceResponse.java @@ -18,11 +18,9 @@ import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.message.ProduceResponseData; -import org.apache.kafka.common.network.Send; +import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.SendBuilder; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.record.RecordBatch; import java.nio.ByteBuffer; @@ -60,6 +58,7 @@ public class ProduceResponse extends AbstractResponse { private final ProduceResponseData data; public ProduceResponse(ProduceResponseData produceResponseData) { + super(ApiKeys.PRODUCE); this.data = produceResponseData; } @@ -82,11 +81,6 @@ public ProduceResponse(Map responses, int thr this(toData(responses, throttleTimeMs)); } - @Override - protected Send toSend(String destination, ResponseHeader header, short apiVersion) { - return SendBuilder.buildResponseSend(destination, header, this.data, apiVersion); - } - private static ProduceResponseData toData(Map responses, int throttleTimeMs) { ProduceResponseData data = new ProduceResponseData().setThrottleTimeMs(throttleTimeMs); responses.forEach((tp, response) -> { @@ -113,14 +107,6 @@ private static ProduceResponseData toData(Map return data; } - /** - * Visible for testing. - */ - @Override - public Struct toStruct(short version) { - return data.toStruct(version); - } - public ProduceResponseData data() { return this.data; } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/RenewDelegationTokenRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/RenewDelegationTokenRequest.java index 2c83f06adda62..2c364aef3dff7 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/RenewDelegationTokenRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/RenewDelegationTokenRequest.java @@ -21,8 +21,8 @@ import org.apache.kafka.common.message.RenewDelegationTokenRequestData; import org.apache.kafka.common.message.RenewDelegationTokenResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; public class RenewDelegationTokenRequest extends AbstractRequest { @@ -33,18 +33,9 @@ public RenewDelegationTokenRequest(RenewDelegationTokenRequestData data, short v this.data = data; } - public RenewDelegationTokenRequest(Struct struct, short version) { - super(ApiKeys.RENEW_DELEGATION_TOKEN, version); - this.data = new RenewDelegationTokenRequestData(struct, version); - } - public static RenewDelegationTokenRequest parse(ByteBuffer buffer, short version) { - return new RenewDelegationTokenRequest(ApiKeys.RENEW_DELEGATION_TOKEN.parseRequest(version, buffer), version); - } - - @Override - protected Struct toStruct() { - return data.toStruct(version()); + return new RenewDelegationTokenRequest(new RenewDelegationTokenRequestData( + new ByteBufferAccessor(buffer), version), version); } public RenewDelegationTokenRequestData data() { diff --git a/clients/src/main/java/org/apache/kafka/common/requests/RenewDelegationTokenResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/RenewDelegationTokenResponse.java index dfbb5c8518fad..2ef840378acb3 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/RenewDelegationTokenResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/RenewDelegationTokenResponse.java @@ -21,23 +21,21 @@ import org.apache.kafka.common.message.RenewDelegationTokenResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; public class RenewDelegationTokenResponse extends AbstractResponse { private final RenewDelegationTokenResponseData data; public RenewDelegationTokenResponse(RenewDelegationTokenResponseData data) { + super(ApiKeys.RENEW_DELEGATION_TOKEN); this.data = data; } - public RenewDelegationTokenResponse(Struct struct, short version) { - data = new RenewDelegationTokenResponseData(struct, version); - } - public static RenewDelegationTokenResponse parse(ByteBuffer buffer, short version) { - return new RenewDelegationTokenResponse(ApiKeys.RENEW_DELEGATION_TOKEN.responseSchema(version).read(buffer), version); + return new RenewDelegationTokenResponse(new RenewDelegationTokenResponseData( + new ByteBufferAccessor(buffer), version)); } @Override @@ -46,8 +44,8 @@ public Map errorCounts() { } @Override - protected Struct toStruct(short version) { - return data.toStruct(version); + protected RenewDelegationTokenResponseData data() { + return data; } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/RequestContext.java b/clients/src/main/java/org/apache/kafka/common/requests/RequestContext.java index 7136584d93a88..c90e2e5ef7c36 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/RequestContext.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/RequestContext.java @@ -22,7 +22,6 @@ import org.apache.kafka.common.network.ListenerName; import org.apache.kafka.common.network.Send; import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.security.auth.KafkaPrincipal; import org.apache.kafka.common.security.auth.KafkaPrincipalSerde; import org.apache.kafka.common.security.auth.SecurityProtocol; @@ -93,9 +92,7 @@ public RequestAndSize parseRequest(ByteBuffer buffer) { ApiKeys apiKey = header.apiKey(); try { short apiVersion = header.apiVersion(); - Struct struct = apiKey.parseRequest(apiVersion, buffer); - AbstractRequest body = AbstractRequest.parseRequest(apiKey, apiVersion, struct); - return new RequestAndSize(body, struct.sizeOf()); + return AbstractRequest.parseRequest(apiKey, apiVersion, buffer); } catch (Throwable ex) { throw new InvalidRequestException("Error getting request for apiKey: " + apiKey + ", apiVersion: " + header.apiVersion() + @@ -111,8 +108,7 @@ public RequestAndSize parseRequest(ByteBuffer buffer) { * over the network. */ public Send buildResponseSend(AbstractResponse body) { - ResponseHeader responseHeader = header.toResponseHeader(); - return body.toSend(connectionId, responseHeader, apiVersion()); + return body.toSend(connectionId, header.toResponseHeader(), apiVersion()); } /** @@ -126,8 +122,7 @@ public Send buildResponseSend(AbstractResponse body) { * so we do not lose the benefit of "zero copy" transfers from disk. */ public ByteBuffer buildResponseEnvelopePayload(AbstractResponse body) { - ResponseHeader responseHeader = header.toResponseHeader(); - return RequestUtils.serialize(responseHeader.toStruct(), body.toStruct(header.apiVersion())); + return body.serializeWithHeader(header.toResponseHeader(), apiVersion()); } private boolean isUnsupportedApiVersionsRequest() { diff --git a/clients/src/main/java/org/apache/kafka/common/requests/RequestHeader.java b/clients/src/main/java/org/apache/kafka/common/requests/RequestHeader.java index d145f49280852..29539491d1469 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/RequestHeader.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/RequestHeader.java @@ -21,7 +21,7 @@ import org.apache.kafka.common.message.RequestHeaderData; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; -import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.protocol.ObjectSerializationCache; import java.nio.ByteBuffer; @@ -32,20 +32,13 @@ public class RequestHeader implements AbstractRequestResponse { private final RequestHeaderData data; private final short headerVersion; - public RequestHeader(Struct struct, short headerVersion) { - this(new RequestHeaderData(struct, headerVersion), headerVersion); - } - - public RequestHeader(ApiKeys requestApiKey, - short requestVersion, - String clientId, - int correlationId) { - this(new RequestHeaderData() - .setRequestApiKey(requestApiKey.id) - .setRequestApiVersion(requestVersion) - .setClientId(clientId) - .setCorrelationId(correlationId), - ApiKeys.forId(requestApiKey.id).requestHeaderVersion(requestVersion)); + public RequestHeader(ApiKeys requestApiKey, short requestVersion, String clientId, int correlationId) { + this(new RequestHeaderData(). + setRequestApiKey(requestApiKey.id). + setRequestApiVersion(requestVersion). + setClientId(clientId). + setCorrelationId(correlationId), + requestApiKey.requestHeaderVersion(requestVersion)); } public RequestHeader(RequestHeaderData data, short headerVersion) { @@ -53,10 +46,6 @@ public RequestHeader(RequestHeaderData data, short headerVersion) { this.headerVersion = headerVersion; } - public Struct toStruct() { - return this.data.toStruct(headerVersion); - } - public ApiKeys apiKey() { return ApiKeys.forId(data.requestApiKey()); } @@ -81,14 +70,23 @@ public RequestHeaderData data() { return data; } + public void write(ByteBuffer buffer, ObjectSerializationCache serializationCache) { + data.write(new ByteBufferAccessor(buffer), serializationCache, headerVersion); + } + + public int size(ObjectSerializationCache serializationCache) { + return data.size(serializationCache, headerVersion); + } + public ResponseHeader toResponseHeader() { - return new ResponseHeader(data.correlationId(), - apiKey().responseHeaderVersion(apiVersion())); + return new ResponseHeader(data.correlationId(), apiKey().responseHeaderVersion(apiVersion())); } public static RequestHeader parse(ByteBuffer buffer) { short apiKey = -1; try { + // We derive the header version from the request api version, so we read that first. + // The request api version is part of `RequestHeaderData`, so we reset the buffer position after the read. int position = buffer.position(); apiKey = buffer.getShort(); short apiVersion = buffer.getShort(); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/RequestUtils.java b/clients/src/main/java/org/apache/kafka/common/requests/RequestUtils.java index ff42914ebff34..4fa701f76476d 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/RequestUtils.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/RequestUtils.java @@ -17,8 +17,10 @@ package org.apache.kafka.common.requests; import org.apache.kafka.common.message.ProduceRequestData; -import org.apache.kafka.common.protocol.types.Field; -import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.protocol.ByteBufferAccessor; +import org.apache.kafka.common.protocol.Message; +import org.apache.kafka.common.protocol.MessageSizeAccumulator; +import org.apache.kafka.common.protocol.ObjectSerializationCache; import org.apache.kafka.common.record.BaseRecords; import org.apache.kafka.common.record.RecordBatch; import org.apache.kafka.common.record.Records; @@ -32,28 +34,11 @@ public final class RequestUtils { private RequestUtils() {} - static void setLeaderEpochIfExists(Struct struct, Field.Int32 leaderEpochField, Optional leaderEpoch) { - struct.setIfExists(leaderEpochField, leaderEpoch.orElse(RecordBatch.NO_PARTITION_LEADER_EPOCH)); - } - - static Optional getLeaderEpoch(Struct struct, Field.Int32 leaderEpochField) { - int leaderEpoch = struct.getOrElse(leaderEpochField, RecordBatch.NO_PARTITION_LEADER_EPOCH); - return getLeaderEpoch(leaderEpoch); - } - static Optional getLeaderEpoch(int leaderEpoch) { return leaderEpoch == RecordBatch.NO_PARTITION_LEADER_EPOCH ? Optional.empty() : Optional.of(leaderEpoch); } - public static ByteBuffer serialize(Struct headerStruct, Struct bodyStruct) { - ByteBuffer buffer = ByteBuffer.allocate(headerStruct.sizeOf() + bodyStruct.sizeOf()); - headerStruct.writeTo(buffer); - bodyStruct.writeTo(buffer); - buffer.rewind(); - return buffer; - } - // visible for testing public static boolean hasIdempotentRecords(ProduceRequest request) { return flags(request).getKey(); @@ -72,7 +57,7 @@ public static boolean hasTransactionalRecords(ProduceRequest request) { public static AbstractMap.SimpleEntry flags(ProduceRequest request) { boolean hasIdempotentRecords = false; boolean hasTransactionalRecords = false; - for (ProduceRequestData.TopicProduceData tpd : request.dataOrException().topicData()) { + for (ProduceRequestData.TopicProduceData tpd : request.data().topicData()) { for (ProduceRequestData.PartitionProduceData ppd : tpd.partitionData()) { BaseRecords records = ppd.records(); if (records instanceof Records) { @@ -90,4 +75,37 @@ public static AbstractMap.SimpleEntry flags(ProduceRequest req } return new AbstractMap.SimpleEntry<>(hasIdempotentRecords, hasTransactionalRecords); } + + public static MessageSizeAccumulator size( + ObjectSerializationCache serializationCache, + Message header, + short headerVersion, + Message apiMessage, + short apiVersion + ) { + MessageSizeAccumulator messageSize = new MessageSizeAccumulator(); + if (header != null) + header.addSize(messageSize, serializationCache, headerVersion); + apiMessage.addSize(messageSize, serializationCache, apiVersion); + return messageSize; + } + + public static ByteBuffer serialize( + Message header, + short headerVersion, + Message apiMessage, + short apiVersion + ) { + ObjectSerializationCache serializationCache = new ObjectSerializationCache(); + MessageSizeAccumulator messageSize = RequestUtils.size(serializationCache, header, headerVersion, apiMessage, apiVersion); + + ByteBuffer buffer = ByteBuffer.allocate(messageSize.totalSize()); + ByteBufferAccessor bufferWritable = new ByteBufferAccessor(buffer); + if (header != null) + header.write(bufferWritable, serializationCache, headerVersion); + apiMessage.write(bufferWritable, serializationCache, apiVersion); + + buffer.rewind(); + return buffer; + } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ResponseHeader.java b/clients/src/main/java/org/apache/kafka/common/requests/ResponseHeader.java index 3fe7d9ebf4c7b..d45e28dc1d81d 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ResponseHeader.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ResponseHeader.java @@ -18,7 +18,7 @@ import org.apache.kafka.common.message.ResponseHeaderData; import org.apache.kafka.common.protocol.ByteBufferAccessor; -import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.protocol.ObjectSerializationCache; import java.nio.ByteBuffer; import java.util.Objects; @@ -30,10 +30,6 @@ public class ResponseHeader implements AbstractRequestResponse { private final ResponseHeaderData data; private final short headerVersion; - public ResponseHeader(Struct struct, short headerVersion) { - this(new ResponseHeaderData(struct, headerVersion), headerVersion); - } - public ResponseHeader(int correlationId, short headerVersion) { this(new ResponseHeaderData().setCorrelationId(correlationId), headerVersion); } @@ -43,12 +39,8 @@ public ResponseHeader(ResponseHeaderData data, short headerVersion) { this.headerVersion = headerVersion; } - public int sizeOf() { - return toStruct().sizeOf(); - } - - public Struct toStruct() { - return data.toStruct(headerVersion); + public int size(ObjectSerializationCache serializationCache) { + return data().size(serializationCache, headerVersion); } public int correlationId() { @@ -63,6 +55,18 @@ public ResponseHeaderData data() { return data; } + public void write(ByteBuffer buffer, ObjectSerializationCache serializationCache) { + data.write(new ByteBufferAccessor(buffer), serializationCache, headerVersion); + } + + @Override + public String toString() { + return "ResponseHeader(" + + "correlationId=" + data.correlationId() + + ", headerVersion=" + headerVersion + + ")"; + } + public static ResponseHeader parse(ByteBuffer buffer, short headerVersion) { return new ResponseHeader( new ResponseHeaderData(new ByteBufferAccessor(buffer), headerVersion), diff --git a/clients/src/main/java/org/apache/kafka/common/requests/SaslAuthenticateRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/SaslAuthenticateRequest.java index 5fdfbec66a923..35da4caf9c5c1 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/SaslAuthenticateRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/SaslAuthenticateRequest.java @@ -19,7 +19,7 @@ import org.apache.kafka.common.message.SaslAuthenticateRequestData; import org.apache.kafka.common.message.SaslAuthenticateResponseData; import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import java.nio.ByteBuffer; @@ -49,28 +49,17 @@ public SaslAuthenticateRequest build(short version) { @Override public String toString() { - StringBuilder bld = new StringBuilder(); - bld.append("(type=SaslAuthenticateRequest)"); - return bld.toString(); + return "(type=SaslAuthenticateRequest)"; } } private final SaslAuthenticateRequestData data; - public SaslAuthenticateRequest(SaslAuthenticateRequestData data) { - this(data, ApiKeys.SASL_AUTHENTICATE.latestVersion()); - } - public SaslAuthenticateRequest(SaslAuthenticateRequestData data, short version) { super(ApiKeys.SASL_AUTHENTICATE, version); this.data = data; } - public SaslAuthenticateRequest(Struct struct, short version) { - super(ApiKeys.SASL_AUTHENTICATE, version); - this.data = new SaslAuthenticateRequestData(struct, version); - } - public SaslAuthenticateRequestData data() { return data; } @@ -85,12 +74,8 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { } public static SaslAuthenticateRequest parse(ByteBuffer buffer, short version) { - return new SaslAuthenticateRequest(ApiKeys.SASL_AUTHENTICATE.parseRequest(version, buffer), version); - } - - @Override - protected Struct toStruct() { - return data.toStruct(version()); + return new SaslAuthenticateRequest(new SaslAuthenticateRequestData(new ByteBufferAccessor(buffer), version), + version); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/SaslAuthenticateResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/SaslAuthenticateResponse.java index 650252bbd7200..2e1a2189a39be 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/SaslAuthenticateResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/SaslAuthenticateResponse.java @@ -18,13 +18,12 @@ import org.apache.kafka.common.message.SaslAuthenticateResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.Map; - /** * Response from SASL server which for a SASL challenge as defined by the SASL protocol * for the mechanism configured for the client. @@ -34,13 +33,10 @@ public class SaslAuthenticateResponse extends AbstractResponse { private final SaslAuthenticateResponseData data; public SaslAuthenticateResponse(SaslAuthenticateResponseData data) { + super(ApiKeys.SASL_AUTHENTICATE); this.data = data; } - public SaslAuthenticateResponse(Struct struct, short version) { - this.data = new SaslAuthenticateResponseData(struct, version); - } - /** * Possible error codes: * SASL_AUTHENTICATION_FAILED(57) : Authentication failed @@ -67,11 +63,16 @@ public byte[] saslAuthBytes() { } @Override - public Struct toStruct(short version) { - return data.toStruct(version); + public int throttleTimeMs() { + return DEFAULT_THROTTLE_TIME; + } + + @Override + protected SaslAuthenticateResponseData data() { + return data; } public static SaslAuthenticateResponse parse(ByteBuffer buffer, short version) { - return new SaslAuthenticateResponse(ApiKeys.SASL_AUTHENTICATE.parseResponse(version, buffer), version); + return new SaslAuthenticateResponse(new SaslAuthenticateResponseData(new ByteBufferAccessor(buffer), version)); } -} \ No newline at end of file +} diff --git a/clients/src/main/java/org/apache/kafka/common/requests/SaslHandshakeRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/SaslHandshakeRequest.java index c2b5f729256a8..e64dd6b2cb792 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/SaslHandshakeRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/SaslHandshakeRequest.java @@ -20,7 +20,7 @@ import org.apache.kafka.common.message.SaslHandshakeRequestData; import org.apache.kafka.common.message.SaslHandshakeResponseData; import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import java.nio.ByteBuffer; @@ -56,20 +56,11 @@ public String toString() { private final SaslHandshakeRequestData data; - public SaslHandshakeRequest(SaslHandshakeRequestData data) { - this(data, ApiKeys.SASL_HANDSHAKE.latestVersion()); - } - public SaslHandshakeRequest(SaslHandshakeRequestData data, short version) { super(ApiKeys.SASL_HANDSHAKE, version); this.data = data; } - public SaslHandshakeRequest(Struct struct, short version) { - super(ApiKeys.SASL_HANDSHAKE, version); - this.data = new SaslHandshakeRequestData(struct, version); - } - public SaslHandshakeRequestData data() { return data; } @@ -82,12 +73,7 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { } public static SaslHandshakeRequest parse(ByteBuffer buffer, short version) { - return new SaslHandshakeRequest(ApiKeys.SASL_HANDSHAKE.parseRequest(version, buffer), version); - } - - @Override - protected Struct toStruct() { - return data.toStruct(version()); + return new SaslHandshakeRequest(new SaslHandshakeRequestData(new ByteBufferAccessor(buffer), version), version); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/SaslHandshakeResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/SaslHandshakeResponse.java index 161c075aa5318..b0ac2efcdec5f 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/SaslHandshakeResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/SaslHandshakeResponse.java @@ -18,8 +18,8 @@ import org.apache.kafka.common.message.SaslHandshakeResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.List; @@ -34,13 +34,10 @@ public class SaslHandshakeResponse extends AbstractResponse { private final SaslHandshakeResponseData data; public SaslHandshakeResponse(SaslHandshakeResponseData data) { + super(ApiKeys.SASL_HANDSHAKE); this.data = data; } - public SaslHandshakeResponse(Struct struct, short version) { - this.data = new SaslHandshakeResponseData(struct, version); - } - /* * Possible error codes: * UNSUPPORTED_SASL_MECHANISM(33): Client mechanism not enabled in server @@ -56,8 +53,13 @@ public Map errorCounts() { } @Override - public Struct toStruct(short version) { - return data.toStruct(version); + public int throttleTimeMs() { + return DEFAULT_THROTTLE_TIME; + } + + @Override + protected SaslHandshakeResponseData data() { + return data; } public List enabledMechanisms() { @@ -65,6 +67,6 @@ public List enabledMechanisms() { } public static SaslHandshakeResponse parse(ByteBuffer buffer, short version) { - return new SaslHandshakeResponse(ApiKeys.SASL_HANDSHAKE.parseResponse(version, buffer), version); + return new SaslHandshakeResponse(new SaslHandshakeResponseData(new ByteBufferAccessor(buffer), version)); } -} \ No newline at end of file +} diff --git a/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaRequest.java index 8ab8cc38ecad7..2b0aef766935b 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaRequest.java @@ -25,8 +25,8 @@ import org.apache.kafka.common.message.StopReplicaResponseData; import org.apache.kafka.common.message.StopReplicaResponseData.StopReplicaPartitionError; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.utils.MappedIterator; import org.apache.kafka.common.utils.Utils; @@ -103,10 +103,6 @@ private StopReplicaRequest(StopReplicaRequestData data, short version) { this.data = data; } - public StopReplicaRequest(Struct struct, short version) { - this(new StopReplicaRequestData(struct, version), version); - } - @Override public StopReplicaResponse getErrorResponse(int throttleTimeMs, Throwable e) { Errors error = Errors.forException(e); @@ -211,15 +207,11 @@ public long brokerEpoch() { } public static StopReplicaRequest parse(ByteBuffer buffer, short version) { - return new StopReplicaRequest(ApiKeys.STOP_REPLICA.parseRequest(version, buffer), version); - } - - public StopReplicaRequestData data() { - return data; + return new StopReplicaRequest(new StopReplicaRequestData(new ByteBufferAccessor(buffer), version), version); } @Override - protected Struct toStruct() { - return data.toStruct(version()); + protected StopReplicaRequestData data() { + return data; } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaResponse.java index 78b0d98c51a4a..27a2502e75103 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/StopReplicaResponse.java @@ -19,8 +19,8 @@ import org.apache.kafka.common.message.StopReplicaResponseData; import org.apache.kafka.common.message.StopReplicaResponseData.StopReplicaPartitionError; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.Collections; @@ -39,13 +39,10 @@ public class StopReplicaResponse extends AbstractResponse { private final StopReplicaResponseData data; public StopReplicaResponse(StopReplicaResponseData data) { + super(ApiKeys.STOP_REPLICA); this.data = data; } - public StopReplicaResponse(Struct struct, short version) { - data = new StopReplicaResponseData(struct, version); - } - public List partitionErrors() { return data.partitionErrors(); } @@ -65,12 +62,17 @@ public Map errorCounts() { } public static StopReplicaResponse parse(ByteBuffer buffer, short version) { - return new StopReplicaResponse(ApiKeys.STOP_REPLICA.parseResponse(version, buffer), version); + return new StopReplicaResponse(new StopReplicaResponseData(new ByteBufferAccessor(buffer), version)); + } + + @Override + public int throttleTimeMs() { + return DEFAULT_THROTTLE_TIME; } @Override - protected Struct toStruct(short version) { - return data.toStruct(version); + protected StopReplicaResponseData data() { + return data; } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/SyncGroupRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/SyncGroupRequest.java index f3b648791f1ee..18e5eb8b80964 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/SyncGroupRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/SyncGroupRequest.java @@ -20,8 +20,8 @@ import org.apache.kafka.common.message.SyncGroupRequestData; import org.apache.kafka.common.message.SyncGroupResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.HashMap; @@ -60,11 +60,6 @@ public SyncGroupRequest(SyncGroupRequestData data, short version) { this.data = data; } - public SyncGroupRequest(Struct struct, short version) { - super(ApiKeys.SYNC_GROUP, version); - this.data = new SyncGroupRequestData(struct, version); - } - @Override public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { return new SyncGroupResponse(new SyncGroupResponseData() @@ -93,11 +88,11 @@ public boolean areMandatoryProtocolTypeAndNamePresent() { } public static SyncGroupRequest parse(ByteBuffer buffer, short version) { - return new SyncGroupRequest(ApiKeys.SYNC_GROUP.parseRequest(version, buffer), version); + return new SyncGroupRequest(new SyncGroupRequestData(new ByteBufferAccessor(buffer), version), version); } @Override - protected Struct toStruct() { - return data.toStruct(version()); + protected SyncGroupRequestData data() { + return data; } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/SyncGroupResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/SyncGroupResponse.java index 7f02d2d25bc7e..a26b6c900bbdf 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/SyncGroupResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/SyncGroupResponse.java @@ -18,8 +18,8 @@ import org.apache.kafka.common.message.SyncGroupResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.Map; @@ -29,18 +29,10 @@ public class SyncGroupResponse extends AbstractResponse { public final SyncGroupResponseData data; public SyncGroupResponse(SyncGroupResponseData data) { + super(ApiKeys.SYNC_GROUP); this.data = data; } - public SyncGroupResponse(Struct struct) { - short latestVersion = (short) (SyncGroupResponseData.SCHEMAS.length - 1); - this.data = new SyncGroupResponseData(struct, latestVersion); - } - - public SyncGroupResponse(Struct struct, short version) { - this.data = new SyncGroupResponseData(struct, version); - } - @Override public int throttleTimeMs() { return data.throttleTimeMs(); @@ -56,8 +48,8 @@ public Map errorCounts() { } @Override - protected Struct toStruct(short version) { - return data.toStruct(version); + protected SyncGroupResponseData data() { + return data; } @Override @@ -66,7 +58,7 @@ public String toString() { } public static SyncGroupResponse parse(ByteBuffer buffer, short version) { - return new SyncGroupResponse(ApiKeys.SYNC_GROUP.parseResponse(version, buffer), version); + return new SyncGroupResponse(new SyncGroupResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitRequest.java index bee9c134cfa29..055793cf54b15 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitRequest.java @@ -25,8 +25,8 @@ import org.apache.kafka.common.message.TxnOffsetCommitResponseData.TxnOffsetCommitResponsePartition; import org.apache.kafka.common.message.TxnOffsetCommitResponseData.TxnOffsetCommitResponseTopic; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.record.RecordBatch; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -129,11 +129,6 @@ public TxnOffsetCommitRequest(TxnOffsetCommitRequestData data, short version) { this.data = data; } - public TxnOffsetCommitRequest(Struct struct, short version) { - super(ApiKeys.TXN_OFFSET_COMMIT, version); - this.data = new TxnOffsetCommitRequestData(struct, version); - } - public Map offsets() { List topics = data.topics(); Map offsetMap = new HashMap<>(); @@ -173,8 +168,8 @@ static List getTopics(Map getErrorResponseTopics(List requestTopics, @@ -206,7 +201,8 @@ public TxnOffsetCommitResponse getErrorResponse(int throttleTimeMs, Throwable e) } public static TxnOffsetCommitRequest parse(ByteBuffer buffer, short version) { - return new TxnOffsetCommitRequest(ApiKeys.TXN_OFFSET_COMMIT.parseRequest(version, buffer), version); + return new TxnOffsetCommitRequest(new TxnOffsetCommitRequestData( + new ByteBufferAccessor(buffer), version), version); } public static class CommittedOffset { diff --git a/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitResponse.java index c71719dad33a0..1ead0ec6d6073 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/TxnOffsetCommitResponse.java @@ -21,8 +21,8 @@ import org.apache.kafka.common.message.TxnOffsetCommitResponseData.TxnOffsetCommitResponsePartition; import org.apache.kafka.common.message.TxnOffsetCommitResponseData.TxnOffsetCommitResponseTopic; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -50,14 +50,12 @@ public class TxnOffsetCommitResponse extends AbstractResponse { public final TxnOffsetCommitResponseData data; public TxnOffsetCommitResponse(TxnOffsetCommitResponseData data) { + super(ApiKeys.TXN_OFFSET_COMMIT); this.data = data; } - public TxnOffsetCommitResponse(Struct struct, short version) { - this.data = new TxnOffsetCommitResponseData(struct, version); - } - public TxnOffsetCommitResponse(int requestThrottleMs, Map responseData) { + super(ApiKeys.TXN_OFFSET_COMMIT); Map responseTopicDataMap = new HashMap<>(); for (Map.Entry entry : responseData.entrySet()) { @@ -80,8 +78,8 @@ public TxnOffsetCommitResponse(int requestThrottleMs, Map errors() { } public static TxnOffsetCommitResponse parse(ByteBuffer buffer, short version) { - return new TxnOffsetCommitResponse(ApiKeys.TXN_OFFSET_COMMIT.parseResponse(version, buffer), version); + return new TxnOffsetCommitResponse(new TxnOffsetCommitResponseData(new ByteBufferAccessor(buffer), version)); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/UpdateFeaturesRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/UpdateFeaturesRequest.java index 3276c0abbe438..d61056f64bedc 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/UpdateFeaturesRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/UpdateFeaturesRequest.java @@ -23,7 +23,7 @@ import org.apache.kafka.common.message.UpdateFeaturesResponseData.UpdatableFeatureResult; import org.apache.kafka.common.message.UpdateFeaturesResponseData.UpdatableFeatureResultCollection; import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.protocol.ByteBufferAccessor; public class UpdateFeaturesRequest extends AbstractRequest { @@ -54,11 +54,6 @@ public UpdateFeaturesRequest(UpdateFeaturesRequestData data, short version) { this.data = data; } - public UpdateFeaturesRequest(Struct struct, short version) { - super(ApiKeys.UPDATE_FEATURES, version); - this.data = new UpdateFeaturesRequestData(struct, version); - } - @Override public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { final ApiError apiError = ApiError.fromThrowable(e); @@ -73,11 +68,7 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { final UpdateFeaturesResponseData responseData = new UpdateFeaturesResponseData() .setThrottleTimeMs(throttleTimeMs) .setResults(results); - return new UpdateFeaturesResponse(responseData); } - - @Override - protected Struct toStruct() { - return data.toStruct(version()); + return new UpdateFeaturesResponse(responseData); } public UpdateFeaturesRequestData data() { @@ -85,8 +76,7 @@ public UpdateFeaturesRequestData data() { } public static UpdateFeaturesRequest parse(ByteBuffer buffer, short version) { - return new UpdateFeaturesRequest( - ApiKeys.UPDATE_FEATURES.parseRequest(version, buffer), version); + return new UpdateFeaturesRequest(new UpdateFeaturesRequestData(new ByteBufferAccessor(buffer), version), version); } public static boolean isDeleteRequest(UpdateFeaturesRequestData.FeatureUpdateKey update) { diff --git a/clients/src/main/java/org/apache/kafka/common/requests/UpdateFeaturesResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/UpdateFeaturesResponse.java index 5754f13595c8b..2c28842cd89d6 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/UpdateFeaturesResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/UpdateFeaturesResponse.java @@ -19,12 +19,13 @@ import java.nio.ByteBuffer; import java.util.Map; import java.util.stream.Collectors; + import org.apache.kafka.common.message.UpdateFeaturesResponseData; import org.apache.kafka.common.message.UpdateFeaturesResponseData.UpdatableFeatureResult; import org.apache.kafka.common.message.UpdateFeaturesResponseData.UpdatableFeatureResultCollection; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; /** @@ -40,18 +41,10 @@ public class UpdateFeaturesResponse extends AbstractResponse { private final UpdateFeaturesResponseData data; public UpdateFeaturesResponse(UpdateFeaturesResponseData data) { + super(ApiKeys.UPDATE_FEATURES); this.data = data; } - public UpdateFeaturesResponse(Struct struct) { - final short latestVersion = (short) (UpdateFeaturesResponseData.SCHEMAS.length - 1); - this.data = new UpdateFeaturesResponseData(struct, latestVersion); - } - - public UpdateFeaturesResponse(Struct struct, short version) { - this.data = new UpdateFeaturesResponseData(struct, version); - } - public Map errors() { return data.results().valuesSet().stream().collect( Collectors.toMap( @@ -69,11 +62,6 @@ public int throttleTimeMs() { return data.throttleTimeMs(); } - @Override - protected Struct toStruct(short version) { - return data.toStruct(version); - } - @Override public String toString() { return data.toString(); @@ -84,7 +72,7 @@ public UpdateFeaturesResponseData data() { } public static UpdateFeaturesResponse parse(ByteBuffer buffer, short version) { - return new UpdateFeaturesResponse(ApiKeys.UPDATE_FEATURES.parseResponse(version, buffer), version); + return new UpdateFeaturesResponse(new UpdateFeaturesResponseData(new ByteBufferAccessor(buffer), version)); } public static UpdateFeaturesResponse createWithErrors(ApiError topLevelError, Map updateErrors, int throttleTimeMs) { diff --git a/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java index 7ae861b5c3df0..a0ef3471f12dc 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataRequest.java @@ -25,8 +25,8 @@ import org.apache.kafka.common.message.UpdateMetadataResponseData; import org.apache.kafka.common.network.ListenerName; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.security.auth.SecurityProtocol; import org.apache.kafka.common.utils.FlattenedIterator; import org.apache.kafka.common.utils.Utils; @@ -166,10 +166,6 @@ private static String listenerNameFromSecurityProtocol(UpdateMetadataEndpoint en return ListenerName.forSecurityProtocol(securityProtocol).value(); } - public UpdateMetadataRequest(Struct struct, short version) { - this(new UpdateMetadataRequestData(struct, version), version); - } - @Override public int controllerId() { return data.controllerId(); @@ -205,15 +201,11 @@ public List liveBrokers() { } @Override - protected Struct toStruct() { - return data.toStruct(version()); - } - - public UpdateMetadataRequestData data() { + protected UpdateMetadataRequestData data() { return data; } public static UpdateMetadataRequest parse(ByteBuffer buffer, short version) { - return new UpdateMetadataRequest(ApiKeys.UPDATE_METADATA.parseRequest(version, buffer), version); + return new UpdateMetadataRequest(new UpdateMetadataRequestData(new ByteBufferAccessor(buffer), version), version); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataResponse.java index c7d803f560eb1..4749067a9a786 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/UpdateMetadataResponse.java @@ -18,8 +18,8 @@ import org.apache.kafka.common.message.UpdateMetadataResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.Map; @@ -29,13 +29,10 @@ public class UpdateMetadataResponse extends AbstractResponse { private final UpdateMetadataResponseData data; public UpdateMetadataResponse(UpdateMetadataResponseData data) { + super(ApiKeys.UPDATE_METADATA); this.data = data; } - public UpdateMetadataResponse(Struct struct, short version) { - this(new UpdateMetadataResponseData(struct, version)); - } - public Errors error() { return Errors.forCode(data.errorCode()); } @@ -45,12 +42,18 @@ public Map errorCounts() { return errorCounts(error()); } + @Override + public int throttleTimeMs() { + return DEFAULT_THROTTLE_TIME; + } + public static UpdateMetadataResponse parse(ByteBuffer buffer, short version) { - return new UpdateMetadataResponse(ApiKeys.UPDATE_METADATA.parseResponse(version, buffer), version); + return new UpdateMetadataResponse(new UpdateMetadataResponseData(new ByteBufferAccessor(buffer), version)); } @Override - protected Struct toStruct(short version) { - return data.toStruct(version); + protected UpdateMetadataResponseData data() { + return data; } + } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/VoteRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/VoteRequest.java index 9fe5cfc128a01..d31d019206f0b 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/VoteRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/VoteRequest.java @@ -20,12 +20,14 @@ import org.apache.kafka.common.message.VoteRequestData; import org.apache.kafka.common.message.VoteResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; +import java.nio.ByteBuffer; import java.util.Collections; public class VoteRequest extends AbstractRequest { + public static class Builder extends AbstractRequest.Builder { private final VoteRequestData data; @@ -52,14 +54,9 @@ private VoteRequest(VoteRequestData data, short version) { this.data = data; } - public VoteRequest(Struct struct, short version) { - super(ApiKeys.VOTE, version); - this.data = new VoteRequestData(struct, version); - } - @Override - protected Struct toStruct() { - return data.toStruct(version()); + protected VoteRequestData data() { + return data; } @Override @@ -68,6 +65,10 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { .setErrorCode(Errors.forException(e).code())); } + public static VoteRequest parse(ByteBuffer buffer, short version) { + return new VoteRequest(new VoteRequestData(new ByteBufferAccessor(buffer), version), version); + } + public static VoteRequestData singletonRequest(TopicPartition topicPartition, int candidateEpoch, int candidateId, diff --git a/clients/src/main/java/org/apache/kafka/common/requests/VoteResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/VoteResponse.java index ab45d918dc4e8..5956fe0077e99 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/VoteResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/VoteResponse.java @@ -20,8 +20,8 @@ import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.message.VoteResponseData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.Collections; @@ -45,23 +45,10 @@ public class VoteResponse extends AbstractResponse { public final VoteResponseData data; public VoteResponse(VoteResponseData data) { + super(ApiKeys.VOTE); this.data = data; } - public VoteResponse(Struct struct, short version) { - this.data = new VoteResponseData(struct, version); - } - - public VoteResponse(Struct struct) { - short latestVersion = (short) (VoteResponseData.SCHEMAS.length - 1); - this.data = new VoteResponseData(struct, latestVersion); - } - - @Override - protected Struct toStruct(short version) { - return data.toStruct(version); - } - public static VoteResponseData singletonResponse(Errors topLevelError, TopicPartition topicPartition, Errors partitionLevelError, @@ -95,7 +82,17 @@ public Map errorCounts() { return errors; } + @Override + protected VoteResponseData data() { + return data; + } + + @Override + public int throttleTimeMs() { + return DEFAULT_THROTTLE_TIME; + } + public static VoteResponse parse(ByteBuffer buffer, short version) { - return new VoteResponse(ApiKeys.VOTE.responseSchema(version).read(buffer), version); + return new VoteResponse(new VoteResponseData(new ByteBufferAccessor(buffer), version)); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/WriteTxnMarkersRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/WriteTxnMarkersRequest.java index 88bde08dc4488..5998ca7e07e7e 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/WriteTxnMarkersRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/WriteTxnMarkersRequest.java @@ -21,8 +21,8 @@ import org.apache.kafka.common.message.WriteTxnMarkersRequestData.WritableTxnMarker; import org.apache.kafka.common.message.WriteTxnMarkersRequestData.WritableTxnMarkerTopic; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -141,14 +141,9 @@ private WriteTxnMarkersRequest(WriteTxnMarkersRequestData data, short version) { this.data = data; } - public WriteTxnMarkersRequest(Struct struct, short version) { - super(ApiKeys.WRITE_TXN_MARKERS, version); - this.data = new WriteTxnMarkersRequestData(struct, version); - } - @Override - protected Struct toStruct() { - return data.toStruct(version()); + protected WriteTxnMarkersRequestData data() { + return data; } @Override @@ -190,7 +185,7 @@ public List markers() { } public static WriteTxnMarkersRequest parse(ByteBuffer buffer, short version) { - return new WriteTxnMarkersRequest(ApiKeys.WRITE_TXN_MARKERS.parseRequest(version, buffer), version); + return new WriteTxnMarkersRequest(new WriteTxnMarkersRequestData(new ByteBufferAccessor(buffer), version), version); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/WriteTxnMarkersResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/WriteTxnMarkersResponse.java index 8fdcb99c9333a..fd2a834d24569 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/WriteTxnMarkersResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/WriteTxnMarkersResponse.java @@ -22,8 +22,8 @@ import org.apache.kafka.common.message.WriteTxnMarkersResponseData.WritableTxnMarkerResult; import org.apache.kafka.common.message.WriteTxnMarkersResponseData.WritableTxnMarkerTopicResult; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -49,10 +49,10 @@ */ public class WriteTxnMarkersResponse extends AbstractResponse { - private final Map> errors; - public final WriteTxnMarkersResponseData data; + private final WriteTxnMarkersResponseData data; public WriteTxnMarkersResponse(Map> errors) { + super(ApiKeys.WRITE_TXN_MARKERS); List markers = new ArrayList<>(); for (Map.Entry> markerEntry : errors.entrySet()) { Map responseTopicDataMap = new HashMap<>(); @@ -74,48 +74,53 @@ public WriteTxnMarkersResponse(Map> errors) { .setTopics(new ArrayList<>(responseTopicDataMap.values())) ); } - - this.errors = errors; this.data = new WriteTxnMarkersResponseData() .setMarkers(markers); } - public WriteTxnMarkersResponse(Struct struct, short version) { - this.data = new WriteTxnMarkersResponseData(struct, version); - this.errors = new HashMap<>(); + public WriteTxnMarkersResponse(WriteTxnMarkersResponseData data) { + super(ApiKeys.WRITE_TXN_MARKERS); + this.data = data; + } + + @Override + public WriteTxnMarkersResponseData data() { + return data; + } + + public Map> errorsByProducerId() { + Map> errors = new HashMap<>(); for (WritableTxnMarkerResult marker : data.markers()) { Map topicPartitionErrorsMap = new HashMap<>(); - for (WritableTxnMarkerTopicResult topic: marker.topics()) { - for (WritableTxnMarkerPartitionResult partitionResult: topic.partitions()) { + for (WritableTxnMarkerTopicResult topic : marker.topics()) { + for (WritableTxnMarkerPartitionResult partitionResult : topic.partitions()) { topicPartitionErrorsMap.put(new TopicPartition(topic.name(), partitionResult.partitionIndex()), - Errors.forCode(partitionResult.errorCode())); + Errors.forCode(partitionResult.errorCode())); } } errors.put(marker.producerId(), topicPartitionErrorsMap); } + return errors; } @Override - protected Struct toStruct(short version) { - return data.toStruct(version); - } - - public Map errors(long producerId) { - return errors.get(producerId); + public int throttleTimeMs() { + return DEFAULT_THROTTLE_TIME; } @Override public Map errorCounts() { Map errorCounts = new HashMap<>(); - errors.values().forEach(allErrors -> - allErrors.values().forEach(error -> - updateErrorCounts(errorCounts, error) - ) - ); + for (WritableTxnMarkerResult marker : data.markers()) { + for (WritableTxnMarkerTopicResult topic : marker.topics()) { + for (WritableTxnMarkerPartitionResult partitionResult : topic.partitions()) + updateErrorCounts(errorCounts, Errors.forCode(partitionResult.errorCode())); + } + } return errorCounts; } public static WriteTxnMarkersResponse parse(ByteBuffer buffer, short version) { - return new WriteTxnMarkersResponse(ApiKeys.WRITE_TXN_MARKERS.parseResponse(version, buffer), version); + return new WriteTxnMarkersResponse(new WriteTxnMarkersResponseData(new ByteBufferAccessor(buffer), version)); } } diff --git a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java index 00a4bfc64cfbe..5694d853a590f 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java +++ b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java @@ -58,6 +58,7 @@ import javax.security.sasl.SaslClient; import javax.security.sasl.SaslException; import java.io.IOException; +import java.nio.BufferUnderflowException; import java.nio.ByteBuffer; import java.nio.channels.SelectionKey; import java.security.Principal; @@ -432,13 +433,16 @@ private boolean sendSaslClientToken(byte[] serverToken, boolean isInitial) throw byte[] saslToken = createSaslToken(serverToken, isInitial); if (saslToken != null) { ByteBuffer tokenBuf = ByteBuffer.wrap(saslToken); - if (saslAuthenticateVersion != DISABLE_KAFKA_SASL_AUTHENTICATE_HEADER) { + Send send; + if (saslAuthenticateVersion == DISABLE_KAFKA_SASL_AUTHENTICATE_HEADER) { + send = new NetworkSend(node, tokenBuf); + } else { SaslAuthenticateRequestData data = new SaslAuthenticateRequestData() .setAuthBytes(tokenBuf.array()); SaslAuthenticateRequest request = new SaslAuthenticateRequest.Builder(data).build(saslAuthenticateVersion); - tokenBuf = request.serialize(nextRequestHeader(ApiKeys.SASL_AUTHENTICATE, saslAuthenticateVersion)); + send = request.toSend(node, nextRequestHeader(ApiKeys.SASL_AUTHENTICATE, saslAuthenticateVersion)); } - send(new NetworkSend(node, tokenBuf)); + send(send); return true; } } @@ -573,7 +577,7 @@ private AbstractResponse receiveKafkaResponse() throws IOException { currentRequestHeader = null; return response; } - } catch (SchemaException | IllegalArgumentException e) { + } catch (BufferUnderflowException | SchemaException | IllegalArgumentException e) { /* * Account for the fact that during re-authentication there may be responses * arriving for requests that were sent in the past. diff --git a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticator.java b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticator.java index 924ae4431d9ad..e6a2c5693bd83 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticator.java +++ b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticator.java @@ -170,7 +170,7 @@ public SaslServerAuthenticator(Map configs, List enabledMechanisms = (List) this.configs.get(BrokerSecurityConfigs.SASL_ENABLED_MECHANISMS_CONFIG); if (enabledMechanisms == null || enabledMechanisms.isEmpty()) throw new IllegalArgumentException("No SASL mechanisms are enabled"); - this.enabledMechanisms = new ArrayList(new HashSet(enabledMechanisms)); + this.enabledMechanisms = new ArrayList<>(new HashSet<>(enabledMechanisms)); for (String mechanism : this.enabledMechanisms) { if (!callbackHandlers.containsKey(mechanism)) throw new IllegalArgumentException("Callback handler not specified for SASL mechanism " + mechanism); @@ -470,7 +470,7 @@ private void handleSaslToken(byte[] clientToken) throws IOException { String errorMessage = "Authentication failed during " + reauthInfo.authenticationOrReauthenticationText() + " due to invalid credentials with SASL mechanism " + saslMechanism; - sendKafkaResponse(requestContext, new SaslAuthenticateResponse( + buildResponseOnAuthenticateFailure(requestContext, new SaslAuthenticateResponse( new SaslAuthenticateResponseData() .setErrorCode(Errors.SASL_AUTHENTICATION_FAILED.code()) .setErrorMessage(errorMessage))); @@ -576,8 +576,8 @@ private void handleApiVersionsRequest(RequestContext context, ApiVersionsRequest else if (!apiVersionsRequest.isValid()) sendKafkaResponse(context, apiVersionsRequest.getErrorResponse(0, Errors.INVALID_REQUEST.exception())); else { - metadataRegistry.registerClientInformation(new ClientInformation(apiVersionsRequest.data.clientSoftwareName(), - apiVersionsRequest.data.clientSoftwareVersion())); + metadataRegistry.registerClientInformation(new ClientInformation(apiVersionsRequest.data().clientSoftwareName(), + apiVersionsRequest.data().clientSoftwareVersion())); sendKafkaResponse(context, apiVersionsResponse()); setSaslState(SaslState.HANDSHAKE_REQUEST); } @@ -676,8 +676,7 @@ else if (connectionsMaxReauthMs == null) retvalSessionLifetimeMs = zeroIfNegative(credentialExpirationMs - authenticationEndMs); else retvalSessionLifetimeMs = zeroIfNegative( - Math.min(credentialExpirationMs - authenticationEndMs, - connectionsMaxReauthMs)); + Math.min(credentialExpirationMs - authenticationEndMs, connectionsMaxReauthMs)); if (retvalSessionLifetimeMs > 0L) sessionExpirationTimeNanos = authenticationEndNanos + 1000 * 1000 * retvalSessionLifetimeMs; } diff --git a/clients/src/main/java/org/apache/kafka/common/utils/Utils.java b/clients/src/main/java/org/apache/kafka/common/utils/Utils.java index 5a8aefec8a00c..3f4a9312fdc3e 100755 --- a/clients/src/main/java/org/apache/kafka/common/utils/Utils.java +++ b/clients/src/main/java/org/apache/kafka/common/utils/Utils.java @@ -1321,4 +1321,9 @@ public static long getDateTime(String timestamp) throws ParseException, IllegalA return date.getTime(); } } + + @SuppressWarnings("unchecked") + public static Iterator covariantCast(Iterator iterator) { + return (Iterator) iterator; + } } diff --git a/clients/src/test/java/org/apache/kafka/clients/MetadataTest.java b/clients/src/test/java/org/apache/kafka/clients/MetadataTest.java index ba965db500d52..58180008d668d 100644 --- a/clients/src/test/java/org/apache/kafka/clients/MetadataTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/MetadataTest.java @@ -30,7 +30,7 @@ import org.apache.kafka.common.message.MetadataResponseData.MetadataResponseTopicCollection; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.protocol.MessageTestUtil; import org.apache.kafka.common.requests.MetadataRequest; import org.apache.kafka.common.requests.MetadataResponse; import org.apache.kafka.common.utils.LogContext; @@ -41,6 +41,7 @@ import org.junit.Test; import java.net.InetSocketAddress; +import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -68,7 +69,7 @@ public class MetadataTest { new ClusterResourceListeners()); private static MetadataResponse emptyMetadataResponse() { - return MetadataResponse.prepareResponse( + return TestUtils.metadataResponse( Collections.emptyList(), null, -1, @@ -196,8 +197,8 @@ public void testIgnoreLeaderEpochInOlderMetadataResponse() { .setBrokers(new MetadataResponseBrokerCollection()); for (short version = ApiKeys.METADATA.oldestVersion(); version < 9; version++) { - Struct struct = data.toStruct(version); - MetadataResponse response = new MetadataResponse(struct, version); + ByteBuffer buffer = MessageTestUtil.messageToByteBuffer(data, version); + MetadataResponse response = MetadataResponse.parse(buffer, version); assertFalse(response.hasReliableLeaderEpochs()); metadata.updateWithCurrentRequestVersion(response, false, 100); assertTrue(metadata.partitionMetadataIfCurrent(tp).isPresent()); @@ -206,8 +207,8 @@ public void testIgnoreLeaderEpochInOlderMetadataResponse() { } for (short version = 9; version <= ApiKeys.METADATA.latestVersion(); version++) { - Struct struct = data.toStruct(version); - MetadataResponse response = new MetadataResponse(struct, version); + ByteBuffer buffer = MessageTestUtil.messageToByteBuffer(data, version); + MetadataResponse response = MetadataResponse.parse(buffer, version); assertTrue(response.hasReliableLeaderEpochs()); metadata.updateWithCurrentRequestVersion(response, false, 100); assertTrue(metadata.partitionMetadataIfCurrent(tp).isPresent()); @@ -244,7 +245,7 @@ public void testStaleMetadata() { .setTopics(topics) .setBrokers(new MetadataResponseBrokerCollection()); - metadata.updateWithCurrentRequestVersion(new MetadataResponse(data), false, 100); + metadata.updateWithCurrentRequestVersion(new MetadataResponse(data, ApiKeys.METADATA.latestVersion()), false, 100); // Older epoch with changed ISR should be ignored partitionMetadata @@ -256,7 +257,7 @@ public void testStaleMetadata() { .setOfflineReplicas(Collections.emptyList()) .setErrorCode(Errors.NONE.code()); - metadata.updateWithCurrentRequestVersion(new MetadataResponse(data), false, 101); + metadata.updateWithCurrentRequestVersion(new MetadataResponse(data, ApiKeys.METADATA.latestVersion()), false, 101); assertEquals(Optional.of(10), metadata.lastSeenLeaderEpoch(tp)); assertTrue(metadata.partitionMetadataIfCurrent(tp).isPresent()); @@ -757,12 +758,14 @@ public void testLeaderMetadataInconsistentWithBrokerMetadata() { metadata.updateWithCurrentRequestVersion(new MetadataResponse(new MetadataResponseData() .setTopics(buildTopicCollection(tp.topic(), firstPartitionMetadata)) - .setBrokers(buildBrokerCollection(Arrays.asList(node0, node1, node2)))), + .setBrokers(buildBrokerCollection(Arrays.asList(node0, node1, node2))), + ApiKeys.METADATA.latestVersion()), false, 10L); metadata.updateWithCurrentRequestVersion(new MetadataResponse(new MetadataResponseData() .setTopics(buildTopicCollection(tp.topic(), secondPartitionMetadata)) - .setBrokers(buildBrokerCollection(Arrays.asList(node1, node2)))), + .setBrokers(buildBrokerCollection(Arrays.asList(node1, node2))), + ApiKeys.METADATA.latestVersion()), false, 20L); assertNull(metadata.fetch().leaderFor(tp)); diff --git a/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java b/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java index 0615c9c8f3427..dd7ca94c45f08 100644 --- a/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java @@ -30,13 +30,13 @@ import org.apache.kafka.common.network.NetworkReceive; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.requests.AbstractResponse; import org.apache.kafka.common.requests.ApiVersionsResponse; import org.apache.kafka.common.requests.MetadataRequest; import org.apache.kafka.common.requests.MetadataResponse; import org.apache.kafka.common.requests.ProduceRequest; +import org.apache.kafka.common.requests.ProduceResponse; import org.apache.kafka.common.requests.RequestHeader; -import org.apache.kafka.common.requests.ResponseHeader; import org.apache.kafka.common.security.authenticator.SaslClientAuthenticator; import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.MockTime; @@ -185,9 +185,10 @@ public void testUnsupportedVersionDuringInternalMetadataRequest() { private void checkSimpleRequestResponse(NetworkClient networkClient) { awaitReady(networkClient, node); // has to be before creating any request, as it may send ApiVersionsRequest and its response is mocked with correlation id 0 + short requestVersion = PRODUCE.latestVersion(); ProduceRequest.Builder builder = new ProduceRequest.Builder( - PRODUCE.latestVersion(), - PRODUCE.latestVersion(), + requestVersion, + requestVersion, new ProduceRequestData() .setAcks((short) 1) .setTimeoutMs(1000)); @@ -197,18 +198,9 @@ private void checkSimpleRequestResponse(NetworkClient networkClient) { networkClient.send(request, time.milliseconds()); networkClient.poll(1, time.milliseconds()); assertEquals(1, networkClient.inFlightRequestCount()); - ResponseHeader respHeader = - new ResponseHeader(request.correlationId(), - request.apiKey().responseHeaderVersion(PRODUCE.latestVersion())); - Struct resp = new ProduceResponseData() - .setThrottleTimeMs(100) - .toStruct(PRODUCE.latestVersion()); - Struct responseHeaderStruct = respHeader.toStruct(); - int size = responseHeaderStruct.sizeOf() + resp.sizeOf(); - ByteBuffer buffer = ByteBuffer.allocate(size); - responseHeaderStruct.writeTo(buffer); - resp.writeTo(buffer); - buffer.flip(); + ProduceResponse produceResponse = new ProduceResponse(new ProduceResponseData()); + ByteBuffer buffer = produceResponse.serializeWithHeader(requestVersion, request.correlationId()); + buffer.rewind(); selector.completeReceive(new NetworkReceive(node.idString(), buffer)); List responses = networkClient.poll(1, time.milliseconds()); assertEquals(1, responses.size()); @@ -219,7 +211,7 @@ private void checkSimpleRequestResponse(NetworkClient networkClient) { } private void delayedApiVersionsResponse(int correlationId, short version, ApiVersionsResponse response) { - ByteBuffer buffer = response.serialize(ApiKeys.API_VERSIONS, version, correlationId); + ByteBuffer buffer = response.serializeWithHeader(version, correlationId); selector.delayedReceive(new DelayedReceive(node.idString(), new NetworkReceive(node.idString(), buffer))); } @@ -502,9 +494,10 @@ public void testConnectionSetupTimeout() { public void testConnectionThrottling() { // Instrument the test to return a response with a 100ms throttle delay. awaitReady(client, node); + short requestVersion = PRODUCE.latestVersion(); ProduceRequest.Builder builder = new ProduceRequest.Builder( - PRODUCE.latestVersion(), - PRODUCE.latestVersion(), + requestVersion, + requestVersion, new ProduceRequestData() .setAcks((short) 1) .setTimeoutMs(1000)); @@ -513,18 +506,10 @@ public void testConnectionThrottling() { defaultRequestTimeoutMs, handler); client.send(request, time.milliseconds()); client.poll(1, time.milliseconds()); - ResponseHeader respHeader = - new ResponseHeader(request.correlationId(), - request.apiKey().responseHeaderVersion(PRODUCE.latestVersion())); - Struct resp = new ProduceResponseData() - .setThrottleTimeMs(100) - .toStruct(PRODUCE.latestVersion()); - Struct responseHeaderStruct = respHeader.toStruct(); - int size = responseHeaderStruct.sizeOf() + resp.sizeOf(); - ByteBuffer buffer = ByteBuffer.allocate(size); - responseHeaderStruct.writeTo(buffer); - resp.writeTo(buffer); - buffer.flip(); + int throttleTime = 100; + ProduceResponse produceResponse = new ProduceResponse(new ProduceResponseData().setThrottleTimeMs(throttleTime)); + ByteBuffer buffer = produceResponse.serializeWithHeader(requestVersion, request.correlationId()); + buffer.rewind(); selector.completeReceive(new NetworkReceive(node.idString(), buffer)); client.poll(1, time.milliseconds()); @@ -603,23 +588,15 @@ private int sendEmptyProduceRequest(String nodeId) { return request.correlationId(); } - private void sendResponse(ResponseHeader respHeader, Struct response) { - Struct responseHeaderStruct = respHeader.toStruct(); - int size = responseHeaderStruct.sizeOf() + response.sizeOf(); - ByteBuffer buffer = ByteBuffer.allocate(size); - responseHeaderStruct.writeTo(buffer); - response.writeTo(buffer); - buffer.flip(); + private void sendResponse(AbstractResponse response, short version, int correlationId) { + ByteBuffer buffer = response.serializeWithHeader(version, correlationId); + buffer.rewind(); selector.completeReceive(new NetworkReceive(node.idString(), buffer)); } private void sendThrottledProduceResponse(int correlationId, int throttleMs, short version) { - Struct resp = new ProduceResponseData() - .setThrottleTimeMs(throttleMs) - .toStruct(version); - sendResponse(new ResponseHeader(correlationId, - PRODUCE.responseHeaderVersion(version)), - resp); + ProduceResponse response = new ProduceResponse(new ProduceResponseData().setThrottleTimeMs(throttleMs)); + sendResponse(response, version, correlationId); } @Test @@ -706,7 +683,7 @@ public void testAuthenticationFailureWithInFlightMetadataRequest() { RequestHeader header = parseHeader(requestBuffer); assertEquals(ApiKeys.METADATA, header.apiKey()); - ByteBuffer responseBuffer = metadataResponse.serialize(ApiKeys.METADATA, header.apiVersion(), header.correlationId()); + ByteBuffer responseBuffer = metadataResponse.serializeWithHeader(header.apiVersion(), header.correlationId()); selector.delayedReceive(new DelayedReceive(node1.idString(), new NetworkReceive(node1.idString(), responseBuffer))); int initialUpdateVersion = metadata.updateVersion(); diff --git a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java index 1d2dd4ae23997..071120e94015b 100644 --- a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java @@ -116,7 +116,7 @@ import org.apache.kafka.common.message.OffsetDeleteResponseData.OffsetDeleteResponseTopicCollection; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.protocol.Message; import org.apache.kafka.common.quota.ClientQuotaAlteration; import org.apache.kafka.common.quota.ClientQuotaEntity; import org.apache.kafka.common.quota.ClientQuotaFilter; @@ -458,11 +458,12 @@ private static MetadataResponse prepareMetadataResponse(Cluster cluster, Errors .setPartitions(pms); metadata.add(tm); } - return MetadataResponse.prepareResponse(0, - metadata, + return MetadataResponse.prepareResponse(true, + 0, cluster.nodes(), cluster.clusterResource().clusterId(), cluster.controller().id(), + metadata, MetadataResponse.AUTHORIZED_OPERATIONS_OMITTED); } @@ -560,7 +561,7 @@ public void testConnectionFailureOnMetadataUpdate() throws Exception { env.kafkaClient().setNodeApiVersions(NodeApiVersions.create()); env.kafkaClient().prepareResponse(request -> request instanceof MetadataRequest, null, true); env.kafkaClient().prepareResponse(request -> request instanceof MetadataRequest, - MetadataResponse.prepareResponse(discoveredCluster.nodes(), discoveredCluster.clusterResource().clusterId(), + TestUtils.metadataResponse(discoveredCluster.nodes(), discoveredCluster.clusterResource().clusterId(), 1, Collections.emptyList())); env.kafkaClient().prepareResponse(body -> body instanceof CreateTopicsRequest, prepareCreateTopicsResponse("myTopic", Errors.NONE)); @@ -585,7 +586,7 @@ public void testUnreachableBootstrapServer() throws Exception { Cluster discoveredCluster = mockCluster(3, 0); env.kafkaClient().setNodeApiVersions(NodeApiVersions.create()); env.kafkaClient().prepareResponse(body -> body instanceof MetadataRequest, - MetadataResponse.prepareResponse(discoveredCluster.nodes(), discoveredCluster.clusterResource().clusterId(), + TestUtils.metadataResponse(discoveredCluster.nodes(), discoveredCluster.clusterResource().clusterId(), 1, Collections.emptyList())); env.kafkaClient().prepareResponse(body -> body instanceof CreateTopicsRequest, prepareCreateTopicsResponse("myTopic", Errors.NONE)); @@ -702,7 +703,7 @@ public void testCreateTopicsHandleNotControllerException() throws Exception { env.kafkaClient().prepareResponseFrom( prepareCreateTopicsResponse("myTopic", Errors.NOT_CONTROLLER), env.cluster().nodeById(0)); - env.kafkaClient().prepareResponse(MetadataResponse.prepareResponse(env.cluster().nodes(), + env.kafkaClient().prepareResponse(TestUtils.metadataResponse(env.cluster().nodes(), env.cluster().clusterResource().clusterId(), 1, Collections.emptyList())); @@ -1046,7 +1047,7 @@ public void testMetadataRetries() throws Exception { env.kafkaClient().prepareResponse(null, true); // The next one succeeds and gives us the controller id - env.kafkaClient().prepareResponse(MetadataResponse.prepareResponse(initializedCluster.nodes(), + env.kafkaClient().prepareResponse(TestUtils.metadataResponse(initializedCluster.nodes(), initializedCluster.clusterResource().clusterId(), initializedCluster.controller().id(), Collections.emptyList())); @@ -1056,7 +1057,7 @@ public void testMetadataRetries() throws Exception { MetadataResponse.PartitionMetadata partitionMetadata = new MetadataResponse.PartitionMetadata( Errors.NONE, new TopicPartition(topic, 0), Optional.of(leader.id()), Optional.of(10), singletonList(leader.id()), singletonList(leader.id()), singletonList(leader.id())); - env.kafkaClient().prepareResponse(MetadataResponse.prepareResponse(initializedCluster.nodes(), + env.kafkaClient().prepareResponse(TestUtils.metadataResponse(initializedCluster.nodes(), initializedCluster.clusterResource().clusterId(), 1, singletonList(new MetadataResponse.TopicMetadata(Errors.NONE, topic, false, singletonList(partitionMetadata), MetadataResponse.AUTHORIZED_OPERATIONS_OMITTED)))); @@ -1175,17 +1176,18 @@ public void testDescribeAcls() throws Exception { // Test a call where we get back ACL1 and ACL2. env.kafkaClient().prepareResponse(new DescribeAclsResponse(new DescribeAclsResponseData() - .setResources(DescribeAclsResponse.aclsResources(asList(ACL1, ACL2))))); + .setResources(DescribeAclsResponse.aclsResources(asList(ACL1, ACL2))), ApiKeys.DESCRIBE_ACLS.latestVersion())); assertCollectionIs(env.adminClient().describeAcls(FILTER1).values().get(), ACL1, ACL2); // Test a call where we get back no results. - env.kafkaClient().prepareResponse(new DescribeAclsResponse(new DescribeAclsResponseData())); + env.kafkaClient().prepareResponse(new DescribeAclsResponse(new DescribeAclsResponseData(), + ApiKeys.DESCRIBE_ACLS.latestVersion())); assertTrue(env.adminClient().describeAcls(FILTER2).values().get().isEmpty()); // Test a call where we get back an error. env.kafkaClient().prepareResponse(new DescribeAclsResponse(new DescribeAclsResponseData() .setErrorCode(Errors.SECURITY_DISABLED.code()) - .setErrorMessage("Security is disabled"))); + .setErrorMessage("Security is disabled"), ApiKeys.DESCRIBE_ACLS.latestVersion())); TestUtils.assertFutureError(env.adminClient().describeAcls(FILTER2).values(), SecurityDisabledException.class); // Test a call where we supply an invalid filter. @@ -1238,7 +1240,8 @@ public void testDeleteAcls() throws Exception { DeleteAclsResponse.matchingAcl(ACL2, ApiError.NONE))), new DeleteAclsResponseData.DeleteAclsFilterResult() .setErrorCode(Errors.SECURITY_DISABLED.code()) - .setErrorMessage("No security"))))); + .setErrorMessage("No security"))), + ApiKeys.DELETE_ACLS.latestVersion())); DeleteAclsResult results = env.adminClient().deleteAcls(asList(FILTER1, FILTER2)); Map> filterResults = results.values(); FilterResults filter1Results = filterResults.get(FILTER1).get(); @@ -1258,8 +1261,13 @@ public void testDeleteAcls() throws Exception { DeleteAclsResponse.matchingAcl(ACL1, ApiError.NONE), new DeleteAclsResponseData.DeleteAclsMatchingAcl() .setErrorCode(Errors.SECURITY_DISABLED.code()) - .setErrorMessage("No security"))), - new DeleteAclsResponseData.DeleteAclsFilterResult())))); + .setErrorMessage("No security") + .setPermissionType(AclPermissionType.ALLOW.code()) + .setOperation(AclOperation.ALTER.code()) + .setResourceType(ResourceType.CLUSTER.code()) + .setPatternType(FILTER2.patternFilter().patternType().code()))), + new DeleteAclsResponseData.DeleteAclsFilterResult())), + ApiKeys.DELETE_ACLS.latestVersion())); results = env.adminClient().deleteAcls(asList(FILTER1, FILTER2)); assertTrue(results.values().get(FILTER2).get().values().isEmpty()); TestUtils.assertFutureError(results.all(), SecurityDisabledException.class); @@ -1271,7 +1279,8 @@ public void testDeleteAcls() throws Exception { new DeleteAclsResponseData.DeleteAclsFilterResult() .setMatchingAcls(asList(DeleteAclsResponse.matchingAcl(ACL1, ApiError.NONE))), new DeleteAclsResponseData.DeleteAclsFilterResult() - .setMatchingAcls(asList(DeleteAclsResponse.matchingAcl(ACL2, ApiError.NONE))))))); + .setMatchingAcls(asList(DeleteAclsResponse.matchingAcl(ACL2, ApiError.NONE))))), + ApiKeys.DELETE_ACLS.latestVersion())); results = env.adminClient().deleteAcls(asList(FILTER1, FILTER2)); Collection deleted = results.all().get(); assertCollectionIs(deleted, ACL1, ACL2); @@ -1307,7 +1316,8 @@ public void testElectLeaders() throws Exception { electionResults.add(electionResult); - env.kafkaClient().prepareResponse(new ElectLeadersResponse(0, Errors.NONE.code(), electionResults)); + env.kafkaClient().prepareResponse(new ElectLeadersResponse(0, Errors.NONE.code(), + electionResults, ApiKeys.ELECT_LEADERS.latestVersion())); ElectLeadersResult results = env.adminClient().electLeaders( electionType, new HashSet<>(asList(topic1, topic2))); @@ -1320,7 +1330,8 @@ public void testElectLeaders() throws Exception { partition2Result.setErrorCode(ApiError.NONE.error().code()); partition2Result.setErrorMessage(ApiError.NONE.message()); - env.kafkaClient().prepareResponse(new ElectLeadersResponse(0, Errors.NONE.code(), electionResults)); + env.kafkaClient().prepareResponse(new ElectLeadersResponse(0, Errors.NONE.code(), electionResults, + ApiKeys.ELECT_LEADERS.latestVersion())); results = env.adminClient().electLeaders(electionType, new HashSet<>(asList(topic1, topic2))); assertFalse(results.partitions().get().get(topic1).isPresent()); assertFalse(results.partitions().get().get(topic2).isPresent()); @@ -1848,7 +1859,7 @@ public void testDeleteRecordsTopicAuthorizationError() { topics.add(new MetadataResponse.TopicMetadata(Errors.TOPIC_AUTHORIZATION_FAILED, topic, false, Collections.emptyList())); - env.kafkaClient().prepareResponse(MetadataResponse.prepareResponse(env.cluster().nodes(), + env.kafkaClient().prepareResponse(TestUtils.metadataResponse(env.cluster().nodes(), env.cluster().clusterResource().clusterId(), env.cluster().controller().id(), topics)); Map recordsToDelete = new HashMap<>(); @@ -1881,7 +1892,7 @@ public void testDeleteRecordsMultipleSends() throws Exception { List topicMetadata = new ArrayList<>(); topicMetadata.add(new MetadataResponse.TopicMetadata(Errors.NONE, topic, false, partitionMetadata)); - env.kafkaClient().prepareResponse(MetadataResponse.prepareResponse(env.cluster().nodes(), + env.kafkaClient().prepareResponse(TestUtils.metadataResponse(env.cluster().nodes(), env.cluster().clusterResource().clusterId(), env.cluster().controller().id(), topicMetadata)); env.kafkaClient().prepareResponseFrom(new DeleteRecordsResponse(new DeleteRecordsResponseData().setTopics( @@ -1969,7 +1980,7 @@ public void testDeleteRecords() throws Exception { t.add(new MetadataResponse.TopicMetadata(Errors.NONE, "my_topic", false, p)); - env.kafkaClient().prepareResponse(MetadataResponse.prepareResponse(cluster.nodes(), cluster.clusterResource().clusterId(), cluster.controller().id(), t)); + env.kafkaClient().prepareResponse(TestUtils.metadataResponse(cluster.nodes(), cluster.clusterResource().clusterId(), cluster.controller().id(), t)); env.kafkaClient().prepareResponse(new DeleteRecordsResponse(m)); Map recordsToDelete = new HashMap<>(); @@ -2032,20 +2043,20 @@ public void testDescribeCluster() throws Exception { env.kafkaClient().setNodeApiVersions(NodeApiVersions.create()); // Prepare the metadata response used for the first describe cluster - MetadataResponse response = MetadataResponse.prepareResponse(0, - Collections.emptyList(), + MetadataResponse response = TestUtils.metadataResponse(0, env.cluster().nodes(), env.cluster().clusterResource().clusterId(), 2, + Collections.emptyList(), MetadataResponse.AUTHORIZED_OPERATIONS_OMITTED); env.kafkaClient().prepareResponse(response); // Prepare the metadata response used for the second describe cluster - MetadataResponse response2 = MetadataResponse.prepareResponse(0, - Collections.emptyList(), + MetadataResponse response2 = TestUtils.metadataResponse(0, env.cluster().nodes(), env.cluster().clusterResource().clusterId(), 3, + Collections.emptyList(), 1 << AclOperation.DESCRIBE.code() | 1 << AclOperation.ALTER.code()); env.kafkaClient().prepareResponse(response2); @@ -2072,14 +2083,14 @@ public void testListConsumerGroups() throws Exception { // Empty metadata response should be retried env.kafkaClient().prepareResponse( - MetadataResponse.prepareResponse( + TestUtils.metadataResponse( Collections.emptyList(), env.cluster().clusterResource().clusterId(), -1, Collections.emptyList())); env.kafkaClient().prepareResponse( - MetadataResponse.prepareResponse( + TestUtils.metadataResponse( env.cluster().nodes(), env.cluster().clusterResource().clusterId(), env.cluster().controller().id(), @@ -2185,7 +2196,7 @@ public void testListConsumerGroupsMetadataFailure() throws Exception { // Empty metadata causes the request to fail since we have no list of brokers // to send the ListGroups requests to env.kafkaClient().prepareResponse( - MetadataResponse.prepareResponse( + TestUtils.metadataResponse( Collections.emptyList(), env.cluster().clusterResource().clusterId(), -1, @@ -3446,7 +3457,7 @@ public void testAlterPartitionReassignments() throws Exception { .setName("B") .setPartitions(Collections.singletonList(normalPartitionResponse))) ); - MetadataResponse controllerNodeResponse = MetadataResponse.prepareResponse(env.cluster().nodes(), + MetadataResponse controllerNodeResponse = TestUtils.metadataResponse(env.cluster().nodes(), env.cluster().clusterResource().clusterId(), 1, Collections.emptyList()); AlterPartitionReassignmentsResponseData normalResponse = new AlterPartitionReassignmentsResponseData() @@ -3574,7 +3585,7 @@ public void testListPartitionReassignments() throws Exception { ListPartitionReassignmentsResponseData notControllerData = new ListPartitionReassignmentsResponseData() .setErrorCode(Errors.NOT_CONTROLLER.code()) .setErrorMessage(Errors.NOT_CONTROLLER.message()); - MetadataResponse controllerNodeResponse = MetadataResponse.prepareResponse(env.cluster().nodes(), + MetadataResponse controllerNodeResponse = TestUtils.metadataResponse(env.cluster().nodes(), env.cluster().clusterResource().clusterId(), 1, Collections.emptyList()); ListPartitionReassignmentsResponseData reassignmentsData = new ListPartitionReassignmentsResponseData() .setTopics(Arrays.asList(tp1Reassignment, tp2Reassignment)); @@ -4031,7 +4042,7 @@ public void testUpdateFeaturesHandleNotControllerException() throws Exception { 0), env.cluster().nodeById(0)); final int controllerId = 1; - env.kafkaClient().prepareResponse(MetadataResponse.prepareResponse(env.cluster().nodes(), + env.kafkaClient().prepareResponse(TestUtils.metadataResponse(env.cluster().nodes(), env.cluster().clusterResource().clusterId(), controllerId, Collections.emptyList())); @@ -4539,7 +4550,7 @@ public void testDescribeClientQuotas() throws Exception { responseData.put(entity1, Collections.singletonMap("consumer_byte_rate", 10000.0)); responseData.put(entity2, Collections.singletonMap("producer_byte_rate", 20000.0)); - env.kafkaClient().prepareResponse(new DescribeClientQuotasResponse(responseData, 0)); + env.kafkaClient().prepareResponse(DescribeClientQuotasResponse.fromQuotaEntities(responseData, 0)); ClientQuotaFilter filter = ClientQuotaFilter.contains(asList(ClientQuotaFilterComponent.ofEntity(ClientQuotaEntity.USER, value))); @@ -4593,7 +4604,7 @@ public void testAlterClientQuotas() throws Exception { responseData.put(unauthorizedEntity, new ApiError(Errors.CLUSTER_AUTHORIZATION_FAILED, "Authorization failed")); responseData.put(invalidEntity, new ApiError(Errors.INVALID_REQUEST, "Invalid quota entity")); - env.kafkaClient().prepareResponse(new AlterClientQuotasResponse(responseData, 0)); + env.kafkaClient().prepareResponse(AlterClientQuotasResponse.fromQuotaEntities(responseData, 0)); List entries = new ArrayList<>(3); entries.add(new ClientQuotaAlteration(goodEntity, Collections.singleton(new ClientQuotaAlteration.Op("consumer_byte_rate", 10000.0)))); @@ -4915,15 +4926,21 @@ public void testDescribeLogDirsPartialFailure() throws Exception { @Test public void testHasCoordinatorMoved() { Map errors = new HashMap<>(); - AbstractResponse response = new AbstractResponse() { + AbstractResponse response = new AbstractResponse(ApiKeys.OFFSET_COMMIT) { @Override public Map errorCounts() { return errors; } + @Override - protected Struct toStruct(short version) { + protected Message data() { return null; } + + @Override + public int throttleTimeMs() { + return DEFAULT_THROTTLE_TIME; + } }; assertFalse(ConsumerGroupOperationContext.hasCoordinatorMoved(response)); diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java index ab36ddf1a4b62..ad8fc03ac631d 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/KafkaConsumerTest.java @@ -2408,7 +2408,7 @@ public void testSubscriptionOnInvalidTopic() { List topicMetadata = new ArrayList<>(); topicMetadata.add(new MetadataResponse.TopicMetadata(Errors.INVALID_TOPIC_EXCEPTION, invalidTopicName, false, Collections.emptyList())); - MetadataResponse updateResponse = MetadataResponse.prepareResponse(cluster.nodes(), + MetadataResponse updateResponse = TestUtils.metadataResponse(cluster.nodes(), cluster.clusterResource().clusterId(), cluster.controller().id(), topicMetadata); diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java index ae2ab8a79b303..902401a77445f 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerCoordinatorTest.java @@ -1595,7 +1595,7 @@ private void testInternalTopicInclusion(boolean includeInternalTopics) { MetadataResponse.TopicMetadata topicMetadata = new MetadataResponse.TopicMetadata(Errors.NONE, Topic.GROUP_METADATA_TOPIC_NAME, true, singletonList(partitionMetadata)); - client.updateMetadata(MetadataResponse.prepareResponse(singletonList(node), "clusterId", node.id(), + client.updateMetadata(TestUtils.metadataResponse(singletonList(node), "clusterId", node.id(), singletonList(topicMetadata))); coordinator.maybeUpdateSubscriptionMetadata(); diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerMetadataTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerMetadataTest.java index 330f88e210ead..f71aa2f5956fe 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerMetadataTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/ConsumerMetadataTest.java @@ -75,7 +75,7 @@ private void testPatternSubscription(boolean includeInternalTopics) { topics.add(topicMetadata("__matching_topic", false)); topics.add(topicMetadata("non_matching_topic", false)); - MetadataResponse response = MetadataResponse.prepareResponse(singletonList(node), + MetadataResponse response = TestUtils.metadataResponse(singletonList(node), "clusterId", node.id(), topics); metadata.updateWithCurrentRequestVersion(response, false, time.milliseconds()); @@ -151,7 +151,7 @@ private void testBasicSubscription(Set expectedTopics, Set expec for (String expectedInternalTopic : expectedInternalTopics) topics.add(topicMetadata(expectedInternalTopic, true)); - MetadataResponse response = MetadataResponse.prepareResponse(singletonList(node), + MetadataResponse response = TestUtils.metadataResponse(singletonList(node), "clusterId", node.id(), topics); metadata.updateWithCurrentRequestVersion(response, false, time.milliseconds()); diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java index 4d31390e6792c..0ee5c76e2b22b 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java @@ -2031,7 +2031,7 @@ public void testGetTopicMetadataOfflinePartitions() { altTopics.add(alteredTopic); } Node controller = originalResponse.controller(); - MetadataResponse altered = MetadataResponse.prepareResponse( + MetadataResponse altered = TestUtils.metadataResponse( originalResponse.brokers(), originalResponse.clusterId(), controller != null ? controller.id() : MetadataResponse.NO_CONTROLLER_ID, @@ -2065,7 +2065,8 @@ public void testQuotaMetrics() { time, true, new ApiVersions(), throttleTimeSensor, new LogContext()); ByteBuffer buffer = ApiVersionsResponse.createApiVersionsResponse( - 400, RecordBatch.CURRENT_MAGIC_VALUE).serialize(ApiKeys.API_VERSIONS, ApiKeys.API_VERSIONS.latestVersion(), 0); + 400, RecordBatch.CURRENT_MAGIC_VALUE).serializeWithHeader(ApiKeys.API_VERSIONS.latestVersion(), 0); + selector.delayedReceive(new DelayedReceive(node.idString(), new NetworkReceive(node.idString(), buffer))); while (!client.ready(node, time.milliseconds())) { client.poll(1, time.milliseconds()); @@ -2082,9 +2083,7 @@ public void testQuotaMetrics() { client.send(request, time.milliseconds()); client.poll(1, time.milliseconds()); FetchResponse response = fullFetchResponse(tp0, nextRecords, Errors.NONE, i, throttleTimeMs); - buffer = response.serialize(ApiKeys.FETCH, - ApiKeys.FETCH.latestVersion(), - request.correlationId()); + buffer = response.serializeWithHeader(ApiKeys.FETCH.latestVersion(), request.correlationId()); selector.completeReceive(new NetworkReceive(node.idString(), buffer)); client.poll(1, time.milliseconds()); // If a throttled response is received, advance the time to ensure progress. @@ -4507,7 +4506,7 @@ private MetadataResponse newMetadataResponse(String topic, Errors error) { MetadataResponse.TopicMetadata topicMetadata = new MetadataResponse.TopicMetadata(error, topic, false, partitionsMetadata); List brokers = new ArrayList<>(initialUpdateResponse.brokers()); - return MetadataResponse.prepareResponse(brokers, initialUpdateResponse.clusterId(), + return TestUtils.metadataResponse(brokers, initialUpdateResponse.clusterId(), initialUpdateResponse.controller().id(), Collections.singletonList(topicMetadata)); } diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/KafkaProducerTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/KafkaProducerTest.java index 8153c2f7b4acf..96901c5c26b38 100644 --- a/clients/src/test/java/org/apache/kafka/clients/producer/KafkaProducerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/producer/KafkaProducerTest.java @@ -1086,7 +1086,7 @@ public void testSendToInvalidTopic() throws Exception { List topicMetadata = new ArrayList<>(); topicMetadata.add(new MetadataResponse.TopicMetadata(Errors.INVALID_TOPIC_EXCEPTION, invalidTopicName, false, Collections.emptyList())); - MetadataResponse updateResponse = MetadataResponse.prepareResponse( + MetadataResponse updateResponse = TestUtils.metadataResponse( new ArrayList<>(initialUpdateResponse.brokers()), initialUpdateResponse.clusterId(), initialUpdateResponse.controller().id(), diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java index a0c8d5639aafc..6ded3f72de291 100644 --- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java @@ -160,7 +160,7 @@ public void tearDown() { private static Map partitionRecords(ProduceRequest request) { Map partitionRecords = new HashMap<>(); - request.dataOrException().topicData().forEach(tpData -> tpData.partitionData().forEach(p -> { + request.data().topicData().forEach(tpData -> tpData.partitionData().forEach(p -> { TopicPartition tp = new TopicPartition(tpData.name(), p.index()); partitionRecords.put(tp, (MemoryRecords) p.records()); })); @@ -285,8 +285,9 @@ public void testQuotaMetrics() { 1000, 1000, 64 * 1024, 64 * 1024, 1000, 10 * 1000, 127 * 1000, ClientDnsLookup.USE_ALL_DNS_IPS, time, true, new ApiVersions(), throttleTimeSensor, logContext); - ByteBuffer buffer = ApiVersionsResponse.createApiVersionsResponse( - 400, RecordBatch.CURRENT_MAGIC_VALUE).serialize(ApiKeys.API_VERSIONS, ApiKeys.API_VERSIONS.latestVersion(), 0); + ByteBuffer buffer = ApiVersionsResponse.createApiVersionsResponse(400, RecordBatch.CURRENT_MAGIC_VALUE). + serializeWithHeader(ApiKeys.API_VERSIONS.latestVersion(), 0); + selector.delayedReceive(new DelayedReceive(node.idString(), new NetworkReceive(node.idString(), buffer))); while (!client.ready(node, time.milliseconds())) { client.poll(1, time.milliseconds()); @@ -305,8 +306,7 @@ public void testQuotaMetrics() { client.send(request, time.milliseconds()); client.poll(1, time.milliseconds()); ProduceResponse response = produceResponse(tp0, i, Errors.NONE, throttleTimeMs); - buffer = response. - serialize(ApiKeys.PRODUCE, ApiKeys.PRODUCE.latestVersion(), request.correlationId()); + buffer = response.serializeWithHeader(ApiKeys.PRODUCE.latestVersion(), request.correlationId()); selector.completeReceive(new NetworkReceive(node.idString(), buffer)); client.poll(1, time.milliseconds()); // If a throttled response is received, advance the time to ensure progress. diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java index cd71aabb3dfd0..70b52c5df05c2 100644 --- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java @@ -3356,7 +3356,7 @@ private void prepareProduceResponse(Errors error, final long producerId, final s private MockClient.RequestMatcher produceRequestMatcher(final long producerId, final short epoch, TopicPartition tp) { return body -> { ProduceRequest produceRequest = (ProduceRequest) body; - MemoryRecords records = produceRequest.dataOrException().topicData() + MemoryRecords records = produceRequest.data().topicData() .stream() .filter(t -> t.name().equals(tp.topic())) .findAny() diff --git a/clients/src/test/java/org/apache/kafka/common/network/NioEchoServer.java b/clients/src/test/java/org/apache/kafka/common/network/NioEchoServer.java index b66d66baba628..1d6894b4dacd0 100644 --- a/clients/src/test/java/org/apache/kafka/common/network/NioEchoServer.java +++ b/clients/src/test/java/org/apache/kafka/common/network/NioEchoServer.java @@ -91,7 +91,7 @@ public NioEchoServer(ListenerName listenerName, SecurityProtocol securityProtoco public NioEchoServer(ListenerName listenerName, SecurityProtocol securityProtocol, AbstractConfig config, String serverHost, ChannelBuilder channelBuilder, CredentialCache credentialCache, int failedAuthenticationDelayMs, Time time) throws Exception { - this(listenerName, securityProtocol, config, serverHost, channelBuilder, credentialCache, 100, time, + this(listenerName, securityProtocol, config, serverHost, channelBuilder, credentialCache, failedAuthenticationDelayMs, time, new DelegationTokenCache(ScramMechanism.mechanismNames())); } diff --git a/clients/src/test/java/org/apache/kafka/common/protocol/MessageTestUtil.java b/clients/src/test/java/org/apache/kafka/common/protocol/MessageTestUtil.java index 945246d9b7538..a093148b891aa 100644 --- a/clients/src/test/java/org/apache/kafka/common/protocol/MessageTestUtil.java +++ b/clients/src/test/java/org/apache/kafka/common/protocol/MessageTestUtil.java @@ -20,16 +20,12 @@ import java.nio.ByteBuffer; public final class MessageTestUtil { - public static int messageSize(Message message, short version) { - return message.size(new ObjectSerializationCache(), version); - } - public static ByteBuffer messageToByteBuffer(Message message, short version) { ObjectSerializationCache cache = new ObjectSerializationCache(); int size = message.size(cache, version); ByteBuffer bytes = ByteBuffer.allocate(size); message.write(new ByteBufferAccessor(bytes), cache, version); - bytes.flip(); + bytes.rewind(); return bytes; } } diff --git a/clients/src/test/java/org/apache/kafka/common/requests/AddPartitionsToTxnResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/AddPartitionsToTxnResponseTest.java index 7d49fb5509f04..92c38e557b93d 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/AddPartitionsToTxnResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/AddPartitionsToTxnResponseTest.java @@ -67,7 +67,7 @@ public void testConstructorWithErrorResponse() { } @Test - public void testConstructorWithStruct() { + public void testParse() { AddPartitionsToTxnTopicResultCollection topicCollection = new AddPartitionsToTxnTopicResultCollection(); @@ -87,14 +87,13 @@ public void testConstructorWithStruct() { AddPartitionsToTxnResponseData data = new AddPartitionsToTxnResponseData() .setResults(topicCollection) .setThrottleTimeMs(throttleTimeMs); + AddPartitionsToTxnResponse response = new AddPartitionsToTxnResponse(data); for (short version = 0; version <= ApiKeys.ADD_PARTITIONS_TO_TXN.latestVersion(); version++) { - AddPartitionsToTxnResponse response = new AddPartitionsToTxnResponse(data.toStruct(version), version); - assertEquals(expectedErrorCounts, response.errorCounts()); - - assertEquals(throttleTimeMs, response.throttleTimeMs()); - - assertEquals(version >= 1, response.shouldClientThrottle(version)); + AddPartitionsToTxnResponse parsedResponse = AddPartitionsToTxnResponse.parse(response.serializeBody(version), version); + assertEquals(expectedErrorCounts, parsedResponse.errorCounts()); + assertEquals(throttleTimeMs, parsedResponse.throttleTimeMs()); + assertEquals(version >= 1, parsedResponse.shouldClientThrottle(version)); } } } diff --git a/clients/src/test/java/org/apache/kafka/common/requests/CreateAclsRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/CreateAclsRequestTest.java index 06b4b9f4bb2b2..02a24ba54eaba 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/CreateAclsRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/CreateAclsRequestTest.java @@ -23,12 +23,12 @@ import org.apache.kafka.common.acl.AclPermissionType; import org.apache.kafka.common.errors.UnsupportedVersionException; import org.apache.kafka.common.message.CreateAclsRequestData; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.resource.PatternType; import org.apache.kafka.common.resource.ResourcePattern; import org.apache.kafka.common.resource.ResourceType; import org.junit.Test; +import java.nio.ByteBuffer; import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; @@ -53,30 +53,30 @@ public class CreateAclsRequestTest { @Test(expected = UnsupportedVersionException.class) public void shouldThrowOnV0IfNotLiteral() { - new CreateAclsRequest(V0, data(PREFIXED_ACL1)); + new CreateAclsRequest(data(PREFIXED_ACL1), V0); } @Test(expected = IllegalArgumentException.class) public void shouldThrowOnIfUnknown() { - new CreateAclsRequest(V0, data(UNKNOWN_ACL1)); + new CreateAclsRequest(data(UNKNOWN_ACL1), V0); } @Test public void shouldRoundTripV0() { - final CreateAclsRequest original = new CreateAclsRequest(V0, data(LITERAL_ACL1, LITERAL_ACL2)); - final Struct struct = original.toStruct(); + final CreateAclsRequest original = new CreateAclsRequest(data(LITERAL_ACL1, LITERAL_ACL2), V0); + final ByteBuffer buffer = original.serializeBody(); - final CreateAclsRequest result = new CreateAclsRequest(struct, V0); + final CreateAclsRequest result = CreateAclsRequest.parse(buffer, V0); assertRequestEquals(original, result); } @Test public void shouldRoundTripV1() { - final CreateAclsRequest original = new CreateAclsRequest(V1, data(LITERAL_ACL1, PREFIXED_ACL1)); - final Struct struct = original.toStruct(); + final CreateAclsRequest original = new CreateAclsRequest(data(LITERAL_ACL1, PREFIXED_ACL1), V1); + final ByteBuffer buffer = original.serializeBody(); - final CreateAclsRequest result = new CreateAclsRequest(struct, V1); + final CreateAclsRequest result = CreateAclsRequest.parse(buffer, V1); assertRequestEquals(original, result); } diff --git a/clients/src/test/java/org/apache/kafka/common/requests/DeleteAclsRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/DeleteAclsRequestTest.java index 19b65ed088e1c..9d2e741fbe0bd 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/DeleteAclsRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/DeleteAclsRequestTest.java @@ -23,12 +23,12 @@ import org.apache.kafka.common.acl.AclPermissionType; import org.apache.kafka.common.errors.UnsupportedVersionException; import org.apache.kafka.common.message.DeleteAclsRequestData; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.resource.PatternType; import org.apache.kafka.common.resource.ResourcePatternFilter; import org.apache.kafka.common.resource.ResourceType; import org.junit.Test; +import java.nio.ByteBuffer; import java.util.stream.Collectors; import static java.util.Arrays.asList; @@ -64,9 +64,9 @@ public void shouldThrowOnUnknownElements() { @Test public void shouldRoundTripLiteralV0() { final DeleteAclsRequest original = new DeleteAclsRequest.Builder(requestData(LITERAL_FILTER)).build(V0); - final Struct struct = original.toStruct(); + final ByteBuffer buffer = original.serializeBody(); - final DeleteAclsRequest result = new DeleteAclsRequest(struct, V0); + final DeleteAclsRequest result = DeleteAclsRequest.parse(buffer, V0); assertRequestEquals(original, result); } @@ -82,7 +82,7 @@ public void shouldRoundTripAnyV0AsLiteral() { ANY_FILTER.entryFilter())) ).build(V0); - final DeleteAclsRequest result = new DeleteAclsRequest(original.toStruct(), V0); + final DeleteAclsRequest result = DeleteAclsRequest.parse(original.serializeBody(), V0); assertRequestEquals(expected, result); } @@ -92,9 +92,9 @@ public void shouldRoundTripV1() { final DeleteAclsRequest original = new DeleteAclsRequest.Builder( requestData(LITERAL_FILTER, PREFIXED_FILTER, ANY_FILTER) ).build(V1); - final Struct struct = original.toStruct(); + final ByteBuffer buffer = original.serializeBody(); - final DeleteAclsRequest result = new DeleteAclsRequest(struct, V1); + final DeleteAclsRequest result = DeleteAclsRequest.parse(buffer, V1); assertRequestEquals(original, result); } diff --git a/clients/src/test/java/org/apache/kafka/common/requests/DeleteAclsResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/DeleteAclsResponseTest.java index aaebc0c91a005..96f04ebf29e61 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/DeleteAclsResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/DeleteAclsResponseTest.java @@ -23,14 +23,16 @@ import org.apache.kafka.common.message.DeleteAclsResponseData; import org.apache.kafka.common.message.DeleteAclsResponseData.DeleteAclsFilterResult; import org.apache.kafka.common.message.DeleteAclsResponseData.DeleteAclsMatchingAcl; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.resource.PatternType; import org.apache.kafka.common.resource.ResourceType; import org.junit.Test; +import java.nio.ByteBuffer; + import static java.util.Arrays.asList; import static java.util.Collections.singletonList; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; public class DeleteAclsResponseTest { private static final short V0 = 0; @@ -81,41 +83,47 @@ public class DeleteAclsResponseTest { private static final DeleteAclsFilterResult UNKNOWN_RESPONSE = new DeleteAclsFilterResult().setMatchingAcls(asList( UNKNOWN_ACL)); - @Test(expected = UnsupportedVersionException.class) + @Test public void shouldThrowOnV0IfNotLiteral() { - new DeleteAclsResponse(new DeleteAclsResponseData() - .setThrottleTimeMs(10) - .setFilterResults(singletonList(PREFIXED_RESPONSE)) - ).toStruct(V0); + assertThrows(UnsupportedVersionException.class, () -> new DeleteAclsResponse( + new DeleteAclsResponseData() + .setThrottleTimeMs(10) + .setFilterResults(singletonList(PREFIXED_RESPONSE)), + V0)); } - @Test(expected = IllegalArgumentException.class) + @Test public void shouldThrowOnIfUnknown() { - new DeleteAclsResponse(new DeleteAclsResponseData() - .setThrottleTimeMs(10) - .setFilterResults(singletonList(UNKNOWN_RESPONSE)) - ).toStruct(V1); + assertThrows(IllegalArgumentException.class, () -> new DeleteAclsResponse( + new DeleteAclsResponseData() + .setThrottleTimeMs(10) + .setFilterResults(singletonList(UNKNOWN_RESPONSE)), + V1)); } @Test public void shouldRoundTripV0() { - final DeleteAclsResponse original = new DeleteAclsResponse(new DeleteAclsResponseData() - .setThrottleTimeMs(10) - .setFilterResults(singletonList(LITERAL_RESPONSE))); - final Struct struct = original.toStruct(V0); - - final DeleteAclsResponse result = new DeleteAclsResponse(struct, V0); + final DeleteAclsResponse original = new DeleteAclsResponse( + new DeleteAclsResponseData() + .setThrottleTimeMs(10) + .setFilterResults(singletonList(LITERAL_RESPONSE)), + V0); + final ByteBuffer buffer = original.serializeBody(V0); + + final DeleteAclsResponse result = DeleteAclsResponse.parse(buffer, V0); assertEquals(original.filterResults(), result.filterResults()); } @Test public void shouldRoundTripV1() { - final DeleteAclsResponse original = new DeleteAclsResponse(new DeleteAclsResponseData() - .setThrottleTimeMs(10) - .setFilterResults(asList(LITERAL_RESPONSE, PREFIXED_RESPONSE))); - final Struct struct = original.toStruct(V1); - - final DeleteAclsResponse result = new DeleteAclsResponse(struct, V1); + final DeleteAclsResponse original = new DeleteAclsResponse( + new DeleteAclsResponseData() + .setThrottleTimeMs(10) + .setFilterResults(asList(LITERAL_RESPONSE, PREFIXED_RESPONSE)), + V1); + final ByteBuffer buffer = original.serializeBody(V1); + + final DeleteAclsResponse result = DeleteAclsResponse.parse(buffer, V1); assertEquals(original.filterResults(), result.filterResults()); } diff --git a/clients/src/test/java/org/apache/kafka/common/requests/DescribeAclsRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/DescribeAclsRequestTest.java index e9203aebe2e64..e346ba6c395de 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/DescribeAclsRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/DescribeAclsRequestTest.java @@ -22,7 +22,6 @@ import org.apache.kafka.common.acl.AclOperation; import org.apache.kafka.common.acl.AclPermissionType; import org.apache.kafka.common.errors.UnsupportedVersionException; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.resource.PatternType; import org.apache.kafka.common.resource.ResourcePatternFilter; import org.apache.kafka.common.resource.ResourceType; @@ -60,9 +59,7 @@ public void shouldThrowIfUnknown() { @Test public void shouldRoundTripLiteralV0() { final DescribeAclsRequest original = new DescribeAclsRequest.Builder(LITERAL_FILTER).build(V0); - final Struct struct = original.toStruct(); - - final DescribeAclsRequest result = new DescribeAclsRequest(struct, V0); + final DescribeAclsRequest result = DescribeAclsRequest.parse(original.serializeBody(), V0); assertRequestEquals(original, result); } @@ -77,39 +74,28 @@ public void shouldRoundTripAnyV0AsLiteral() { PatternType.LITERAL), ANY_FILTER.entryFilter())).build(V0); - final Struct struct = original.toStruct(); - final DescribeAclsRequest result = new DescribeAclsRequest(struct, V0); - + final DescribeAclsRequest result = DescribeAclsRequest.parse(original.serializeBody(), V0); assertRequestEquals(expected, result); } @Test public void shouldRoundTripLiteralV1() { final DescribeAclsRequest original = new DescribeAclsRequest.Builder(LITERAL_FILTER).build(V1); - final Struct struct = original.toStruct(); - - final DescribeAclsRequest result = new DescribeAclsRequest(struct, V1); - + final DescribeAclsRequest result = DescribeAclsRequest.parse(original.serializeBody(), V1); assertRequestEquals(original, result); } @Test public void shouldRoundTripPrefixedV1() { final DescribeAclsRequest original = new DescribeAclsRequest.Builder(PREFIXED_FILTER).build(V1); - final Struct struct = original.toStruct(); - - final DescribeAclsRequest result = new DescribeAclsRequest(struct, V1); - + final DescribeAclsRequest result = DescribeAclsRequest.parse(original.serializeBody(), V1); assertRequestEquals(original, result); } @Test public void shouldRoundTripAnyV1() { final DescribeAclsRequest original = new DescribeAclsRequest.Builder(ANY_FILTER).build(V1); - final Struct struct = original.toStruct(); - - final DescribeAclsRequest result = new DescribeAclsRequest(struct, V1); - + final DescribeAclsRequest result = DescribeAclsRequest.parse(original.serializeBody(), V1); assertRequestEquals(original, result); } diff --git a/clients/src/test/java/org/apache/kafka/common/requests/DescribeAclsResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/DescribeAclsResponseTest.java index 84500650d6cf6..56bfe8557ddfe 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/DescribeAclsResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/DescribeAclsResponseTest.java @@ -26,12 +26,12 @@ import org.apache.kafka.common.message.DescribeAclsResponseData.AclDescription; import org.apache.kafka.common.message.DescribeAclsResponseData.DescribeAclsResource; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.resource.PatternType; import org.apache.kafka.common.resource.ResourcePattern; import org.apache.kafka.common.resource.ResourceType; import org.junit.Test; +import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collections; import java.util.HashSet; @@ -82,21 +82,21 @@ public class DescribeAclsResponseTest { @Test(expected = UnsupportedVersionException.class) public void shouldThrowOnV0IfNotLiteral() { - buildResponse(10, Errors.NONE, Collections.singletonList(PREFIXED_ACL1)).toStruct(V0); + buildResponse(10, Errors.NONE, Collections.singletonList(PREFIXED_ACL1)).serializeBody(V0); } @Test(expected = IllegalArgumentException.class) public void shouldThrowIfUnknown() { - buildResponse(10, Errors.NONE, Collections.singletonList(UNKNOWN_ACL)).toStruct(V0); + buildResponse(10, Errors.NONE, Collections.singletonList(UNKNOWN_ACL)).serializeBody(V0); } @Test public void shouldRoundTripV0() { List resources = Arrays.asList(LITERAL_ACL1, LITERAL_ACL2); final DescribeAclsResponse original = buildResponse(10, Errors.NONE, resources); - final Struct struct = original.toStruct(V0); + final ByteBuffer buffer = original.serializeBody(V0); - final DescribeAclsResponse result = new DescribeAclsResponse(struct, V0); + final DescribeAclsResponse result = DescribeAclsResponse.parse(buffer, V0); assertResponseEquals(original, result); final DescribeAclsResponse result2 = buildResponse(10, Errors.NONE, DescribeAclsResponse.aclsResources( @@ -108,9 +108,9 @@ public void shouldRoundTripV0() { public void shouldRoundTripV1() { List resources = Arrays.asList(LITERAL_ACL1, PREFIXED_ACL1); final DescribeAclsResponse original = buildResponse(100, Errors.NONE, resources); - final Struct struct = original.toStruct(V1); + final ByteBuffer buffer = original.serializeBody(V1); - final DescribeAclsResponse result = new DescribeAclsResponse(struct, V1); + final DescribeAclsResponse result = DescribeAclsResponse.parse(buffer, V1); assertResponseEquals(original, result); final DescribeAclsResponse result2 = buildResponse(100, Errors.NONE, DescribeAclsResponse.aclsResources( diff --git a/clients/src/test/java/org/apache/kafka/common/requests/EndTxnResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/EndTxnResponseTest.java index f7b5850916c23..c1d4367c929e0 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/EndTxnResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/EndTxnResponseTest.java @@ -29,21 +29,24 @@ public class EndTxnResponseTest { @Test - public void testConstructorWithStruct() { + public void testConstructor() { int throttleTimeMs = 10; EndTxnResponseData data = new EndTxnResponseData() - .setErrorCode(Errors.NOT_COORDINATOR.code()) - .setThrottleTimeMs(throttleTimeMs); + .setErrorCode(Errors.NOT_COORDINATOR.code()) + .setThrottleTimeMs(throttleTimeMs); Map expectedErrorCounts = Collections.singletonMap(Errors.NOT_COORDINATOR, 1); for (short version = 0; version <= ApiKeys.END_TXN.latestVersion(); version++) { - EndTxnResponse response = new EndTxnResponse(data.toStruct(version), version); + EndTxnResponse response = new EndTxnResponse(data); assertEquals(expectedErrorCounts, response.errorCounts()); - assertEquals(throttleTimeMs, response.throttleTimeMs()); + assertEquals(version >= 1, response.shouldClientThrottle(version)); + response = EndTxnResponse.parse(response.serializeBody(version), version); + assertEquals(expectedErrorCounts, response.errorCounts()); + assertEquals(throttleTimeMs, response.throttleTimeMs()); assertEquals(version >= 1, response.shouldClientThrottle(version)); } } diff --git a/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrRequestTest.java index 2235a8f643cf8..a8c0bd5e778cb 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/LeaderAndIsrRequestTest.java @@ -25,7 +25,6 @@ import org.apache.kafka.common.message.LeaderAndIsrRequestData.LeaderAndIsrPartitionState; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.MessageTestUtil; import org.apache.kafka.test.TestUtils; import org.junit.Test; @@ -129,7 +128,7 @@ public void testVersionLogic() { assertEquals(2, request.controllerEpoch()); assertEquals(3, request.brokerEpoch()); - ByteBuffer byteBuffer = MessageTestUtil.messageToByteBuffer(request.data(), request.version()); + ByteBuffer byteBuffer = request.serializeBody(); LeaderAndIsrRequest deserializedRequest = new LeaderAndIsrRequest(new LeaderAndIsrRequestData( new ByteBufferAccessor(byteBuffer), version), version); @@ -163,10 +162,7 @@ public void testTopicPartitionGroupingSizeReduction() { LeaderAndIsrRequest v2 = builder.build((short) 2); LeaderAndIsrRequest v1 = builder.build((short) 1); - int size2 = MessageTestUtil.messageSize(v2.data(), v2.version()); - int size1 = MessageTestUtil.messageSize(v1.data(), v1.version()); - - assertTrue("Expected v2 < v1: v2=" + size2 + ", v1=" + size1, size2 < size1); + assertTrue("Expected v2 < v1: v2=" + v2.sizeInBytes() + ", v1=" + v1.sizeInBytes(), v2.sizeInBytes() < v1.sizeInBytes()); } private Set iterableToSet(Iterable iterable) { diff --git a/clients/src/test/java/org/apache/kafka/common/requests/LeaveGroupResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/LeaveGroupResponseTest.java index a620ed0d08230..99c863627b9f2 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/LeaveGroupResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/LeaveGroupResponseTest.java @@ -20,9 +20,11 @@ import org.apache.kafka.common.message.LeaveGroupResponseData.MemberResponse; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.MessageTestUtil; import org.junit.Before; import org.junit.Test; +import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -58,29 +60,6 @@ public void setUp() { ); } - @Test - public void testConstructorWithStruct() { - Map expectedErrorCounts = Collections.singletonMap(Errors.NOT_COORDINATOR, 1); - - LeaveGroupResponseData responseData = new LeaveGroupResponseData() - .setErrorCode(Errors.NOT_COORDINATOR.code()) - .setThrottleTimeMs(throttleTimeMs); - for (short version = 0; version <= ApiKeys.LEAVE_GROUP.latestVersion(); version++) { - LeaveGroupResponse leaveGroupResponse = new LeaveGroupResponse(responseData.toStruct(version), version); - - assertEquals(expectedErrorCounts, leaveGroupResponse.errorCounts()); - - if (version >= 1) { - assertEquals(throttleTimeMs, leaveGroupResponse.throttleTimeMs()); - } else { - assertEquals(DEFAULT_THROTTLE_TIME, leaveGroupResponse.throttleTimeMs()); - } - - assertEquals(Errors.NOT_COORDINATOR, leaveGroupResponse.error()); - } - } - - @Test public void testConstructorWithMemberResponses() { Map expectedErrorCounts = new HashMap<>(); @@ -126,19 +105,42 @@ public void testShouldThrottle() { } @Test - public void testEqualityWithStruct() { + public void testEqualityWithSerialization() { LeaveGroupResponseData responseData = new LeaveGroupResponseData() - .setErrorCode(Errors.NONE.code()) - .setThrottleTimeMs(throttleTimeMs); + .setErrorCode(Errors.NONE.code()) + .setThrottleTimeMs(throttleTimeMs); for (short version = 0; version <= ApiKeys.LEAVE_GROUP.latestVersion(); version++) { - LeaveGroupResponse primaryResponse = new LeaveGroupResponse(responseData.toStruct(version), version); - - LeaveGroupResponse secondaryResponse = new LeaveGroupResponse(responseData.toStruct(version), version); + LeaveGroupResponse primaryResponse = LeaveGroupResponse.parse( + MessageTestUtil.messageToByteBuffer(responseData, version), version); + LeaveGroupResponse secondaryResponse = LeaveGroupResponse.parse( + MessageTestUtil.messageToByteBuffer(responseData, version), version); assertEquals(primaryResponse, primaryResponse); assertEquals(primaryResponse, secondaryResponse); assertEquals(primaryResponse.hashCode(), secondaryResponse.hashCode()); + } + } + + @Test + public void testParse() { + Map expectedErrorCounts = Collections.singletonMap(Errors.NOT_COORDINATOR, 1); + LeaveGroupResponseData data = new LeaveGroupResponseData() + .setErrorCode(Errors.NOT_COORDINATOR.code()) + .setThrottleTimeMs(throttleTimeMs); + + for (short version = 0; version <= ApiKeys.LEAVE_GROUP.latestVersion(); version++) { + ByteBuffer buffer = MessageTestUtil.messageToByteBuffer(data, version); + LeaveGroupResponse leaveGroupResponse = LeaveGroupResponse.parse(buffer, version); + assertEquals(expectedErrorCounts, leaveGroupResponse.errorCounts()); + + if (version >= 1) { + assertEquals(throttleTimeMs, leaveGroupResponse.throttleTimeMs()); + } else { + assertEquals(DEFAULT_THROTTLE_TIME, leaveGroupResponse.throttleTimeMs()); + } + + assertEquals(Errors.NOT_COORDINATOR, leaveGroupResponse.error()); } } diff --git a/clients/src/test/java/org/apache/kafka/common/requests/ListOffsetRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/ListOffsetRequestTest.java index 9d7c2aa7cfd42..22e7c5736286e 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/ListOffsetRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/ListOffsetRequestTest.java @@ -35,6 +35,7 @@ import org.apache.kafka.common.message.ListOffsetResponseData.ListOffsetTopicResponse; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.MessageTestUtil; import org.junit.Test; public class ListOffsetRequestTest { @@ -52,7 +53,7 @@ public void testDuplicatePartitions() { ListOffsetRequestData data = new ListOffsetRequestData() .setTopics(topics) .setReplicaId(-1); - ListOffsetRequest request = new ListOffsetRequest(data.toStruct((short) 0), (short) 0); + ListOffsetRequest request = ListOffsetRequest.parse(MessageTestUtil.messageToByteBuffer(data, (short) 0), (short) 0); assertEquals(Collections.singleton(new TopicPartition("topic", 0)), request.duplicatePartitions()); } diff --git a/clients/src/test/java/org/apache/kafka/common/requests/OffsetCommitResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/OffsetCommitResponseTest.java index 62229168b2405..5e3a85739e50d 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/OffsetCommitResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/OffsetCommitResponseTest.java @@ -22,9 +22,11 @@ import org.apache.kafka.common.message.OffsetCommitResponseData.OffsetCommitResponseTopic; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.MessageTestUtil; import org.junit.Before; import org.junit.Test; +import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -69,22 +71,23 @@ public void testConstructorWithErrorResponse() { } @Test - public void testConstructorWithStruct() { + public void testParse() { OffsetCommitResponseData data = new OffsetCommitResponseData() - .setTopics(Arrays.asList( - new OffsetCommitResponseTopic().setPartitions( - Collections.singletonList(new OffsetCommitResponsePartition() - .setPartitionIndex(partitionOne) - .setErrorCode(errorOne.code()))), - new OffsetCommitResponseTopic().setPartitions( - Collections.singletonList(new OffsetCommitResponsePartition() - .setPartitionIndex(partitionTwo) - .setErrorCode(errorTwo.code()))) - )) - .setThrottleTimeMs(throttleTimeMs); + .setTopics(Arrays.asList( + new OffsetCommitResponseTopic().setPartitions( + Collections.singletonList(new OffsetCommitResponsePartition() + .setPartitionIndex(partitionOne) + .setErrorCode(errorOne.code()))), + new OffsetCommitResponseTopic().setPartitions( + Collections.singletonList(new OffsetCommitResponsePartition() + .setPartitionIndex(partitionTwo) + .setErrorCode(errorTwo.code()))) + )) + .setThrottleTimeMs(throttleTimeMs); for (short version = 0; version <= ApiKeys.OFFSET_COMMIT.latestVersion(); version++) { - OffsetCommitResponse response = new OffsetCommitResponse(data.toStruct(version), version); + ByteBuffer buffer = MessageTestUtil.messageToByteBuffer(data, version); + OffsetCommitResponse response = OffsetCommitResponse.parse(buffer, version); assertEquals(expectedErrorCounts, response.errorCounts()); if (version >= 3) { @@ -96,4 +99,5 @@ public void testConstructorWithStruct() { assertEquals(version >= 4, response.shouldClientThrottle(version)); } } + } diff --git a/clients/src/test/java/org/apache/kafka/common/requests/OffsetFetchResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/OffsetFetchResponseTest.java index 2bfa8b594525d..f20a74412880a 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/OffsetFetchResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/OffsetFetchResponseTest.java @@ -108,7 +108,7 @@ public void testStructBuild() { for (short version = 0; version <= ApiKeys.OFFSET_FETCH.latestVersion(); version++) { Struct struct = latestResponse.data.toStruct(version); - OffsetFetchResponse oldResponse = new OffsetFetchResponse(struct, version); + OffsetFetchResponse oldResponse = OffsetFetchResponse.parse(latestResponse.serializeBody(version), version); if (version <= 1) { assertFalse(struct.hasField(ERROR_CODE)); @@ -148,9 +148,7 @@ public void testStructBuild() { Map responseData = oldResponse.responseData(); assertEquals(expectedDataMap, responseData); - responseData.forEach( - (tp, data) -> assertTrue(data.hasError()) - ); + responseData.forEach((tp, data) -> assertTrue(data.hasError())); } } diff --git a/clients/src/test/java/org/apache/kafka/common/requests/OffsetsForLeaderEpochRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/OffsetsForLeaderEpochRequestTest.java index 37460cc3a5bae..f11a4837747ce 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/OffsetsForLeaderEpochRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/OffsetsForLeaderEpochRequestTest.java @@ -49,8 +49,7 @@ public void testDefaultReplicaId() { OffsetsForLeaderEpochRequest.Builder builder = OffsetsForLeaderEpochRequest.Builder.forFollower( version, Collections.emptyMap(), replicaId); OffsetsForLeaderEpochRequest request = builder.build(); - OffsetsForLeaderEpochRequest parsed = (OffsetsForLeaderEpochRequest) AbstractRequest.parseRequest( - ApiKeys.OFFSET_FOR_LEADER_EPOCH, version, request.toStruct()); + OffsetsForLeaderEpochRequest parsed = OffsetsForLeaderEpochRequest.parse(request.serializeBody(), version); if (version < 3) assertEquals(OffsetsForLeaderEpochRequest.DEBUGGING_REPLICA_ID, parsed.replicaId()); else @@ -58,4 +57,4 @@ public void testDefaultReplicaId() { } } -} \ No newline at end of file +} diff --git a/clients/src/test/java/org/apache/kafka/common/requests/ProduceRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/ProduceRequestTest.java index 156af86b53756..7bf53eceb3038 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/ProduceRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/ProduceRequestTest.java @@ -299,7 +299,7 @@ private void assertThrowsInvalidRecordExceptionForAllVersions(ProduceRequest.Bui private void assertThrowsInvalidRecordException(ProduceRequest.Builder builder, short version) { try { - builder.build(version).toStruct(); + builder.build(version).serializeBody(); fail("Builder did not raise " + InvalidRecordException.class.getName() + " as expected"); } catch (RuntimeException e) { assertTrue("Unexpected exception type " + e.getClass().getName(), diff --git a/clients/src/test/java/org/apache/kafka/common/requests/ProduceResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/ProduceResponseTest.java index 37cac84db0c15..9f3f1e877e33a 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/ProduceResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/ProduceResponseTest.java @@ -18,10 +18,8 @@ package org.apache.kafka.common.requests; import org.apache.kafka.common.TopicPartition; -import org.apache.kafka.common.message.ProduceResponseData; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.record.RecordBatch; import org.junit.Test; @@ -48,15 +46,12 @@ public void produceResponseV5Test() { ProduceResponse v5Response = new ProduceResponse(responseData, 10); short version = 5; - ByteBuffer buffer = v5Response.serialize(ApiKeys.PRODUCE, version, 0); + ByteBuffer buffer = v5Response.serializeWithHeader(version, 0); buffer.rewind(); ResponseHeader.parse(buffer, ApiKeys.PRODUCE.responseHeaderVersion(version)); // throw away. - - Struct deserializedStruct = ApiKeys.PRODUCE.parseResponse(version, buffer); - ProduceResponse v5FromBytes = (ProduceResponse) AbstractResponse.parseResponse(ApiKeys.PRODUCE, - deserializedStruct, version); + buffer, version); assertEquals(1, v5FromBytes.responses().size()); assertTrue(v5FromBytes.responses().containsKey(tp0)); @@ -79,12 +74,6 @@ public void produceResponseVersionTest() { assertEquals("Throttle time must be zero", 0, v0Response.throttleTimeMs()); assertEquals("Throttle time must be 10", 10, v1Response.throttleTimeMs()); assertEquals("Throttle time must be 10", 10, v2Response.throttleTimeMs()); - assertEquals("Should use schema version 0", ApiKeys.PRODUCE.responseSchema((short) 0), - v0Response.toStruct((short) 0).schema()); - assertEquals("Should use schema version 1", ApiKeys.PRODUCE.responseSchema((short) 1), - v1Response.toStruct((short) 1).schema()); - assertEquals("Should use schema version 2", ApiKeys.PRODUCE.responseSchema((short) 2), - v2Response.toStruct((short) 2).schema()); assertEquals("Response data does not match", responseData, v0Response.responses()); assertEquals("Response data does not match", responseData, v1Response.responses()); assertEquals("Response data does not match", responseData, v2Response.responses()); @@ -103,9 +92,7 @@ public void produceResponseRecordErrorsTest() { for (short ver = 0; ver <= PRODUCE.latestVersion(); ver++) { ProduceResponse response = new ProduceResponse(responseData); - Struct struct = response.toStruct(ver); - assertEquals("Should use schema version " + ver, ApiKeys.PRODUCE.responseSchema(ver), struct.schema()); - ProduceResponse.PartitionResponse deserialized = new ProduceResponse(new ProduceResponseData(struct, ver)).responses().get(tp); + ProduceResponse.PartitionResponse deserialized = ProduceResponse.parse(response.serializeBody(ver), ver).responses().get(tp); if (ver >= 8) { assertEquals(1, deserialized.recordErrors.size()); assertEquals(3, deserialized.recordErrors.get(0).batchIndex); diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestContextTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestContextTest.java index 47d49e618e9ab..1f2b567639832 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/RequestContextTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestContextTest.java @@ -24,7 +24,6 @@ import org.apache.kafka.common.network.Send; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.security.auth.KafkaPrincipal; import org.apache.kafka.common.security.auth.SecurityProtocol; import org.junit.Test; @@ -73,9 +72,8 @@ public void testSerdeUnsupportedApiVersionRequest() throws Exception { ApiKeys.API_VERSIONS.responseHeaderVersion(header.apiVersion())); assertEquals(correlationId, responseHeader.correlationId()); - Struct struct = ApiKeys.API_VERSIONS.parseResponse((short) 0, responseBuffer); - ApiVersionsResponse response = (ApiVersionsResponse) - AbstractResponse.parseResponse(ApiKeys.API_VERSIONS, struct, (short) 0); + ApiVersionsResponse response = (ApiVersionsResponse) AbstractResponse.parseResponse(ApiKeys.API_VERSIONS, + responseBuffer, (short) 0); assertEquals(Errors.UNSUPPORTED_VERSION.code(), response.data.errorCode()); assertTrue(response.data.apiKeys().isEmpty()); } diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestHeaderTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestHeaderTest.java index a3fa922359b4d..8cf5fb2c0b72b 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/RequestHeaderTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestHeaderTest.java @@ -17,12 +17,12 @@ package org.apache.kafka.common.requests; import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.protocol.ObjectSerializationCache; +import org.apache.kafka.test.TestUtils; import org.junit.Test; import java.nio.ByteBuffer; -import static org.apache.kafka.test.TestUtils.toBuffer; import static org.junit.Assert.assertEquals; public class RequestHeaderTest { @@ -30,11 +30,11 @@ public class RequestHeaderTest { @Test public void testSerdeControlledShutdownV0() { // Verify that version 0 of controlled shutdown does not include the clientId field - + short apiVersion = 0; int correlationId = 2342; ByteBuffer rawBuffer = ByteBuffer.allocate(32); rawBuffer.putShort(ApiKeys.CONTROLLED_SHUTDOWN.id); - rawBuffer.putShort((short) 0); + rawBuffer.putShort(apiVersion); rawBuffer.putInt(correlationId); rawBuffer.flip(); @@ -45,8 +45,7 @@ public void testSerdeControlledShutdownV0() { assertEquals("", deserialized.clientId()); assertEquals(0, deserialized.headerVersion()); - Struct serialized = deserialized.toStruct(); - ByteBuffer serializedBuffer = toBuffer(serialized); + ByteBuffer serializedBuffer = TestUtils.serializeRequestHeader(deserialized); assertEquals(ApiKeys.CONTROLLED_SHUTDOWN.id, serializedBuffer.getShort(0)); assertEquals(0, serializedBuffer.getShort(2)); @@ -56,9 +55,11 @@ public void testSerdeControlledShutdownV0() { @Test public void testRequestHeaderV1() { - RequestHeader header = new RequestHeader(ApiKeys.FIND_COORDINATOR, (short) 1, "", 10); + short apiVersion = 1; + RequestHeader header = new RequestHeader(ApiKeys.FIND_COORDINATOR, apiVersion, "", 10); assertEquals(1, header.headerVersion()); - ByteBuffer buffer = toBuffer(header.toStruct()); + + ByteBuffer buffer = TestUtils.serializeRequestHeader(header); assertEquals(10, buffer.remaining()); RequestHeader deserialized = RequestHeader.parse(buffer); assertEquals(header, deserialized); @@ -66,9 +67,11 @@ public void testRequestHeaderV1() { @Test public void testRequestHeaderV2() { - RequestHeader header = new RequestHeader(ApiKeys.CREATE_DELEGATION_TOKEN, (short) 2, "", 10); + short apiVersion = 2; + RequestHeader header = new RequestHeader(ApiKeys.CREATE_DELEGATION_TOKEN, apiVersion, "", 10); assertEquals(2, header.headerVersion()); - ByteBuffer buffer = toBuffer(header.toStruct()); + + ByteBuffer buffer = TestUtils.serializeRequestHeader(header); assertEquals(11, buffer.remaining()); RequestHeader deserialized = RequestHeader.parse(buffer); assertEquals(header, deserialized); @@ -80,7 +83,10 @@ public void parseHeaderFromBufferWithNonZeroPosition() { buffer.position(10); RequestHeader header = new RequestHeader(ApiKeys.FIND_COORDINATOR, (short) 1, "", 10); - header.toStruct().writeTo(buffer); + ObjectSerializationCache serializationCache = new ObjectSerializationCache(); + // size must be called before write to avoid an NPE with the current implementation + header.size(serializationCache); + header.write(buffer, serializationCache); int limit = buffer.position(); buffer.position(10); buffer.limit(limit); diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java index 40d9b32876c3c..b62376c8881d6 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java @@ -35,6 +35,7 @@ import org.apache.kafka.common.errors.UnsupportedVersionException; import org.apache.kafka.common.message.AddOffsetsToTxnRequestData; import org.apache.kafka.common.message.AddOffsetsToTxnResponseData; +import org.apache.kafka.common.message.AlterClientQuotasResponseData; import org.apache.kafka.common.message.AlterConfigsResponseData; import org.apache.kafka.common.message.AlterPartitionReassignmentsRequestData; import org.apache.kafka.common.message.AlterPartitionReassignmentsResponseData; @@ -80,6 +81,7 @@ import org.apache.kafka.common.message.DescribeAclsResponseData; import org.apache.kafka.common.message.DescribeAclsResponseData.AclDescription; import org.apache.kafka.common.message.DescribeAclsResponseData.DescribeAclsResource; +import org.apache.kafka.common.message.DescribeClientQuotasResponseData; import org.apache.kafka.common.message.DescribeConfigsRequestData; import org.apache.kafka.common.message.DescribeConfigsResponseData; import org.apache.kafka.common.message.DescribeConfigsResponseData.DescribeConfigsResourceResult; @@ -94,6 +96,7 @@ import org.apache.kafka.common.message.ExpireDelegationTokenRequestData; import org.apache.kafka.common.message.ExpireDelegationTokenResponseData; import org.apache.kafka.common.message.FetchRequestData; +import org.apache.kafka.common.message.FetchResponseData; import org.apache.kafka.common.message.FindCoordinatorRequestData; import org.apache.kafka.common.message.HeartbeatRequestData; import org.apache.kafka.common.message.HeartbeatResponseData; @@ -160,8 +163,6 @@ import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.ObjectSerializationCache; -import org.apache.kafka.common.protocol.types.SchemaException; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.quota.ClientQuotaAlteration; import org.apache.kafka.common.quota.ClientQuotaEntity; import org.apache.kafka.common.quota.ClientQuotaFilter; @@ -182,17 +183,16 @@ import org.apache.kafka.common.security.token.delegation.TokenInformation; import org.apache.kafka.common.utils.SecurityUtils; import org.apache.kafka.common.utils.Utils; +import org.apache.kafka.test.TestUtils; import org.junit.Test; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; +import java.nio.BufferUnderflowException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; @@ -210,7 +210,6 @@ import static org.apache.kafka.common.protocol.ApiKeys.LIST_OFFSETS; import static org.apache.kafka.common.protocol.ApiKeys.SYNC_GROUP; import static org.apache.kafka.common.requests.FetchMetadata.INVALID_SESSION_ID; -import static org.apache.kafka.test.TestUtils.toBuffer; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -455,7 +454,7 @@ public void testSerialization() throws Exception { checkResponse(createCreateAclsResponse(), ApiKeys.CREATE_ACLS.latestVersion(), true); checkRequest(createDeleteAclsRequest(), true); checkErrorResponse(createDeleteAclsRequest(), new SecurityDisabledException("Security is not enabled."), true); - checkResponse(createDeleteAclsResponse(), ApiKeys.DELETE_ACLS.latestVersion(), true); + checkResponse(createDeleteAclsResponse(ApiKeys.DELETE_ACLS.latestVersion()), ApiKeys.DELETE_ACLS.latestVersion(), true); checkRequest(createAlterConfigsRequest(), false); checkErrorResponse(createAlterConfigsRequest(), unknownServerException, true); checkResponse(createAlterConfigsResponse(), 0, false); @@ -514,12 +513,15 @@ public void testSerialization() throws Exception { @Test public void testResponseHeader() { ResponseHeader header = createResponseHeader((short) 1); - ByteBuffer buffer = toBuffer(header.toStruct()); + ObjectSerializationCache serializationCache = new ObjectSerializationCache(); + ByteBuffer buffer = ByteBuffer.allocate(header.size(serializationCache)); + header.write(buffer, serializationCache); + buffer.flip(); ResponseHeader deserialized = ResponseHeader.parse(buffer, header.headerVersion()); assertEquals(header.correlationId(), deserialized.correlationId()); } - private void checkOlderFetchVersions() throws Exception { + private void checkOlderFetchVersions() { int latestVersion = FETCH.latestVersion(); for (int i = 0; i < latestVersion; ++i) { if (i > 7) { @@ -530,12 +532,15 @@ private void checkOlderFetchVersions() throws Exception { } } - private void verifyDescribeConfigsResponse(DescribeConfigsResponse expected, DescribeConfigsResponse actual, int version) throws Exception { + private void verifyDescribeConfigsResponse(DescribeConfigsResponse expected, DescribeConfigsResponse actual, + int version) { for (Map.Entry resource : expected.resultMap().entrySet()) { List actualEntries = actual.resultMap().get(resource.getKey()).configs(); - Iterator expectedEntries = expected.resultMap().get(resource.getKey()).configs().iterator(); - for (DescribeConfigsResourceResult actualEntry : actualEntries) { - DescribeConfigsResourceResult expectedEntry = expectedEntries.next(); + List expectedEntries = expected.resultMap().get(resource.getKey()).configs(); + assertEquals(expectedEntries.size(), actualEntries.size()); + for (int i = 0; i < actualEntries.size(); ++i) { + DescribeConfigsResourceResult actualEntry = actualEntries.get(i); + DescribeConfigsResourceResult expectedEntry = expectedEntries.get(i); assertEquals(expectedEntry.name(), actualEntry.name()); assertEquals("Non-matching values for " + actualEntry.name() + " in version " + version, expectedEntry.value(), actualEntry.value()); @@ -550,84 +555,69 @@ private void verifyDescribeConfigsResponse(DescribeConfigsResponse expected, Des assertEquals("Non-matching configType for " + actualEntry.name() + " in version " + version, expectedEntry.configType(), actualEntry.configType()); } - if (version == 1 || version == 3 || (expectedEntry.configSource() != DescribeConfigsResponse.ConfigSource.DYNAMIC_BROKER_CONFIG.id() && - expectedEntry.configSource() != DescribeConfigsResponse.ConfigSource.DYNAMIC_DEFAULT_BROKER_CONFIG.id())) - assertEquals("Non-matching configSource for " + actualEntry.name() + " in version " + version, - expectedEntry.configSource(), actualEntry.configSource()); - else + if (version == 0) { assertEquals("Non matching configSource for " + actualEntry.name() + " in version " + version, DescribeConfigsResponse.ConfigSource.STATIC_BROKER_CONFIG.id(), actualEntry.configSource()); + } else { + assertEquals("Non-matching configSource for " + actualEntry.name() + " in version " + version, + expectedEntry.configSource(), actualEntry.configSource()); + } } } } - private void checkDescribeConfigsResponseVersions() throws Exception { - DescribeConfigsResponse response = createDescribeConfigsResponse((short) 0); - DescribeConfigsResponse deserialized0 = (DescribeConfigsResponse) deserialize(response, - response.toStruct((short) 0), (short) 0); - verifyDescribeConfigsResponse(response, deserialized0, 0); - - response = createDescribeConfigsResponse((short) 1); - DescribeConfigsResponse deserialized1 = (DescribeConfigsResponse) deserialize(response, - response.toStruct((short) 1), (short) 1); - verifyDescribeConfigsResponse(response, deserialized1, 1); - - response = createDescribeConfigsResponse((short) 3); - DescribeConfigsResponse deserialized3 = (DescribeConfigsResponse) deserialize(response, - response.toStruct((short) 3), (short) 3); - verifyDescribeConfigsResponse(response, deserialized3, 3); + private void checkDescribeConfigsResponseVersions() { + for (int version = ApiKeys.DESCRIBE_CONFIGS.oldestVersion(); version < ApiKeys.DESCRIBE_CONFIGS.latestVersion(); ++version) { + short apiVersion = (short) version; + DescribeConfigsResponse response = createDescribeConfigsResponse(apiVersion); + DescribeConfigsResponse deserialized0 = (DescribeConfigsResponse) AbstractResponse.parseResponse(ApiKeys.DESCRIBE_CONFIGS, + response.serializeBody(apiVersion), apiVersion); + verifyDescribeConfigsResponse(response, deserialized0, apiVersion); + } } private void checkErrorResponse(AbstractRequest req, Throwable e, boolean checkEqualityAndHashCode) { AbstractResponse response = req.getErrorResponse(e); checkResponse(response, req.version(), checkEqualityAndHashCode); if (e instanceof UnknownServerException) { - String responseStr = response.toStruct(req.version()).toString(); - assertFalse(String.format("Unknown message included in response for %s: %s ", req.api, responseStr), + String responseStr = response.toString(); + assertFalse(String.format("Unknown message included in response for %s: %s ", req.apiKey(), responseStr), responseStr.contains(e.getMessage())); } } - private void checkRequest(AbstractRequest req, boolean checkEqualityAndHashCode) { + private void checkRequest(AbstractRequest req, boolean checkEquality) { // Check that we can serialize, deserialize and serialize again - // Check for equality and hashCode of the Struct only if indicated (it is likely to fail if any of the fields + // Check for equality of the ByteBuffer only if indicated (it is likely to fail if any of the fields // in the request is a HashMap with multiple elements since ordering of the elements may vary) try { - Struct struct = req.toStruct(); - AbstractRequest deserialized = AbstractRequest.parseRequest(req.api, req.version(), struct); - Struct struct2 = deserialized.toStruct(); - if (checkEqualityAndHashCode) { - assertEquals(struct, struct2); - assertEquals(struct.hashCode(), struct2.hashCode()); - } + ByteBuffer serializedBytes = req.serializeBody(); + AbstractRequest deserialized = AbstractRequest.parseRequest(req.apiKey(), req.version(), serializedBytes).request; + ByteBuffer serializedBytes2 = deserialized.serializeBody(); + serializedBytes.rewind(); + if (checkEquality) + assertEquals("Request " + req + "failed equality test", serializedBytes, serializedBytes2); } catch (Exception e) { throw new RuntimeException("Failed to deserialize request " + req + " with type " + req.getClass(), e); } } - private void checkResponse(AbstractResponse response, int version, boolean checkEqualityAndHashCode) { + private void checkResponse(AbstractResponse response, int version, boolean checkEquality) { // Check that we can serialize, deserialize and serialize again // Check for equality and hashCode of the Struct only if indicated (it is likely to fail if any of the fields // in the response is a HashMap with multiple elements since ordering of the elements may vary) try { - Struct struct = response.toStruct((short) version); - AbstractResponse deserialized = (AbstractResponse) deserialize(response, struct, (short) version); - Struct struct2 = deserialized.toStruct((short) version); - if (checkEqualityAndHashCode) { - assertEquals(struct, struct2); - assertEquals(struct.hashCode(), struct2.hashCode()); - } + ByteBuffer serializedBytes = response.serializeBody((short) version); + AbstractResponse deserialized = AbstractResponse.parseResponse(response.apiKey(), serializedBytes, (short) version); + ByteBuffer serializedBytes2 = deserialized.serializeBody((short) version); + serializedBytes.rewind(); + if (checkEquality) + assertEquals("Response " + response + "failed equality test", serializedBytes, serializedBytes2); } catch (Exception e) { throw new RuntimeException("Failed to deserialize response " + response + " with type " + response.getClass(), e); } } - private AbstractRequestResponse deserialize(AbstractRequestResponse req, Struct struct, short version) throws NoSuchMethodException, IllegalAccessException, InvocationTargetException { - ByteBuffer buffer = toBuffer(struct); - Method deserializer = req.getClass().getDeclaredMethod("parse", ByteBuffer.class, Short.TYPE); - return (AbstractRequestResponse) deserializer.invoke(null, buffer, version); - } - @Test(expected = UnsupportedVersionException.class) public void cannotUseFindCoordinatorV0ToFindTransactionCoordinator() { FindCoordinatorRequest.Builder builder = new FindCoordinatorRequest.Builder( @@ -665,7 +655,7 @@ public void testPartitionSize() { @Test public void produceRequestToStringTest() { ProduceRequest request = createProduceRequest(ApiKeys.PRODUCE.latestVersion()); - assertEquals(1, request.dataOrException().topicData().size()); + assertEquals(1, request.data().topicData().size()); assertFalse(request.toString(false).contains("partitionSizes")); assertTrue(request.toString(false).contains("numPartitions=1")); assertTrue(request.toString(true).contains("partitionSizes")); @@ -673,7 +663,7 @@ public void produceRequestToStringTest() { request.clearPartitionRecords(); try { - request.dataOrException(); + request.data(); fail("dataOrException should fail after clearPartitionRecords()"); } catch (IllegalStateException e) { // OK @@ -723,10 +713,6 @@ public void fetchResponseVersionTest() { FetchResponse v1Response = new FetchResponse<>(Errors.NONE, responseData, 10, INVALID_SESSION_ID); assertEquals("Throttle time must be zero", 0, v0Response.throttleTimeMs()); assertEquals("Throttle time must be 10", 10, v1Response.throttleTimeMs()); - assertEquals("Should use schema version 0", FETCH.responseSchema((short) 0), - v0Response.toStruct((short) 0).schema()); - assertEquals("Should use schema version 1", FETCH.responseSchema((short) 1), - v1Response.toStruct((short) 1).schema()); assertEquals("Response data does not match", responseData, v0Response.responseData()); assertEquals("Response data does not match", responseData, v1Response.responseData()); } @@ -748,7 +734,7 @@ public void testFetchResponseV4() { 6, FetchResponse.INVALID_LOG_START_OFFSET, Optional.empty(), emptyList(), records)); FetchResponse response = new FetchResponse<>(Errors.NONE, responseData, 10, INVALID_SESSION_ID); - FetchResponse deserialized = FetchResponse.parse(toBuffer(response.toStruct((short) 4)), (short) 4); + FetchResponse deserialized = FetchResponse.parse(response.serializeBody((short) 4), (short) 4); assertEquals(responseData, deserialized.responseData()); } @@ -781,19 +767,17 @@ private void verifyFetchResponseFullWrite(short apiVersion, FetchResponse ApiVersionsResponse.parse(ByteBuffer.allocate(0), version)); } @Test - public void testApiVersionResponseStructParsing() { - Struct struct = ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE.toStruct(ApiKeys.API_VERSIONS.latestVersion()); - ApiVersionsResponse response = ApiVersionsResponse. - fromStruct(struct, ApiKeys.API_VERSIONS.latestVersion()); + public void testApiVersionResponseParsing() { + ByteBuffer buffer = ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE.serializeBody(ApiKeys.API_VERSIONS.latestVersion()); + ApiVersionsResponse response = ApiVersionsResponse.parse(buffer, ApiKeys.API_VERSIONS.latestVersion()); assertEquals(Errors.NONE.code(), response.data.errorCode()); } @@ -1003,7 +985,7 @@ public void testInitProducerIdRequestVersions() { setTransactionalId("abracadabra"). setProducerId(123)); final UnsupportedVersionException exception = assertThrows( - UnsupportedVersionException.class, () -> bld.build((short) 2).toStruct()); + UnsupportedVersionException.class, () -> bld.build((short) 2).serializeBody()); assertTrue(exception.getMessage().contains("Attempted to write a non-default producerId at version 2")); bld.build((short) 3); } @@ -1367,7 +1349,7 @@ private MetadataResponse createMetadataResponse() { new TopicPartition("topic3", 0), Optional.empty(), Optional.empty(), replicas, isr, offlineReplicas)))); - return MetadataResponse.prepareResponse(asList(node), null, MetadataResponse.NO_CONTROLLER_ID, allTopicMetadata); + return TestUtils.metadataResponse(asList(node), null, MetadataResponse.NO_CONTROLLER_ID, allTopicMetadata); } private OffsetCommitRequest createOffsetCommitRequest(int version) { @@ -1679,7 +1661,7 @@ private SaslHandshakeResponse createSaslHandshakeResponse() { private SaslAuthenticateRequest createSaslAuthenticateRequest() { SaslAuthenticateRequestData data = new SaslAuthenticateRequestData().setAuthBytes(new byte[0]); - return new SaslAuthenticateRequest(data); + return new SaslAuthenticateRequest(data, ApiKeys.SASL_AUTHENTICATE.latestVersion()); } private SaslAuthenticateResponse createSaslAuthenticateResponse() { @@ -2040,7 +2022,7 @@ private DeleteAclsRequest createDeleteAclsRequest() { return new DeleteAclsRequest.Builder(data).build(); } - private DeleteAclsResponse createDeleteAclsResponse() { + private DeleteAclsResponse createDeleteAclsResponse(int version) { List filterResults = new ArrayList<>(); filterResults.add(new DeleteAclsResponseData.DeleteAclsFilterResult().setMatchingAcls(asList( new DeleteAclsResponseData.DeleteAclsMatchingAcl() @@ -2064,7 +2046,7 @@ private DeleteAclsResponse createDeleteAclsResponse() { .setErrorMessage("No security")); return new DeleteAclsResponse(new DeleteAclsResponseData() .setThrottleTimeMs(0) - .setFilterResults(filterResults)); + .setFilterResults(filterResults), (short) version); } private DescribeConfigsRequest createDescribeConfigsRequest(int version) { @@ -2350,7 +2332,7 @@ private ElectLeadersResponse createElectLeadersResponse() { partitionResult.setErrorMessage(Errors.UNKNOWN_TOPIC_OR_PARTITION.message()); electionResult.partitionResult().add(partitionResult); - return new ElectLeadersResponse(200, Errors.NONE.code(), electionResults); + return new ElectLeadersResponse(200, Errors.NONE.code(), electionResults, ApiKeys.ELECT_LEADERS.latestVersion()); } private IncrementalAlterConfigsRequest createIncrementalAlterConfigsRequest() { @@ -2511,8 +2493,15 @@ private DescribeClientQuotasRequest createDescribeClientQuotasRequest() { } private DescribeClientQuotasResponse createDescribeClientQuotasResponse() { - ClientQuotaEntity entity = new ClientQuotaEntity(Collections.singletonMap(ClientQuotaEntity.USER, "user")); - return new DescribeClientQuotasResponse(Collections.singletonMap(entity, Collections.singletonMap("request_percentage", 1.0)), 0); + DescribeClientQuotasResponseData data = new DescribeClientQuotasResponseData().setEntries(asList( + new DescribeClientQuotasResponseData.EntryData() + .setEntity(asList(new DescribeClientQuotasResponseData.EntityData() + .setEntityType(ClientQuotaEntity.USER) + .setEntityName("user"))) + .setValues(asList(new DescribeClientQuotasResponseData.ValueData() + .setKey("request_percentage") + .setValue(1.0))))); + return new DescribeClientQuotasResponse(data); } private AlterClientQuotasRequest createAlterClientQuotasRequest() { @@ -2523,8 +2512,12 @@ private AlterClientQuotasRequest createAlterClientQuotasRequest() { } private AlterClientQuotasResponse createAlterClientQuotasResponse() { - ClientQuotaEntity entity = new ClientQuotaEntity(Collections.singletonMap(ClientQuotaEntity.USER, "user")); - return new AlterClientQuotasResponse(Collections.singletonMap(entity, ApiError.NONE), 0); + AlterClientQuotasResponseData data = new AlterClientQuotasResponseData() + .setEntries(asList(new AlterClientQuotasResponseData.EntryData() + .setEntity(asList(new AlterClientQuotasResponseData.EntityData() + .setEntityType(ClientQuotaEntity.USER) + .setEntityName("user"))))); + return new AlterClientQuotasResponse(data); } /** @@ -2544,7 +2537,7 @@ public void testErrorCountsIncludesNone() { assertEquals(Integer.valueOf(1), createCreatePartitionsResponse().errorCounts().get(Errors.NONE)); assertEquals(Integer.valueOf(1), createCreateTokenResponse().errorCounts().get(Errors.NONE)); assertEquals(Integer.valueOf(1), createCreateTopicResponse().errorCounts().get(Errors.NONE)); - assertEquals(Integer.valueOf(1), createDeleteAclsResponse().errorCounts().get(Errors.NONE)); + assertEquals(Integer.valueOf(1), createDeleteAclsResponse(ApiKeys.DELETE_ACLS.latestVersion()).errorCounts().get(Errors.NONE)); assertEquals(Integer.valueOf(1), createDeleteGroupsResponse().errorCounts().get(Errors.NONE)); assertEquals(Integer.valueOf(1), createDeleteTopicsResponse().errorCounts().get(Errors.NONE)); assertEquals(Integer.valueOf(1), createDescribeAclsResponse().errorCounts().get(Errors.NONE)); diff --git a/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaRequestTest.java index a038aaf9de479..243efa9bbf4b9 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/StopReplicaRequestTest.java @@ -129,12 +129,12 @@ public void testTopicStatesNormalization() { List topicStates = topicStates(true); for (short version = STOP_REPLICA.oldestVersion(); version < STOP_REPLICA.latestVersion(); version++) { - // Create a request for version to get its struct + // Create a request for version to get its serialized form StopReplicaRequest baseRequest = new StopReplicaRequest.Builder(version, 0, 1, 0, true, topicStates).build(version); - // Construct the request from the struct - StopReplicaRequest request = new StopReplicaRequest(baseRequest.toStruct(), version); + // Construct the request from the buffer + StopReplicaRequest request = StopReplicaRequest.parse(baseRequest.serializeBody(), version); Map partitionStates = StopReplicaRequestTest.partitionStates(request.topicStates()); @@ -164,12 +164,12 @@ public void testPartitionStatesNormalization() { List topicStates = topicStates(true); for (short version = STOP_REPLICA.oldestVersion(); version < STOP_REPLICA.latestVersion(); version++) { - // Create a request for version to get its struct + // Create a request for version to get its serialized form StopReplicaRequest baseRequest = new StopReplicaRequest.Builder(version, 0, 1, 0, true, topicStates).build(version); - // Construct the request from the struct - StopReplicaRequest request = new StopReplicaRequest(baseRequest.toStruct(), version); + // Construct the request from the buffer + StopReplicaRequest request = StopReplicaRequest.parse(baseRequest.serializeBody(), version); Map partitionStates = request.partitionStates(); assertEquals(6, partitionStates.size()); diff --git a/clients/src/test/java/org/apache/kafka/common/requests/TxnOffsetCommitResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/TxnOffsetCommitResponseTest.java index be4f5df359a05..bf3e9c8d2115e 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/TxnOffsetCommitResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/TxnOffsetCommitResponseTest.java @@ -17,10 +17,8 @@ package org.apache.kafka.common.requests; import org.apache.kafka.common.message.TxnOffsetCommitResponseData; -import org.apache.kafka.common.message.TxnOffsetCommitResponseData.TxnOffsetCommitResponsePartition; -import org.apache.kafka.common.message.TxnOffsetCommitResponseData.TxnOffsetCommitResponseTopic; import org.apache.kafka.common.protocol.ApiKeys; - +import org.apache.kafka.common.protocol.MessageTestUtil; import org.junit.Test; import java.util.Arrays; @@ -42,26 +40,27 @@ public void testConstructorWithErrorResponse() { @Test @Override - public void testConstructorWithStruct() { + public void testParse() { TxnOffsetCommitResponseData data = new TxnOffsetCommitResponseData() .setThrottleTimeMs(throttleTimeMs) .setTopics(Arrays.asList( - new TxnOffsetCommitResponseTopic().setPartitions( - Collections.singletonList(new TxnOffsetCommitResponsePartition() - .setPartitionIndex(partitionOne) - .setErrorCode(errorOne.code()))), - new TxnOffsetCommitResponseTopic().setPartitions( - Collections.singletonList(new TxnOffsetCommitResponsePartition() - .setPartitionIndex(partitionTwo) - .setErrorCode(errorTwo.code())) - ) - )); + new TxnOffsetCommitResponseData.TxnOffsetCommitResponseTopic().setPartitions( + Collections.singletonList(new TxnOffsetCommitResponseData.TxnOffsetCommitResponsePartition() + .setPartitionIndex(partitionOne) + .setErrorCode(errorOne.code()))), + new TxnOffsetCommitResponseData.TxnOffsetCommitResponseTopic().setPartitions( + Collections.singletonList(new TxnOffsetCommitResponseData.TxnOffsetCommitResponsePartition() + .setPartitionIndex(partitionTwo) + .setErrorCode(errorTwo.code()))) + )); for (short version = 0; version <= ApiKeys.TXN_OFFSET_COMMIT.latestVersion(); version++) { - TxnOffsetCommitResponse response = new TxnOffsetCommitResponse(data.toStruct(version), version); + TxnOffsetCommitResponse response = TxnOffsetCommitResponse.parse( + MessageTestUtil.messageToByteBuffer(data, version), version); assertEquals(expectedErrorCounts, response.errorCounts()); assertEquals(throttleTimeMs, response.throttleTimeMs()); assertEquals(version >= 1, response.shouldClientThrottle(version)); } } + } diff --git a/clients/src/test/java/org/apache/kafka/common/requests/UpdateMetadataRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/UpdateMetadataRequestTest.java index 6bd83c6eeb80c..8da3fedef2a08 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/UpdateMetadataRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/UpdateMetadataRequestTest.java @@ -26,7 +26,6 @@ import org.apache.kafka.common.network.ListenerName; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.MessageTestUtil; import org.apache.kafka.common.security.auth.SecurityProtocol; import org.apache.kafka.test.TestUtils; import org.junit.Test; @@ -157,7 +156,7 @@ public void testVersionLogic() { assertEquals(2, request.controllerEpoch()); assertEquals(3, request.brokerEpoch()); - ByteBuffer byteBuffer = MessageTestUtil.messageToByteBuffer(request.data(), request.version()); + ByteBuffer byteBuffer = request.serializeBody(); UpdateMetadataRequest deserializedRequest = new UpdateMetadataRequest(new UpdateMetadataRequestData( new ByteBufferAccessor(byteBuffer), version), version); @@ -207,8 +206,7 @@ public void testTopicPartitionGroupingSizeReduction() { UpdateMetadataRequest.Builder builder = new UpdateMetadataRequest.Builder((short) 5, 0, 0, 0, partitionStates, Collections.emptyList()); - assertTrue(MessageTestUtil.messageSize(builder.build((short) 5).data(), (short) 5) < - MessageTestUtil.messageSize(builder.build((short) 4).data(), (short) 4)); + assertTrue(builder.build((short) 5).sizeInBytes() < builder.build((short) 4).sizeInBytes()); } private Set iterableToSet(Iterable iterable) { diff --git a/clients/src/test/java/org/apache/kafka/common/requests/WriteTxnMarkersRequestTest.java b/clients/src/test/java/org/apache/kafka/common/requests/WriteTxnMarkersRequestTest.java index d869201b52662..12fa71b92e015 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/WriteTxnMarkersRequestTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/WriteTxnMarkersRequestTest.java @@ -72,7 +72,7 @@ public void testGetErrorResponse() { request.getErrorResponse(throttleTimeMs, Errors.UNKNOWN_PRODUCER_ID.exception()); assertEquals(Collections.singletonMap( - topicPartition, Errors.UNKNOWN_PRODUCER_ID), errorResponse.errors(producerId)); + topicPartition, Errors.UNKNOWN_PRODUCER_ID), errorResponse.errorsByProducerId().get(producerId)); assertEquals(Collections.singletonMap(Errors.UNKNOWN_PRODUCER_ID, 1), errorResponse.errorCounts()); // Write txn marker has no throttle time defined in response. assertEquals(0, errorResponse.throttleTimeMs()); diff --git a/clients/src/test/java/org/apache/kafka/common/requests/WriteTxnMarkersResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/WriteTxnMarkersResponseTest.java index 4cec88c22e923..3860b186f072e 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/WriteTxnMarkersResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/WriteTxnMarkersResponseTest.java @@ -46,15 +46,15 @@ public void setUp() { errorMap.put(producerIdOne, Collections.singletonMap(tp1, pidOneError)); errorMap.put(producerIdTwo, Collections.singletonMap(tp2, pidTwoError)); } - @Test - public void testConstructorWithStruct() { + @Test + public void testConstructor() { Map expectedErrorCounts = new HashMap<>(); expectedErrorCounts.put(Errors.UNKNOWN_PRODUCER_ID, 1); expectedErrorCounts.put(Errors.INVALID_PRODUCER_EPOCH, 1); WriteTxnMarkersResponse response = new WriteTxnMarkersResponse(errorMap); assertEquals(expectedErrorCounts, response.errorCounts()); - assertEquals(Collections.singletonMap(tp1, pidOneError), response.errors(producerIdOne)); - assertEquals(Collections.singletonMap(tp2, pidTwoError), response.errors(producerIdTwo)); + assertEquals(Collections.singletonMap(tp1, pidOneError), response.errorsByProducerId().get(producerIdOne)); + assertEquals(Collections.singletonMap(tp2, pidTwoError), response.errorsByProducerId().get(producerIdTwo)); } } diff --git a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorFailureDelayTest.java b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorFailureDelayTest.java index 599345a110ac3..19003ed56da90 100644 --- a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorFailureDelayTest.java +++ b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorFailureDelayTest.java @@ -45,6 +45,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.Map; @@ -55,7 +56,7 @@ public class SaslAuthenticatorFailureDelayTest { private static final int BUFFER_SIZE = 4 * 1024; - private final MockTime time = new MockTime(10); + private final MockTime time = new MockTime(1); private NioEchoServer server; private Selector selector; private ChannelBuilder channelBuilder; @@ -118,6 +119,38 @@ public void testInvalidPasswordSaslPlain() throws Exception { server.verifyAuthenticationMetrics(0, 1); } + /** + * Tests that SASL/SCRAM clients with invalid password fail authentication with + * connection close delay if configured. + */ + @Test + public void testInvalidPasswordSaslScram() throws Exception { + String node = "0"; + SecurityProtocol securityProtocol = SecurityProtocol.SASL_SSL; + TestJaasConfig jaasConfig = configureMechanisms("SCRAM-SHA-256", Collections.singletonList("SCRAM-SHA-256")); + jaasConfig.setClientOptions("SCRAM-SHA-256", TestJaasConfig.USERNAME, "invalidpassword"); + + server = createEchoServer(securityProtocol); + createAndCheckClientAuthenticationFailure(securityProtocol, node, "SCRAM-SHA-256", null); + server.verifyAuthenticationMetrics(0, 1); + } + + /** + * Tests that clients with disabled SASL mechanism fail authentication with + * connection close delay if configured. + */ + @Test + public void testDisabledSaslMechanism() throws Exception { + String node = "0"; + SecurityProtocol securityProtocol = SecurityProtocol.SASL_SSL; + TestJaasConfig jaasConfig = configureMechanisms("SCRAM-SHA-256", Collections.singletonList("SCRAM-SHA-256")); + jaasConfig.setClientOptions("PLAIN", TestJaasConfig.USERNAME, "invalidpassword"); + + server = createEchoServer(securityProtocol); + createAndCheckClientAuthenticationFailure(securityProtocol, node, "SCRAM-SHA-256", null); + server.verifyAuthenticationMetrics(0, 1); + } + /** * Tests client connection close before response for authentication failure is sent. */ @@ -215,7 +248,7 @@ private void createAndCheckClientAuthenticationFailure(SecurityProtocol security Exception exception = finalState.exception(); assertTrue("Invalid exception class " + exception.getClass(), exception instanceof SaslAuthenticationException); if (expectedErrorMessage == null) - expectedErrorMessage = "Authentication failed due to invalid credentials with SASL mechanism " + mechanism; + expectedErrorMessage = "Authentication failed during authentication due to invalid credentials with SASL mechanism " + mechanism; assertEquals(expectedErrorMessage, exception.getMessage()); } diff --git a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java index 5c1ce3cfc0653..77bd9197476a9 100644 --- a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java +++ b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java @@ -132,6 +132,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -1602,7 +1603,7 @@ public void testConvertListOffsetResponseToSaslHandshakeResponse() { .setOffset(0) .setTimestamp(0))))); ListOffsetResponse response = new ListOffsetResponse(data); - ByteBuffer buffer = response.serialize(ApiKeys.LIST_OFFSETS, LIST_OFFSETS.latestVersion(), 0); + ByteBuffer buffer = response.serializeWithHeader(LIST_OFFSETS.latestVersion(), 0); final RequestHeader header0 = new RequestHeader(LIST_OFFSETS, LIST_OFFSETS.latestVersion(), "id", SaslClientAuthenticator.MIN_RESERVED_CORRELATION_ID); Assert.assertThrows(SchemaException.class, () -> NetworkClient.parseResponse(buffer.duplicate(), header0)); final RequestHeader header1 = new RequestHeader(LIST_OFFSETS, LIST_OFFSETS.latestVersion(), "id", 1); @@ -1681,19 +1682,18 @@ public void testCannotReauthenticateAgainFasterThanOneSecond() throws Exception * instead */ time.sleep((long) (CONNECTIONS_MAX_REAUTH_MS_VALUE * 1.1)); - NetworkTestUtils.checkClientConnection(selector, node, 1, 1); - fail("Expected a failure when trying to re-authenticate to quickly, but that did not occur"); - } catch (AssertionError e) { + AssertionError exception = assertThrows(AssertionError.class, + () -> NetworkTestUtils.checkClientConnection(selector, node, 1, 1)); String expectedResponseTextRegex = "\\w-" + node; String receivedResponseTextRegex = ".*" + OAuthBearerLoginModule.OAUTHBEARER_MECHANISM; assertTrue( "Should have received the SaslHandshakeRequest bytes back since we re-authenticated too quickly, " + "but instead we got our generated message echoed back, implying re-auth succeeded when it " + - "should not have: " + e, - e.getMessage().matches( + "should not have: " + exception, + exception.getMessage().matches( ".*<\\[" + expectedResponseTextRegex + "]>.*<\\[" + receivedResponseTextRegex + ".*?]>")); server.verifyReauthenticationMetrics(1, 0); // unchanged - } finally { + } finally { selector.close(); selector = null; } diff --git a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticatorTest.java b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticatorTest.java index 0b4c63bc42cb1..a4d2d771f5c17 100644 --- a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticatorTest.java +++ b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticatorTest.java @@ -29,11 +29,11 @@ import org.apache.kafka.common.requests.ApiVersionsRequest; import org.apache.kafka.common.security.auth.AuthenticateCallbackHandler; import org.apache.kafka.common.security.auth.SecurityProtocol; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.requests.RequestHeader; import org.apache.kafka.common.security.plain.PlainLoginModule; import org.apache.kafka.common.utils.AppInfoParser; import org.apache.kafka.common.utils.Time; +import org.apache.kafka.test.TestUtils; import org.junit.Test; import javax.security.auth.Subject; @@ -79,16 +79,16 @@ public void testUnexpectedRequestType() throws IOException { SaslServerAuthenticator authenticator = setupAuthenticator(configs, transportLayer, SCRAM_SHA_256.mechanismName(), new DefaultChannelMetadataRegistry()); - final RequestHeader header = new RequestHeader(ApiKeys.METADATA, (short) 0, "clientId", 13243); - final Struct headerStruct = header.toStruct(); + RequestHeader header = new RequestHeader(ApiKeys.METADATA, (short) 0, "clientId", 13243); + ByteBuffer headerBuffer = TestUtils.serializeRequestHeader(header); when(transportLayer.read(any(ByteBuffer.class))).then(invocation -> { - invocation.getArgument(0).putInt(headerStruct.sizeOf()); + invocation.getArgument(0).putInt(headerBuffer.remaining()); return 4; }).then(invocation -> { // serialize only the request header. the authenticator should not parse beyond this - headerStruct.writeTo(invocation.getArgument(0)); - return headerStruct.sizeOf(); + invocation.getArgument(0).put(headerBuffer.duplicate()); + return headerBuffer.remaining(); }); try { @@ -113,7 +113,7 @@ public void testLatestApiVersionsRequest() throws IOException { "apache-kafka-java", AppInfoParser.getVersion()); } - public void testApiVersionsRequest(short version, String expectedSoftwareName, + private void testApiVersionsRequest(short version, String expectedSoftwareName, String expectedSoftwareVersion) throws IOException { TransportLayer transportLayer = mock(TransportLayer.class, Answers.RETURNS_DEEP_STUBS); Map configs = Collections.singletonMap(BrokerSecurityConfigs.SASL_ENABLED_MECHANISMS_CONFIG, @@ -122,21 +122,23 @@ public void testApiVersionsRequest(short version, String expectedSoftwareName, SaslServerAuthenticator authenticator = setupAuthenticator(configs, transportLayer, SCRAM_SHA_256.mechanismName(), metadataRegistry); - final RequestHeader header = new RequestHeader(ApiKeys.API_VERSIONS, version, "clientId", 0); - final Struct headerStruct = header.toStruct(); + RequestHeader header = new RequestHeader(ApiKeys.API_VERSIONS, version, "clientId", 0); + ByteBuffer headerBuffer = TestUtils.serializeRequestHeader(header); - final ApiVersionsRequest request = new ApiVersionsRequest.Builder().build(version); - final Struct requestStruct = request.data.toStruct(version); + ApiVersionsRequest request = new ApiVersionsRequest.Builder().build(version); + ByteBuffer requestBuffer = request.serializeBody(); + requestBuffer.rewind(); when(transportLayer.socketChannel().socket().getInetAddress()).thenReturn(InetAddress.getLoopbackAddress()); when(transportLayer.read(any(ByteBuffer.class))).then(invocation -> { - invocation.getArgument(0).putInt(headerStruct.sizeOf() + requestStruct.sizeOf()); + invocation.getArgument(0).putInt(headerBuffer.remaining() + requestBuffer.remaining()); return 4; }).then(invocation -> { - headerStruct.writeTo(invocation.getArgument(0)); - requestStruct.writeTo(invocation.getArgument(0)); - return headerStruct.sizeOf() + requestStruct.sizeOf(); + invocation.getArgument(0) + .put(headerBuffer.duplicate()) + .put(requestBuffer.duplicate()); + return headerBuffer.remaining() + requestBuffer.remaining(); }); authenticator.authenticate(); diff --git a/clients/src/test/java/org/apache/kafka/test/TestUtils.java b/clients/src/test/java/org/apache/kafka/test/TestUtils.java index f006a20ba2dda..50a79c7dbb19e 100644 --- a/clients/src/test/java/org/apache/kafka/test/TestUtils.java +++ b/clients/src/test/java/org/apache/kafka/test/TestUtils.java @@ -24,11 +24,13 @@ import org.apache.kafka.common.PartitionInfo; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.internals.Topic; +import org.apache.kafka.common.message.MetadataResponseData; import org.apache.kafka.common.network.NetworkReceive; import org.apache.kafka.common.network.Send; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.protocol.ObjectSerializationCache; +import org.apache.kafka.common.record.RecordBatch; import org.apache.kafka.common.requests.ByteBufferChannel; import org.apache.kafka.common.requests.MetadataResponse; import org.apache.kafka.common.requests.RequestHeader; @@ -116,6 +118,41 @@ public static Cluster clusterWith(final int nodes, final Map to return new Cluster("kafka-cluster", asList(ns), parts, Collections.emptySet(), Collections.emptySet()); } + public static MetadataResponse metadataResponse(Collection brokers, + String clusterId, int controllerId, + List topicMetadataList) { + return metadataResponse(MetadataResponse.DEFAULT_THROTTLE_TIME, brokers, clusterId, controllerId, + topicMetadataList, MetadataResponse.AUTHORIZED_OPERATIONS_OMITTED); + } + + public static MetadataResponse metadataResponse(int throttleTimeMs, Collection brokers, + String clusterId, int controllerId, + List topicMetadatas, + int clusterAuthorizedOperations) { + List topics = new ArrayList<>(); + topicMetadatas.forEach(topicMetadata -> { + MetadataResponseData.MetadataResponseTopic metadataResponseTopic = new MetadataResponseData.MetadataResponseTopic(); + metadataResponseTopic + .setErrorCode(topicMetadata.error().code()) + .setName(topicMetadata.topic()) + .setIsInternal(topicMetadata.isInternal()) + .setTopicAuthorizedOperations(topicMetadata.authorizedOperations()); + + for (MetadataResponse.PartitionMetadata partitionMetadata : topicMetadata.partitionMetadata()) { + metadataResponseTopic.partitions().add(new MetadataResponseData.MetadataResponsePartition() + .setErrorCode(partitionMetadata.error.code()) + .setPartitionIndex(partitionMetadata.partition()) + .setLeaderId(partitionMetadata.leaderId.orElse(MetadataResponse.NO_LEADER_ID)) + .setLeaderEpoch(partitionMetadata.leaderEpoch.orElse(RecordBatch.NO_PARTITION_LEADER_EPOCH)) + .setReplicaNodes(partitionMetadata.replicaIds) + .setIsrNodes(partitionMetadata.inSyncReplicaIds) + .setOfflineReplicas(partitionMetadata.offlineReplicaIds)); + } + topics.add(metadataResponseTopic); + }); + return MetadataResponse.prepareResponse(true, throttleTimeMs, brokers, clusterId, controllerId, + topics, clusterAuthorizedOperations); } + public static MetadataResponse metadataUpdateWith(final int numNodes, final Map topicPartitionCounts) { return metadataUpdateWith("kafka-cluster", numNodes, topicPartitionCounts); @@ -197,7 +234,15 @@ public static MetadataResponse metadataUpdateWith(final String clusterId, Topic.isInternal(topic), Collections.emptyList())); } - return MetadataResponse.prepareResponse(nodes, clusterId, 0, topicMetadata, responseVersion); + return metadataResponse(nodes, clusterId, 0, topicMetadata); + } + + public static ByteBuffer serializeRequestHeader(RequestHeader header) { + ObjectSerializationCache serializationCache = new ObjectSerializationCache(); + ByteBuffer buffer = ByteBuffer.allocate(header.size(serializationCache)); + header.write(buffer, serializationCache); + buffer.rewind(); + return buffer; } @FunctionalInterface @@ -526,13 +571,6 @@ public static Set toSet(Collection collection) { return new HashSet<>(collection); } - public static ByteBuffer toBuffer(Struct struct) { - ByteBuffer buffer = ByteBuffer.allocate(struct.sizeOf()); - struct.writeTo(buffer); - buffer.rewind(); - return buffer; - } - public static ByteBuffer toBuffer(Send send) { ByteBufferChannel channel = new ByteBufferChannel(send.size()); try { diff --git a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorConnectorsIntegrationTest.java b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorConnectorsIntegrationTest.java index a71b28c0aa9a1..d0715dc8452a9 100644 --- a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorConnectorsIntegrationTest.java +++ b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorConnectorsIntegrationTest.java @@ -24,6 +24,7 @@ import org.apache.kafka.common.TopicPartition; import org.apache.kafka.connect.util.clusters.EmbeddedConnectCluster; import org.apache.kafka.connect.util.clusters.EmbeddedKafkaCluster; +import org.apache.kafka.connect.util.clusters.UngracefulShutdownException; import org.apache.kafka.test.IntegrationTest; import org.apache.kafka.common.utils.Exit; import org.junit.After; @@ -44,13 +45,11 @@ import java.util.Properties; import java.util.Set; import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; import java.util.stream.IntStream; import static org.apache.kafka.test.TestUtils.waitForCondition; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; @@ -75,14 +74,45 @@ public class MirrorConnectorsIntegrationTest { private static final int RECORD_CONSUME_DURATION_MS = 20_000; private static final int OFFSET_SYNC_DURATION_MS = 30_000; - private final AtomicBoolean exited = new AtomicBoolean(false); + private volatile boolean shuttingDown; private Map mm2Props; private MirrorMakerConfig mm2Config; private EmbeddedConnectCluster primary; private EmbeddedConnectCluster backup; + private Exit.Procedure exitProcedure; + private Exit.Procedure haltProcedure; + @Before public void setup() throws InterruptedException { + shuttingDown = false; + exitProcedure = (code, message) -> { + if (shuttingDown) { + // ignore this since we're shutting down Connect and Kafka and timing isn't always great + return; + } + if (code != 0) { + String exitMessage = "Abrupt service exit with code " + code + " and message " + message; + log.warn(exitMessage); + throw new UngracefulShutdownException(exitMessage); + } + }; + haltProcedure = (code, message) -> { + if (shuttingDown) { + // ignore this since we're shutting down Connect and Kafka and timing isn't always great + return; + } + if (code != 0) { + String haltMessage = "Abrupt service halt with code " + code + " and message " + message; + log.warn(haltMessage); + throw new UngracefulShutdownException(haltMessage); + } + }; + // Override the exit and halt procedure that Connect and Kafka will use. For these integration tests, + // we don't want to exit the JVM and instead simply want to fail the test + Exit.setExitProcedure(exitProcedure); + Exit.setHaltProcedure(haltProcedure); + Properties brokerProps = new Properties(); brokerProps.put("auto.create.topics.enable", "false"); @@ -116,6 +146,7 @@ public void setup() throws InterruptedException { .numBrokers(1) .brokerProps(brokerProps) .workerProps(primaryWorkerProps) + .maskExitProcedures(false) .build(); backup = new EmbeddedConnectCluster.Builder() @@ -124,6 +155,7 @@ public void setup() throws InterruptedException { .numBrokers(1) .brokerProps(brokerProps) .workerProps(backupWorkerProps) + .maskExitProcedures(false) .build(); primary.start(); @@ -164,8 +196,6 @@ public void setup() throws InterruptedException { mm2Props.put("primary.bootstrap.servers", primary.kafka().bootstrapServers()); mm2Props.put("backup.bootstrap.servers", backup.kafka().bootstrapServers()); mm2Config = new MirrorMakerConfig(mm2Props); - - Exit.setExitProcedure((status, errorCode) -> exited.set(true)); } @@ -194,20 +224,27 @@ private void waitUntilMirrorMakerIsRunning(EmbeddedConnectCluster connectCluster @After public void close() { - for (String x : primary.connectors()) { - primary.deleteConnector(x); - } - for (String x : backup.connectors()) { - backup.deleteConnector(x); - } - deleteAllTopics(primary.kafka()); - deleteAllTopics(backup.kafka()); - primary.stop(); - backup.stop(); try { - assertFalse(exited.get()); + for (String x : primary.connectors()) { + primary.deleteConnector(x); + } + for (String x : backup.connectors()) { + backup.deleteConnector(x); + } + deleteAllTopics(primary.kafka()); + deleteAllTopics(backup.kafka()); } finally { - Exit.resetExitProcedure(); + shuttingDown = true; + try { + try { + primary.stop(); + } finally { + backup.stop(); + } + } finally { + Exit.resetExitProcedure(); + Exit.resetHaltProcedure(); + } } } @@ -305,6 +342,9 @@ public void testReplication() throws InterruptedException { Map primaryOffsets = primaryClient.remoteConsumerOffsets(consumerGroupName, "backup", Duration.ofMillis(CHECKPOINT_DURATION_MS)); + primaryClient.close(); + backupClient.close(); + // Failback consumer group to primary cluster primaryConsumer = primary.kafka().createConsumer(consumerProps); primaryConsumer.assign(allPartitions("test-topic-1", "backup.test-topic-1")); diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSourceTask.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSourceTask.java index 5874e3c26e25f..bc7df97476b30 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSourceTask.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSourceTask.java @@ -102,9 +102,7 @@ class WorkerSourceTask extends WorkerTask { private CountDownLatch stopRequestedLatch; private Map taskConfig; - private boolean finishedStart = false; - private boolean startedShutdownBeforeStartCompleted = false; - private boolean stopped = false; + private boolean started = false; public WorkerSourceTask(ConnectorTaskId id, SourceTask task, @@ -166,8 +164,12 @@ public void initialize(TaskConfig taskConfig) { @Override protected void close() { - if (!shouldPause()) { - tryStop(); + if (started) { + try { + task.stop(); + } catch (Throwable t) { + log.warn("Could not stop task", t); + } } if (producer != null) { try { @@ -206,39 +208,21 @@ public void cancel() { public void stop() { super.stop(); stopRequestedLatch.countDown(); - synchronized (this) { - if (finishedStart) - tryStop(); - else - startedShutdownBeforeStartCompleted = true; - } - } - - private synchronized void tryStop() { - if (!stopped) { - try { - task.stop(); - stopped = true; - } catch (Throwable t) { - log.warn("Could not stop task", t); - } - } } @Override public void execute() { try { + // If we try to start the task at all by invoking initialize, then count this as + // "started" and expect a subsequent call to the task's stop() method + // to properly clean up any resources allocated by its initialize() or + // start() methods. If the task throws an exception during stop(), + // the worst thing that happens is another exception gets logged for an already- + // failed task + started = true; task.initialize(new WorkerSourceTaskContext(offsetReader, this, configState)); task.start(taskConfig); log.info("{} Source task finished initialization and start", this); - synchronized (this) { - if (startedShutdownBeforeStartCompleted) { - tryStop(); - return; - } - finishedStart = true; - } - while (!isStopping()) { if (shouldPause()) { onPause(); diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/BlockingConnectorTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/BlockingConnectorTest.java index 9a8e1fa76aeb7..abc9a938c6606 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/BlockingConnectorTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/BlockingConnectorTest.java @@ -16,17 +16,26 @@ */ package org.apache.kafka.connect.integration; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; +import org.apache.kafka.clients.producer.RecordMetadata; +import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.config.AbstractConfig; import org.apache.kafka.common.config.Config; import org.apache.kafka.common.config.ConfigDef; +import org.apache.kafka.connect.connector.Connector; import org.apache.kafka.connect.connector.ConnectorContext; import org.apache.kafka.connect.connector.Task; import org.apache.kafka.connect.runtime.Worker; import org.apache.kafka.connect.runtime.rest.errors.ConnectRestException; import org.apache.kafka.connect.runtime.rest.resources.ConnectorsResource; +import org.apache.kafka.connect.sink.SinkConnector; +import org.apache.kafka.connect.sink.SinkRecord; +import org.apache.kafka.connect.sink.SinkTask; +import org.apache.kafka.connect.sink.SinkTaskContext; import org.apache.kafka.connect.source.SourceConnector; import org.apache.kafka.connect.source.SourceRecord; import org.apache.kafka.connect.source.SourceTask; +import org.apache.kafka.connect.source.SourceTaskContext; import org.apache.kafka.connect.util.clusters.EmbeddedConnectCluster; import org.apache.kafka.test.IntegrationTest; import org.junit.After; @@ -36,16 +45,21 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Properties; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.IntStream; import static org.apache.kafka.connect.runtime.ConnectorConfig.CONNECTOR_CLASS_CONFIG; import static org.apache.kafka.connect.runtime.ConnectorConfig.TASKS_MAX_CONFIG; +import static org.apache.kafka.connect.runtime.SinkConnectorConfig.TOPICS_CONFIG; import static org.apache.kafka.test.TestUtils.waitForCondition; import static org.junit.Assert.assertThrows; @@ -67,6 +81,33 @@ public class BlockingConnectorTest { private static final long RECORD_TRANSFER_DURATION_MS = TimeUnit.SECONDS.toMillis(30); private static final long REST_REQUEST_TIMEOUT = Worker.CONNECTOR_GRACEFUL_SHUTDOWN_TIMEOUT_MS * 2; + private static final String CONNECTOR_INITIALIZE = "Connector::initialize"; + private static final String CONNECTOR_INITIALIZE_WITH_TASK_CONFIGS = "Connector::initializeWithTaskConfigs"; + private static final String CONNECTOR_START = "Connector::start"; + private static final String CONNECTOR_RECONFIGURE = "Connector::reconfigure"; + private static final String CONNECTOR_TASK_CLASS = "Connector::taskClass"; + private static final String CONNECTOR_TASK_CONFIGS = "Connector::taskConfigs"; + private static final String CONNECTOR_STOP = "Connector::stop"; + private static final String CONNECTOR_VALIDATE = "Connector::validate"; + private static final String CONNECTOR_CONFIG = "Connector::config"; + private static final String CONNECTOR_VERSION = "Connector::version"; + private static final String TASK_START = "Task::start"; + private static final String TASK_STOP = "Task::stop"; + private static final String TASK_VERSION = "Task::version"; + private static final String SINK_TASK_INITIALIZE = "SinkTask::initialize"; + private static final String SINK_TASK_PUT = "SinkTask::put"; + private static final String SINK_TASK_FLUSH = "SinkTask::flush"; + private static final String SINK_TASK_PRE_COMMIT = "SinkTask::preCommit"; + private static final String SINK_TASK_OPEN = "SinkTask::open"; + private static final String SINK_TASK_ON_PARTITIONS_ASSIGNED = "SinkTask::onPartitionsAssigned"; + private static final String SINK_TASK_CLOSE = "SinkTask::close"; + private static final String SINK_TASK_ON_PARTITIONS_REVOKED = "SinkTask::onPartitionsRevoked"; + private static final String SOURCE_TASK_INITIALIZE = "SourceTask::initialize"; + private static final String SOURCE_TASK_POLL = "SourceTask::poll"; + private static final String SOURCE_TASK_COMMIT = "SourceTask::commit"; + private static final String SOURCE_TASK_COMMIT_RECORD = "SourceTask::commitRecord"; + private static final String SOURCE_TASK_COMMIT_RECORD_WITH_METADATA = "SourceTask::commitRecordWithMetadata"; + private EmbeddedConnectCluster connect; private ConnectorHandle normalConnectorHandle; @@ -101,18 +142,18 @@ public void close() { // stop all Connect, Kafka and Zk threads. connect.stop(); ConnectorsResource.resetRequestTimeout(); - BlockingConnector.resetBlockLatch(); + Block.resetBlockLatch(); } @Test public void testBlockInConnectorValidate() throws Exception { log.info("Starting test testBlockInConnectorValidate"); - assertThrows(ConnectRestException.class, () -> createConnectorWithBlock(ValidateBlockingConnector.class)); + assertThrows(ConnectRestException.class, () -> createConnectorWithBlock(ValidateBlockingConnector.class, CONNECTOR_VALIDATE)); // Will NOT assert that connector has failed, since the request should fail before it's even created // Connector should already be blocked so this should return immediately, but check just to // make sure that it actually did block - BlockingConnector.waitForBlock(); + Block.waitForBlock(); createNormalConnector(); verifyNormalConnector(); @@ -121,12 +162,12 @@ public void testBlockInConnectorValidate() throws Exception { @Test public void testBlockInConnectorConfig() throws Exception { log.info("Starting test testBlockInConnectorConfig"); - assertThrows(ConnectRestException.class, () -> createConnectorWithBlock(ConfigBlockingConnector.class)); + assertThrows(ConnectRestException.class, () -> createConnectorWithBlock(ConfigBlockingConnector.class, CONNECTOR_CONFIG)); // Will NOT assert that connector has failed, since the request should fail before it's even created // Connector should already be blocked so this should return immediately, but check just to // make sure that it actually did block - BlockingConnector.waitForBlock(); + Block.waitForBlock(); createNormalConnector(); verifyNormalConnector(); @@ -135,8 +176,8 @@ public void testBlockInConnectorConfig() throws Exception { @Test public void testBlockInConnectorInitialize() throws Exception { log.info("Starting test testBlockInConnectorInitialize"); - createConnectorWithBlock(InitializeBlockingConnector.class); - BlockingConnector.waitForBlock(); + createConnectorWithBlock(InitializeBlockingConnector.class, CONNECTOR_INITIALIZE); + Block.waitForBlock(); createNormalConnector(); verifyNormalConnector(); @@ -145,8 +186,8 @@ public void testBlockInConnectorInitialize() throws Exception { @Test public void testBlockInConnectorStart() throws Exception { log.info("Starting test testBlockInConnectorStart"); - createConnectorWithBlock(BlockingConnector.START); - BlockingConnector.waitForBlock(); + createConnectorWithBlock(BlockingConnector.class, CONNECTOR_START); + Block.waitForBlock(); createNormalConnector(); verifyNormalConnector(); @@ -155,10 +196,54 @@ public void testBlockInConnectorStart() throws Exception { @Test public void testBlockInConnectorStop() throws Exception { log.info("Starting test testBlockInConnectorStop"); - createConnectorWithBlock(BlockingConnector.STOP); + createConnectorWithBlock(BlockingConnector.class, CONNECTOR_STOP); + waitForConnectorStart(BLOCKING_CONNECTOR_NAME); + connect.deleteConnector(BLOCKING_CONNECTOR_NAME); + Block.waitForBlock(); + + createNormalConnector(); + verifyNormalConnector(); + } + + @Test + public void testBlockInSourceTaskStart() throws Exception { + log.info("Starting test testBlockInSourceTaskStart"); + createConnectorWithBlock(BlockingSourceConnector.class, TASK_START); + Block.waitForBlock(); + + createNormalConnector(); + verifyNormalConnector(); + } + + @Test + public void testBlockInSourceTaskStop() throws Exception { + log.info("Starting test testBlockInSourceTaskStop"); + createConnectorWithBlock(BlockingSourceConnector.class, TASK_STOP); waitForConnectorStart(BLOCKING_CONNECTOR_NAME); connect.deleteConnector(BLOCKING_CONNECTOR_NAME); - BlockingConnector.waitForBlock(); + Block.waitForBlock(); + + createNormalConnector(); + verifyNormalConnector(); + } + + @Test + public void testBlockInSinkTaskStart() throws Exception { + log.info("Starting test testBlockInSinkTaskStart"); + createConnectorWithBlock(BlockingSinkConnector.class, TASK_START); + Block.waitForBlock(); + + createNormalConnector(); + verifyNormalConnector(); + } + + @Test + public void testBlockInSinkTaskStop() throws Exception { + log.info("Starting test testBlockInSinkTaskStop"); + createConnectorWithBlock(BlockingSinkConnector.class, TASK_STOP); + waitForConnectorStart(BLOCKING_CONNECTOR_NAME); + connect.deleteConnector(BLOCKING_CONNECTOR_NAME); + Block.waitForBlock(); createNormalConnector(); verifyNormalConnector(); @@ -167,50 +252,41 @@ public void testBlockInConnectorStop() throws Exception { @Test public void testWorkerRestartWithBlockInConnectorStart() throws Exception { log.info("Starting test testWorkerRestartWithBlockInConnectorStart"); - createConnectorWithBlock(BlockingConnector.START); + createConnectorWithBlock(BlockingConnector.class, CONNECTOR_START); // First instance of the connector should block on startup - BlockingConnector.waitForBlock(); + Block.waitForBlock(); createNormalConnector(); connect.removeWorker(); connect.addWorker(); // After stopping the only worker and restarting it, a new instance of the blocking // connector should be created and we can ensure that it blocks again - BlockingConnector.waitForBlock(); + Block.waitForBlock(); verifyNormalConnector(); } @Test public void testWorkerRestartWithBlockInConnectorStop() throws Exception { log.info("Starting test testWorkerRestartWithBlockInConnectorStop"); - createConnectorWithBlock(BlockingConnector.STOP); + createConnectorWithBlock(BlockingConnector.class, CONNECTOR_STOP); waitForConnectorStart(BLOCKING_CONNECTOR_NAME); createNormalConnector(); waitForConnectorStart(NORMAL_CONNECTOR_NAME); connect.removeWorker(); - BlockingConnector.waitForBlock(); + Block.waitForBlock(); connect.addWorker(); waitForConnectorStart(BLOCKING_CONNECTOR_NAME); verifyNormalConnector(); } - private void createConnectorWithBlock(String block) { - Map props = baseBlockingConnectorProps(); - props.put(BlockingConnector.BLOCK_CONFIG, block); - log.info("Creating connector with block during {}", block); - try { - connect.configureConnector(BLOCKING_CONNECTOR_NAME, props); - } catch (RuntimeException e) { - log.info("Failed to create connector", e); - throw e; - } - } - - private void createConnectorWithBlock(Class connectorClass) { - Map props = baseBlockingConnectorProps(); + private void createConnectorWithBlock(Class connectorClass, String block) { + Map props = new HashMap<>(); props.put(CONNECTOR_CLASS_CONFIG, connectorClass.getName()); - log.info("Creating blocking connector of type {}", connectorClass.getSimpleName()); + props.put(TASKS_MAX_CONFIG, "1"); + props.put(TOPICS_CONFIG, "t1"); // Required for sink connectors + props.put(Block.BLOCK_CONFIG, Objects.requireNonNull(block)); + log.info("Creating blocking connector of type {} with block in {}", connectorClass.getSimpleName(), block); try { connect.configureConnector(BLOCKING_CONNECTOR_NAME, props); } catch (RuntimeException e) { @@ -219,13 +295,6 @@ private void createConnectorWithBlock(Class connect } } - private Map baseBlockingConnectorProps() { - Map result = new HashMap<>(); - result.put(CONNECTOR_CLASS_CONFIG, BlockingConnector.class.getName()); - result.put(TASKS_MAX_CONFIG, "1"); - return result; - } - private void createNormalConnector() { connect.kafka().createTopic(TEST_TOPIC, 3); @@ -263,63 +332,43 @@ private void verifyNormalConnector() throws InterruptedException { normalConnectorHandle.awaitCommits(RECORD_TRANSFER_DURATION_MS); } - public static class BlockingConnector extends SourceConnector { - + private static class Block { private static CountDownLatch blockLatch; - private String block; + private final String block; public static final String BLOCK_CONFIG = "block"; - public static final String INITIALIZE = "initialize"; - public static final String INITIALIZE_WITH_TASK_CONFIGS = "initializeWithTaskConfigs"; - public static final String START = "start"; - public static final String RECONFIGURE = "reconfigure"; - public static final String TASK_CLASS = "taskClass"; - public static final String TASK_CONFIGS = "taskConfigs"; - public static final String STOP = "stop"; - public static final String VALIDATE = "validate"; - public static final String CONFIG = "config"; - public static final String VERSION = "version"; - - private static final ConfigDef CONFIG_DEF = new ConfigDef() - .define( - BLOCK_CONFIG, - ConfigDef.Type.STRING, - "", - ConfigDef.Importance.MEDIUM, - "Where to block indefinitely, e.g., 'start', 'initialize', 'taskConfigs', 'version'" - ); - - // No-args constructor required by the framework - public BlockingConnector() { - this(null); - } - - protected BlockingConnector(String block) { - this.block = block; - synchronized (BlockingConnector.class) { - if (blockLatch != null) { - blockLatch.countDown(); - } - blockLatch = new CountDownLatch(1); - } + private static ConfigDef config() { + return new ConfigDef() + .define( + BLOCK_CONFIG, + ConfigDef.Type.STRING, + "", + ConfigDef.Importance.MEDIUM, + "Where to block indefinitely, e.g., 'Connector::start', 'Connector::initialize', " + + "'Connector::taskConfigs', 'Task::version', 'SinkTask::put', 'SourceTask::poll'" + ); } public static void waitForBlock() throws InterruptedException { - synchronized (BlockingConnector.class) { + synchronized (Block.class) { if (blockLatch == null) { throw new IllegalArgumentException("No connector has been created yet"); } } - + log.debug("Waiting for connector to block"); blockLatch.await(); log.debug("Connector should now be blocked"); } + // Note that there is only ever at most one global block latch at a time, which makes tests that + // use blocks in multiple places impossible. If necessary, this can be addressed in the future by + // adding support for multiple block latches at a time, possibly identifiable by a connector/task + // ID, the location of the expected block, or both. public static void resetBlockLatch() { - synchronized (BlockingConnector.class) { + synchronized (Block.class) { if (blockLatch != null) { blockLatch.countDown(); blockLatch = null; @@ -327,81 +376,114 @@ public static void resetBlockLatch() { } } + public Block(Map props) { + this(new AbstractConfig(config(), props).getString(BLOCK_CONFIG)); + } + + public Block(String block) { + this.block = block; + synchronized (Block.class) { + if (blockLatch != null) { + blockLatch.countDown(); + } + blockLatch = new CountDownLatch(1); + } + } + + public Map taskConfig() { + return Collections.singletonMap(BLOCK_CONFIG, block); + } + + public void maybeBlockOn(String block) { + if (block.equals(this.block)) { + log.info("Will block on {}", block); + blockLatch.countDown(); + while (true) { + try { + Thread.sleep(Long.MAX_VALUE); + } catch (InterruptedException e) { + // No-op. Just keep blocking. + } + } + } else { + log.debug("Will not block on {}", block); + } + } + } + + // Used to test blocks in Connector (as opposed to Task) methods + public static class BlockingConnector extends SourceConnector { + + private Block block; + + // No-args constructor required by the framework + public BlockingConnector() { + this(null); + } + + protected BlockingConnector(String block) { + this.block = new Block(block); + } + @Override public void initialize(ConnectorContext ctx) { - maybeBlockOn(INITIALIZE); + block.maybeBlockOn(CONNECTOR_INITIALIZE); super.initialize(ctx); } @Override public void initialize(ConnectorContext ctx, List> taskConfigs) { - maybeBlockOn(INITIALIZE_WITH_TASK_CONFIGS); + block.maybeBlockOn(CONNECTOR_INITIALIZE_WITH_TASK_CONFIGS); super.initialize(ctx, taskConfigs); } @Override public void start(Map props) { - this.block = new AbstractConfig(CONFIG_DEF, props).getString(BLOCK_CONFIG); - maybeBlockOn(START); + this.block = new Block(props); + block.maybeBlockOn(CONNECTOR_START); } @Override public void reconfigure(Map props) { + block.maybeBlockOn(CONNECTOR_RECONFIGURE); super.reconfigure(props); - maybeBlockOn(RECONFIGURE); } @Override public Class taskClass() { - maybeBlockOn(TASK_CLASS); + block.maybeBlockOn(CONNECTOR_TASK_CLASS); return BlockingTask.class; } @Override public List> taskConfigs(int maxTasks) { - maybeBlockOn(TASK_CONFIGS); + block.maybeBlockOn(CONNECTOR_TASK_CONFIGS); return Collections.singletonList(Collections.emptyMap()); } @Override public void stop() { - maybeBlockOn(STOP); + block.maybeBlockOn(CONNECTOR_STOP); } @Override public Config validate(Map connectorConfigs) { - maybeBlockOn(VALIDATE); + block.maybeBlockOn(CONNECTOR_VALIDATE); return super.validate(connectorConfigs); } @Override public ConfigDef config() { - maybeBlockOn(CONFIG); - return CONFIG_DEF; + block.maybeBlockOn(CONNECTOR_CONFIG); + return Block.config(); } @Override public String version() { - maybeBlockOn(VERSION); + block.maybeBlockOn(CONNECTOR_VERSION); return "0.0.0"; } - protected void maybeBlockOn(String block) { - if (block.equals(this.block)) { - log.info("Will block on {}", block); - blockLatch.countDown(); - while (true) { - try { - Thread.sleep(Long.MAX_VALUE); - } catch (InterruptedException e) { - // No-op. Just keep blocking. - } - } - } else { - log.debug("Will not block on {}", block); - } - } - public static class BlockingTask extends SourceTask { @Override public void start(Map props) { @@ -426,19 +508,287 @@ public String version() { // Some methods are called before Connector::start, so we use this as a workaround public static class InitializeBlockingConnector extends BlockingConnector { public InitializeBlockingConnector() { - super(INITIALIZE); + super(CONNECTOR_INITIALIZE); } } public static class ConfigBlockingConnector extends BlockingConnector { public ConfigBlockingConnector() { - super(CONFIG); + super(CONNECTOR_CONFIG); } } public static class ValidateBlockingConnector extends BlockingConnector { public ValidateBlockingConnector() { - super(VALIDATE); + super(CONNECTOR_VALIDATE); + } + } + + // Used to test blocks in SourceTask methods + public static class BlockingSourceConnector extends SourceConnector { + + private Map props; + private final Class taskClass; + + // No-args constructor required by the framework + public BlockingSourceConnector() { + this(BlockingSourceTask.class); + } + + protected BlockingSourceConnector(Class taskClass) { + this.taskClass = taskClass; + } + + @Override + public void start(Map props) { + this.props = props; + } + + @Override + public Class taskClass() { + return taskClass; + } + + @Override + public List> taskConfigs(int maxTasks) { + return IntStream.range(0, maxTasks) + .mapToObj(i -> new HashMap<>(props)) + .collect(Collectors.toList()); + } + + @Override + public void stop() { + } + + @Override + public Config validate(Map connectorConfigs) { + return super.validate(connectorConfigs); + } + + @Override + public ConfigDef config() { + return Block.config(); + } + + @Override + public String version() { + return "0.0.0"; + } + + public static class BlockingSourceTask extends SourceTask { + private Block block; + + // No-args constructor required by the framework + public BlockingSourceTask() { + this(null); + } + + protected BlockingSourceTask(String block) { + this.block = new Block(block); + } + + @Override + public void start(Map props) { + this.block = new Block(props); + block.maybeBlockOn(TASK_START); + } + + @Override + public List poll() { + block.maybeBlockOn(SOURCE_TASK_POLL); + return null; + } + + @Override + public void stop() { + block.maybeBlockOn(TASK_STOP); + } + + @Override + public String version() { + block.maybeBlockOn(TASK_VERSION); + return "0.0.0"; + } + + @Override + public void initialize(SourceTaskContext context) { + block.maybeBlockOn(SOURCE_TASK_INITIALIZE); + super.initialize(context); + } + + @Override + public void commit() throws InterruptedException { + block.maybeBlockOn(SOURCE_TASK_COMMIT); + super.commit(); + } + + @Override + @SuppressWarnings("deprecation") + public void commitRecord(SourceRecord record) throws InterruptedException { + block.maybeBlockOn(SOURCE_TASK_COMMIT_RECORD); + super.commitRecord(record); + } + + @Override + public void commitRecord(SourceRecord record, RecordMetadata metadata) throws InterruptedException { + block.maybeBlockOn(SOURCE_TASK_COMMIT_RECORD_WITH_METADATA); + super.commitRecord(record, metadata); + } + } + } + + public static class TaskInitializeBlockingSourceConnector extends BlockingSourceConnector { + public TaskInitializeBlockingSourceConnector() { + super(InitializeBlockingSourceTask.class); + } + + public static class InitializeBlockingSourceTask extends BlockingSourceTask { + public InitializeBlockingSourceTask() { + super(SOURCE_TASK_INITIALIZE); + } + } + } + + // Used to test blocks in SinkTask methods + public static class BlockingSinkConnector extends SinkConnector { + + private Map props; + private final Class taskClass; + + // No-args constructor required by the framework + public BlockingSinkConnector() { + this(BlockingSinkTask.class); + } + + protected BlockingSinkConnector(Class taskClass) { + this.taskClass = taskClass; + } + + @Override + public void start(Map props) { + this.props = props; + } + + @Override + public Class taskClass() { + return taskClass; + } + + @Override + public List> taskConfigs(int maxTasks) { + return IntStream.rangeClosed(0, maxTasks) + .mapToObj(i -> new HashMap<>(props)) + .collect(Collectors.toList()); + } + + @Override + public void stop() { + } + + @Override + public Config validate(Map connectorConfigs) { + return super.validate(connectorConfigs); + } + + @Override + public ConfigDef config() { + return Block.config(); + } + + @Override + public String version() { + return "0.0.0"; + } + + public static class BlockingSinkTask extends SinkTask { + private Block block; + + // No-args constructor required by the framework + public BlockingSinkTask() { + this(null); + } + + protected BlockingSinkTask(String block) { + this.block = new Block(block); + } + + @Override + public void start(Map props) { + this.block = new Block(props); + block.maybeBlockOn(TASK_START); + } + + @Override + public void put(Collection records) { + block.maybeBlockOn(SINK_TASK_PUT); + } + + @Override + public void stop() { + block.maybeBlockOn(TASK_STOP); + } + + @Override + public String version() { + block.maybeBlockOn(TASK_VERSION); + return "0.0.0"; + } + + @Override + public void initialize(SinkTaskContext context) { + block.maybeBlockOn(SINK_TASK_INITIALIZE); + super.initialize(context); + } + + @Override + public void flush(Map currentOffsets) { + block.maybeBlockOn(SINK_TASK_FLUSH); + super.flush(currentOffsets); + } + + @Override + public Map preCommit(Map currentOffsets) { + block.maybeBlockOn(SINK_TASK_PRE_COMMIT); + return super.preCommit(currentOffsets); + } + + @Override + public void open(Collection partitions) { + block.maybeBlockOn(SINK_TASK_OPEN); + super.open(partitions); + } + + @Override + @SuppressWarnings("deprecation") + public void onPartitionsAssigned(Collection partitions) { + block.maybeBlockOn(SINK_TASK_ON_PARTITIONS_ASSIGNED); + super.onPartitionsAssigned(partitions); + } + + @Override + public void close(Collection partitions) { + block.maybeBlockOn(SINK_TASK_CLOSE); + super.close(partitions); + } + + @Override + @SuppressWarnings("deprecation") + public void onPartitionsRevoked(Collection partitions) { + block.maybeBlockOn(SINK_TASK_ON_PARTITIONS_REVOKED); + super.onPartitionsRevoked(partitions); + } + } + } + + public static class TaskInitializeBlockingSinkConnector extends BlockingSinkConnector { + public TaskInitializeBlockingSinkConnector() { + super(InitializeBlockingSinkTask.class); + } + + public static class InitializeBlockingSinkTask extends BlockingSinkTask { + public InitializeBlockingSinkTask() { + super(SINK_TASK_INITIALIZE); + } } } diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/ErrorHandlingTaskTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/ErrorHandlingTaskTest.java index c471b0381cbe3..70bbfc6590cd1 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/ErrorHandlingTaskTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/ErrorHandlingTaskTest.java @@ -236,9 +236,6 @@ public void testSourceTasksCloseErrorReporters() { createSourceTask(initialState, retryWithToleranceOperator); - sourceTask.stop(); - PowerMock.expectLastCall(); - expectClose(); reporter.close(); @@ -263,9 +260,6 @@ public void testCloseErrorReportersExceptionPropagation() { createSourceTask(initialState, retryWithToleranceOperator); - sourceTask.stop(); - PowerMock.expectLastCall(); - expectClose(); // Even though the reporters throw exceptions, they should both still be closed. diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/ErrorHandlingTaskWithTopicCreationTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/ErrorHandlingTaskWithTopicCreationTest.java index aba64457dbc7c..9c115ac517821 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/ErrorHandlingTaskWithTopicCreationTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/ErrorHandlingTaskWithTopicCreationTest.java @@ -250,9 +250,6 @@ public void testSourceTasksCloseErrorReporters() { createSourceTask(initialState, retryWithToleranceOperator); - sourceTask.stop(); - PowerMock.expectLastCall(); - expectClose(); reporter.close(); @@ -277,9 +274,6 @@ public void testCloseErrorReportersExceptionPropagation() { createSourceTask(initialState, retryWithToleranceOperator); - sourceTask.stop(); - PowerMock.expectLastCall(); - expectClose(); // Even though the reporters throw exceptions, they should both still be closed. diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/util/TopicAdminTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/util/TopicAdminTest.java index dd3062e4159bc..e4b456df61663 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/util/TopicAdminTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/util/TopicAdminTest.java @@ -38,6 +38,7 @@ import org.apache.kafka.common.message.CreateTopicsResponseData.CreatableTopicResult; import org.apache.kafka.common.message.DescribeConfigsResponseData; import org.apache.kafka.common.message.MetadataResponseData; +import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.requests.ApiError; import org.apache.kafka.common.requests.CreateTopicsResponse; @@ -553,7 +554,7 @@ private MetadataResponse describeTopicResponse(ApiError error, NewTopic... topic .setName(topic.name()) .setErrorCode(error.error().code())); } - return new MetadataResponse(response); + return new MetadataResponse(response, ApiKeys.METADATA.latestVersion()); } private DescribeConfigsResponse describeConfigsResponseWithUnsupportedVersion(NewTopic... topics) { diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/EmbeddedConnectCluster.java b/connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/EmbeddedConnectCluster.java index 170ad3490974f..4c270d25f16b3 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/EmbeddedConnectCluster.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/util/clusters/EmbeddedConnectCluster.java @@ -153,8 +153,10 @@ public void stop() { log.error("Could not stop kafka", e); throw new RuntimeException("Could not stop brokers", e); } finally { - Exit.resetExitProcedure(); - Exit.resetHaltProcedure(); + if (maskExitProcedures) { + Exit.resetExitProcedure(); + Exit.resetHaltProcedure(); + } } } diff --git a/core/src/main/scala/kafka/controller/ControllerChannelManager.scala b/core/src/main/scala/kafka/controller/ControllerChannelManager.scala index 1391d345c685f..f217776109cad 100755 --- a/core/src/main/scala/kafka/controller/ControllerChannelManager.scala +++ b/core/src/main/scala/kafka/controller/ControllerChannelManager.scala @@ -271,7 +271,7 @@ class RequestSendThread(val controllerId: Int, val response = clientResponse.responseBody stateChangeLogger.withControllerEpoch(controllerContext.epoch).trace(s"Received response " + - s"${response.toString(requestHeader.apiVersion)} for request $api with correlation id " + + s"$response for request $api with correlation id " + s"${requestHeader.correlationId} sent to broker $brokerNode") if (callback != null) { diff --git a/core/src/main/scala/kafka/coordinator/transaction/TransactionMarkerRequestCompletionHandler.scala b/core/src/main/scala/kafka/coordinator/transaction/TransactionMarkerRequestCompletionHandler.scala index c2efdbd934672..848e0fa65ceeb 100644 --- a/core/src/main/scala/kafka/coordinator/transaction/TransactionMarkerRequestCompletionHandler.scala +++ b/core/src/main/scala/kafka/coordinator/transaction/TransactionMarkerRequestCompletionHandler.scala @@ -89,10 +89,11 @@ class TransactionMarkerRequestCompletionHandler(brokerId: Int, val writeTxnMarkerResponse = response.responseBody.asInstanceOf[WriteTxnMarkersResponse] + val responseErrors = writeTxnMarkerResponse.errorsByProducerId; for (txnIdAndMarker <- txnIdAndMarkerEntries.asScala) { val transactionalId = txnIdAndMarker.txnId val txnMarker = txnIdAndMarker.txnMarkerEntry - val errors = writeTxnMarkerResponse.errors(txnMarker.producerId) + val errors = responseErrors.get(txnMarker.producerId) if (errors == null) throw new IllegalStateException(s"WriteTxnMarkerResponse does not contain expected error map for producer id ${txnMarker.producerId}") diff --git a/core/src/main/scala/kafka/log/Log.scala b/core/src/main/scala/kafka/log/Log.scala index a490c97a650ae..9677dc4f6c7a7 100644 --- a/core/src/main/scala/kafka/log/Log.scala +++ b/core/src/main/scala/kafka/log/Log.scala @@ -770,9 +770,7 @@ class Log(@volatile private var _dir: File, baseOffset = 0, config, time = time, - fileAlreadyExists = false, - initFileSize = this.initFileSize, - preallocate = false)) + initFileSize = this.initFileSize)) } 0 } @@ -858,7 +856,6 @@ class Log(@volatile private var _dir: File, baseOffset = logStartOffset, config, time = time, - fileAlreadyExists = false, initFileSize = this.initFileSize, preallocate = config.preallocate)) } @@ -2124,7 +2121,6 @@ class Log(@volatile private var _dir: File, baseOffset = newOffset, config = config, time = time, - fileAlreadyExists = false, initFileSize = initFileSize, preallocate = config.preallocate)) updateLogEndOffset(newOffset) diff --git a/core/src/main/scala/kafka/network/RequestChannel.scala b/core/src/main/scala/kafka/network/RequestChannel.scala index 3945e8e15c17c..39ca69f1f1893 100644 --- a/core/src/main/scala/kafka/network/RequestChannel.scala +++ b/core/src/main/scala/kafka/network/RequestChannel.scala @@ -20,7 +20,6 @@ package kafka.network import java.net.InetAddress import java.nio.ByteBuffer import java.util.concurrent._ - import com.typesafe.scalalogging.Logger import com.yammer.metrics.core.Meter import kafka.log.LogConfig @@ -34,7 +33,7 @@ import org.apache.kafka.common.memory.MemoryPool import org.apache.kafka.common.message.IncrementalAlterConfigsRequestData import org.apache.kafka.common.message.IncrementalAlterConfigsRequestData._ import org.apache.kafka.common.network.Send -import org.apache.kafka.common.protocol.{ApiKeys, Errors} +import org.apache.kafka.common.protocol.{ApiKeys, Errors, ObjectSerializationCache} import org.apache.kafka.common.requests._ import org.apache.kafka.common.security.auth.KafkaPrincipal import org.apache.kafka.common.utils.{Sanitizer, Time} @@ -99,11 +98,14 @@ object RequestChannel extends Logging { private val bodyAndSize: RequestAndSize = context.parseRequest(buffer) def header: RequestHeader = context.header + def sizeOfBodyInBytes: Int = bodyAndSize.size - // most request types are parsed entirely into objects at this point. for those we can release the underlying buffer. - // some (like produce, or any time the schema contains fields of types BYTES or NULLABLE_BYTES) retain a reference - // to the buffer. for those requests we cannot release the buffer early, but only when request processing is done. + def sizeInBytes: Int = header.size(new ObjectSerializationCache) + sizeOfBodyInBytes + + //most request types are parsed entirely into objects at this point. for those we can release the underlying buffer. + //some (like produce, or any time the schema contains fields of types BYTES or NULLABLE_BYTES) retain a reference + //to the buffer. for those requests we cannot release the buffer early, but only when request processing is done. if (!header.apiKey.requiresDelayedAllocation) { releaseBuffer() } @@ -123,7 +125,7 @@ object RequestChannel extends Logging { def responseString(response: AbstractResponse): Option[String] = { if (RequestChannel.isRequestLoggingEnabled) - Some(response.toString(context.apiVersion)) + Some(response.toString) else None } diff --git a/core/src/main/scala/kafka/raft/KafkaMetadataLog.scala b/core/src/main/scala/kafka/raft/KafkaMetadataLog.scala index c5225953b1500..039b299481dd4 100644 --- a/core/src/main/scala/kafka/raft/KafkaMetadataLog.scala +++ b/core/src/main/scala/kafka/raft/KafkaMetadataLog.scala @@ -16,6 +16,7 @@ */ package kafka.raft +import java.nio.file.NoSuchFileException import java.util.Optional import kafka.log.{AppendOrigin, Log} @@ -24,6 +25,10 @@ import org.apache.kafka.common.record.{MemoryRecords, Records} import org.apache.kafka.common.{KafkaException, TopicPartition} import org.apache.kafka.raft import org.apache.kafka.raft.{LogAppendInfo, LogFetchInfo, LogOffsetMetadata, Isolation, ReplicatedLog} +import org.apache.kafka.snapshot.FileRawSnapshotReader +import org.apache.kafka.snapshot.FileRawSnapshotWriter +import org.apache.kafka.snapshot.RawSnapshotReader +import org.apache.kafka.snapshot.RawSnapshotWriter import scala.compat.java8.OptionConverters._ @@ -141,6 +146,18 @@ class KafkaMetadataLog( topicPartition } + override def createSnapshot(snapshotId: raft.OffsetAndEpoch): RawSnapshotWriter = { + FileRawSnapshotWriter.create(log.dir.toPath, snapshotId) + } + + override def readSnapshot(snapshotId: raft.OffsetAndEpoch): Optional[RawSnapshotReader] = { + try { + Optional.of(FileRawSnapshotReader.open(log.dir.toPath, snapshotId)) + } catch { + case e: NoSuchFileException => Optional.empty() + } + } + override def close(): Unit = { log.close() } diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala index 072ea5304bbbb..6aa71505acf4f 100644 --- a/core/src/main/scala/kafka/server/KafkaApis.scala +++ b/core/src/main/scala/kafka/server/KafkaApis.scala @@ -24,7 +24,6 @@ import java.util import java.util.{Collections, Optional} import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.atomic.AtomicInteger - import kafka.admin.{AdminUtils, RackAwareMode} import kafka.api.ElectLeadersRequestOps import kafka.api.{ApiVersion, KAFKA_0_11_0_IV0, KAFKA_2_3_IV0} @@ -51,7 +50,7 @@ import org.apache.kafka.common.internals.{FatalExitError, Topic} import org.apache.kafka.common.internals.Topic.{GROUP_METADATA_TOPIC_NAME, TRANSACTION_STATE_TOPIC_NAME, isInternal} import org.apache.kafka.common.message.CreateTopicsRequestData.CreatableTopic import org.apache.kafka.common.message.CreatePartitionsResponseData.CreatePartitionsTopicResult -import org.apache.kafka.common.message.{AddOffsetsToTxnResponseData, AlterConfigsResponseData, AlterPartitionReassignmentsResponseData, AlterReplicaLogDirsResponseData, CreateAclsResponseData, CreatePartitionsResponseData, CreateTopicsResponseData, DeleteAclsResponseData, DeleteGroupsResponseData, DeleteRecordsResponseData, DeleteTopicsResponseData, DescribeAclsResponseData, DescribeConfigsResponseData, DescribeGroupsResponseData, DescribeLogDirsResponseData, EndTxnResponseData, ExpireDelegationTokenResponseData, FindCoordinatorResponseData, HeartbeatResponseData, InitProducerIdResponseData, JoinGroupResponseData, LeaveGroupResponseData, ListGroupsResponseData, ListOffsetResponseData, ListPartitionReassignmentsResponseData, MetadataResponseData, OffsetCommitRequestData, OffsetCommitResponseData, OffsetDeleteResponseData, RenewDelegationTokenResponseData, SaslAuthenticateResponseData, SaslHandshakeResponseData, StopReplicaResponseData, SyncGroupResponseData, UpdateMetadataResponseData} +import org.apache.kafka.common.message.{AddOffsetsToTxnResponseData, AlterClientQuotasResponseData, AlterConfigsResponseData, AlterPartitionReassignmentsResponseData, AlterReplicaLogDirsResponseData, CreateAclsResponseData, CreatePartitionsResponseData, CreateTopicsResponseData, DeleteAclsResponseData, DeleteGroupsResponseData, DeleteRecordsResponseData, DeleteTopicsResponseData, DescribeAclsResponseData, DescribeClientQuotasResponseData, DescribeConfigsResponseData, DescribeGroupsResponseData, DescribeLogDirsResponseData, EndTxnResponseData, ExpireDelegationTokenResponseData, FindCoordinatorResponseData, HeartbeatResponseData, InitProducerIdResponseData, JoinGroupResponseData, LeaveGroupResponseData, ListGroupsResponseData, ListOffsetResponseData, ListPartitionReassignmentsResponseData, MetadataResponseData, OffsetCommitRequestData, OffsetCommitResponseData, OffsetDeleteResponseData, RenewDelegationTokenResponseData, SaslAuthenticateResponseData, SaslHandshakeResponseData, StopReplicaResponseData, SyncGroupResponseData, UpdateMetadataResponseData} import org.apache.kafka.common.message.CreateTopicsResponseData.{CreatableTopicResult, CreatableTopicResultCollection} import org.apache.kafka.common.message.DeleteGroupsResponseData.{DeletableGroupResult, DeletableGroupResultCollection} import org.apache.kafka.common.message.AlterPartitionReassignmentsResponseData.{ReassignablePartitionResponse, ReassignableTopicResponse} @@ -569,7 +568,7 @@ class KafkaApis(val requestChannel: RequestChannel, */ def handleProduceRequest(request: RequestChannel.Request): Unit = { val produceRequest = request.body[ProduceRequest] - val numBytesAppended = request.header.toStruct.sizeOf + request.sizeOfBodyInBytes + val requestSize = request.sizeInBytes val (hasIdempotentRecords, hasTransactionalRecords) = { val flags = RequestUtils.flags(produceRequest) @@ -595,9 +594,9 @@ class KafkaApis(val requestChannel: RequestChannel, val authorizedRequestInfo = mutable.Map[TopicPartition, MemoryRecords]() // cache the result to avoid redundant authorization calls val authorizedTopics = filterByAuthorized(request.context, WRITE, TOPIC, - produceRequest.dataOrException().topicData().asScala)(_.name()) + produceRequest.data().topicData().asScala)(_.name()) - produceRequest.dataOrException.topicData.forEach(topic => topic.partitionData.forEach { partition => + produceRequest.data.topicData.forEach(topic => topic.partitionData.forEach { partition => val topicPartition = new TopicPartition(topic.name, partition.index) // This caller assumes the type is MemoryRecords and that is true on current serialization // We cast the type to avoid causing big change to code base. @@ -641,7 +640,7 @@ class KafkaApis(val requestChannel: RequestChannel, // have been violated. If both quotas have been violated, use the max throttle time between the two quotas. Note // that the request quota is not enforced if acks == 0. val timeMs = time.milliseconds() - val bandwidthThrottleTimeMs = quotas.produce.maybeRecordAndGetThrottleTimeMs(request, numBytesAppended, timeMs) + val bandwidthThrottleTimeMs = quotas.produce.maybeRecordAndGetThrottleTimeMs(request, requestSize, timeMs) val requestThrottleTimeMs = if (produceRequest.acks == 0) 0 else quotas.request.maybeRecordAndGetThrottleTimeMs(request, timeMs) @@ -1345,11 +1344,12 @@ class KafkaApis(val requestChannel: RequestChannel, sendResponseMaybeThrottle(request, requestThrottleMs => MetadataResponse.prepareResponse( + requestVersion, requestThrottleMs, - completeTopicMetadata.asJava, brokers.flatMap(_.getNode(request.context.listenerName)).asJava, clusterId, metadataCache.getControllerId.getOrElse(MetadataResponse.NO_CONTROLLER_ID), + completeTopicMetadata.asJava, clusterAuthorizedOperations )) } @@ -2506,7 +2506,8 @@ class KafkaApis(val requestChannel: RequestChannel, new DescribeAclsResponse(new DescribeAclsResponseData() .setErrorCode(Errors.SECURITY_DISABLED.code) .setErrorMessage("No Authorizer is configured on the broker") - .setThrottleTimeMs(requestThrottleMs))) + .setThrottleTimeMs(requestThrottleMs), + describeAclsRequest.version)) case Some(auth) => val filter = describeAclsRequest.filter val returnedAcls = new util.HashSet[AclBinding]() @@ -2514,7 +2515,8 @@ class KafkaApis(val requestChannel: RequestChannel, sendResponseMaybeThrottle(request, requestThrottleMs => new DescribeAclsResponse(new DescribeAclsResponseData() .setThrottleTimeMs(requestThrottleMs) - .setResources(DescribeAclsResponse.aclsResources(returnedAcls)))) + .setResources(DescribeAclsResponse.aclsResources(returnedAcls)), + describeAclsRequest.version)) } } @@ -2585,9 +2587,11 @@ class KafkaApis(val requestChannel: RequestChannel, def sendResponseCallback(): Unit = { val filterResults = deleteResults.map(_.get).map(DeleteAclsResponse.filterResult).asJava sendResponseMaybeThrottle(request, requestThrottleMs => - new DeleteAclsResponse(new DeleteAclsResponseData() - .setThrottleTimeMs(requestThrottleMs) - .setFilterResults(filterResults))) + new DeleteAclsResponse( + new DeleteAclsResponseData() + .setThrottleTimeMs(requestThrottleMs) + .setFilterResults(filterResults), + deleteAclsRequest.version)) } alterAclsPurgatory.tryCompleteElseWatch(config.connectionsMaxIdleMs, deleteResults, sendResponseCallback) } @@ -3125,11 +3129,30 @@ class KafkaApis(val requestChannel: RequestChannel, val describeClientQuotasRequest = request.body[DescribeClientQuotasRequest] if (authorize(request.context, DESCRIBE_CONFIGS, CLUSTER, CLUSTER_NAME)) { - val result = adminManager.describeClientQuotas(describeClientQuotasRequest.filter).map { case (quotaEntity, quotaConfigs) => - quotaEntity -> quotaConfigs.map { case (key, value) => key -> Double.box(value) }.asJava - }.asJava + val result = adminManager.describeClientQuotas(describeClientQuotasRequest.filter) + + val entriesData = result.iterator.map { case (quotaEntity, quotaValues) => + val entityData = quotaEntity.entries.asScala.iterator.map { case (entityType, entityName) => + new DescribeClientQuotasResponseData.EntityData() + .setEntityType(entityType) + .setEntityName(entityName) + }.toBuffer + + val valueData = quotaValues.iterator.map { case (key, value) => + new DescribeClientQuotasResponseData.ValueData() + .setKey(key) + .setValue(value) + }.toBuffer + + new DescribeClientQuotasResponseData.EntryData() + .setEntity(entityData.asJava) + .setValues(valueData.asJava) + }.toBuffer + sendResponseMaybeThrottle(request, requestThrottleMs => - new DescribeClientQuotasResponse(result, requestThrottleMs)) + new DescribeClientQuotasResponse(new DescribeClientQuotasResponseData() + .setThrottleTimeMs(requestThrottleMs) + .setEntries(entriesData.asJava))) } else { sendResponseMaybeThrottle(request, requestThrottleMs => describeClientQuotasRequest.getErrorResponse(requestThrottleMs, Errors.CLUSTER_AUTHORIZATION_FAILED.exception)) @@ -3140,10 +3163,26 @@ class KafkaApis(val requestChannel: RequestChannel, val alterClientQuotasRequest = request.body[AlterClientQuotasRequest] if (authorize(request.context, ALTER_CONFIGS, CLUSTER, CLUSTER_NAME)) { - val result = adminManager.alterClientQuotas(alterClientQuotasRequest.entries().asScala.toSeq, - alterClientQuotasRequest.validateOnly()).asJava + val result = adminManager.alterClientQuotas(alterClientQuotasRequest.entries.asScala, + alterClientQuotasRequest.validateOnly) + + val entriesData = result.iterator.map { case (quotaEntity, apiError) => + val entityData = quotaEntity.entries.asScala.iterator.map { case (key, value) => + new AlterClientQuotasResponseData.EntityData() + .setEntityType(key) + .setEntityName(value) + }.toBuffer + + new AlterClientQuotasResponseData.EntryData() + .setErrorCode(apiError.error.code) + .setErrorMessage(apiError.message) + .setEntity(entityData.asJava) + }.toBuffer + sendResponseMaybeThrottle(request, requestThrottleMs => - new AlterClientQuotasResponse(result, requestThrottleMs)) + new AlterClientQuotasResponse(new AlterClientQuotasResponseData() + .setThrottleTimeMs(requestThrottleMs) + .setEntries(entriesData.asJava))) } else { sendResponseMaybeThrottle(request, requestThrottleMs => alterClientQuotasRequest.getErrorResponse(requestThrottleMs, Errors.CLUSTER_AUTHORIZATION_FAILED.exception)) diff --git a/core/src/main/scala/kafka/tools/StreamsResetter.java b/core/src/main/scala/kafka/tools/StreamsResetter.java index d356681e11242..f72a3b6905400 100644 --- a/core/src/main/scala/kafka/tools/StreamsResetter.java +++ b/core/src/main/scala/kafka/tools/StreamsResetter.java @@ -489,19 +489,7 @@ private Map getTopicPartitionOffsetFromResetPlan(final Str private void resetByDuration(final Consumer client, final Set inputTopicPartitions, final Duration duration) { - final Instant now = Instant.now(); - final long timestamp = now.minus(duration).toEpochMilli(); - - final Map topicPartitionsAndTimes = new HashMap<>(inputTopicPartitions.size()); - for (final TopicPartition topicPartition : inputTopicPartitions) { - topicPartitionsAndTimes.put(topicPartition, timestamp); - } - - final Map topicPartitionsAndOffset = client.offsetsForTimes(topicPartitionsAndTimes); - - for (final TopicPartition topicPartition : inputTopicPartitions) { - client.seek(topicPartition, topicPartitionsAndOffset.get(topicPartition).offset()); - } + resetToDatetime(client, inputTopicPartitions, Instant.now().minus(duration).toEpochMilli()); } private void resetToDatetime(final Consumer client, diff --git a/core/src/main/scala/kafka/tools/TestRaftRequestHandler.scala b/core/src/main/scala/kafka/tools/TestRaftRequestHandler.scala index a8f7cd93838c3..fefd274927df2 100644 --- a/core/src/main/scala/kafka/tools/TestRaftRequestHandler.scala +++ b/core/src/main/scala/kafka/tools/TestRaftRequestHandler.scala @@ -94,7 +94,7 @@ class TestRaftRequestHandler( case Some(response) => val responseSend = request.context.buildResponseSend(response) val responseString = - if (RequestChannel.isRequestLoggingEnabled) Some(response.toString(request.context.apiVersion)) + if (RequestChannel.isRequestLoggingEnabled) Some(response.toString) else None new RequestChannel.SendResponse(request, responseSend, responseString, None) case None => diff --git a/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala index 26fec0cc81df9..e8cce7abe8118 100644 --- a/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala +++ b/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala @@ -196,7 +196,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest { ApiKeys.ALTER_CONFIGS -> ((resp: AlterConfigsResponse) => resp.errors.get(new ConfigResource(ConfigResource.Type.TOPIC, tp.topic)).error), ApiKeys.INIT_PRODUCER_ID -> ((resp: InitProducerIdResponse) => resp.error), - ApiKeys.WRITE_TXN_MARKERS -> ((resp: WriteTxnMarkersResponse) => resp.errors(producerId).get(tp)), + ApiKeys.WRITE_TXN_MARKERS -> ((resp: WriteTxnMarkersResponse) => resp.errorsByProducerId.get(producerId).get(tp)), ApiKeys.ADD_PARTITIONS_TO_TXN -> ((resp: AddPartitionsToTxnResponse) => resp.errors.get(tp)), ApiKeys.ADD_OFFSETS_TO_TXN -> ((resp: AddOffsetsToTxnResponse) => Errors.forCode(resp.data.errorCode)), ApiKeys.END_TXN -> ((resp: EndTxnResponse) => resp.error), @@ -1808,7 +1808,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest { resources: Set[ResourceType], isAuthorized: Boolean, topicExists: Boolean = true): AbstractResponse = { - val apiKey = request.api + val apiKey = request.apiKey val response = connectAndReceive[AbstractResponse](request) val error = requestKeyToError(apiKey).asInstanceOf[AbstractResponse => Errors](response) diff --git a/core/src/test/scala/unit/kafka/network/RequestChannelTest.scala b/core/src/test/scala/unit/kafka/network/RequestChannelTest.scala index 18b838232048f..aa1abb4026557 100644 --- a/core/src/test/scala/unit/kafka/network/RequestChannelTest.scala +++ b/core/src/test/scala/unit/kafka/network/RequestChannelTest.scala @@ -165,7 +165,7 @@ class RequestChannelTest { } def request(req: AbstractRequest): RequestChannel.Request = { - val buffer = req.serialize(new RequestHeader(req.api, req.version, "client-id", 1)) + val buffer = req.serializeWithHeader(new RequestHeader(req.apiKey, req.version, "client-id", 1)) val requestContext = newRequestContext(buffer) new network.RequestChannel.Request(processor = 1, requestContext, @@ -195,4 +195,4 @@ class RequestChannelTest { private def toMap(config: IncrementalAlterConfigsRequestData.AlterableConfigCollection): Map[String, String] = { config.asScala.map(e => e.name -> e.value).toMap } -} \ No newline at end of file +} diff --git a/core/src/test/scala/unit/kafka/network/SocketServerTest.scala b/core/src/test/scala/unit/kafka/network/SocketServerTest.scala index d8b76e05a9217..49d1490c96e75 100644 --- a/core/src/test/scala/unit/kafka/network/SocketServerTest.scala +++ b/core/src/test/scala/unit/kafka/network/SocketServerTest.scala @@ -112,7 +112,7 @@ class SocketServerTest { } def sendApiRequest(socket: Socket, request: AbstractRequest, header: RequestHeader) = { - val byteBuffer = request.serialize(header) + val byteBuffer = request.serializeWithHeader(header) byteBuffer.rewind() val serializedBytes = new Array[Byte](byteBuffer.remaining) byteBuffer.get(serializedBytes) @@ -141,7 +141,7 @@ class SocketServerTest { } def processRequest(channel: RequestChannel, request: RequestChannel.Request): Unit = { - val byteBuffer = request.body[AbstractRequest].serialize(request.header) + val byteBuffer = request.body[AbstractRequest].serializeWithHeader(request.header) byteBuffer.rewind() val send = new NetworkSend(request.context.connectionId, byteBuffer) @@ -214,7 +214,7 @@ class SocketServerTest { .setTransactionalId(null)) .build() val emptyHeader = new RequestHeader(ApiKeys.PRODUCE, emptyRequest.version, clientId, correlationId) - val byteBuffer = emptyRequest.serialize(emptyHeader) + val byteBuffer = emptyRequest.serializeWithHeader(emptyHeader) byteBuffer.rewind() val serializedBytes = new Array[Byte](byteBuffer.remaining) @@ -225,7 +225,7 @@ class SocketServerTest { private def apiVersionRequestBytes(clientId: String, version: Short): Array[Byte] = { val request = new ApiVersionsRequest.Builder().build(version) val header = new RequestHeader(ApiKeys.API_VERSIONS, request.version(), clientId, -1) - val buffer = request.serialize(header) + val buffer = request.serializeWithHeader(header) buffer.rewind() val bytes = new Array[Byte](buffer.remaining()) buffer.get(bytes) @@ -378,7 +378,7 @@ class SocketServerTest { val correlationId = 57 val header = new RequestHeader(ApiKeys.VOTE, 0, "", correlationId) val request = new VoteRequest.Builder(new VoteRequestData()).build() - val byteBuffer = request.serialize(header) + val byteBuffer = request.serializeWithHeader(header) byteBuffer.rewind() val socket = connect() @@ -687,7 +687,7 @@ class SocketServerTest { // Mimic a primitive request handler that fetches the request from RequestChannel and place a response with a // throttled channel. val request = receiveRequest(server.dataPlaneRequestChannel) - val byteBuffer = request.body[AbstractRequest].serialize(request.header) + val byteBuffer = request.body[AbstractRequest].serializeWithHeader(request.header) val send = new NetworkSend(request.context.connectionId, byteBuffer) def channelThrottlingCallback(response: RequestChannel.Response): Unit = { server.dataPlaneRequestChannel.sendResponse(response) @@ -967,7 +967,7 @@ class SocketServerTest { .build() val emptyHeader = new RequestHeader(ApiKeys.PRODUCE, emptyRequest.version, clientId, correlationId) - val byteBuffer = emptyRequest.serialize(emptyHeader) + val byteBuffer = emptyRequest.serializeWithHeader(emptyHeader) byteBuffer.rewind() val serializedBytes = new Array[Byte](byteBuffer.remaining) byteBuffer.get(serializedBytes) diff --git a/core/src/test/scala/unit/kafka/server/AbstractCreateTopicsRequestTest.scala b/core/src/test/scala/unit/kafka/server/AbstractCreateTopicsRequestTest.scala index cfe799ac9e717..10f73a88ae40d 100644 --- a/core/src/test/scala/unit/kafka/server/AbstractCreateTopicsRequestTest.scala +++ b/core/src/test/scala/unit/kafka/server/AbstractCreateTopicsRequestTest.scala @@ -24,7 +24,6 @@ import kafka.network.SocketServer import kafka.utils.TestUtils import org.apache.kafka.common.message.CreateTopicsRequestData import org.apache.kafka.common.message.CreateTopicsRequestData._ -import org.apache.kafka.common.protocol.types.Struct import org.apache.kafka.common.protocol.Errors import org.apache.kafka.common.requests._ import org.junit.Assert.{assertEquals, assertFalse, assertNotNull, assertTrue} @@ -151,15 +150,6 @@ abstract class AbstractCreateTopicsRequestTest extends BaseRequestTest { protected def error(error: Errors, errorMessage: Option[String] = None): ApiError = new ApiError(error, errorMessage.orNull) - protected def addPartitionsAndReplicationFactorToFirstTopic(request: CreateTopicsRequest) = { - val struct = request.toStruct - val topics = struct.getArray("create_topic_requests") - val firstTopic = topics(0).asInstanceOf[Struct] - firstTopic.set("num_partitions", 1) - firstTopic.set("replication_factor", 1.toShort) - new CreateTopicsRequest(struct, request.version) - } - protected def validateErrorCreateTopicsRequests(request: CreateTopicsRequest, expectedResponse: Map[String, ApiError], checkErrorMessage: Boolean = true): Unit = { diff --git a/core/src/test/scala/unit/kafka/server/BaseClientQuotaManagerTest.scala b/core/src/test/scala/unit/kafka/server/BaseClientQuotaManagerTest.scala index 3a4dd9487d181..d9f64e8e416f7 100644 --- a/core/src/test/scala/unit/kafka/server/BaseClientQuotaManagerTest.scala +++ b/core/src/test/scala/unit/kafka/server/BaseClientQuotaManagerTest.scala @@ -63,7 +63,7 @@ class BaseClientQuotaManagerTest { listenerName: ListenerName = ListenerName.forSecurityProtocol(SecurityProtocol.PLAINTEXT)): (T, RequestChannel.Request) = { val request = builder.build() - val buffer = request.serialize(new RequestHeader(builder.apiKey, request.version, "", 0)) + val buffer = request.serializeWithHeader(new RequestHeader(builder.apiKey, request.version, "", 0)) val requestChannelMetrics: RequestChannel.Metrics = EasyMock.createNiceMock(classOf[RequestChannel.Metrics]) // read the header from the buffer first so that the body can be read next from the Request constructor diff --git a/core/src/test/scala/unit/kafka/server/BaseRequestTest.scala b/core/src/test/scala/unit/kafka/server/BaseRequestTest.scala index bf54cdb19157f..a78b37e6a50ba 100644 --- a/core/src/test/scala/unit/kafka/server/BaseRequestTest.scala +++ b/core/src/test/scala/unit/kafka/server/BaseRequestTest.scala @@ -97,8 +97,7 @@ abstract class BaseRequestTest extends IntegrationTestHarness { val responseBuffer = ByteBuffer.wrap(responseBytes) ResponseHeader.parse(responseBuffer, apiKey.responseHeaderVersion(version)) - val responseStruct = apiKey.parseResponse(version, responseBuffer) - AbstractResponse.parseResponse(apiKey, responseStruct, version) match { + AbstractResponse.parseResponse(apiKey, responseBuffer, version) match { case response: T => response case response => throw new ClassCastException(s"Expected response with type ${classTag.runtimeClass}, but found ${response.getClass}") @@ -111,7 +110,7 @@ abstract class BaseRequestTest extends IntegrationTestHarness { correlationId: Option[Int] = None) (implicit classTag: ClassTag[T], nn: NotNothing[T]): T = { send(request, socket, clientId, correlationId) - receive[T](socket, request.api, request.version) + receive[T](socket, request.apiKey, request.version) } def connectAndReceive[T <: AbstractResponse](request: AbstractRequest, @@ -130,12 +129,12 @@ abstract class BaseRequestTest extends IntegrationTestHarness { socket: Socket, clientId: String = "client-id", correlationId: Option[Int] = None): Unit = { - val header = nextRequestHeader(request.api, request.version, clientId, correlationId) + val header = nextRequestHeader(request.apiKey, request.version, clientId, correlationId) sendWithHeader(request, header, socket) } def sendWithHeader(request: AbstractRequest, header: RequestHeader, socket: Socket): Unit = { - val serializedBytes = request.serialize(header).array + val serializedBytes = request.serializeWithHeader(header).array sendRequest(socket, serializedBytes) } diff --git a/core/src/test/scala/unit/kafka/server/EdgeCaseRequestTest.scala b/core/src/test/scala/unit/kafka/server/EdgeCaseRequestTest.scala index 2a3cd0f611c37..2ed64509c459a 100755 --- a/core/src/test/scala/unit/kafka/server/EdgeCaseRequestTest.scala +++ b/core/src/test/scala/unit/kafka/server/EdgeCaseRequestTest.scala @@ -139,10 +139,11 @@ class EdgeCaseRequestTest extends KafkaServerTestHarness { .setTimeoutMs(10000) .setTransactionalId(null)) .build() - val byteBuffer = ByteBuffer.allocate(headerBytes.length + request.toStruct.sizeOf) + val bodyBytes = request.serializeBody + val byteBuffer = ByteBuffer.allocate(headerBytes.length + bodyBytes.remaining()) byteBuffer.put(headerBytes) - request.toStruct.writeTo(byteBuffer) - (byteBuffer.array(), request.api.responseHeaderVersion(version)) + byteBuffer.put(bodyBytes) + (byteBuffer.array(), request.apiKey.responseHeaderVersion(version)) } val response = requestAndReceive(serializedBytes) diff --git a/core/src/test/scala/unit/kafka/server/ForwardingManagerTest.scala b/core/src/test/scala/unit/kafka/server/ForwardingManagerTest.scala index 4d9451e2a31b4..b6624d6c2a205 100644 --- a/core/src/test/scala/unit/kafka/server/ForwardingManagerTest.scala +++ b/core/src/test/scala/unit/kafka/server/ForwardingManagerTest.scala @@ -60,8 +60,7 @@ class ForwardingManagerTest { val request = buildRequest(requestHeader, requestBuffer, clientPrincipal) val responseBody = new AlterConfigsResponse(new AlterConfigsResponseData()) - val responseBuffer = responseBody.serialize(ApiKeys.ALTER_CONFIGS, - requestBody.version, requestCorrelationId + 1) + val responseBuffer = responseBody.serializeWithHeader(requestBody.version, requestCorrelationId + 1) Mockito.when(brokerToController.sendRequest( any(classOf[EnvelopeRequest.Builder]), @@ -113,12 +112,12 @@ class ForwardingManagerTest { correlationId: Int ): (RequestHeader, ByteBuffer) = { val header = new RequestHeader( - body.api, + body.apiKey, body.version, "clientId", correlationId ) - val buffer = body.serialize(header) + val buffer = body.serializeWithHeader(header) // Fast-forward buffer to start of the request as `RequestChannel.Request` expects RequestHeader.parse(buffer) diff --git a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala index 5cc7f4bc66974..b89fb2efdb1b3 100644 --- a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala +++ b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala @@ -352,7 +352,7 @@ class KafkaApisTest { configResource -> new AlterConfigsRequest.Config( Seq(new AlterConfigsRequest.ConfigEntry("foo", "bar")).asJava)) val alterConfigsRequest = new AlterConfigsRequest.Builder(configs.asJava, false).build(requestHeader.apiVersion) - val serializedRequestData = alterConfigsRequest.serialize(requestHeader) + val serializedRequestData = alterConfigsRequest.serializeWithHeader(requestHeader) val capturedResponse = expectNoThrottling() @@ -375,7 +375,7 @@ class KafkaApisTest { clientId, 0) val leaveGroupRequest = new LeaveGroupRequest.Builder("group", Collections.singletonList(new MemberIdentity())).build(requestHeader.apiVersion) - val serializedRequestData = leaveGroupRequest.serialize(requestHeader) + val serializedRequestData = leaveGroupRequest.serializeWithHeader(requestHeader) val capturedResponse = expectNoThrottling() @@ -1358,7 +1358,7 @@ class KafkaApisTest { val markersResponse = readResponse(writeTxnMarkersRequest, capturedResponse) .asInstanceOf[WriteTxnMarkersResponse] - assertEquals(expectedErrors, markersResponse.errors(1)) + assertEquals(expectedErrors, markersResponse.errorsByProducerId.get(1L)) } @Test @@ -1377,7 +1377,7 @@ class KafkaApisTest { val markersResponse = readResponse(writeTxnMarkersRequest, capturedResponse) .asInstanceOf[WriteTxnMarkersResponse] - assertEquals(expectedErrors, markersResponse.errors(1)) + assertEquals(expectedErrors, markersResponse.errorsByProducerId.get(1L)) } @Test @@ -1412,7 +1412,7 @@ class KafkaApisTest { val markersResponse = readResponse(writeTxnMarkersRequest, capturedResponse) .asInstanceOf[WriteTxnMarkersResponse] - assertEquals(expectedErrors, markersResponse.errors(1)) + assertEquals(expectedErrors, markersResponse.errorsByProducerId.get(1L)) EasyMock.verify(replicaManager) } @@ -1549,7 +1549,7 @@ class KafkaApisTest { val markersResponse = readResponse(writeTxnMarkersRequest, capturedResponse) .asInstanceOf[WriteTxnMarkersResponse] - assertEquals(expectedErrors, markersResponse.errors(1)) + assertEquals(expectedErrors, markersResponse.errorsByProducerId.get(1L)) EasyMock.verify(replicaManager) } @@ -2826,8 +2826,8 @@ class KafkaApisTest { ): RequestChannel.Request = { val listenerName = ListenerName.forSecurityProtocol(SecurityProtocol.PLAINTEXT) - val requestHeader = new RequestHeader(request.api, request.version, clientId, 0) - val requestBuffer = request.serialize(requestHeader) + val requestHeader = new RequestHeader(request.apiKey, request.version, clientId, 0) + val requestBuffer = request.serializeWithHeader(requestHeader) val requestContext = new RequestContext(requestHeader, "1", InetAddress.getLocalHost, KafkaPrincipal.ANONYMOUS, listenerName, SecurityProtocol.PLAINTEXT, ClientInformation.EMPTY, fromPrivilegedListener) @@ -2837,7 +2837,7 @@ class KafkaApisTest { requestBuffer, new Array[Byte](0), InetAddress.getLocalHost.getAddress - ).build().serialize(envelopeHeader) + ).build().serializeWithHeader(envelopeHeader) val envelopeContext = new RequestContext(envelopeHeader, "1", InetAddress.getLocalHost, KafkaPrincipal.ANONYMOUS, listenerName, SecurityProtocol.PLAINTEXT, ClientInformation.EMPTY, fromPrivilegedListener, principalSerde.asJava) @@ -2868,8 +2868,8 @@ class KafkaApisTest { listenerName: ListenerName = ListenerName.forSecurityProtocol(SecurityProtocol.PLAINTEXT), fromPrivilegedListener: Boolean = false, requestHeader: Option[RequestHeader] = None): RequestChannel.Request = { - val buffer = request.serialize(requestHeader.getOrElse( - new RequestHeader(request.api, request.version, clientId, 0))) + val buffer = request.serializeWithHeader(requestHeader.getOrElse( + new RequestHeader(request.apiKey, request.version, clientId, 0))) // read the header from the buffer first so that the body can be read next from the Request constructor val header = RequestHeader.parse(buffer) @@ -2881,7 +2881,7 @@ class KafkaApisTest { } private def readResponse(request: AbstractRequest, capturedResponse: Capture[RequestChannel.Response]) = { - val api = request.api + val api = request.apiKey val response = capturedResponse.getValue assertTrue(s"Unexpected response type: ${response.getClass}", response.isInstanceOf[SendResponse]) val sendResponse = response.asInstanceOf[SendResponse] @@ -2891,8 +2891,7 @@ class KafkaApisTest { channel.close() channel.buffer.getInt() // read the size ResponseHeader.parse(channel.buffer, api.responseHeaderVersion(request.version)) - val struct = api.responseSchema(request.version).read(channel.buffer) - AbstractResponse.parseResponse(api, struct, request.version) + AbstractResponse.parseResponse(api, channel.buffer, request.version) } private def expectNoThrottling(): Capture[RequestChannel.Response] = { diff --git a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala index ccd8904198c7b..aedb5194e2870 100644 --- a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala +++ b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala @@ -610,7 +610,7 @@ class RequestQuotaTest extends BaseRequestTest { 0 ) val embedRequestData = new AlterClientQuotasRequest.Builder( - List.empty.asJava, false).build().serialize(requestHeader) + List.empty.asJava, false).build().serializeWithHeader(requestHeader) new EnvelopeRequest.Builder(embedRequestData, new Array[Byte](0), InetAddress.getByName("192.168.1.1").getAddress) @@ -670,11 +670,9 @@ class RequestQuotaTest extends BaseRequestTest { } private def checkRequestThrottleTime(apiKey: ApiKeys): Unit = { - // Request until throttled using client-id with default small quota val clientId = apiKey.toString val client = Client(clientId, apiKey) - val throttled = client.runUntil(_.throttleTimeMs > 0) assertTrue(s"Response not throttled: $client", throttled) diff --git a/core/src/test/scala/unit/kafka/server/SaslApiVersionsRequestTest.scala b/core/src/test/scala/unit/kafka/server/SaslApiVersionsRequestTest.scala index 6d1f8cafeae18..ca276d982d538 100644 --- a/core/src/test/scala/unit/kafka/server/SaslApiVersionsRequestTest.scala +++ b/core/src/test/scala/unit/kafka/server/SaslApiVersionsRequestTest.scala @@ -18,11 +18,10 @@ package kafka.server import java.net.Socket import java.util.Collections - import kafka.api.{KafkaSasl, SaslSetup} import kafka.utils.JaasTestUtils import org.apache.kafka.common.message.SaslHandshakeRequestData -import org.apache.kafka.common.protocol.Errors +import org.apache.kafka.common.protocol.{ApiKeys, Errors} import org.apache.kafka.common.requests.{ApiVersionsRequest, ApiVersionsResponse, SaslHandshakeRequest, SaslHandshakeResponse} import org.apache.kafka.common.security.auth.SecurityProtocol import org.junit.Assert._ @@ -91,7 +90,8 @@ class SaslApiVersionsRequestTest extends AbstractApiVersionsRequestTest with Sas } private def sendSaslHandshakeRequestValidateResponse(socket: Socket): Unit = { - val request = new SaslHandshakeRequest(new SaslHandshakeRequestData().setMechanism("PLAIN")) + val request = new SaslHandshakeRequest(new SaslHandshakeRequestData().setMechanism("PLAIN"), + ApiKeys.SASL_HANDSHAKE.latestVersion) val response = sendAndReceive[SaslHandshakeResponse](request, socket) assertEquals(Errors.NONE, response.error) assertEquals(Collections.singletonList("PLAIN"), response.enabledMechanisms) diff --git a/core/src/test/scala/unit/kafka/server/ThrottledChannelExpirationTest.scala b/core/src/test/scala/unit/kafka/server/ThrottledChannelExpirationTest.scala index 02e36413b860a..7e92084086e55 100644 --- a/core/src/test/scala/unit/kafka/server/ThrottledChannelExpirationTest.scala +++ b/core/src/test/scala/unit/kafka/server/ThrottledChannelExpirationTest.scala @@ -50,7 +50,7 @@ class ThrottledChannelExpirationTest { listenerName: ListenerName = ListenerName.forSecurityProtocol(SecurityProtocol.PLAINTEXT)): (T, RequestChannel.Request) = { val request = builder.build() - val buffer = request.serialize(new RequestHeader(builder.apiKey, request.version, "", 0)) + val buffer = request.serializeWithHeader(new RequestHeader(builder.apiKey, request.version, "", 0)) val requestChannelMetrics: RequestChannel.Metrics = EasyMock.createNiceMock(classOf[RequestChannel.Metrics]) // read the header from the buffer first so that the body can be read next from the Request constructor diff --git a/docs/streams/upgrade-guide.html b/docs/streams/upgrade-guide.html index 758ea746b1183..f24d065f57412 100644 --- a/docs/streams/upgrade-guide.html +++ b/docs/streams/upgrade-guide.html @@ -156,6 +156,11 @@

Streams API We added a --force option in StreamsResetter to force remove left-over members on broker side when long session time out was configured as per KIP-571.

+

+ We added Suppressed.withLoggingDisabled() and Suppressed.withLoggingEnabled(config) + methods to allow disabling or configuring of the changelog topic and allows for configuration of the changelog topic + as per KIP-446. +

Streams API changes in 2.5.0

diff --git a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/common/FetchRequestBenchmark.java b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/common/FetchRequestBenchmark.java index 5f68a89ef4971..0d0b1e46a86dd 100644 --- a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/common/FetchRequestBenchmark.java +++ b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/common/FetchRequestBenchmark.java @@ -20,7 +20,6 @@ import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.network.Send; import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.requests.AbstractRequest; import org.apache.kafka.common.requests.ByteBufferChannel; import org.apache.kafka.common.requests.FetchRequest; @@ -39,6 +38,7 @@ import org.openjdk.jmh.annotations.Warmup; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.HashMap; import java.util.Map; import java.util.Optional; @@ -67,7 +67,7 @@ public class FetchRequestBenchmark { FetchRequest replicaRequest; - Struct requestStruct; + ByteBuffer requestBuffer; @Setup(Level.Trial) public void setup() { @@ -86,14 +86,13 @@ public void setup() { .build(ApiKeys.FETCH.latestVersion()); this.replicaRequest = FetchRequest.Builder.forReplica(ApiKeys.FETCH.latestVersion(), 1, 0, 0, fetchData) .build(ApiKeys.FETCH.latestVersion()); - this.requestStruct = this.consumerRequest.data().toStruct(ApiKeys.FETCH.latestVersion()); + this.requestBuffer = this.consumerRequest.serializeBody(); } @Benchmark - public short testFetchRequestFromStruct() { - AbstractRequest request = AbstractRequest.parseRequest(ApiKeys.FETCH, ApiKeys.FETCH.latestVersion(), requestStruct); - return request.version(); + public short testFetchRequestFromBuffer() { + return AbstractRequest.parseRequest(ApiKeys.FETCH, ApiKeys.FETCH.latestVersion(), requestBuffer).request.version(); } @Benchmark diff --git a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/metadata/MetadataRequestBenchmark.java b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/metadata/MetadataRequestBenchmark.java index 413781fa365d5..4ac4c56d4c18f 100644 --- a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/metadata/MetadataRequestBenchmark.java +++ b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/metadata/MetadataRequestBenchmark.java @@ -200,7 +200,7 @@ public void tearDown() { private RequestChannel.Request buildAllTopicMetadataRequest() { MetadataRequest metadataRequest = MetadataRequest.Builder.allTopics().build(); - ByteBuffer buffer = metadataRequest.serialize(new RequestHeader(metadataRequest.api, + ByteBuffer buffer = metadataRequest.serializeWithHeader(new RequestHeader(metadataRequest.apiKey(), metadataRequest.version(), "", 0)); RequestHeader header = RequestHeader.parse(buffer); diff --git a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/producer/ProducerRequestBenchmark.java b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/producer/ProducerRequestBenchmark.java index 58b35f354f92e..22d49551dd5e8 100644 --- a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/producer/ProducerRequestBenchmark.java +++ b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/producer/ProducerRequestBenchmark.java @@ -19,7 +19,6 @@ import org.apache.kafka.common.message.ProduceRequestData; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.record.CompressionType; import org.apache.kafka.common.record.MemoryRecords; import org.apache.kafka.common.record.RecordBatch; @@ -85,9 +84,4 @@ public ProduceResponse constructorErrorResponse() { return REQUEST.getErrorResponse(0, Errors.INVALID_REQUEST.exception()); } - @Benchmark - @OutputTimeUnit(TimeUnit.NANOSECONDS) - public Struct constructorStruct() { - return REQUEST.toStruct(); - } } diff --git a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/producer/ProducerResponseBenchmark.java b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/producer/ProducerResponseBenchmark.java index 307e194431818..431880a96dc18 100644 --- a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/producer/ProducerResponseBenchmark.java +++ b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/producer/ProducerResponseBenchmark.java @@ -19,7 +19,6 @@ import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.requests.AbstractResponse; import org.apache.kafka.common.requests.ProduceResponse; import org.openjdk.jmh.annotations.Benchmark; @@ -38,8 +37,6 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; -import static org.apache.kafka.common.protocol.ApiKeys.PRODUCE; - @State(Scope.Benchmark) @Fork(value = 1) @Warmup(iterations = 5) @@ -79,10 +76,4 @@ private static ProduceResponse response() { public AbstractResponse constructorProduceResponse() { return response(); } - - @Benchmark - @OutputTimeUnit(TimeUnit.NANOSECONDS) - public Struct constructorStruct() { - return RESPONSE.toStruct(PRODUCE.latestVersion()); - } } diff --git a/raft/src/main/java/org/apache/kafka/raft/KafkaRaftClient.java b/raft/src/main/java/org/apache/kafka/raft/KafkaRaftClient.java index 66553f66f3332..37c1846047af5 100644 --- a/raft/src/main/java/org/apache/kafka/raft/KafkaRaftClient.java +++ b/raft/src/main/java/org/apache/kafka/raft/KafkaRaftClient.java @@ -23,14 +23,14 @@ import org.apache.kafka.common.message.BeginQuorumEpochRequestData; import org.apache.kafka.common.message.BeginQuorumEpochResponseData; import org.apache.kafka.common.message.DescribeQuorumRequestData; -import org.apache.kafka.common.message.DescribeQuorumResponseData; import org.apache.kafka.common.message.DescribeQuorumResponseData.ReplicaState; +import org.apache.kafka.common.message.DescribeQuorumResponseData; import org.apache.kafka.common.message.EndQuorumEpochRequestData; import org.apache.kafka.common.message.EndQuorumEpochResponseData; import org.apache.kafka.common.message.FetchRequestData; import org.apache.kafka.common.message.FetchResponseData; -import org.apache.kafka.common.message.LeaderChangeMessage; import org.apache.kafka.common.message.LeaderChangeMessage.Voter; +import org.apache.kafka.common.message.LeaderChangeMessage; import org.apache.kafka.common.message.VoteRequestData; import org.apache.kafka.common.message.VoteResponseData; import org.apache.kafka.common.metrics.Metrics; @@ -61,6 +61,7 @@ import org.apache.kafka.raft.internals.MemoryBatchReader; import org.apache.kafka.raft.internals.RecordsBatchReader; import org.apache.kafka.raft.internals.ThresholdPurgatory; +import org.apache.kafka.snapshot.SnapshotWriter; import org.slf4j.Logger; import java.io.IOException; @@ -1815,6 +1816,18 @@ public CompletableFuture shutdown(int timeoutMs) { return shutdownComplete; } + @Override + public SnapshotWriter createSnapshot(OffsetAndEpoch snapshotId) throws IOException { + return new SnapshotWriter<>( + log.createSnapshot(snapshotId), + MAX_BATCH_SIZE, + memoryPool, + time, + CompressionType.NONE, + serde + ); + } + private void close() { kafkaRaftMetrics.close(); } diff --git a/raft/src/main/java/org/apache/kafka/raft/QuorumState.java b/raft/src/main/java/org/apache/kafka/raft/QuorumState.java index d1eea74dbb5e2..3764308bddde8 100644 --- a/raft/src/main/java/org/apache/kafka/raft/QuorumState.java +++ b/raft/src/main/java/org/apache/kafka/raft/QuorumState.java @@ -35,26 +35,26 @@ * only valid state transitions. Below we define the possible state transitions and * how they are triggered: * - * Unattached|Resigned => + * Unattached|Resigned transitions to: * Unattached: After learning of a new election with a higher epoch * Voted: After granting a vote to a candidate * Candidate: After expiration of the election timeout * Follower: After discovering a leader with an equal or larger epoch * - * Voted => + * Voted transitions to: * Unattached: After learning of a new election with a higher epoch * Candidate: After expiration of the election timeout * - * Candidate => + * Candidate transitions to: * Unattached: After learning of a new election with a higher epoch * Candidate: After expiration of the election timeout * Leader: After receiving a majority of votes * - * Leader => + * Leader transitions to: * Unattached: After learning of a new election with a higher epoch * Resigned: When shutting down gracefully * - * Follower => + * Follower transitions to: * Unattached: After learning of a new election with a higher epoch * Candidate: After expiration of the fetch timeout * Follower: After discovering a leader with a larger epoch @@ -63,11 +63,11 @@ * states are not possible for observers, so the only transitions that are possible * are between Unattached and Follower. * - * Unattached => + * Unattached transitions to: * Unattached: After learning of a new election with a higher epoch * Follower: After discovering a leader with an equal or larger epoch * - * Follower => + * Follower transitions to: * Unattached: After learning of a new election with a higher epoch * Follower: After discovering a leader with a larger epoch * diff --git a/raft/src/main/java/org/apache/kafka/raft/RaftClient.java b/raft/src/main/java/org/apache/kafka/raft/RaftClient.java index 1763bd36a05fb..ba58cb2cfb5c8 100644 --- a/raft/src/main/java/org/apache/kafka/raft/RaftClient.java +++ b/raft/src/main/java/org/apache/kafka/raft/RaftClient.java @@ -16,6 +16,8 @@ */ package org.apache.kafka.raft; +import org.apache.kafka.snapshot.SnapshotWriter; + import java.io.IOException; import java.util.List; import java.util.concurrent.CompletableFuture; @@ -100,4 +102,15 @@ default void handleResign() {} */ CompletableFuture shutdown(int timeoutMs); + /** + * Create a writable snapshot file for a given offset and epoch. + * + * The RaftClient assumes that the snapshot return will contain the records up to but + * not including the end offset in the snapshot id. See {@link SnapshotWriter} for + * details on how to use this object. + * + * @param snapshotId the end offset and epoch that identifies the snapshot + * @return a writable snapshot + */ + SnapshotWriter createSnapshot(OffsetAndEpoch snapshotId) throws IOException; } diff --git a/raft/src/main/java/org/apache/kafka/raft/ReplicatedLog.java b/raft/src/main/java/org/apache/kafka/raft/ReplicatedLog.java index d0572947b5436..b56e9bdbff9a9 100644 --- a/raft/src/main/java/org/apache/kafka/raft/ReplicatedLog.java +++ b/raft/src/main/java/org/apache/kafka/raft/ReplicatedLog.java @@ -18,8 +18,11 @@ import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.record.Records; +import org.apache.kafka.snapshot.RawSnapshotReader; +import org.apache.kafka.snapshot.RawSnapshotWriter; import java.io.Closeable; +import java.io.IOException; import java.util.Optional; import java.util.OptionalLong; @@ -149,6 +152,29 @@ default OptionalLong truncateToEndOffset(OffsetAndEpoch endOffset) { return OptionalLong.of(truncationOffset); } + /** + * Create a writable snapshot for the given snapshot id. + * + * See {@link RawSnapshotWriter} for details on how to use this object. + * + * @param snapshotId the end offset and epoch that identifies the snapshot + * @return a writable snapshot + */ + RawSnapshotWriter createSnapshot(OffsetAndEpoch snapshotId) throws IOException; + + /** + * Opens a readable snapshot for the given snapshot id. + * + * Returns an Optional with a readable snapshot, if the snapshot exists, otherwise + * returns an empty Optional. See {@link RawSnapshotReader} for details on how to + * use this object. + * + * @param snapshotId the end offset and epoch that identifies the snapshot + * @return an Optional with a readable snapshot, if the snapshot exists, otherwise + * returns an empty Optional + */ + Optional readSnapshot(OffsetAndEpoch snapshotId) throws IOException; + default void close() {} } diff --git a/raft/src/main/java/org/apache/kafka/snapshot/FileRawSnapshotReader.java b/raft/src/main/java/org/apache/kafka/snapshot/FileRawSnapshotReader.java new file mode 100644 index 0000000000000..d517006d4d68f --- /dev/null +++ b/raft/src/main/java/org/apache/kafka/snapshot/FileRawSnapshotReader.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.snapshot; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.file.Path; +import java.util.Iterator; +import org.apache.kafka.common.record.FileRecords; +import org.apache.kafka.common.record.RecordBatch; +import org.apache.kafka.common.utils.Utils; +import org.apache.kafka.raft.OffsetAndEpoch; + +public final class FileRawSnapshotReader implements RawSnapshotReader { + private final FileRecords fileRecords; + private final OffsetAndEpoch snapshotId; + + private FileRawSnapshotReader(FileRecords fileRecords, OffsetAndEpoch snapshotId) { + this.fileRecords = fileRecords; + this.snapshotId = snapshotId; + } + + @Override + public OffsetAndEpoch snapshotId() { + return snapshotId; + } + + @Override + public long sizeInBytes() { + return fileRecords.sizeInBytes(); + } + + @Override + public Iterator iterator() { + return Utils.covariantCast(fileRecords.batchIterator()); + } + + @Override + public int read(ByteBuffer buffer, long position) throws IOException { + return fileRecords.channel().read(buffer, position); + } + + @Override + public void close() throws IOException { + fileRecords.close(); + } + + /** + * Opens a snapshot for reading. + * + * @param logDir the directory for the topic partition + * @param snapshotId the end offset and epoch for the snapshotId + * @throws java.nio.file.NoSuchFileException if the snapshot doesn't exist + * @throws IOException for any IO error while opening the snapshot + */ + public static FileRawSnapshotReader open(Path logDir, OffsetAndEpoch snapshotId) throws IOException { + FileRecords fileRecords = FileRecords.open( + Snapshots.snapshotPath(logDir, snapshotId).toFile(), + false, // mutable + true, // fileAlreadyExists + 0, // initFileSize + false // preallocate + ); + + return new FileRawSnapshotReader(fileRecords, snapshotId); + } +} diff --git a/raft/src/main/java/org/apache/kafka/snapshot/FileRawSnapshotWriter.java b/raft/src/main/java/org/apache/kafka/snapshot/FileRawSnapshotWriter.java new file mode 100644 index 0000000000000..934a12481ccf1 --- /dev/null +++ b/raft/src/main/java/org/apache/kafka/snapshot/FileRawSnapshotWriter.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.snapshot; + +import org.apache.kafka.common.utils.Utils; +import org.apache.kafka.raft.OffsetAndEpoch; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; + +public final class FileRawSnapshotWriter implements RawSnapshotWriter { + private final Path tempSnapshotPath; + private final FileChannel channel; + private final OffsetAndEpoch snapshotId; + private boolean frozen = false; + + private FileRawSnapshotWriter( + Path tempSnapshotPath, + FileChannel channel, + OffsetAndEpoch snapshotId + ) { + this.tempSnapshotPath = tempSnapshotPath; + this.channel = channel; + this.snapshotId = snapshotId; + } + + @Override + public OffsetAndEpoch snapshotId() { + return snapshotId; + } + + @Override + public long sizeInBytes() throws IOException { + return channel.size(); + } + + @Override + public void append(ByteBuffer buffer) throws IOException { + if (frozen) { + throw new IllegalStateException( + String.format("Append is not supported. Snapshot is already frozen: id = %s; temp path = %s", snapshotId, tempSnapshotPath) + ); + } + + Utils.writeFully(channel, buffer); + } + + @Override + public boolean isFrozen() { + return frozen; + } + + @Override + public void freeze() throws IOException { + if (frozen) { + throw new IllegalStateException( + String.format("Freeze is not supported. Snapshot is already frozen: id = %s; temp path = %s", snapshotId, tempSnapshotPath) + ); + } + + channel.close(); + frozen = true; + + // Set readonly and ignore the result + if (!tempSnapshotPath.toFile().setReadOnly()) { + throw new IOException(String.format("Unable to set file (%s) as read-only", tempSnapshotPath)); + } + + Path destination = Snapshots.moveRename(tempSnapshotPath, snapshotId); + Utils.atomicMoveWithFallback(tempSnapshotPath, destination); + } + + @Override + public void close() throws IOException { + try { + channel.close(); + } finally { + // This is a noop if freeze was called before calling close + Files.deleteIfExists(tempSnapshotPath); + } + } + + /** + * Create a snapshot writer for topic partition log dir and snapshot id. + * + * @param logDir the directory for the topic partition + * @param snapshotId the end offset and epoch for the snapshotId + * @throws IOException for any IO error while creating the snapshot + */ + public static FileRawSnapshotWriter create(Path logDir, OffsetAndEpoch snapshotId) throws IOException { + Path path = Snapshots.createTempFile(logDir, snapshotId); + + return new FileRawSnapshotWriter( + path, + FileChannel.open(path, Utils.mkSet(StandardOpenOption.WRITE, StandardOpenOption.APPEND)), + snapshotId + ); + } +} diff --git a/raft/src/main/java/org/apache/kafka/snapshot/RawSnapshotReader.java b/raft/src/main/java/org/apache/kafka/snapshot/RawSnapshotReader.java new file mode 100644 index 0000000000000..6b10fa89bba59 --- /dev/null +++ b/raft/src/main/java/org/apache/kafka/snapshot/RawSnapshotReader.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.snapshot; + +import java.io.Closeable; +import java.io.IOException; +import java.nio.ByteBuffer; +import org.apache.kafka.common.record.RecordBatch; +import org.apache.kafka.raft.OffsetAndEpoch; + +/** + * Interface for reading snapshots as a sequence of records. + */ +public interface RawSnapshotReader extends Closeable, Iterable { + /** + * Returns the end offset and epoch for the snapshot. + */ + public OffsetAndEpoch snapshotId(); + + /** + * Returns the number of bytes for the snapshot. + * + * @throws IOException for any IO error while reading the size + */ + public long sizeInBytes() throws IOException; + + /** + * Reads bytes from position into the given buffer. + * + * It is not guarantee that the given buffer will be filled. + * + * @param buffer byte buffer to put the read files + * @param position the starting position in the snapshot to read + * @return the number of bytes read + * @throws IOException for any IO error while reading the snapshot + */ + public int read(ByteBuffer buffer, long position) throws IOException; +} diff --git a/raft/src/main/java/org/apache/kafka/snapshot/RawSnapshotWriter.java b/raft/src/main/java/org/apache/kafka/snapshot/RawSnapshotWriter.java new file mode 100644 index 0000000000000..001d2ef37c78e --- /dev/null +++ b/raft/src/main/java/org/apache/kafka/snapshot/RawSnapshotWriter.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.snapshot; + +import java.io.Closeable; +import java.io.IOException; +import java.nio.ByteBuffer; +import org.apache.kafka.raft.OffsetAndEpoch; + +/** + * Interface for writing snapshot as a sequence of records. + */ +public interface RawSnapshotWriter extends Closeable { + /** + * Returns the end offset and epoch for the snapshot. + */ + public OffsetAndEpoch snapshotId(); + + /** + * Returns the number of bytes for the snapshot. + * + * @throws IOException for any IO error while reading the size + */ + public long sizeInBytes() throws IOException; + + /** + * Fully appends the buffer to the snapshot. + * + * If the method returns without an exception the given buffer was fully writing the + * snapshot. + * + * @param buffer the buffer to append + * @throws IOException for any IO error during append + */ + public void append(ByteBuffer buffer) throws IOException; + + /** + * Returns true if the snapshot has been frozen, otherwise false is returned. + * + * Modification to the snapshot are not allowed once it is frozen. + */ + public boolean isFrozen(); + + /** + * Freezes the snapshot and marking it as immutable. + * + * @throws IOException for any IO error during freezing + */ + public void freeze() throws IOException; + + /** + * Closes the snapshot writer. + * + * If close is called without first calling freeze the the snapshot is aborted. + * + * @throws IOException for any IO error during close + */ + public void close() throws IOException; +} diff --git a/raft/src/main/java/org/apache/kafka/snapshot/SnapshotWriter.java b/raft/src/main/java/org/apache/kafka/snapshot/SnapshotWriter.java new file mode 100644 index 0000000000000..0232e93ac380f --- /dev/null +++ b/raft/src/main/java/org/apache/kafka/snapshot/SnapshotWriter.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.snapshot; + +import java.io.Closeable; +import java.io.IOException; +import java.util.List; +import org.apache.kafka.common.memory.MemoryPool; +import org.apache.kafka.common.record.CompressionType; +import org.apache.kafka.common.utils.Time; +import org.apache.kafka.raft.OffsetAndEpoch; +import org.apache.kafka.raft.RecordSerde; +import org.apache.kafka.raft.internals.BatchAccumulator.CompletedBatch; +import org.apache.kafka.raft.internals.BatchAccumulator; + +/** + * A type for writing a snapshot fora given end offset and epoch. + * + * A snapshot writer can be used to append objects until freeze is called. When freeze is + * called the snapshot is validated and marked as immutable. After freeze is called any + * append will fail with an exception. + * + * It is assumed that the content of the snapshot represents all of the records for the + * topic partition from offset 0 up to but not including the end offset in the snapshot + * id. + * + * @see org.apache.kafka.raft.RaftClient#createSnapshot(OffsetAndEpoch) + */ +final public class SnapshotWriter implements Closeable { + final private RawSnapshotWriter snapshot; + final private BatchAccumulator accumulator; + final private Time time; + + /** + * Initializes a new instance of the class. + * + * @param snapshot the low level snapshot writer + * @param maxBatchSize the maximum size in byte for a batch + * @param memoryPool the memory pool for buffer allocation + * @param time the clock implementation + * @param compressionType the compression algorithm to use + * @param serde the record serialization and deserialization implementation + */ + public SnapshotWriter( + RawSnapshotWriter snapshot, + int maxBatchSize, + MemoryPool memoryPool, + Time time, + CompressionType compressionType, + RecordSerde serde + ) { + this.snapshot = snapshot; + this.time = time; + + this.accumulator = new BatchAccumulator<>( + snapshot.snapshotId().epoch, + 0, + Integer.MAX_VALUE, + maxBatchSize, + memoryPool, + time, + compressionType, + serde + ); + } + + /** + * Returns the end offset and epoch for the snapshot. + */ + public OffsetAndEpoch snapshotId() { + return snapshot.snapshotId(); + } + + /** + * Returns true if the snapshot has been frozen, otherwise false is returned. + * + * Modification to the snapshot are not allowed once it is frozen. + */ + public boolean isFrozen() { + return snapshot.isFrozen(); + } + + /** + * Appends a list of values to the snapshot. + * + * The list of record passed are guaranteed to get written together. + * + * @param records the list of records to append to the snapshot + * @throws IOException for any IO error while appending + * @throws IllegalStateException if append is called when isFrozen is true + */ + public void append(List records) throws IOException { + if (snapshot.isFrozen()) { + String message = String.format( + "Append not supported. Snapshot is already frozen: id = {}.", + snapshot.snapshotId() + ); + + throw new IllegalStateException(message); + } + + accumulator.append(snapshot.snapshotId().epoch, records); + + if (accumulator.needsDrain(time.milliseconds())) { + appendBatches(accumulator.drain()); + } + } + + /** + * Freezes the snapshot by flushing all pending writes and marking it as immutable. + * + * @throws IOException for any IO error during freezing + */ + public void freeze() throws IOException { + appendBatches(accumulator.drain()); + snapshot.freeze(); + accumulator.close(); + } + + /** + * Closes the snapshot writer. + * + * If close is called without first calling freeze the the snapshot is aborted. + * + * @throws IOException for any IO error during close + */ + public void close() throws IOException { + snapshot.close(); + accumulator.close(); + } + + private void appendBatches(List> batches) throws IOException { + try { + for (CompletedBatch batch : batches) { + snapshot.append(batch.data.buffer()); + } + } finally { + batches.forEach(CompletedBatch::release); + } + } +} diff --git a/raft/src/main/java/org/apache/kafka/snapshot/Snapshots.java b/raft/src/main/java/org/apache/kafka/snapshot/Snapshots.java new file mode 100644 index 0000000000000..8abf160b1a82e --- /dev/null +++ b/raft/src/main/java/org/apache/kafka/snapshot/Snapshots.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.snapshot; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.text.NumberFormat; +import org.apache.kafka.raft.OffsetAndEpoch; + +final class Snapshots { + private static final String SUFFIX = ".checkpoint"; + private static final String PARTIAL_SUFFIX = String.format("%s.part", SUFFIX); + + private static final NumberFormat OFFSET_FORMATTER = NumberFormat.getInstance(); + private static final NumberFormat EPOCH_FORMATTER = NumberFormat.getInstance(); + + static { + OFFSET_FORMATTER.setMinimumIntegerDigits(20); + OFFSET_FORMATTER.setGroupingUsed(false); + + EPOCH_FORMATTER.setMinimumIntegerDigits(10); + EPOCH_FORMATTER.setGroupingUsed(false); + } + + static Path snapshotDir(Path logDir) { + return logDir; + } + + static Path snapshotPath(Path logDir, OffsetAndEpoch snapshotId) { + return snapshotDir(logDir).resolve(filenameFromSnapshotId(snapshotId) + SUFFIX); + } + + static String filenameFromSnapshotId(OffsetAndEpoch snapshotId) { + return String.format("%s-%s", OFFSET_FORMATTER.format(snapshotId.offset), EPOCH_FORMATTER.format(snapshotId.epoch)); + } + + static Path moveRename(Path source, OffsetAndEpoch snapshotId) { + return source.resolveSibling(filenameFromSnapshotId(snapshotId) + SUFFIX); + } + + static Path createTempFile(Path logDir, OffsetAndEpoch snapshotId) throws IOException { + Path dir = snapshotDir(logDir); + + // Create the snapshot directory if it doesn't exists + Files.createDirectories(dir); + + String prefix = String.format("%s-", filenameFromSnapshotId(snapshotId)); + + return Files.createTempFile(dir, prefix, PARTIAL_SUFFIX); + } +} diff --git a/raft/src/test/java/org/apache/kafka/raft/MockLog.java b/raft/src/test/java/org/apache/kafka/raft/MockLog.java index 57c519e001d7a..1e8c1b53fcb72 100644 --- a/raft/src/test/java/org/apache/kafka/raft/MockLog.java +++ b/raft/src/test/java/org/apache/kafka/raft/MockLog.java @@ -26,13 +26,19 @@ import org.apache.kafka.common.record.Records; import org.apache.kafka.common.record.SimpleRecord; import org.apache.kafka.common.record.TimestampType; +import org.apache.kafka.common.utils.ByteBufferOutputStream; import org.apache.kafka.common.utils.Utils; +import org.apache.kafka.snapshot.RawSnapshotReader; +import org.apache.kafka.snapshot.RawSnapshotWriter; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.OptionalLong; @@ -45,6 +51,7 @@ public class MockLog implements ReplicatedLog { private final List epochStartOffsets = new ArrayList<>(); private final List log = new ArrayList<>(); + private final Map snapshots = new HashMap<>(); private final TopicPartition topicPartition; private long nextId = ID_GENERATOR.getAndIncrement(); @@ -351,6 +358,16 @@ public void initializeLeaderEpoch(int epoch) { epochStartOffsets.add(new EpochStartOffset(epoch, startOffset)); } + @Override + public RawSnapshotWriter createSnapshot(OffsetAndEpoch snapshotId) { + return new MockRawSnapshotWriter(snapshotId); + } + + @Override + public Optional readSnapshot(OffsetAndEpoch snapshotId) { + return Optional.ofNullable(snapshots.get(snapshotId)); + } + static class MockOffsetMetadata implements OffsetMetadata { final long id; @@ -472,4 +489,94 @@ private EpochStartOffset(int epoch, long startOffset) { } } + final class MockRawSnapshotWriter implements RawSnapshotWriter { + private final OffsetAndEpoch snapshotId; + private ByteBufferOutputStream data; + private boolean frozen; + + public MockRawSnapshotWriter(OffsetAndEpoch snapshotId) { + this.snapshotId = snapshotId; + this.data = new ByteBufferOutputStream(0); + this.frozen = false; + } + + @Override + public OffsetAndEpoch snapshotId() { + return snapshotId; + } + + @Override + public long sizeInBytes() { + return data.position(); + } + + @Override + public void append(ByteBuffer buffer) { + if (frozen) { + throw new RuntimeException("Snapshot is already frozen " + snapshotId); + } + + data.write(buffer); + } + + @Override + public boolean isFrozen() { + return frozen; + } + + @Override + public void freeze() { + if (frozen) { + throw new RuntimeException("Snapshot is already frozen " + snapshotId); + } + + frozen = true; + ByteBuffer buffer = data.buffer(); + buffer.flip(); + + snapshots.putIfAbsent(snapshotId, new MockRawSnapshotReader(snapshotId, buffer)); + } + + @Override + public void close() {} + } + + final static class MockRawSnapshotReader implements RawSnapshotReader { + private final OffsetAndEpoch snapshotId; + private final MemoryRecords data; + + MockRawSnapshotReader(OffsetAndEpoch snapshotId, ByteBuffer data) { + this.snapshotId = snapshotId; + this.data = MemoryRecords.readableRecords(data); + } + + @Override + public OffsetAndEpoch snapshotId() { + return snapshotId; + } + + @Override + public long sizeInBytes() { + return data.sizeInBytes(); + } + + @Override + public Iterator iterator() { + return Utils.covariantCast(data.batchIterator()); + } + + @Override + public int read(ByteBuffer buffer, long position) { + ByteBuffer copy = data.buffer(); + copy.position((int) position); + copy.limit((int) position + Math.min(copy.remaining(), buffer.remaining())); + + buffer.put(copy); + + return copy.remaining(); + } + + @Override + public void close() {} + } } diff --git a/raft/src/test/java/org/apache/kafka/raft/RaftClientTestContext.java b/raft/src/test/java/org/apache/kafka/raft/RaftClientTestContext.java index 4d9db4aaf90a5..0e803e5ebef4b 100644 --- a/raft/src/test/java/org/apache/kafka/raft/RaftClientTestContext.java +++ b/raft/src/test/java/org/apache/kafka/raft/RaftClientTestContext.java @@ -78,7 +78,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -final class RaftClientTestContext { +public final class RaftClientTestContext { private static final StringSerde STRING_SERDE = new StringSerde(); final TopicPartition metadataPartition = Builder.METADATA_PARTITION; @@ -91,9 +91,9 @@ final class RaftClientTestContext { private final QuorumStateStore quorumStateStore; final int localId; - final KafkaRaftClient client; + public final KafkaRaftClient client; final Metrics metrics; - final MockLog log; + public final MockLog log; final MockNetworkChannel channel; final MockTime time; final MockListener listener; @@ -124,7 +124,7 @@ public static final class Builder { private int appendLingerMs = DEFAULT_APPEND_LINGER_MS; private MemoryPool memoryPool = MemoryPool.NONE; - Builder(int localId, Set voters) { + public Builder(int localId, Set voters) { this.voters = voters; this.localId = localId; } @@ -175,7 +175,7 @@ Builder withRequestTimeoutMs(int requestTimeoutMs) { return this; } - RaftClientTestContext build() throws IOException { + public RaftClientTestContext build() throws IOException { Metrics metrics = new Metrics(time); MockNetworkChannel channel = new MockNetworkChannel(); LogContext logContext = new LogContext(); diff --git a/raft/src/test/java/org/apache/kafka/snapshot/FileRawSnapshotTest.java b/raft/src/test/java/org/apache/kafka/snapshot/FileRawSnapshotTest.java new file mode 100644 index 0000000000000..e52943558f2cc --- /dev/null +++ b/raft/src/test/java/org/apache/kafka/snapshot/FileRawSnapshotTest.java @@ -0,0 +1,289 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.snapshot; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.Iterator; +import java.util.stream.IntStream; +import org.apache.kafka.common.record.BufferSupplier.GrowableBufferSupplier; +import org.apache.kafka.common.record.CompressionType; +import org.apache.kafka.common.record.MemoryRecords; +import org.apache.kafka.common.record.RecordBatch; +import org.apache.kafka.common.record.SimpleRecord; +import org.apache.kafka.common.record.Record; +import org.apache.kafka.raft.OffsetAndEpoch; +import org.apache.kafka.test.TestUtils; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public final class FileRawSnapshotTest { + @Test + public void testWritingSnapshot() throws IOException { + Path tempDir = TestUtils.tempDirectory().toPath(); + OffsetAndEpoch offsetAndEpoch = new OffsetAndEpoch(10L, 3); + int bufferSize = 256; + int batches = 10; + int expectedSize = 0; + + try (FileRawSnapshotWriter snapshot = FileRawSnapshotWriter.create(tempDir, offsetAndEpoch)) { + assertEquals(0, snapshot.sizeInBytes()); + + MemoryRecords records = buildRecords(ByteBuffer.wrap(randomBytes(bufferSize))); + for (int i = 0; i < batches; i++) { + snapshot.append(records.buffer()); + expectedSize += records.sizeInBytes(); + } + + assertEquals(expectedSize, snapshot.sizeInBytes()); + + snapshot.freeze(); + } + + // File should exist and the size should be the sum of all the buffers + assertTrue(Files.exists(Snapshots.snapshotPath(tempDir, offsetAndEpoch))); + assertEquals(expectedSize, Files.size(Snapshots.snapshotPath(tempDir, offsetAndEpoch))); + } + + @Test + public void testWriteReadSnapshot() throws IOException { + Path tempDir = TestUtils.tempDirectory().toPath(); + OffsetAndEpoch offsetAndEpoch = new OffsetAndEpoch(10L, 3); + int bufferSize = 256; + int batches = 10; + + ByteBuffer expectedBuffer = ByteBuffer.wrap(randomBytes(bufferSize)); + + try (FileRawSnapshotWriter snapshot = FileRawSnapshotWriter.create(tempDir, offsetAndEpoch)) { + MemoryRecords records = buildRecords(expectedBuffer); + for (int i = 0; i < batches; i++) { + snapshot.append(records.buffer()); + } + + snapshot.freeze(); + } + + try (FileRawSnapshotReader snapshot = FileRawSnapshotReader.open(tempDir, offsetAndEpoch)) { + int countBatches = 0; + int countRecords = 0; + for (RecordBatch batch : snapshot) { + countBatches += 1; + + Iterator records = batch.streamingIterator(new GrowableBufferSupplier()); + while (records.hasNext()) { + Record record = records.next(); + + countRecords += 1; + + assertFalse(record.hasKey()); + assertTrue(record.hasValue()); + assertEquals(bufferSize, record.value().remaining()); + assertEquals(expectedBuffer, record.value()); + } + } + + assertEquals(batches, countBatches); + assertEquals(batches, countRecords); + } + } + + @Test + public void testBatchWriteReadSnapshot() throws IOException { + Path tempDir = TestUtils.tempDirectory().toPath(); + OffsetAndEpoch offsetAndEpoch = new OffsetAndEpoch(10L, 3); + int bufferSize = 256; + int batchSize = 3; + int batches = 10; + + try (FileRawSnapshotWriter snapshot = FileRawSnapshotWriter.create(tempDir, offsetAndEpoch)) { + for (int i = 0; i < batches; i++) { + ByteBuffer[] buffers = IntStream + .range(0, batchSize) + .mapToObj(ignore -> ByteBuffer.wrap(randomBytes(bufferSize))).toArray(ByteBuffer[]::new); + + snapshot.append(buildRecords(buffers).buffer()); + } + + snapshot.freeze(); + } + + try (FileRawSnapshotReader snapshot = FileRawSnapshotReader.open(tempDir, offsetAndEpoch)) { + int countBatches = 0; + int countRecords = 0; + for (RecordBatch batch : snapshot) { + countBatches += 1; + + Iterator records = batch.streamingIterator(new GrowableBufferSupplier()); + while (records.hasNext()) { + Record record = records.next(); + + countRecords += 1; + + assertFalse(record.hasKey()); + assertTrue(record.hasValue()); + assertEquals(bufferSize, record.value().remaining()); + } + } + + assertEquals(batches, countBatches); + assertEquals(batches * batchSize, countRecords); + } + } + + @Test + public void testBufferWriteReadSnapshot() throws IOException { + Path tempDir = TestUtils.tempDirectory().toPath(); + OffsetAndEpoch offsetAndEpoch = new OffsetAndEpoch(10L, 3); + int bufferSize = 256; + int batchSize = 3; + int batches = 10; + int expectedSize = 0; + + try (FileRawSnapshotWriter snapshot = FileRawSnapshotWriter.create(tempDir, offsetAndEpoch)) { + for (int i = 0; i < batches; i++) { + ByteBuffer[] buffers = IntStream + .range(0, batchSize) + .mapToObj(ignore -> ByteBuffer.wrap(randomBytes(bufferSize))).toArray(ByteBuffer[]::new); + + MemoryRecords records = buildRecords(buffers); + snapshot.append(records.buffer()); + expectedSize += records.sizeInBytes(); + } + + assertEquals(expectedSize, snapshot.sizeInBytes()); + + snapshot.freeze(); + } + + // File should exist and the size should be the sum of all the buffers + assertTrue(Files.exists(Snapshots.snapshotPath(tempDir, offsetAndEpoch))); + assertEquals(expectedSize, Files.size(Snapshots.snapshotPath(tempDir, offsetAndEpoch))); + + try (FileRawSnapshotReader snapshot = FileRawSnapshotReader.open(tempDir, offsetAndEpoch)) { + int countBatches = 0; + int countRecords = 0; + + for (RecordBatch batch : snapshot) { + countBatches += 1; + + Iterator records = batch.streamingIterator(new GrowableBufferSupplier()); + while (records.hasNext()) { + Record record = records.next(); + + countRecords += 1; + + assertFalse(record.hasKey()); + assertTrue(record.hasValue()); + assertEquals(bufferSize, record.value().remaining()); + } + } + + assertEquals(batches, countBatches); + assertEquals(batches * batchSize, countRecords); + } + } + + @Test + public void testAbortedSnapshot() throws IOException { + Path tempDir = TestUtils.tempDirectory().toPath(); + OffsetAndEpoch offsetAndEpoch = new OffsetAndEpoch(20L, 2); + int bufferSize = 256; + int batches = 10; + + try (FileRawSnapshotWriter snapshot = FileRawSnapshotWriter.create(tempDir, offsetAndEpoch)) { + MemoryRecords records = buildRecords(ByteBuffer.wrap(randomBytes(bufferSize))); + for (int i = 0; i < batches; i++) { + snapshot.append(records.buffer()); + } + } + + // File should not exist since freeze was not called before + assertFalse(Files.exists(Snapshots.snapshotPath(tempDir, offsetAndEpoch))); + assertEquals(0, Files.list(Snapshots.snapshotDir(tempDir)).count()); + } + + @Test + public void testAppendToFrozenSnapshot() throws IOException { + Path tempDir = TestUtils.tempDirectory().toPath(); + OffsetAndEpoch offsetAndEpoch = new OffsetAndEpoch(10L, 3); + int bufferSize = 256; + int batches = 10; + + try (FileRawSnapshotWriter snapshot = FileRawSnapshotWriter.create(tempDir, offsetAndEpoch)) { + MemoryRecords records = buildRecords(ByteBuffer.wrap(randomBytes(bufferSize))); + for (int i = 0; i < batches; i++) { + snapshot.append(records.buffer()); + } + + snapshot.freeze(); + + assertThrows(RuntimeException.class, () -> snapshot.append(records.buffer())); + } + + // File should exist and the size should be greater than the sum of all the buffers + assertTrue(Files.exists(Snapshots.snapshotPath(tempDir, offsetAndEpoch))); + assertTrue(Files.size(Snapshots.snapshotPath(tempDir, offsetAndEpoch)) > bufferSize * batches); + } + + @Test + public void testCreateSnapshotWithSameId() throws IOException { + Path tempDir = TestUtils.tempDirectory().toPath(); + OffsetAndEpoch offsetAndEpoch = new OffsetAndEpoch(20L, 2); + int bufferSize = 256; + int batches = 1; + + try (FileRawSnapshotWriter snapshot = FileRawSnapshotWriter.create(tempDir, offsetAndEpoch)) { + MemoryRecords records = buildRecords(ByteBuffer.wrap(randomBytes(bufferSize))); + for (int i = 0; i < batches; i++) { + snapshot.append(records.buffer()); + } + + snapshot.freeze(); + } + + // Create another snapshot with the same id + try (FileRawSnapshotWriter snapshot = FileRawSnapshotWriter.create(tempDir, offsetAndEpoch)) { + MemoryRecords records = buildRecords(ByteBuffer.wrap(randomBytes(bufferSize))); + for (int i = 0; i < batches; i++) { + snapshot.append(records.buffer()); + } + + snapshot.freeze(); + } + } + + private static byte[] randomBytes(int size) { + byte[] array = new byte[size]; + + TestUtils.SEEDED_RANDOM.nextBytes(array); + + return array; + } + + private static MemoryRecords buildRecords(ByteBuffer... buffers) { + return MemoryRecords.withRecords( + CompressionType.NONE, + Arrays.stream(buffers).map(buffer -> new SimpleRecord(buffer)).toArray(SimpleRecord[]::new) + ); + } +} diff --git a/raft/src/test/java/org/apache/kafka/snapshot/SnapshotWriterTest.java b/raft/src/test/java/org/apache/kafka/snapshot/SnapshotWriterTest.java new file mode 100644 index 0000000000000..3957640284e59 --- /dev/null +++ b/raft/src/test/java/org/apache/kafka/snapshot/SnapshotWriterTest.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.snapshot; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Random; +import java.util.Set; +import org.apache.kafka.common.record.BufferSupplier.GrowableBufferSupplier; +import org.apache.kafka.common.utils.Utils; +import org.apache.kafka.raft.OffsetAndEpoch; +import org.apache.kafka.raft.RaftClientTestContext; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; + +final public class SnapshotWriterTest { + private final int localId = 0; + private final Set voters = Collections.singleton(localId); + + @Test + public void testWritingSnapshot() throws IOException { + OffsetAndEpoch id = new OffsetAndEpoch(10L, 3); + List> expected = buildRecords(3, 3); + RaftClientTestContext context = new RaftClientTestContext.Builder(localId, voters).build(); + + try (SnapshotWriter snapshot = context.client.createSnapshot(id)) { + expected.forEach(batch -> { + assertDoesNotThrow(() -> snapshot.append(batch)); + }); + snapshot.freeze(); + } + + try (RawSnapshotReader reader = context.log.readSnapshot(id).get()) { + assertSnapshot(expected, reader); + } + } + + @Test + public void testAbortedSnapshot() throws IOException { + OffsetAndEpoch id = new OffsetAndEpoch(10L, 3); + List> expected = buildRecords(3, 3); + RaftClientTestContext context = new RaftClientTestContext.Builder(localId, voters).build(); + + try (SnapshotWriter snapshot = context.client.createSnapshot(id)) { + expected.forEach(batch -> { + assertDoesNotThrow(() -> snapshot.append(batch)); + }); + } + + assertFalse(context.log.readSnapshot(id).isPresent()); + } + + @Test + public void testAppendToFrozenSnapshot() throws IOException { + OffsetAndEpoch id = new OffsetAndEpoch(10L, 3); + List> expected = buildRecords(3, 3); + RaftClientTestContext context = new RaftClientTestContext.Builder(localId, voters).build(); + + try (SnapshotWriter snapshot = context.client.createSnapshot(id)) { + expected.forEach(batch -> { + assertDoesNotThrow(() -> snapshot.append(batch)); + }); + + snapshot.freeze(); + + assertThrows(RuntimeException.class, () -> snapshot.append(expected.get(0))); + } + } + + private List> buildRecords(int recordsPerBatch, int batches) { + Random random = new Random(0); + List> result = new ArrayList<>(batches); + for (int i = 0; i < batches; i++) { + List batch = new ArrayList<>(recordsPerBatch); + for (int j = 0; j < recordsPerBatch; j++) { + batch.add(String.valueOf(random.nextInt())); + } + result.add(batch); + } + + return result; + } + + private void assertSnapshot(List> batches, RawSnapshotReader reader) { + List expected = new ArrayList<>(); + batches.forEach(expected::addAll); + + List actual = new ArrayList<>(expected.size()); + reader.forEach(batch -> { + batch.streamingIterator(new GrowableBufferSupplier()).forEachRemaining(record -> { + actual.add(Utils.utf8(record.value())); + }); + }); + + assertEquals(expected, actual); + } +} diff --git a/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java b/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java index 44ce3079ac457..4fb2b2fbefea8 100644 --- a/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java +++ b/streams/src/main/java/org/apache/kafka/streams/KafkaStreams.java @@ -84,6 +84,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Properties; import java.util.Set; import java.util.TreeMap; @@ -161,11 +162,20 @@ public class KafkaStreams implements AutoCloseable { private final ProcessorTopology taskTopology; private final ProcessorTopology globalTaskTopology; private final long totalCacheSize; + private final StreamStateListener streamStateListener; + private final StateRestoreListener delegatingStateRestoreListener; + private final Map threadState; + private final ArrayList storeProviders; + private final UUID processId; + private final KafkaClientSupplier clientSupplier; + private final InternalTopologyBuilder internalTopologyBuilder; GlobalStreamThread globalStreamThread; private KafkaStreams.StateListener stateListener; private StateRestoreListener globalStateRestoreListener; private boolean oldHandler; + private java.util.function.Consumer streamsUncaughtExceptionHandler; + private final Object changeThreadCount = new Object(); // container states /** @@ -397,6 +407,7 @@ public void setUncaughtExceptionHandler(final StreamsUncaughtExceptionHandler st final Consumer handler = exception -> handleStreamsUncaughtException(exception, streamsUncaughtExceptionHandler); synchronized (stateLock) { if (state == State.CREATED) { + this.streamsUncaughtExceptionHandler = handler; Objects.requireNonNull(streamsUncaughtExceptionHandler); for (final StreamThread thread : threads) { thread.setStreamsUncaughtExceptionHandler(handler); @@ -755,7 +766,7 @@ private KafkaStreams(final InternalTopologyBuilder internalTopologyBuilder, this.config = config; this.time = time; // The application ID is a required config and hence should always have value - final UUID processId = UUID.randomUUID(); + processId = UUID.randomUUID(); final String userClientId = config.getString(StreamsConfig.CLIENT_ID_CONFIG); final String applicationId = config.getString(StreamsConfig.APPLICATION_ID_CONFIG); if (userClientId.length() <= 0) { @@ -765,7 +776,7 @@ private KafkaStreams(final InternalTopologyBuilder internalTopologyBuilder, } final LogContext logContext = new LogContext(String.format("stream-client [%s] ", clientId)); this.log = logContext.logger(getClass()); - + this.clientSupplier = clientSupplier; final MetricConfig metricConfig = new MetricConfig() .samples(config.getInt(StreamsConfig.METRICS_NUM_SAMPLES_CONFIG)) .recordLevel(Sensor.RecordingLevel.forName(config.getString(StreamsConfig.METRICS_RECORDING_LEVEL_CONFIG))) @@ -792,7 +803,7 @@ private KafkaStreams(final InternalTopologyBuilder internalTopologyBuilder, ClientMetrics.addStateMetric(streamsMetrics, (metricsConfig, now) -> state); log.info("Kafka Streams version: {}", ClientMetrics.version()); log.info("Kafka Streams commit ID: {}", ClientMetrics.commitId()); - + this.internalTopologyBuilder = internalTopologyBuilder; // re-write the physical topology according to the config internalTopologyBuilder.rewriteTopology(config); @@ -820,18 +831,18 @@ private KafkaStreams(final InternalTopologyBuilder internalTopologyBuilder, throw new TopologyException("Topology has no stream threads and no global threads, " + "must subscribe to at least one source topic or global table."); } - + oldHandler = false; totalCacheSize = config.getLong(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG); final long cacheSizePerThread = getCacheSizePerThread(numStreamThreads); final boolean hasPersistentStores = taskTopology.hasPersistentLocalStore() || (hasGlobalTopology && globalTaskTopology.hasPersistentGlobalStore()); - + streamsUncaughtExceptionHandler = this::defaultStreamsUncaughtExceptionHandler; try { stateDirectory = new StateDirectory(config, time, hasPersistentStores); } catch (final ProcessorStateException fatal) { throw new StreamsException(fatal); } - final StateRestoreListener delegatingStateRestoreListener = new DelegatingStateRestoreListener(); + delegatingStateRestoreListener = new DelegatingStateRestoreListener(); GlobalStreamThread.State globalThreadState = null; if (hasGlobalTopology) { final String globalThreadId = clientId + "-GlobalStreamThread"; @@ -845,7 +856,7 @@ private KafkaStreams(final InternalTopologyBuilder internalTopologyBuilder, time, globalThreadId, delegatingStateRestoreListener, - this::defaultStreamsUncaughtExceptionHandler + streamsUncaughtExceptionHandler ); globalThreadState = globalStreamThread.state(); } @@ -853,59 +864,110 @@ private KafkaStreams(final InternalTopologyBuilder internalTopologyBuilder, // use client id instead of thread client id since this admin client may be shared among threads adminClient = clientSupplier.getAdmin(config.getAdminConfigs(ClientUtils.getSharedAdminClientId(clientId))); - final Map threadState = new HashMap<>(numStreamThreads); - final ArrayList storeProviders = new ArrayList<>(); - for (int i = 0; i < numStreamThreads; i++) { - final StreamThread streamThread = StreamThread.create( - internalTopologyBuilder, - config, - clientSupplier, - adminClient, - processId, - clientId, - streamsMetrics, - time, - streamsMetadataState, - cacheSizePerThread, - stateDirectory, - delegatingStateRestoreListener, - i + 1, - KafkaStreams.this::closeToError, - this::defaultStreamsUncaughtExceptionHandler - ); - threads.add(streamThread); - threadState.put(streamThread.getId(), streamThread.state()); - storeProviders.add(new StreamThreadStateStoreProvider(streamThread)); - } - - ClientMetrics.addNumAliveStreamThreadMetric(streamsMetrics, (metricsConfig, now) -> - Math.toIntExact(threads.stream().filter(thread -> thread.state().isAlive()).count())); - - final StreamStateListener streamStateListener = new StreamStateListener(threadState, globalThreadState); + threadState = new HashMap<>(numStreamThreads); + storeProviders = new ArrayList<>(); + streamStateListener = new StreamStateListener(threadState, globalThreadState); if (hasGlobalTopology) { globalStreamThread.setStateListener(streamStateListener); } - for (final StreamThread thread : threads) { - thread.setStateListener(streamStateListener); + for (int i = 1; i <= numStreamThreads; i++) { + createStreamThread(cacheSizePerThread, i); } + ClientMetrics.addNumAliveStreamThreadMetric(streamsMetrics, (metricsConfig, now) -> + Math.toIntExact(threads.stream().filter(thread -> thread.state().isAlive()).count())); + final GlobalStateStoreProvider globalStateStoreProvider = new GlobalStateStoreProvider(internalTopologyBuilder.globalStateStores()); queryableStoreProvider = new QueryableStoreProvider(storeProviders, globalStateStoreProvider); stateDirCleaner = setupStateDirCleaner(); - oldHandler = false; maybeWarnAboutCodeInRocksDBConfigSetter(log, config); rocksDBMetricsRecordingService = maybeCreateRocksDBMetricsRecordingService(clientId, config); } + private StreamThread createStreamThread(final long cacheSizePerThread, final int threadIdx) { + final StreamThread streamThread = StreamThread.create( + internalTopologyBuilder, + config, + clientSupplier, + adminClient, + processId, + clientId, + streamsMetrics, + time, + streamsMetadataState, + cacheSizePerThread, + stateDirectory, + delegatingStateRestoreListener, + threadIdx, + KafkaStreams.this::closeToError, + streamsUncaughtExceptionHandler + ); + streamThread.setStateListener(streamStateListener); + threads.add(streamThread); + threadState.put(streamThread.getId(), streamThread.state()); + storeProviders.add(new StreamThreadStateStoreProvider(streamThread)); + return streamThread; + } + + /** + * Adds and starts a stream thread in addition to the stream threads that are already running in this + * Kafka Streams client. + *

+ * Since the number of stream threads increases, the sizes of the caches in the new stream thread + * and the existing stream threads are adapted so that the sum of the cache sizes over all stream + * threads does not exceed the total cache size specified in configuration + * {@link StreamsConfig#CACHE_MAX_BYTES_BUFFERING_CONFIG}. + *

+ * Stream threads can only be added if this Kafka Streams client is in state RUNNING or REBALANCING. + * + * @return name of the added stream thread or empty if a new stream thread could not be added + */ + public Optional addStreamThread() { + synchronized (changeThreadCount) { + if (isRunningOrRebalancing()) { + final int threadIdx = getNextThreadIndex(); + final long cacheSizePerThread = getCacheSizePerThread(threads.size() + 1); + resizeThreadCache(cacheSizePerThread); + final StreamThread streamThread = createStreamThread(cacheSizePerThread, threadIdx); + synchronized (stateLock) { + if (isRunningOrRebalancing()) { + streamThread.start(); + return Optional.of(streamThread.getName()); + } else { + streamThread.shutdown(); + threads.remove(streamThread); + resizeThreadCache(getCacheSizePerThread(threads.size())); + return Optional.empty(); + } + } + } + } + return Optional.empty(); + } + + private int getNextThreadIndex() { + final HashSet names = new HashSet<>(); + for (final StreamThread streamThread: threads) { + names.add(streamThread.getName()); + } + final String baseName = clientId + "-StreamThread-"; + for (int i = 1; i <= threads.size(); i++) { + final String name = baseName + i; + if (!names.contains(name)) { + return i; + } + } + return threads.size() + 1; + } + private long getCacheSizePerThread(final int numStreamThreads) { return totalCacheSize / (numStreamThreads + ((globalTaskTopology != null) ? 1 : 0)); } - private void resizeThreadCache(final int numStreamThreads) { - final long cacheSizePreThread = getCacheSizePerThread(numStreamThreads); + private void resizeThreadCache(final long cacheSizePerThread) { for (final StreamThread streamThread: threads) { - streamThread.resizeCache(cacheSizePreThread); + streamThread.resizeCache(cacheSizePerThread); } } diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/ThreadCache.java b/streams/src/main/java/org/apache/kafka/streams/state/internals/ThreadCache.java index af3d767011329..9e99d3b26ecf0 100644 --- a/streams/src/main/java/org/apache/kafka/streams/state/internals/ThreadCache.java +++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/ThreadCache.java @@ -76,6 +76,9 @@ public void resize(final long newCacheSizeBytes) { final boolean shrink = newCacheSizeBytes < maxCacheSizeBytes; maxCacheSizeBytes = newCacheSizeBytes; if (shrink) { + if (caches.values().isEmpty()) { + return; + } final CircularIterator circularIterator = new CircularIterator<>(caches.values()); while (sizeBytes() > maxCacheSizeBytes) { final NamedCache cache = circularIterator.next(); diff --git a/streams/src/test/java/org/apache/kafka/streams/KafkaStreamsTest.java b/streams/src/test/java/org/apache/kafka/streams/KafkaStreamsTest.java index b2e397191a05c..20de1047b57e8 100644 --- a/streams/src/test/java/org/apache/kafka/streams/KafkaStreamsTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/KafkaStreamsTest.java @@ -78,6 +78,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Properties; import java.util.UUID; import java.util.concurrent.Executors; @@ -312,6 +313,9 @@ private void prepareStreamThread(final StreamThread thread, final boolean termin StreamThread.State.PARTITIONS_ASSIGNED); return null; }).anyTimes(); + thread.resizeCache(EasyMock.anyLong()); + EasyMock.expectLastCall().anyTimes(); + EasyMock.expect(thread.getName()).andStubReturn("newThread"); thread.shutdown(); EasyMock.expectLastCall().andAnswer(() -> { supplier.consumer.close(); @@ -588,6 +592,46 @@ public void testCloseIsIdempotent() { closeCount, MockMetricsReporter.CLOSE_COUNT.get()); } + @Test + public void shouldAddThreadWhenRunning() throws InterruptedException { + props.put(StreamsConfig.NUM_STREAM_THREADS_CONFIG, 1); + final KafkaStreams streams = new KafkaStreams(getBuilderWithSource().build(), props, supplier, time); + streams.start(); + final int oldSize = streams.threads.size(); + TestUtils.waitForCondition(() -> streams.state() == KafkaStreams.State.RUNNING, 15L, "wait until running"); + assertThat(streams.addStreamThread(), equalTo(Optional.of("newThread"))); + assertThat(streams.threads.size(), equalTo(oldSize + 1)); + } + + @Test + public void shouldNotAddThreadWhenCreated() { + final KafkaStreams streams = new KafkaStreams(getBuilderWithSource().build(), props, supplier, time); + final int oldSize = streams.threads.size(); + assertThat(streams.addStreamThread(), equalTo(Optional.empty())); + assertThat(streams.threads.size(), equalTo(oldSize)); + } + + @Test + public void shouldNotAddThreadWhenClosed() { + final KafkaStreams streams = new KafkaStreams(getBuilderWithSource().build(), props, supplier, time); + final int oldSize = streams.threads.size(); + streams.close(); + assertThat(streams.addStreamThread(), equalTo(Optional.empty())); + assertThat(streams.threads.size(), equalTo(oldSize)); + } + + @Test + public void shouldNotAddThreadWhenError() { + final KafkaStreams streams = new KafkaStreams(getBuilderWithSource().build(), props, supplier, time); + final int oldSize = streams.threads.size(); + streams.start(); + streamThreadOne.shutdown(); + streamThreadTwo.shutdown(); + assertThat(streams.addStreamThread(), equalTo(Optional.empty())); + assertThat(streams.threads.size(), equalTo(oldSize)); + } + + @Test public void testCannotStartOnceClosed() { final KafkaStreams streams = new KafkaStreams(getBuilderWithSource().build(), props, supplier, time); diff --git a/streams/src/test/java/org/apache/kafka/streams/StreamsBuilderTest.java b/streams/src/test/java/org/apache/kafka/streams/StreamsBuilderTest.java index e24ad1d427d33..8a9c165c5cb3a 100644 --- a/streams/src/test/java/org/apache/kafka/streams/StreamsBuilderTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/StreamsBuilderTest.java @@ -112,7 +112,7 @@ public void process(final Record record) { } } ); - try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), new Properties())) { + try (final TopologyTestDriver driver = new TopologyTestDriver(builder.build())) { final TestInputTopic inputTopic = driver.createInputTopic("topic", new StringSerializer(), new StringSerializer()); inputTopic.pipeInput("hey", "there"); diff --git a/streams/src/test/java/org/apache/kafka/streams/TopologyTest.java b/streams/src/test/java/org/apache/kafka/streams/TopologyTest.java index 3fe80f90f4b1e..f42e0b9f14800 100644 --- a/streams/src/test/java/org/apache/kafka/streams/TopologyTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/TopologyTest.java @@ -37,7 +37,6 @@ import org.apache.kafka.test.MockApiProcessorSupplier; import org.apache.kafka.test.MockKeyValueStore; import org.apache.kafka.test.MockProcessorSupplier; -import org.apache.kafka.test.TestUtils; import org.easymock.EasyMock; import org.junit.Assert; import org.junit.Test; @@ -48,9 +47,6 @@ import java.util.Set; import java.util.regex.Pattern; -import static org.apache.kafka.common.utils.Utils.mkEntry; -import static org.apache.kafka.common.utils.Utils.mkMap; -import static org.apache.kafka.common.utils.Utils.mkProperties; import static java.time.Duration.ofMillis; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.MatcherAssert.assertThat; @@ -353,8 +349,7 @@ public void shouldThrowOnUnassignedStateStoreAccess() { .addProcessor(badNodeName, new LocalMockProcessorSupplier(), sourceNodeName); try { - new TopologyTestDriver(topology, mkProperties( - mkMap(mkEntry(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getAbsolutePath())))); + new TopologyTestDriver(topology); fail("Should have thrown StreamsException"); } catch (final StreamsException e) { final String error = e.toString(); diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/AdjustStreamThreadCountTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/AdjustStreamThreadCountTest.java new file mode 100644 index 0000000000000..649fc86b82511 --- /dev/null +++ b/streams/src/test/java/org/apache/kafka/streams/integration/AdjustStreamThreadCountTest.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.streams.integration; + +import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.streams.KafkaStreams; +import org.apache.kafka.streams.StreamsBuilder; +import org.apache.kafka.streams.StreamsConfig; +import org.apache.kafka.streams.integration.utils.EmbeddedKafkaCluster; +import org.apache.kafka.streams.integration.utils.IntegrationTestUtils; +import org.apache.kafka.streams.processor.ThreadMetadata; +import org.apache.kafka.test.IntegrationTest; +import org.apache.kafka.test.StreamsTestUtils; +import org.apache.kafka.test.TestUtils; +import org.hamcrest.CoreMatchers; +import org.junit.After; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.TestName; + +import java.io.IOException; +import java.util.Optional; +import java.util.Properties; + +import static org.apache.kafka.common.utils.Utils.mkEntry; +import static org.apache.kafka.common.utils.Utils.mkMap; +import static org.apache.kafka.common.utils.Utils.mkObjectProperties; +import static org.apache.kafka.streams.integration.utils.IntegrationTestUtils.purgeLocalStreamsState; +import static org.apache.kafka.streams.integration.utils.IntegrationTestUtils.safeUniqueTestName; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.MatcherAssert.assertThat; + +@Category(IntegrationTest.class) +public class AdjustStreamThreadCountTest { + + @ClassRule + public static final EmbeddedKafkaCluster CLUSTER = new EmbeddedKafkaCluster(1); + + @Rule + public TestName testName = new TestName(); + + private static String inputTopic; + private static StreamsBuilder builder; + private static Properties properties; + private static String appId = ""; + + @Before + public void setup() { + final String testId = safeUniqueTestName(getClass(), testName); + appId = "appId_" + testId; + inputTopic = "input" + testId; + IntegrationTestUtils.cleanStateBeforeTest(CLUSTER, inputTopic); + + builder = new StreamsBuilder(); + builder.stream(inputTopic); + + properties = mkObjectProperties( + mkMap( + mkEntry(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers()), + mkEntry(StreamsConfig.APPLICATION_ID_CONFIG, appId), + mkEntry(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getPath()), + mkEntry(StreamsConfig.NUM_STREAM_THREADS_CONFIG, 2), + mkEntry(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.StringSerde.class), + mkEntry(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.StringSerde.class) + ) + ); + } + + @After + public void teardown() throws IOException { + purgeLocalStreamsState(properties); + } + + @Test + public void shouldAddStreamThread() throws Exception { + try (final KafkaStreams kafkaStreams = new KafkaStreams(builder.build(), properties)) { + StreamsTestUtils.startKafkaStreamsAndWaitForRunningState(kafkaStreams); + final int oldThreadCount = kafkaStreams.localThreadsMetadata().size(); + assertThat(kafkaStreams.localThreadsMetadata().stream().map(t -> t.threadName().split("-StreamThread-")[1]).sorted().toArray(), equalTo(new String[] {"1", "2"})); + + final Optional name = kafkaStreams.addStreamThread(); + + assertThat(name, CoreMatchers.not(Optional.empty())); + TestUtils.waitForCondition( + () -> kafkaStreams.localThreadsMetadata().stream().sequential() + .map(ThreadMetadata::threadName).anyMatch(t -> t.equals(name.orElse(""))), + "Wait for the thread to be added" + ); + assertThat(kafkaStreams.localThreadsMetadata().size(), equalTo(oldThreadCount + 1)); + assertThat(kafkaStreams.localThreadsMetadata().stream().map(t -> t.threadName().split("-StreamThread-")[1]).sorted().toArray(), equalTo(new String[] {"1", "2", "3"})); + TestUtils.waitForCondition(() -> kafkaStreams.state() == KafkaStreams.State.RUNNING, "wait for running"); + } + } +} diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/AbstractStreamTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/AbstractStreamTest.java index 8eb1b4dc58e4f..4bc7779f5ed6a 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/AbstractStreamTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/AbstractStreamTest.java @@ -22,7 +22,6 @@ import org.apache.kafka.test.NoopValueTransformer; import org.apache.kafka.test.NoopValueTransformerWithKey; import org.apache.kafka.streams.StreamsBuilder; -import org.apache.kafka.streams.StreamsConfig; import org.apache.kafka.streams.TestInputTopic; import org.apache.kafka.streams.TopologyTestDriver; import org.apache.kafka.streams.kstream.Consumed; @@ -35,8 +34,8 @@ import org.apache.kafka.streams.processor.Processor; import org.apache.kafka.streams.processor.ProcessorSupplier; import org.apache.kafka.test.MockProcessorSupplier; -import org.apache.kafka.test.TestUtils; import org.junit.Test; + import java.util.Random; import static org.easymock.EasyMock.createMock; @@ -44,9 +43,6 @@ import static org.easymock.EasyMock.replay; import static org.easymock.EasyMock.verify; import static org.junit.Assert.assertTrue; -import static org.apache.kafka.common.utils.Utils.mkEntry; -import static org.apache.kafka.common.utils.Utils.mkMap; -import static org.apache.kafka.common.utils.Utils.mkProperties; public class AbstractStreamTest { @@ -86,8 +82,8 @@ public void testShouldBeExtensible() { stream.randomFilter().process(supplier); - final TopologyTestDriver driver = new TopologyTestDriver(builder.build(), mkProperties( - mkMap(mkEntry(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getAbsolutePath())))); + final TopologyTestDriver driver = new TopologyTestDriver(builder.build()); + final TestInputTopic inputTopic = driver.createInputTopic(topicName, new IntegerSerializer(), new StringSerializer()); for (final int expectedKey : expectedKeys) { inputTopic.pipeInput(expectedKey, "V" + expectedKey); diff --git a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamTransformTest.java b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamTransformTest.java index 2b2b9ba947153..20b1d3a896676 100644 --- a/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamTransformTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/kstream/internals/KStreamTransformTest.java @@ -79,7 +79,6 @@ public void close() { } try (final TopologyTestDriver driver = new TopologyTestDriver( builder.build(), - new Properties(), Instant.ofEpochMilli(0L))) { final TestInputTopic inputTopic = driver.createInputTopic(TOPIC_NAME, new IntegerSerializer(), new IntegerSerializer()); diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorNodeTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorNodeTest.java index 398b3a45c0417..ed445a747ee91 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorNodeTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/ProcessorNodeTest.java @@ -36,7 +36,7 @@ import java.util.Collections; import java.util.LinkedHashMap; import java.util.Map; -import java.util.Properties; + import static org.apache.kafka.streams.processor.internals.metrics.StreamsMetricsImpl.ROLLUP_VALUE; import static org.hamcrest.CoreMatchers.containsString; @@ -171,7 +171,7 @@ public void testTopologyLevelClassCastException() { }); final Topology topology = builder.build(); - final TopologyTestDriver testDriver = new TopologyTestDriver(topology, new Properties()); + final TopologyTestDriver testDriver = new TopologyTestDriver(topology); final TestInputTopic topic = testDriver.createInputTopic("streams-plaintext-input", new StringSerializer(), new StringSerializer()); final StreamsException se = assertThrows(StreamsException.class, () -> topic.pipeInput("a-key", "a value")); diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java index 59bb66f9f210d..157f9e5efcd58 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamThreadTest.java @@ -2518,7 +2518,8 @@ public void runAndVerifyFailedStreamThreadRecording(final boolean shouldFail) { new AtomicInteger(), new AtomicLong(Long.MAX_VALUE), null, - e -> { } + e -> { }, + null ) { @Override void runOnce() { diff --git a/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java b/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java index 876746a0f2998..8a9007651b8a8 100644 --- a/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java +++ b/streams/test-utils/src/main/java/org/apache/kafka/streams/TopologyTestDriver.java @@ -99,6 +99,7 @@ import java.util.Properties; import java.util.Queue; import java.util.Set; +import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Supplier; @@ -242,6 +243,16 @@ public void onBatchRestored(final TopicPartition topicPartition, final String st public void onRestoreEnd(final TopicPartition topicPartition, final String storeName, final long totalRestored) {} }; + /** + * Create a new test diver instance. + * Default test properties are used to initialize the driver instance + * + * @param topology the topology to be tested + */ + public TopologyTestDriver(final Topology topology) { + this(topology, new Properties()); + } + /** * Create a new test diver instance. * Initialized the internally mocked wall-clock time with {@link System#currentTimeMillis() current system time}. @@ -254,6 +265,18 @@ public TopologyTestDriver(final Topology topology, this(topology, config, null); } + /** + * Create a new test diver instance. + * + * @param topology the topology to be tested + * @param initialWallClockTimeMs the initial value of internally mocked wall-clock time + */ + public TopologyTestDriver(final Topology topology, + final Instant initialWallClockTimeMs) { + this(topology, new Properties(), initialWallClockTimeMs); + } + + /** * Create a new test diver instance. * @@ -299,7 +322,8 @@ private TopologyTestDriver(final InternalTopologyBuilder builder, final Properties configCopy = new Properties(); configCopy.putAll(config); configCopy.putIfAbsent(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "dummy-bootstrap-host:0"); - configCopy.putIfAbsent(StreamsConfig.APPLICATION_ID_CONFIG, "dummy-topology-test-driver-app-id"); + // provide randomized dummy app-id if it's not specified + configCopy.putIfAbsent(StreamsConfig.APPLICATION_ID_CONFIG, "dummy-topology-test-driver-app-id-" + ThreadLocalRandom.current().nextInt()); final StreamsConfig streamsConfig = new ClientUtils.QuietStreamsConfig(configCopy); logIfTaskIdleEnabled(streamsConfig); diff --git a/streams/test-utils/src/test/java/org/apache/kafka/streams/TestTopicsTest.java b/streams/test-utils/src/test/java/org/apache/kafka/streams/TestTopicsTest.java index 463bd3fac1ca0..6d24f1f45bea4 100644 --- a/streams/test-utils/src/test/java/org/apache/kafka/streams/TestTopicsTest.java +++ b/streams/test-utils/src/test/java/org/apache/kafka/streams/TestTopicsTest.java @@ -42,7 +42,6 @@ import java.util.List; import java.util.Map; import java.util.NoSuchElementException; -import java.util.Properties; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.equalTo; @@ -75,7 +74,7 @@ public void setup() { final KStream source = builder.stream(INPUT_TOPIC_MAP, Consumed.with(longSerde, stringSerde)); final KStream mapped = source.map((key, value) -> new KeyValue<>(value, key)); mapped.to(OUTPUT_TOPIC_MAP, Produced.with(stringSerde, longSerde)); - testDriver = new TopologyTestDriver(builder.build(), new Properties()); + testDriver = new TopologyTestDriver(builder.build()); } @After diff --git a/streams/test-utils/src/test/java/org/apache/kafka/streams/TopologyTestDriverTest.java b/streams/test-utils/src/test/java/org/apache/kafka/streams/TopologyTestDriverTest.java index 6dc3a3705d375..6cdb65840462c 100644 --- a/streams/test-utils/src/test/java/org/apache/kafka/streams/TopologyTestDriverTest.java +++ b/streams/test-utils/src/test/java/org/apache/kafka/streams/TopologyTestDriverTest.java @@ -471,7 +471,7 @@ public void shouldCloseProcessor() { @Test public void shouldThrowForUnknownTopic() { - testDriver = new TopologyTestDriver(new Topology(), config); + testDriver = new TopologyTestDriver(new Topology()); assertThrows( IllegalArgumentException.class, () -> testDriver.pipeRecord( @@ -485,7 +485,7 @@ public void shouldThrowForUnknownTopic() { @Test public void shouldThrowForMissingTime() { - testDriver = new TopologyTestDriver(new Topology(), config); + testDriver = new TopologyTestDriver(new Topology()); assertThrows( IllegalStateException.class, () -> testDriver.pipeRecord( @@ -506,7 +506,7 @@ public void shouldThrowForUnknownTopicDeprecated() { new ByteArraySerializer(), new ByteArraySerializer()); - testDriver = new TopologyTestDriver(new Topology(), config); + testDriver = new TopologyTestDriver(new Topology()); try { testDriver.pipeInput(consumerRecordFactory.create((byte[]) null)); fail("Should have throw IllegalArgumentException"); @@ -517,7 +517,7 @@ public void shouldThrowForUnknownTopicDeprecated() { @Test public void shouldThrowNoSuchElementExceptionForUnusedOutputTopicWithDynamicRouting() { - testDriver = new TopologyTestDriver(setupSourceSinkTopology(), config); + testDriver = new TopologyTestDriver(setupSourceSinkTopology()); final TestOutputTopic outputTopic = new TestOutputTopic<>( testDriver, "unused-topic", @@ -531,7 +531,7 @@ public void shouldThrowNoSuchElementExceptionForUnusedOutputTopicWithDynamicRout @Test public void shouldCaptureSinkTopicNamesIfWrittenInto() { - testDriver = new TopologyTestDriver(setupSourceSinkTopology(), config); + testDriver = new TopologyTestDriver(setupSourceSinkTopology()); assertThat(testDriver.producedTopicNames(), is(Collections.emptySet())); @@ -577,7 +577,7 @@ public void shouldCaptureGlobalTopicNameIfWrittenInto() { builder.globalTable(SOURCE_TOPIC_1, Materialized.as("globalTable")); builder.stream(SOURCE_TOPIC_2).to(SOURCE_TOPIC_1); - testDriver = new TopologyTestDriver(builder.build(), config); + testDriver = new TopologyTestDriver(builder.build()); assertThat(testDriver.producedTopicNames(), is(Collections.emptySet())); @@ -590,7 +590,7 @@ public void shouldCaptureGlobalTopicNameIfWrittenInto() { @Test public void shouldProcessRecordForTopic() { - testDriver = new TopologyTestDriver(setupSourceSinkTopology(), config); + testDriver = new TopologyTestDriver(setupSourceSinkTopology()); pipeRecord(SOURCE_TOPIC_1, testRecord1); final ProducerRecord outputRecord = testDriver.readRecord(SINK_TOPIC_1); @@ -602,7 +602,7 @@ public void shouldProcessRecordForTopic() { @Test public void shouldSetRecordMetadata() { - testDriver = new TopologyTestDriver(setupSingleProcessorTopology(), config); + testDriver = new TopologyTestDriver(setupSingleProcessorTopology()); pipeRecord(SOURCE_TOPIC_1, testRecord1); @@ -624,7 +624,7 @@ private void pipeRecord(final String topic, final TestRecord rec //Test not migrated to non-deprecated methods, topic handling not based on record any more @Test public void shouldSendRecordViaCorrectSourceTopicDeprecated() { - testDriver = new TopologyTestDriver(setupMultipleSourceTopology(SOURCE_TOPIC_1, SOURCE_TOPIC_2), config); + testDriver = new TopologyTestDriver(setupMultipleSourceTopology(SOURCE_TOPIC_1, SOURCE_TOPIC_2)); final List processedRecords1 = mockProcessors.get(0).processedRecords; final List processedRecords2 = mockProcessors.get(1).processedRecords; @@ -677,7 +677,7 @@ public void shouldUseSourceSpecificDeserializersDeprecated() { }, processor); - testDriver = new TopologyTestDriver(topology, config); + testDriver = new TopologyTestDriver(topology); final org.apache.kafka.streams.test.ConsumerRecordFactory source1Factory = new org.apache.kafka.streams.test.ConsumerRecordFactory<>( @@ -739,7 +739,7 @@ public void shouldUseSourceSpecificDeserializers() { }, processor); - testDriver = new TopologyTestDriver(topology, config); + testDriver = new TopologyTestDriver(topology); final Long source1Key = 42L; final String source1Value = "anyString"; @@ -772,7 +772,7 @@ public void shouldUseSourceSpecificDeserializers() { @Test public void shouldPassRecordHeadersIntoSerializersAndDeserializers() { - testDriver = new TopologyTestDriver(setupSourceSinkTopology(), config); + testDriver = new TopologyTestDriver(setupSourceSinkTopology()); final AtomicBoolean passedHeadersToKeySerializer = new AtomicBoolean(false); final AtomicBoolean passedHeadersToValueSerializer = new AtomicBoolean(false); @@ -832,7 +832,7 @@ public void shouldUseSinkSpecificSerializers() { topology.addSink("sink-1", SINK_TOPIC_1, Serdes.Long().serializer(), Serdes.String().serializer(), sourceName1); topology.addSink("sink-2", SINK_TOPIC_2, Serdes.Integer().serializer(), Serdes.Double().serializer(), sourceName2); - testDriver = new TopologyTestDriver(topology, config); + testDriver = new TopologyTestDriver(topology); final Long source1Key = 42L; final String source1Value = "anyString"; @@ -867,7 +867,7 @@ public void shouldUseSinkSpecificSerializers() { //Test not migrated to non-deprecated methods, List processing now in TestInputTopic @Test public void shouldProcessConsumerRecordList() { - testDriver = new TopologyTestDriver(setupMultipleSourceTopology(SOURCE_TOPIC_1, SOURCE_TOPIC_2), config); + testDriver = new TopologyTestDriver(setupMultipleSourceTopology(SOURCE_TOPIC_1, SOURCE_TOPIC_2)); final List processedRecords1 = mockProcessors.get(0).processedRecords; final List processedRecords2 = mockProcessors.get(1).processedRecords; @@ -892,7 +892,7 @@ record = processedRecords2.get(0); @Test public void shouldForwardRecordsFromSubtopologyToSubtopology() { - testDriver = new TopologyTestDriver(setupTopologyWithTwoSubtopologies(), config); + testDriver = new TopologyTestDriver(setupTopologyWithTwoSubtopologies()); pipeRecord(SOURCE_TOPIC_1, testRecord1); @@ -909,7 +909,7 @@ public void shouldForwardRecordsFromSubtopologyToSubtopology() { @Test public void shouldPopulateGlobalStore() { - testDriver = new TopologyTestDriver(setupGlobalStoreTopology(SOURCE_TOPIC_1), config); + testDriver = new TopologyTestDriver(setupGlobalStoreTopology(SOURCE_TOPIC_1)); final KeyValueStore globalStore = testDriver.getKeyValueStore(SOURCE_TOPIC_1 + "-globalStore"); Assert.assertNotNull(globalStore); @@ -924,8 +924,8 @@ public void shouldPopulateGlobalStore() { public void shouldPunctuateOnStreamsTime() { final MockPunctuator mockPunctuator = new MockPunctuator(); testDriver = new TopologyTestDriver( - setupSingleProcessorTopology(10L, PunctuationType.STREAM_TIME, mockPunctuator), - config); + setupSingleProcessorTopology(10L, PunctuationType.STREAM_TIME, mockPunctuator) + ); final List expectedPunctuations = new LinkedList<>(); @@ -1052,7 +1052,7 @@ public void shouldReturnAllStores() { "globalProcessorName", voidProcessorSupplier); - testDriver = new TopologyTestDriver(topology, config); + testDriver = new TopologyTestDriver(topology); final Set expectedStoreNames = new HashSet<>(); expectedStoreNames.add("store"); @@ -1096,7 +1096,7 @@ private void shouldReturnCorrectStoreTypeOnly(final boolean persistent) { globalTimestampedKeyValueStoreName); - testDriver = new TopologyTestDriver(topology, config); + testDriver = new TopologyTestDriver(topology); // verify state stores assertNotNull(testDriver.getKeyValueStore(keyValueStoreName)); @@ -1175,7 +1175,7 @@ private void shouldThrowIfBuiltInStoreIsAccessedWithUntypedMethod(final boolean globalTimestampedKeyValueStoreName); - testDriver = new TopologyTestDriver(topology, config); + testDriver = new TopologyTestDriver(topology); { final IllegalArgumentException e = assertThrows( @@ -1367,7 +1367,7 @@ public void shouldReturnAllStoresNames() { "globalProcessorName", voidProcessorSupplier); - testDriver = new TopologyTestDriver(topology, config); + testDriver = new TopologyTestDriver(topology); final Set expectedStoreNames = new HashSet<>(); expectedStoreNames.add("store"); @@ -1515,7 +1515,7 @@ public void shouldAllowPrePopulatingStatesStoresWithCachingEnabled() { Serdes.Long()).withCachingEnabled(), // intentionally turn on caching to achieve better test coverage "aggregator"); - testDriver = new TopologyTestDriver(topology, config); + testDriver = new TopologyTestDriver(topology); store = testDriver.getKeyValueStore("aggStore"); store.put("a", 21L); @@ -1655,7 +1655,7 @@ public void shouldProcessFromSourceThatMatchPattern() { topology.addSource(sourceName, pattern2Source1); topology.addSink("sink", SINK_TOPIC_1, sourceName); - testDriver = new TopologyTestDriver(topology, config); + testDriver = new TopologyTestDriver(topology); pipeRecord(SOURCE_TOPIC_1, testRecord1); final ProducerRecord outputRecord = testDriver.readRecord(SINK_TOPIC_1); @@ -1674,7 +1674,7 @@ public void shouldThrowPatternNotValidForTopicNameException() { topology.addSource(sourceName, pattern2Source1); topology.addSink("sink", SINK_TOPIC_1, sourceName); - testDriver = new TopologyTestDriver(topology, config); + testDriver = new TopologyTestDriver(topology); try { pipeRecord(SOURCE_TOPIC_1, testRecord1); } catch (final TopologyException exception) { @@ -1737,7 +1737,7 @@ public void process(final Record record) { topology.addSink("recursiveSink", "input", new StringSerializer(), new StringSerializer(), "recursiveProcessor"); topology.addSink("sink", "output", new StringSerializer(), new StringSerializer(), "recursiveProcessor"); - try (final TopologyTestDriver topologyTestDriver = new TopologyTestDriver(topology, new Properties())) { + try (final TopologyTestDriver topologyTestDriver = new TopologyTestDriver(topology)) { final TestInputTopic in = topologyTestDriver.createInputTopic("input", new StringSerializer(), new StringSerializer()); final TestOutputTopic out = topologyTestDriver.createOutputTopic("output", new StringDeserializer(), new StringDeserializer()); @@ -1809,7 +1809,7 @@ public void process(final Record record) { topology.addSink("sink", "output", new StringSerializer(), new StringSerializer(), "recursiveProcessor"); topology.addSink("globalSink", "global-topic", new StringSerializer(), new StringSerializer(), "recursiveProcessor"); - try (final TopologyTestDriver topologyTestDriver = new TopologyTestDriver(topology, new Properties())) { + try (final TopologyTestDriver topologyTestDriver = new TopologyTestDriver(topology)) { final TestInputTopic in = topologyTestDriver.createInputTopic("input", new StringSerializer(), new StringSerializer()); final TestOutputTopic globalTopic = topologyTestDriver.createOutputTopic("global-topic", new StringDeserializer(), new StringDeserializer()); diff --git a/tools/src/main/java/org/apache/kafka/trogdor/workload/RoundTripWorker.java b/tools/src/main/java/org/apache/kafka/trogdor/workload/RoundTripWorker.java index 643555a8e95ee..6f448cc2ed31c 100644 --- a/tools/src/main/java/org/apache/kafka/trogdor/workload/RoundTripWorker.java +++ b/tools/src/main/java/org/apache/kafka/trogdor/workload/RoundTripWorker.java @@ -254,8 +254,8 @@ public void run() { spec.valueGenerator().generate(messageIndex)); producer.send(record, (metadata, exception) -> { if (exception == null) { + lock.lock(); try { - lock.lock(); unackedSends -= 1; if (unackedSends <= 0) unackedSendsAreZero.signalAll(); @@ -272,8 +272,8 @@ public void run() { } catch (Throwable e) { WorkerUtils.abort(log, "ProducerRunnable", e, doneFuture); } finally { + lock.lock(); try { - lock.lock(); log.info("{}: ProducerRunnable is exiting. messagesSent={}; uniqueMessagesSent={}; " + "ackedSends={}/{}.", id, messagesSent, uniqueMessagesSent, spec.maxMessages() - unackedSends, spec.maxMessages()); @@ -359,8 +359,8 @@ public void run() { if (toReceiveTracker.removePending(messageIndex)) { uniqueMessagesReceived++; if (uniqueMessagesReceived >= spec.maxMessages()) { + lock.lock(); try { - lock.lock(); log.info("{}: Consumer received the full count of {} unique messages. " + "Waiting for all {} sends to be acked...", id, spec.maxMessages(), unackedSends); while (unackedSends > 0)