From 8cd184678f471c742a5d313046b0757373489c93 Mon Sep 17 00:00:00 2001 From: "Colin P. Mccabe" Date: Fri, 23 Nov 2018 20:30:46 -0800 Subject: [PATCH 01/14] KAFKA-7609: Add Protocol Generator for Kafka --- .gitignore | 1 + build.gradle | 22 + buildSrc/build.gradle | 26 + .../org/apache/kafka/message/CodeBuffer.java | 85 ++ .../org/apache/kafka/message/FieldSpec.java | 142 ++ .../org/apache/kafka/message/FieldType.java | 281 ++++ .../apache/kafka/message/HeaderGenerator.java | 78 + .../kafka/message/MessageDataGenerator.java | 1279 +++++++++++++++++ .../message/MessageFactoryGenerator.java | 125 ++ .../kafka/message/MessageGenerator.java | 200 +++ .../org/apache/kafka/message/MessageSpec.java | 76 + .../apache/kafka/message/MessageSpecType.java | 31 + .../apache/kafka/message/SchemaGenerator.java | 243 ++++ .../org/apache/kafka/message/StructSpec.java | 70 + .../org/apache/kafka/message/Versions.java | 116 ++ .../kafka/task/ProcessMessagesTask.java | 68 + checkstyle/import-control.xml | 10 + checkstyle/suppressions.xml | 9 +- .../common/protocol/ByteBufferAccessor.java | 78 + .../apache/kafka/common/protocol/Message.java | 97 ++ .../kafka/common/protocol/MessageUtil.java | 60 + .../kafka/common/protocol/Readable.java | 57 + .../kafka/common/protocol/Writable.java | 71 + .../kafka/common/protocol/types/Struct.java | 15 + .../org/apache/kafka/common/utils/Bytes.java | 2 + .../utils/ImplicitLinkedHashMultiSet.java | 142 ++ .../common/utils/ImplicitLinkedHashSet.java | 234 ++- .../message/AddOffsetsToTxnRequest.json | 32 + .../message/AddOffsetsToTxnResponse.json | 28 + .../message/AddPartitionsToTxnRequest.json | 37 + .../message/AddPartitionsToTxnResponse.json | 38 + .../message/AlterConfigsRequest.json | 40 + .../message/AlterConfigsResponse.json | 37 + .../message/AlterReplicaLogDirsRequest.json | 36 + .../message/AlterReplicaLogDirsResponse.json | 38 + .../resources/message/ApiVersionsRequest.json | 24 + .../message/ApiVersionsResponse.json | 38 + .../message/ControlledShutdownRequest.json | 34 + .../message/ControlledShutdownResponse.json | 33 + .../resources/message/CreateAclsRequest.json | 41 + .../resources/message/CreateAclsResponse.json | 33 + .../message/CreateDelegationTokenRequest.json | 33 + .../CreateDelegationTokenResponse.json | 42 + .../message/CreatePartitionsRequest.json | 40 + .../message/CreatePartitionsResponse.json | 35 + .../message/CreateTopicsRequest.json | 51 + .../message/CreateTopicsResponse.json | 37 + .../resources/message/DeleteAclsRequest.json | 41 + .../resources/message/DeleteAclsResponse.json | 55 + .../message/DeleteGroupsRequest.json | 26 + .../message/DeleteGroupsResponse.json | 33 + .../message/DeleteRecordsRequest.json | 38 + .../message/DeleteRecordsResponse.json | 40 + .../message/DeleteTopicsRequest.json | 28 + .../message/DeleteTopicsResponse.json | 35 + .../message/DescribeAclsRequest.json | 38 + .../message/DescribeAclsResponse.json | 51 + .../message/DescribeConfigsRequest.json | 36 + .../message/DescribeConfigsResponse.json | 65 + .../DescribeDelegationTokenRequest.json | 31 + .../DescribeDelegationTokenResponse.json | 52 + .../message/DescribeGroupsRequest.json | 26 + .../message/DescribeGroupsResponse.json | 57 + .../message/DescribeLogDirsRequest.json | 31 + .../message/DescribeLogDirsResponse.json | 48 + .../main/resources/message/EndTxnRequest.json | 32 + .../resources/message/EndTxnResponse.json | 28 + .../message/ExpireDelegationTokenRequest.json | 28 + .../ExpireDelegationTokenResponse.json | 30 + .../main/resources/message/FetchRequest.json | 89 ++ .../main/resources/message/FetchResponse.json | 77 + .../message/FindCoordinatorRequest.json | 29 + .../message/FindCoordinatorResponse.json | 37 + .../resources/message/HeartbeatRequest.json | 30 + .../resources/message/HeartbeatResponse.json | 29 + .../message/InitProducerIdRequest.json | 28 + .../message/InitProducerIdResponse.json | 32 + .../resources/message/JoinGroupRequest.json | 44 + .../resources/message/JoinGroupResponse.json | 44 + .../message/LeaderAndIsrRequest.json | 86 ++ .../message/LeaderAndIsrResponse.json | 37 + .../resources/message/LeaveGroupRequest.json | 28 + .../resources/message/LeaveGroupResponse.json | 29 + .../resources/message/ListGroupsRequest.json | 24 + .../resources/message/ListGroupsResponse.json | 36 + .../resources/message/ListOffsetRequest.json | 48 + .../resources/message/ListOffsetResponse.json | 48 + .../resources/message/MetadataRequest.json | 37 + .../resources/message/MetadataResponse.json | 81 ++ .../message/OffsetCommitRequest.json | 59 + .../message/OffsetCommitResponse.json | 44 + .../resources/message/OffsetFetchRequest.json | 37 + .../message/OffsetFetchResponse.json | 54 + .../message/OffsetForLeaderEpochRequest.json | 40 + .../message/OffsetForLeaderEpochResponse.json | 43 + .../resources/message/ProduceRequest.json | 52 + .../resources/message/ProduceResponse.json | 53 + clients/src/main/resources/message/README.md | 219 +++ .../message/RenewDelegationTokenRequest.json | 28 + .../message/RenewDelegationTokenResponse.json | 30 + .../main/resources/message/RequestHeader.json | 30 + .../resources/message/ResponseHeader.json | 24 + .../message/SaslAuthenticateRequest.json | 26 + .../message/SaslAuthenticateResponse.json | 32 + .../message/SaslHandshakeRequest.json | 26 + .../message/SaslHandshakeResponse.json | 28 + .../resources/message/StopReplicaRequest.json | 47 + .../message/StopReplicaResponse.json | 35 + .../resources/message/SyncGroupRequest.json | 37 + .../resources/message/SyncGroupResponse.json | 31 + .../message/TxnOffsetCommitRequest.json | 50 + .../message/TxnOffsetCommitResponse.json | 39 + .../message/UpdateMetadataRequest.json | 105 ++ .../message/UpdateMetadataResponse.json | 26 + .../message/WriteTxnMarkersRequest.json | 41 + .../message/WriteTxnMarkersResponse.json | 40 + .../common/protocol/MessageUtilTest.java | 59 + .../utils/ImplicitLinkedHashMultiSetTest.java | 163 +++ .../utils/ImplicitLinkedHashSetTest.java | 20 +- .../kafka/message/MessageGeneratorTest.java | 59 + .../org/apache/kafka/message/MessageTest.java | 308 ++++ .../apache/kafka/message/VersionsTest.java | 85 ++ gradle/spotbugs-exclude.xml | 6 + 123 files changed, 7993 insertions(+), 70 deletions(-) create mode 100644 buildSrc/build.gradle create mode 100644 buildSrc/src/main/java/org/apache/kafka/message/CodeBuffer.java create mode 100644 buildSrc/src/main/java/org/apache/kafka/message/FieldSpec.java create mode 100644 buildSrc/src/main/java/org/apache/kafka/message/FieldType.java create mode 100644 buildSrc/src/main/java/org/apache/kafka/message/HeaderGenerator.java create mode 100644 buildSrc/src/main/java/org/apache/kafka/message/MessageDataGenerator.java create mode 100644 buildSrc/src/main/java/org/apache/kafka/message/MessageFactoryGenerator.java create mode 100644 buildSrc/src/main/java/org/apache/kafka/message/MessageGenerator.java create mode 100644 buildSrc/src/main/java/org/apache/kafka/message/MessageSpec.java create mode 100644 buildSrc/src/main/java/org/apache/kafka/message/MessageSpecType.java create mode 100644 buildSrc/src/main/java/org/apache/kafka/message/SchemaGenerator.java create mode 100644 buildSrc/src/main/java/org/apache/kafka/message/StructSpec.java create mode 100644 buildSrc/src/main/java/org/apache/kafka/message/Versions.java create mode 100644 buildSrc/src/main/java/org/apache/kafka/task/ProcessMessagesTask.java create mode 100644 clients/src/main/java/org/apache/kafka/common/protocol/ByteBufferAccessor.java create mode 100644 clients/src/main/java/org/apache/kafka/common/protocol/Message.java create mode 100644 clients/src/main/java/org/apache/kafka/common/protocol/MessageUtil.java create mode 100644 clients/src/main/java/org/apache/kafka/common/protocol/Readable.java create mode 100644 clients/src/main/java/org/apache/kafka/common/protocol/Writable.java create mode 100644 clients/src/main/java/org/apache/kafka/common/utils/ImplicitLinkedHashMultiSet.java create mode 100644 clients/src/main/resources/message/AddOffsetsToTxnRequest.json create mode 100644 clients/src/main/resources/message/AddOffsetsToTxnResponse.json create mode 100644 clients/src/main/resources/message/AddPartitionsToTxnRequest.json create mode 100644 clients/src/main/resources/message/AddPartitionsToTxnResponse.json create mode 100644 clients/src/main/resources/message/AlterConfigsRequest.json create mode 100644 clients/src/main/resources/message/AlterConfigsResponse.json create mode 100644 clients/src/main/resources/message/AlterReplicaLogDirsRequest.json create mode 100644 clients/src/main/resources/message/AlterReplicaLogDirsResponse.json create mode 100644 clients/src/main/resources/message/ApiVersionsRequest.json create mode 100644 clients/src/main/resources/message/ApiVersionsResponse.json create mode 100644 clients/src/main/resources/message/ControlledShutdownRequest.json create mode 100644 clients/src/main/resources/message/ControlledShutdownResponse.json create mode 100644 clients/src/main/resources/message/CreateAclsRequest.json create mode 100644 clients/src/main/resources/message/CreateAclsResponse.json create mode 100644 clients/src/main/resources/message/CreateDelegationTokenRequest.json create mode 100644 clients/src/main/resources/message/CreateDelegationTokenResponse.json create mode 100644 clients/src/main/resources/message/CreatePartitionsRequest.json create mode 100644 clients/src/main/resources/message/CreatePartitionsResponse.json create mode 100644 clients/src/main/resources/message/CreateTopicsRequest.json create mode 100644 clients/src/main/resources/message/CreateTopicsResponse.json create mode 100644 clients/src/main/resources/message/DeleteAclsRequest.json create mode 100644 clients/src/main/resources/message/DeleteAclsResponse.json create mode 100644 clients/src/main/resources/message/DeleteGroupsRequest.json create mode 100644 clients/src/main/resources/message/DeleteGroupsResponse.json create mode 100644 clients/src/main/resources/message/DeleteRecordsRequest.json create mode 100644 clients/src/main/resources/message/DeleteRecordsResponse.json create mode 100644 clients/src/main/resources/message/DeleteTopicsRequest.json create mode 100644 clients/src/main/resources/message/DeleteTopicsResponse.json create mode 100644 clients/src/main/resources/message/DescribeAclsRequest.json create mode 100644 clients/src/main/resources/message/DescribeAclsResponse.json create mode 100644 clients/src/main/resources/message/DescribeConfigsRequest.json create mode 100644 clients/src/main/resources/message/DescribeConfigsResponse.json create mode 100644 clients/src/main/resources/message/DescribeDelegationTokenRequest.json create mode 100644 clients/src/main/resources/message/DescribeDelegationTokenResponse.json create mode 100644 clients/src/main/resources/message/DescribeGroupsRequest.json create mode 100644 clients/src/main/resources/message/DescribeGroupsResponse.json create mode 100644 clients/src/main/resources/message/DescribeLogDirsRequest.json create mode 100644 clients/src/main/resources/message/DescribeLogDirsResponse.json create mode 100644 clients/src/main/resources/message/EndTxnRequest.json create mode 100644 clients/src/main/resources/message/EndTxnResponse.json create mode 100644 clients/src/main/resources/message/ExpireDelegationTokenRequest.json create mode 100644 clients/src/main/resources/message/ExpireDelegationTokenResponse.json create mode 100644 clients/src/main/resources/message/FetchRequest.json create mode 100644 clients/src/main/resources/message/FetchResponse.json create mode 100644 clients/src/main/resources/message/FindCoordinatorRequest.json create mode 100644 clients/src/main/resources/message/FindCoordinatorResponse.json create mode 100644 clients/src/main/resources/message/HeartbeatRequest.json create mode 100644 clients/src/main/resources/message/HeartbeatResponse.json create mode 100644 clients/src/main/resources/message/InitProducerIdRequest.json create mode 100644 clients/src/main/resources/message/InitProducerIdResponse.json create mode 100644 clients/src/main/resources/message/JoinGroupRequest.json create mode 100644 clients/src/main/resources/message/JoinGroupResponse.json create mode 100644 clients/src/main/resources/message/LeaderAndIsrRequest.json create mode 100644 clients/src/main/resources/message/LeaderAndIsrResponse.json create mode 100644 clients/src/main/resources/message/LeaveGroupRequest.json create mode 100644 clients/src/main/resources/message/LeaveGroupResponse.json create mode 100644 clients/src/main/resources/message/ListGroupsRequest.json create mode 100644 clients/src/main/resources/message/ListGroupsResponse.json create mode 100644 clients/src/main/resources/message/ListOffsetRequest.json create mode 100644 clients/src/main/resources/message/ListOffsetResponse.json create mode 100644 clients/src/main/resources/message/MetadataRequest.json create mode 100644 clients/src/main/resources/message/MetadataResponse.json create mode 100644 clients/src/main/resources/message/OffsetCommitRequest.json create mode 100644 clients/src/main/resources/message/OffsetCommitResponse.json create mode 100644 clients/src/main/resources/message/OffsetFetchRequest.json create mode 100644 clients/src/main/resources/message/OffsetFetchResponse.json create mode 100644 clients/src/main/resources/message/OffsetForLeaderEpochRequest.json create mode 100644 clients/src/main/resources/message/OffsetForLeaderEpochResponse.json create mode 100644 clients/src/main/resources/message/ProduceRequest.json create mode 100644 clients/src/main/resources/message/ProduceResponse.json create mode 100644 clients/src/main/resources/message/README.md create mode 100644 clients/src/main/resources/message/RenewDelegationTokenRequest.json create mode 100644 clients/src/main/resources/message/RenewDelegationTokenResponse.json create mode 100644 clients/src/main/resources/message/RequestHeader.json create mode 100644 clients/src/main/resources/message/ResponseHeader.json create mode 100644 clients/src/main/resources/message/SaslAuthenticateRequest.json create mode 100644 clients/src/main/resources/message/SaslAuthenticateResponse.json create mode 100644 clients/src/main/resources/message/SaslHandshakeRequest.json create mode 100644 clients/src/main/resources/message/SaslHandshakeResponse.json create mode 100644 clients/src/main/resources/message/StopReplicaRequest.json create mode 100644 clients/src/main/resources/message/StopReplicaResponse.json create mode 100644 clients/src/main/resources/message/SyncGroupRequest.json create mode 100644 clients/src/main/resources/message/SyncGroupResponse.json create mode 100644 clients/src/main/resources/message/TxnOffsetCommitRequest.json create mode 100644 clients/src/main/resources/message/TxnOffsetCommitResponse.json create mode 100644 clients/src/main/resources/message/UpdateMetadataRequest.json create mode 100644 clients/src/main/resources/message/UpdateMetadataResponse.json create mode 100644 clients/src/main/resources/message/WriteTxnMarkersRequest.json create mode 100644 clients/src/main/resources/message/WriteTxnMarkersResponse.json create mode 100755 clients/src/test/java/org/apache/kafka/common/protocol/MessageUtilTest.java create mode 100644 clients/src/test/java/org/apache/kafka/common/utils/ImplicitLinkedHashMultiSetTest.java create mode 100644 clients/src/test/java/org/apache/kafka/message/MessageGeneratorTest.java create mode 100644 clients/src/test/java/org/apache/kafka/message/MessageTest.java create mode 100644 clients/src/test/java/org/apache/kafka/message/VersionsTest.java diff --git a/.gitignore b/.gitignore index fe191eed44b32..a31643f00af6e 100644 --- a/.gitignore +++ b/.gitignore @@ -52,3 +52,4 @@ docs/generated/ kafkatest.egg-info/ systest/ *.swp +clients/src/generated diff --git a/build.gradle b/build.gradle index 65d774d377486..be77596160dfc 100644 --- a/build.gradle +++ b/build.gradle @@ -839,6 +839,7 @@ project(':clients') { testRuntime libs.slf4jlog4j testRuntime libs.jacksonDatabind testRuntime libs.jacksonJDK8Datatypes + testCompile libs.jacksonJaxrsJsonProvider } task determineCommitId { @@ -887,6 +888,27 @@ project(':clients') { delete "$buildDir/kafka/" } + task processMessages(type:org.apache.kafka.task.ProcessMessagesTask) { + inputDirectory = file("src/main/resources/message") + outputDirectory = file("src/generated/java/org/apache/kafka/message") + } + + sourceSets { + main { + java { + srcDirs = ["src/generated/java", "src/main/java"] + } + } + test { + java { + srcDirs = ["src/generated/java", "src/test/java", + "$rootDir/buildSrc/src/main/java/org/apache/kafka/message/"] + } + } + } + + compileJava.dependsOn 'processMessages' + javadoc { include "**/org/apache/kafka/clients/admin/*" include "**/org/apache/kafka/clients/consumer/*" diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle new file mode 100644 index 0000000000000..2c53f6c0a2cfd --- /dev/null +++ b/buildSrc/build.gradle @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +repositories { + mavenCentral() +} + +dependencies { + compile "com.fasterxml.jackson.core:jackson-databind:2.9.6" + compile "com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:2.9.6" +} + +// Workaround for https://github.com/gradle/gradle/issues/1271 +test.enabled=false diff --git a/buildSrc/src/main/java/org/apache/kafka/message/CodeBuffer.java b/buildSrc/src/main/java/org/apache/kafka/message/CodeBuffer.java new file mode 100644 index 0000000000000..fa3b2db6e9479 --- /dev/null +++ b/buildSrc/src/main/java/org/apache/kafka/message/CodeBuffer.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.message; + +import java.io.IOException; +import java.io.Writer; +import java.util.ArrayList; + +public class CodeBuffer { + private final ArrayList lines; + private int indent; + + public CodeBuffer() { + this.lines = new ArrayList<>(); + this.indent = 0; + } + + public int indent() { + return indent; + } + + public void incrementIndent() { + indent++; + } + + public void decrementIndent() { + indent--; + if (indent < 0) { + throw new RuntimeException("Indent < 0"); + } + } + + public void printf(String format, Object... args) throws IOException { + lines.add(String.format(indentSpaces() + format, args)); + } + + public void write(Writer writer) throws IOException { + for (String line : lines) { + writer.write(line); + } + } + + public void write(CodeBuffer other) { + for (String line : lines) { + other.lines.add(other.indentSpaces() + line); + } + } + + private String indentSpaces() { + StringBuilder bld = new StringBuilder(); + for (int i = 0; i < indent; i++) { + bld.append(" "); + } + return bld.toString(); + } + + @Override + public boolean equals(Object other) { + if ((other == null) || (!(other instanceof CodeBuffer))) { + return false; + } + CodeBuffer o = (CodeBuffer) other; + return lines.equals(o.lines); + } + + @Override + public int hashCode() { + return lines.hashCode(); + } +} diff --git a/buildSrc/src/main/java/org/apache/kafka/message/FieldSpec.java b/buildSrc/src/main/java/org/apache/kafka/message/FieldSpec.java new file mode 100644 index 0000000000000..365925698cec9 --- /dev/null +++ b/buildSrc/src/main/java/org/apache/kafka/message/FieldSpec.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.message; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.List; +import java.util.Objects; + +public final class FieldSpec { + private final StructSpec struct; + + private final FieldType type; + + private final boolean mapKey; + + private final Versions nullableVersions; + + private final String fieldDefault; + + private final boolean ignorable; + + private final String about; + + @JsonCreator + public FieldSpec(@JsonProperty("name") String name, + @JsonProperty("versions") String versions, + @JsonProperty("fields") List fields, + @JsonProperty("type") String type, + @JsonProperty("mapKey") boolean mapKey, + @JsonProperty("nullableVersions") String nullableVersions, + @JsonProperty("default") String fieldDefault, + @JsonProperty("ignorable") boolean ignorable, + @JsonProperty("about") String about) { + this.struct = new StructSpec(name, versions, fields); + this.type = FieldType.parse(Objects.requireNonNull(type)); + this.mapKey = mapKey; + this.nullableVersions = Versions.parse(nullableVersions, Versions.NONE); + if (!this.nullableVersions.empty()) { + if (!this.type.canBeNullable()) { + throw new RuntimeException("Type " + this.type + " cannot be nullable."); + } + } + this.fieldDefault = fieldDefault == null ? "" : fieldDefault; + this.ignorable = ignorable; + this.about = about == null ? "" : about; + if (!this.struct.fields().isEmpty()) { + if (!this.type.isArray()) { + throw new RuntimeException("Non-array field " + name + " cannot have fields"); + } + } + } + + public StructSpec struct() { + return struct; + } + + @JsonProperty("name") + public String name() { + return struct.name(); + } + + String capitalizedCamelCaseName() { + return MessageGenerator.capitalizeFirst(struct.name()); + } + + String camelCaseName() { + return MessageGenerator.lowerCaseFirst(struct.name()); + } + + String snakeCaseName() { + return MessageGenerator.toSnakeCase(struct.name()); + } + + @JsonProperty("versions") + public String versionsString() { + return struct.versionsString(); + } + + @JsonProperty("fields") + public List fields() { + return struct.fields(); + } + + @JsonProperty("type") + public String typeString() { + return type.toString(); + } + + public FieldType type() { + return type; + } + + @JsonProperty("mapKey") + public boolean mapKey() { + return mapKey; + } + + public Versions nullableVersions() { + return nullableVersions; + } + + @JsonProperty("nullableVersions") + public String nullableVersionsString() { + return nullableVersions.toString(); + } + + @JsonProperty("default") + public String defaultString() { + return fieldDefault; + } + + boolean hasKeys() { + return struct.hasKeys(); + } + + @JsonProperty("ignorable") + public boolean ignorable() { + return ignorable; + } + + @JsonProperty("about") + public String about() { + return about; + } +}; diff --git a/buildSrc/src/main/java/org/apache/kafka/message/FieldType.java b/buildSrc/src/main/java/org/apache/kafka/message/FieldType.java new file mode 100644 index 0000000000000..ad21adc934e6b --- /dev/null +++ b/buildSrc/src/main/java/org/apache/kafka/message/FieldType.java @@ -0,0 +1,281 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.message; + +import java.util.Optional; + +public interface FieldType { + static final String STRUCT_PREFIX = "[]"; + + final class BoolFieldType implements FieldType { + private static final BoolFieldType INSTANCE = new BoolFieldType(); + private static final String NAME = "bool"; + + @Override + public Optional fixedLength() { + return Optional.of(1); + } + + @Override + public String toString() { + return NAME; + } + } + + final class Int8FieldType implements FieldType { + private static final Int8FieldType INSTANCE = new Int8FieldType(); + private static final String NAME = "int8"; + + @Override + public Optional fixedLength() { + return Optional.of(1); + } + + @Override + public String toString() { + return NAME; + } + } + + final class Int16FieldType implements FieldType { + private static final Int16FieldType INSTANCE = new Int16FieldType(); + private static final String NAME = "int16"; + + @Override + public Optional fixedLength() { + return Optional.of(2); + } + + @Override + public String toString() { + return NAME; + } + } + + final class Int32FieldType implements FieldType { + private static final Int32FieldType INSTANCE = new Int32FieldType(); + private static final String NAME = "int32"; + + @Override + public Optional fixedLength() { + return Optional.of(4); + } + + @Override + public String toString() { + return NAME; + } + } + + final class Int64FieldType implements FieldType { + private static final Int64FieldType INSTANCE = new Int64FieldType(); + private static final String NAME = "int64"; + + @Override + public Optional fixedLength() { + return Optional.of(8); + } + + @Override + public String toString() { + return NAME; + } + } + + final class StringFieldType implements FieldType { + private static final StringFieldType INSTANCE = new StringFieldType(); + private static final String NAME = "string"; + + @Override + public boolean isString() { + return true; + } + + @Override + public boolean canBeNullable() { + return true; + } + + @Override + public String toString() { + return NAME; + } + } + + final class BytesFieldType implements FieldType { + private static final BytesFieldType INSTANCE = new BytesFieldType(); + private static final String NAME = "bytes"; + + @Override + public boolean isBytes() { + return true; + } + + @Override + public boolean canBeNullable() { + return true; + } + + @Override + public String toString() { + return NAME; + } + } + + final class StructType implements FieldType { + private final String type; + + StructType(String type) { + this.type = type; + } + + @Override + public boolean isStruct() { + return true; + } + + @Override + public String toString() { + return type; + } + } + + final class ArrayType implements FieldType { + private final FieldType elementType; + + ArrayType(FieldType elementType) { + this.elementType = elementType; + } + + @Override + public boolean isArray() { + return true; + } + + @Override + public boolean isStructArray() { + return elementType.isStruct(); + } + + @Override + public boolean canBeNullable() { + return true; + } + + public FieldType elementType() { + return elementType; + } + + @Override + public String toString() { + return "[]" + elementType.toString(); + } + } + + static FieldType parse(String string) { + string = string.trim(); + switch (string) { + case BoolFieldType.NAME: + return BoolFieldType.INSTANCE; + case Int8FieldType.NAME: + return Int8FieldType.INSTANCE; + case Int16FieldType.NAME: + return Int16FieldType.INSTANCE; + case Int32FieldType.NAME: + return Int32FieldType.INSTANCE; + case Int64FieldType.NAME: + return Int64FieldType.INSTANCE; + case StringFieldType.NAME: + return StringFieldType.INSTANCE; + case BytesFieldType.NAME: + return BytesFieldType.INSTANCE; + default: + if (string.startsWith(STRUCT_PREFIX)) { + String elementTypeString = string.substring(STRUCT_PREFIX.length()); + if (string.length() == 0) { + throw new RuntimeException("Can't parse array type " + string + + ". No element type found."); + } + FieldType elementType = parse(elementTypeString); + if (elementType.isArray()) { + throw new RuntimeException("Can't have an array of arrays. " + + "Use an array of structs containing an array instead."); + } + return new ArrayType(elementType); + } else if (MessageGenerator.firstIsCapitalized(string)) { + return new StructType(string); + } else { + throw new RuntimeException("Can't parse type " + string); + } + } + } + + /** + * Returns true if this is an array type. + */ + default boolean isArray() { + return false; + } + + /** + * Returns true if this is an array of structures. + */ + default boolean isStructArray() { + return false; + } + + /** + * Returns true if this is a string type. + */ + default boolean isString() { + return false; + } + + /** + * Returns true if this is a bytes type. + */ + default boolean isBytes() { + return false; + } + + /** + * Returns true if this is a struct type. + */ + default boolean isStruct() { + return false; + } + + /** + * Returns true if this field type is compatible with nullability. + */ + default boolean canBeNullable() { + return false; + } + + /** + * Gets the fixed length of the field, or None if the field is variable-length. + */ + default Optional fixedLength() { + return Optional.empty(); + } + + /** + * Convert the field type to a JSON string. + */ + String toString(); +}; diff --git a/buildSrc/src/main/java/org/apache/kafka/message/HeaderGenerator.java b/buildSrc/src/main/java/org/apache/kafka/message/HeaderGenerator.java new file mode 100644 index 0000000000000..b1099084042c5 --- /dev/null +++ b/buildSrc/src/main/java/org/apache/kafka/message/HeaderGenerator.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.message; + +import java.util.TreeSet; + +/** + * The Kafka header generator. + */ +public final class HeaderGenerator { + private static final String[] HEADER = new String[] { + "/*", + " * Licensed to the Apache Software Foundation (ASF) under one or more", + " * contributor license agreements. See the NOTICE file distributed with", + " * this work for additional information regarding copyright ownership.", + " * The ASF licenses this file to You under the Apache License, Version 2.0", + " * (the \"License\"); you may not use this file except in compliance with", + " * the License. You may obtain a copy of the License at", + " *", + " * http://www.apache.org/licenses/LICENSE-2.0", + " *", + " * Unless required by applicable law or agreed to in writing, software", + " * distributed under the License is distributed on an \"AS IS\" BASIS,", + " * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.", + " * See the License for the specific language governing permissions and", + " * limitations under the License.", + " */", + "", + "// THIS CODE IS AUTOMATICALLY GENERATED. DO NOT EDIT.", + "" + }; + + static final String PACKAGE = "org.apache.kafka.message"; + + private final CodeBuffer buffer; + + private final TreeSet imports; + + public HeaderGenerator() { + this.buffer = new CodeBuffer(); + this.imports = new TreeSet<>(); + } + + public void addImport(String newImport) { + this.imports.add(newImport); + } + + public void generate() throws Exception { + for (int i = 0; i < HEADER.length; i++) { + buffer.printf("%s%n", HEADER[i]); + } + buffer.printf("package %s;%n", PACKAGE); + buffer.printf("%n"); + for (String newImport : imports) { + buffer.printf("import %s;%n", newImport); + } + buffer.printf("%n"); + } + + public CodeBuffer buffer() { + return buffer; + } +}; diff --git a/buildSrc/src/main/java/org/apache/kafka/message/MessageDataGenerator.java b/buildSrc/src/main/java/org/apache/kafka/message/MessageDataGenerator.java new file mode 100644 index 0000000000000..711fed84298c2 --- /dev/null +++ b/buildSrc/src/main/java/org/apache/kafka/message/MessageDataGenerator.java @@ -0,0 +1,1279 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.message; + +import java.io.Writer; +import java.util.HashSet; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Generates Kafka MessageData classes. + */ +public final class MessageDataGenerator { + private final HeaderGenerator headerGenerator; + private final SchemaGenerator schemaGenerator; + private final CodeBuffer buffer; + + MessageDataGenerator() { + this.headerGenerator = new HeaderGenerator(); + this.schemaGenerator = new SchemaGenerator(headerGenerator); + this.buffer = new CodeBuffer(); + } + + void generate(MessageSpec message) throws Exception { + if (message.struct().versions().contains(Short.MAX_VALUE)) { + throw new RuntimeException("Message " + message.name() + " does " + + "not specify a maximum version."); + } + schemaGenerator.generateSchemas(message); + generateClass(Optional.of(message), + message.name() + "Data", + message.struct(), + message.struct().versions()); + headerGenerator.generate(); + } + + void write(Writer writer) throws Exception { + headerGenerator.buffer().write(writer); + buffer.write(writer); + } + + private void generateClass(Optional topLevelMessageSpec, + String className, + StructSpec struct, + Versions parentVersions) throws Exception { + buffer.printf("%n"); + boolean isTopLevel = topLevelMessageSpec.isPresent(); + boolean isSetElement = struct.hasKeys(); // Check if the class is inside a set. + if (isTopLevel && isSetElement) { + throw new RuntimeException("Cannot set mapKey on top level fields."); + } + generateClassHeader(className, isTopLevel, isSetElement); + buffer.incrementIndent(); + generateFieldDeclarations(struct, isSetElement); + buffer.printf("%n"); + schemaGenerator.writeSchema(className, buffer); + generateClassConstructors(className, struct); + buffer.printf("%n"); + generateShortAccessor("apiKey", isTopLevel ? + topLevelMessageSpec.get().apiKey() : -1); + buffer.printf("%n"); + generateShortAccessor("lowestSupportedVersion", parentVersions.lowest()); + buffer.printf("%n"); + generateShortAccessor("highestSupportedVersion", parentVersions.highest()); + buffer.printf("%n"); + generateClassReader(className, struct, parentVersions); + buffer.printf("%n"); + generateClassWriter(className, struct, parentVersions); + buffer.printf("%n"); + generateClassFromStruct(className, struct, parentVersions); + buffer.printf("%n"); + generateClassToStruct(className, struct, parentVersions); + buffer.printf("%n"); + generateClassSize(className, struct, parentVersions); + buffer.printf("%n"); + generateClassEquals(className, struct, isSetElement); + buffer.printf("%n"); + generateClassHashCode(struct, isSetElement); + buffer.printf("%n"); + generateClassToString(className, struct); + generateFieldAccessors(struct, isSetElement); + generateFieldMutators(struct, className, isSetElement); + + if (!isTopLevel) { + buffer.decrementIndent(); + buffer.printf("}%n"); + } + generateSubclasses(className, struct, parentVersions, isSetElement); + if (isTopLevel) { + buffer.decrementIndent(); + buffer.printf("}%n"); + } + } + + private void generateClassHeader(String className, boolean isTopLevel, + boolean isSetElement) throws Exception { + Set implementedInterfaces = new HashSet<>(); + implementedInterfaces.add("Message"); + headerGenerator.addImport(MessageGenerator.MESSAGE_CLASS); + if (isSetElement) { + headerGenerator.addImport(MessageGenerator.IMPLICIT_LINKED_HASH_MULTI_SET_CLASS); + implementedInterfaces.add("ImplicitLinkedHashMultiSet.Element"); + } + Set classModifiers = new HashSet<>(); + classModifiers.add("public"); + if (!isTopLevel) { + classModifiers.add("static"); + } + buffer.printf("%s class %s implements %s {%n", + String.join(" ", classModifiers), + className, + String.join(", ", implementedInterfaces)); + } + + private void generateSubclasses(String className, StructSpec struct, + Versions parentVersions, boolean isSetElement) throws Exception { + for (FieldSpec field : struct.fields()) { + if (field.type().isStructArray()) { + FieldType.ArrayType arrayType = (FieldType.ArrayType) field.type(); + generateClass(Optional.empty(), + arrayType.elementType().toString(), + field.struct(), + parentVersions.intersect(struct.versions())); + } + } + if (isSetElement) { + generateHashSet(className, struct); + } + } + + private void generateHashSet(String className, StructSpec struct) throws Exception { + buffer.printf("%n"); + headerGenerator.addImport(MessageGenerator.IMPLICIT_LINKED_HASH_MULTI_SET_CLASS); + buffer.printf("public static class %s extends ImplicitLinkedHashMultiSet<%s> {%n", + hashSetType(className), className); + buffer.incrementIndent(); + generateHashSetZeroArgConstructor(className); + generateHashSetSizeArgConstructor(className); + generateHashSetIteratorConstructor(className); + generateHashSetFindMethod(className, struct); + generateHashSetFindAllMethod(className, struct); + buffer.decrementIndent(); + buffer.printf("}%n"); + } + + private void generateHashSetZeroArgConstructor(String className) throws Exception { + buffer.printf("public %s() {%n", hashSetType(className)); + buffer.incrementIndent(); + buffer.printf("super();%n"); + buffer.decrementIndent(); + buffer.printf("}%n"); + buffer.printf("%n"); + } + + private void generateHashSetSizeArgConstructor(String className) throws Exception { + buffer.printf("public %s(int expectedNumElements) {%n", hashSetType(className)); + buffer.incrementIndent(); + buffer.printf("super(expectedNumElements);%n"); + buffer.decrementIndent(); + buffer.printf("}%n"); + buffer.printf("%n"); + } + + private void generateHashSetIteratorConstructor(String className) throws Exception { + headerGenerator.addImport(MessageGenerator.ITERATOR_CLASS); + buffer.printf("public %s(Iterator<%s> iterator) {%n", hashSetType(className), className); + buffer.incrementIndent(); + buffer.printf("super(iterator);%n"); + buffer.decrementIndent(); + buffer.printf("}%n"); + buffer.printf("%n"); + } + + private void generateHashSetFindMethod(String className, StructSpec struct) + throws Exception { + headerGenerator.addImport(MessageGenerator.LIST_CLASS); + buffer.printf("public %s find(%s) {%n", className, + commaSeparatedHashSetFieldAndTypes(struct)); + buffer.incrementIndent(); + generateKeyElement(className, struct); + headerGenerator.addImport(MessageGenerator.IMPLICIT_LINKED_HASH_MULTI_SET_CLASS); + buffer.printf("return find(key);%n"); + buffer.decrementIndent(); + buffer.printf("}%n"); + buffer.printf("%n"); + } + + private void generateHashSetFindAllMethod(String className, StructSpec struct) + throws Exception { + headerGenerator.addImport(MessageGenerator.LIST_CLASS); + buffer.printf("public List<%s> findAll(%s) {%n", className, + commaSeparatedHashSetFieldAndTypes(struct)); + buffer.incrementIndent(); + generateKeyElement(className, struct); + headerGenerator.addImport(MessageGenerator.IMPLICIT_LINKED_HASH_MULTI_SET_CLASS); + buffer.printf("return findAll(key);%n"); + buffer.decrementIndent(); + buffer.printf("}%n"); + buffer.printf("%n"); + } + + private void generateKeyElement(String className, StructSpec struct) throws Exception { + buffer.printf("%s key = new %s();%n", className, className); + for (FieldSpec field : struct.fields()) { + if (field.mapKey()) { + buffer.printf("key.set%s(%s);%n", + field.capitalizedCamelCaseName(), + field.camelCaseName()); + } + } + } + + private String commaSeparatedHashSetFieldAndTypes(StructSpec struct) { + return struct.fields().stream(). + filter(f -> f.mapKey()). + map(f -> String.format("%s %s", fieldConcreteJavaType(f), f.camelCaseName())). + collect(Collectors.joining(", ")); + } + + private void generateFieldDeclarations(StructSpec struct, boolean isSetElement) + throws Exception { + for (FieldSpec field : struct.fields()) { + generateFieldDeclaration(field); + } + if (isSetElement) { + buffer.printf("private int next;%n"); + buffer.printf("private int prev;%n"); + } + } + + private void generateFieldDeclaration(FieldSpec field) throws Exception { + buffer.printf("private %s %s;%n", + fieldAbstractJavaType(field), field.camelCaseName()); + } + + private void generateFieldAccessors(StructSpec struct, boolean isSetElement) + throws Exception { + for (FieldSpec field : struct.fields()) { + generateFieldAccessor(field); + } + if (isSetElement) { + buffer.printf("%n"); + buffer.printf("@Override%n"); + generateAccessor("int", "next", "next"); + + buffer.printf("%n"); + buffer.printf("@Override%n"); + generateAccessor("int", "prev", "prev"); + } + } + + private void generateFieldMutators(StructSpec struct, + String className, boolean isSetElement) throws Exception { + for (FieldSpec field : struct.fields()) { + generateFieldMutator(className, field); + } + if (isSetElement) { + buffer.printf("%n"); + buffer.printf("@Override%n"); + generateSetter("int", "setNext", "next"); + + buffer.printf("%n"); + buffer.printf("@Override%n"); + generateSetter("int", "setPrev", "prev"); + } + } + + private static String hashSetType(String baseType) { + return baseType + "Set"; + } + + private String fieldAbstractJavaType(FieldSpec field) { + if (field.type() instanceof FieldType.BoolFieldType) { + return "boolean"; + } else if (field.type() instanceof FieldType.Int8FieldType) { + return "byte"; + } else if (field.type() instanceof FieldType.Int16FieldType) { + return "short"; + } else if (field.type() instanceof FieldType.Int32FieldType) { + return "int"; + } else if (field.type() instanceof FieldType.Int64FieldType) { + return "long"; + } else if (field.type().isString()) { + return "String"; + } else if (field.type().isBytes()) { + return "byte[]"; + } else if (field.type().isStruct()) { + return MessageGenerator.capitalizeFirst(field.typeString()); + } else if (field.type().isArray()) { + FieldType.ArrayType arrayType = (FieldType.ArrayType) field.type(); + if (field.hasKeys()) { + headerGenerator.addImport(MessageGenerator.IMPLICIT_LINKED_HASH_MULTI_SET_CLASS); + return hashSetType(arrayType.elementType().toString()); + } else { + headerGenerator.addImport(MessageGenerator.LIST_CLASS); + return "List<" + getBoxedJavaType(arrayType.elementType()) + ">"; + } + } else { + throw new RuntimeException("Unknown field type " + field.type()); + } + } + + private String fieldConcreteJavaType(FieldSpec field) { + if (field.type().isArray()) { + FieldType.ArrayType arrayType = (FieldType.ArrayType) field.type(); + if (field.hasKeys()) { + headerGenerator.addImport(MessageGenerator.IMPLICIT_LINKED_HASH_MULTI_SET_CLASS); + return hashSetType(arrayType.elementType().toString()); + } else { + headerGenerator.addImport(MessageGenerator.ARRAYLIST_CLASS); + return "ArrayList<" + getBoxedJavaType(arrayType.elementType()) + ">"; + } + } else { + return fieldAbstractJavaType(field); + } + } + + private void generateClassConstructors(String className, StructSpec struct) throws Exception { + headerGenerator.addImport(MessageGenerator.READABLE_CLASS); + buffer.printf("public %s(Readable readable, short version) {%n", className); + buffer.incrementIndent(); + initializeArrayDefaults(struct); + buffer.printf("read(readable, version);%n"); + buffer.decrementIndent(); + buffer.printf("}%n"); + buffer.printf("%n"); + headerGenerator.addImport(MessageGenerator.STRUCT_CLASS); + buffer.printf("public %s(Struct struct, short version) {%n", className); + buffer.incrementIndent(); + initializeArrayDefaults(struct); + buffer.printf("fromStruct(struct, version);%n"); + buffer.decrementIndent(); + buffer.printf("}%n"); + buffer.printf("%n"); + buffer.printf("public %s() {%n", className); + buffer.incrementIndent(); + for (FieldSpec field : struct.fields()) { + buffer.printf("this.%s = %s;%n", + field.camelCaseName(), fieldDefault(field)); + } + buffer.decrementIndent(); + buffer.printf("}%n"); + } + + private void initializeArrayDefaults(StructSpec struct) throws Exception { + for (FieldSpec field : struct.fields()) { + if (field.type().isArray()) { + buffer.printf("this.%s = %s;%n", + field.camelCaseName(), fieldDefault(field)); + } + } + } + + private void generateShortAccessor(String name, short val) throws Exception { + buffer.printf("@Override%n"); + buffer.printf("public short %s() {%n", name); + buffer.incrementIndent(); + buffer.printf("return %d;%n", val); + buffer.decrementIndent(); + buffer.printf("}%n"); + } + + private void generateClassReader(String className, StructSpec struct, + Versions parentVersions) throws Exception { + headerGenerator.addImport(MessageGenerator.READABLE_CLASS); + buffer.printf("@Override%n"); + buffer.printf("public void read(Readable readable, short version) {%n"); + buffer.incrementIndent(); + if (generateInverseVersionCheck(parentVersions, struct.versions())) { + buffer.incrementIndent(); + headerGenerator.addImport(MessageGenerator.UNSUPPORTED_VERSION_EXCEPTION_CLASS); + buffer.printf("throw new UnsupportedVersionException(\"Can't read " + + "version \" + version + \" of %s\");%n", className); + buffer.decrementIndent(); + buffer.printf("}%n"); + } + Versions curVersions = parentVersions.intersect(struct.versions()); + if (curVersions.empty()) { + throw new RuntimeException("Version ranges " + parentVersions + + " and " + struct.versions() + " have no versions in common."); + } + for (FieldSpec field : struct.fields()) { + generateFieldReader(field, curVersions); + } + buffer.decrementIndent(); + buffer.printf("}%n"); + } + + private void generateFieldReader(FieldSpec field, + Versions curVersions) throws Exception { + if (field.type().isArray()) { + boolean maybeAbsent = + generateVersionCheck(curVersions, field.struct().versions()); + if (!maybeAbsent) { + buffer.printf("{%n"); + buffer.incrementIndent(); + } + boolean hasKeys = field.hasKeys(); + buffer.printf("int arrayLength = readable.readInt();%n"); + buffer.printf("if (arrayLength < 0) {%n"); + buffer.incrementIndent(); + buffer.printf("this.%s.clear(%s);%n", + field.camelCaseName(), + hasKeys ? "0" : ""); + buffer.decrementIndent(); + buffer.printf("} else {%n"); + buffer.incrementIndent(); + buffer.printf("this.%s.clear(%s);%n", + field.camelCaseName(), + hasKeys ? "arrayLength" : ""); + buffer.printf("for (int i = 0; i < arrayLength; i++) {%n"); + buffer.incrementIndent(); + FieldType.ArrayType arrayType = (FieldType.ArrayType) field.type(); + buffer.printf("this.%s.add(%s);%n", + field.camelCaseName(), readFieldFromReadable(arrayType.elementType())); + buffer.decrementIndent(); + buffer.printf("}%n"); + buffer.decrementIndent(); + buffer.printf("}%n"); + if (maybeAbsent) { + generateSetDefault(field); + } else { + buffer.decrementIndent(); + buffer.printf("}%n"); + } + } else { + boolean maybeAbsent = + generateVersionCheck(curVersions, field.struct().versions()); + buffer.printf("this.%s = %s;%n", + field.camelCaseName(), + readFieldFromReadable(field.type())); + if (maybeAbsent) { + generateSetDefault(field); + } + } + } + + private String readFieldFromReadable(FieldType type) { + if (type instanceof FieldType.BoolFieldType) { + return "readable.readByte() != 0"; + } else if (type instanceof FieldType.Int8FieldType) { + return "readable.readByte()"; + } else if (type instanceof FieldType.Int16FieldType) { + return "readable.readShort()"; + } else if (type instanceof FieldType.Int32FieldType) { + return "readable.readInt()"; + } else if (type instanceof FieldType.Int64FieldType) { + return "readable.readLong()"; + } else if (type.isString()) { + return "readable.readNullableString()"; + } else if (type.isBytes()) { + return "readable.readNullableBytes()"; + } else if (type.isStruct()) { + return String.format("new %s(readable, version)", type.toString()); + } else { + throw new RuntimeException("Unsupported field type " + type); + } + } + + private void generateClassFromStruct(String className, StructSpec struct, + Versions parentVersions) throws Exception { + headerGenerator.addImport(MessageGenerator.STRUCT_CLASS); + buffer.printf("@Override%n"); + buffer.printf("public void fromStruct(Struct struct, short version) {%n"); + buffer.incrementIndent(); + if (generateInverseVersionCheck(parentVersions, struct.versions())) { + buffer.incrementIndent(); + headerGenerator.addImport(MessageGenerator.UNSUPPORTED_VERSION_EXCEPTION_CLASS); + buffer.printf("throw new UnsupportedVersionException(\"Can't read " + + "version \" + version + \" of %s\");%n", className); + buffer.decrementIndent(); + buffer.printf("}%n"); + } + Versions curVersions = parentVersions.intersect(struct.versions()); + if (curVersions.empty()) { + throw new RuntimeException("Version ranges " + parentVersions + + " and " + struct.versions() + " have no versions in common."); + } + for (FieldSpec field : struct.fields()) { + generateFieldFromStruct(field, curVersions); + } + buffer.decrementIndent(); + buffer.printf("}%n"); + } + + private void generateFieldFromStruct(FieldSpec field, + Versions curVersions) throws Exception { + if (field.type().isArray()) { + boolean maybeAbsent = + generateVersionCheck(curVersions, field.struct().versions()); + if (!maybeAbsent) { + buffer.printf("{%n"); + buffer.incrementIndent(); + } + headerGenerator.addImport(MessageGenerator.STRUCT_CLASS); + buffer.printf("Object[] nestedObjects = struct.getArray(\"%s\");%n", + field.snakeCaseName()); + boolean maybeNull = false; + if (!curVersions.intersect(field.nullableVersions()).empty()) { + maybeNull = true; + buffer.printf("if (nestedObjects == null) {%n", field.camelCaseName()); + buffer.incrementIndent(); + buffer.printf("this.%s = null;%n", field.camelCaseName()); + buffer.decrementIndent(); + buffer.printf("} else {%n"); + buffer.incrementIndent(); + } + FieldType.ArrayType arrayType = (FieldType.ArrayType) field.type(); + FieldType elementType = arrayType.elementType(); + buffer.printf("this.%s = new %s(nestedObjects.length);%n", + field.camelCaseName(), fieldConcreteJavaType(field)); + buffer.printf("for (Object nestedObject : nestedObjects) {%n"); + buffer.incrementIndent(); + if (elementType.isStruct()) { + buffer.printf("this.%s.add(new %s((Struct) nestedObject, version));%n", + field.camelCaseName(), elementType.toString()); + } else { + buffer.printf("this.%s.add((%s) nestedObject);%n", + field.camelCaseName(), getBoxedJavaType(elementType)); + } + buffer.decrementIndent(); + buffer.printf("}%n"); + if (maybeNull) { + buffer.decrementIndent(); + buffer.printf("}%n"); + } + if (maybeAbsent) { + generateSetDefault(field); + } else { + buffer.decrementIndent(); + buffer.printf("}%n"); + } + } else { + boolean maybeAbsent = + generateVersionCheck(curVersions, field.struct().versions()); + buffer.printf("this.%s = %s;%n", + field.camelCaseName(), + readFieldFromStruct(field.type(), field.snakeCaseName())); + if (maybeAbsent) { + generateSetDefault(field); + } + } + } + + private String getBoxedJavaType(FieldType type) { + if (type instanceof FieldType.BoolFieldType) { + return "Boolean"; + } else if (type instanceof FieldType.Int8FieldType) { + return "Byte"; + } else if (type instanceof FieldType.Int16FieldType) { + return "Short"; + } else if (type instanceof FieldType.Int32FieldType) { + return "Integer"; + } else if (type instanceof FieldType.Int64FieldType) { + return "Long"; + } else if (type.isString()) { + return "String"; + } else if (type.isStruct()) { + return type.toString(); + } else { + throw new RuntimeException("Unsupported field type " + type); + } + } + + private String readFieldFromStruct(FieldType type, String name) { + if (type instanceof FieldType.BoolFieldType) { + return String.format("struct.getBoolean(\"%s\")", name); + } else if (type instanceof FieldType.Int8FieldType) { + return String.format("struct.getByte(\"%s\")", name); + } else if (type instanceof FieldType.Int16FieldType) { + return String.format("struct.getShort(\"%s\")", name); + } else if (type instanceof FieldType.Int32FieldType) { + return String.format("struct.getInt(\"%s\")", name); + } else if (type instanceof FieldType.Int64FieldType) { + return String.format("struct.getLong(\"%s\")", name); + } else if (type.isString()) { + return String.format("struct.getString(\"%s\")", name); + } else if (type.isBytes()) { + return String.format("struct.getByteArray(\"%s\")", name); + } else if (type.isStruct()) { + return String.format("new %s(struct, version)", type.toString()); + } else { + throw new RuntimeException("Unsupported field type " + type); + } + } + + private void generateClassWriter(String className, StructSpec struct, + Versions parentVersions) throws Exception { + headerGenerator.addImport(MessageGenerator.WRITABLE_CLASS); + buffer.printf("@Override%n"); + buffer.printf("public void write(Writable writable, short version) {%n"); + buffer.incrementIndent(); + if (generateInverseVersionCheck(parentVersions, struct.versions())) { + buffer.incrementIndent(); + headerGenerator.addImport(MessageGenerator.UNSUPPORTED_VERSION_EXCEPTION_CLASS); + buffer.printf("throw new UnsupportedVersionException(\"Can't write " + + "version \" + version + \" of %s\");%n", className); + buffer.decrementIndent(); + buffer.printf("}%n"); + } + Versions curVersions = parentVersions.intersect(struct.versions()); + if (curVersions.empty()) { + throw new RuntimeException("Version ranges " + parentVersions + + " and " + struct.versions() + " have no versions in common."); + } + for (FieldSpec field : struct.fields()) { + generateFieldWriter(field, curVersions); + } + buffer.decrementIndent(); + buffer.printf("}%n"); + } + + private String writeFieldToWritable(FieldType type, boolean nullable, String name) { + if (type instanceof FieldType.BoolFieldType) { + return String.format("writable.writeByte(%s ? (byte) 1 : (byte) 0)", name); + } else if (type instanceof FieldType.Int8FieldType) { + return String.format("writable.writeByte(%s)", name); + } else if (type instanceof FieldType.Int16FieldType) { + return String.format("writable.writeShort(%s)", name); + } else if (type instanceof FieldType.Int32FieldType) { + return String.format("writable.writeInt(%s)", name); + } else if (type instanceof FieldType.Int64FieldType) { + return String.format("writable.writeLong(%s)", name); + } else if (type instanceof FieldType.StringFieldType) { + if (nullable) { + return String.format("writable.writeNullableString(%s)", name); + } else { + return String.format("writable.writeString(%s)", name); + } + } else if (type instanceof FieldType.BytesFieldType) { + if (nullable) { + return String.format("writable.writeNullableBytes(%s)", name); + } else { + return String.format("writable.writeBytes(%s)", name); + } + } else if (type instanceof FieldType.StructType) { + return String.format("%s.write(writable, version)", name); + } else { + throw new RuntimeException("Unsupported field type " + type); + } + } + + private void generateFieldWriter(FieldSpec field, Versions curVersions) throws Exception { + if (field.type().isArray()) { + boolean maybeAbsent = + generateVersionCheck(curVersions, field.struct().versions()); + boolean maybeNull = generateNullCheck(curVersions, field); + if (maybeNull) { + buffer.printf("writable.writeInt(-1);%n"); + buffer.decrementIndent(); + buffer.printf("} else {%n"); + buffer.incrementIndent(); + } + buffer.printf("writable.writeInt(%s.size());%n", field.camelCaseName()); + FieldType.ArrayType arrayType = (FieldType.ArrayType) field.type(); + FieldType elementType = arrayType.elementType(); + String nestedTypeName = elementType.isStruct() ? + elementType.toString() : getBoxedJavaType(elementType); + buffer.printf("for (%s element : %s) {%n", + nestedTypeName, field.camelCaseName()); + buffer.incrementIndent(); + buffer.printf("%s;%n", writeFieldToWritable(elementType, false, "element")); + buffer.decrementIndent(); + buffer.printf("}%n"); + if (maybeNull) { + buffer.decrementIndent(); + buffer.printf("}%n"); + } + if (maybeAbsent) { + buffer.decrementIndent(); + buffer.printf("}%n"); + } + } else { + boolean maybeAbsent = + generateVersionCheck(curVersions, field.struct().versions()); + buffer.printf("%s;%n", writeFieldToWritable(field.type(), + !field.nullableVersions().empty(), + field.camelCaseName())); + if (maybeAbsent) { + buffer.decrementIndent(); + buffer.printf("}%n"); + } + } + } + + private void generateClassToStruct(String className, StructSpec struct, + Versions parentVersions) throws Exception { + headerGenerator.addImport(MessageGenerator.STRUCT_CLASS); + buffer.printf("@Override%n"); + buffer.printf("public Struct toStruct(short version) {%n"); + buffer.incrementIndent(); + if (generateInverseVersionCheck(parentVersions, struct.versions())) { + buffer.incrementIndent(); + headerGenerator.addImport(MessageGenerator.UNSUPPORTED_VERSION_EXCEPTION_CLASS); + buffer.printf("throw new UnsupportedVersionException(\"Can't write " + + "version \" + version + \" of %s\");%n", className); + buffer.decrementIndent(); + buffer.printf("}%n"); + } + Versions curVersions = parentVersions.intersect(struct.versions()); + if (curVersions.empty()) { + throw new RuntimeException("Version ranges " + parentVersions + + " and " + struct.versions() + " have no versions in common."); + } + buffer.printf("Struct struct = new Struct(SCHEMAS[version]);%n"); + for (FieldSpec field : struct.fields()) { + generateFieldToStruct(field, curVersions); + } + buffer.printf("return struct;%n"); + buffer.decrementIndent(); + buffer.printf("}%n"); + } + + private void generateFieldToStruct(FieldSpec field, Versions curVersions) throws Exception { + if ((!field.type().canBeNullable()) && + (!field.nullableVersions().empty())) { + throw new RuntimeException("Fields of type " + field.type() + + " cannot be nullable."); + } + if ((field.type() instanceof FieldType.BoolFieldType) || + (field.type() instanceof FieldType.Int8FieldType) || + (field.type() instanceof FieldType.Int16FieldType) || + (field.type() instanceof FieldType.Int32FieldType) || + (field.type() instanceof FieldType.Int64FieldType) || + (field.type() instanceof FieldType.StringFieldType)) { + boolean maybeAbsent = + generateVersionCheck(curVersions, field.struct().versions()); + buffer.printf("struct.set(\"%s\", this.%s);%n", + field.snakeCaseName(), field.camelCaseName()); + if (maybeAbsent) { + buffer.decrementIndent(); + buffer.printf("}%n"); + } + } else if (field.type().isBytes()) { + boolean maybeAbsent = + generateVersionCheck(curVersions, field.struct().versions()); + buffer.printf("struct.setByteArray(\"%s\", this.%s);%n", + field.snakeCaseName(), field.camelCaseName()); + if (maybeAbsent) { + buffer.decrementIndent(); + buffer.printf("}%n"); + } + } else if (field.type().isArray()) { + boolean maybeAbsent = + generateVersionCheck(curVersions, field.struct().versions()); + if (!maybeAbsent) { + buffer.printf("{%n"); + buffer.incrementIndent(); + } + boolean maybeNull = generateNullCheck(curVersions, field); + if (maybeNull) { + buffer.printf("struct.set(\"%s\", null);%n", field.snakeCaseName()); + buffer.decrementIndent(); + buffer.printf("} else {%n"); + buffer.incrementIndent(); + } + FieldType.ArrayType arrayType = (FieldType.ArrayType) field.type(); + FieldType elementType = arrayType.elementType(); + String boxdElementType = elementType.isStruct() ? "Struct" : getBoxedJavaType(elementType); + buffer.printf("%s[] nestedObjects = new %s[%s.size()];%n", + boxdElementType, boxdElementType, field.camelCaseName()); + buffer.printf("int i = 0;%n"); + buffer.printf("for (%s element : this.%s) {%n", + getBoxedJavaType(arrayType.elementType()), field.camelCaseName()); + buffer.incrementIndent(); + if (elementType.isStruct()) { + buffer.printf("nestedObjects[i++] = element.toStruct(version);%n"); + } else { + buffer.printf("nestedObjects[i++] = element;%n"); + } + buffer.decrementIndent(); + buffer.printf("}%n"); + buffer.printf("struct.set(\"%s\", (Object[]) nestedObjects);%n", + field.snakeCaseName()); + if (maybeNull) { + buffer.decrementIndent(); + buffer.printf("}%n"); + } + buffer.decrementIndent(); + buffer.printf("}%n"); + } else { + throw new RuntimeException("Unsupported field type " + field.type()); + } + } + + private void generateClassSize(String className, StructSpec struct, + Versions parentVersions) throws Exception { + buffer.printf("@Override%n"); + buffer.printf("public int size(short version) {%n"); + buffer.incrementIndent(); + buffer.printf("int size = 0;%n"); + if (generateInverseVersionCheck(parentVersions, struct.versions())) { + buffer.incrementIndent(); + headerGenerator.addImport(MessageGenerator.UNSUPPORTED_VERSION_EXCEPTION_CLASS); + buffer.printf("throw new UnsupportedVersionException(\"Can't size " + + "version \" + version + \" of %s\");%n", className); + buffer.decrementIndent(); + buffer.printf("}%n"); + } + Versions curVersions = parentVersions.intersect(struct.versions()); + if (curVersions.empty()) { + throw new RuntimeException("Version ranges " + parentVersions + + " and " + struct.versions() + " have no versions in common."); + } + for (FieldSpec field : struct.fields()) { + generateFieldSize(field, curVersions); + } + buffer.printf("return size;%n"); + buffer.decrementIndent(); + buffer.printf("}%n"); + } + + private void generateVariableLengthFieldSize(String fieldName, FieldType type, boolean nullable) + throws Exception { + if (type instanceof FieldType.StringFieldType) { + buffer.printf("size += 2;%n"); + if (nullable) { + buffer.printf("if (%s != null) {%n", fieldName); + buffer.incrementIndent(); + } + headerGenerator.addImport(MessageGenerator.MESSAGE_UTIL_CLASS); + buffer.printf("size += MessageUtil.utf8Length(%s);%n", fieldName); + if (nullable) { + buffer.decrementIndent(); + buffer.printf("}%n"); + } + } else if (type instanceof FieldType.BytesFieldType) { + buffer.printf("size += 4;%n"); + if (nullable) { + buffer.printf("if (%s != null) {%n", fieldName); + buffer.incrementIndent(); + } + buffer.printf("size += %s.length;%n", fieldName); + if (nullable) { + buffer.decrementIndent(); + buffer.printf("}%n"); + } + } else if (type instanceof FieldType.StructType) { + buffer.printf("size += %s.size(version);%n", fieldName); + } else { + throw new RuntimeException("Unsupported type " + type); + } + } + + private void generateFieldSize(FieldSpec field, Versions curVersions) throws Exception { + if (field.type().fixedLength().isPresent()) { + boolean maybeAbsent = + generateVersionCheck(curVersions, field.struct().versions()); + buffer.printf("size += %d;%n", field.type().fixedLength().get()); + if (maybeAbsent) { + buffer.decrementIndent(); + generateAbsentValueCheck(field); + } + } else if (field.type().isString() || field.type().isBytes() || field.type().isStruct()) { + boolean nullable = !curVersions.intersect(field.nullableVersions()).empty(); + boolean maybeAbsent = + generateVersionCheck(curVersions, field.struct().versions()); + generateVariableLengthFieldSize(field.camelCaseName(), field.type(), nullable); + if (maybeAbsent) { + buffer.decrementIndent(); + generateAbsentValueCheck(field); + } + } else if (field.type().isArray()) { + boolean maybeAbsent = + generateVersionCheck(curVersions, field.struct().versions()); + boolean maybeNull = generateNullCheck(curVersions, field); + if (maybeNull) { + buffer.printf("size += 4;%n"); + buffer.decrementIndent(); + buffer.printf("} else {%n"); + buffer.incrementIndent(); + } + buffer.printf("size += 4;%n"); + FieldType.ArrayType arrayType = (FieldType.ArrayType) field.type(); + FieldType elementType = arrayType.elementType(); + if (elementType.fixedLength().isPresent()) { + buffer.printf("size += %s.size() * %d;%n", + field.camelCaseName(), + elementType.fixedLength().get()); + } else if (elementType instanceof FieldType.ArrayType) { + throw new RuntimeException("Arrays of arrays are not supported " + + "(use a struct)."); + } else { + buffer.printf("for (%s element : %s) {%n", + getBoxedJavaType(elementType), field.camelCaseName()); + buffer.incrementIndent(); + generateVariableLengthFieldSize("element", elementType, false); + buffer.decrementIndent(); + buffer.printf("}%n"); + } + if (maybeNull) { + buffer.decrementIndent(); + buffer.printf("}%n"); + } + if (maybeAbsent) { + buffer.decrementIndent(); + generateAbsentValueCheck(field); + } + } else { + throw new RuntimeException("Unsupported field type " + field.type()); + } + } + + private void generateAbsentValueCheck(FieldSpec field) throws Exception { + if (field.ignorable()) { + buffer.printf("}%n"); + return; + } + buffer.printf("} else {%n"); + buffer.incrementIndent(); + headerGenerator.addImport(MessageGenerator.UNSUPPORTED_VERSION_EXCEPTION_CLASS); + if (field.type().isArray()) { + buffer.printf("if (!%s.isEmpty()) {%n", field.camelCaseName()); + } else if (field.type().isBytes()) { + buffer.printf("if (%s.length != 0) {%n", field.camelCaseName()); + } else if (field.type().isString()) { + buffer.printf("if (%s.equals(%s)) {%n", field.camelCaseName(), fieldDefault(field)); + } else if (field.type() instanceof FieldType.BoolFieldType) { + buffer.printf("if (%s%s) {%n", + fieldDefault(field).equals("true") ? "!" : "", + field.camelCaseName()); + } else { + buffer.printf("if (%s != %s) {%n", field.camelCaseName(), fieldDefault(field)); + } + buffer.incrementIndent(); + buffer.printf("throw new UnsupportedVersionException(" + + "\"Attempted to write a non-default %s at version \" + version);%n", + field.camelCaseName()); + buffer.decrementIndent(); + buffer.printf("}%n"); + buffer.decrementIndent(); + buffer.printf("}%n"); + } + + private void generateClassEquals(String className, StructSpec struct, boolean onlyMapKeys) + throws Exception { + buffer.printf("@Override%n"); + buffer.printf("public boolean equals(Object obj) {%n"); + buffer.incrementIndent(); + buffer.printf("if (!(obj instanceof %s)) return false;%n", className); + if (!struct.fields().isEmpty()) { + buffer.printf("%s other = (%s) obj;%n", className, className); + for (FieldSpec field : struct.fields()) { + if ((!onlyMapKeys) || field.mapKey()) { + generateFieldEquals(field); + } + } + } + buffer.printf("return true;%n"); + buffer.decrementIndent(); + buffer.printf("}%n"); + } + + private void generateFieldEquals(FieldSpec field) throws Exception { + if (field.type().isString() || field.type().isArray() || field.type().isStruct()) { + buffer.printf("if (this.%s == null) {%n", field.camelCaseName()); + buffer.incrementIndent(); + buffer.printf("if (other.%s != null) return false;%n", field.camelCaseName()); + buffer.decrementIndent(); + buffer.printf("} else {%n"); + buffer.incrementIndent(); + buffer.printf("if (!this.%s.equals(other.%s)) return false;%n", + field.camelCaseName(), field.camelCaseName()); + buffer.decrementIndent(); + buffer.printf("}%n"); + } else if (field.type().isBytes()) { + // Arrays#equals handles nulls. + headerGenerator.addImport(MessageGenerator.ARRAYS_CLASS); + buffer.printf("if (!Arrays.equals(this.%s, other.%s)) return false;%n", + field.camelCaseName(), field.camelCaseName()); + } else { + buffer.printf("if (%s != other.%s) return false;%n", + field.camelCaseName(), field.camelCaseName()); + } + } + + private void generateClassHashCode(StructSpec struct, boolean onlyMapKeys) throws Exception { + buffer.printf("@Override%n"); + buffer.printf("public int hashCode() {%n"); + buffer.incrementIndent(); + buffer.printf("int hashCode = 0;%n"); + for (FieldSpec field : struct.fields()) { + if ((!onlyMapKeys) || field.mapKey()) { + generateFieldHashCode(field); + } + } + buffer.printf("return hashCode;%n"); + buffer.decrementIndent(); + buffer.printf("}%n"); + } + + private void generateFieldHashCode(FieldSpec field) throws Exception { + if (field.type() instanceof FieldType.BoolFieldType) { + buffer.printf("hashCode = 31 * hashCode + (%s ? 1231 : 1237);%n", + field.camelCaseName()); + } else if ((field.type() instanceof FieldType.Int8FieldType) || + (field.type() instanceof FieldType.Int16FieldType) || + (field.type() instanceof FieldType.Int32FieldType)) { + buffer.printf("hashCode = 31 * hashCode + %s;%n", + field.camelCaseName()); + } else if (field.type() instanceof FieldType.Int64FieldType) { + buffer.printf("hashCode = 31 * hashCode + ((int) (%s >> 32) ^ (int) %s);%n", + field.camelCaseName(), field.camelCaseName()); + } else if (field.type().isString()) { + buffer.printf("hashCode = 31 * hashCode + (%s == null ? 0 : %s.hashCode());%n", + field.camelCaseName(), field.camelCaseName()); + } else if (field.type().isBytes()) { + headerGenerator.addImport(MessageGenerator.ARRAYS_CLASS); + buffer.printf("hashCode = 31 * hashCode + Arrays.hashCode(%s);%n", + field.camelCaseName()); + } else if (field.type().isStruct() || field.type().isArray()) { + buffer.printf("hashCode = 31 * hashCode + (%s == null ? 0 : %s.hashCode());%n", + field.camelCaseName(), field.camelCaseName()); + } else { + throw new RuntimeException("Unsupported field type " + field.type()); + } + } + + private void generateClassToString(String className, StructSpec struct) throws Exception { + buffer.printf("@Override%n"); + buffer.printf("public String toString() {%n"); + buffer.incrementIndent(); + buffer.printf("return \"%s(\"%n", className); + buffer.incrementIndent(); + String prefix = ""; + for (FieldSpec field : struct.fields()) { + generateFieldToString(prefix, field); + prefix = ", "; + } + buffer.printf("+ \")\";%n"); + buffer.decrementIndent(); + buffer.decrementIndent(); + buffer.printf("}%n"); + } + + private void generateFieldToString(String prefix, FieldSpec field) throws Exception { + if (field.type() instanceof FieldType.BoolFieldType) { + buffer.printf("+ \"%s%s=\" + (%s ? \"true\" : \"false\")%n", + prefix, field.camelCaseName(), field.camelCaseName()); + } else if ((field.type() instanceof FieldType.Int8FieldType) || + (field.type() instanceof FieldType.Int16FieldType) || + (field.type() instanceof FieldType.Int32FieldType) || + (field.type() instanceof FieldType.Int64FieldType)) { + buffer.printf("+ \"%s%s=\" + %s%n", + prefix, field.camelCaseName(), field.camelCaseName()); + } else if (field.type().isString()) { + buffer.printf("+ \"%s%s='\" + %s + \"'\"%n", + prefix, field.camelCaseName(), field.camelCaseName()); + } else if (field.type().isBytes()) { + headerGenerator.addImport(MessageGenerator.ARRAYS_CLASS); + buffer.printf("+ \"%s%s=\" + Arrays.toString(%s)%n", + prefix, field.camelCaseName(), field.camelCaseName()); + } else if (field.type().isStruct()) { + buffer.printf("+ \"%s%s=\" + %s.toString()%n", + prefix, field.camelCaseName(), field.camelCaseName()); + } else if (field.type().isArray()) { + headerGenerator.addImport(MessageGenerator.MESSAGE_UTIL_CLASS); + buffer.printf("+ \"%s%s=\" + MessageUtil.deepToString(%s.iterator())%n", + prefix, field.camelCaseName(), field.camelCaseName()); + } else { + throw new RuntimeException("Unsupported field type " + field.type()); + } + } + + private boolean generateNullCheck(Versions prevVersions, FieldSpec field) throws Exception { + if (prevVersions.intersect(field.nullableVersions()).empty()) { + return false; + } + buffer.printf("if (%s == null) {%n", field.camelCaseName()); + buffer.incrementIndent(); + return true; + } + + private boolean generateNonNullCheck(Versions prevVersions, FieldSpec field) throws Exception { + if (prevVersions.intersect(field.nullableVersions()).empty()) { + return false; + } + buffer.printf("if (%s != null) {%n", field.camelCaseName()); + buffer.incrementIndent(); + return true; + } + + private boolean generateVersionCheck(Versions prev, Versions cur) throws Exception { + if (cur.lowest() > prev.lowest()) { + if (cur.highest() < prev.highest()) { + buffer.printf("if ((version >= %d) && (version <= %d)) {%n", + cur.lowest(), cur.highest()); + buffer.incrementIndent(); + return true; + } else { + buffer.printf("if (version >= %d) {%n", cur.lowest()); + buffer.incrementIndent(); + return true; + } + } else { + if (cur.highest() < prev.highest()) { + buffer.printf("if (version <= %d) {%n", cur.highest()); + buffer.incrementIndent(); + return true; + } else { + return false; + } + } + } + + private boolean generateInverseVersionCheck(Versions prev, Versions cur) throws Exception { + if (cur.lowest() > prev.lowest()) { + if (cur.highest() < prev.highest()) { + buffer.printf("if ((version < %d) || (version > %d)) {%n", + cur.lowest(), cur.highest()); + return true; + } else { + buffer.printf("if (version < %d) {%n", cur.lowest()); + return true; + } + } else { + if (cur.highest() < prev.highest()) { + buffer.printf("if (version > %d) {%n", cur.highest()); + return true; + } else { + return false; + } + } + } + + private void generateSetDefault(FieldSpec field) throws Exception { + buffer.decrementIndent(); + buffer.printf("} else {%n"); + buffer.incrementIndent(); + buffer.printf("this.%s = %s;%n", field.camelCaseName(), fieldDefault(field)); + buffer.decrementIndent(); + buffer.printf("}%n"); + } + + private String fieldDefault(FieldSpec field) { + if (field.type() instanceof FieldType.BoolFieldType) { + if (field.defaultString().isEmpty()) { + return "false"; + } else if (field.defaultString().equalsIgnoreCase("true")) { + return "true"; + } else if (field.defaultString().equalsIgnoreCase("false")) { + return "false"; + } else { + throw new RuntimeException("Invalid default for boolean field " + + field.name() + ": " + field.defaultString()); + } + } else if (field.type() instanceof FieldType.Int8FieldType) { + if (field.defaultString().isEmpty()) { + return "(byte) 0"; + } else { + try { + Byte.decode(field.defaultString()); + } catch (NumberFormatException e) { + throw new RuntimeException("Invalid default for int8 field " + + field.name() + ": " + field.defaultString(), e); + } + return "(byte) " + field.defaultString(); + } + } else if (field.type() instanceof FieldType.Int16FieldType) { + if (field.defaultString().isEmpty()) { + return "(short) 0"; + } else { + try { + Short.decode(field.defaultString()); + } catch (NumberFormatException e) { + throw new RuntimeException("Invalid default for int16 field " + + field.name() + ": " + field.defaultString(), e); + } + return "(short) " + field.defaultString(); + } + } else if (field.type() instanceof FieldType.Int32FieldType) { + if (field.defaultString().isEmpty()) { + return "0"; + } else { + try { + Integer.decode(field.defaultString()); + } catch (NumberFormatException e) { + throw new RuntimeException("Invalid default for int32 field " + + field.name() + ": " + field.defaultString(), e); + } + return field.defaultString(); + } + } else if (field.type() instanceof FieldType.Int64FieldType) { + if (field.defaultString().isEmpty()) { + return "0L"; + } else { + try { + Integer.decode(field.defaultString()); + } catch (NumberFormatException e) { + throw new RuntimeException("Invalid default for int64 field " + + field.name() + ": " + field.defaultString(), e); + } + return field.defaultString() + "L"; + } + } else if (field.type() instanceof FieldType.StringFieldType) { + return "\"" + field.defaultString() + "\""; + } else if (field.type().isBytes()) { + if (!field.defaultString().isEmpty()) { + throw new RuntimeException("Invalid default for bytes field " + + field.name() + ": custom defaults are not supported for bytes fields."); + } + headerGenerator.addImport(MessageGenerator.BYTES_CLASS); + return "Bytes.EMPTY"; + } else if (field.type().isStruct()) { + if (!field.defaultString().isEmpty()) { + throw new RuntimeException("Invalid default for struct field " + + field.name() + ": custom defaults are not supported for struct fields."); + } + return "new " + field.type().toString() + "()"; + } else if (field.type().isArray()) { + if (!field.defaultString().isEmpty()) { + throw new RuntimeException("Invalid default for array field " + + field.name() + ": custom defaults are not supported for array fields."); + } + FieldType.ArrayType arrayType = (FieldType.ArrayType) field.type(); + if (field.hasKeys()) { + return "new " + hashSetType(arrayType.elementType().toString()) + "(0)"; + } else { + headerGenerator.addImport(MessageGenerator.ARRAYLIST_CLASS); + return "new ArrayList<" + getBoxedJavaType(arrayType.elementType()) + ">()"; + } + } else { + throw new RuntimeException("Unsupported field type " + field.type()); + } + } + + private void generateFieldAccessor(FieldSpec field) throws Exception { + buffer.printf("%n"); + generateAccessor(fieldAbstractJavaType(field), field.camelCaseName(), + field.camelCaseName()); + } + + private void generateAccessor(String javaType, String functionName, String memberName) + throws Exception { + buffer.printf("public %s %s() {%n", javaType, functionName); + buffer.incrementIndent(); + buffer.printf("return this.%s;%n", memberName); + buffer.decrementIndent(); + buffer.printf("}%n"); + } + + private void generateFieldMutator(String className, FieldSpec field) throws Exception { + buffer.printf("%n"); + buffer.printf("public %s set%s(%s v) {%n", + className, + field.capitalizedCamelCaseName(), + fieldAbstractJavaType(field)); + buffer.incrementIndent(); + buffer.printf("this.%s = v;%n", field.camelCaseName()); + buffer.printf("return this;%n"); + buffer.decrementIndent(); + buffer.printf("}%n"); + } + + private void generateSetter(String javaType, String functionName, String memberName) + throws Exception { + buffer.printf("public void %s(%s v) {%n", functionName, javaType); + buffer.incrementIndent(); + buffer.printf("this.%s = v;%n", memberName); + buffer.decrementIndent(); + buffer.printf("}%n"); + } +}; diff --git a/buildSrc/src/main/java/org/apache/kafka/message/MessageFactoryGenerator.java b/buildSrc/src/main/java/org/apache/kafka/message/MessageFactoryGenerator.java new file mode 100644 index 0000000000000..ab520b1918d63 --- /dev/null +++ b/buildSrc/src/main/java/org/apache/kafka/message/MessageFactoryGenerator.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.message; + +import java.io.BufferedWriter; +import java.io.IOException; +import java.util.Map; +import java.util.TreeMap; + +public final class MessageFactoryGenerator { + private final TreeMap requestApis; + private final TreeMap responseApis; + private final HeaderGenerator headerGenerator; + private final CodeBuffer buffer; + + public void registerMessageType(MessageSpec spec) { + if (spec.type() == MessageSpecType.REQUEST) { + if (requestApis.containsKey(Short.valueOf(spec.apiKey()))) { + throw new RuntimeException("Found more than one request with " + + "API key " + spec.apiKey()); + } + requestApis.put(spec.apiKey(), spec.generatedClassName()); + } else if (spec.type() == MessageSpecType.RESPONSE) { + if (responseApis.containsKey(Short.valueOf(spec.apiKey()))) { + throw new RuntimeException("Found more than one response with " + + "API key " + spec.apiKey()); + } + responseApis.put(spec.apiKey(), spec.generatedClassName()); + } + } + + public MessageFactoryGenerator() { + this.requestApis = new TreeMap<>(); + this.responseApis = new TreeMap<>(); + this.headerGenerator = new HeaderGenerator(); + this.buffer = new CodeBuffer(); + } + + public void generate() throws Exception { + buffer.printf("public final class MessageFactory {%n"); + buffer.incrementIndent(); + generateFactoryMethod("request", requestApis); + buffer.printf("%n"); + generateFactoryMethod("response", responseApis); + buffer.printf("%n"); + generateSchemasAccessor("request", requestApis); + buffer.printf("%n"); + generateSchemasAccessor("response", responseApis); + buffer.decrementIndent(); + buffer.printf("}%n"); + headerGenerator.generate(); + } + + public void generateFactoryMethod(String type, TreeMap apis) + throws Exception { + headerGenerator.addImport(MessageGenerator.MESSAGE_CLASS); + buffer.printf("public static Message new%s(short apiKey) {%n", + MessageGenerator.capitalizeFirst(type)); + buffer.incrementIndent(); + buffer.printf("switch (apiKey) {%n"); + buffer.incrementIndent(); + for (Map.Entry entry : apis.entrySet()) { + buffer.printf("case %d:%n", entry.getKey()); + buffer.incrementIndent(); + buffer.printf("return new %s();%n", entry.getValue()); + buffer.decrementIndent(); + } + buffer.printf("default:%n"); + buffer.incrementIndent(); + headerGenerator.addImport(MessageGenerator.UNSUPPORTED_VERSION_EXCEPTION_CLASS); + buffer.printf("throw new UnsupportedVersionException(\"Unsupported %s API key \"" + + " + apiKey);%n", type); + buffer.decrementIndent(); + buffer.decrementIndent(); + buffer.printf("}%n"); + buffer.decrementIndent(); + buffer.printf("}%n"); + } + + public void generateSchemasAccessor(String type, TreeMap apis) + throws Exception { + headerGenerator.addImport(MessageGenerator.SCHEMA_CLASS); + buffer.printf("public static Schema[] %sSchemas(short apiKey) {%n", + MessageGenerator.lowerCaseFirst(type)); + buffer.incrementIndent(); + buffer.printf("switch (apiKey) {%n"); + buffer.incrementIndent(); + for (Map.Entry entry : apis.entrySet()) { + buffer.printf("case %d:%n", entry.getKey()); + buffer.incrementIndent(); + buffer.printf("return %s.SCHEMAS;%n", entry.getValue()); + buffer.decrementIndent(); + } + buffer.printf("default:%n"); + buffer.incrementIndent(); + headerGenerator.addImport(MessageGenerator.UNSUPPORTED_VERSION_EXCEPTION_CLASS); + buffer.printf("throw new UnsupportedVersionException(\"Unsupported %s API key \"" + + " + apiKey);%n", type); + buffer.decrementIndent(); + buffer.decrementIndent(); + buffer.printf("}%n"); + buffer.decrementIndent(); + buffer.printf("}%n"); + } + + public void write(BufferedWriter writer) throws IOException { + headerGenerator.buffer().write(writer); + buffer.write(writer); + } +}; diff --git a/buildSrc/src/main/java/org/apache/kafka/message/MessageGenerator.java b/buildSrc/src/main/java/org/apache/kafka/message/MessageGenerator.java new file mode 100644 index 0000000000000..f20d350c331da --- /dev/null +++ b/buildSrc/src/main/java/org/apache/kafka/message/MessageGenerator.java @@ -0,0 +1,200 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.message; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; + +import java.io.BufferedWriter; +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Locale; +import java.util.Map; + +/** + * The Kafka message generator. + */ +public final class MessageGenerator { + static final String JSON_SUFFIX = ".json"; + + static final String JSON_GLOB = "*" + JSON_SUFFIX; + + static final String JAVA_SUFFIX = ".java"; + + static final String MESSAGE_FACTORY_JAVA = "MessageFactory.java"; + + static final String MESSAGE_CLASS = "org.apache.kafka.common.protocol.Message"; + + static final String MESSAGE_UTIL_CLASS = "org.apache.kafka.common.protocol.MessageUtil"; + + static final String READABLE_CLASS = "org.apache.kafka.common.protocol.Readable"; + + static final String WRITABLE_CLASS = "org.apache.kafka.common.protocol.Writable"; + + static final String ARRAYS_CLASS = "java.util.Arrays"; + + static final String LIST_CLASS = "java.util.List"; + + static final String ARRAYLIST_CLASS = "java.util.ArrayList"; + + static final String IMPLICIT_LINKED_HASH_MULTI_SET_CLASS = + "org.apache.kafka.common.utils.ImplicitLinkedHashMultiSet"; + + static final String UNSUPPORTED_VERSION_EXCEPTION_CLASS = + "org.apache.kafka.common.errors.UnsupportedVersionException"; + + static final String ITERATOR_CLASS = "java.util.Iterator"; + + static final String TYPE_CLASS = "org.apache.kafka.common.protocol.types.Type"; + + static final String FIELD_CLASS = "org.apache.kafka.common.protocol.types.Field"; + + static final String SCHEMA_CLASS = "org.apache.kafka.common.protocol.types.Schema"; + + static final String ARRAYOF_CLASS = "org.apache.kafka.common.protocol.types.ArrayOf"; + + static final String STRUCT_CLASS = "org.apache.kafka.common.protocol.types.Struct"; + + static final String BYTES_CLASS = "org.apache.kafka.common.utils.Bytes"; + + /** + * The Jackson serializer we use for JSON objects. + */ + static final ObjectMapper JSON_SERDE; + + static { + JSON_SERDE = new ObjectMapper(); + JSON_SERDE.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false); + JSON_SERDE.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, true); + JSON_SERDE.configure(JsonParser.Feature.ALLOW_COMMENTS, true); + JSON_SERDE.setSerializationInclusion(JsonInclude.Include.NON_EMPTY); + } + + public static Map getOutputFiles(String outputDir, String inputDir) throws Exception { + HashMap outputFiles = new HashMap<>(); + for (Path inputPath : Files.newDirectoryStream(Paths.get(inputDir), JSON_GLOB)) { + String jsonName = inputPath.getFileName().toString(); + String javaName = jsonName.substring(0, jsonName.length() - JSON_SUFFIX.length()) + "Data.java"; + File outputFile = new File(outputDir, javaName); + outputFiles.put(outputFile.toString(), outputFile); + } + File factoryFile = new File(outputDir, MESSAGE_FACTORY_JAVA); + outputFiles.put(factoryFile.toString(), factoryFile); + return outputFiles; + } + + public static void processDirectories(String outputDir, String inputDir) throws Exception { + Files.createDirectories(Paths.get(outputDir)); + int numProcessed = 0; + MessageFactoryGenerator messageFactoryGenerator = new MessageFactoryGenerator(); + HashSet outputFileNames = new HashSet<>(); + for (Path inputPath : Files.newDirectoryStream(Paths.get(inputDir), JSON_GLOB)) { + try { + MessageSpec spec = JSON_SERDE. + readValue(inputPath.toFile(), MessageSpec.class); + String javaName = spec.generatedClassName() + JAVA_SUFFIX; + outputFileNames.add(javaName); + Path outputPath = Paths.get(outputDir, javaName); + try (BufferedWriter writer = Files.newBufferedWriter(outputPath)) { + MessageDataGenerator generator = new MessageDataGenerator(); + generator.generate(spec); + generator.write(writer); + } + numProcessed++; + messageFactoryGenerator.registerMessageType(spec); + } catch (Exception e) { + throw new RuntimeException("Exception while processing " + inputPath.toString(), e); + } + } + Path factoryOutputPath = Paths.get(outputDir, MESSAGE_FACTORY_JAVA); + outputFileNames.add(MESSAGE_FACTORY_JAVA); + try (BufferedWriter writer = Files.newBufferedWriter(factoryOutputPath)) { + messageFactoryGenerator.generate(); + messageFactoryGenerator.write(writer); + } + numProcessed++; + for (Path outputPath : Files.newDirectoryStream(Paths.get(outputDir))) { + if (!outputFileNames.contains(outputPath.getFileName().toString())) { + Files.delete(outputPath); + } + } + System.out.printf("MessageGenerator: processed %d Kafka message JSON files(s).%n", numProcessed); + } + + static String capitalizeFirst(String string) { + if (string.isEmpty()) { + return string; + } + return string.substring(0, 1).toUpperCase(Locale.ENGLISH) + + string.substring(1); + } + + static String lowerCaseFirst(String string) { + if (string.isEmpty()) { + return string; + } + return string.substring(0, 1).toLowerCase(Locale.ENGLISH) + + string.substring(1); + } + + static boolean firstIsCapitalized(String string) { + if (string.isEmpty()) { + return false; + } + return Character.isUpperCase(string.charAt(0)); + } + + static String toSnakeCase(String string) { + StringBuilder bld = new StringBuilder(); + boolean prevWasCapitalized = true; + for (int i = 0; i < string.length(); i++) { + char c = string.charAt(i); + if (Character.isUpperCase(c)) { + if (!prevWasCapitalized) { + bld.append('_'); + } + bld.append(Character.toLowerCase(c)); + prevWasCapitalized = true; + } else { + bld.append(c); + prevWasCapitalized = false; + } + } + return bld.toString(); + } + + private final static String USAGE = "MessageGenerator: [output Java file] [input JSON file]"; + + public static void main(String[] args) throws Exception { + if (args.length == 0) { + System.out.println(USAGE); + System.exit(0); + } else if (args.length != 2) { + System.out.println(USAGE); + System.exit(1); + } + processDirectories(args[0], args[1]); + } +}; diff --git a/buildSrc/src/main/java/org/apache/kafka/message/MessageSpec.java b/buildSrc/src/main/java/org/apache/kafka/message/MessageSpec.java new file mode 100644 index 0000000000000..e9936ec8ac8b6 --- /dev/null +++ b/buildSrc/src/main/java/org/apache/kafka/message/MessageSpec.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.message; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.List; +import java.util.Objects; + +public final class MessageSpec { + private final StructSpec struct; + + private final short apiKey; + + private final MessageSpecType type; + + @JsonCreator + public MessageSpec(@JsonProperty("name") String name, + @JsonProperty("validVersions") String validVersions, + @JsonProperty("fields") List fields, + @JsonProperty("apiKey") Short apiKey, + @JsonProperty("type") MessageSpecType type) { + this.struct = new StructSpec(name, validVersions, fields); + this.apiKey = apiKey == null ? -1 : apiKey; + this.type = Objects.requireNonNull(type); + } + + public StructSpec struct() { + return struct; + } + + @JsonProperty("name") + public String name() { + return struct.name(); + } + + @JsonProperty("validVersions") + public String validVersionsString() { + return struct.versionsString(); + } + + @JsonProperty("fields") + public List fields() { + return struct.fields(); + } + + @JsonProperty("apiKey") + public short apiKey() { + return apiKey; + } + + @JsonProperty("type") + public MessageSpecType type() { + return type; + } + + public String generatedClassName() { + return struct.name() + "Data"; + } +}; diff --git a/buildSrc/src/main/java/org/apache/kafka/message/MessageSpecType.java b/buildSrc/src/main/java/org/apache/kafka/message/MessageSpecType.java new file mode 100644 index 0000000000000..419a2ac3655c7 --- /dev/null +++ b/buildSrc/src/main/java/org/apache/kafka/message/MessageSpecType.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.message; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public enum MessageSpecType { + @JsonProperty("request") + REQUEST, + + @JsonProperty("response") + RESPONSE, + + @JsonProperty("header") + HEADER; +}; diff --git a/buildSrc/src/main/java/org/apache/kafka/message/SchemaGenerator.java b/buildSrc/src/main/java/org/apache/kafka/message/SchemaGenerator.java new file mode 100644 index 0000000000000..d443488b21d4d --- /dev/null +++ b/buildSrc/src/main/java/org/apache/kafka/message/SchemaGenerator.java @@ -0,0 +1,243 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.message; + +import java.util.HashMap; +import java.util.Map; +import java.util.TreeMap; + +/** + * Generates Schemas for Kafka MessageData classes. + */ +final class SchemaGenerator { + /** + * Schema information for a particular message. + */ + static class MessageInfo { + /** + * The versions of this message that we want to generate a schema for. + * This will be constrained by the valid versions for the parent objects. + * For example, if the parent message is valid for versions 0 and 1, + * we will only generate a version 0 and version 1 schema for child classes, + * even if their valid versions are "0+". + */ + private final Versions versions; + + /** + * Maps versions to schema declaration code. If the schema for a + * particular version is the same as that of a previous version, + * there will be no entry in the map for it. + */ + private final TreeMap schemaForVersion; + + MessageInfo(Versions versions) { + this.versions = versions; + this.schemaForVersion = new TreeMap<>(); + } + } + + /** + * The header file generator. This is shared with the MessageDataGenerator + * instance that owns this SchemaGenerator. + */ + private final HeaderGenerator headerGenerator; + + /** + * Maps message names to message information. + */ + private final Map messages; + + SchemaGenerator(HeaderGenerator headerGenerator) { + this.headerGenerator = headerGenerator; + this.messages = new HashMap<>(); + } + + void generateSchemas(MessageSpec message) throws Exception { + generateSchemas(message.generatedClassName(), message.struct(), + message.struct().versions()); + } + + void generateSchemas(String className, StructSpec struct, + Versions parentVersions) throws Exception { + Versions versions = parentVersions.intersect(struct.versions()); + MessageInfo messageInfo = messages.get(className); + if (messageInfo != null) { + return; + } + messageInfo = new MessageInfo(versions); + messages.put(className, messageInfo); + // Process the leaf classes first. + for (FieldSpec field : struct.fields()) { + if (field.type().isStructArray()) { + FieldType.ArrayType arrayType = (FieldType.ArrayType) field.type(); + generateSchemas(arrayType.elementType().toString(), field.struct(), versions); + } else if (field.type().isStruct()) { + generateSchemas(field.type().toString(), field.struct(), versions); + } + } + CodeBuffer prev = null; + for (short v = versions.lowest(); v <= versions.highest(); v++) { + CodeBuffer cur = new CodeBuffer(); + generateSchemaForVersion(struct, v, cur); + // If this schema version is different from the previous one, + // create a new map entry. + if (!cur.equals(prev)) { + messageInfo.schemaForVersion.put(v, cur); + } + prev = cur; + } + } + + private void generateSchemaForVersion(StructSpec struct, short version, + CodeBuffer buffer) throws Exception { + // Find the last valid field index. + int lastValidIndex = struct.fields().size() - 1; + while (true) { + if (lastValidIndex < 0) { + break; + } + FieldSpec field = struct.fields().get(lastValidIndex); + if (field.struct().versions().contains(version)) { + break; + } + lastValidIndex--; + } + + headerGenerator.addImport(MessageGenerator.SCHEMA_CLASS); + buffer.printf("new Schema(%n"); + buffer.incrementIndent(); + for (int i = 0; i <= lastValidIndex; i++) { + FieldSpec field = struct.fields().get(i); + if (!field.struct().versions().contains(version)) { + continue; + } + headerGenerator.addImport(MessageGenerator.FIELD_CLASS); + buffer.printf("new Field(\"%s\", %s, \"%s\")%s%n", + field.snakeCaseName(), + fieldTypeToSchemaType(field, version), + field.about(), + i == lastValidIndex ? "" : ","); + } + buffer.decrementIndent(); + buffer.printf(");%n"); + } + + private String fieldTypeToSchemaType(FieldSpec field, short version) { + return fieldTypeToSchemaType(field.type(), + field.nullableVersions().contains(version), + version); + } + + private String fieldTypeToSchemaType(FieldType type, boolean nullable, short version) { + if (type instanceof FieldType.BoolFieldType) { + headerGenerator.addImport(MessageGenerator.TYPE_CLASS); + if (nullable) { + throw new RuntimeException("Type " + type + " cannot be nullable."); + } + return "Type.BOOLEAN"; + } else if (type instanceof FieldType.Int8FieldType) { + headerGenerator.addImport(MessageGenerator.TYPE_CLASS); + if (nullable) { + throw new RuntimeException("Type " + type + " cannot be nullable."); + } + return "Type.INT8"; + } else if (type instanceof FieldType.Int16FieldType) { + headerGenerator.addImport(MessageGenerator.TYPE_CLASS); + if (nullable) { + throw new RuntimeException("Type " + type + " cannot be nullable."); + } + return "Type.INT16"; + } else if (type instanceof FieldType.Int32FieldType) { + headerGenerator.addImport(MessageGenerator.TYPE_CLASS); + if (nullable) { + throw new RuntimeException("Type " + type + " cannot be nullable."); + } + return "Type.INT32"; + } else if (type instanceof FieldType.Int64FieldType) { + headerGenerator.addImport(MessageGenerator.TYPE_CLASS); + if (nullable) { + throw new RuntimeException("Type " + type + " cannot be nullable."); + } + return "Type.INT64"; + } else if (type instanceof FieldType.StringFieldType) { + headerGenerator.addImport(MessageGenerator.TYPE_CLASS); + return nullable ? "Type.NULLABLE_STRING" : "Type.STRING"; + } else if (type instanceof FieldType.BytesFieldType) { + headerGenerator.addImport(MessageGenerator.TYPE_CLASS); + return nullable ? "Type.NULLABLE_BYTES" : "Type.BYTES"; + } else if (type.isArray()) { + headerGenerator.addImport(MessageGenerator.ARRAYOF_CLASS); + FieldType.ArrayType arrayType = (FieldType.ArrayType) type; + String prefix = nullable ? "ArrayOf.nullable" : "new ArrayOf"; + return String.format("%s(%s)", prefix, + fieldTypeToSchemaType(arrayType.elementType(), false, version)); + } else if (type.isStruct()) { + if (nullable) { + throw new RuntimeException("Type " + type + " cannot be nullable."); + } + return String.format("%s.SCHEMA_%d", type.toString(), + floorVersion(type.toString(), version)); + } else { + throw new RuntimeException("Unsupported type " + type); + } + } + + /** + * Find the lowest schema version for a given class that is the same as the + * given version. + */ + private short floorVersion(String className, short v) { + MessageInfo message = messages.get(className); + return message.schemaForVersion.floorKey(v); + } + + /** + * Write the message schema to the provided buffer. + * + * @param className The class name. + * @param buffer The destination buffer. + */ + void writeSchema(String className, CodeBuffer buffer) throws Exception { + MessageInfo messageInfo = messages.get(className); + Versions versions = messageInfo.versions; + + for (short v = versions.lowest(); v <= versions.highest(); v++) { + CodeBuffer declaration = messageInfo.schemaForVersion.get(v); + if (declaration == null) { + buffer.printf("public static final Schema SCHEMA_%d = SCHEMA_%d;%n", v, v - 1); + } else { + buffer.printf("public static final Schema SCHEMA_%d =%n", v); + buffer.incrementIndent(); + declaration.write(buffer); + buffer.decrementIndent(); + } + buffer.printf("%n"); + } + buffer.printf("public static final Schema[] SCHEMAS = new Schema[] {%n"); + buffer.incrementIndent(); + for (short v = 0; v < versions.lowest(); v++) { + buffer.printf("null%s%n", (v == versions.highest()) ? "" : ","); + } + for (short v = versions.lowest(); v <= versions.highest(); v++) { + buffer.printf("SCHEMA_%d%s%n", v, (v == versions.highest()) ? "" : ","); + } + buffer.decrementIndent(); + buffer.printf("};%n"); + buffer.printf("%n"); + } +}; diff --git a/buildSrc/src/main/java/org/apache/kafka/message/StructSpec.java b/buildSrc/src/main/java/org/apache/kafka/message/StructSpec.java new file mode 100644 index 0000000000000..3fe337077d57f --- /dev/null +++ b/buildSrc/src/main/java/org/apache/kafka/message/StructSpec.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.message; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +public final class StructSpec { + private final String name; + + private final Versions versions; + + private final List fields; + + private final boolean hasKeys; + + @JsonCreator + public StructSpec(@JsonProperty("name") String name, + @JsonProperty("versions") String versions, + @JsonProperty("fields") List fields) { + this.name = Objects.requireNonNull(name); + this.versions = Versions.parse(versions, Versions.ALL); + this.fields = Collections.unmodifiableList(fields == null ? + Collections.emptyList() : new ArrayList<>(fields)); + this.hasKeys = fields == null ? false : fields.stream().anyMatch(f -> f.mapKey()); + } + + @JsonProperty + public String name() { + return name; + } + + public Versions versions() { + return versions; + } + + @JsonProperty + public String versionsString() { + return versions.toString(); + } + + @JsonProperty + public List fields() { + return fields; + } + + boolean hasKeys() { + return hasKeys; + } +}; diff --git a/buildSrc/src/main/java/org/apache/kafka/message/Versions.java b/buildSrc/src/main/java/org/apache/kafka/message/Versions.java new file mode 100644 index 0000000000000..6b7d1004c18dd --- /dev/null +++ b/buildSrc/src/main/java/org/apache/kafka/message/Versions.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.message; + +import java.util.Objects; + +public final class Versions { + private final short lowest; + private final short highest; + + public static Versions parse(String input, Versions defaultVersions) { + if (input == null) { + return defaultVersions; + } + String trimmedInput = input.trim(); + if (trimmedInput.length() == 0) { + return defaultVersions; + } + if (trimmedInput.equals(NONE_STRING)) { + return NONE; + } + if (trimmedInput.endsWith("+")) { + return new Versions(Short.parseShort( + trimmedInput.substring(0, trimmedInput.length() - 1)), + Short.MAX_VALUE); + } else { + int dashIndex = trimmedInput.indexOf("-"); + if (dashIndex < 0) { + short version = Short.parseShort(trimmedInput); + return new Versions(version, version); + } + return new Versions( + Short.parseShort(trimmedInput.substring(0, dashIndex)), + Short.parseShort(trimmedInput.substring(dashIndex + 1))); + } + } + + public static final Versions ALL = new Versions((short) 0, Short.MAX_VALUE); + + public static final Versions NONE = new Versions((short) 0, (short) -1); + + public static final String NONE_STRING = "none"; + + public Versions(short lowest, short highest) { + this.lowest = lowest; + this.highest = highest; + } + + public short lowest() { + return lowest; + } + + public short highest() { + return highest; + } + + public boolean empty() { + return lowest > highest; + } + + @Override + public String toString() { + if (empty()) { + return NONE_STRING; + } else if (lowest == highest) { + return String.valueOf(lowest); + } else if (highest == Short.MAX_VALUE) { + return String.format("%d+", lowest); + } else { + return String.format("%d-%d", lowest, highest); + } + } + + public Versions intersect(Versions other) { + short newLowest = lowest > other.lowest ? lowest : other.lowest; + short newHighest = highest < other.highest ? highest : other.highest; + if (newLowest > newHighest) { + return Versions.NONE; + } + return new Versions(newLowest, newHighest); + } + + public boolean contains(short version) { + return version >= lowest && version <= highest; + } + + @Override + public int hashCode() { + return Objects.hash(lowest, highest); + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof Versions)) { + return false; + } + Versions otherVersions = (Versions) other; + return lowest == otherVersions.lowest && + highest == otherVersions.highest; + } +}; diff --git a/buildSrc/src/main/java/org/apache/kafka/task/ProcessMessagesTask.java b/buildSrc/src/main/java/org/apache/kafka/task/ProcessMessagesTask.java new file mode 100644 index 0000000000000..ddedbdadf026a --- /dev/null +++ b/buildSrc/src/main/java/org/apache/kafka/task/ProcessMessagesTask.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.task; + +import org.apache.kafka.message.MessageGenerator; +import org.gradle.api.DefaultTask; +import org.gradle.api.tasks.InputDirectory; +import org.gradle.api.tasks.OutputFiles; +import org.gradle.api.tasks.TaskAction; + +import java.io.File; +import java.util.Map; + +/** + * A gradle task which processes a directory full of JSON files into an output directory. + */ +public class ProcessMessagesTask extends DefaultTask { + /** + * The directory where we should read the input JSON from. + */ + public File inputDirectory; + + /** + * The directory that we should write output JSON to. + */ + public File outputDirectory; + + @InputDirectory + public File getInputDirectory() { + return inputDirectory; + } + + /** + * Define the task outputs. + * + * Gradle consults this to see if the task is up-to-date. + */ + @OutputFiles + public Map getOutputFiles() throws Exception { + return MessageGenerator.getOutputFiles( + outputDirectory.toString(), inputDirectory.toString()); + } + + @TaskAction + public void run() { + try { + MessageGenerator.processDirectories( + outputDirectory.toString(), inputDirectory.toString()); + } catch (Exception e) { + throw new RuntimeException(e); + } + } +}; diff --git a/checkstyle/import-control.xml b/checkstyle/import-control.xml index 91d23f6424ba4..d4f137b634fcf 100644 --- a/checkstyle/import-control.xml +++ b/checkstyle/import-control.xml @@ -53,6 +53,7 @@ + @@ -155,6 +156,7 @@ + @@ -290,6 +292,7 @@ + @@ -385,4 +388,11 @@ + + + + + + + diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml index 75ad799c149c8..7ac8c7d78721c 100644 --- a/checkstyle/suppressions.xml +++ b/checkstyle/suppressions.xml @@ -51,7 +51,7 @@ files="(Utils|Topic|KafkaLZ4BlockOutputStream|AclData).java"/> + files="(ConsumerCoordinator|Fetcher|Sender|KafkaProducer|BufferPool|ConfigDef|RecordAccumulator|KerberosLogin|AbstractRequest|AbstractResponse|Selector|SslFactory|SslTransportLayer|SaslClientAuthenticator|SaslClientCallbackHandler|SaslServerAuthenticator|SchemaGenerator).java"/> @@ -218,4 +218,11 @@ + + + + + diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/ByteBufferAccessor.java b/clients/src/main/java/org/apache/kafka/common/protocol/ByteBufferAccessor.java new file mode 100644 index 0000000000000..60a923a70362d --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/common/protocol/ByteBufferAccessor.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.common.protocol; + +import java.nio.ByteBuffer; + +public class ByteBufferAccessor implements Readable, Writable { + private final ByteBuffer buf; + + public ByteBufferAccessor(ByteBuffer buf) { + this.buf = buf; + } + + @Override + public byte readByte() { + return buf.get(); + } + + @Override + public short readShort() { + return buf.getShort(); + } + + @Override + public int readInt() { + return buf.getInt(); + } + + @Override + public long readLong() { + return buf.getLong(); + } + + @Override + public void readArray(byte[] arr) { + buf.get(arr); + } + + @Override + public void writeByte(byte val) { + buf.put(val); + } + + @Override + public void writeShort(short val) { + buf.putShort(val); + } + + @Override + public void writeInt(int val) { + buf.putInt(val); + } + + @Override + public void writeLong(long val) { + buf.putLong(val); + } + + @Override + public void writeArray(byte[] arr) { + buf.put(arr); + } +} diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/Message.java b/clients/src/main/java/org/apache/kafka/common/protocol/Message.java new file mode 100644 index 0000000000000..881f84aec44d3 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/common/protocol/Message.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.common.protocol; + +import org.apache.kafka.common.protocol.types.Struct; + +/** + * An object that can serialize itself. The serialization protocol is versioned. + * Messages also implement toString, equals, and hashCode. + */ +public interface Message { + /** + * Returns the API key of this message, or -1 if there is none. + */ + short apiKey(); + + /** + * Returns the lowest supported API key of this message, inclusive. + */ + short lowestSupportedVersion(); + + /** + * Returns the highest supported API key of this message, inclusive. + */ + short highestSupportedVersion(); + + /** + * Returns the number of bytes it would take to write out this message. + * + * @param version The version to use. + * + * @throws org.apache.kafka.common.errors.UnsupportedVersionException + * If the specified version is too new to be supported + * by this software. + */ + int size(short version); + + /** + * Writes out this message to the given ByteBuffer. + * + * @param writable The destination writable. + * @param version The version to use. + * + * @throws org.apache.kafka.common.errors.UnsupportedVersionException + * If the specified version is too new to be supported + * by this software. + */ + void write(Writable writable, short version); + + /** + * Reads this message from the given ByteBuffer. This will overwrite all + * relevant fields with information from the byte buffer. + * + * @param readable The source readable. + * @param version The version to use. + * + * @throws org.apache.kafka.common.errors.UnsupportedVersionException + * If the specified version is too new to be supported + * by this software. + */ + void read(Readable readable, short version); + + /** + * Reads this message from the a Struct object. This will overwrite all + * relevant fields with information from the Struct. + * + * @param struct The source struct. + * @param version The version to use. + */ + void fromStruct(Struct struct, short version); + + /** + * Writes out this message to a Struct. + * + * @param version The version to use. + * + * @throws org.apache.kafka.common.errors.UnsupportedVersionException + * If the specified version is too new to be supported + * by this software. + */ + Struct toStruct(short version); +} diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/MessageUtil.java b/clients/src/main/java/org/apache/kafka/common/protocol/MessageUtil.java new file mode 100644 index 0000000000000..da12ab13e0c36 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/common/protocol/MessageUtil.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.common.protocol; + +import java.util.Iterator; + +public final class MessageUtil { + /** + * Get the length of the UTF8 representation of a string, without allocating + * a byte buffer for the string. + */ + public static short utf8Length(CharSequence input) { + int count = 0, len = input.length(); + for (int i = 0; i < len; i++) { + char c = input.charAt(i); + if (c <= 0x7f) { + count++; + } else if (c <= 0x7ff) { + count += 2; + } else if (Character.isHighSurrogate(c)) { + count += 4; + i++; + } else { + count += 3; + } + } + if (count > Short.MAX_VALUE) { + throw new RuntimeException("String " + input + " is too long to serialize."); + } + return (short) count; + } + + public static String deepToString(Iterator iter) { + StringBuilder bld = new StringBuilder("["); + String prefix = ""; + while (iter.hasNext()) { + Object object = iter.next(); + bld.append(prefix); + bld.append(object.toString()); + prefix = ", "; + } + bld.append("]"); + return bld.toString(); + } +} diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/Readable.java b/clients/src/main/java/org/apache/kafka/common/protocol/Readable.java new file mode 100644 index 0000000000000..a527239ca486c --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/common/protocol/Readable.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.common.protocol; + +import java.nio.charset.StandardCharsets; + +public interface Readable { + byte readByte(); + short readShort(); + int readInt(); + long readLong(); + void readArray(byte[] arr); + + /** + * Read a Kafka-delimited string from a byte buffer. The UTF-8 string + * length is stored in a two-byte short. If the length is negative, the + * string is null. + */ + default String readNullableString() { + int length = readShort(); + if (length < 0) { + return null; + } + byte[] arr = new byte[length]; + readArray(arr); + return new String(arr, StandardCharsets.UTF_8); + } + + /** + * Read a Kafka-delimited array from a byte buffer. The array length is + * stored in a four-byte short. + */ + default byte[] readNullableBytes() { + int length = readInt(); + if (length < 0) { + return null; + } + byte[] arr = new byte[length]; + readArray(arr); + return arr; + } +} diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/Writable.java b/clients/src/main/java/org/apache/kafka/common/protocol/Writable.java new file mode 100644 index 0000000000000..9478ed3ec6225 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/common/protocol/Writable.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.common.protocol; + +import java.nio.charset.StandardCharsets; + +public interface Writable { + void writeByte(byte val); + void writeShort(short val); + void writeInt(int val); + void writeLong(long val); + void writeArray(byte[] arr); + + /** + * Write a nullable byte array delimited by a four-byte length prefix. + */ + default void writeNullableBytes(byte[] arr) { + if (arr == null) { + writeInt(-1); + } else { + writeBytes(arr); + } + } + + /** + * Write a byte array delimited by a four-byte length prefix. + */ + default void writeBytes(byte[] arr) { + writeInt(arr.length); + writeArray(arr); + } + + /** + * Write a nullable string delimited by a two-byte length prefix. + */ + default void writeNullableString(String string) { + if (string == null) { + writeShort((short) -1); + } else { + writeString(string); + } + } + + /** + * Write a string delimited by a two-byte length prefix. + */ + default void writeString(String string) { + byte[] arr = string.getBytes(StandardCharsets.UTF_8); + if (arr.length > Short.MAX_VALUE) { + throw new RuntimeException("Can't store string longer than " + + Short.MAX_VALUE); + } + writeShort((short) arr.length); + writeArray(arr); + } +} diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/types/Struct.java b/clients/src/main/java/org/apache/kafka/common/protocol/types/Struct.java index f54d14c7554ce..b93c0cf7cbcda 100644 --- a/clients/src/main/java/org/apache/kafka/common/protocol/types/Struct.java +++ b/clients/src/main/java/org/apache/kafka/common/protocol/types/Struct.java @@ -282,6 +282,16 @@ public ByteBuffer getBytes(String name) { return (ByteBuffer) result; } + public byte[] getByteArray(String name) { + Object result = get(name); + if (result instanceof byte[]) + return (byte[]) result; + ByteBuffer buf = (ByteBuffer) result; + byte[] arr = new byte[buf.remaining()]; + buf.get(arr); + return arr; + } + /** * Set the given field to the specified value * @@ -346,6 +356,11 @@ public Struct set(Field.ComplexArray def, Object[] value) { return set(def.name, value); } + public Struct setByteArray(String name, byte[] value) { + ByteBuffer buf = value == null ? null : ByteBuffer.wrap(value); + return set(name, buf); + } + public Struct setIfExists(Field.Array def, Object[] value) { return setIfExists(def.name, value); } diff --git a/clients/src/main/java/org/apache/kafka/common/utils/Bytes.java b/clients/src/main/java/org/apache/kafka/common/utils/Bytes.java index e531d1f1760f3..19cd711928fc8 100644 --- a/clients/src/main/java/org/apache/kafka/common/utils/Bytes.java +++ b/clients/src/main/java/org/apache/kafka/common/utils/Bytes.java @@ -25,6 +25,8 @@ */ public class Bytes implements Comparable { + public static final byte[] EMPTY = new byte[0]; + private static final char[] HEX_CHARS_UPPER = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F'}; private final byte[] bytes; diff --git a/clients/src/main/java/org/apache/kafka/common/utils/ImplicitLinkedHashMultiSet.java b/clients/src/main/java/org/apache/kafka/common/utils/ImplicitLinkedHashMultiSet.java new file mode 100644 index 0000000000000..2eb53f6f1d892 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/common/utils/ImplicitLinkedHashMultiSet.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.common.utils; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; + +/** + * A memory-efficient hash multiset which tracks the order of insertion of elements. + * See org.apache.kafka.common.utils.ImplicitLinkedHashSet for implementation details. + * + * This class is a multi-set because it allows multiple elements to be inserted that are + * equal to each other. + * + * We use reference equality when adding elements to the set. A new element A can + * be added if there is no existing element B such that A == B. If an element B + * exists such that A.equals(B), A will still be added. + * + * When deleting an element A from the set, we will try to delete the element B such + * that A == B. If no such element can be found, we will try to delete an element B + * such that A.equals(B). + * + * contains() and find() are unchanged from the base class-- they will look for element + * based on object equality, not reference equality. + * + * This multiset does not allow null elements. It does not have internal synchronization. + */ +public class ImplicitLinkedHashMultiSet + extends ImplicitLinkedHashSet { + public ImplicitLinkedHashMultiSet() { + super(0); + } + + public ImplicitLinkedHashMultiSet(int expectedNumElements) { + super(expectedNumElements); + } + + public ImplicitLinkedHashMultiSet(Iterator iter) { + super(iter); + } + + + /** + * Adds a new element to the appropriate place in the elements array. + * + * @param newElement The new element to add. + * @param addElements The elements array. + * @return The index at which the element was inserted, or INVALID_INDEX + * if the element could not be inserted. + */ + @Override + int addInternal(Element newElement, Element[] addElements) { + int slot = slot(addElements, newElement); + for (int seen = 0; seen < addElements.length; seen++) { + Element element = addElements[slot]; + if (element == null) { + addElements[slot] = newElement; + return slot; + } + if (element == newElement) { + return INVALID_INDEX; + } + slot = (slot + 1) % addElements.length; + } + throw new RuntimeException("Not enough hash table slots to add a new element."); + } + + /** + * Find an element matching an example element. + * + * @param key The element to match. + * + * @return The match index, or INVALID_INDEX if no match was found. + */ + @Override + int findElementToRemove(Object key) { + if (key == null) { + return INVALID_INDEX; + } + int slot = slot(elements, key); + int bestSlot = INVALID_INDEX; + for (int seen = 0; seen < elements.length; seen++) { + Element element = elements[slot]; + if (element == null) { + return bestSlot; + } + if (key == element) { + return slot; + } else if (key.equals(element)) { + bestSlot = slot; + } + slot = (slot + 1) % elements.length; + } + return INVALID_INDEX; + } + + /** + * Returns all of the elements e in the collection such that + * key.equals(e) and key.hashCode() == e.hashCode(). + * + * @param key The element to match. + * + * @return All of the matching elements. + */ + final public List findAll(E key) { + if (key == null) { + return Collections.emptyList(); + } + ArrayList results = new ArrayList<>(); + int slot = slot(elements, key); + for (int seen = 0; seen < elements.length; seen++) { + Element element = elements[slot]; + if (element == null) { + break; + } + if (key.equals(element)) { + @SuppressWarnings("unchecked") + E result = (E) elements[slot]; + results.add(result); + } + slot = (slot + 1) % elements.length; + } + return results; + } +} diff --git a/clients/src/main/java/org/apache/kafka/common/utils/ImplicitLinkedHashSet.java b/clients/src/main/java/org/apache/kafka/common/utils/ImplicitLinkedHashSet.java index 701684dd0b451..75fc9ee8273c5 100644 --- a/clients/src/main/java/org/apache/kafka/common/utils/ImplicitLinkedHashSet.java +++ b/clients/src/main/java/org/apache/kafka/common/utils/ImplicitLinkedHashSet.java @@ -22,35 +22,57 @@ import java.util.NoSuchElementException; /** - * A LinkedHashSet which is more memory-efficient than the standard implementation. + * A memory-efficient hash set which tracks the order of insertion of elements. * - * This set preserves the order of insertion. The order of iteration will always be - * the order of insertion. + * Like java.util.LinkedHashSet, this collection maintains a linked list of elements. + * However, rather than using a separate linked list, this collection embeds the next + * and previous fields into the elements themselves. This reduces memory consumption, + * because it means that we only have to store one Java object per element, rather + * than multiple. * - * This collection requires previous and next indexes to be embedded into each - * element. Using array indices rather than pointers saves space on large heaps - * where pointer compression is not in use. It also reduces the amount of time - * the garbage collector has to spend chasing pointers. + * The next and previous fields are stored as array indices rather than pointers. + * This ensures that the fields only take 32 bits, even when pointers are 64 bits. + * It also makes the garbage collector's job easier, because it reduces the number of + * pointers that it must chase. * * This class uses linear probing. Unlike HashMap (but like HashTable), we don't force * the size to be a power of 2. This saves memory. * - * This class does not have internal synchronization. + * This set does not allow null elements. It does not have internal synchronization. */ -@SuppressWarnings("unchecked") public class ImplicitLinkedHashSet extends AbstractSet { public interface Element { int prev(); - void setPrev(int e); + void setPrev(int prev); int next(); - void setNext(int e); + void setNext(int next); } + /** + * A special index value used to indicate that the next or previous field is + * the head. + */ private static final int HEAD_INDEX = -1; + /** + * A special index value used for next and previous indices which have not + * been initialized. + */ public static final int INVALID_INDEX = -2; + /** + * The minimum new capacity for a non-empty implicit hash set. + */ + private static final int MIN_NONEMPTY_CAPACITY = 5; + + /** + * A static empty array used to avoid object allocations when the capacity is zero. + */ + private static final Element[] EMPTY_ELEMENTS = new Element[0]; + private static class HeadElement implements Element { + static final HeadElement EMPTY = new HeadElement(); + private int prev = HEAD_INDEX; private int next = HEAD_INDEX; @@ -122,7 +144,9 @@ public E next() { } cur = next; next = indexToElement(head, elements, cur.next()); - return (E) cur; + @SuppressWarnings("unchecked") + E returnValue = (E) cur; + return returnValue; } @Override @@ -137,16 +161,23 @@ public void remove() { private Element head; - private Element[] elements; + Element[] elements; private int size; + /** + * Returns an iterator that will yield every element in the set. + * The elements will be returned in the order that they were inserted in. + * + * Do not modify the set while you are iterating over it (except by calling + * remove on the iterator itself, of course.) + */ @Override - public Iterator iterator() { + final public Iterator iterator() { return new ImplicitLinkedHashSetIterator(); } - private static int slot(Element[] curElements, Element e) { + final int slot(Element[] curElements, Object e) { return (e.hashCode() & 0x7fffffff) % curElements.length; } @@ -158,17 +189,20 @@ private static int slot(Element[] curElements, Element e) { * Therefore, we must search forward in the array until we hit a null, before * concluding that the element is not present. * - * @param example The element to match. - * @return The match index, or INVALID_INDEX if no match was found. + * @param key The element to match. + * @return The match index, or INVALID_INDEX if no match was found. */ - private int findIndex(E example) { - int slot = slot(elements, example); + final private int findIndexOfEqualElement(Object key) { + if (key == null) { + return INVALID_INDEX; + } + int slot = slot(elements, key); for (int seen = 0; seen < elements.length; seen++) { Element element = elements[slot]; if (element == null) { return INVALID_INDEX; } - if (element.equals(example)) { + if (key.equals(element)) { return slot; } slot = (slot + 1) % elements.length; @@ -177,43 +211,66 @@ private int findIndex(E example) { } /** - * Find the element which equals() the given example element. + * An element e in the collection such that e.equals(key) and + * e.hashCode() == key.hashCode(). * - * @param example The example element. - * @return Null if no element was found; the element, otherwise. + * @param key The element to match. + * @return The matching element, or null if there were none. */ - public E find(E example) { - int index = findIndex(example); + final public E find(E key) { + int index = findIndexOfEqualElement(key); if (index == INVALID_INDEX) { return null; } - return (E) elements[index]; + @SuppressWarnings("unchecked") + E result = (E) elements[index]; + return result; } /** * Returns the number of elements in the set. */ @Override - public int size() { + final public int size() { return size; } + /** + * Returns true if there is at least one element e in the collection such + * that key.equals(e) and key.hashCode() == e.hashCode(). + * + * @param key The object to try to match. + */ @Override - public boolean contains(Object o) { - E example = null; - try { - example = (E) o; - } catch (ClassCastException e) { - return false; + final public boolean contains(Object key) { + return findIndexOfEqualElement(key) != INVALID_INDEX; + } + + private static int calculateCapacity(int expectedNumElements) { + // Avoid using even-sized capacities, to get better key distribution. + int newCapacity = (2 * expectedNumElements) + 1; + // Don't use a capacity that is too small. + if (newCapacity < MIN_NONEMPTY_CAPACITY) { + return MIN_NONEMPTY_CAPACITY; } - return find(example) != null; + return newCapacity; } + /** + * Add a new element to the collection. + * + * @param newElement The new element. + * + * @return True if the element was added to the collection; + * false if it was not, because there was an existing equal element. + */ @Override - public boolean add(E newElement) { + final public boolean add(E newElement) { + if (newElement == null) { + return false; + } if ((size + 1) >= elements.length / 2) { - // Avoid using even-sized capacities, to get better key distribution. - changeCapacity((2 * elements.length) + 1); + changeCapacity(calculateCapacity(elements.length)); } int slot = addInternal(newElement, elements); if (slot >= 0) { @@ -224,7 +281,7 @@ public boolean add(E newElement) { return false; } - public void mustAdd(E newElement) { + final public void mustAdd(E newElement) { if (!add(newElement)) { throw new RuntimeException("Unable to add " + newElement); } @@ -236,10 +293,9 @@ public void mustAdd(E newElement) { * @param newElement The new element to add. * @param addElements The elements array. * @return The index at which the element was inserted, or INVALID_INDEX - * if the element could not be inserted because there was already - * an equivalent element. + * if the element could not be inserted. */ - private static int addInternal(Element newElement, Element[] addElements) { + int addInternal(Element newElement, Element[] addElements) { int slot = slot(addElements, newElement); for (int seen = 0; seen < addElements.length; seen++) { Element element = addElements[slot]; @@ -270,18 +326,35 @@ private void changeCapacity(int newCapacity) { this.size = oldSize; } + /** + * Remove the first element e such that key.equals(e) + * and key.hashCode == e.hashCode. + * + * @param key The object to try to match. + * @return True if an element was removed; false otherwise. + */ @Override - public boolean remove(Object o) { - E example = null; - try { - example = (E) o; - } catch (ClassCastException e) { - return false; - } - int slot = findIndex(example); + final public boolean remove(Object key) { + int slot = findElementToRemove(key); if (slot == INVALID_INDEX) { return false; } + removeElementAtSlot(slot); + return true; + } + + int findElementToRemove(Object key) { + return findIndexOfEqualElement(key); + } + + /** + * Remove an element in a particular slot. + * + * @param slot The slot of the element to remove. + * + * @return True if an element was removed; false otherwise. + */ + private boolean removeElementAtSlot(int slot) { size--; removeFromList(head, elements, slot); slot = (slot + 1) % elements.length; @@ -328,27 +401,64 @@ private void reseat(int prevSlot) { elements[newSlot] = element; } - @Override - public void clear() { - reset(elements.length); + /** + * Create a new ImplicitLinkedHashSet. + */ + public ImplicitLinkedHashSet() { + this(0); } - public ImplicitLinkedHashSet() { - this(5); + /** + * Create a new ImplicitLinkedHashSet. + * + * @param expectedNumElements The number of elements we expect to have in this set. + * This is used to optimize by setting the capacity ahead + * of time rather than growing incrementally. + */ + public ImplicitLinkedHashSet(int expectedNumElements) { + clear(expectedNumElements); } - public ImplicitLinkedHashSet(int initialCapacity) { - reset(initialCapacity); + /** + * Create a new ImplicitLinkedHashSet. + * + * @param iter We will add all the elements accessible through this iterator + * to the set. + */ + public ImplicitLinkedHashSet(Iterator iter) { + clear(0); + while (iter.hasNext()) { + mustAdd(iter.next()); + } } - private void reset(int capacity) { - this.head = new HeadElement(); - // Avoid using even-sized capacities, to get better key distribution. - this.elements = new Element[(2 * capacity) + 1]; - this.size = 0; + /** + * Removes all of the elements from this set. + */ + @Override + final public void clear() { + clear(elements.length); + } + + /** + * Removes all of the elements from this set, and resets the set capacity + * based on the provided expected number of elements. + */ + final public void clear(int expectedNumElements) { + if (expectedNumElements == 0) { + // Optimize away object allocations for empty sets. + this.head = HeadElement.EMPTY; + this.elements = EMPTY_ELEMENTS; + this.size = 0; + } else { + this.head = new HeadElement(); + this.elements = new Element[calculateCapacity(expectedNumElements)]; + this.size = 0; + } } - int numSlots() { + // Visible for testing + final int numSlots() { return elements.length; } } diff --git a/clients/src/main/resources/message/AddOffsetsToTxnRequest.json b/clients/src/main/resources/message/AddOffsetsToTxnRequest.json new file mode 100644 index 0000000000000..981650f6f4c0b --- /dev/null +++ b/clients/src/main/resources/message/AddOffsetsToTxnRequest.json @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 25, + "type": "request", + "name": "AddOffsetsToTxnRequest", + // Version 1 is the same as version 0. + "validVersions": "0-1", + "fields": [ + { "name": "TransactionalId", "type": "string", "versions": "0+", + "about": "The transactional id corresponding to the transaction."}, + { "name": "ProducerId", "type": "int64", "versions": "0+", + "about": "Current producer id in use by the transactional id." }, + { "name": "ProducerEpoch", "type": "int16", "versions": "0+", + "about": "Current epoch associated with the producer id." }, + { "name": "GroupId", "type": "string", "versions": "0+", + "about": "The unique group identifier." } + ] +} diff --git a/clients/src/main/resources/message/AddOffsetsToTxnResponse.json b/clients/src/main/resources/message/AddOffsetsToTxnResponse.json new file mode 100644 index 0000000000000..080974226fa7a --- /dev/null +++ b/clients/src/main/resources/message/AddOffsetsToTxnResponse.json @@ -0,0 +1,28 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 25, + "type": "response", + "name": "AddOffsetsToTxnResponse", + // Starting in version 1, on quota violation brokers send out responses before throttling. + "validVersions": "0-1", + "fields": [ + { "name": "ThrottleTimeMs", "type": "int32", "versions": "0+", + "about": "Duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." }, + { "name": "ErrorCode", "type": "int16", "versions": "0+", + "about": "The response error code, or 0 if there was no error." } + ] +} diff --git a/clients/src/main/resources/message/AddPartitionsToTxnRequest.json b/clients/src/main/resources/message/AddPartitionsToTxnRequest.json new file mode 100644 index 0000000000000..1c71fa7855c4d --- /dev/null +++ b/clients/src/main/resources/message/AddPartitionsToTxnRequest.json @@ -0,0 +1,37 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 24, + "type": "request", + "name": "AddPartitionsToTxnRequest", + // Version 1 is the same as version 0. + "validVersions": "0-1", + "fields": [ + { "name": "TransactionalId", "type": "string", "versions": "0+", + "about": "The transactional id corresponding to the transaction."}, + { "name": "ProducerId", "type": "int64", "versions": "0+", + "about": "Current producer id in use by the transactional id." }, + { "name": "ProducerEpoch", "type": "int16", "versions": "0+", + "about": "Current epoch associated with the producer id." }, + { "name": "Topics", "type": "[]AddPartitionsToTxnTopic", "versions": "0+", + "about": "The partitions to add to the transation.", "fields": [ + { "name": "Name", "type": "string", "versions": "0+", "mapKey": true, + "about": "The name of the topic." }, + { "name": "Partitions", "type": "[]int32", "versions": "0+", + "about": "The partition indexes to add to the transaction" } + ]} + ] +} diff --git a/clients/src/main/resources/message/AddPartitionsToTxnResponse.json b/clients/src/main/resources/message/AddPartitionsToTxnResponse.json new file mode 100644 index 0000000000000..50ae5cd6a4a2d --- /dev/null +++ b/clients/src/main/resources/message/AddPartitionsToTxnResponse.json @@ -0,0 +1,38 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 24, + "type": "response", + "name": "AddPartitionsToTxnResponse", + // Starting in version 1, on quota violation brokers send out responses before throttling. + "validVersions": "0-1", + "fields": [ + { "name": "ThrottleTimeMs", "type": "int32", "versions": "0+", + "about": "Duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." }, + { "name": "Results", "type": "[]AddPartitionsToTxnTopicResult", "versions": "0+", + "about": "The results for each topic.", "fields": [ + { "name": "Name", "type": "string", "versions": "0+", "mapKey": true, + "about": "The topic name." }, + { "name": "Results", "type": "[]AddPartitionsToTxnPartitionResult", "versions": "0+", + "about": "The results for each partition", "fields": [ + { "name": "PartitionIndex", "type": "int32", "versions": "0+", "mapKey": true, + "about": "The partition indexes." }, + { "name": "ErrorCode", "type": "int16", "versions": "0+", + "about": "The response error code."} + ]} + ]} + ] +} diff --git a/clients/src/main/resources/message/AlterConfigsRequest.json b/clients/src/main/resources/message/AlterConfigsRequest.json new file mode 100644 index 0000000000000..6d037654d0c50 --- /dev/null +++ b/clients/src/main/resources/message/AlterConfigsRequest.json @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 33, + "type": "request", + "name": "AlterConfigsRequest", + // Version 1 is the same as version 0. + "validVersions": "0-1", + "fields": [ + { "name": "Resources", "type": "[]AlterConfigsResource", "versions": "0+", + "about": "The updates for each resource.", "fields": [ + { "name": "ResourceType", "type": "int8", "versions": "0+", "mapKey": true, + "about": "The resource type." }, + { "name": "ResourceName", "type": "string", "versions": "0+", "mapKey": true, + "about": "The resource name." }, + { "name": "Configs", "type": "[]AlterableConfig", "versions": "0+", + "about": "The configurations.", "fields": [ + { "name": "Name", "type": "string", "versions": "0+", "mapKey": true, + "about": "The configuration key name." }, + { "name": "Value", "type": "string", "versions": "0+", "nullableVersions": "0+", + "about": "The value to set for the configuration key."} + ]} + ]}, + { "name": "ValidateOnly", "type": "bool", "versions": "0+", + "about": "True if we should validate the request, but not change the configurations."} + ] +} diff --git a/clients/src/main/resources/message/AlterConfigsResponse.json b/clients/src/main/resources/message/AlterConfigsResponse.json new file mode 100644 index 0000000000000..135a46741366d --- /dev/null +++ b/clients/src/main/resources/message/AlterConfigsResponse.json @@ -0,0 +1,37 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 33, + "type": "response", + "name": "AlterConfigsResponse", + // Starting in version 1, on quota violation brokers send out responses before throttling. + "validVersions": "0-1", + "fields": [ + { "name": "ThrottleTimeMs", "type": "int32", "versions": "0+", + "about": "Duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." }, + { "name": "Resources", "type": "[]AlterConfigsResourceResponse", "versions": "0+", + "about": "The responses for each resource.", "fields": [ + { "name": "ErrorCode", "type": "int16", "versions": "0+", + "about": "The resource error code." }, + { "name": "ErrorMessage", "type": "string", "nullableVersions": "0+", "versions": "0+", + "about": "The resource error message, or null if there was no error." }, + { "name": "ResourceType", "type": "int8", "versions": "0+", + "about": "The resource type." }, + { "name": "ResourceName", "type": "string", "versions": "0+", + "about": "The resource name." } + ]} + ] +} diff --git a/clients/src/main/resources/message/AlterReplicaLogDirsRequest.json b/clients/src/main/resources/message/AlterReplicaLogDirsRequest.json new file mode 100644 index 0000000000000..4a00249afeef0 --- /dev/null +++ b/clients/src/main/resources/message/AlterReplicaLogDirsRequest.json @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 34, + "type": "request", + "name": "AlterReplicaLogDirsRequest", + // Version 1 is the same as version 0. + "validVersions": "0-1", + "fields": [ + { "name": "Dirs", "type": "[]AlterReplicaLogDir", "versions": "0+", + "about": "The alterations to make for each directory.", "fields": [ + { "name": "Path", "type": "string", "versions": "0+", "mapKey": true, + "about": "The absolute directory path." }, + { "name": "Topics", "type": "[]AlterReplicaLogDirTopic", "versions": "0+", + "about": "The topics to add to the directory.", "fields": [ + { "name": "Name", "type": "string", "versions": "0+", "mapKey": true, + "about": "The topic name." }, + { "name": "Partitions", "type": "[]int32", "versions": "0+", + "about": "The partition indexes." } + ]} + ]} + ] +} diff --git a/clients/src/main/resources/message/AlterReplicaLogDirsResponse.json b/clients/src/main/resources/message/AlterReplicaLogDirsResponse.json new file mode 100644 index 0000000000000..2551a15985c9b --- /dev/null +++ b/clients/src/main/resources/message/AlterReplicaLogDirsResponse.json @@ -0,0 +1,38 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 34, + "type": "response", + "name": "AlterReplicaLogDirsResponse", + // Starting in version 1, on quota violation brokers send out responses before throttling. + "validVersions": "0-1", + "fields": [ + { "name": "ThrottleTimeMs", "type": "int32", "versions": "0+", + "about": "Duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." }, + { "name": "Results", "type": "[]AlterReplicaLogDirTopicResult", "versions": "0+", + "about": "The results for each topic.", "fields": [ + { "name": "TopicName", "type": "string", "versions": "0+", + "about": "The name of the topic." }, + { "name": "Partitions", "type": "[]AlterReplicaLogDirPartitionResult", "versions": "0+", + "about": "The results for each partition.", "fields": [ + { "name": "PartitionIndex", "type": "int32", "versions": "0+", + "about": "The partition index."}, + { "name": "ErrorCode", "type": "int16", "versions": "0+", + "about": "The error code, or 0 if there was no error." } + ]} + ]} + ] +} diff --git a/clients/src/main/resources/message/ApiVersionsRequest.json b/clients/src/main/resources/message/ApiVersionsRequest.json new file mode 100644 index 0000000000000..d6ddb5475c7b8 --- /dev/null +++ b/clients/src/main/resources/message/ApiVersionsRequest.json @@ -0,0 +1,24 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 18, + "type": "request", + "name": "ApiVersionsRequest", + // Versions 0 through 2 of ApiVersionsRequest are the same. + "validVersions": "0-2", + "fields": [ + ] +} diff --git a/clients/src/main/resources/message/ApiVersionsResponse.json b/clients/src/main/resources/message/ApiVersionsResponse.json new file mode 100644 index 0000000000000..21fc84a9798c2 --- /dev/null +++ b/clients/src/main/resources/message/ApiVersionsResponse.json @@ -0,0 +1,38 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 18, + "type": "response", + "name": "ApiVersionsResponse", + // Version 1 adds throttle time to the response. + // Starting in version 2, on quota violation, brokers send out responses before throttling. + "validVersions": "0-2", + "fields": [ + { "name": "ErrorCode", "type": "int16", "versions": "0+", + "about": "The top-level error code." }, + { "name": "ApiKeys", "type": "[]ApiVersionsResponseKey", "versions": "0+", + "about": "The APIs supported by the broker.", "fields": [ + { "name": "Index", "type": "int16", "versions": "0+", "mapKey": true, + "about": "The API index." }, + { "name": "MinVersion", "type": "int16", "versions": "0+", + "about": "The minimum supported version, inclusive." }, + { "name": "MaxVersion", "type": "int16", "versions": "0+", + "about": "The maximum supported version, inclusive." } + ]}, + { "name": "ThrottleTimeMs", "type": "int32", "versions": "1+", "ignorable": true, + "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." } + ] +} diff --git a/clients/src/main/resources/message/ControlledShutdownRequest.json b/clients/src/main/resources/message/ControlledShutdownRequest.json new file mode 100644 index 0000000000000..60ceaa59cdcff --- /dev/null +++ b/clients/src/main/resources/message/ControlledShutdownRequest.json @@ -0,0 +1,34 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 7, + "type": "request", + "name": "ControlledShutdownRequest", + // Version 0 of ControlledShutdownRequest has a non-standard request header + // which does not include clientId. Version 1 and later use the standard + // request header. + // + // Version 1 is the same as version 0. + // + // Version 2 adds BrokerEpoch. + "validVersions": "0-2", + "fields": [ + { "name": "BrokerId", "type": "int32", "versions": "0+", + "about": "The id of the broker for which controlled shutdown has been requested." }, + { "name": "BrokerEpoch", "type": "int64", "versions": "2+", "default": "-1", "ignorable": true, + "about": "The broker epoch." } + ] +} diff --git a/clients/src/main/resources/message/ControlledShutdownResponse.json b/clients/src/main/resources/message/ControlledShutdownResponse.json new file mode 100644 index 0000000000000..d0fbcf2fbc87e --- /dev/null +++ b/clients/src/main/resources/message/ControlledShutdownResponse.json @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 7, + "type": "response", + "name": "ControlledShutdownResponse", + // Versions 1 and 2 are the same as version 0. + "validVersions": "0-2", + "fields": [ + { "name": "ErrorCode", "type": "int16", "versions": "0+", + "about": "The top-level error code." }, + { "name": "RemainingPartitions", "type": "[]RemainingPartition", "versions": "0+", + "about": "The partitions that the broker still leads.", "fields": [ + { "name": "TopicName", "type": "string", "versions": "0+", "mapKey": true, + "about": "The name of the topic." }, + { "name": "PartitionIndex", "type": "int32", "versions": "0+", "mapKey": true, + "about": "The index of the partition." } + ]} + ] +} diff --git a/clients/src/main/resources/message/CreateAclsRequest.json b/clients/src/main/resources/message/CreateAclsRequest.json new file mode 100644 index 0000000000000..0e022a8954dc2 --- /dev/null +++ b/clients/src/main/resources/message/CreateAclsRequest.json @@ -0,0 +1,41 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 30, + "type": "request", + "name": "CreateAclsRequest", + // Version 1 adds resource pattern type. + "validVersions": "0-1", + "fields": [ + { "name": "Creations", "type": "[]CreatableAcl", "versions": "0+", + "about": "The ACLs that we want to create.", "fields": [ + { "name": "ResourceType", "type": "int8", "versions": "0+", + "about": "The type of the resource." }, + { "name": "ResourceName", "type": "string", "versions": "0+", + "about": "The resource name for the ACL." }, + { "name": "ResourcePatternType", "type": "int8", "versions": "1+", "default": "3", + "about": "The pattern type for the ACL." }, + { "name": "Principal", "type": "string", "versions": "0+", + "about": "The principal for the ACL." }, + { "name": "Host", "type": "string", "versions": "0+", + "about": "The host for the ACL." }, + { "name": "Operation", "type": "int8", "versions": "0+", + "about": "The operation type for the ACL (read, write, etc.)." }, + { "name": "PermissionType", "type": "int8", "versions": "0+", + "about": "The permission type for the ACL (allow, deny, etc.)." } + ]} + ] +} diff --git a/clients/src/main/resources/message/CreateAclsResponse.json b/clients/src/main/resources/message/CreateAclsResponse.json new file mode 100644 index 0000000000000..ede3fef9058c4 --- /dev/null +++ b/clients/src/main/resources/message/CreateAclsResponse.json @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 30, + "type": "response", + "name": "CreateAclsResponse", + // Starting in version 1, on quota violation, brokers send out responses before throttling. + "validVersions": "0-1", + "fields": [ + { "name": "ThrottleTimeMs", "type": "int32", "versions": "0+", + "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." }, + { "name": "Results", "type": "[]CreatableAclResult", "versions": "0+", + "about": "The results for each ACL creation.", "fields": [ + { "name": "ErrorCode", "type": "int16", "versions": "0+", + "about": "The result error, or zero if there was no error." }, + { "name": "ErrorMessage", "type": "string", "nullableVersions": "0+", "versions": "0+", + "about": "The result message, or null if there was no error." } + ]} + ] +} diff --git a/clients/src/main/resources/message/CreateDelegationTokenRequest.json b/clients/src/main/resources/message/CreateDelegationTokenRequest.json new file mode 100644 index 0000000000000..2de43b9654261 --- /dev/null +++ b/clients/src/main/resources/message/CreateDelegationTokenRequest.json @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 38, + "type": "request", + "name": "CreateDelegationTokenRequest", + // Version 1 is the same as version 0. + "validVersions": "0-1", + "fields": [ + { "name": "Renewers", "type": "[]CreatableRenewers", "versions": "0+", + "about": "A list of those who are allowed to renew this token before it expires.", "fields": [ + { "name": "PrincipalType", "type": "string", "versions": "0+", + "about": "The type of the Kafka principal." }, + { "name": "PrincipalName", "type": "string", "versions": "0+", + "about": "The name of the Kafka principal." } + ]}, + { "name": "MaxLifetimeMs", "type": "int64", "versions": "0+", + "about": "The maximum lifetime of the token in milliseconds, or -1 to use the server side default." } + ] +} diff --git a/clients/src/main/resources/message/CreateDelegationTokenResponse.json b/clients/src/main/resources/message/CreateDelegationTokenResponse.json new file mode 100644 index 0000000000000..61367e4991c60 --- /dev/null +++ b/clients/src/main/resources/message/CreateDelegationTokenResponse.json @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 38, + "type": "response", + "name": "CreateDelegationTokenResponse", + // Starting in version 1, on quota violation, brokers send out responses before throttling. + "validVersions": "0-1", + "fields": [ + { "name": "ErrorCode", "type": "int16", "versions": "0+", + "about": "The top-level error, or zero if there was no error."}, + { "name": "PrincipalType", "type": "string", "versions": "0+", + "about": "The principal type of the token owner." }, + { "name": "PrincipalName", "type": "string", "versions": "0+", + "about": "The name of the token owner." }, + { "name": "IssueTimestampMs", "type": "int64", "versions": "0+", + "about": "When this token was generated." }, + { "name": "ExpiryTimestampMs", "type": "int64", "versions": "0+", + "about": "When this token expires." }, + { "name": "MaxTimestampMs", "type": "int64", "versions": "0+", + "about": "The maximum lifetime of this token." }, + { "name": "TokenId", "type": "string", "versions": "0+", + "about": "The token UUID." }, + { "name": "Hmac", "type": "bytes", "versions": "0+", + "about": "HMAC of the delegation token." }, + { "name": "ThrottleTimeMs", "type": "int32", "versions": "0+", + "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." } + ] +} diff --git a/clients/src/main/resources/message/CreatePartitionsRequest.json b/clients/src/main/resources/message/CreatePartitionsRequest.json new file mode 100644 index 0000000000000..2dc75c7cca445 --- /dev/null +++ b/clients/src/main/resources/message/CreatePartitionsRequest.json @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 37, + "type": "request", + "name": "CreatePartitionsRequest", + // Version 1 is the same as version 0. + "validVersions": "0-1", + "fields": [ + { "name": "Topics", "type": "[]CreatePartitionsTopic", "versions": "0+", + "about": "Each topic that we want to create new partitions inside.", "fields": [ + { "name": "Name", "type": "string", "versions": "0+", + "about": "The topic name." }, + { "name": "Count", "type": "int32", "versions": "0+", + "about": "The new partition count." }, + { "name": "Assignments", "type": "[]CreatePartitionsAssignment", "versions": "0+", "nullableVersions": "0+", + "about": "The new partition assignments.", "fields": [ + { "name": "BrokerIds", "type": "[]int32", "versions": "0+", + "about": "The assigned broker IDs." } + ]} + ]}, + { "name": "TimeoutMs", "type": "int32", "versions": "0+", + "about": "The time in ms to wait for the partitions to be created." }, + { "name": "ValidateOnly", "type": "bool", "versions": "0+", + "about": "If true, then validate the request, but don't actually increase the number of partitions." } + ] +} diff --git a/clients/src/main/resources/message/CreatePartitionsResponse.json b/clients/src/main/resources/message/CreatePartitionsResponse.json new file mode 100644 index 0000000000000..2a0c01ed935fb --- /dev/null +++ b/clients/src/main/resources/message/CreatePartitionsResponse.json @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 37, + "type": "response", + "name": "CreatePartitionsResponse", + // Starting in version 1, on quota violation, brokers send out responses before throttling. + "validVersions": "0-1", + "fields": [ + { "name": "ThrottleTimeMs", "type": "int32", "versions": "0+", + "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." }, + { "name": "Results", "type": "[]CreatePartitionsTopicResult", "versions": "0+", + "about": "The partition creation results for each topic.", "fields": [ + { "name": "Name", "type": "string", "versions": "0+", + "about": "The topic name." }, + { "name": "ErrorCode", "type": "int16", "versions": "0+", + "about": "The result error, or zero if there was no error."}, + { "name": "ErrorMessage", "type": "string", "versions": "0+", "nullableVersions": "0+", + "about": "The result message, or null if there was no error."} + ]} + ] +} diff --git a/clients/src/main/resources/message/CreateTopicsRequest.json b/clients/src/main/resources/message/CreateTopicsRequest.json new file mode 100644 index 0000000000000..b76d13dbf8d31 --- /dev/null +++ b/clients/src/main/resources/message/CreateTopicsRequest.json @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 19, + "type": "request", + "name": "CreateTopicsRequest", + // Version 1 adds validateOnly. + "validVersions": "0-3", + "fields": [ + { "name": "Topics", "type": "[]CreatableTopic", "versions": "0+", + "about": "The topics to create.", "fields": [ + { "name": "Name", "type": "string", "versions": "0+", + "about": "The topic name." }, + { "name": "NumPartitions", "type": "int32", "versions": "0+", + "about": "The number of partitions to create in the topic, or -1 if we are specifying a manual partition assignment." }, + { "name": "ReplicationFactor", "type": "int16", "versions": "0+", + "about": "The number of replicas to create for each partition in the topic, or -1 if we are specifying a manual partition assignment." }, + { "name": "Assignments", "type": "[]CreatableReplicaAssignment", "versions": "0+", + "about": "The manual partition assignment, or the empty array if we are using automatic assignment.", "fields": [ + { "name": "PartitionIndex", "type": "int32", "versions": "0+", "mapKey": true, + "about": "The partition index." }, + { "name": "BrokerIds", "type": "[]int32", "versions": "0+", + "about": "The brokers to place the partition on." } + ]}, + { "name": "Configs", "type": "[]CreateableTopicConfig", "versions": "0+", + "about": "The custom topic configurations to set.", "fields": [ + { "name": "Name", "type": "string", "versions": "0+" , "mapKey": true, + "about": "The configuration name." }, + { "name": "Value", "type": "string", "versions": "0+", "nullableVersions": "0+", + "about": "The configuration value." } + ]} + ]}, + { "name": "timeoutMs", "type": "int32", "versions": "0+", + "about": "How long to wait in milliseconds before timing out the request." }, + { "name": "validateOnly", "type": "bool", "versions": "1+", "default": "false", "ignorable": false, + "about": "If true, check that the topics can be created as specified, but don't create anything." } + ] +} diff --git a/clients/src/main/resources/message/CreateTopicsResponse.json b/clients/src/main/resources/message/CreateTopicsResponse.json new file mode 100644 index 0000000000000..49e4d7b888bf9 --- /dev/null +++ b/clients/src/main/resources/message/CreateTopicsResponse.json @@ -0,0 +1,37 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 19, + "type": "response", + "name": "CreateTopicsResponse", + // Version 1 adds a per-topic error message string. + // Version 2 adds the throttle time. + // Starting in version 3, on quota violation, brokers send out responses before throttling. + "validVersions": "0-3", + "fields": [ + { "name": "ThrottleTimeMs", "type": "int32", "versions": "2+", "ignorable": true, + "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." }, + { "name": "Topics", "type": "[]CreatableTopicResult", "versions": "0+", + "about": "Results for each topic we tried to create.", "fields": [ + { "name": "Name", "type": "string", "versions": "0+", "mapKey": true, + "about": "The topic name." }, + { "name": "ErrorCode", "type": "int16", "versions": "0+", + "about": "The error code, or 0 if there was no error." }, + { "name": "ErrorMessage", "type": "string", "versions": "1+", "nullableVersions": "0+", "ignorable": true, + "about": "The error message, or null if there was no error." } + ]} + ] +} diff --git a/clients/src/main/resources/message/DeleteAclsRequest.json b/clients/src/main/resources/message/DeleteAclsRequest.json new file mode 100644 index 0000000000000..3a4aed173349f --- /dev/null +++ b/clients/src/main/resources/message/DeleteAclsRequest.json @@ -0,0 +1,41 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 31, + "type": "request", + "name": "DeleteAclsRequest", + // Version 1 adds the pattern type. + "validVersions": "0-1", + "fields": [ + { "name": "Filters", "type": "[]DeleteAclsFilter", "versions": "0+", + "about": "The filters to use when deleting ACLs.", "fields": [ + { "name": "ResourceTypeFilter", "type": "int8", "versions": "0+", + "about": "The resource type." }, + { "name": "ResourceNameFilter", "type": "string", "versions": "0+", "nullableVersions": "0+", + "about": "The resource name." }, + { "name": "PatternTypeFilter", "type": "int8", "versions": "1+", "default": "3", "ignorable": false, + "about": "The pattern type." }, + { "name": "PrincipalFilter", "type": "string", "versions": "0+", "nullableVersions": "0+", + "about": "The principal filter, or null to accept all principals." }, + { "name": "HostFilter", "type": "string", "versions": "0+", "nullableVersions": "0+", + "about": "The host filter, or null to accept all hosts." }, + { "name": "Operation", "type": "int8", "versions": "0+", + "about": "The ACL operation." }, + { "name": "PermissionType", "type": "int8", "versions": "0+", + "about": "The permission type." } + ]} + ] +} diff --git a/clients/src/main/resources/message/DeleteAclsResponse.json b/clients/src/main/resources/message/DeleteAclsResponse.json new file mode 100644 index 0000000000000..d752888866990 --- /dev/null +++ b/clients/src/main/resources/message/DeleteAclsResponse.json @@ -0,0 +1,55 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 31, + "type": "response", + "name": "DeleteAclsResponse", + // Version 1 adds the resource pattern type. + // Starting in version 1, on quota violation, brokers send out responses before throttling. + "validVersions": "0-1", + "fields": [ + { "name": "ThrottleTimeMs", "type": "int32", "versions": "0+", + "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." }, + { "name": "FilterResults", "type": "[]DeleteAclsFilterResult", "versions": "0+", + "about": "The results for each filter.", "fields": [ + { "name": "ErrorCode", "type": "int16", "versions": "0+", + "about": "The error code, or 0 if the filter succeeded." }, + { "name": "ErrorMessage", "type": "string", "versions": "0+", "nullableVersions": "0+", + "about": "The error message, or null if the filter succeeded." }, + { "name": "MatchingAcls", "type": "[]DeleteAclsMatchingAcl", "versions": "0+", + "about": "The ACLs which matched this filter.", "fields": [ + { "name": "ErrorCode", "type": "int16", "versions": "0+", + "about": "The deletion error code, or 0 if the deletion succeeded." }, + { "name": "ErrorMessage", "type": "string", "versions": "0+", "nullableVersions": "0+", + "about": "The deletion error message, or null if the deletion succeeded." }, + { "name": "ResourceType", "type": "int8", "versions": "0+", + "about": "The ACL resource type." }, + { "name": "ResourceName", "type": "string", "versions": "0+", + "about": "The ACL resource name." }, + { "name": "PatternType", "type": "int8", "versions": "1+", "default": "3", "ignorable": false, + "about": "The ACL resource pattern type." }, + { "name": "Principal", "type": "string", "versions": "0+", + "about": "The ACL principal." }, + { "name": "Host", "type": "string", "versions": "0+", + "about": "The ACL host." }, + { "name": "Operation", "type": "int8", "versions": "0+", + "about": "The ACL operation." }, + { "name": "PermissionType", "type": "int8", "versions": "0+", + "about": "The ACL permission type." } + ]} + ]} + ] +} diff --git a/clients/src/main/resources/message/DeleteGroupsRequest.json b/clients/src/main/resources/message/DeleteGroupsRequest.json new file mode 100644 index 0000000000000..8dd8172de30b7 --- /dev/null +++ b/clients/src/main/resources/message/DeleteGroupsRequest.json @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 42, + "type": "request", + "name": "DeleteGroupsRequest", + // Version 1 is the same as version 0. + "validVersions": "0-1", + "fields": [ + { "name": "GroupsNames", "type": "[]string", "versions": "0+", + "about": "The group names to delete." } + ] +} diff --git a/clients/src/main/resources/message/DeleteGroupsResponse.json b/clients/src/main/resources/message/DeleteGroupsResponse.json new file mode 100644 index 0000000000000..818331bf61b93 --- /dev/null +++ b/clients/src/main/resources/message/DeleteGroupsResponse.json @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 42, + "type": "response", + "name": "DeleteGroupsResponse", + // Starting in version 1, on quota violation, brokers send out responses before throttling. + "validVersions": "0-1", + "fields": [ + { "name": "ThrottleTimeMs", "type": "int32", "versions": "0+", + "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." }, + { "name": "Results", "type": "[]DeletableGroupResult", "versions": "0+", + "about": "The deletion results", "fields": [ + { "name": "GroupId", "type": "string", "versions": "0+", "mapKey": true, + "about": "The group id" }, + { "name": "ErrorCode", "type": "int16", "versions": "0+", + "about": "The deletion error, or 0 if the deletion succeeded." } + ]} + ] +} diff --git a/clients/src/main/resources/message/DeleteRecordsRequest.json b/clients/src/main/resources/message/DeleteRecordsRequest.json new file mode 100644 index 0000000000000..be6c5e755f954 --- /dev/null +++ b/clients/src/main/resources/message/DeleteRecordsRequest.json @@ -0,0 +1,38 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 21, + "type": "request", + "name": "DeleteRecordsRequest", + // Version 1 is the same as version 0. + "validVersions": "0-1", + "fields": [ + { "name": "Topics", "type": "[]DeleteRecordsTopic", "versions": "0+", + "about": "Each topic that we want to delete records from.", "fields": [ + { "name": "Name", "type": "string", "versions": "0+", + "about": "The topic name." }, + { "name": "Partitions", "type": "[]DeleteRecordsPartition", "versions": "0+", + "about": "Each partition that we want to delete records from.", "fields": [ + { "name": "PartitionIndex", "type": "int32", "versions": "0+", + "about": "The partition index." }, + { "name": "Offset", "type": "int64", "versions": "0+", + "about": "The deletion offset." } + ]} + ]}, + { "name": "TimeoutMs", "type": "int32", "versions": "0+", + "about": "How long to wait for the deletion to complete, in milliseconds." } + ] +} diff --git a/clients/src/main/resources/message/DeleteRecordsResponse.json b/clients/src/main/resources/message/DeleteRecordsResponse.json new file mode 100644 index 0000000000000..88ac4ab874cfd --- /dev/null +++ b/clients/src/main/resources/message/DeleteRecordsResponse.json @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 21, + "type": "response", + "name": "DeleteRecordsResponse", + // Starting in version 1, on quota violation, brokers send out responses before throttling. + "validVersions": "0-1", + "fields": [ + { "name": "ThrottleTimeMs", "type": "int32", "versions": "0+", + "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." }, + { "name": "Topics", "type": "[]DeleteRecordsTopicResult", "versions": "0+", + "about": "Each topic that we wanted to delete records from.", "fields": [ + { "name": "Name", "type": "string", "versions": "0+", + "about": "The topic name." }, + { "name": "Partitions", "type": "[]DeleteRecordsPartitionResult", "versions": "0+", + "about": "Each partition that we wanted to delete records from.", "fields": [ + { "name": "PartitionIndex", "type": "int32", "versions": "0+", + "about": "The partition index." }, + { "name": "LowWatermark", "type": "int64", "versions": "0+", + "about": "The partition low water mark." }, + { "name": "ErrorCode", "type": "int16", "versions": "0+", + "about": "The deletion error code, or 0 if the deletion succeeded." } + ]} + ]} + ] +} diff --git a/clients/src/main/resources/message/DeleteTopicsRequest.json b/clients/src/main/resources/message/DeleteTopicsRequest.json new file mode 100644 index 0000000000000..269a3c01734d6 --- /dev/null +++ b/clients/src/main/resources/message/DeleteTopicsRequest.json @@ -0,0 +1,28 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 20, + "type": "request", + "name": "DeleteTopicsRequest", + // Versions 0, 1, 2, and 3 are the same. + "validVersions": "0-3", + "fields": [ + { "name": "TopicNames", "type": "[]string", "versions": "0+", + "about": "The names of the topics to delete" }, + { "name": "TimeoutMs", "type": "int32", "versions": "0+", + "about": "The length of time in milliseconds to wait for the deletions to complete." } + ] +} diff --git a/clients/src/main/resources/message/DeleteTopicsResponse.json b/clients/src/main/resources/message/DeleteTopicsResponse.json new file mode 100644 index 0000000000000..cf0837bbf2065 --- /dev/null +++ b/clients/src/main/resources/message/DeleteTopicsResponse.json @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 20, + "type": "response", + "name": "DeleteTopicsResponse", + // Version 1 adds the throttle time. + // Starting in version 2, on quota violation, brokers send out responses before throttling. + // Starting in version 3, a TOPIC_DELETION_DISABLED error code may be returned. + "validVersions": "0-3", + "fields": [ + { "name": "throttleTimeMs", "type": "int32", "versions": "1+", + "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." }, + { "name": "Responses", "type": "[]DeletableTopicResult", "versions": "0+", + "about": "The results for each topic.", "fields": [ + { "name": "Name", "type": "string", "versions": "0+", + "about": "The topic name" }, + { "name": "ErrorCode", "type": "int16", "versions": "0+", + "about": "The deletion error, or 0 if the deletion succeeded." } + ]} + ] +} diff --git a/clients/src/main/resources/message/DescribeAclsRequest.json b/clients/src/main/resources/message/DescribeAclsRequest.json new file mode 100644 index 0000000000000..9be6bbe24617f --- /dev/null +++ b/clients/src/main/resources/message/DescribeAclsRequest.json @@ -0,0 +1,38 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 29, + "type": "request", + "name": "DescribeAclsRequest", + // Version 1 adds resource pattern type. + "validVersions": "0-1", + "fields": [ + { "name": "ResourceType", "type": "int8", "versions": "0+", + "about": "The resource type." }, + { "name": "ResourceNameFilter", "type": "string", "versions": "0+", "nullableVersions": "0+", + "about": "The resource name, or null to match any resource name." }, + { "name": "ResourcePatternType", "type": "int8", "versions": "1+", "default": "3", "ignorable": false, + "about": "The resource pattern to match." }, + { "name": "PrincipalFilter", "type": "string", "versions": "0+", "nullableVersions": "0+", + "about": "The principal to match, or null to match any principal." }, + { "name": "HostFilter", "type": "string", "versions": "0+", "nullableVersions": "0+", + "about": "The host to match, or null to match any host." }, + { "name": "Operation", "type": "int8", "versions": "0+", + "about": "The operation to match." }, + { "name": "PermissionType", "type": "int8", "versions": "0+", + "about": "The permission type to match." } + ] +} diff --git a/clients/src/main/resources/message/DescribeAclsResponse.json b/clients/src/main/resources/message/DescribeAclsResponse.json new file mode 100644 index 0000000000000..0fdf0c0335864 --- /dev/null +++ b/clients/src/main/resources/message/DescribeAclsResponse.json @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 29, + "type": "response", + "name": "DescribeAclsResponse", + // Version 1 adds PatternType. + // Starting in version 1, on quota violation, brokers send out responses before throttling. + "validVersions": "0-1", + "fields": [ + { "name": "ThrottleTimeMs", "type": "int32", "versions": "0+", + "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." }, + { "name": "ErrorCode", "type": "int16", "versions": "0+", + "about": "The error code, or 0 if there was no error." }, + { "name": "ErrorMessage", "type": "string", "versions": "0+", "nullableVersions": "0+", + "about": "The error message, or null if there was no error." }, + { "name": "Resources", "type": "[]DescribeAclsResource", "versions": "0+", + "about": "Each Resource that is referenced in an ACL.", "fields": [ + { "name": "Type", "type": "int8", "versions": "0+", + "about": "The resource type." }, + { "name": "Name", "type": "string", "versions": "0+", + "about": "The resource name." }, + { "name": "PatternType", "type": "int8", "versions": "1+", "default": "3", "ignorable": false, + "about": "The resource pattern type." }, + { "name": "Acls", "type": "[]AclDescription", "versions": "0+", + "about": "The ACLs.", "fields": [ + { "name": "Principal", "type": "string", "versions": "0+", + "about": "The ACL principal." }, + { "name": "Host", "type": "string", "versions": "0+", + "about": "The ACL host." }, + { "name": "Operation", "type": "int8", "versions": "0+", + "about": "The ACL operation." }, + { "name": "PermissionType", "type": "int8", "versions": "0+", + "about": "The ACL permission type." } + ]} + ]} + ] +} diff --git a/clients/src/main/resources/message/DescribeConfigsRequest.json b/clients/src/main/resources/message/DescribeConfigsRequest.json new file mode 100644 index 0000000000000..5113d4d8dc4b9 --- /dev/null +++ b/clients/src/main/resources/message/DescribeConfigsRequest.json @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 32, + "type": "request", + "name": "DescribeConfigsRequest", + // Version 1 adds IncludeSynoyms. + // Version 2 is the same as version 1. + "validVersions": "0-2", + "fields": [ + { "name": "Resources", "type": "[]DescribeConfigsResource", "versions": "0+", + "about": "The resources whose configurations we want to describe.", "fields": [ + { "name": "ResourceType", "type": "int8", "versions": "0+", + "about": "The resource type." }, + { "name": "ResourceName", "type": "string", "versions": "0+", + "about": "The resource name." }, + { "name": "ConfigurationKeys", "type": "[]string", "versions": "0+", "nullableVersions": "0+", + "about": "The configuration keys to list, or null to list all configuration keys." } + ]}, + { "name": "IncludeSynoyms", "type": "bool", "versions": "1+", "default": "false", "ignorable": false, + "about": "True if we should include all synonyms." } + ] +} diff --git a/clients/src/main/resources/message/DescribeConfigsResponse.json b/clients/src/main/resources/message/DescribeConfigsResponse.json new file mode 100644 index 0000000000000..89cb145b1d678 --- /dev/null +++ b/clients/src/main/resources/message/DescribeConfigsResponse.json @@ -0,0 +1,65 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 32, + "type": "response", + "name": "DescribeConfigsResponse", + // Version 1 adds ConfigSource and the synonyms. + // Starting in version 2, on quota violation, brokers send out responses before throttling. + "validVersions": "0-2", + "fields": [ + { "name": "ThrottleTimeMs", "type": "int32", "versions": "0+", + "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." }, + { "name": "Results", "type": "[]DescribeConfigsResult", "versions": "0+", + "about": "The results for each resource.", "fields": [ + { "name": "ErrorCode", "type": "int16", "versions": "0+", + "about": "The error code, or 0 if we were able to successfully describe the configurations." }, + { "name": "ErrorMessage", "type": "string", "versions": "0+", "nullableVersions": "0+", + "about": "The error message, or null if we were able to successfully describe the configurations." }, + { "name": "ResourceType", "type": "int8", "versions": "0+", + "about": "The resource type." }, + { "name": "ResourceName", "type": "string", "versions": "0+", + "about": "The resource name." }, + { "name": "Configs", "type": "[]DescribeConfigsResourceResult", "versions": "0+", + "about": "Each listed configuration.", "fields": [ + { "name": "Name", "type": "string", "versions": "0+", + "about": "The configuration name." }, + { "name": "Value", "type": "string", "versions": "0+", "nullableVersions": "0+", + "about": "The configuration value." }, + { "name": "ReadOnly", "type": "bool", "versions": "0+", + "about": "True if the configuration is read-only." }, + { "name": "IsDefault", "type": "bool", "versions": "0", + "about": "True if the configuration is not set." }, + // Note: the v0 default for this field that shouldd be exposed to callers is + // context-dependent. For example, if the resource is a broker, this should default to 4. + // -1 is just a placeholder value. + { "name": "ConfigSource", "type": "int8", "versions": "1+", "default": "-1", "ignorable": true, + "about": "The configuration source." }, + { "name": "IsSensitive", "type": "bool", "versions": "0+", + "about": "True if this configuration is sensitive." }, + { "name": "Synonyms", "type": "[]DescribeConfigsSynonym", "versions": "1+", "ignorable": true, + "about": "The synonyms for this configuration key.", "fields": [ + { "name": "Name", "type": "string", "versions": "1+", + "about": "The synonym name." }, + { "name": "Value", "type": "string", "versions": "1+", "nullableVersions": "0+", + "about": "The synonym value." }, + { "name": "Source", "type": "int8", "versions": "1+", + "about": "The synonym source." } + ]} + ]} + ]} + ] +} diff --git a/clients/src/main/resources/message/DescribeDelegationTokenRequest.json b/clients/src/main/resources/message/DescribeDelegationTokenRequest.json new file mode 100644 index 0000000000000..e9ec3baffc5c2 --- /dev/null +++ b/clients/src/main/resources/message/DescribeDelegationTokenRequest.json @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 41, + "type": "request", + "name": "DescribeDelegationTokenRequest", + // Version 1 is the same as version 0. + "validVersions": "0-1", + "fields": [ + { "name": "Owners", "type": "[]DescribeDelegationTokenOwner", "versions": "0+", "nullableVersions": "0+", + "about": "Each owner that we want to describe delegation tokens for, or null to describe all tokens.", "fields": [ + { "name": "PrincipalType", "type": "string", "versions": "0+", + "about": "The owner principal type." }, + { "name": "PrincipalName", "type": "string", "versions": "0+", + "about": "The owner principal name." } + ]} + ] +} diff --git a/clients/src/main/resources/message/DescribeDelegationTokenResponse.json b/clients/src/main/resources/message/DescribeDelegationTokenResponse.json new file mode 100644 index 0000000000000..627246d254edf --- /dev/null +++ b/clients/src/main/resources/message/DescribeDelegationTokenResponse.json @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 41, + "type": "response", + "name": "DescribeDelegationTokenResponse", + // Starting in version 1, on quota violation, brokers send out responses before throttling. + "validVersions": "0-1", + "fields": [ + { "name": "ErrorCode", "type": "int16", "versions": "0+", + "about": "The error code, or 0 if there was no error." }, + { "name": "Tokens", "type": "[]DescribedDelegationToken", "versions": "0+", + "about": "The tokens.", "fields": [ + { "name": "PrincipalType", "type": "string", "versions": "0+", + "about": "The token principal type." }, + { "name": "PrincipalName", "type": "string", "versions": "0+", + "about": "The token principal name." }, + { "name": "IssueTimestamp", "type": "int64", "versions": "0+", + "about": "The token issue timestamp in milliseconds." }, + { "name": "ExpiryTimestamp", "type": "int64", "versions": "0+", + "about": "The token expiry timestamp in milliseconds." }, + { "name": "MaxTimestamp", "type": "int64", "versions": "0+", + "about": "The token maximum timestamp length in milliseconds." }, + { "name": "TokenId", "type": "string", "versions": "0+", + "about": "The token ID." }, + { "name": "Hmac", "type": "bytes", "versions": "0+", + "about": "The token HMAC." }, + { "name": "Renewers", "type": "[]DescribedDelegationTokenRenewer", "versions": "0+", + "about": "Those who are able to renew this token before it expires.", "fields": [ + { "name": "PrincipalType", "type": "string", "versions": "0+", + "about": "The renewer principal type" }, + { "name": "PrincipalName", "type": "string", "versions": "0+", + "about": "The renewer principal name" } + ]} + ]}, + { "name": "ThrottleTimeMs", "type": "int32", "versions": "0+", + "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." } + ] +} diff --git a/clients/src/main/resources/message/DescribeGroupsRequest.json b/clients/src/main/resources/message/DescribeGroupsRequest.json new file mode 100644 index 0000000000000..8039a7e4d6f21 --- /dev/null +++ b/clients/src/main/resources/message/DescribeGroupsRequest.json @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 15, + "type": "request", + "name": "DescribeGroupsRequest", + // Versions 1 and 2 are the same as version 0. + "validVersions": "0-2", + "fields": [ + { "name": "Groups", "type": "[]string", "versions": "0+", + "about": "The names of the groups to describe" } + ] +} diff --git a/clients/src/main/resources/message/DescribeGroupsResponse.json b/clients/src/main/resources/message/DescribeGroupsResponse.json new file mode 100644 index 0000000000000..c5fcd823a3206 --- /dev/null +++ b/clients/src/main/resources/message/DescribeGroupsResponse.json @@ -0,0 +1,57 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 15, + "type": "response", + "name": "DescribeGroupsResponse", + // Version 1 added throttle time. + // Starting in version 2, on quota violation, brokers send out responses before throttling. + "validVersions": "0-2", + "fields": [ + { "name": "ThrottleTimeMs", "type": "int32", "versions": "1+", "ignorable": true, + "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." }, + { "name": "Groups", "type": "[]DescribedGroup", "versions": "0+", + "about": "Each described group.", "fields": [ + { "name": "ErrorCode", "type": "int16", "versions": "0+", + "about": "The describe error, or 0 if there was no error." }, + { "name": "GroupId", "type": "string", "versions": "0+", + "about": "The group ID string." }, + { "name": "GroupState", "type": "string", "versions": "0+", + "about": "The group state string, or the empty string." }, + { "name": "ProtocolType", "type": "string", "versions": "0+", + "about": "The group protocol type, or the empty string." }, + // ProtocolData is currently only filled in if the group state is in the Stable state. + { "name": "ProtocolData", "type": "string", "versions": "0+", + "about": "The group protocol data, or the empty string." }, + // N.B. If the group is in the Dead state, the members array will always be empty. + { "name": "Members", "type": "[]DescribedGroupMember", "versions": "0+", + "about": "The group members.", "fields": [ + { "name": "MemberId", "type": "string", "versions": "0+", + "about": "The member ID assigned by the group coordinator." }, + { "name": "ClientId", "type": "string", "versions": "0+", + "about": "The client ID used in the member's latest join group request." }, + { "name": "ClientHost", "type": "string", "versions": "0+", + "about": "The client host." }, + // This is currently only provided if the group is in the Stable state. + { "name": "MemberMetadata", "type": "bytes", "versions": "0+", + "about": "The metadata corresponding to the current group protocol in use." }, + // This is currently only provided if the group is in the Stable state. + { "name": "MemberAssignment", "type": "bytes", "versions": "0+", + "about": "The current assignment provided by the group leader." } + ]} + ]} + ] +} diff --git a/clients/src/main/resources/message/DescribeLogDirsRequest.json b/clients/src/main/resources/message/DescribeLogDirsRequest.json new file mode 100644 index 0000000000000..b17cd4b3eb3b4 --- /dev/null +++ b/clients/src/main/resources/message/DescribeLogDirsRequest.json @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 35, + "type": "request", + "name": "DescribeLogDirsRequest", + // Version 1 is the same as version 0. + "validVersions": "0-1", + "fields": [ + { "name": "Topics", "type": "[]DescribableLogDirTopic", "versions": "0+", "nullableVersions": "0+", + "about": "Each topic that we want to describe log directories for, or null for all topics.", "fields": [ + { "name": "Topic", "type": "string", "versions": "0+", + "about": "The topic name" }, + { "name": "PartitionIndex", "type": "[]int32", "versions": "0+", + "about": "The partition indxes." } + ]} + ] +} diff --git a/clients/src/main/resources/message/DescribeLogDirsResponse.json b/clients/src/main/resources/message/DescribeLogDirsResponse.json new file mode 100644 index 0000000000000..85355d3a4eaa8 --- /dev/null +++ b/clients/src/main/resources/message/DescribeLogDirsResponse.json @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 35, + "type": "response", + "name": "DescribeLogDirsResponse", + // Starting in version 1, on quota violation, brokers send out responses before throttling. + "validVersions": "0-1", + "fields": [ + { "name": "ThrottleTimeMs", "type": "int32", "versions": "0+", + "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." }, + { "name": "Results", "type": "[]DescribeLogDirsResult", "versions": "0+", + "about": "The log directories.", "fields": [ + { "name": "ErrorCode", "type": "int16", "versions": "0+", + "about": "The error code, or 0 if there was no error." }, + { "name": "LogDir", "type": "string", "versions": "0+", + "about": "The absolute log directory path." }, + { "name": "Topics", "type": "[]DescribeLogDirsTopic", "versions": "0+", + "about": "Each topic.", "fields": [ + { "name": "Name", "type": "string", "versions": "0+", + "about": "The topic name." }, + { "name": "Partitions", "type": "[]DescribeLogDirsPartition", "versions": "0+", "fields": [ + { "name": "PartitionIndex", "type": "int32", "versions": "0+", + "about": "The partition index." }, + { "name": "PartitionSize", "type": "int64", "versions": "0+", + "about": "The size of the log segments in this partition in bytes." }, + { "name": "OffsetLag", "type": "int64", "versions": "0+", + "about": "The lag of the log's LEO w.r.t. partition's HW (if it is the current log for the partition) or current replica's LEO (if it is the future log for the partition)" }, + { "name": "IsFutureKey", "type": "bool", "versions": "0+", + "about": "True if this log is created by AlterReplicaLogDirsRequest and will replace the current log of the replica in the future." } + ]} + ]} + ]} + ] +} diff --git a/clients/src/main/resources/message/EndTxnRequest.json b/clients/src/main/resources/message/EndTxnRequest.json new file mode 100644 index 0000000000000..ebf122439b910 --- /dev/null +++ b/clients/src/main/resources/message/EndTxnRequest.json @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 26, + "type": "request", + "name": "EndTxnRequest", + // Version 1 is the same as version 0. + "validVersions": "0-1", + "fields": [ + { "name": "TransactionalId", "type": "string", "versions": "0+", + "about": "The ID of the transaction to end." }, + { "name": "ProducerId", "type": "int64", "versions": "0+", + "about": "The producer ID." }, + { "name": "ProducerEpoch", "type": "int16", "versions": "0+", + "about": "The current epoch associated with the producer." }, + { "name": "Committed", "type": "bool", "versions": "0+", + "about": "True if the transaction was committed, false if it was aborted." } + ] +} diff --git a/clients/src/main/resources/message/EndTxnResponse.json b/clients/src/main/resources/message/EndTxnResponse.json new file mode 100644 index 0000000000000..b0d4010a7e408 --- /dev/null +++ b/clients/src/main/resources/message/EndTxnResponse.json @@ -0,0 +1,28 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 26, + "type": "response", + "name": "EndTxnResponse", + // Starting in version 1, on quota violation, brokers send out responses before throttling. + "validVersions": "0-1", + "fields": [ + { "name": "ThrottleTimeMs", "type": "int32", "versions": "0+", + "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." }, + { "name": "ErrorCode", "type": "int16", "versions": "0+", + "about": "The error code, or 0 if there was no error." } + ] +} diff --git a/clients/src/main/resources/message/ExpireDelegationTokenRequest.json b/clients/src/main/resources/message/ExpireDelegationTokenRequest.json new file mode 100644 index 0000000000000..fdc50d0559f43 --- /dev/null +++ b/clients/src/main/resources/message/ExpireDelegationTokenRequest.json @@ -0,0 +1,28 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 40, + "type": "request", + "name": "ExpireDelegationTokenRequest", + // Version 1 is the same as version 0. + "validVersions": "0-1", + "fields": [ + { "name": "Hmac", "type": "bytes", "versions": "0+", + "about": "The HMAC of the delegation token to be expired." }, + { "name": "ExpiryTimePeriodMs", "type": "int64", "versions": "0+", + "about": "The expiry time period in milliseconds." } + ] +} diff --git a/clients/src/main/resources/message/ExpireDelegationTokenResponse.json b/clients/src/main/resources/message/ExpireDelegationTokenResponse.json new file mode 100644 index 0000000000000..53fd5a0a7ea1c --- /dev/null +++ b/clients/src/main/resources/message/ExpireDelegationTokenResponse.json @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 40, + "type": "response", + "name": "ExpireDelegationTokenResponse", + // Starting in version 1, on quota violation, brokers send out responses before throttling. + "validVersions": "0-1", + "fields": [ + { "name": "ErrorCode", "type": "int16", "versions": "0+", + "about": "The error code, or 0 if there was no error." }, + { "name": "ExpiryTimestampMs", "type": "int64", "versions": "0+", + "about": "The timestamp in milliseconds at which this token expires." }, + { "name": "ThrottleTimeMs", "type": "int32", "versions": "0+", + "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." } + ] +} diff --git a/clients/src/main/resources/message/FetchRequest.json b/clients/src/main/resources/message/FetchRequest.json new file mode 100644 index 0000000000000..ee1e88b65e562 --- /dev/null +++ b/clients/src/main/resources/message/FetchRequest.json @@ -0,0 +1,89 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 1, + "type": "request", + "name": "FetchRequest", + // + // Version 1 is the same as version 0. + // + // Starting in Version 2, the requestor must be able to handle Kafka Log + // Message format version 1. + // + // Version 3 adds MaxBytes. Starting in version 3, the partition ordering in + // the request is now relevant. Partitions will be processed in the order + // they appear in the request. + // + // Version 4 adds IsolationLevel. Starting in version 4, the reqestor must be + // able to handle Kafka log message format version 2. + // + // Version 5 adds LogStartOffset to indicate the earliest available offset of + // partition data that can be consumed. + // + // Version 6 is the same as version 5. + // + // Version 7 adds incremental fetch request support. + // + // Version 8 is the same as version 7. + // + // Version 9 adds CurrentLeaderEpoch, as described in KIP-320. + // + // Version 10 indicates that we can use the ZStd compression algorithm, as + // described in KIP-110. + // + "validVersions": "0-10", + "fields": [ + { "name": "ReplicaId", "type": "int32", "versions": "0+", + "about": "The broker ID of the follower, of -1 if this request is from a consumer." }, + { "name": "MaxWait", "type": "int32", "versions": "0+", + "about": "The maximum time in milliseconds to wait for the response." }, + { "name": "MinBytes", "type": "int32", "versions": "0+", + "about": "The minimum bytes to accumulate in the response." }, + { "name": "MaxBytes", "type": "int32", "versions": "3+", "default": "0x7fffffff", "ignorable": true, + "about": "The maximum bytes to fetch. See KIP-74 for cases where this limit may not be honored." }, + { "name": "IsolationLevel", "type": "int8", "versions": "4+", "default": "0", "ignorable": false, + "about": "This setting controls the visibility of transactional records. Using READ_UNCOMMITTED (isolation_level = 0) makes all records visible. With READ_COMMITTED (isolation_level = 1), non-transactional and COMMITTED transactional records are visible. To be more concrete, READ_COMMITTED returns all data from offsets smaller than the current LSO (last stable offset), and enables the inclusion of the list of aborted transactions in the result, which allows consumers to discard ABORTED transactional records" }, + { "name": "SessionId", "type": "int32", "versions": "7+", "default": "0", "ignorable": false, + "about": "The fetch session ID." }, + { "name": "Epoch", "type": "int32", "versions": "7+", "default": "-1", "ignorable": false, + "about": "The fetch session ID." }, + { "name": "Topics", "type": "[]FetchableTopic", "versions": "0+", + "about": "The topics to fetch.", "fields": [ + { "name": "Name", "type": "string", "versions": "0+", + "about": "The name of the topic to fetch." }, + { "name": "FetchPartitions", "type": "[]FetchPartition", "versions": "0+", + "about": "The partitions to fetch.", "fields": [ + { "name": "PartitionIndex", "type": "int32", "versions": "0+", + "about": "The partition index." }, + { "name": "CurrentLeaderEpoch", "type": "int32", "versions": "9+", "default": "-1", "ignorable": true, + "about": "The current leader epoch of the partition." }, + { "name": "FetchOffset", "type": "int64", "versions": "0+", + "about": "The message offset." }, + { "name": "LogStartOffset", "type": "int64", "versions": "5+", "default": "-1", "ignorable": false, + "about": "The earliest available offset of the follower replica. The field is only used when the request is sent by the follower."}, + { "name": "MaxBytes", "type": "int32", "versions": "0+", + "about": "The maximum bytes to fetch from this partition. See KIP-74 for cases where this limit may not be honored." } + ]} + ]}, + { "name": "Forgotten", "type": "[]ForgottenTopic", "versions": "7+", "ignorable": false, + "about": "In an incremental fetch request, the partitions to remove.", "fields": [ + { "name": "Name", "type": "string", "versions": "7+", + "about": "The partition name." }, + { "name": "ForgottenPartitionIndexes", "type": "[]int32", "versions": "7+", + "about": "The partitions indexes to forget." } + ]} + ] +} diff --git a/clients/src/main/resources/message/FetchResponse.json b/clients/src/main/resources/message/FetchResponse.json new file mode 100644 index 0000000000000..afee391c71065 --- /dev/null +++ b/clients/src/main/resources/message/FetchResponse.json @@ -0,0 +1,77 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 1, + "type": "response", + "name": "FetchResponse", + // + // Version 1 adds throttle time. + // + // Version 2 and 3 are the same as version 1. + // + // Version 4 adds features for transactional consumption. + // + // Version 5 adds LogStartOffset to indicate the earliest available offset of + // partition data that can be consumed. + // + // Starting in version 6, we may return KAFKA_STORAGE_ERROR as an error code. + // + // Version 7 adds incremental fetch request support. + // + // Starting in version 8, on quota violation, brokers send out responses before throttling. + // + // Version 9 is the same as version 8. + // + // Version 10 indicates that the response data can use the ZStd compression + // algorithm, as described in KIP-110. + // + "validVersions": "0-10", + "fields": [ + { "name": "ThrottleTimeMs", "type": "int32", "versions": "1+", "ignorable": true, + "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." }, + { "name": "ErrorCode", "type": "int16", "versions": "7+", "ignorable": false, + "about": "The top level response error code." }, + { "name": "SessionId", "type": "int32", "versions": "7+", "default": "0", "ignorable": false, + "about": "The fetch session ID, or 0 if this is not part of a fetch session." }, + { "name": "Topics", "type": "[]FetchableTopicResponse", "versions": "0+", + "about": "The response topics.", "fields": [ + { "name": "Name", "type": "string", "versions": "0+", + "about": "The topic name." }, + { "name": "Partitions", "type": "[]FetchablePartitionResponse", "versions": "0+", + "about": "The topic partitions.", "fields": [ + { "name": "PartitionIndex", "type": "int32", "versions": "0+", + "about": "The partiiton index." }, + { "name": "ErrorCode", "type": "int16", "versions": "0+", + "about": "The error code, or 0 if there was no fetch error." }, + { "name": "HighWatermark", "type": "int64", "versions": "0+", + "about": "The current high water mark." }, + { "name": "LastStableOffset", "type": "int64", "versions": "4+", "default": "-1", "ignorable": true, + "about": "The last stable offset (or LSO) of the partition. This is the last offset such that the state of all transactional records prior to this offset have been decided (ABORTED or COMMITTED)" }, + { "name": "LogStartOffset", "type": "int64", "versions": "5+", "default": "-1", "ignorable": true, + "about": "The current log start offset." }, + { "name": "Aborted", "type": "[]AbortedTransaction", "versions": "4+", "nullableVersions": "4+", "ignorable": false, + "about": "The aborted transactions.", "fields": [ + { "name": "ProducerId", "type": "int64", "versions": "4+", + "about": "The producer id associated with the aborted transaction." }, + { "name": "FirstOffset", "type": "int64", "versions": "4+", + "about": "The first offset in the aborted transaction." } + ]}, + { "name": "Records", "type": "bytes", "versions": "0+", "nullableVersions": "0+", + "about": "The record data." } + ]} + ]} + ] +} diff --git a/clients/src/main/resources/message/FindCoordinatorRequest.json b/clients/src/main/resources/message/FindCoordinatorRequest.json new file mode 100644 index 0000000000000..8ef3f7420e2f3 --- /dev/null +++ b/clients/src/main/resources/message/FindCoordinatorRequest.json @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 10, + "type": "request", + "name": "FindCoordinatorRequest", + // Version 1 adds KeyType. + // Version 2 is the same as version 1. + "validVersions": "0-2", + "fields": [ + { "name": "Key", "type": "string", "versions": "0+", + "about": "The coordinator key." }, + { "name": "KeyType", "type": "int8", "versions": "1+", "default": "0", "ignorable": false, + "about": "The coordinator key type. (Group, transaction, etc.)" } + ] +} diff --git a/clients/src/main/resources/message/FindCoordinatorResponse.json b/clients/src/main/resources/message/FindCoordinatorResponse.json new file mode 100644 index 0000000000000..aed8f1a8ae323 --- /dev/null +++ b/clients/src/main/resources/message/FindCoordinatorResponse.json @@ -0,0 +1,37 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 10, + "type": "response", + "name": "FindCoordinatorResponse", + // Version 1 adds throttle time and error messages. + // Starting in version 2, on quota violation, brokers send out responses before throttling. + "validVersions": "0-2", + "fields": [ + { "name": "ThrottleTimeMs", "type": "int32", "versions": "1+", "ignorable": true, + "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." }, + { "name": "ErrorCode", "type": "int16", "versions": "0+", + "about": "The error code, or 0 if there was no error." }, + { "name": "ErrorMessage", "type": "string", "versions": "1+", "nullableVersions": "1+", "ignorable": true, + "about": "The error message, or null if there was no error." }, + { "name": "NodeId", "type": "int32", "versions": "0+", + "about": "The node id." }, + { "name": "Host", "type": "string", "versions": "0+", + "about": "The host name." }, + { "name": "Port", "type": "int32", "versions": "0+", + "about": "The port." } + ] +} diff --git a/clients/src/main/resources/message/HeartbeatRequest.json b/clients/src/main/resources/message/HeartbeatRequest.json new file mode 100644 index 0000000000000..61cb20e034369 --- /dev/null +++ b/clients/src/main/resources/message/HeartbeatRequest.json @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 12, + "type": "request", + "name": "HeartbeatRequest", + // Version 1 and version 2 are the same as version 0. + "validVersions": "0-2", + "fields": [ + { "name": "GroupId", "type": "string", "versions": "0+", + "about": "The group id." }, + { "name": "Generationid", "type": "int32", "versions": "0+", + "about": "The generation of the group." }, + { "name": "MemberId", "type": "string", "versions": "0+", + "about": "The member ID." } + ] +} diff --git a/clients/src/main/resources/message/HeartbeatResponse.json b/clients/src/main/resources/message/HeartbeatResponse.json new file mode 100644 index 0000000000000..c19ba37ad1774 --- /dev/null +++ b/clients/src/main/resources/message/HeartbeatResponse.json @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 12, + "type": "response", + "name": "HeartbeatResponse", + // Version 1 adds throttle time. + // Starting in version 2, on quota violation, brokers send out responses before throttling. + "validVersions": "0-2", + "fields": [ + { "name": "ThrottleTimeMs", "type": "int32", "versions": "1+", "ignorable": true, + "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." }, + { "name": "ErrorCode", "type": "int16", "versions": "0+", + "about": "The error code, or 0 if there was no error." } + ] +} diff --git a/clients/src/main/resources/message/InitProducerIdRequest.json b/clients/src/main/resources/message/InitProducerIdRequest.json new file mode 100644 index 0000000000000..8bf2ce35748d4 --- /dev/null +++ b/clients/src/main/resources/message/InitProducerIdRequest.json @@ -0,0 +1,28 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 22, + "type": "request", + "name": "InitProducerIdRequest", + // Version 1 is the same as version 0. + "validVersions": "0-1", + "fields": [ + { "name": "TransactionalId", "type": "string", "versions": "0+", "nullableVersions": "0+", + "about": "The transactional id, or null if the producer is not transactional." }, + { "name": "TransactionTimeoutMs", "type": "int32", "versions": "0+", + "about": "The time in ms to wait for before aborting idle transactions sent by this producer." } + ] +} diff --git a/clients/src/main/resources/message/InitProducerIdResponse.json b/clients/src/main/resources/message/InitProducerIdResponse.json new file mode 100644 index 0000000000000..b251051ec9022 --- /dev/null +++ b/clients/src/main/resources/message/InitProducerIdResponse.json @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 22, + "type": "response", + "name": "InitProducerIdResponse", + // Starting in version 1, on quota violation, brokers send out responses before throttling. + "validVersions": "0-1", + "fields": [ + { "name": "ThrottleTimeMs", "type": "int32", "versions": "0+", "ignorable": true, + "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." }, + { "name": "ErrorCode", "type": "int16", "versions": "0+", + "about": "The error code, or 0 if there was no error." }, + { "name": "ProducerId", "type": "int64", "versions": "0+", + "about": "The current producer id." }, + { "name": "ProducerEpoch", "type": "int16", "versions": "0+", + "about": "The current epoch associated with the producer id." } + ] +} diff --git a/clients/src/main/resources/message/JoinGroupRequest.json b/clients/src/main/resources/message/JoinGroupRequest.json new file mode 100644 index 0000000000000..50812e76255a2 --- /dev/null +++ b/clients/src/main/resources/message/JoinGroupRequest.json @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 11, + "type": "request", + "name": "JoinGroupRequest", + // Version 1 adds RebalanceTimeoutMs. + // Version 2 and 3 are the same as version 1. + "validVersions": "0-3", + "fields": [ + { "name": "GroupId", "type": "string", "versions": "0+", + "about": "The group identifier." }, + { "name": "SessionTimeoutMs", "type": "int32", "versions": "0+", + "about": "The coordinator considers the consumer dead if it receives no heartbeat after this timeout in milliseconds." }, + // Note: if RebalanceTimeoutMs is not present, SessionTimeoutMs should be + // used instead. The default of -1 here is just intended as a placeholder. + { "name": "RebalanceTimeoutMs", "type": "int32", "versions": "1+", "default": "-1", "ignorable": true, + "about": "The maximum time in milliseconds that the coordinator will wait for each member to rejoin when rebalancing the group." }, + { "name": "MemberId", "type": "string", "versions": "0+", + "about": "The member id assigned by the group coordinator." }, + { "name": "ProtocolType", "type": "string", "versions": "0+", + "about": "The unique name the for class of protocols implemented by the group we want to join." }, + { "name": "Protocols", "type": "[]JoinGroupRequestProtocol", "versions": "0+", + "about": "The list of protocols that the member supports.", "fields": [ + { "name": "Name", "type": "string", "versions": "0+", "mapKey": true, + "about": "The protocol name." }, + { "name": "Metadata", "type": "bytes", "versions": "0+", + "about": "The protocol metadata." } + ]} + ] +} diff --git a/clients/src/main/resources/message/JoinGroupResponse.json b/clients/src/main/resources/message/JoinGroupResponse.json new file mode 100644 index 0000000000000..42a28c9b2b5b9 --- /dev/null +++ b/clients/src/main/resources/message/JoinGroupResponse.json @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 11, + "type": "response", + "name": "JoinGroupResponse", + // Version 1 is the same as version 0. + // Version 2 adds throttle time. + // Starting in version 3, on quota violation, brokers send out responses before throttling. + "validVersions": "0-3", + "fields": [ + { "name": "ThrottleTimeMs", "type": "int32", "versions": "2+", "ignorable": true, + "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." }, + { "name": "ErrorCode", "type": "int16", "versions": "0+", + "about": "The error code, or 0 if there was no error." }, + { "name": "GenerationId", "type": "int32", "versions": "0+", + "about": "The generation ID of the group." }, + { "name": "ProtocolName", "type": "string", "versions": "0+", + "about": "The group protocol selected by the coordinator." }, + { "name": "Leader", "type": "string", "versions": "0+", + "about": "The leader of the group." }, + { "name": "MemberId", "type": "string", "versions": "0+", + "about": "The member ID assigned by the group coordinator." }, + { "name": "Members", "type": "[]JoinGroupResponseMember", "versions": "0+", "fields": [ + { "name": "MemberId", "type": "string", "versions": "0+", + "about": "The group member ID." }, + { "name": "Metadata", "type": "bytes", "versions": "0+", + "about": "The group member metadata." } + ]} + ] +} diff --git a/clients/src/main/resources/message/LeaderAndIsrRequest.json b/clients/src/main/resources/message/LeaderAndIsrRequest.json new file mode 100644 index 0000000000000..b8988351c8da0 --- /dev/null +++ b/clients/src/main/resources/message/LeaderAndIsrRequest.json @@ -0,0 +1,86 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 4, + "type": "request", + "name": "LeaderAndIsrRequest", + // Version 1 adds IsNew. + // + // Version 2 adds broker epoch and reorganizes the partitions by topic. + "validVersions": "0-2", + "fields": [ + { "name": "ControllerId", "type": "int32", "versions": "0+", + "about": "The current controller ID." }, + { "name": "ControllerEpoch", "type": "int32", "versions": "0+", + "about": "The current controller epoch." }, + { "name": "BrokerEpoch", "type": "int64", "versions": "2+", "ignorable": true, "default": "-1", + "about": "The current broker epoch." }, + { "name": "TopicStates", "type": "[]LeaderAndIsrRequestTopicState", "versions": "2+", + "about": "Each topic.", "fields": [ + { "name": "Name", "type": "string", "versions": "2+", + "about": "The topic name." }, + { "name": "PartitionStates", "type": "[]LeaderAndIsrRequestPartitionState", "versions": "0+", + "about": "The state of each partition", "fields": [ + { "name": "PartitionIndex", "type": "int32", "versions": "0+", + "about": "The partition index." }, + { "name": "ControllerEpoch", "type": "int32", "versions": "0+", + "about": "The controller epoch." }, + { "name": "LeaderKey", "type": "int32", "versions": "0+", + "about": "The broker ID of the leader." }, + { "name": "LeaderEpoch", "type": "int32", "versions": "0+", + "about": "The leader epoch." }, + { "name": "IsrReplicas", "type": "[]int32", "versions": "0+", + "about": "The in-sync replica IDs." }, + { "name": "ZkVersion", "type": "int32", "versions": "0+", + "about": "The ZooKeeper version." }, + { "name": "Replicas", "type": "[]int32", "versions": "0+", + "about": "The replica IDs." }, + { "name": "IsNew", "type": "bool", "versions": "1+", "default": "false", "ignorable": true, + "about": "Whether the replica should have existed on the broker or not." } + ]} + ]}, + { "name": "PartitionStatesV0", "type": "[]LeaderAndIsrRequestPartitionStateV0", "versions": "0-1", + "about": "The state of each partition", "fields": [ + { "name": "TopicName", "type": "string", "versions": "0-1", + "about": "The topic name." }, + { "name": "PartitionIndex", "type": "int32", "versions": "0-1", + "about": "The partition index." }, + { "name": "ControllerEpoch", "type": "int32", "versions": "0-1", + "about": "The controller epoch." }, + { "name": "LeaderKey", "type": "int32", "versions": "0-1", + "about": "The broker ID of the leader." }, + { "name": "LeaderEpoch", "type": "int32", "versions": "0-1", + "about": "The leader epoch." }, + { "name": "IsrReplicas", "type": "[]int32", "versions": "0-1", + "about": "The in-sync replica IDs." }, + { "name": "ZkVersion", "type": "int32", "versions": "0-1", + "about": "The ZooKeeper version." }, + { "name": "Replicas", "type": "[]int32", "versions": "0-1", + "about": "The replica IDs." }, + { "name": "IsNew", "type": "bool", "versions": "1", "default": "false", "ignorable": true, + "about": "Whether the replica should have existed on the broker or not." } + ]}, + { "name": "LiveLeaders", "type": "[]LeaderAndIsrLiveLeader", "versions": "0+", + "about": "The current live leaders.", "fields": [ + { "name": "BrokerId", "type": "int32", "versions": "0+", + "about": "The leader's broker ID." }, + { "name": "HostName", "type": "string", "versions": "0+", + "about": "The leader's hostname." }, + { "name": "Port", "type": "int32", "versions": "0+", + "about": "The leader's port." } + ]} + ] +} diff --git a/clients/src/main/resources/message/LeaderAndIsrResponse.json b/clients/src/main/resources/message/LeaderAndIsrResponse.json new file mode 100644 index 0000000000000..e4e1e09098c46 --- /dev/null +++ b/clients/src/main/resources/message/LeaderAndIsrResponse.json @@ -0,0 +1,37 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 4, + "type": "response", + "name": "LeaderAndIsrResponse", + // Version 1 adds KAFKA_STORAGE_ERROR as a valid error code. + // + // Version 2 is the same as version 1. + "validVersions": "0-2", + "fields": [ + { "name": "ErrorCode", "type": "int16", "versions": "0+", + "about": "The error code, or 0 if there was no error." }, + { "name": "Partitions", "type": "[]LeaderAndIsrResponsePartition", "versions": "0+", + "about": "Each partition.", "fields": [ + { "name": "TopicName", "type": "string", "versions": "0+", + "about": "The topic name." }, + { "name": "PartitionIndex", "type": "int32", "versions": "0+", + "about": "The partition index." }, + { "name": "ErrorCode", "type": "int16", "versions": "0+", + "about": "The partition error code, or 0 if there was no error." } + ]} + ] +} diff --git a/clients/src/main/resources/message/LeaveGroupRequest.json b/clients/src/main/resources/message/LeaveGroupRequest.json new file mode 100644 index 0000000000000..9448705fd66b4 --- /dev/null +++ b/clients/src/main/resources/message/LeaveGroupRequest.json @@ -0,0 +1,28 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 13, + "type": "request", + "name": "LeaveGroupRequest", + // Version 1 and 2 are the same as version 0. + "validVersions": "0-2", + "fields": [ + { "name": "GroupId", "type": "string", "versions": "0+", + "about": "The ID of the group to leave." }, + { "name": "MemberId", "type": "string", "versions": "0+", + "about": "The member ID to remove from the group." } + ] +} diff --git a/clients/src/main/resources/message/LeaveGroupResponse.json b/clients/src/main/resources/message/LeaveGroupResponse.json new file mode 100644 index 0000000000000..0d887cdcb0289 --- /dev/null +++ b/clients/src/main/resources/message/LeaveGroupResponse.json @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 13, + "type": "response", + "name": "LeaveGroupResponse", + // Version 1 adds the throttle time. + // Starting in version 2, on quota violation, brokers send out responses before throttling. + "validVersions": "0-2", + "fields": [ + { "name": "ThrottleTimeMs", "type": "int32", "versions": "1+", "ignorable": true, + "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." }, + { "name": "ErrorCode", "type": "int16", "versions": "0+", + "about": "The error code, or 0 if there was no error." } + ] +} diff --git a/clients/src/main/resources/message/ListGroupsRequest.json b/clients/src/main/resources/message/ListGroupsRequest.json new file mode 100644 index 0000000000000..b0c5aa0e3e81d --- /dev/null +++ b/clients/src/main/resources/message/ListGroupsRequest.json @@ -0,0 +1,24 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 16, + "type": "request", + "name": "ListGroupsRequest", + // Version 1 and 2 are the same as version 0. + "validVersions": "0-2", + "fields": [ + ] +} diff --git a/clients/src/main/resources/message/ListGroupsResponse.json b/clients/src/main/resources/message/ListGroupsResponse.json new file mode 100644 index 0000000000000..2dc83fa0121d8 --- /dev/null +++ b/clients/src/main/resources/message/ListGroupsResponse.json @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 16, + "type": "response", + "name": "ListGroupsResponse", + // Version 1 adds the throttle time. + // Starting in version 2, on quota violation, brokers send out responses before throttling. + "validVersions": "0-2", + "fields": [ + { "name": "ThrottleTimeMs", "type": "int32", "versions": "1+", "ignorable": true, + "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." }, + { "name": "ErrorCode", "type": "int16", "versions": "0+", + "about": "The error code, or 0 if there was no error." }, + { "name": "Groups", "type": "[]ListedGroup", "versions": "0+", + "about": "Each group in the response.", "fields": [ + { "name": "GroupId", "type": "string", "versions": "0+", + "about": "The group ID." }, + { "name": "ProtocolType", "type": "string", "versions": "0+", + "about": "The group protocol type." } + ]} + ] +} diff --git a/clients/src/main/resources/message/ListOffsetRequest.json b/clients/src/main/resources/message/ListOffsetRequest.json new file mode 100644 index 0000000000000..d9d089a795743 --- /dev/null +++ b/clients/src/main/resources/message/ListOffsetRequest.json @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 2, + "type": "request", + "name": "ListOffsetRequest", + // Version 1 removes MaxNumOffsets. From this version forward, only a single + // offset can be returned. + // Version 2 adds the isolation level, which is used for transactional reads. + // Version 3 is the same as version 2. + // Version 4 adds the current leader epoch, which is used for fencing. + "validVersions": "0-4", + "fields": [ + { "name": "ReplicaId", "type": "int32", "versions": "0+", + "about": "The broker ID of the requestor, or -1 if this request is being made by a normal consumer." }, + { "name": "IsolationLevel", "type": "int8", "versions": "2+", + "about": "This setting controls the visibility of transactional records. Using READ_UNCOMMITTED (isolation_level = 0) makes all records visible. With READ_COMMITTED (isolation_level = 1), non-transactional and COMMITTED transactional records are visible. To be more concrete, READ_COMMITTED returns all data from offsets smaller than the current LSO (last stable offset), and enables the inclusion of the list of aborted transactions in the result, which allows consumers to discard ABORTED transactional records" }, + { "name": "Topics", "type": "[]ListOffsetTopic", "versions": "0+", + "about": "Each topic in the request.", "fields": [ + { "name": "Name", "type": "string", "versions": "0+", + "about": "The topic name." }, + { "name": "Partitions", "type": "[]ListOffsetPartition", "versions": "0+", + "about": "Each partition in the request.", "fields": [ + { "name": "PartitionIndex", "type": "int32", "versions": "0+", + "about": "The partition index." }, + { "name": "CurrentLeaderEpoch", "type": "int32", "versions": "4+", + "about": "The current leader epoch." }, + { "name": "Timestamp", "type": "int64", "versions": "0+", + "about": "The current timestamp." }, + { "name": "MaxNumOffsets", "type": "int32", "versions": "0", + "about": "The maximum number of offsets to report." } + ]} + ]} + ] +} diff --git a/clients/src/main/resources/message/ListOffsetResponse.json b/clients/src/main/resources/message/ListOffsetResponse.json new file mode 100644 index 0000000000000..933c0e778cb9a --- /dev/null +++ b/clients/src/main/resources/message/ListOffsetResponse.json @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 2, + "type": "response", + "name": "ListOffsetResponse", + // Version 1 removes the offsets array in favor of returning a single offset. + // Version 1 also adds the timestamp associated with the returned offset. + // Version 2 adds the throttle time. + // Starting in version 3, on quota violation, brokers send out responses before throttling. + // Version 4 adds the leader epoch, which is used for fencing. + "validVersions": "0-4", + "fields": [ + { "name": "ThrottleTimeMs", "type": "int32", "versions": "2+", "ignorable": true, + "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." }, + { "name": "Topics", "type": "[]ListOffsetTopicResponse", "versions": "0+", + "about": "Each topic in the response.", "fields": [ + { "name": "Name", "type": "string", "versions": "0+", + "about": "The topic name" }, + { "name": "Partitions", "type": "[]ListOffsetPartitionResponse", "versions": "0+", + "about": "Each partition in the response.", "fields": [ + { "name": "PartitionIndex", "type": "int32", "versions": "0+", + "about": "The partition index." }, + { "name": "ErrorCode", "type": "int16", "versions": "0+", + "about": "The partition error code, or 0 if there was no error." }, + { "name": "OldStyleOffsets", "type": "[]int64", "versions": "0", "ignorable": false, + "about": "The result offsets." }, + { "name": "Timestamp", "type": "int64", "versions": "1+" }, + { "name": "Offset", "type": "int64", "versions": "1+", + "about": "The timestamp associated with the returned offset." }, + { "name": "LeaderEpoch", "type": "int32", "versions": "4+" } + ]} + ]} + ] +} diff --git a/clients/src/main/resources/message/MetadataRequest.json b/clients/src/main/resources/message/MetadataRequest.json new file mode 100644 index 0000000000000..74f3fab5cd585 --- /dev/null +++ b/clients/src/main/resources/message/MetadataRequest.json @@ -0,0 +1,37 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 3, + "type": "request", + "name": "MetadataRequest", + "validVersions": "0-7", + "fields": [ + // In version 0, an empty array indicates "request metadata for all topics." In version 1 and + // higher, an empty array indicates "request metadata for no topics," and a null array is used to + // indiate "request metadata for all topics." + // + // Version 2 and 3 are the same as version 1. + // + // Version 4 adds AllowAutoTopicCreation. + { "name": "Topics", "type": "[]MetadataRequestTopic", "versions": "0+", "nullableVersions": "1+", + "about": "The topics to fetch metadata for.", "fields": [ + { "name": "Name", "type": "string", "versions": "0+", + "about": "The topic name." } + ]}, + { "name": "AllowAutoTopicCreation", "type": "bool", "versions": "4+", "default": "true", "ignorable": false, + "about": "If this is true, the broker may auto-create topics that we requested which do not already exist, if it is configured to do so." } + ] +} diff --git a/clients/src/main/resources/message/MetadataResponse.json b/clients/src/main/resources/message/MetadataResponse.json new file mode 100644 index 0000000000000..5de87af48093d --- /dev/null +++ b/clients/src/main/resources/message/MetadataResponse.json @@ -0,0 +1,81 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 3, + "type": "response", + "name": "MetadataResponse", + // Version 1 adds fields for the rack of each broker, the controller id, and + // whether or not the topic is internal. + // + // Version 2 adds the cluster ID field. + // + // Version 3 adds the throttle time. + // + // Version 4 is the same as version 3. + // + // Version 5 adds a per-partition offline_replicas field. This field specifies + // the list of replicas that are offline. + // + // Starting in version 6, on quota violation, brokers send out responses before throttling. + // + // Version 7 adds the leader epoch to the partition metadata. + "validVersions": "0-7", + "fields": [ + { "name": "ThrottleTimeMs", "type": "int32", "versions": "3+", + "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." }, + { "name": "Brokers", "type": "[]MetadataResponseBroker", "versions": "0+", + "about": "Each broker in the response.", "fields": [ + { "name": "NodeId", "type": "int32", "versions": "0+", "mapKey": true, + "about": "The broker ID." }, + { "name": "Host", "type": "string", "versions": "0+", + "about": "The broker hostname." }, + { "name": "Port", "type": "int32", "versions": "0+", + "about": "The broker port." }, + { "name": "Rack", "type": "string", "versions": "1+", "nullableVersions": "1+", "ignorable": true, + "about": "The rack of the broker, or null if it has not been assigned to a rack." } + ]}, + { "name": "ClusterId", "type": "string", "nullableVersions": "2+", "versions": "2+", "ignorable": true, + "about": "The cluster ID that responding broker belongs to." }, + { "name": "ControllerId", "type": "int32", "versions": "1+", + "about": "The ID of the controller broker." }, + { "name": "Topics", "type": "[]MetadataResponseTopic", "versions": "0+", + "about": "Each topic in the response.", "fields": [ + { "name": "ErrorCode", "type": "int16", "versions": "0+", + "about": "The topic error, or 0 if there was no error." }, + { "name": "Name", "type": "string", "versions": "0+", "mapKey": true, + "about": "The topic name." }, + { "name": "IsInternal", "type": "bool", "versions": "1+", "default": "false", "ignorable": true, + "about": "True if the topic is internal." }, + { "name": "Partitions", "type": "[]MetadataResponsePartition", "versions": "0+", + "about": "Each partition in the topic.", "fields": [ + { "name": "ErrorCode", "type": "int16", "versions": "0+", + "about": "The partition error, or 0 if there was no error." }, + { "name": "PartitionIndex", "type": "int32", "versions": "0+", + "about": "The partition index." }, + { "name": "LeaderId", "type": "int32", "versions": "0+", + "about": "The ID of the leader broker." }, + { "name": "LeaderEpoch", "type": "int32", "versions": "7+", "default": "-1", "ignorable": true, + "about": "The leader epoch of this partition." }, + { "name": "ReplicaNodes", "type": "[]int32", "versions": "0+", + "about": "The set of all nodes that host this partition." }, + { "name": "IsrNodes", "type": "[]int32", "versions": "0+", + "about": "The set of nodes that are in sync with the leader for this partition." }, + { "name": "OfflineReplicas", "type": "[]int32", "versions": "5+", "ignorable": true, + "about": "The set of offline replicas of this partition." } + ]} + ]} + ] +} diff --git a/clients/src/main/resources/message/OffsetCommitRequest.json b/clients/src/main/resources/message/OffsetCommitRequest.json new file mode 100644 index 0000000000000..ebccedef58772 --- /dev/null +++ b/clients/src/main/resources/message/OffsetCommitRequest.json @@ -0,0 +1,59 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 8, + "type": "request", + "name": "OffsetCommitRequest", + // Version 1 adds timestamp and group membership information, as well as the commit timestamp. + // + // Version 2 adds retention time. It removes the commit timestamp added in version 1. + // + // Version 3 and 4 are the same as version 2. + // + // Version 5 removes the retention time, which is now controlled only by a broker configuration. + // + // Version 6 adds the leader epoch for fencing. + "validVersions": "0-6", + "fields": [ + { "name": "GroupId", "type": "string", "versions": "0+", + "about": "The unique group identifier." }, + { "name": "GenerationId", "type": "int32", "versions": "1+", "default": "-1", "ignorable": true, + "about": "The generation of the group." }, + { "name": "MemberId", "type": "string", "versions": "1+", "ignorable": true, + "about": "The member ID assigned by the group coordinator." }, + { "name": "RetentionTimeMs", "type": "int64", "versions": "2-4", "default": "-1", "ignorable": true, + "about": "The time period in ms to retain the offset." }, + { "name": "Topics", "type": "[]OffsetCommitRequestTopic", "versions": "0+", + "about": "The topics to commit offsets for.", "fields": [ + { "name": "Name", "type": "string", "versions": "0+", + "about": "The topic name." }, + { "name": "Partitions", "type": "[]OffsetCommitRequestPartition", "versions": "0+", + "about": "Each partition to commit offsets for.", "fields": [ + { "name": "PartitionIndex", "type": "int32", "versions": "0+", + "about": "The partition index." }, + { "name": "CommittedOffset", "type": "int64", "versions": "0+", + "about": "The message offset to be committed." }, + { "name": "CommittedLeaderEpoch", "type": "int32", "versions": "6+", "default": "-1", "ignorable": true, + "about": "The leader epoch of this partition." }, + // CommitTimestamp has been removed from v2 and later. + { "name": "CommitTimestamp", "type": "int64", "versions": "1", "default": "-1", + "about": "The timestamp of the commit." }, + { "name": "CommittedMetadata", "type": "string", "versions": "0+", "nullableVersions": "0+", + "about": "Any associated metadata the client wants to keep." } + ]} + ]} + ] +} diff --git a/clients/src/main/resources/message/OffsetCommitResponse.json b/clients/src/main/resources/message/OffsetCommitResponse.json new file mode 100644 index 0000000000000..39daa56562f50 --- /dev/null +++ b/clients/src/main/resources/message/OffsetCommitResponse.json @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 8, + "type": "response", + "name": "OffsetCommitResponse", + // Versions 1 and 2 are the same as version 0. + // + // Version 3 adds the throttle time to the response. + // + // Starting in version 4, on quota violation, brokers send out responses before throttling. + // + // Versions 5 and 6 are the same as version 4. + "validVersions": "0-6", + "fields": [ + { "name": "ThrottleTimeMs", "type": "int32", "versions": "3+", "ignorable": true, + "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." }, + { "name": "Topics", "type": "[]OffsetCommitResponseTopic", "versions": "0+", + "about": "The responses for each topic.", "fields": [ + { "name": "Name", "type": "string", "versions": "0+", + "about": "The topic name." }, + { "name": "Partitions", "type": "[]OffsetCommitResponsePartition", "versions": "0+", + "about": "The responses for each partition in the topic.", "fields": [ + { "name": "PartitionIndex", "type": "int32", "versions": "0+", + "about": "The partition index." }, + { "name": "ErrorCode", "type": "int16", "versions": "0+", + "about": "The error code, or 0 if there was no error." } + ]} + ]} + ] +} diff --git a/clients/src/main/resources/message/OffsetFetchRequest.json b/clients/src/main/resources/message/OffsetFetchRequest.json new file mode 100644 index 0000000000000..e634f7c1a279b --- /dev/null +++ b/clients/src/main/resources/message/OffsetFetchRequest.json @@ -0,0 +1,37 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 9, + "type": "request", + "name": "OffsetFetchRequest", + // Starting in version 1, the broker supports fetching offsets from the internal __consumer_offsets topic. + // + // Starting in version 2, the request can contain a null topics array to indicate that offsets + // for all topics should be fetched. + // + // Version 3, 4, and 5 are the same as version 2. + "validVersions": "0-5", + "fields": [ + { "name": "GroupId", "type": "string", "versions": "0+", + "about": "The group to fetch offsets for." }, + { "name": "Topics", "type": "[]OffsetFetchRequestTopic", "versions": "0+", "nullableVersions": "2+", + "about": "Each topic we would like to fetch offsets for, or null to fetch offsets for all topics.", "fields": [ + { "name": "Name", "type": "string", "versions": "0+" }, + { "name": "PartitionIndexes", "type": "[]int32", "versions": "0+", + "about": "The partition indexes we would like to fetch offsets for." } + ]} + ] +} diff --git a/clients/src/main/resources/message/OffsetFetchResponse.json b/clients/src/main/resources/message/OffsetFetchResponse.json new file mode 100644 index 0000000000000..70fd27712791f --- /dev/null +++ b/clients/src/main/resources/message/OffsetFetchResponse.json @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 9, + "type": "response", + "name": "OffsetFetchResponse", + // Version 1 is the same as version 0. + // + // Version 2 adds a top-level error code. + // + // Version 3 adds the throttle time. + // + // Starting in version 4, on quota violation, brokers send out responses before throttling. + // + // Version 5 adds the leader epoch to the committed offset. + "validVersions": "0-5", + "fields": [ + { "name": "ThrottleTimeMs", "type": "int32", "versions": "3+", "ignorable": true, + "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." }, + { "name": "Topics", "type": "[]OffsetFetchResponseTopic", "versions": "0+", + "about": "The responses per topic.", "fields": [ + { "name": "Name", "type": "string", "versions": "0+", + "about": "The topic name." }, + { "name": "Partitions", "type": "[]OffsetFetchResponsePartition", "versions": "0+", + "about": "The responses per partition", "fields": [ + { "name": "PartitionIndex", "type": "int32", "versions": "0+", + "about": "The partition index." }, + { "name": "CommittedOffset", "type": "int64", "versions": "0+", + "about": "The committed message offset." }, + { "name": "CommittedLeaderEpoch", "type": "int32", "versions": "5+", + "about": "The leader epoch." }, + { "name": "Metadata", "type": "string", "versions": "0+", "nullableVersions": "0+", + "about": "The partition metadata." }, + { "name": "ErrorCode", "type": "int16", "versions": "0+", + "about": "The error code, or 0 if there was no error." } + ]} + ]}, + { "name": "ErrorCode", "type": "int16", "versions": "2+", "default": "0", "ignorable": false, + "about": "The top-level error code, or 0 if there was no error." } + ] +} diff --git a/clients/src/main/resources/message/OffsetForLeaderEpochRequest.json b/clients/src/main/resources/message/OffsetForLeaderEpochRequest.json new file mode 100644 index 0000000000000..40227edae5a66 --- /dev/null +++ b/clients/src/main/resources/message/OffsetForLeaderEpochRequest.json @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 23, + "type": "request", + "name": "OffsetForLeaderEpochRequest", + // Version 1 is the same as version 0. + // + // Version 2 adds the current leader epoch to support fencing. + "validVersions": "0-2", + "fields": [ + { "name": "Topics", "type": "[]OffsetForLeaderTopic", "versions": "0+", + "about": "Each topic to get offsets for.", "fields": [ + { "name": "Name", "type": "string", "versions": "0+", + "about": "The topic name." }, + { "name": "Partitions", "type": "[]OffsetForLeaderPartition", "versions": "0+", + "about": "Each partition to get offsets for.", "fields": [ + { "name": "PartitionIndex", "type": "int32", "versions": "0+", + "about": "The partition index." }, + { "name": "CurrentLeaderEpoch", "type": "int32", "versions": "2+", "default": "-1", "ignorable": true, + "about": "An epoch used to fence consumers/replicas with old metadata. If the epoch provided by the client is larger than the current epoch known to the broker, then the UNKNOWN_LEADER_EPOCH error code will be returned. If the provided epoch is smaller, then the FENCED_LEADER_EPOCH error code will be returned." }, + { "name": "LeaderEpoch", "type": "int32", "versions": "0+", + "about": "The epoch to look up an offset for." } + ]} + ]} + ] +} diff --git a/clients/src/main/resources/message/OffsetForLeaderEpochResponse.json b/clients/src/main/resources/message/OffsetForLeaderEpochResponse.json new file mode 100644 index 0000000000000..26bd490fdaf91 --- /dev/null +++ b/clients/src/main/resources/message/OffsetForLeaderEpochResponse.json @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 23, + "type": "response", + "name": "OffsetForLeaderEpochResponse", + // Version 1 added the leader epoch to the response. + // Version 2 added the throttle time. + "validVersions": "0-2", + "fields": [ + { "name": "ThrottleTimeMs", "type": "int32", "versions": "2+", "ignorable": true, + "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." }, + { "name": "Topics", "type": "[]OffsetForLeaderTopicResult", "versions": "0+", + "about": "Each topic we fetched offsets for.", "fields": [ + { "name": "Name", "type": "string", "versions": "0+", + "about": "The topic name." }, + { "name": "Partitions", "type": "[]OffsetForLeaderPartitionResult", "versions": "0+", + "about": "Each partition in the topic we fetched offsets for.", "fields": [ + { "name": "ErrorCode", "type": "int16", "versions": "0+", + "about": "The error code 0, or if there was no error." }, + { "name": "PartitionIndex", "type": "int32", "versions": "0+", + "about": "The partition index." }, + { "name": "LeaderEpoch", "type": "int32", "versions": "1+", "default": "-1", "ignorable": true, + "about": "The leader epoch of the partition." }, + { "name": "EndOffset", "type": "int64", "versions": "0+", + "about": "The end offset of the epoch." } + ]} + ]} + ] +} diff --git a/clients/src/main/resources/message/ProduceRequest.json b/clients/src/main/resources/message/ProduceRequest.json new file mode 100644 index 0000000000000..4f35db2e712ca --- /dev/null +++ b/clients/src/main/resources/message/ProduceRequest.json @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 0, + "type": "request", + "name": "ProduceRequest", + // Version 1 and 2 are the same as version 0. + // + // Version 3 adds the transactional ID, which is used for authorization when attempting to write + // transactional data. Version 3 also adds support for Kafka Message Format v2. + // + // Version 4 is the same as version 3, but the requestor must be prepared to handle a + // KAFKA_STORAGE_ERROR. + // + // Version 5 and 6 are the same as version 3. + // + // Starting in version 7, records can be produced using ZStandard compression. See KIP-110. + "validVersions": "0-7", + "fields": [ + { "name": "TransactionalId", "type": "string", "versions": "3+", "nullableVersions": "0+", + "about": "The transactional ID, or null if the producer is not transactional." }, + { "name": "Acks", "type": "int16", "versions": "0+", + "about": "The number of acknowledgments the producer requires the leader to have received before considering a request complete. Allowed values: 0 for no acknowledgments, 1 for only the leader and -1 for the full ISR." }, + { "name": "TimeoutMs", "type": "int32", "versions": "0+", + "about": "The timeout to await a response in miliseconds." }, + { "name": "Topics", "type": "[]TopicProduceData", "versions": "0+", + "about": "Each topic to produce to.", "fields": [ + { "name": "Name", "type": "string", "versions": "0+", + "about": "The topic name." }, + { "name": "Partitions", "type": "[]PartitionProduceData", "versions": "0+", + "about": "Each partition to produce to.", "fields": [ + { "name": "PartitionIndex", "type": "int32", "versions": "0+", + "about": "The partition index." }, + { "name": "Records", "type": "bytes", "versions": "0+", "nullableVersions": "0+", + "about": "The record data to be produced." } + ]} + ]} + ] +} diff --git a/clients/src/main/resources/message/ProduceResponse.json b/clients/src/main/resources/message/ProduceResponse.json new file mode 100644 index 0000000000000..14d38aa6917db --- /dev/null +++ b/clients/src/main/resources/message/ProduceResponse.json @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 0, + "type": "response", + "name": "ProduceResponse", + // Version 1 added the throttle time. + // + // Version 2 added the log append time. + // + // Version 3 is the same as version 2. + // + // Version 4 added KAFKA_STORAGE_ERROR as a possible error code. + // + // Version 5 added LogStartOffset to filter out spurious + // OutOfOrderSequenceExceptions on the client. + "validVersions": "0-7", + "fields": [ + { "name": "Responses", "type": "[]TopicProduceResponse", "versions": "0+", + "about": "Each produce response", "fields": [ + { "name": "Name", "type": "string", "versions": "0+", + "about": "The topic name" }, + { "name": "Partitions", "type": "[]PartitionProduceResponse", "versions": "0+", + "about": "Each partition that we produced to within the topic.", "fields": [ + { "name": "PartitionIndex", "type": "int32", "versions": "0+", + "about": "The partition index." }, + { "name": "ErrorCode", "type": "int16", "versions": "0+", + "about": "The error code, or 0 if there was no error." }, + { "name": "BaseOffset", "type": "int64", "versions": "0+", + "about": "The base offset." }, + { "name": "LogAppendTimeMs", "type": "int64", "versions": "2+", "default": "-1", "ignorable": true, + "about": "The timestamp returned by broker after appending the messages. If CreateTime is used for the topic, the timestamp will be -1. If LogAppendTime is used for the topic, the timestamp will be the broker local time when the messages are appended." }, + { "name": "LogStartOffset", "type": "int64", "versions": "5+", "default": "-1", "ignorable": true, + "about": "The log start offset." } + ]} + ]}, + { "name": "ThrottleTimeMs", "type": "int32", "versions": "1+", "ignorable": true, + "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." } + ] +} diff --git a/clients/src/main/resources/message/README.md b/clients/src/main/resources/message/README.md new file mode 100644 index 0000000000000..5648f37812d7a --- /dev/null +++ b/clients/src/main/resources/message/README.md @@ -0,0 +1,219 @@ +Apache Kafka Message Definitions +================================ + +Introduction +------------ +The JSON files in this directory define the Apache Kafka message protocol. +This protocol describes what information clients and servers send to each +other, and how it is serialized. Note that this version of JSON supports +comments. Comments begin with a double forward slash. + +When Kafka is compiled, these specification files are translated into Java code +to read and write messages. Any change to these JSON files will trigger a +recompilation of this generated code. + +These specification files replace an older system where hand-written +serialization code was used. Over time, we will migrate all messages to using +automatically generated serialization and deserialization code. + +Requests and Responses +---------------------- +The Kafka protocol features requests and responses. Requests are sent to a +server in order to get a response. Each request is uniquely identified by a +16-bit integer called the "api key". The API key of the response will always +match that of the request. + +Each message has a unique 16-bit version number. The schema might be different +for each version of the message. Sometimes, the version is incremented even +though the schema has not changed. This may indicate that the server should +behave differently in some way. The version of a response must always match +the version of the corresponding request. + +Each request or response has a top-level field named "validVersions." This +specifies the versions of the protocol that our code understands. For example, +specifying "0-2" indicates that we understand versions 0, 1, and 2. You must +always specify the highest message version which is supported. + +The only old message versions that are no longer supported are version 0 of +MetadataRequest and MetadataResponse. In general, since we adopted KIP-97, +dropping support for old message versions is no longer allowed without a KIP. +Therefore, please be careful not to increase the lower end of the version +support interval for any message. + +MessageData Objects +------------------- +Using the JSON files in this directory, we generate Java code for MessageData +objects. These objects store request and response data for kafka. MessageData +objects do not contain a version number. Instead, a single MessageData object +represents every possible version of a Message. This makes working with +messages more convenient, because the same code path can be used for every +version of a message. + +Fields +------ +Each message contains an array of fields. Fields specify the data that should +be sent with the message. In general, fields have a name, a type, and version +information associated with them. + +The order that fields appear in a message is important. Fields which come +first in the message definition will be sent first over the network. Changing +the order of the fields in a message is an incompatible change. + +In each new message version, we may add or subtract fields. For example, if we +are creating a new version 3 of a message, we can add a new field with the +version spec "3+". This specifies that the field only appears in version 3 and +later. If a field is being removed, we should change its version from "0+" to +"0-2" to indicate that it will not appear in version 3 and later. + +Field Types +----------- +There are several primitive field types available. + +* "boolean": either true or false. This takes up 1 byte on the wire. + +* "int8": an 8-bit integer. This also takes up 1 byte on the wire. + +* "int16": a 16-bit integer. This takes up 2 bytes on the wire. + +* "int32": a 32-bit integer. This takes up 4 bytes on the wire. + +* "int64": a 64-bit integer. This takes up 8 bytes on the wire. + +* "string": a string. This must be less than 64kb in size when serialized as UTF-8. This takes up 2 bytes on the wire, plus the length of the string when serialized to UTF-8. + +* "bytes": binary data. This takes up 4 bytes on the wire, plus the length of the bytes. + +In addition to these primitive field types, there is also an array type. Array +types start with a "[]" and end with the name of the element type. For +example, []Foo declares an array of "Foo" objects. Array fields have their own +array of fields, which specifies what is in the contained objects. + +Nullable Fields +--------------- +Booleans and ints can never be null. However, fields that are strings, bytes, +or arrays may optionally be "nullable." When a field is "nullable," that +simply means that we are prepared to serialize and deserialize null entries for +that field. + +If you want to declare a field as nullable, you set "nullableVersions" for that +field. Nullability is implemented as a version range in order to accomodate a +very common pattern in Kafka where a field that was originally not nullable +becomes nullable in a later version. + +If a field is declared as non-nullable, and it is present in the message +version you are using, you should set it to a non-null value before serializing +the message. Otherwise, you will get a runtime error. + +Serializing Messages +-------------------- +The Message#write method writes out a message to a buffer. The fields that are +written out will depend on the version number that you supply to write(). When +you write out a message using an older version, fields that are too old to be +present in the schema will be omitted. + +When working with older message versions, please verify that the older message +schema includes all the data that needs to be sent. For example, it is probably +OK to skip sending a timeout field. However, a field which radically alters the +meaning of the request, such as a "validateOnly" boolean, should not be ignored. + +It's often useful to know how much space a message will take up before writing +it out to a buffer. You can find this out by calling the Message#size method. + +You can also convert a message to a Struct by calling Message#toStruct. This +allows you to use the functions that serialize Structs to buffers. + +Deserializing Messages +---------------------- +Message objects may be deserialized using the Message#read method. This method +overwrites all the data currently in the message object with new data. + +You can also deserialize a message from a Struct by calling Message#fromStruct. +The Struct will not be modified. + +Any fields in the message object that are not present in the version that you +are deserializing will be reset to default values. Unless a custom default has +been set: + +* Integer fields default to 0. + +* Booleans default to false. + +* Strings default to the empty string. + +* Bytes fields default to the empty byte array. + +* Array fields default to empty. + +Null is never used as a default for any field. + +Custom Default Values +--------------------- +You may set a custom default for fields that are integers, booleans, or strings. +Just add a "default" entry in the JSON object. The custom default overrides the +normal default for the type. So for example, you could make a boolean field +default to true rather than false, and so forth. + +Note that the default must be valid for the field type. So the default for an +int16 field must by an integer that fits in 16 bits, and so forth. You may +specify hex or octal values, as long as they are prefixed with 0x or 0. It is +currently not possible to specify a custom default for bytes or array fields. + +Custom defaults are useful when an older message version lacked some +information. For example, if an older request lacked a timeout field, you may +want to specify that the server should assume that the timeout for such a +request is 5000 ms (or some other arbitrary value.) + +Ignorable Fields +---------------- +When we write messages using an older or newer format, not all fields may be +present. The message receiver will fill in the default value for the field +during deserialization. Therefore, if the source field was set to a non-default +value, that information will be lost. + +In some cases, this information loss is acceptable. For example, if a timeout +field does not get preserved, this is not a problem. However, in other cases, +the field is really quite important and should not be discarded. One example is +a "verify only" boolean which changes the whole meaning of the request. + +By default, we assume that information loss is not acceptable. The message +serialization code will throw an exception if the ignored field is not set to +the default value. If information loss for a field is OK, please set +"ignorable" to true for the field to disable this behavior. When ignorable is +set to true, the field may be silently omitted during serialization. + +Hash Sets +--------- +One very common pattern in Kafka is to load array elements from a message into +a Map or Set for easier access. The message protocol makes this easier with +the "mapKey" concept. + +If some of the elemements of an array are annotated with "mapKey": true, the +entire array will be treated as a linked hash set rather than a list. Elements +in this set will be accessible in O(1) time with an automatically generated +"find" function. The order of elements in the set will still be preserved, +however. New entries that are added to the set always show up as last in the +ordering. + +Incompatible Changes +-------------------- +It's very important to avoid making incompatible changes to the message +protocol. Here are some examples of incompatible changes: + +#### Making changes to a protocol version which has already been released. +Protocol versions that have been released must be regarded as done. If there +were mistakes, they should be corrected in a new version rather than changing +the existing version. + +#### Re-ordering existing fields. +It is OK to add new fields before or after existing fields. However, existing +fields should not be re-ordered with respect to each other. + +#### Changing the default of an existing field. +You must never change the default of a field which already exists. Otherwise, +new clients and old servers will not agree on the default, and so forth. + +#### Changing the type of an existing field. +One exception is that an array of primitives may be changed to an array of +structures containing the same data, as long as the conversion is done +correctly. The Kafka protocol does not do any "boxing" of structures, so an +array of structs that contain a single int32 is the same as an array of int32s. diff --git a/clients/src/main/resources/message/RenewDelegationTokenRequest.json b/clients/src/main/resources/message/RenewDelegationTokenRequest.json new file mode 100644 index 0000000000000..ba1db7ede5eb8 --- /dev/null +++ b/clients/src/main/resources/message/RenewDelegationTokenRequest.json @@ -0,0 +1,28 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 39, + "type": "request", + "name": "RenewDelegationTokenRequest", + // Version 1 is the same as version 0. + "validVersions": "0-1", + "fields": [ + { "name": "Hmac", "type": "bytes", "versions": "0+", + "about": "The HMAC of the delegation token to be renewed." }, + { "name": "RenewPeriodMs", "type": "int64", "versions": "0+", + "about": "The renewal time period in milliseconds." } + ] +} diff --git a/clients/src/main/resources/message/RenewDelegationTokenResponse.json b/clients/src/main/resources/message/RenewDelegationTokenResponse.json new file mode 100644 index 0000000000000..aed734a879645 --- /dev/null +++ b/clients/src/main/resources/message/RenewDelegationTokenResponse.json @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 39, + "type": "response", + "name": "RenewDelegationTokenResponse", + // Starting in version 1, on quota violation, brokers send out responses before throttling. + "validVersions": "0-1", + "fields": [ + { "name": "ErrorCode", "type": "int16", "versions": "0+", + "about": "The error code, or 0 if there was no error." }, + { "name": "ExpiryTimestampMs", "type": "int64", "versions": "0+", + "about": "The timestamp in milliseconds at which this token expires." }, + { "name": "ThrottleTimeMs", "type": "int32", "versions": "0+", + "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." } + ] +} diff --git a/clients/src/main/resources/message/RequestHeader.json b/clients/src/main/resources/message/RequestHeader.json new file mode 100644 index 0000000000000..d24dcf0e98755 --- /dev/null +++ b/clients/src/main/resources/message/RequestHeader.json @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "type": "header", + "name": "RequestHeader", + "validVersions": "0", + "fields": [ + { "name": "RequestApiKey", "type": "int16", "versions": "0+", + "about": "The API key of this request." }, + { "name": "RequestApiVersion", "type": "int16", "versions": "0+", + "about": "The API version of this request." }, + { "name": "CorrelationId", "type": "int32", "versions": "0+", + "about": "The correlation ID of this request." }, + { "name": "ClientId", "type": "string", "versions": "0+", + "about": "The client ID string." } + ] +} diff --git a/clients/src/main/resources/message/ResponseHeader.json b/clients/src/main/resources/message/ResponseHeader.json new file mode 100644 index 0000000000000..d44825977da7d --- /dev/null +++ b/clients/src/main/resources/message/ResponseHeader.json @@ -0,0 +1,24 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "type": "header", + "name": "ResponseHeader", + "validVersions": "0", + "fields": [ + { "name": "CorrelationId", "type": "int32", "versions": "0+", + "about": "The correlation ID of this response." } + ] +} diff --git a/clients/src/main/resources/message/SaslAuthenticateRequest.json b/clients/src/main/resources/message/SaslAuthenticateRequest.json new file mode 100644 index 0000000000000..96f6f3aa4efb4 --- /dev/null +++ b/clients/src/main/resources/message/SaslAuthenticateRequest.json @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 36, + "type": "request", + "name": "SaslAuthenticateRequest", + // Version 1 is the same as version 0. + "validVersions": "0-1", + "fields": [ + { "name": "AuthBytes", "type": "bytes", "versions": "0+", + "about": "The SASL authentication bytes from the client, as defined by the SASL mechanism." } + ] +} diff --git a/clients/src/main/resources/message/SaslAuthenticateResponse.json b/clients/src/main/resources/message/SaslAuthenticateResponse.json new file mode 100644 index 0000000000000..f6644f6d3459d --- /dev/null +++ b/clients/src/main/resources/message/SaslAuthenticateResponse.json @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 36, + "type": "response", + "name": "SaslAuthenticateResponse", + // Version 1 adds the session lifetime. + "validVersions": "0-1", + "fields": [ + { "name": "ErrorCode", "type": "int16", "versions": "0+", + "about": "The error code, or 0 if there was no error." }, + { "name": "ErrorMessage", "type": "string", "versions": "0+", "nullableVersions": "0+", + "about": "The error message, or null if there was no error." }, + { "name": "AuthBytes", "type": "bytes", "versions": "0+", + "about": "The SASL authentication bytes from the server, as defined by the SASL mechanism." }, + { "name": "SessionLifetimeMs", "type": "int64", "versions": "1+", "default": "0", + "about": "The SASL authentication bytes from the server, as defined by the SASL mechanism." } + ] +} diff --git a/clients/src/main/resources/message/SaslHandshakeRequest.json b/clients/src/main/resources/message/SaslHandshakeRequest.json new file mode 100644 index 0000000000000..7ad0a4716ef74 --- /dev/null +++ b/clients/src/main/resources/message/SaslHandshakeRequest.json @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 17, + "type": "request", + "name": "SaslHandshakeRequest", + // Version 1 supports SASL_AUTHENTICATE. + "validVersions": "0-1", + "fields": [ + { "name": "Mechanism", "type": "string", "versions": "0+", + "about": "The SASL mechanism chosen by the client." } + ] +} diff --git a/clients/src/main/resources/message/SaslHandshakeResponse.json b/clients/src/main/resources/message/SaslHandshakeResponse.json new file mode 100644 index 0000000000000..821f3290833cf --- /dev/null +++ b/clients/src/main/resources/message/SaslHandshakeResponse.json @@ -0,0 +1,28 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 17, + "type": "response", + "name": "SaslHandshakeResponse", + // Version 1 is the same as version 0. + "validVersions": "0-1", + "fields": [ + { "name": "ErrorCode", "type": "int16", "versions": "0+", + "about": "The error code, or 0 if there was no error." }, + { "name": "Mechanisms", "type": "[]string", "versions": "0+", + "about": "The mechanisms enabled in the server." } + ] +} diff --git a/clients/src/main/resources/message/StopReplicaRequest.json b/clients/src/main/resources/message/StopReplicaRequest.json new file mode 100644 index 0000000000000..dffa11da9ee4c --- /dev/null +++ b/clients/src/main/resources/message/StopReplicaRequest.json @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 5, + "type": "request", + "name": "StopReplicaRequest", + // Version 1 adds the broker epoch and reorganizes the partitions to be stored + // per topic. + "validVersions": "0-1", + "fields": [ + { "name": "ControllerId", "type": "int32", "versions": "0+", + "about": "The controller id." }, + { "name": "ControllerEpoch", "type": "int32", "versions": "0+", + "about": "The controller epoch." }, + { "name": "BrokerEpoch", "type": "int64", "versions": "1+", "default": "-1", "ignorable": true, + "about": "The broker epoch." }, + { "name": "DeletePartitions", "type": "bool", "versions": "0+", + "about": "Whether these partitions should be deleted." }, + { "name": "PartitionsV0", "type": "[]StopReplicaRequestPartitionV0", "versions": "0", + "about": "The partitions to stop.", "fields": [ + { "name": "TopicName", "type": "string", "versions": "0", + "about": "The topic name." }, + { "name": "PartitionIndex", "type": "int32", "versions": "0", + "about": "The partition index." } + ]}, + { "name": "Topics", "type": "[]StopReplicaRequestTopic", "versions": "1+", + "about": "The topics to stop.", "fields": [ + { "name": "Name", "type": "string", "versions": "1+", + "about": "The topic name." }, + { "name": "PartitionIndexes", "type": "[]int32", "versions": "1+", + "about": "The partition indexes." } + ]} + ] +} diff --git a/clients/src/main/resources/message/StopReplicaResponse.json b/clients/src/main/resources/message/StopReplicaResponse.json new file mode 100644 index 0000000000000..55daac5e9c391 --- /dev/null +++ b/clients/src/main/resources/message/StopReplicaResponse.json @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 5, + "type": "response", + "name": "StopReplicaResponse", + // Version 1 is the same as version 0. + "validVersions": "0-1", + "fields": [ + { "name": "ErrorCode", "type": "int16", "versions": "0+", + "about": "The top-level error code, or 0 if there was no top-level error." }, + { "name": "Partitions", "type": "[]StopReplicaResponsePartition", "versions": "0+", + "about": "The responses for each partition.", "fields": [ + { "name": "TopicName", "type": "string", "versions": "0+", + "about": "The topic name." }, + { "name": "PartitionIndex", "type": "int32", "versions": "0+", + "about": "The partition index." }, + { "name": "ErrorCode", "type": "int16", "versions": "0+", + "about": "The partition error code, or 0 if there was no partition error." } + ]} + ] +} diff --git a/clients/src/main/resources/message/SyncGroupRequest.json b/clients/src/main/resources/message/SyncGroupRequest.json new file mode 100644 index 0000000000000..ec910a03e60a7 --- /dev/null +++ b/clients/src/main/resources/message/SyncGroupRequest.json @@ -0,0 +1,37 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 14, + "type": "request", + "name": "SyncGroupRequest", + // Versions 1 and 2 are the same as version 0. + "validVersions": "0-2", + "fields": [ + { "name": "GroupId", "type": "string", "versions": "0+", + "about": "The unique group identifier." }, + { "name": "GenerationId", "type": "int32", "versions": "0+", + "about": "The generation of the group." }, + { "name": "MemberId", "type": "string", "versions": "0+", + "about": "The member ID assigned by the group." }, + { "name": "Assignments", "type": "[]SyncGroupRequestAssignment", "versions": "0+", + "about": "Each assignment.", "fields": [ + { "name": "MemberId", "type": "string", "versions": "0+", + "about": "The ID of the member to assign." }, + { "name": "Assignment", "type": "bytes", "versions": "0+", + "about": "The member assignment." } + ]} + ] +} diff --git a/clients/src/main/resources/message/SyncGroupResponse.json b/clients/src/main/resources/message/SyncGroupResponse.json new file mode 100644 index 0000000000000..0faa158223bd3 --- /dev/null +++ b/clients/src/main/resources/message/SyncGroupResponse.json @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 14, + "type": "response", + "name": "SyncGroupResponse", + // Version 1 adds throttle time. + // Starting in version 2, on quota violation, brokers send out responses before throttling. + "validVersions": "0-2", + "fields": [ + { "name": "ThrottleTimeMs", "type": "int32", "versions": "1+", "ignorable": true, + "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." }, + { "name": "ErrorCode", "type": "int16", "versions": "0+", + "about": "The error code, or 0 if there was no error." }, + { "name": "Assignment", "type": "bytes", "versions": "0+", + "about": "The member assignment." } + ] +} diff --git a/clients/src/main/resources/message/TxnOffsetCommitRequest.json b/clients/src/main/resources/message/TxnOffsetCommitRequest.json new file mode 100644 index 0000000000000..357d1d0ad5c52 --- /dev/null +++ b/clients/src/main/resources/message/TxnOffsetCommitRequest.json @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 28, + "type": "request", + "name": "TxnOffsetCommitRequest", + // Version 1 is the same as version 0. + // + // Version 2 adds the committed leader epoch. + "validVersions": "0-2", + "fields": [ + { "name": "TransactionalId", "type": "string", "versions": "0+", + "about": "The ID of the transaction." }, + { "name": "GroupId", "type": "string", "versions": "0+", + "about": "The ID of the group." }, + { "name": "ProducerId", "type": "int64", "versions": "0+", + "about": "The current producer ID in use by the transactional ID." }, + { "name": "ProducerEpoch", "type": "int16", "versions": "0+", + "about": "The current epoch associated with the producer ID." }, + { "name": "Topics", "type" : "[]TxnOffsetCommitRequestTopic", "versions": "0+", + "about": "Each topic that we want to committ offsets for.", "fields": [ + { "name": "Name", "type": "string", "versions": "0+", + "about": "The topic name." }, + { "name": "Partitions", "type": "[]TxnOffsetCommitRequestPartition", "versions": "0+", + "about": "The partitions inside the topic that we want to committ offsets for.", "fields": [ + { "name": "PartitionIndex", "type": "int32", "versions": "0+", + "about": "The index of the partition within the topic." }, + { "name": "CommittedOffset", "type": "int64", "versions": "0+", + "about": "The message offset to be committed." }, + { "name": "CommittedLeaderEpoch", "type": "int32", "versions": "2+", "default": "-1", "ignorable": true, + "about": "The leader epoch of the last consumed record." }, + { "name": "CommittedMetadata", "type": "string", "versions": "0+", "nullableVersions": "0+", + "about": "Any associated metadata the client wants to keep." } + ]} + ]} + ] +} diff --git a/clients/src/main/resources/message/TxnOffsetCommitResponse.json b/clients/src/main/resources/message/TxnOffsetCommitResponse.json new file mode 100644 index 0000000000000..58667cb503661 --- /dev/null +++ b/clients/src/main/resources/message/TxnOffsetCommitResponse.json @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 28, + "type": "response", + "name": "TxnOffsetCommitResponse", + // Starting in version 1, on quota violation, brokers send out responses before throttling. + // Version 2 is the same as version 1. + "validVersions": "0-2", + "fields": [ + { "name": "ThrottleTimeMs", "type": "int32", "versions": "0+", + "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." }, + { "name": "Topics", "type": "[]TxnOffsetCommitResponseTopic", "versions": "0+", + "about": "The responses for each topic.", "fields": [ + { "name": "Name", "type": "string", "versions": "0+", + "about": "The topic name." }, + { "name": "Partitions", "type": "[]TxnOffsetCommitResponsePartition", "versions": "0+", + "about": "The responses for each partition in the topic.", "fields": [ + { "name": "PartitionIndex", "type": "int32", "versions": "0+", + "about": "The partitition index." }, + { "name": "ErrorCode", "type": "int16", "versions": "0+", + "about": "The error code, or 0 if there was no error." } + ]} + ]} + ] +} diff --git a/clients/src/main/resources/message/UpdateMetadataRequest.json b/clients/src/main/resources/message/UpdateMetadataRequest.json new file mode 100644 index 0000000000000..07e0f0346eec8 --- /dev/null +++ b/clients/src/main/resources/message/UpdateMetadataRequest.json @@ -0,0 +1,105 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 6, + "type": "request", + "name": "UpdateMetadataRequest", + // Version 1 allows specifying multiple endpoints for each broker. + // + // Version 2 adds the rack. + // + // Version 3 adds the listener name. + // + // Version 4 adds the offline replica list. + // + // Version 5 adds the broker epoch field and normalizes partitions by topic. + "validVersions": "0-5", + "fields": [ + { "name": "ControllerId", "type": "int32", "versions": "0+", + "about": "The controller id." }, + { "name": "ControllerEpoch", "type": "int32", "versions": "0+", + "about": "The controller epoch." }, + { "name": "BrokerEpoch", "type": "int64", "versions": "5+", "ignorable": true, "default": "-1", + "about": "The broker epoch." }, + { "name": "TopicStates", "type": "[]UpdateMetadataRequestTopicState", "versions": "5+", + "about": "Each topic that we would like to update.", "fields": [ + { "name": "TopicName", "type": "string", "versions": "0+", + "about": "The topic name." }, + { "name": "PartitionStates", "type": "[]UpdateMetadataPartitionState", "versions": "5+", + "about": "The partition that we would like to update.", "fields": [ + { "name": "PartitionIndex", "type": "int32", "versions": "5+", + "about": "The partition index." }, + { "name": "ControllerEpoch", "type": "int32", "versions": "5+", + "about": "The controller epoch." }, + { "name": "Leader", "type": "int32", "versions": "5+", + "about": "The ID of the broker which is the current partition leader." }, + { "name": "LeaderEpoch", "type": "int32", "versions": "5+", + "about": "The leader epoch of this partition." }, + { "name": "Isr", "type": "[]int32", "versions": "5+", + "about": "The brokers which are in the ISR for this partition." }, + { "name": "ZkVersion", "type": "int32", "versions": "5+", + "about": "The Zookeeper version." }, + { "name": "Replicas", "type": "[]int32", "versions": "5+", + "about": "All the replicas of this partition." }, + { "name": "OfflineReplicas", "type": "[]int32", "versions": "5+", + "about": "The replicas of this partition which are offline." } + ]} + ]}, + { "name": "PartitionStatesV0", "type": "[]UpdateMetadataRequestPartitionStateV0", "versions": "0-4", + "about": "Each partition that we would like to update.", "fields": [ + { "name": "TopicName", "type": "string", "versions": "0-4", + "about": "The topic name." }, + { "name": "PartitionIndex", "type": "int32", "versions": "0-4", + "about": "The partition index." }, + { "name": "ControllerEpoch", "type": "int32", "versions": "0-4", + "about": "The controller epoch." }, + { "name": "Leader", "type": "int32", "versions": "0-4", + "about": "The ID of the broker which is the current partition leader." }, + { "name": "LeaderEpoch", "type": "int32", "versions": "0-4", + "about": "The leader epoch of this partition." }, + { "name": "Isr", "type": "[]int32", "versions": "0-4", + "about": "The brokers which are in the ISR for this partition." }, + { "name": "ZkVersion", "type": "int32", "versions": "0-4", + "about": "The Zookeeper version." }, + { "name": "Replicas", "type": "[]int32", "versions": "0-4", + "about": "All the replicas of this partition." }, + { "name": "OfflineReplicas", "type": "[]int32", "versions": "4", + "about": "The replicas of this partition which are offline." } + ]}, + { "name": "Brokers", "type": "[]UpdateMetadataRequestBroker", "versions": "0+", "fields": [ + { "name": "Id", "type": "int32", "versions": "0+" }, + // Version 0 of the protocol only allowed specifying a single host and + // port per broker, rather than an array of endpoints. + { "name": "V0Host", "type": "string", "versions": "0", "ignorable": true, + "about": "The broker hostname." }, + { "name": "V0Port", "type": "int32", "versions": "0", "ignorable": true, + "about": "The broker port." }, + { "name": "Endpoints", "type": "[]UpdateMetadataRequestEndpoint", "versions": "1+", + "about": "The broker endpoints.", "fields": [ + { "name": "Port", "type": "int32", "versions": "1+", + "about": "The port of this endpoint" }, + { "name": "Host", "type": "string", "versions": "1+", + "about": "The hostname of this endpoint" }, + { "name": "Listener", "type": "string", "versions": "3+", + "about": "The listener name." }, + { "name": "SecurityProtocol", "type": "int16", "versions": "1+", + "about": "The security protocol type." } + ]}, + { "name": "Rack", "type": "string", "versions": "2+", "nullableVersions": "0+", "ignorable": true, + "about": "The rack which this broker belongs to." } + ]} + ] +} diff --git a/clients/src/main/resources/message/UpdateMetadataResponse.json b/clients/src/main/resources/message/UpdateMetadataResponse.json new file mode 100644 index 0000000000000..5069a63d77345 --- /dev/null +++ b/clients/src/main/resources/message/UpdateMetadataResponse.json @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 6, + "type": "response", + "name": "UpdateMetadataResponse", + // Versions 1, 2, 3, 4, and 5 are the same as version 0 + "validVersions": "0-5", + "fields": [ + { "name": "ErrorCode", "type": "int16", "versions": "0+", + "about": "The error code, or 0 if there was no error." } + ] +} diff --git a/clients/src/main/resources/message/WriteTxnMarkersRequest.json b/clients/src/main/resources/message/WriteTxnMarkersRequest.json new file mode 100644 index 0000000000000..89868fc9710b4 --- /dev/null +++ b/clients/src/main/resources/message/WriteTxnMarkersRequest.json @@ -0,0 +1,41 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 27, + "type": "request", + "name": "WriteTxnMarkersRequest", + "validVersions": "0", + "fields": [ + { "name": "Markers", "type": "[]WritableTxnMarker", "versions": "0+", + "about": "The transaction markers to be written.", "fields": [ + { "name": "ProducerId", "type": "int64", "versions": "0+", + "about": "The current producer ID."}, + { "name": "ProducerEpoch", "type": "int16", "versions": "0+", + "about": "The current epoch associated with the producer ID." }, + { "name": "TransactionResult", "type": "bool", "versions": "0+", + "about": "The result of the transaction to write to the partitions (false = ABORT, true = COMMIT)." }, + { "name": "Topics", "type": "[]WritableTxnMarkerTopic", "versions": "0+", + "about": "Each topic that we want to write transaction marker(s) for.", "fields": [ + { "name": "Name", "type": "string", "versions": "0+", + "about": "The topic name." }, + { "name": "PartitionIndexes", "type": "[]int32", "versions": "0+", + "about": "The indexes of the partitions to write transaction markers for." } + ]}, + { "name": "CoordinatorEpoch", "type": "int32", "versions": "0+", + "about": "Epoch associated with the transaction state partition hosted by this transaction coordinator" } + ]} + ] +} diff --git a/clients/src/main/resources/message/WriteTxnMarkersResponse.json b/clients/src/main/resources/message/WriteTxnMarkersResponse.json new file mode 100644 index 0000000000000..ca080548f7594 --- /dev/null +++ b/clients/src/main/resources/message/WriteTxnMarkersResponse.json @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 27, + "type": "response", + "name": "WriteTxnMarkersResponse", + "validVersions": "0", + "fields": [ + { "name": "Markers", "type": "[]WritableTxnMarkerResult", "versions": "0+", + "about": "The results for writing makers.", "fields": [ + { "name": "ProducerId", "type": "int64", "versions": "0+", + "about": "The current producer ID in use by the transactional ID." }, + { "name": "Topics", "type": "[]WritableTxnMarkerTopicResult", "versions": "0+", + "about": "The results by topic.", "fields": [ + { "name": "Name", "type": "string", "versions": "0+", + "about": "The topic name." }, + { "name": "Partitions", "type": "[]WritableTxnMarkerPartitionResult", "versions": "0+", + "about": "The results by partition.", "fields": [ + { "name": "PartitionIndex", "type": "int32", "versions": "0+", + "about": "The partition index." }, + { "name": "ErrorCode", "type": "int16", "versions": "0+", + "about": "The error code, or 0 if there was no error." } + ]} + ]} + ]} + ] +} diff --git a/clients/src/test/java/org/apache/kafka/common/protocol/MessageUtilTest.java b/clients/src/test/java/org/apache/kafka/common/protocol/MessageUtilTest.java new file mode 100755 index 0000000000000..b327314539fe1 --- /dev/null +++ b/clients/src/test/java/org/apache/kafka/common/protocol/MessageUtilTest.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.common.protocol; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.Timeout; + +import java.nio.charset.StandardCharsets; +import java.util.Arrays; + +import static org.junit.Assert.assertEquals; + +public final class MessageUtilTest { + @Rule + final public Timeout globalTimeout = Timeout.millis(120000); + + @Test + public void testSimpleUtf8Lengths() { + validateUtf8Length(""); + validateUtf8Length("abc"); + validateUtf8Length("This is a test string."); + } + + @Test + public void testMultibyteUtf8Lengths() { + validateUtf8Length("A\u00ea\u00f1\u00fcC"); + validateUtf8Length("\ud801\udc00"); + validateUtf8Length("M\u00fcO"); + } + + private void validateUtf8Length(String string) { + byte[] arr = string.getBytes(StandardCharsets.UTF_8); + assertEquals(arr.length, MessageUtil.utf8Length(string)); + } + + @Test + public void testDeepToString() { + assertEquals("[1, 2, 3]", + MessageUtil.deepToString(Arrays.asList(1, 2, 3).iterator())); + assertEquals("[foo]", + MessageUtil.deepToString(Arrays.asList("foo").iterator())); + } +} diff --git a/clients/src/test/java/org/apache/kafka/common/utils/ImplicitLinkedHashMultiSetTest.java b/clients/src/test/java/org/apache/kafka/common/utils/ImplicitLinkedHashMultiSetTest.java new file mode 100644 index 0000000000000..950deb8bfb657 --- /dev/null +++ b/clients/src/test/java/org/apache/kafka/common/utils/ImplicitLinkedHashMultiSetTest.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.common.utils; + +import org.apache.kafka.common.utils.ImplicitLinkedHashSetTest.TestElement; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.Timeout; + +import java.util.Iterator; +import java.util.LinkedList; +import java.util.Random; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * A unit test for ImplicitLinkedHashMultiSet. + */ +public class ImplicitLinkedHashMultiSetTest { + @Rule + final public Timeout globalTimeout = Timeout.millis(120000); + + @Test + public void testNullForbidden() { + ImplicitLinkedHashMultiSet multiSet = new ImplicitLinkedHashMultiSet<>(); + assertFalse(multiSet.add(null)); + } + + @Test + public void testInsertDelete() { + ImplicitLinkedHashMultiSet multiSet = new ImplicitLinkedHashMultiSet<>(100); + TestElement e1 = new TestElement(1); + TestElement e2 = new TestElement(1); + TestElement e3 = new TestElement(2); + multiSet.mustAdd(e1); + multiSet.mustAdd(e2); + multiSet.mustAdd(e3); + assertFalse(multiSet.add(e3)); + assertEquals(3, multiSet.size()); + expectExactTraversal(multiSet.findAll(e1).iterator(), e1, e2); + expectExactTraversal(multiSet.findAll(e3).iterator(), e3); + multiSet.remove(e2); + expectExactTraversal(multiSet.findAll(e1).iterator(), e1); + assertTrue(multiSet.contains(e2)); + } + + @Test + public void testTraversal() { + ImplicitLinkedHashMultiSet multiSet = new ImplicitLinkedHashMultiSet<>(); + expectExactTraversal(multiSet.iterator()); + TestElement e1 = new TestElement(1); + TestElement e2 = new TestElement(1); + TestElement e3 = new TestElement(2); + assertTrue(multiSet.add(e1)); + assertTrue(multiSet.add(e2)); + assertTrue(multiSet.add(e3)); + expectExactTraversal(multiSet.iterator(), e1, e2, e3); + assertTrue(multiSet.remove(e2)); + expectExactTraversal(multiSet.iterator(), e1, e3); + assertTrue(multiSet.remove(e1)); + expectExactTraversal(multiSet.iterator(), e3); + } + + static void expectExactTraversal(Iterator iterator, TestElement... sequence) { + int i = 0; + while (iterator.hasNext()) { + TestElement element = iterator.next(); + assertTrue("Iterator yieled " + (i + 1) + " elements, but only " + + sequence.length + " were expected.", i < sequence.length); + if (sequence[i] != element) { + fail("Iterator value number " + (i + 1) + " was incorrect."); + } + i = i + 1; + } + assertTrue("Iterator yieled " + (i + 1) + " elements, but " + + sequence.length + " were expected.", i == sequence.length); + } + + @Test + public void testEnlargement() { + ImplicitLinkedHashMultiSet multiSet = new ImplicitLinkedHashMultiSet<>(5); + assertEquals(11, multiSet.numSlots()); + TestElement[] testElements = { + new TestElement(100), + new TestElement(101), + new TestElement(102), + new TestElement(100), + new TestElement(101), + new TestElement(105) + }; + for (int i = 0; i < testElements.length; i++) { + assertTrue(multiSet.add(testElements[i])); + } + for (int i = 0; i < testElements.length; i++) { + assertFalse(multiSet.add(testElements[i])); + } + assertEquals(23, multiSet.numSlots()); + assertEquals(testElements.length, multiSet.size()); + expectExactTraversal(multiSet.iterator(), testElements); + multiSet.remove(testElements[1]); + assertEquals(23, multiSet.numSlots()); + assertEquals(5, multiSet.size()); + expectExactTraversal(multiSet.iterator(), + testElements[0], testElements[2], testElements[3], testElements[4], testElements[5]); + } + + @Test + public void testManyInsertsAndDeletes() { + Random random = new Random(123); + LinkedList existing = new LinkedList<>(); + ImplicitLinkedHashMultiSet multiSet = new ImplicitLinkedHashMultiSet<>(); + for (int i = 0; i < 100; i++) { + for (int j = 0; j < 4; j++) { + TestElement testElement = new TestElement(random.nextInt()); + multiSet.mustAdd(testElement); + existing.add(testElement); + } + int elementToRemove = random.nextInt(multiSet.size()); + Iterator iter1 = multiSet.iterator(); + Iterator iter2 = existing.iterator(); + for (int j = 0; j <= elementToRemove; j++) { + iter1.next(); + iter2.next(); + } + iter1.remove(); + iter2.remove(); + expectTraversal(multiSet.iterator(), existing.iterator()); + } + } + + void expectTraversal(Iterator iter, Iterator expectedIter) { + int i = 0; + while (iter.hasNext()) { + TestElement element = iter.next(); + Assert.assertTrue("Iterator yieled " + (i + 1) + " elements, but only " + + i + " were expected.", expectedIter.hasNext()); + TestElement expected = expectedIter.next(); + assertTrue("Iterator value number " + (i + 1) + " was incorrect.", + expected == element); + i = i + 1; + } + Assert.assertFalse("Iterator yieled " + i + " elements, but at least " + + (i + 1) + " were expected.", expectedIter.hasNext()); + } +} diff --git a/clients/src/test/java/org/apache/kafka/common/utils/ImplicitLinkedHashSetTest.java b/clients/src/test/java/org/apache/kafka/common/utils/ImplicitLinkedHashSetTest.java index 249f6f8d4dbcf..156eba23c0e5d 100644 --- a/clients/src/test/java/org/apache/kafka/common/utils/ImplicitLinkedHashSetTest.java +++ b/clients/src/test/java/org/apache/kafka/common/utils/ImplicitLinkedHashSetTest.java @@ -22,6 +22,7 @@ import org.junit.rules.Timeout; import java.util.ArrayList; +import java.util.Collection; import java.util.Iterator; import java.util.LinkedHashSet; import java.util.List; @@ -38,7 +39,7 @@ public class ImplicitLinkedHashSetTest { @Rule final public Timeout globalTimeout = Timeout.millis(120000); - private final static class TestElement implements ImplicitLinkedHashSet.Element { + final static class TestElement implements ImplicitLinkedHashSet.Element { private int prev = ImplicitLinkedHashSet.INVALID_INDEX; private int next = ImplicitLinkedHashSet.INVALID_INDEX; private final int val; @@ -86,6 +87,12 @@ public int hashCode() { } } + @Test + public void testNullForbidden() { + ImplicitLinkedHashMultiSet multiSet = new ImplicitLinkedHashMultiSet<>(); + assertFalse(multiSet.add(null)); + } + @Test public void testInsertDelete() { ImplicitLinkedHashSet set = new ImplicitLinkedHashSet<>(100); @@ -106,7 +113,7 @@ public void testInsertDelete() { assertEquals(0, set.size()); } - private static void expectTraversal(Iterator iterator, Integer... sequence) { + static void expectTraversal(Iterator iterator, Integer... sequence) { int i = 0; while (iterator.hasNext()) { TestElement element = iterator.next(); @@ -120,8 +127,7 @@ private static void expectTraversal(Iterator iterator, Integer... s sequence.length + " were expected.", i == sequence.length); } - private static void expectTraversal(Iterator iter, - Iterator expectedIter) { + static void expectTraversal(Iterator iter, Iterator expectedIter) { int i = 0; while (iter.hasNext()) { TestElement element = iter.next(); @@ -138,7 +144,7 @@ private static void expectTraversal(Iterator iter, @Test public void testTraversal() { - ImplicitLinkedHashSet set = new ImplicitLinkedHashSet<>(100); + ImplicitLinkedHashSet set = new ImplicitLinkedHashSet<>(); expectTraversal(set.iterator()); assertTrue(set.add(new TestElement(2))); expectTraversal(set.iterator(), 2); @@ -226,8 +232,8 @@ private void addRandomElement(Random random, LinkedHashSet existing, set.add(new TestElement(next)); } - private void removeRandomElement(Random random, LinkedHashSet existing, - ImplicitLinkedHashSet set) { + private void removeRandomElement(Random random, Collection existing, + ImplicitLinkedHashSet set) { int removeIdx = random.nextInt(existing.size()); Iterator iter = existing.iterator(); Integer element = null; diff --git a/clients/src/test/java/org/apache/kafka/message/MessageGeneratorTest.java b/clients/src/test/java/org/apache/kafka/message/MessageGeneratorTest.java new file mode 100644 index 0000000000000..1fd7c15a58a6a --- /dev/null +++ b/clients/src/test/java/org/apache/kafka/message/MessageGeneratorTest.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.message; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.Timeout; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertEquals; + +public class MessageGeneratorTest { + @Rule + final public Timeout globalTimeout = Timeout.millis(120000); + + @Test + public void testCapitalizeFirst() throws Exception { + assertEquals("", MessageGenerator.capitalizeFirst("")); + assertEquals("AbC", MessageGenerator.capitalizeFirst("abC")); + } + + @Test + public void testLowerCaseFirst() throws Exception { + assertEquals("", MessageGenerator.lowerCaseFirst("")); + assertEquals("fORTRAN", MessageGenerator.lowerCaseFirst("FORTRAN")); + assertEquals("java", MessageGenerator.lowerCaseFirst("java")); + } + + @Test + public void testFirstIsCapitalized() throws Exception { + assertFalse(MessageGenerator.firstIsCapitalized("")); + assertTrue(MessageGenerator.firstIsCapitalized("FORTRAN")); + assertFalse(MessageGenerator.firstIsCapitalized("java")); + } + + @Test + public void testToSnakeCase() throws Exception { + assertEquals("", MessageGenerator.toSnakeCase("")); + assertEquals("foo_bar_baz", MessageGenerator.toSnakeCase("FooBarBaz")); + assertEquals("foo_bar_baz", MessageGenerator.toSnakeCase("fooBarBaz")); + assertEquals("fortran", MessageGenerator.toSnakeCase("FORTRAN")); + } +}; diff --git a/clients/src/test/java/org/apache/kafka/message/MessageTest.java b/clients/src/test/java/org/apache/kafka/message/MessageTest.java new file mode 100644 index 0000000000000..120da9962a556 --- /dev/null +++ b/clients/src/test/java/org/apache/kafka/message/MessageTest.java @@ -0,0 +1,308 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.message; + +import org.apache.kafka.common.errors.UnsupportedVersionException; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; +import org.apache.kafka.common.protocol.Message; +import org.apache.kafka.common.protocol.types.ArrayOf; +import org.apache.kafka.common.protocol.types.BoundField; +import org.apache.kafka.common.protocol.types.Schema; +import org.apache.kafka.common.protocol.types.Struct; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import org.apache.kafka.common.protocol.types.Type; +import org.apache.kafka.common.utils.Utils; +import org.apache.kafka.message.AddPartitionsToTxnRequestData.AddPartitionsToTxnTopic; +import org.apache.kafka.message.AddPartitionsToTxnRequestData.AddPartitionsToTxnTopicSet; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.Timeout; + +import static org.junit.Assert.assertEquals; + +public final class MessageTest { + @Rule + final public Timeout globalTimeout = Timeout.millis(120000); + + /** + * Test serializing and deserializing some messages. + */ + @Test + public void testRoundTrips() throws Exception { + testMessageRoundTrips(new MetadataRequestData().setTopics( + Arrays.asList(new MetadataRequestData.MetadataRequestTopic[] { + new MetadataRequestData.MetadataRequestTopic().setName("foo"), + new MetadataRequestData.MetadataRequestTopic().setName("bar") + })), (short) 6); + testMessageRoundTrips(new AddOffsetsToTxnRequestData(). + setTransactionalId("foobar"). + setProducerId(0xbadcafebadcafeL). + setProducerEpoch((short) 123). + setGroupId("baaz"), (short) 1); + testMessageRoundTrips(new AddOffsetsToTxnResponseData(). + setThrottleTimeMs(42). + setErrorCode((short) 0), (short) 0); + testMessageRoundTrips(new AddPartitionsToTxnRequestData(). + setTransactionalId("blah"). + setProducerId(0xbadcafebadcafeL). + setProducerEpoch((short) 30000). + setTopics(new AddPartitionsToTxnTopicSet(Collections.singletonList( + new AddPartitionsToTxnTopic(). + setName("Topic"). + setPartitions(Collections.singletonList(1))).iterator()))); + testMessageRoundTrips(new CreateTopicsRequestData(). + setTimeoutMs(1000).setTopics(Collections.emptyList())); + testMessageRoundTrips(new DescribeAclsRequestData(). + setResourceType((byte) 42). + setResourceNameFilter(null). + setResourcePatternType((byte) 3). + setPrincipalFilter("abc"). + setHostFilter(null). + setOperation((byte) 0). + setPermissionType((byte) 0), (short) 0); + } + + private void testMessageRoundTrips(Message message) throws Exception { + testMessageRoundTrips(message, message.highestSupportedVersion()); + } + + private void testMessageRoundTrips(Message message, short version) throws Exception { + testStructRoundTrip(message, version); + testByteBufferRoundTrip(message, version); + } + + private void testByteBufferRoundTrip(Message message, short version) throws Exception { + int size = message.size(version); + ByteBuffer buf = ByteBuffer.allocate(size); + ByteBufferAccessor byteBufferAccessor = new ByteBufferAccessor(buf); + message.write(byteBufferAccessor, version); + assertEquals(size, buf.position()); + Message message2 = message.getClass().newInstance(); + buf.flip(); + message2.read(byteBufferAccessor, version); + assertEquals(size, buf.position()); + assertEquals(message, message2); + assertEquals(message.hashCode(), message2.hashCode()); + assertEquals(message.toString(), message2.toString()); + } + + private void testStructRoundTrip(Message message, short version) throws Exception { + Struct struct = message.toStruct(version); + Message message2 = message.getClass().newInstance(); + message2.fromStruct(struct, version); + assertEquals(message, message2); + assertEquals(message.hashCode(), message2.hashCode()); + assertEquals(message.toString(), message2.toString()); + } + + /** + * Verify that the JSON files support the same message versions as the + * schemas accessible through the ApiKey class. + */ + @Test + public void testMessageVersions() throws Exception { + for (ApiKeys apiKey : ApiKeys.values()) { + Message message; + try { + message = MessageFactory.newRequest(apiKey.id); + } catch (UnsupportedVersionException e) { + throw new RuntimeException("No request message spec found for API " + apiKey); + } + if (apiKey.latestVersion() > message.highestSupportedVersion()) { + throw new RuntimeException("Request message spec for " + apiKey + " only " + + "supports versions up to " + message.highestSupportedVersion()); + } + try { + message = MessageFactory.newResponse(apiKey.id); + } catch (UnsupportedVersionException e) { + throw new RuntimeException("No response message spec found for API " + apiKey); + } + if (apiKey.latestVersion() > message.highestSupportedVersion()) { + throw new RuntimeException("Response message spec for " + apiKey + " only " + + "supports versions up to " + message.highestSupportedVersion()); + } + } + } + + /** + * Test that the JSON request files match the schemas accessible through the ApiKey class. + */ + @Test + public void testRequestSchemas() throws Exception { + for (ApiKeys apiKey : ApiKeys.values()) { + Schema[] manualSchemas = apiKey.requestSchemas; + Schema[] generatedSchemas = MessageFactory.requestSchemas(apiKey.id); + Assert.assertEquals("Mismatching request SCHEMAS lengths " + + "for api key " + apiKey, manualSchemas.length, generatedSchemas.length); + for (int v = 0; v < manualSchemas.length; v++) { + try { + if (generatedSchemas[v] != null) { + compareTypes(manualSchemas[v], generatedSchemas[v]); + } + } catch (Exception e) { + throw new RuntimeException("Failed to compare request schemas " + + "for version " + v + " of " + apiKey, e); + } + } + } + } + + /** + * Test that the JSON response files match the schemas accessible through the ApiKey class. + */ + @Test + public void testResponseSchemas() throws Exception { + for (ApiKeys apiKey : ApiKeys.values()) { + Schema[] manualSchemas = apiKey.responseSchemas; + Schema[] generatedSchemas = MessageFactory.responseSchemas(apiKey.id); + Assert.assertEquals("Mismatching response SCHEMAS lengths " + + "for api key " + apiKey, manualSchemas.length, generatedSchemas.length); + for (int v = 0; v < manualSchemas.length; v++) { + try { + if (generatedSchemas[v] != null) { + compareTypes(manualSchemas[v], generatedSchemas[v]); + } + } catch (Exception e) { + throw new RuntimeException("Failed to compare response schemas " + + "for version " + v + " of " + apiKey, e); + } + } + } + } + + private static class NamedType { + final String name; + final Type type; + + NamedType(String name, Type type) { + this.name = name; + this.type = type; + } + + boolean hasSimilarType(NamedType other) { + if (type.getClass().equals(other.type.getClass())) { + return true; + } + if (type.getClass().equals(Type.RECORDS.getClass())) { + if (other.type.getClass().equals(Type.NULLABLE_BYTES.getClass())) { + return true; + } + } else if (type.getClass().equals(Type.NULLABLE_BYTES.getClass())) { + if (other.type.getClass().equals(Type.RECORDS.getClass())) { + return true; + } + } + return false; + } + + @Override + public String toString() { + return name + "[" + type + "]"; + } + } + + private static void compareTypes(Schema schemaA, Schema schemaB) { + compareTypes(new NamedType("schemaA", schemaA), + new NamedType("schemaB", schemaB)); + } + + private static void compareTypes(NamedType typeA, NamedType typeB) { + List listA = flatten(typeA); + List listB = flatten(typeB); + if (listA.size() != listB.size()) { + throw new RuntimeException("Can't match up structures: typeA has " + + Utils.join(listA, ", ") + ", but typeB has " + + Utils.join(listB, ", ")); + } + for (int i = 0; i < listA.size(); i++) { + NamedType entryA = listA.get(i); + NamedType entryB = listB.get(i); + if (!entryA.hasSimilarType(entryB)) { + throw new RuntimeException("Type " + entryA + " in schema A " + + "does not match type " + entryB + " in schema B."); + } + if (entryA.type.isNullable() != entryB.type.isNullable()) { + throw new RuntimeException(String.format( + "Type %s in Schema A is %s, but type %s in " + + "Schema B is %s", + entryA, entryA.type.isNullable() ? "nullable" : "non-nullable", + entryB, entryB.type.isNullable() ? "nullable" : "non-nullable")); + } + if (entryA.type instanceof ArrayOf) { + compareTypes(new NamedType(entryA.name, ((ArrayOf) entryA.type).type()), + new NamedType(entryB.name, ((ArrayOf) entryB.type).type())); + } + } + } + + /** + * We want to remove Schema nodes from the hierarchy before doing + * our comparison. The reason is because Schema nodes don't change what + * is written to the wire. Schema(STRING, Schema(INT, SHORT)) is equivalent to + * Schema(STRING, INT, SHORT). This function translates schema nodes into their + * component types. + */ + private static List flatten(NamedType type) { + if (!(type.type instanceof Schema)) { + return Collections.singletonList(type); + } + Schema schema = (Schema) type.type; + ArrayList results = new ArrayList<>(); + for (BoundField field : schema.fields()) { + results.addAll(flatten(new NamedType(field.def.name, field.def.type))); + } + return results; + } + + @Test + public void testDefaultValues() throws Exception { + verifySizeRaisesUve((short) 0, "validateOnly", + new CreateTopicsRequestData().setValidateOnly(true)); + verifySizeSucceeds((short) 0, + new CreateTopicsRequestData().setValidateOnly(false)); + verifySizeSucceeds((short) 0, + new OffsetCommitRequestData().setRetentionTimeMs(123)); + verifySizeRaisesUve((short) 5, "forgotten", + new FetchRequestData().setForgotten(Collections.singletonList( + new FetchRequestData.ForgottenTopic().setName("foo")))); + } + + private void verifySizeRaisesUve(short version, String problemFieldName, + Message message) throws Exception { + try { + message.size(version); + Assert.fail("Expected to see an UnsupportedVersionException when writing " + + message + " at version " + version); + } catch (UnsupportedVersionException e) { + Assert.assertTrue("Expected to get an error message about " + problemFieldName, + e.getMessage().contains(problemFieldName)); + } + } + + private void verifySizeSucceeds(short version, Message message) throws Exception { + message.size(version); + } +}; diff --git a/clients/src/test/java/org/apache/kafka/message/VersionsTest.java b/clients/src/test/java/org/apache/kafka/message/VersionsTest.java new file mode 100644 index 0000000000000..43644a9f477bc --- /dev/null +++ b/clients/src/test/java/org/apache/kafka/message/VersionsTest.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.message; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.Timeout; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; + +public class VersionsTest { + @Rule + final public Timeout globalTimeout = Timeout.millis(120000); + + private static Versions newVersions(int lower, int higher) { + if ((lower < Short.MIN_VALUE) || (lower > Short.MAX_VALUE)) { + throw new RuntimeException("lower bound out of range."); + } + if ((higher < Short.MIN_VALUE) || (higher > Short.MAX_VALUE)) { + throw new RuntimeException("higher bound out of range."); + } + return new Versions((short) lower, (short) higher); + } + + @Test + public void testVersionsParse() { + assertEquals(Versions.NONE, Versions.parse(null, Versions.NONE)); + assertEquals(Versions.ALL, Versions.parse(" ", Versions.ALL)); + assertEquals(Versions.ALL, Versions.parse("", Versions.ALL)); + assertEquals(newVersions(4, 5), Versions.parse(" 4-5 ", null)); + } + + @Test + public void testRoundTrips() { + testRoundTrip(Versions.ALL, "0+"); + testRoundTrip(newVersions(1, 3), "1-3"); + testRoundTrip(newVersions(2, 2), "2"); + testRoundTrip(newVersions(3, Short.MAX_VALUE), "3+"); + testRoundTrip(Versions.NONE, "none"); + } + + private void testRoundTrip(Versions versions, String string) { + assertEquals(string, versions.toString()); + assertEquals(versions, Versions.parse(versions.toString(), null)); + } + + @Test + public void testIntersections() { + assertEquals(newVersions(2, 3), + newVersions(1, 3).intersect( + newVersions(2, 4))); + assertEquals(newVersions(3, 3), + newVersions(0, Short.MAX_VALUE).intersect( + newVersions(3, 3))); + assertEquals(Versions.NONE, + newVersions(9, Short.MAX_VALUE).intersect( + newVersions(2, 8))); + } + + @Test + public void testContains() { + assertTrue(newVersions(2, 3).contains((short) 3)); + assertTrue(newVersions(2, 3).contains((short) 2)); + assertFalse(newVersions(0, 1).contains((short) 2)); + assertTrue(newVersions(0, Short.MAX_VALUE).contains((short) 100)); + assertFalse(newVersions(2, Short.MAX_VALUE).contains((short) 0)); + } +}; diff --git a/gradle/spotbugs-exclude.xml b/gradle/spotbugs-exclude.xml index a954baf68d493..08313227544f0 100644 --- a/gradle/spotbugs-exclude.xml +++ b/gradle/spotbugs-exclude.xml @@ -153,6 +153,12 @@ For a detailed description of spotbugs bug categories, see https://spotbugs.read + + + + + + - + From a5c9bc46c72c13f7ca227f3f7c1b8ef9116f9b99 Mon Sep 17 00:00:00 2001 From: "Colin P. Mccabe" Date: Wed, 2 Jan 2019 15:45:55 -0800 Subject: [PATCH 03/14] Require struct versions to be set --- .../src/main/java/org/apache/kafka/message/StructSpec.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/buildSrc/src/main/java/org/apache/kafka/message/StructSpec.java b/buildSrc/src/main/java/org/apache/kafka/message/StructSpec.java index 0653ee97bdf93..cd24ea99b4356 100644 --- a/buildSrc/src/main/java/org/apache/kafka/message/StructSpec.java +++ b/buildSrc/src/main/java/org/apache/kafka/message/StructSpec.java @@ -39,7 +39,11 @@ public StructSpec(@JsonProperty("name") String name, @JsonProperty("versions") String versions, @JsonProperty("fields") List fields) { this.name = Objects.requireNonNull(name); - this.versions = Versions.parse(versions, Versions.ALL); + this.versions = Versions.parse(versions, null); + if (this.versions == null) { + throw new RuntimeException("You must specify the version of the " + + name + " structure."); + } this.fields = Collections.unmodifiableList(fields == null ? Collections.emptyList() : new ArrayList<>(fields)); this.hasKeys = fields == null ? false : fields.stream().anyMatch(f -> f.mapKey()); From b7cffd252d2876b544c8825739d58cb13c8c2b7d Mon Sep 17 00:00:00 2001 From: "Colin P. Mccabe" Date: Thu, 3 Jan 2019 11:52:03 -0800 Subject: [PATCH 04/14] More style fixes and etc. --- build.gradle | 2 +- .../apache/kafka/message/HeaderGenerator.java | 2 +- .../kafka/message/MessageDataGenerator.java | 2 +- checkstyle/import-control.xml | 3 --- .../kafka/common/protocol/MessageUtil.java | 19 ++++--------------- .../common/protocol/MessageUtilTest.java | 2 +- 6 files changed, 8 insertions(+), 22 deletions(-) diff --git a/build.gradle b/build.gradle index 98361a58c1937..423a269ed788e 100644 --- a/build.gradle +++ b/build.gradle @@ -902,7 +902,7 @@ project(':clients') { test { java { srcDirs = ["src/generated/java", "src/test/java", - "$rootDir/buildSrc/src/main/java/org/apache/kafka/message/"] + "$rootDir/buildSrc/src/main/java/org/apache/kafka/common/message/"] } } } diff --git a/buildSrc/src/main/java/org/apache/kafka/message/HeaderGenerator.java b/buildSrc/src/main/java/org/apache/kafka/message/HeaderGenerator.java index 1216406e4a4ea..36f2f5c0c5e5a 100644 --- a/buildSrc/src/main/java/org/apache/kafka/message/HeaderGenerator.java +++ b/buildSrc/src/main/java/org/apache/kafka/message/HeaderGenerator.java @@ -45,7 +45,7 @@ public final class HeaderGenerator { "" }; - static final String PACKAGE = "org.apache.kafka.message"; + static final String PACKAGE = "org.apache.kafka.common.message"; private final CodeBuffer buffer; diff --git a/buildSrc/src/main/java/org/apache/kafka/message/MessageDataGenerator.java b/buildSrc/src/main/java/org/apache/kafka/message/MessageDataGenerator.java index 8c37387254266..1e423df68d972 100644 --- a/buildSrc/src/main/java/org/apache/kafka/message/MessageDataGenerator.java +++ b/buildSrc/src/main/java/org/apache/kafka/message/MessageDataGenerator.java @@ -835,7 +835,7 @@ private void generateVariableLengthFieldSize(String fieldName, FieldType type, b buffer.incrementIndent(); } headerGenerator.addImport(MessageGenerator.MESSAGE_UTIL_CLASS); - buffer.printf("size += MessageUtil.utf8Length(%s);%n", fieldName); + buffer.printf("size += MessageUtil.serializedUtf8Length(%s);%n", fieldName); if (nullable) { buffer.decrementIndent(); buffer.printf("}%n"); diff --git a/checkstyle/import-control.xml b/checkstyle/import-control.xml index cd659faa26658..427eec9b5665e 100644 --- a/checkstyle/import-control.xml +++ b/checkstyle/import-control.xml @@ -53,7 +53,6 @@ - @@ -163,7 +162,6 @@ - @@ -299,7 +297,6 @@ - diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/MessageUtil.java b/clients/src/main/java/org/apache/kafka/common/protocol/MessageUtil.java index da12ab13e0c36..e3fdea7feae3b 100644 --- a/clients/src/main/java/org/apache/kafka/common/protocol/MessageUtil.java +++ b/clients/src/main/java/org/apache/kafka/common/protocol/MessageUtil.java @@ -17,6 +17,8 @@ package org.apache.kafka.common.protocol; +import org.apache.kafka.common.utils.Utils; + import java.util.Iterator; public final class MessageUtil { @@ -24,21 +26,8 @@ public final class MessageUtil { * Get the length of the UTF8 representation of a string, without allocating * a byte buffer for the string. */ - public static short utf8Length(CharSequence input) { - int count = 0, len = input.length(); - for (int i = 0; i < len; i++) { - char c = input.charAt(i); - if (c <= 0x7f) { - count++; - } else if (c <= 0x7ff) { - count += 2; - } else if (Character.isHighSurrogate(c)) { - count += 4; - i++; - } else { - count += 3; - } - } + public static short serializedUtf8Length(CharSequence input) { + int count = Utils.utf8Length(input); if (count > Short.MAX_VALUE) { throw new RuntimeException("String " + input + " is too long to serialize."); } diff --git a/clients/src/test/java/org/apache/kafka/common/protocol/MessageUtilTest.java b/clients/src/test/java/org/apache/kafka/common/protocol/MessageUtilTest.java index b327314539fe1..8620a1b03dc8d 100755 --- a/clients/src/test/java/org/apache/kafka/common/protocol/MessageUtilTest.java +++ b/clients/src/test/java/org/apache/kafka/common/protocol/MessageUtilTest.java @@ -46,7 +46,7 @@ public void testMultibyteUtf8Lengths() { private void validateUtf8Length(String string) { byte[] arr = string.getBytes(StandardCharsets.UTF_8); - assertEquals(arr.length, MessageUtil.utf8Length(string)); + assertEquals(arr.length, MessageUtil.serializedUtf8Length(string)); } @Test From baffbf347673931d744d3d2516af862582a06d9f Mon Sep 17 00:00:00 2001 From: "Colin P. Mccabe" Date: Thu, 3 Jan 2019 12:09:40 -0800 Subject: [PATCH 05/14] Finish renaming --- build.gradle | 2 +- checkstyle/import-control.xml | 5 +++++ .../kafka/{ => common}/message/MessageGeneratorTest.java | 0 .../org/apache/kafka/{ => common}/message/MessageTest.java | 6 +++--- .../org/apache/kafka/{ => common}/message/VersionsTest.java | 0 5 files changed, 9 insertions(+), 4 deletions(-) rename clients/src/test/java/org/apache/kafka/{ => common}/message/MessageGeneratorTest.java (100%) rename clients/src/test/java/org/apache/kafka/{ => common}/message/MessageTest.java (98%) rename clients/src/test/java/org/apache/kafka/{ => common}/message/VersionsTest.java (100%) diff --git a/build.gradle b/build.gradle index 423a269ed788e..98361a58c1937 100644 --- a/build.gradle +++ b/build.gradle @@ -902,7 +902,7 @@ project(':clients') { test { java { srcDirs = ["src/generated/java", "src/test/java", - "$rootDir/buildSrc/src/main/java/org/apache/kafka/common/message/"] + "$rootDir/buildSrc/src/main/java/org/apache/kafka/message/"] } } } diff --git a/checkstyle/import-control.xml b/checkstyle/import-control.xml index 427eec9b5665e..c69f94d25bbce 100644 --- a/checkstyle/import-control.xml +++ b/checkstyle/import-control.xml @@ -213,6 +213,11 @@ + + + + + diff --git a/clients/src/test/java/org/apache/kafka/message/MessageGeneratorTest.java b/clients/src/test/java/org/apache/kafka/common/message/MessageGeneratorTest.java similarity index 100% rename from clients/src/test/java/org/apache/kafka/message/MessageGeneratorTest.java rename to clients/src/test/java/org/apache/kafka/common/message/MessageGeneratorTest.java diff --git a/clients/src/test/java/org/apache/kafka/message/MessageTest.java b/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java similarity index 98% rename from clients/src/test/java/org/apache/kafka/message/MessageTest.java rename to clients/src/test/java/org/apache/kafka/common/message/MessageTest.java index 609c26c4bf3ea..a0f19e5069d4c 100644 --- a/clients/src/test/java/org/apache/kafka/message/MessageTest.java +++ b/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.kafka.message; +package org.apache.kafka.common.message; import org.apache.kafka.common.errors.UnsupportedVersionException; import org.apache.kafka.common.protocol.ApiKeys; @@ -34,8 +34,8 @@ import org.apache.kafka.common.protocol.types.Type; import org.apache.kafka.common.utils.Utils; -import org.apache.kafka.message.AddPartitionsToTxnRequestData.AddPartitionsToTxnTopic; -import org.apache.kafka.message.AddPartitionsToTxnRequestData.AddPartitionsToTxnTopicSet; +import org.apache.kafka.common.message.AddPartitionsToTxnRequestData.AddPartitionsToTxnTopic; +import org.apache.kafka.common.message.AddPartitionsToTxnRequestData.AddPartitionsToTxnTopicSet; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; diff --git a/clients/src/test/java/org/apache/kafka/message/VersionsTest.java b/clients/src/test/java/org/apache/kafka/common/message/VersionsTest.java similarity index 100% rename from clients/src/test/java/org/apache/kafka/message/VersionsTest.java rename to clients/src/test/java/org/apache/kafka/common/message/VersionsTest.java From 3ea267189fe17d6416622cd97ccbaa4cf4d270d8 Mon Sep 17 00:00:00 2001 From: "Colin P. Mccabe" Date: Fri, 4 Jan 2019 14:13:07 -0800 Subject: [PATCH 06/14] Bump ListOffset for KIP-207 --- .../src/main/resources/common/message/ListOffsetRequest.json | 3 ++- .../src/main/resources/common/message/ListOffsetResponse.json | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/clients/src/main/resources/common/message/ListOffsetRequest.json b/clients/src/main/resources/common/message/ListOffsetRequest.json index d9d089a795743..8f2c7383e698c 100644 --- a/clients/src/main/resources/common/message/ListOffsetRequest.json +++ b/clients/src/main/resources/common/message/ListOffsetRequest.json @@ -22,7 +22,8 @@ // Version 2 adds the isolation level, which is used for transactional reads. // Version 3 is the same as version 2. // Version 4 adds the current leader epoch, which is used for fencing. - "validVersions": "0-4", + // Version 5 is the same as version 5. + "validVersions": "0-5", "fields": [ { "name": "ReplicaId", "type": "int32", "versions": "0+", "about": "The broker ID of the requestor, or -1 if this request is being made by a normal consumer." }, diff --git a/clients/src/main/resources/common/message/ListOffsetResponse.json b/clients/src/main/resources/common/message/ListOffsetResponse.json index 933c0e778cb9a..36935daa526a1 100644 --- a/clients/src/main/resources/common/message/ListOffsetResponse.json +++ b/clients/src/main/resources/common/message/ListOffsetResponse.json @@ -22,7 +22,8 @@ // Version 2 adds the throttle time. // Starting in version 3, on quota violation, brokers send out responses before throttling. // Version 4 adds the leader epoch, which is used for fencing. - "validVersions": "0-4", + // Version 5 adds a new error code, OFFSET_NOT_AVAILABLE. + "validVersions": "0-5", "fields": [ { "name": "ThrottleTimeMs", "type": "int32", "versions": "2+", "ignorable": true, "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." }, From e1b2173445a320ff84cbb6607409e6d87ed13583 Mon Sep 17 00:00:00 2001 From: "Colin P. Mccabe" Date: Fri, 4 Jan 2019 17:58:45 -0800 Subject: [PATCH 07/14] Fix some javadoc, use assertTrue instead of Assert.fail in a case --- .../apache/kafka/common/protocol/Message.java | 8 +++---- .../kafka/common/message/MessageTest.java | 24 +++++++++---------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/Message.java b/clients/src/main/java/org/apache/kafka/common/protocol/Message.java index 881f84aec44d3..023c0ee322abe 100644 --- a/clients/src/main/java/org/apache/kafka/common/protocol/Message.java +++ b/clients/src/main/java/org/apache/kafka/common/protocol/Message.java @@ -44,7 +44,7 @@ public interface Message { * * @param version The version to use. * - * @throws org.apache.kafka.common.errors.UnsupportedVersionException + * @throws {@see org.apache.kafka.common.errors.UnsupportedVersionException} * If the specified version is too new to be supported * by this software. */ @@ -56,7 +56,7 @@ public interface Message { * @param writable The destination writable. * @param version The version to use. * - * @throws org.apache.kafka.common.errors.UnsupportedVersionException + * @throws {@see org.apache.kafka.common.errors.UnsupportedVersionException} * If the specified version is too new to be supported * by this software. */ @@ -69,7 +69,7 @@ public interface Message { * @param readable The source readable. * @param version The version to use. * - * @throws org.apache.kafka.common.errors.UnsupportedVersionException + * @throws {@see org.apache.kafka.common.errors.UnsupportedVersionException} * If the specified version is too new to be supported * by this software. */ @@ -89,7 +89,7 @@ public interface Message { * * @param version The version to use. * - * @throws org.apache.kafka.common.errors.UnsupportedVersionException + * @throws {@see org.apache.kafka.common.errors.UnsupportedVersionException} * If the specified version is too new to be supported * by this software. */ diff --git a/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java b/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java index a0f19e5069d4c..1db68b308bd8d 100644 --- a/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java +++ b/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java @@ -42,6 +42,8 @@ import org.junit.rules.Timeout; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; public final class MessageTest { @Rule @@ -128,21 +130,19 @@ public void testMessageVersions() throws Exception { try { message = MessageFactory.newRequest(apiKey.id); } catch (UnsupportedVersionException e) { - Assert.fail("No request message spec found for API " + apiKey); - } - if (apiKey.latestVersion() > message.highestSupportedVersion()) { - Assert.fail("Request message spec for " + apiKey + " only " + - "supports versions up to " + message.highestSupportedVersion()); + fail("No request message spec found for API " + apiKey); } + assertTrue("Request message spec for " + apiKey + " only " + + "supports versions up to " + message.highestSupportedVersion(), + apiKey.latestVersion() <= message.highestSupportedVersion()); try { message = MessageFactory.newResponse(apiKey.id); } catch (UnsupportedVersionException e) { - Assert.fail("No response message spec found for API " + apiKey); - } - if (apiKey.latestVersion() > message.highestSupportedVersion()) { - Assert.fail("Response message spec for " + apiKey + " only " + - "supports versions up to " + message.highestSupportedVersion()); + fail("No response message spec found for API " + apiKey); } + assertTrue("Response message spec for " + apiKey + " only " + + "supports versions up to " + message.highestSupportedVersion(), + apiKey.latestVersion() <= message.highestSupportedVersion()); } } @@ -293,10 +293,10 @@ private void verifySizeRaisesUve(short version, String problemFieldName, Message message) throws Exception { try { message.size(version); - Assert.fail("Expected to see an UnsupportedVersionException when writing " + + fail("Expected to see an UnsupportedVersionException when writing " + message + " at version " + version); } catch (UnsupportedVersionException e) { - Assert.assertTrue("Expected to get an error message about " + problemFieldName, + assertTrue("Expected to get an error message about " + problemFieldName, e.getMessage().contains(problemFieldName)); } } From 2091b2671c14c722b677e2db01762526e1331f00 Mon Sep 17 00:00:00 2001 From: "Colin P. Mccabe" Date: Wed, 9 Jan 2019 16:41:35 -0800 Subject: [PATCH 08/14] Fix namespace for VersionsTest, MessageGeneratorTest --- .../apache/kafka/{common => }/message/MessageGeneratorTest.java | 0 .../java/org/apache/kafka/{common => }/message/VersionsTest.java | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename clients/src/test/java/org/apache/kafka/{common => }/message/MessageGeneratorTest.java (100%) rename clients/src/test/java/org/apache/kafka/{common => }/message/VersionsTest.java (100%) diff --git a/clients/src/test/java/org/apache/kafka/common/message/MessageGeneratorTest.java b/clients/src/test/java/org/apache/kafka/message/MessageGeneratorTest.java similarity index 100% rename from clients/src/test/java/org/apache/kafka/common/message/MessageGeneratorTest.java rename to clients/src/test/java/org/apache/kafka/message/MessageGeneratorTest.java diff --git a/clients/src/test/java/org/apache/kafka/common/message/VersionsTest.java b/clients/src/test/java/org/apache/kafka/message/VersionsTest.java similarity index 100% rename from clients/src/test/java/org/apache/kafka/common/message/VersionsTest.java rename to clients/src/test/java/org/apache/kafka/message/VersionsTest.java From f003e4d459f1b1b1bb2c7a043c93026612791a08 Mon Sep 17 00:00:00 2001 From: "Colin P. Mccabe" Date: Thu, 10 Jan 2019 14:59:26 -0800 Subject: [PATCH 09/14] Some syntax cleanups --- .../org/apache/kafka/message/FieldType.java | 2 +- .../apache/kafka/message/HeaderGenerator.java | 2 +- .../kafka/message/MessageDataGenerator.java | 103 +++++++----------- .../message/MessageFactoryGenerator.java | 8 +- 4 files changed, 47 insertions(+), 68 deletions(-) diff --git a/buildSrc/src/main/java/org/apache/kafka/message/FieldType.java b/buildSrc/src/main/java/org/apache/kafka/message/FieldType.java index 43912c430e69a..4534055a5a86a 100644 --- a/buildSrc/src/main/java/org/apache/kafka/message/FieldType.java +++ b/buildSrc/src/main/java/org/apache/kafka/message/FieldType.java @@ -20,7 +20,7 @@ import java.util.Optional; public interface FieldType { - static final String STRUCT_PREFIX = "[]"; + String STRUCT_PREFIX = "[]"; final class BoolFieldType implements FieldType { private static final BoolFieldType INSTANCE = new BoolFieldType(); diff --git a/buildSrc/src/main/java/org/apache/kafka/message/HeaderGenerator.java b/buildSrc/src/main/java/org/apache/kafka/message/HeaderGenerator.java index 36f2f5c0c5e5a..6de9736af818c 100644 --- a/buildSrc/src/main/java/org/apache/kafka/message/HeaderGenerator.java +++ b/buildSrc/src/main/java/org/apache/kafka/message/HeaderGenerator.java @@ -60,7 +60,7 @@ public void addImport(String newImport) { this.imports.add(newImport); } - public void generate() throws Exception { + public void generate() { for (int i = 0; i < HEADER.length; i++) { buffer.printf("%s%n", HEADER[i]); } diff --git a/buildSrc/src/main/java/org/apache/kafka/message/MessageDataGenerator.java b/buildSrc/src/main/java/org/apache/kafka/message/MessageDataGenerator.java index 1e423df68d972..ab98452d54040 100644 --- a/buildSrc/src/main/java/org/apache/kafka/message/MessageDataGenerator.java +++ b/buildSrc/src/main/java/org/apache/kafka/message/MessageDataGenerator.java @@ -109,7 +109,7 @@ private void generateClass(Optional topLevelMessageSpec, } private void generateClassHeader(String className, boolean isTopLevel, - boolean isSetElement) throws Exception { + boolean isSetElement) { Set implementedInterfaces = new HashSet<>(); implementedInterfaces.add("Message"); headerGenerator.addImport(MessageGenerator.MESSAGE_CLASS); @@ -144,7 +144,7 @@ private void generateSubclasses(String className, StructSpec struct, } } - private void generateHashSet(String className, StructSpec struct) throws Exception { + private void generateHashSet(String className, StructSpec struct) { buffer.printf("%n"); headerGenerator.addImport(MessageGenerator.IMPLICIT_LINKED_HASH_MULTI_SET_CLASS); buffer.printf("public static class %s extends ImplicitLinkedHashMultiSet<%s> {%n", @@ -159,7 +159,7 @@ private void generateHashSet(String className, StructSpec struct) throws Excepti buffer.printf("}%n"); } - private void generateHashSetZeroArgConstructor(String className) throws Exception { + private void generateHashSetZeroArgConstructor(String className) { buffer.printf("public %s() {%n", hashSetType(className)); buffer.incrementIndent(); buffer.printf("super();%n"); @@ -168,7 +168,7 @@ private void generateHashSetZeroArgConstructor(String className) throws Exceptio buffer.printf("%n"); } - private void generateHashSetSizeArgConstructor(String className) throws Exception { + private void generateHashSetSizeArgConstructor(String className) { buffer.printf("public %s(int expectedNumElements) {%n", hashSetType(className)); buffer.incrementIndent(); buffer.printf("super(expectedNumElements);%n"); @@ -177,7 +177,7 @@ private void generateHashSetSizeArgConstructor(String className) throws Exceptio buffer.printf("%n"); } - private void generateHashSetIteratorConstructor(String className) throws Exception { + private void generateHashSetIteratorConstructor(String className) { headerGenerator.addImport(MessageGenerator.ITERATOR_CLASS); buffer.printf("public %s(Iterator<%s> iterator) {%n", hashSetType(className), className); buffer.incrementIndent(); @@ -187,8 +187,7 @@ private void generateHashSetIteratorConstructor(String className) throws Excepti buffer.printf("%n"); } - private void generateHashSetFindMethod(String className, StructSpec struct) - throws Exception { + private void generateHashSetFindMethod(String className, StructSpec struct) { headerGenerator.addImport(MessageGenerator.LIST_CLASS); buffer.printf("public %s find(%s) {%n", className, commaSeparatedHashSetFieldAndTypes(struct)); @@ -201,8 +200,7 @@ private void generateHashSetFindMethod(String className, StructSpec struct) buffer.printf("%n"); } - private void generateHashSetFindAllMethod(String className, StructSpec struct) - throws Exception { + private void generateHashSetFindAllMethod(String className, StructSpec struct) { headerGenerator.addImport(MessageGenerator.LIST_CLASS); buffer.printf("public List<%s> findAll(%s) {%n", className, commaSeparatedHashSetFieldAndTypes(struct)); @@ -215,7 +213,7 @@ private void generateHashSetFindAllMethod(String className, StructSpec struct) buffer.printf("%n"); } - private void generateKeyElement(String className, StructSpec struct) throws Exception { + private void generateKeyElement(String className, StructSpec struct) { buffer.printf("%s key = new %s();%n", className, className); for (FieldSpec field : struct.fields()) { if (field.mapKey()) { @@ -233,8 +231,7 @@ private String commaSeparatedHashSetFieldAndTypes(StructSpec struct) { collect(Collectors.joining(", ")); } - private void generateFieldDeclarations(StructSpec struct, boolean isSetElement) - throws Exception { + private void generateFieldDeclarations(StructSpec struct, boolean isSetElement) { for (FieldSpec field : struct.fields()) { generateFieldDeclaration(field); } @@ -244,13 +241,12 @@ private void generateFieldDeclarations(StructSpec struct, boolean isSetElement) } } - private void generateFieldDeclaration(FieldSpec field) throws Exception { + private void generateFieldDeclaration(FieldSpec field) { buffer.printf("private %s %s;%n", fieldAbstractJavaType(field), field.camelCaseName()); } - private void generateFieldAccessors(StructSpec struct, boolean isSetElement) - throws Exception { + private void generateFieldAccessors(StructSpec struct, boolean isSetElement) { for (FieldSpec field : struct.fields()) { generateFieldAccessor(field); } @@ -265,8 +261,8 @@ private void generateFieldAccessors(StructSpec struct, boolean isSetElement) } } - private void generateFieldMutators(StructSpec struct, - String className, boolean isSetElement) throws Exception { + private void generateFieldMutators(StructSpec struct, String className, + boolean isSetElement) { for (FieldSpec field : struct.fields()) { generateFieldMutator(className, field); } @@ -331,7 +327,7 @@ private String fieldConcreteJavaType(FieldSpec field) { } } - private void generateClassConstructors(String className, StructSpec struct) throws Exception { + private void generateClassConstructors(String className, StructSpec struct) { headerGenerator.addImport(MessageGenerator.READABLE_CLASS); buffer.printf("public %s(Readable readable, short version) {%n", className); buffer.incrementIndent(); @@ -358,7 +354,7 @@ private void generateClassConstructors(String className, StructSpec struct) thro buffer.printf("}%n"); } - private void initializeArrayDefaults(StructSpec struct) throws Exception { + private void initializeArrayDefaults(StructSpec struct) { for (FieldSpec field : struct.fields()) { if (field.type().isArray()) { buffer.printf("this.%s = %s;%n", @@ -367,7 +363,7 @@ private void initializeArrayDefaults(StructSpec struct) throws Exception { } } - private void generateShortAccessor(String name, short val) throws Exception { + private void generateShortAccessor(String name, short val) { buffer.printf("@Override%n"); buffer.printf("public short %s() {%n", name); buffer.incrementIndent(); @@ -377,7 +373,7 @@ private void generateShortAccessor(String name, short val) throws Exception { } private void generateClassReader(String className, StructSpec struct, - Versions parentVersions) throws Exception { + Versions parentVersions) { headerGenerator.addImport(MessageGenerator.READABLE_CLASS); buffer.printf("@Override%n"); buffer.printf("public void read(Readable readable, short version) {%n"); @@ -402,8 +398,7 @@ private void generateClassReader(String className, StructSpec struct, buffer.printf("}%n"); } - private void generateFieldReader(FieldSpec field, - Versions curVersions) throws Exception { + private void generateFieldReader(FieldSpec field, Versions curVersions) { if (field.type().isArray()) { boolean maybeAbsent = generateVersionCheck(curVersions, field.struct().versions()); @@ -474,7 +469,7 @@ private String readFieldFromReadable(FieldType type) { } private void generateClassFromStruct(String className, StructSpec struct, - Versions parentVersions) throws Exception { + Versions parentVersions) { headerGenerator.addImport(MessageGenerator.STRUCT_CLASS); buffer.printf("@Override%n"); buffer.printf("public void fromStruct(Struct struct, short version) {%n"); @@ -499,8 +494,7 @@ private void generateClassFromStruct(String className, StructSpec struct, buffer.printf("}%n"); } - private void generateFieldFromStruct(FieldSpec field, - Versions curVersions) throws Exception { + private void generateFieldFromStruct(FieldSpec field, Versions curVersions) { if (field.type().isArray()) { boolean maybeAbsent = generateVersionCheck(curVersions, field.struct().versions()); @@ -601,7 +595,7 @@ private String readFieldFromStruct(FieldType type, String name) { } private void generateClassWriter(String className, StructSpec struct, - Versions parentVersions) throws Exception { + Versions parentVersions) { headerGenerator.addImport(MessageGenerator.WRITABLE_CLASS); buffer.printf("@Override%n"); buffer.printf("public void write(Writable writable, short version) {%n"); @@ -656,7 +650,7 @@ private String writeFieldToWritable(FieldType type, boolean nullable, String nam } } - private void generateFieldWriter(FieldSpec field, Versions curVersions) throws Exception { + private void generateFieldWriter(FieldSpec field, Versions curVersions) { if (field.type().isArray()) { boolean maybeAbsent = generateVersionCheck(curVersions, field.struct().versions()); @@ -700,7 +694,7 @@ private void generateFieldWriter(FieldSpec field, Versions curVersions) throws E } private void generateClassToStruct(String className, StructSpec struct, - Versions parentVersions) throws Exception { + Versions parentVersions) { headerGenerator.addImport(MessageGenerator.STRUCT_CLASS); buffer.printf("@Override%n"); buffer.printf("public Struct toStruct(short version) {%n"); @@ -727,7 +721,7 @@ private void generateClassToStruct(String className, StructSpec struct, buffer.printf("}%n"); } - private void generateFieldToStruct(FieldSpec field, Versions curVersions) throws Exception { + private void generateFieldToStruct(FieldSpec field, Versions curVersions) { if ((!field.type().canBeNullable()) && (!field.nullableVersions().empty())) { throw new RuntimeException("Fields of type " + field.type() + @@ -800,7 +794,7 @@ private void generateFieldToStruct(FieldSpec field, Versions curVersions) throws } private void generateClassSize(String className, StructSpec struct, - Versions parentVersions) throws Exception { + Versions parentVersions) { buffer.printf("@Override%n"); buffer.printf("public int size(short version) {%n"); buffer.incrementIndent(); @@ -826,8 +820,7 @@ private void generateClassSize(String className, StructSpec struct, buffer.printf("}%n"); } - private void generateVariableLengthFieldSize(String fieldName, FieldType type, boolean nullable) - throws Exception { + private void generateVariableLengthFieldSize(String fieldName, FieldType type, boolean nullable) { if (type instanceof FieldType.StringFieldType) { buffer.printf("size += 2;%n"); if (nullable) { @@ -858,7 +851,7 @@ private void generateVariableLengthFieldSize(String fieldName, FieldType type, b } } - private void generateFieldSize(FieldSpec field, Versions curVersions) throws Exception { + private void generateFieldSize(FieldSpec field, Versions curVersions) { if (field.type().fixedLength().isPresent()) { boolean maybeAbsent = generateVersionCheck(curVersions, field.struct().versions()); @@ -917,7 +910,7 @@ private void generateFieldSize(FieldSpec field, Versions curVersions) throws Exc } } - private void generateAbsentValueCheck(FieldSpec field) throws Exception { + private void generateAbsentValueCheck(FieldSpec field) { if (field.ignorable()) { buffer.printf("}%n"); return; @@ -948,8 +941,7 @@ private void generateAbsentValueCheck(FieldSpec field) throws Exception { buffer.printf("}%n"); } - private void generateClassEquals(String className, StructSpec struct, boolean onlyMapKeys) - throws Exception { + private void generateClassEquals(String className, StructSpec struct, boolean onlyMapKeys) { buffer.printf("@Override%n"); buffer.printf("public boolean equals(Object obj) {%n"); buffer.incrementIndent(); @@ -967,7 +959,7 @@ private void generateClassEquals(String className, StructSpec struct, boolean on buffer.printf("}%n"); } - private void generateFieldEquals(FieldSpec field) throws Exception { + private void generateFieldEquals(FieldSpec field) { if (field.type().isString() || field.type().isArray() || field.type().isStruct()) { buffer.printf("if (this.%s == null) {%n", field.camelCaseName()); buffer.incrementIndent(); @@ -990,7 +982,7 @@ private void generateFieldEquals(FieldSpec field) throws Exception { } } - private void generateClassHashCode(StructSpec struct, boolean onlyMapKeys) throws Exception { + private void generateClassHashCode(StructSpec struct, boolean onlyMapKeys) { buffer.printf("@Override%n"); buffer.printf("public int hashCode() {%n"); buffer.incrementIndent(); @@ -1005,7 +997,7 @@ private void generateClassHashCode(StructSpec struct, boolean onlyMapKeys) throw buffer.printf("}%n"); } - private void generateFieldHashCode(FieldSpec field) throws Exception { + private void generateFieldHashCode(FieldSpec field) { if (field.type() instanceof FieldType.BoolFieldType) { buffer.printf("hashCode = 31 * hashCode + (%s ? 1231 : 1237);%n", field.camelCaseName()); @@ -1032,7 +1024,7 @@ private void generateFieldHashCode(FieldSpec field) throws Exception { } } - private void generateClassToString(String className, StructSpec struct) throws Exception { + private void generateClassToString(String className, StructSpec struct) { buffer.printf("@Override%n"); buffer.printf("public String toString() {%n"); buffer.incrementIndent(); @@ -1049,7 +1041,7 @@ private void generateClassToString(String className, StructSpec struct) throws E buffer.printf("}%n"); } - private void generateFieldToString(String prefix, FieldSpec field) throws Exception { + private void generateFieldToString(String prefix, FieldSpec field) { if (field.type() instanceof FieldType.BoolFieldType) { buffer.printf("+ \"%s%s=\" + (%s ? \"true\" : \"false\")%n", prefix, field.camelCaseName(), field.camelCaseName()); @@ -1078,7 +1070,7 @@ private void generateFieldToString(String prefix, FieldSpec field) throws Except } } - private boolean generateNullCheck(Versions prevVersions, FieldSpec field) throws Exception { + private boolean generateNullCheck(Versions prevVersions, FieldSpec field) { if (prevVersions.intersect(field.nullableVersions()).empty()) { return false; } @@ -1087,16 +1079,7 @@ private boolean generateNullCheck(Versions prevVersions, FieldSpec field) throws return true; } - private boolean generateNonNullCheck(Versions prevVersions, FieldSpec field) throws Exception { - if (prevVersions.intersect(field.nullableVersions()).empty()) { - return false; - } - buffer.printf("if (%s != null) {%n", field.camelCaseName()); - buffer.incrementIndent(); - return true; - } - - private boolean generateVersionCheck(Versions prev, Versions cur) throws Exception { + private boolean generateVersionCheck(Versions prev, Versions cur) { if (cur.lowest() > prev.lowest()) { if (cur.highest() < prev.highest()) { buffer.printf("if ((version >= %d) && (version <= %d)) {%n", @@ -1119,7 +1102,7 @@ private boolean generateVersionCheck(Versions prev, Versions cur) throws Excepti } } - private boolean generateInverseVersionCheck(Versions prev, Versions cur) throws Exception { + private boolean generateInverseVersionCheck(Versions prev, Versions cur) { if (cur.lowest() > prev.lowest()) { if (cur.highest() < prev.highest()) { buffer.printf("if ((version < %d) || (version > %d)) {%n", @@ -1139,7 +1122,7 @@ private boolean generateInverseVersionCheck(Versions prev, Versions cur) throws } } - private void generateSetDefault(FieldSpec field) throws Exception { + private void generateSetDefault(FieldSpec field) { buffer.decrementIndent(); buffer.printf("} else {%n"); buffer.incrementIndent(); @@ -1240,14 +1223,13 @@ private String fieldDefault(FieldSpec field) { } } - private void generateFieldAccessor(FieldSpec field) throws Exception { + private void generateFieldAccessor(FieldSpec field) { buffer.printf("%n"); generateAccessor(fieldAbstractJavaType(field), field.camelCaseName(), field.camelCaseName()); } - private void generateAccessor(String javaType, String functionName, String memberName) - throws Exception { + private void generateAccessor(String javaType, String functionName, String memberName) { buffer.printf("public %s %s() {%n", javaType, functionName); buffer.incrementIndent(); buffer.printf("return this.%s;%n", memberName); @@ -1255,7 +1237,7 @@ private void generateAccessor(String javaType, String functionName, String membe buffer.printf("}%n"); } - private void generateFieldMutator(String className, FieldSpec field) throws Exception { + private void generateFieldMutator(String className, FieldSpec field) { buffer.printf("%n"); buffer.printf("public %s set%s(%s v) {%n", className, @@ -1268,8 +1250,7 @@ private void generateFieldMutator(String className, FieldSpec field) throws Exce buffer.printf("}%n"); } - private void generateSetter(String javaType, String functionName, String memberName) - throws Exception { + private void generateSetter(String javaType, String functionName, String memberName) { buffer.printf("public void %s(%s v) {%n", functionName, javaType); buffer.incrementIndent(); buffer.printf("this.%s = v;%n", memberName); diff --git a/buildSrc/src/main/java/org/apache/kafka/message/MessageFactoryGenerator.java b/buildSrc/src/main/java/org/apache/kafka/message/MessageFactoryGenerator.java index 8a0374603ea71..d754032ca7ae9 100644 --- a/buildSrc/src/main/java/org/apache/kafka/message/MessageFactoryGenerator.java +++ b/buildSrc/src/main/java/org/apache/kafka/message/MessageFactoryGenerator.java @@ -51,7 +51,7 @@ public MessageFactoryGenerator() { this.buffer = new CodeBuffer(); } - public void generate() throws Exception { + public void generate() { buffer.printf("public final class MessageFactory {%n"); buffer.incrementIndent(); generateFactoryMethod("request", requestApis); @@ -66,8 +66,7 @@ public void generate() throws Exception { headerGenerator.generate(); } - public void generateFactoryMethod(String type, TreeMap apis) - throws Exception { + public void generateFactoryMethod(String type, TreeMap apis) { headerGenerator.addImport(MessageGenerator.MESSAGE_CLASS); buffer.printf("public static Message new%s(short apiKey) {%n", MessageGenerator.capitalizeFirst(type)); @@ -92,8 +91,7 @@ public void generateFactoryMethod(String type, TreeMap apis) buffer.printf("}%n"); } - public void generateSchemasAccessor(String type, TreeMap apis) - throws Exception { + public void generateSchemasAccessor(String type, TreeMap apis) { headerGenerator.addImport(MessageGenerator.SCHEMA_CLASS); buffer.printf("public static Schema[] %sSchemas(short apiKey) {%n", MessageGenerator.lowerCaseFirst(type)); From 6386c1a57d1a2088e2c31f49d20ffde1ba1c2b82 Mon Sep 17 00:00:00 2001 From: "Colin P. Mccabe" Date: Thu, 10 Jan 2019 15:19:18 -0800 Subject: [PATCH 10/14] Separate ApiMessage and Message, add Optional, etc. --- buildSrc/build.gradle | 2 +- ...r.java => ApiMessageFactoryGenerator.java} | 18 ++++++------ .../kafka/message/MessageDataGenerator.java | 14 +++++++--- .../kafka/message/MessageGenerator.java | 12 ++++---- .../org/apache/kafka/message/MessageSpec.java | 7 +++-- .../kafka/common/protocol/ApiMessage.java | 28 +++++++++++++++++++ .../apache/kafka/common/protocol/Message.java | 5 ---- .../common/message/RequestHeader.json | 1 + .../common/message/ResponseHeader.json | 1 + .../kafka/common/message/MessageTest.java | 8 +++--- 10 files changed, 65 insertions(+), 31 deletions(-) rename buildSrc/src/main/java/org/apache/kafka/message/{MessageFactoryGenerator.java => ApiMessageFactoryGenerator.java} (89%) create mode 100644 clients/src/main/java/org/apache/kafka/common/protocol/ApiMessage.java diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index 2c53f6c0a2cfd..23a1fc413f78b 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -20,7 +20,7 @@ repositories { dependencies { compile "com.fasterxml.jackson.core:jackson-databind:2.9.6" compile "com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:2.9.6" + compile "com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.9.6" } -// Workaround for https://github.com/gradle/gradle/issues/1271 test.enabled=false diff --git a/buildSrc/src/main/java/org/apache/kafka/message/MessageFactoryGenerator.java b/buildSrc/src/main/java/org/apache/kafka/message/ApiMessageFactoryGenerator.java similarity index 89% rename from buildSrc/src/main/java/org/apache/kafka/message/MessageFactoryGenerator.java rename to buildSrc/src/main/java/org/apache/kafka/message/ApiMessageFactoryGenerator.java index d754032ca7ae9..889b08b80120c 100644 --- a/buildSrc/src/main/java/org/apache/kafka/message/MessageFactoryGenerator.java +++ b/buildSrc/src/main/java/org/apache/kafka/message/ApiMessageFactoryGenerator.java @@ -22,7 +22,7 @@ import java.util.Map; import java.util.TreeMap; -public final class MessageFactoryGenerator { +public final class ApiMessageFactoryGenerator { private final TreeMap requestApis; private final TreeMap responseApis; private final HeaderGenerator headerGenerator; @@ -30,21 +30,21 @@ public final class MessageFactoryGenerator { public void registerMessageType(MessageSpec spec) { if (spec.type() == MessageSpecType.REQUEST) { - if (requestApis.containsKey(spec.apiKey())) { + if (requestApis.containsKey(spec.apiKey().get())) { throw new RuntimeException("Found more than one request with " + - "API key " + spec.apiKey()); + "API key " + spec.apiKey().get()); } - requestApis.put(spec.apiKey(), spec.generatedClassName()); + requestApis.put(spec.apiKey().get(), spec.generatedClassName()); } else if (spec.type() == MessageSpecType.RESPONSE) { - if (responseApis.containsKey(spec.apiKey())) { + if (responseApis.containsKey(spec.apiKey().get())) { throw new RuntimeException("Found more than one response with " + - "API key " + spec.apiKey()); + "API key " + spec.apiKey().get()); } - responseApis.put(spec.apiKey(), spec.generatedClassName()); + responseApis.put(spec.apiKey().get(), spec.generatedClassName()); } } - public MessageFactoryGenerator() { + public ApiMessageFactoryGenerator() { this.requestApis = new TreeMap<>(); this.responseApis = new TreeMap<>(); this.headerGenerator = new HeaderGenerator(); @@ -52,7 +52,7 @@ public MessageFactoryGenerator() { } public void generate() { - buffer.printf("public final class MessageFactory {%n"); + buffer.printf("public final class ApiMessageFactory {%n"); buffer.incrementIndent(); generateFactoryMethod("request", requestApis); buffer.printf("%n"); diff --git a/buildSrc/src/main/java/org/apache/kafka/message/MessageDataGenerator.java b/buildSrc/src/main/java/org/apache/kafka/message/MessageDataGenerator.java index ab98452d54040..636ca2e64a196 100644 --- a/buildSrc/src/main/java/org/apache/kafka/message/MessageDataGenerator.java +++ b/buildSrc/src/main/java/org/apache/kafka/message/MessageDataGenerator.java @@ -72,8 +72,9 @@ private void generateClass(Optional topLevelMessageSpec, schemaGenerator.writeSchema(className, buffer); generateClassConstructors(className, struct); buffer.printf("%n"); - generateShortAccessor("apiKey", isTopLevel ? - topLevelMessageSpec.get().apiKey() : -1); + if (isTopLevel) { + generateShortAccessor("apiKey", topLevelMessageSpec.get().apiKey().get()); + } buffer.printf("%n"); generateShortAccessor("lowestSupportedVersion", parentVersions.lowest()); buffer.printf("%n"); @@ -111,8 +112,13 @@ private void generateClass(Optional topLevelMessageSpec, private void generateClassHeader(String className, boolean isTopLevel, boolean isSetElement) { Set implementedInterfaces = new HashSet<>(); - implementedInterfaces.add("Message"); - headerGenerator.addImport(MessageGenerator.MESSAGE_CLASS); + if (isTopLevel) { + implementedInterfaces.add("ApiMessage"); + headerGenerator.addImport(MessageGenerator.API_MESSAGE_CLASS); + } else { + implementedInterfaces.add("Message"); + headerGenerator.addImport(MessageGenerator.MESSAGE_CLASS); + } if (isSetElement) { headerGenerator.addImport(MessageGenerator.IMPLICIT_LINKED_HASH_MULTI_SET_CLASS); implementedInterfaces.add("ImplicitLinkedHashMultiSet.Element"); diff --git a/buildSrc/src/main/java/org/apache/kafka/message/MessageGenerator.java b/buildSrc/src/main/java/org/apache/kafka/message/MessageGenerator.java index 20012a3619ae7..2abc7046f00da 100644 --- a/buildSrc/src/main/java/org/apache/kafka/message/MessageGenerator.java +++ b/buildSrc/src/main/java/org/apache/kafka/message/MessageGenerator.java @@ -43,7 +43,9 @@ public final class MessageGenerator { static final String JAVA_SUFFIX = ".java"; - static final String MESSAGE_FACTORY_JAVA = "MessageFactory.java"; + static final String API_MESSAGE_FACTORY_JAVA = "ApiMessageFactory.java"; + + static final String API_MESSAGE_CLASS = "org.apache.kafka.common.protocol.ApiMessage"; static final String MESSAGE_CLASS = "org.apache.kafka.common.protocol.Message"; @@ -100,7 +102,7 @@ public static Map getOutputFiles(String outputDir, String inputDir File outputFile = new File(outputDir, javaName); outputFiles.put(outputFile.toString(), outputFile); } - File factoryFile = new File(outputDir, MESSAGE_FACTORY_JAVA); + File factoryFile = new File(outputDir, API_MESSAGE_FACTORY_JAVA); outputFiles.put(factoryFile.toString(), factoryFile); return outputFiles; } @@ -108,7 +110,7 @@ public static Map getOutputFiles(String outputDir, String inputDir public static void processDirectories(String outputDir, String inputDir) throws Exception { Files.createDirectories(Paths.get(outputDir)); int numProcessed = 0; - MessageFactoryGenerator messageFactoryGenerator = new MessageFactoryGenerator(); + ApiMessageFactoryGenerator messageFactoryGenerator = new ApiMessageFactoryGenerator(); HashSet outputFileNames = new HashSet<>(); for (Path inputPath : Files.newDirectoryStream(Paths.get(inputDir), JSON_GLOB)) { try { @@ -128,8 +130,8 @@ public static void processDirectories(String outputDir, String inputDir) throws throw new RuntimeException("Exception while processing " + inputPath.toString(), e); } } - Path factoryOutputPath = Paths.get(outputDir, MESSAGE_FACTORY_JAVA); - outputFileNames.add(MESSAGE_FACTORY_JAVA); + Path factoryOutputPath = Paths.get(outputDir, API_MESSAGE_FACTORY_JAVA); + outputFileNames.add(API_MESSAGE_FACTORY_JAVA); try (BufferedWriter writer = Files.newBufferedWriter(factoryOutputPath)) { messageFactoryGenerator.generate(); messageFactoryGenerator.write(writer); diff --git a/buildSrc/src/main/java/org/apache/kafka/message/MessageSpec.java b/buildSrc/src/main/java/org/apache/kafka/message/MessageSpec.java index 8c03da16e0572..9e3eb38e90d3f 100644 --- a/buildSrc/src/main/java/org/apache/kafka/message/MessageSpec.java +++ b/buildSrc/src/main/java/org/apache/kafka/message/MessageSpec.java @@ -22,11 +22,12 @@ import java.util.List; import java.util.Objects; +import java.util.Optional; public final class MessageSpec { private final StructSpec struct; - private final short apiKey; + private final Optional apiKey; private final MessageSpecType type; @@ -37,7 +38,7 @@ public MessageSpec(@JsonProperty("name") String name, @JsonProperty("apiKey") Short apiKey, @JsonProperty("type") MessageSpecType type) { this.struct = new StructSpec(name, validVersions, fields); - this.apiKey = apiKey == null ? -1 : apiKey; + this.apiKey = apiKey == null ? Optional.empty() : Optional.of(apiKey); this.type = Objects.requireNonNull(type); } @@ -61,7 +62,7 @@ public List fields() { } @JsonProperty("apiKey") - public short apiKey() { + public Optional apiKey() { return apiKey; } diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/ApiMessage.java b/clients/src/main/java/org/apache/kafka/common/protocol/ApiMessage.java new file mode 100644 index 0000000000000..4f175653bf416 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/common/protocol/ApiMessage.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.common.protocol; + +/** + * A Message which is part of the top-level Kafka API. + */ +public interface ApiMessage extends Message { + /** + * Returns the API key of this message, or -1 if there is none. + */ + short apiKey(); +} diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/Message.java b/clients/src/main/java/org/apache/kafka/common/protocol/Message.java index 023c0ee322abe..885c69239c570 100644 --- a/clients/src/main/java/org/apache/kafka/common/protocol/Message.java +++ b/clients/src/main/java/org/apache/kafka/common/protocol/Message.java @@ -24,11 +24,6 @@ * Messages also implement toString, equals, and hashCode. */ public interface Message { - /** - * Returns the API key of this message, or -1 if there is none. - */ - short apiKey(); - /** * Returns the lowest supported API key of this message, inclusive. */ diff --git a/clients/src/main/resources/common/message/RequestHeader.json b/clients/src/main/resources/common/message/RequestHeader.json index d24dcf0e98755..8fb60d592ae73 100644 --- a/clients/src/main/resources/common/message/RequestHeader.json +++ b/clients/src/main/resources/common/message/RequestHeader.json @@ -14,6 +14,7 @@ // limitations under the License. { + "apiKey": -1, "type": "header", "name": "RequestHeader", "validVersions": "0", diff --git a/clients/src/main/resources/common/message/ResponseHeader.json b/clients/src/main/resources/common/message/ResponseHeader.json index d44825977da7d..fabef10b6465b 100644 --- a/clients/src/main/resources/common/message/ResponseHeader.json +++ b/clients/src/main/resources/common/message/ResponseHeader.json @@ -14,6 +14,7 @@ // limitations under the License. { + "apiKey": -1, "type": "header", "name": "ResponseHeader", "validVersions": "0", diff --git a/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java b/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java index 1db68b308bd8d..26ace89e4ba26 100644 --- a/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java +++ b/clients/src/test/java/org/apache/kafka/common/message/MessageTest.java @@ -128,7 +128,7 @@ public void testMessageVersions() throws Exception { for (ApiKeys apiKey : ApiKeys.values()) { Message message = null; try { - message = MessageFactory.newRequest(apiKey.id); + message = ApiMessageFactory.newRequest(apiKey.id); } catch (UnsupportedVersionException e) { fail("No request message spec found for API " + apiKey); } @@ -136,7 +136,7 @@ public void testMessageVersions() throws Exception { "supports versions up to " + message.highestSupportedVersion(), apiKey.latestVersion() <= message.highestSupportedVersion()); try { - message = MessageFactory.newResponse(apiKey.id); + message = ApiMessageFactory.newResponse(apiKey.id); } catch (UnsupportedVersionException e) { fail("No response message spec found for API " + apiKey); } @@ -153,7 +153,7 @@ public void testMessageVersions() throws Exception { public void testRequestSchemas() throws Exception { for (ApiKeys apiKey : ApiKeys.values()) { Schema[] manualSchemas = apiKey.requestSchemas; - Schema[] generatedSchemas = MessageFactory.requestSchemas(apiKey.id); + Schema[] generatedSchemas = ApiMessageFactory.requestSchemas(apiKey.id); Assert.assertEquals("Mismatching request SCHEMAS lengths " + "for api key " + apiKey, manualSchemas.length, generatedSchemas.length); for (int v = 0; v < manualSchemas.length; v++) { @@ -176,7 +176,7 @@ public void testRequestSchemas() throws Exception { public void testResponseSchemas() throws Exception { for (ApiKeys apiKey : ApiKeys.values()) { Schema[] manualSchemas = apiKey.responseSchemas; - Schema[] generatedSchemas = MessageFactory.responseSchemas(apiKey.id); + Schema[] generatedSchemas = ApiMessageFactory.responseSchemas(apiKey.id); Assert.assertEquals("Mismatching response SCHEMAS lengths " + "for api key " + apiKey, manualSchemas.length, generatedSchemas.length); for (int v = 0; v < manualSchemas.length; v++) { From aedc370e01c325442ba3a78e2ed57b90d3817ce8 Mon Sep 17 00:00:00 2001 From: "Colin P. Mccabe" Date: Thu, 10 Jan 2019 15:47:36 -0800 Subject: [PATCH 11/14] StructSpec: stylistic improvement --- buildSrc/src/main/java/org/apache/kafka/message/StructSpec.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildSrc/src/main/java/org/apache/kafka/message/StructSpec.java b/buildSrc/src/main/java/org/apache/kafka/message/StructSpec.java index cd24ea99b4356..ccc568946f1f0 100644 --- a/buildSrc/src/main/java/org/apache/kafka/message/StructSpec.java +++ b/buildSrc/src/main/java/org/apache/kafka/message/StructSpec.java @@ -46,7 +46,7 @@ public StructSpec(@JsonProperty("name") String name, } this.fields = Collections.unmodifiableList(fields == null ? Collections.emptyList() : new ArrayList<>(fields)); - this.hasKeys = fields == null ? false : fields.stream().anyMatch(f -> f.mapKey()); + this.hasKeys = fields == null && fields.stream().anyMatch(f -> f.mapKey()); } @JsonProperty From 744b972270cd6191f412e71f0ed3ea2fa998290e Mon Sep 17 00:00:00 2001 From: "Colin P. Mccabe" Date: Thu, 10 Jan 2019 15:47:43 -0800 Subject: [PATCH 12/14] Versions: add JavaDoc --- .../org/apache/kafka/message/Versions.java | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/buildSrc/src/main/java/org/apache/kafka/message/Versions.java b/buildSrc/src/main/java/org/apache/kafka/message/Versions.java index 3f58bd301c81d..1325836c9d09d 100644 --- a/buildSrc/src/main/java/org/apache/kafka/message/Versions.java +++ b/buildSrc/src/main/java/org/apache/kafka/message/Versions.java @@ -19,8 +19,22 @@ import java.util.Objects; +/** + * A version range. + * + * A range consists of two 16-bit numbers: the lowest version which is accepted, and the highest. + * Ranges are inclusive, meaning that both the lowest and the highest version are valid versions. + * The only exception to this is the NONE range, which contains no versions at all. + * + * Version ranges can be represented as strings. + * + * A single supported version V is represented as "V". + * A bounded range from A to B is represented as "A-B". + * All versions greater than A is represented as "A+". + * The NONE range is represented as an the string "none". + */ public final class Versions { - private final short lowest; + private final short lowest ; private final short highest; public static Versions parse(String input, Versions defaultVersions) { @@ -57,6 +71,12 @@ public static Versions parse(String input, Versions defaultVersions) { public static final String NONE_STRING = "none"; public Versions(short lowest, short highest) { + if ((NONE.lowest != lowest) || (NONE.highest != highest)) { + if ((lowest < 0) || (highest < 0)) { + throw new RuntimeException("Invalid negative version range " + + lowest + " to " + highest); + } + } this.lowest = lowest; this.highest = highest; } From d0957a17309da1d071b4502962895f017d3bf989 Mon Sep 17 00:00:00 2001 From: "Colin P. Mccabe" Date: Thu, 10 Jan 2019 16:42:18 -0800 Subject: [PATCH 13/14] Don't embed Struct into FieldSpec --- .../org/apache/kafka/message/FieldSpec.java | 46 +++++++++++++------ .../kafka/message/MessageDataGenerator.java | 34 +++++++------- .../apache/kafka/message/SchemaGenerator.java | 8 ++-- .../org/apache/kafka/message/StructSpec.java | 2 +- .../org/apache/kafka/message/Versions.java | 17 ++++--- 5 files changed, 63 insertions(+), 44 deletions(-) diff --git a/buildSrc/src/main/java/org/apache/kafka/message/FieldSpec.java b/buildSrc/src/main/java/org/apache/kafka/message/FieldSpec.java index 4ea13fac6d3f6..76ea12adb8c3d 100644 --- a/buildSrc/src/main/java/org/apache/kafka/message/FieldSpec.java +++ b/buildSrc/src/main/java/org/apache/kafka/message/FieldSpec.java @@ -20,11 +20,17 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Objects; public final class FieldSpec { - private final StructSpec struct; + private final String name; + + private final Versions versions; + + private final List fields; private final FieldType type; @@ -48,7 +54,14 @@ public FieldSpec(@JsonProperty("name") String name, @JsonProperty("default") String fieldDefault, @JsonProperty("ignorable") boolean ignorable, @JsonProperty("about") String about) { - this.struct = new StructSpec(name, versions, fields); + this.name = Objects.requireNonNull(name); + this.versions = Versions.parse(versions, null); + if (this.versions == null) { + throw new RuntimeException("You must specify the version of the " + + name + " structure."); + } + this.fields = Collections.unmodifiableList(fields == null ? + Collections.emptyList() : new ArrayList<>(fields)); this.type = FieldType.parse(Objects.requireNonNull(type)); this.mapKey = mapKey; this.nullableVersions = Versions.parse(nullableVersions, Versions.NONE); @@ -60,42 +73,49 @@ public FieldSpec(@JsonProperty("name") String name, this.fieldDefault = fieldDefault == null ? "" : fieldDefault; this.ignorable = ignorable; this.about = about == null ? "" : about; - if (!this.struct.fields().isEmpty()) { + if (!this.fields().isEmpty()) { if (!this.type.isArray()) { throw new RuntimeException("Non-array field " + name + " cannot have fields"); } } } - public StructSpec struct() { - return struct; + public StructSpec toStruct() { + if ((!this.type.isArray()) && (this.type.isStruct())) { + throw new RuntimeException("Field " + name + " cannot be treated as a structure."); + } + return new StructSpec(name, versions.toString(), fields); } @JsonProperty("name") public String name() { - return struct.name(); + return name; } String capitalizedCamelCaseName() { - return MessageGenerator.capitalizeFirst(struct.name()); + return MessageGenerator.capitalizeFirst(name); } String camelCaseName() { - return MessageGenerator.lowerCaseFirst(struct.name()); + return MessageGenerator.lowerCaseFirst(name); } String snakeCaseName() { - return MessageGenerator.toSnakeCase(struct.name()); + return MessageGenerator.toSnakeCase(name); + } + + public Versions versions() { + return versions; } @JsonProperty("versions") public String versionsString() { - return struct.versionsString(); + return versions.toString(); } @JsonProperty("fields") public List fields() { - return struct.fields(); + return fields; } @JsonProperty("type") @@ -126,10 +146,6 @@ public String defaultString() { return fieldDefault; } - boolean hasKeys() { - return struct.hasKeys(); - } - @JsonProperty("ignorable") public boolean ignorable() { return ignorable; diff --git a/buildSrc/src/main/java/org/apache/kafka/message/MessageDataGenerator.java b/buildSrc/src/main/java/org/apache/kafka/message/MessageDataGenerator.java index 636ca2e64a196..f20e54c216594 100644 --- a/buildSrc/src/main/java/org/apache/kafka/message/MessageDataGenerator.java +++ b/buildSrc/src/main/java/org/apache/kafka/message/MessageDataGenerator.java @@ -141,7 +141,7 @@ private void generateSubclasses(String className, StructSpec struct, FieldType.ArrayType arrayType = (FieldType.ArrayType) field.type(); generateClass(Optional.empty(), arrayType.elementType().toString(), - field.struct(), + field.toStruct(), parentVersions.intersect(struct.versions())); } } @@ -306,7 +306,7 @@ private String fieldAbstractJavaType(FieldSpec field) { return MessageGenerator.capitalizeFirst(field.typeString()); } else if (field.type().isArray()) { FieldType.ArrayType arrayType = (FieldType.ArrayType) field.type(); - if (field.hasKeys()) { + if (field.toStruct().hasKeys()) { headerGenerator.addImport(MessageGenerator.IMPLICIT_LINKED_HASH_MULTI_SET_CLASS); return hashSetType(arrayType.elementType().toString()); } else { @@ -321,7 +321,7 @@ private String fieldAbstractJavaType(FieldSpec field) { private String fieldConcreteJavaType(FieldSpec field) { if (field.type().isArray()) { FieldType.ArrayType arrayType = (FieldType.ArrayType) field.type(); - if (field.hasKeys()) { + if (field.toStruct().hasKeys()) { headerGenerator.addImport(MessageGenerator.IMPLICIT_LINKED_HASH_MULTI_SET_CLASS); return hashSetType(arrayType.elementType().toString()); } else { @@ -407,12 +407,12 @@ private void generateClassReader(String className, StructSpec struct, private void generateFieldReader(FieldSpec field, Versions curVersions) { if (field.type().isArray()) { boolean maybeAbsent = - generateVersionCheck(curVersions, field.struct().versions()); + generateVersionCheck(curVersions, field.versions()); if (!maybeAbsent) { buffer.printf("{%n"); buffer.incrementIndent(); } - boolean hasKeys = field.hasKeys(); + boolean hasKeys = field.toStruct().hasKeys(); buffer.printf("int arrayLength = readable.readInt();%n"); buffer.printf("if (arrayLength < 0) {%n"); buffer.incrementIndent(); @@ -442,7 +442,7 @@ private void generateFieldReader(FieldSpec field, Versions curVersions) { } } else { boolean maybeAbsent = - generateVersionCheck(curVersions, field.struct().versions()); + generateVersionCheck(curVersions, field.versions()); buffer.printf("this.%s = %s;%n", field.camelCaseName(), readFieldFromReadable(field.type())); @@ -503,7 +503,7 @@ private void generateClassFromStruct(String className, StructSpec struct, private void generateFieldFromStruct(FieldSpec field, Versions curVersions) { if (field.type().isArray()) { boolean maybeAbsent = - generateVersionCheck(curVersions, field.struct().versions()); + generateVersionCheck(curVersions, field.versions()); if (!maybeAbsent) { buffer.printf("{%n"); buffer.incrementIndent(); @@ -548,7 +548,7 @@ private void generateFieldFromStruct(FieldSpec field, Versions curVersions) { } } else { boolean maybeAbsent = - generateVersionCheck(curVersions, field.struct().versions()); + generateVersionCheck(curVersions, field.versions()); buffer.printf("this.%s = %s;%n", field.camelCaseName(), readFieldFromStruct(field.type(), field.snakeCaseName())); @@ -659,7 +659,7 @@ private String writeFieldToWritable(FieldType type, boolean nullable, String nam private void generateFieldWriter(FieldSpec field, Versions curVersions) { if (field.type().isArray()) { boolean maybeAbsent = - generateVersionCheck(curVersions, field.struct().versions()); + generateVersionCheck(curVersions, field.versions()); boolean maybeNull = generateNullCheck(curVersions, field); if (maybeNull) { buffer.printf("writable.writeInt(-1);%n"); @@ -688,7 +688,7 @@ private void generateFieldWriter(FieldSpec field, Versions curVersions) { } } else { boolean maybeAbsent = - generateVersionCheck(curVersions, field.struct().versions()); + generateVersionCheck(curVersions, field.versions()); buffer.printf("%s;%n", writeFieldToWritable(field.type(), !field.nullableVersions().empty(), field.camelCaseName())); @@ -740,7 +740,7 @@ private void generateFieldToStruct(FieldSpec field, Versions curVersions) { (field.type() instanceof FieldType.Int64FieldType) || (field.type() instanceof FieldType.StringFieldType)) { boolean maybeAbsent = - generateVersionCheck(curVersions, field.struct().versions()); + generateVersionCheck(curVersions, field.versions()); buffer.printf("struct.set(\"%s\", this.%s);%n", field.snakeCaseName(), field.camelCaseName()); if (maybeAbsent) { @@ -749,7 +749,7 @@ private void generateFieldToStruct(FieldSpec field, Versions curVersions) { } } else if (field.type().isBytes()) { boolean maybeAbsent = - generateVersionCheck(curVersions, field.struct().versions()); + generateVersionCheck(curVersions, field.versions()); buffer.printf("struct.setByteArray(\"%s\", this.%s);%n", field.snakeCaseName(), field.camelCaseName()); if (maybeAbsent) { @@ -758,7 +758,7 @@ private void generateFieldToStruct(FieldSpec field, Versions curVersions) { } } else if (field.type().isArray()) { boolean maybeAbsent = - generateVersionCheck(curVersions, field.struct().versions()); + generateVersionCheck(curVersions, field.versions()); if (!maybeAbsent) { buffer.printf("{%n"); buffer.incrementIndent(); @@ -860,7 +860,7 @@ private void generateVariableLengthFieldSize(String fieldName, FieldType type, b private void generateFieldSize(FieldSpec field, Versions curVersions) { if (field.type().fixedLength().isPresent()) { boolean maybeAbsent = - generateVersionCheck(curVersions, field.struct().versions()); + generateVersionCheck(curVersions, field.versions()); buffer.printf("size += %d;%n", field.type().fixedLength().get()); if (maybeAbsent) { buffer.decrementIndent(); @@ -869,7 +869,7 @@ private void generateFieldSize(FieldSpec field, Versions curVersions) { } else if (field.type().isString() || field.type().isBytes() || field.type().isStruct()) { boolean nullable = !curVersions.intersect(field.nullableVersions()).empty(); boolean maybeAbsent = - generateVersionCheck(curVersions, field.struct().versions()); + generateVersionCheck(curVersions, field.versions()); generateVariableLengthFieldSize(field.camelCaseName(), field.type(), nullable); if (maybeAbsent) { buffer.decrementIndent(); @@ -877,7 +877,7 @@ private void generateFieldSize(FieldSpec field, Versions curVersions) { } } else if (field.type().isArray()) { boolean maybeAbsent = - generateVersionCheck(curVersions, field.struct().versions()); + generateVersionCheck(curVersions, field.versions()); boolean maybeNull = generateNullCheck(curVersions, field); if (maybeNull) { buffer.printf("size += 4;%n"); @@ -1218,7 +1218,7 @@ private String fieldDefault(FieldSpec field) { field.name() + ": custom defaults are not supported for array fields."); } FieldType.ArrayType arrayType = (FieldType.ArrayType) field.type(); - if (field.hasKeys()) { + if (field.toStruct().hasKeys()) { return "new " + hashSetType(arrayType.elementType().toString()) + "(0)"; } else { headerGenerator.addImport(MessageGenerator.ARRAYLIST_CLASS); diff --git a/buildSrc/src/main/java/org/apache/kafka/message/SchemaGenerator.java b/buildSrc/src/main/java/org/apache/kafka/message/SchemaGenerator.java index 29baf8c75897c..e89599da0b2b0 100644 --- a/buildSrc/src/main/java/org/apache/kafka/message/SchemaGenerator.java +++ b/buildSrc/src/main/java/org/apache/kafka/message/SchemaGenerator.java @@ -85,9 +85,9 @@ void generateSchemas(String className, StructSpec struct, for (FieldSpec field : struct.fields()) { if (field.type().isStructArray()) { FieldType.ArrayType arrayType = (FieldType.ArrayType) field.type(); - generateSchemas(arrayType.elementType().toString(), field.struct(), versions); + generateSchemas(arrayType.elementType().toString(), field.toStruct(), versions); } else if (field.type().isStruct()) { - generateSchemas(field.type().toString(), field.struct(), versions); + generateSchemas(field.type().toString(), field.toStruct(), versions); } } CodeBuffer prev = null; @@ -112,7 +112,7 @@ private void generateSchemaForVersion(StructSpec struct, short version, break; } FieldSpec field = struct.fields().get(lastValidIndex); - if (field.struct().versions().contains(version)) { + if (field.versions().contains(version)) { break; } lastValidIndex--; @@ -123,7 +123,7 @@ private void generateSchemaForVersion(StructSpec struct, short version, buffer.incrementIndent(); for (int i = 0; i <= lastValidIndex; i++) { FieldSpec field = struct.fields().get(i); - if (!field.struct().versions().contains(version)) { + if (!field.versions().contains(version)) { continue; } headerGenerator.addImport(MessageGenerator.FIELD_CLASS); diff --git a/buildSrc/src/main/java/org/apache/kafka/message/StructSpec.java b/buildSrc/src/main/java/org/apache/kafka/message/StructSpec.java index ccc568946f1f0..a775e77ba27d8 100644 --- a/buildSrc/src/main/java/org/apache/kafka/message/StructSpec.java +++ b/buildSrc/src/main/java/org/apache/kafka/message/StructSpec.java @@ -46,7 +46,7 @@ public StructSpec(@JsonProperty("name") String name, } this.fields = Collections.unmodifiableList(fields == null ? Collections.emptyList() : new ArrayList<>(fields)); - this.hasKeys = fields == null && fields.stream().anyMatch(f -> f.mapKey()); + this.hasKeys = this.fields.stream().anyMatch(f -> f.mapKey()); } @JsonProperty diff --git a/buildSrc/src/main/java/org/apache/kafka/message/Versions.java b/buildSrc/src/main/java/org/apache/kafka/message/Versions.java index 1325836c9d09d..196540b088257 100644 --- a/buildSrc/src/main/java/org/apache/kafka/message/Versions.java +++ b/buildSrc/src/main/java/org/apache/kafka/message/Versions.java @@ -34,7 +34,7 @@ * The NONE range is represented as an the string "none". */ public final class Versions { - private final short lowest ; + private final short lowest; private final short highest; public static Versions parse(String input, Versions defaultVersions) { @@ -66,16 +66,19 @@ public static Versions parse(String input, Versions defaultVersions) { public static final Versions ALL = new Versions((short) 0, Short.MAX_VALUE); - public static final Versions NONE = new Versions((short) 0, (short) -1); + public static final Versions NONE = new Versions(); public static final String NONE_STRING = "none"; + private Versions() { + this.lowest = 0; + this.highest = -1; + } + public Versions(short lowest, short highest) { - if ((NONE.lowest != lowest) || (NONE.highest != highest)) { - if ((lowest < 0) || (highest < 0)) { - throw new RuntimeException("Invalid negative version range " + - lowest + " to " + highest); - } + if ((lowest < 0) || (highest < 0)) { + throw new RuntimeException("Invalid version range " + + lowest + " to " + highest); } this.lowest = lowest; this.highest = highest; From 6d8de7a5c2839b05e6d3143f9ef0e31832234abf Mon Sep 17 00:00:00 2001 From: "Colin P. Mccabe" Date: Fri, 11 Jan 2019 13:21:19 -0800 Subject: [PATCH 14/14] Cleanups --- .../java/org/apache/kafka/message/MessageDataGenerator.java | 2 +- clients/src/main/resources/common/message/RequestHeader.json | 1 - clients/src/main/resources/common/message/ResponseHeader.json | 1 - 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/buildSrc/src/main/java/org/apache/kafka/message/MessageDataGenerator.java b/buildSrc/src/main/java/org/apache/kafka/message/MessageDataGenerator.java index f20e54c216594..1af0ce284652f 100644 --- a/buildSrc/src/main/java/org/apache/kafka/message/MessageDataGenerator.java +++ b/buildSrc/src/main/java/org/apache/kafka/message/MessageDataGenerator.java @@ -73,7 +73,7 @@ private void generateClass(Optional topLevelMessageSpec, generateClassConstructors(className, struct); buffer.printf("%n"); if (isTopLevel) { - generateShortAccessor("apiKey", topLevelMessageSpec.get().apiKey().get()); + generateShortAccessor("apiKey", topLevelMessageSpec.get().apiKey().orElse((short) -1)); } buffer.printf("%n"); generateShortAccessor("lowestSupportedVersion", parentVersions.lowest()); diff --git a/clients/src/main/resources/common/message/RequestHeader.json b/clients/src/main/resources/common/message/RequestHeader.json index 8fb60d592ae73..d24dcf0e98755 100644 --- a/clients/src/main/resources/common/message/RequestHeader.json +++ b/clients/src/main/resources/common/message/RequestHeader.json @@ -14,7 +14,6 @@ // limitations under the License. { - "apiKey": -1, "type": "header", "name": "RequestHeader", "validVersions": "0", diff --git a/clients/src/main/resources/common/message/ResponseHeader.json b/clients/src/main/resources/common/message/ResponseHeader.json index fabef10b6465b..d44825977da7d 100644 --- a/clients/src/main/resources/common/message/ResponseHeader.json +++ b/clients/src/main/resources/common/message/ResponseHeader.json @@ -14,7 +14,6 @@ // limitations under the License. { - "apiKey": -1, "type": "header", "name": "ResponseHeader", "validVersions": "0",