From 696e7987b54bf265b23e8a9755af41a1173c8150 Mon Sep 17 00:00:00 2001 From: Jason Gustafson Date: Thu, 4 Feb 2021 10:04:17 -0800 Subject: [PATCH 1/7] KAFKA-12278; Define api scope --- checkstyle/import-control.xml | 2 + .../apache/kafka/clients/NodeApiVersions.java | 10 +- .../kafka/common/network/ChannelBuilders.java | 16 +- .../common/network/SaslChannelBuilder.java | 12 +- .../apache/kafka/common/protocol/ApiKeys.java | 53 +++--- .../kafka/common/protocol/Protocol.java | 2 +- .../common/requests/ApiVersionsResponse.java | 85 ++++++--- .../SaslServerAuthenticator.java | 13 +- .../message/AddOffsetsToTxnRequest.json | 1 + .../message/AddPartitionsToTxnRequest.json | 1 + .../message/AlterClientQuotasRequest.json | 1 + .../common/message/AlterConfigsRequest.json | 1 + .../common/message/AlterIsrRequest.json | 1 + .../AlterPartitionReassignmentsRequest.json | 1 + .../message/AlterReplicaLogDirsRequest.json | 1 + .../AlterUserScramCredentialsRequest.json | 1 + .../common/message/ApiVersionsRequest.json | 1 + .../message/BeginQuorumEpochRequest.json | 1 + .../message/BrokerHeartbeatRequest.json | 1 + .../message/BrokerRegistrationRequest.json | 1 + .../message/ControlledShutdownRequest.json | 1 + .../common/message/CreateAclsRequest.json | 1 + .../message/CreateDelegationTokenRequest.json | 1 + .../message/CreatePartitionsRequest.json | 1 + .../common/message/CreateTopicsRequest.json | 1 + .../common/message/DeleteAclsRequest.json | 1 + .../common/message/DeleteGroupsRequest.json | 1 + .../common/message/DeleteRecordsRequest.json | 1 + .../common/message/DeleteTopicsRequest.json | 1 + .../common/message/DescribeAclsRequest.json | 1 + .../message/DescribeClientQuotasRequest.json | 1 + .../message/DescribeClusterRequest.json | 1 + .../message/DescribeConfigsRequest.json | 1 + .../DescribeDelegationTokenRequest.json | 1 + .../common/message/DescribeGroupsRequest.json | 1 + .../message/DescribeLogDirsRequest.json | 1 + .../message/DescribeProducersRequest.json | 1 + .../common/message/DescribeQuorumRequest.json | 1 + .../DescribeUserScramCredentialsRequest.json | 1 + .../common/message/ElectLeadersRequest.json | 1 + .../common/message/EndQuorumEpochRequest.json | 1 + .../common/message/EndTxnRequest.json | 1 + .../common/message/EnvelopeRequest.json | 1 + .../message/ExpireDelegationTokenRequest.json | 1 + .../common/message/FetchRequest.json | 1 + .../common/message/FetchSnapshotRequest.json | 1 + .../message/FindCoordinatorRequest.json | 1 + .../common/message/HeartbeatRequest.json | 1 + .../IncrementalAlterConfigsRequest.json | 1 + .../common/message/InitProducerIdRequest.json | 1 + .../common/message/JoinGroupRequest.json | 1 + .../common/message/LeaderAndIsrRequest.json | 1 + .../common/message/LeaveGroupRequest.json | 1 + .../common/message/ListGroupsRequest.json | 1 + .../common/message/ListOffsetsRequest.json | 1 + .../ListPartitionReassignmentsRequest.json | 1 + .../common/message/MetadataRequest.json | 1 + .../common/message/OffsetCommitRequest.json | 1 + .../common/message/OffsetDeleteRequest.json | 1 + .../common/message/OffsetFetchRequest.json | 1 + .../message/OffsetForLeaderEpochRequest.json | 1 + .../common/message/ProduceRequest.json | 1 + .../message/RenewDelegationTokenRequest.json | 1 + .../message/SaslAuthenticateRequest.json | 1 + .../common/message/SaslHandshakeRequest.json | 1 + .../common/message/StopReplicaRequest.json | 1 + .../common/message/SyncGroupRequest.json | 1 + .../message/TxnOffsetCommitRequest.json | 1 + .../message/UnregisterBrokerRequest.json | 1 + .../common/message/UpdateFeaturesRequest.json | 1 + .../common/message/UpdateMetadataRequest.json | 1 + .../resources/common/message/VoteRequest.json | 1 + .../message/WriteTxnMarkersRequest.json | 1 + .../kafka/clients/NetworkClientTest.java | 17 +- .../kafka/clients/NodeApiVersionsTest.java | 36 ++-- .../clients/admin/KafkaAdminClientTest.java | 14 +- .../consumer/internals/FetcherTest.java | 6 +- .../producer/internals/SenderTest.java | 27 +-- .../kafka/common/network/NioEchoServer.java | 5 +- .../network/SaslChannelBuilderTest.java | 14 +- .../common/network/SslTransportLayerTest.java | 24 ++- .../kafka/common/protocol/ApiKeysTest.java | 22 ++- .../requests/ApiVersionsResponseTest.java | 51 ++--- .../common/requests/RequestResponseTest.java | 100 +++++----- .../authenticator/SaslAuthenticatorTest.java | 177 ++++++++---------- .../SaslServerAuthenticatorTest.java | 8 +- .../src/main/scala/kafka/api/ApiVersion.scala | 71 ++++--- .../scala/kafka/network/RequestChannel.scala | 17 +- .../scala/kafka/network/SocketServer.scala | 36 ++-- .../kafka/server/ApiVersionManager.scala | 121 ++++++++++++ .../main/scala/kafka/server/KafkaApis.scala | 58 ++---- .../main/scala/kafka/server/KafkaServer.scala | 40 ++-- .../kafka/tools/TestRaftRequestHandler.scala | 8 +- .../scala/kafka/tools/TestRaftServer.scala | 9 +- .../admin/BrokerApiVersionsCommandTest.scala | 2 +- .../server/GssapiAuthenticationTest.scala | 5 +- .../scala/unit/kafka/api/ApiVersionTest.scala | 23 ++- .../unit/kafka/network/SocketServerTest.scala | 61 +++--- .../AbstractApiVersionsRequestTest.scala | 15 +- .../kafka/server/ApiVersionManagerTest.scala | 104 ++++++++++ .../kafka/server/ApiVersionsRequestTest.scala | 8 +- .../kafka/server/ForwardingManagerTest.scala | 3 +- .../unit/kafka/server/KafkaApisTest.scala | 84 +-------- .../unit/kafka/server/RequestQuotaTest.scala | 6 +- .../server/SaslApiVersionsRequestTest.scala | 4 +- .../message/ApiMessageTypeGenerator.java | 69 ++++++- .../kafka/message/MessageGenerator.java | 6 +- .../org/apache/kafka/message/MessageSpec.java | 16 +- .../apache/kafka/message/RequestApiScope.java | 33 ++++ 109 files changed, 998 insertions(+), 560 deletions(-) create mode 100644 core/src/main/scala/kafka/server/ApiVersionManager.scala create mode 100644 core/src/test/scala/unit/kafka/server/ApiVersionManagerTest.scala create mode 100644 generator/src/main/java/org/apache/kafka/message/RequestApiScope.java diff --git a/checkstyle/import-control.xml b/checkstyle/import-control.xml index 9ec16b9b7c096..aad58b0ddd8cd 100644 --- a/checkstyle/import-control.xml +++ b/checkstyle/import-control.xml @@ -50,6 +50,7 @@ + @@ -108,6 +109,7 @@ + diff --git a/clients/src/main/java/org/apache/kafka/clients/NodeApiVersions.java b/clients/src/main/java/org/apache/kafka/clients/NodeApiVersions.java index 658d481308ead..3c09f0eb4e781 100644 --- a/clients/src/main/java/org/apache/kafka/clients/NodeApiVersions.java +++ b/clients/src/main/java/org/apache/kafka/clients/NodeApiVersions.java @@ -16,9 +16,6 @@ */ package org.apache.kafka.clients; -import java.util.Collection; -import java.util.Collections; -import java.util.LinkedList; import org.apache.kafka.common.errors.UnsupportedVersionException; import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersion; import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersionCollection; @@ -27,7 +24,10 @@ import org.apache.kafka.common.utils.Utils; import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; import java.util.EnumMap; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Optional; @@ -62,7 +62,7 @@ public static NodeApiVersions create() { */ public static NodeApiVersions create(Collection overrides) { List apiVersions = new LinkedList<>(overrides); - for (ApiKeys apiKey : ApiKeys.brokerApis()) { + for (ApiKeys apiKey : ApiKeys.zkBrokerApis()) { boolean exists = false; for (ApiVersion apiVersion : apiVersions) { if (apiVersion.apiKey() == apiKey.id) { @@ -170,7 +170,7 @@ public String toString(boolean lineBreaks) { // Also handle the case where some apiKey types are not specified at all in the given ApiVersions, // which may happen when the remote is too old. - for (ApiKeys apiKey : ApiKeys.brokerApis()) { + for (ApiKeys apiKey : ApiKeys.zkBrokerApis()) { if (!apiKeysText.containsKey(apiKey.id)) { StringBuilder bld = new StringBuilder(); bld.append(apiKey.name).append("("). diff --git a/clients/src/main/java/org/apache/kafka/common/network/ChannelBuilders.java b/clients/src/main/java/org/apache/kafka/common/network/ChannelBuilders.java index ee5ed75828920..b4a1ce87cf1cd 100644 --- a/clients/src/main/java/org/apache/kafka/common/network/ChannelBuilders.java +++ b/clients/src/main/java/org/apache/kafka/common/network/ChannelBuilders.java @@ -21,6 +21,7 @@ import org.apache.kafka.common.config.SslClientAuth; import org.apache.kafka.common.config.internals.BrokerSecurityConfigs; import org.apache.kafka.common.errors.InvalidConfigurationException; +import org.apache.kafka.common.requests.ApiVersionsResponse; import org.apache.kafka.common.security.auth.SecurityProtocol; import org.apache.kafka.common.security.JaasContext; import org.apache.kafka.common.security.authenticator.DefaultKafkaPrincipalBuilder; @@ -40,6 +41,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.function.Supplier; public class ChannelBuilders { private static final Logger log = LoggerFactory.getLogger(ChannelBuilders.class); @@ -77,7 +79,7 @@ public static ChannelBuilder clientChannelBuilder( throw new IllegalArgumentException("`clientSaslMechanism` must be non-null in client mode if `securityProtocol` is `" + securityProtocol + "`"); } return create(securityProtocol, Mode.CLIENT, contextType, config, listenerName, false, clientSaslMechanism, - saslHandshakeRequestEnable, null, null, time, logContext); + saslHandshakeRequestEnable, null, null, time, logContext, null); } /** @@ -89,6 +91,7 @@ public static ChannelBuilder clientChannelBuilder( * @param tokenCache Delegation token cache * @param time the time instance * @param logContext the log context instance + * @param apiVersionSupplier supplier for ApiVersions responses sent prior to authentication * * @return the configured `ChannelBuilder` */ @@ -99,10 +102,11 @@ public static ChannelBuilder serverChannelBuilder(ListenerName listenerName, CredentialCache credentialCache, DelegationTokenCache tokenCache, Time time, - LogContext logContext) { + LogContext logContext, + Supplier apiVersionSupplier) { return create(securityProtocol, Mode.SERVER, JaasContext.Type.SERVER, config, listenerName, isInterBrokerListener, null, true, credentialCache, - tokenCache, time, logContext); + tokenCache, time, logContext, apiVersionSupplier); } private static ChannelBuilder create(SecurityProtocol securityProtocol, @@ -116,7 +120,8 @@ private static ChannelBuilder create(SecurityProtocol securityProtocol, CredentialCache credentialCache, DelegationTokenCache tokenCache, Time time, - LogContext logContext) { + LogContext logContext, + Supplier apiVersionSupplier) { Map configs = channelBuilderConfigs(config, listenerName); ChannelBuilder channelBuilder; @@ -174,7 +179,8 @@ private static ChannelBuilder create(SecurityProtocol securityProtocol, tokenCache, sslClientAuthOverride, time, - logContext); + logContext, + apiVersionSupplier); break; case PLAINTEXT: channelBuilder = new PlaintextChannelBuilder(listenerName); diff --git a/clients/src/main/java/org/apache/kafka/common/network/SaslChannelBuilder.java b/clients/src/main/java/org/apache/kafka/common/network/SaslChannelBuilder.java index 900162614d998..17988db87a650 100644 --- a/clients/src/main/java/org/apache/kafka/common/network/SaslChannelBuilder.java +++ b/clients/src/main/java/org/apache/kafka/common/network/SaslChannelBuilder.java @@ -21,6 +21,7 @@ import org.apache.kafka.common.config.SslConfigs; import org.apache.kafka.common.config.internals.BrokerSecurityConfigs; import org.apache.kafka.common.memory.MemoryPool; +import org.apache.kafka.common.requests.ApiVersionsResponse; import org.apache.kafka.common.security.JaasContext; import org.apache.kafka.common.security.auth.AuthenticateCallbackHandler; import org.apache.kafka.common.security.auth.Login; @@ -85,6 +86,7 @@ public class SaslChannelBuilder implements ChannelBuilder, ListenerReconfigurabl private final DelegationTokenCache tokenCache; private final Map loginManagers; private final Map subjects; + private final Supplier apiVersionSupplier; private SslFactory sslFactory; private Map configs; @@ -108,7 +110,8 @@ public SaslChannelBuilder(Mode mode, DelegationTokenCache tokenCache, String sslClientAuthOverride, Time time, - LogContext logContext) { + LogContext logContext, + Supplier apiVersionSupplier) { this.mode = mode; this.jaasContexts = jaasContexts; this.loginManagers = new HashMap<>(jaasContexts.size()); @@ -126,6 +129,11 @@ public SaslChannelBuilder(Mode mode, this.time = time; this.logContext = logContext; this.log = logContext.logger(getClass()); + this.apiVersionSupplier = apiVersionSupplier; + + if (mode == Mode.SERVER && apiVersionSupplier == null) { + throw new IllegalArgumentException("Server channel builder must provide an ApiVersionResponse supplier"); + } } @SuppressWarnings("unchecked") @@ -266,7 +274,7 @@ protected SaslServerAuthenticator buildServerAuthenticator(Map config ChannelMetadataRegistry metadataRegistry) { return new SaslServerAuthenticator(configs, callbackHandlers, id, subjects, kerberosShortNamer, listenerName, securityProtocol, transportLayer, - connectionsMaxReauthMsByMechanism, metadataRegistry, time); + connectionsMaxReauthMsByMechanism, metadataRegistry, time, apiVersionSupplier); } // Visible to override for testing diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java b/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java index 49a6130a30079..038027096c807 100644 --- a/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java +++ b/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java @@ -21,7 +21,9 @@ import org.apache.kafka.common.protocol.types.Type; import org.apache.kafka.common.record.RecordBatch; +import java.util.ArrayList; import java.util.Arrays; +import java.util.EnumSet; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; @@ -90,19 +92,19 @@ public enum ApiKeys { ALTER_CLIENT_QUOTAS(ApiMessageType.ALTER_CLIENT_QUOTAS, false, true), DESCRIBE_USER_SCRAM_CREDENTIALS(ApiMessageType.DESCRIBE_USER_SCRAM_CREDENTIALS), ALTER_USER_SCRAM_CREDENTIALS(ApiMessageType.ALTER_USER_SCRAM_CREDENTIALS, false, true), - VOTE(ApiMessageType.VOTE, true, RecordBatch.MAGIC_VALUE_V0, false, true), - BEGIN_QUORUM_EPOCH(ApiMessageType.BEGIN_QUORUM_EPOCH, true, RecordBatch.MAGIC_VALUE_V0, false, true), - END_QUORUM_EPOCH(ApiMessageType.END_QUORUM_EPOCH, true, RecordBatch.MAGIC_VALUE_V0, false, true), - DESCRIBE_QUORUM(ApiMessageType.DESCRIBE_QUORUM, true, RecordBatch.MAGIC_VALUE_V0, false, true), + VOTE(ApiMessageType.VOTE, true, RecordBatch.MAGIC_VALUE_V0, false), + BEGIN_QUORUM_EPOCH(ApiMessageType.BEGIN_QUORUM_EPOCH, true, RecordBatch.MAGIC_VALUE_V0, false), + END_QUORUM_EPOCH(ApiMessageType.END_QUORUM_EPOCH, true, RecordBatch.MAGIC_VALUE_V0, false), + DESCRIBE_QUORUM(ApiMessageType.DESCRIBE_QUORUM, true, RecordBatch.MAGIC_VALUE_V0, false), ALTER_ISR(ApiMessageType.ALTER_ISR, true), UPDATE_FEATURES(ApiMessageType.UPDATE_FEATURES, false, true), - ENVELOPE(ApiMessageType.ENVELOPE, true, RecordBatch.MAGIC_VALUE_V0, false, true), - FETCH_SNAPSHOT(ApiMessageType.FETCH_SNAPSHOT, false, RecordBatch.MAGIC_VALUE_V0, false, true), + ENVELOPE(ApiMessageType.ENVELOPE, true, RecordBatch.MAGIC_VALUE_V0, false), + FETCH_SNAPSHOT(ApiMessageType.FETCH_SNAPSHOT, false, RecordBatch.MAGIC_VALUE_V0, false), DESCRIBE_CLUSTER(ApiMessageType.DESCRIBE_CLUSTER), DESCRIBE_PRODUCERS(ApiMessageType.DESCRIBE_PRODUCERS), - BROKER_REGISTRATION(ApiMessageType.BROKER_REGISTRATION, true, RecordBatch.MAGIC_VALUE_V0, false, true), - BROKER_HEARTBEAT(ApiMessageType.BROKER_HEARTBEAT, true, RecordBatch.MAGIC_VALUE_V0, false, true), - UNREGISTER_BROKER(ApiMessageType.UNREGISTER_BROKER, false, RecordBatch.MAGIC_VALUE_V0, true, false); + BROKER_REGISTRATION(ApiMessageType.BROKER_REGISTRATION, true, RecordBatch.MAGIC_VALUE_V0, false), + BROKER_HEARTBEAT(ApiMessageType.BROKER_HEARTBEAT, true, RecordBatch.MAGIC_VALUE_V0, false), + UNREGISTER_BROKER(ApiMessageType.UNREGISTER_BROKER, false, RecordBatch.MAGIC_VALUE_V0, true); // The generator ensures every `ApiMessageType` has a unique id private static final Map ID_TO_TYPE = Arrays.stream(ApiKeys.values()) @@ -120,9 +122,6 @@ public enum ApiKeys { /** indicates the minimum required inter broker magic required to support the API */ public final byte minRequiredInterBrokerMagic; - /** indicates whether this is an API which is only exposed by the KIP-500 controller **/ - public final boolean isControllerOnlyApi; - /** indicates whether the API is enabled for forwarding **/ public final boolean forwardable; @@ -142,24 +141,17 @@ public enum ApiKeys { this(messageType, clusterAction, RecordBatch.MAGIC_VALUE_V0, forwardable); } - ApiKeys(ApiMessageType messageType, boolean clusterAction, byte minRequiredInterBrokerMagic, boolean forwardable) { - this(messageType, clusterAction, minRequiredInterBrokerMagic, forwardable, false); - } - ApiKeys( ApiMessageType messageType, boolean clusterAction, byte minRequiredInterBrokerMagic, - boolean forwardable, - boolean isControllerOnlyApi + boolean forwardable ) { this.messageType = messageType; this.id = messageType.apiKey(); this.name = messageType.name; this.clusterAction = clusterAction; this.minRequiredInterBrokerMagic = minRequiredInterBrokerMagic; - this.isControllerOnlyApi = isControllerOnlyApi; - this.requiresDelayedAllocation = forwardable || shouldRetainsBufferReference(messageType.requestSchemas()); this.forwardable = forwardable; } @@ -195,6 +187,14 @@ public short oldestVersion() { return messageType.lowestSupportedVersion(); } + public List allVersions() { + List versions = new ArrayList<>(latestVersion() - oldestVersion() + 1); + for (short version = oldestVersion(); version < latestVersion(); version++) { + versions.add(version); + } + return versions; + } + public boolean isVersionSupported(short apiVersion) { return apiVersion >= oldestVersion() && apiVersion <= latestVersion(); } @@ -214,7 +214,7 @@ private static String toHtml() { b.append("Name\n"); b.append("Key\n"); b.append(""); - for (ApiKeys key : ApiKeys.brokerApis()) { + for (ApiKeys key : zkBrokerApis()) { b.append("\n"); b.append(""); b.append("" + key.name + ""); @@ -246,10 +246,15 @@ public void visit(Type field) { return hasBuffer.get(); } - public static List brokerApis() { - return Arrays.stream(values()) - .filter(api -> !api.isControllerOnlyApi) + public static EnumSet zkBrokerApis() { + return apisInScope(ApiMessageType.ApiScope.ZK_BROKER); + } + + public static EnumSet apisInScope(ApiMessageType.ApiScope scope) { + List controllerApis = Arrays.stream(ApiKeys.values()) + .filter(apiKey -> apiKey.messageType.scope().contains(scope)) .collect(Collectors.toList()); + return EnumSet.copyOf(controllerApis); } } diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/Protocol.java b/clients/src/main/java/org/apache/kafka/common/protocol/Protocol.java index f31c613a8c51b..d455b26eb2d87 100644 --- a/clients/src/main/java/org/apache/kafka/common/protocol/Protocol.java +++ b/clients/src/main/java/org/apache/kafka/common/protocol/Protocol.java @@ -133,7 +133,7 @@ public static String toHtml() { b.append("\n"); schemaToFieldTableHtml(ResponseHeaderData.SCHEMAS[i], b); } - for (ApiKeys key : ApiKeys.brokerApis()) { + for (ApiKeys key : ApiKeys.zkBrokerApis()) { // Key b.append("
"); b.append(""); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsResponse.java index 2bf9360f126d6..b4fadcdfb66f8 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsResponse.java @@ -19,6 +19,7 @@ import org.apache.kafka.common.feature.Features; import org.apache.kafka.common.feature.FinalizedVersionRange; import org.apache.kafka.common.feature.SupportedVersionRange; +import org.apache.kafka.common.message.ApiMessageType; import org.apache.kafka.common.message.ApiVersionsResponseData; import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersion; import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersionCollection; @@ -29,11 +30,12 @@ import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.record.RecordBatch; +import org.apache.kafka.common.record.RecordVersion; import java.nio.ByteBuffer; import java.util.Map; import java.util.Optional; +import java.util.Set; /** * Possible error codes: @@ -44,9 +46,6 @@ public class ApiVersionsResponse extends AbstractResponse { public static final long UNKNOWN_FINALIZED_FEATURES_EPOCH = -1L; - public static final ApiVersionsResponse DEFAULT_API_VERSIONS_RESPONSE = createApiVersionsResponse( - DEFAULT_THROTTLE_TIME, RecordBatch.CURRENT_MAGIC_VALUE); - private final ApiVersionsResponseData data; public ApiVersionsResponse(ApiVersionsResponseData data) { @@ -96,49 +95,85 @@ public static ApiVersionsResponse parse(ByteBuffer buffer, short version) { } } - public static ApiVersionsResponse createApiVersionsResponse(final int throttleTimeMs, final byte minMagic) { - return createApiVersionsResponse(throttleTimeMs, minMagic, Features.emptySupportedFeatures(), - Features.emptyFinalizedFeatures(), UNKNOWN_FINALIZED_FEATURES_EPOCH); + public static ApiVersionsResponse defaultApiVersionsResponse( + ApiMessageType.ApiScope scope + ) { + return defaultApiVersionsResponse(0, scope); } - private static ApiVersionsResponse createApiVersionsResponse( - final int throttleTimeMs, - final byte minMagic, - final Features latestSupportedFeatures, - final Features finalizedFeatures, - final long finalizedFeaturesEpoch) { + public static ApiVersionsResponse defaultApiVersionsResponse( + int throttleTimeMs, + ApiMessageType.ApiScope scope + ) { + return createApiVersionsResponse(throttleTimeMs, filterApis(RecordVersion.current(), scope)); + } + + public static ApiVersionsResponse createApiVersionsResponse( + int throttleTimeMs, + ApiVersionCollection apiVersions + ) { + return createApiVersionsResponse( + throttleTimeMs, + apiVersions, + Features.emptySupportedFeatures(), + Features.emptyFinalizedFeatures(), + UNKNOWN_FINALIZED_FEATURES_EPOCH + ); + } + + public static ApiVersionsResponse createApiVersionsResponse( + int throttleTimeMs, + ApiVersionCollection apiVersions, + Features latestSupportedFeatures, + Features finalizedFeatures, + long finalizedFeaturesEpoch + ) { return new ApiVersionsResponse( createApiVersionsResponseData( throttleTimeMs, Errors.NONE, - defaultApiKeys(minMagic), + apiVersions, latestSupportedFeatures, finalizedFeatures, - finalizedFeaturesEpoch)); + finalizedFeaturesEpoch + ) + ); } - public static ApiVersionCollection defaultApiKeys(final byte minMagic) { + public static ApiVersionCollection filterApis(RecordVersion minRecordVersion, ApiMessageType.ApiScope scope) { ApiVersionCollection apiKeys = new ApiVersionCollection(); - for (ApiKeys apiKey : ApiKeys.brokerApis()) { - if (apiKey.minRequiredInterBrokerMagic <= minMagic) { + for (ApiKeys apiKey : ApiKeys.apisInScope(scope)) { + if (apiKey.minRequiredInterBrokerMagic <= minRecordVersion.value) { apiKeys.add(ApiVersionsResponse.toApiVersion(apiKey)); } } return apiKeys; } + public static ApiVersionCollection collectApis(Set apiKeys) { + ApiVersionCollection res = new ApiVersionCollection(); + for (ApiKeys apiKey : apiKeys) { + res.add(ApiVersionsResponse.toApiVersion(apiKey)); + } + return res; + } + /** - * Find the commonly agreed ApiVersions between local software and the controller. + * Find the common range of supported API versions between the locally + * known range and that of another set. * - * @param minMagic min inter broker magic + * @param minRecordVersion min inter broker magic * @param activeControllerApiVersions controller ApiVersions * @return commonly agreed ApiVersion collection */ - public static ApiVersionCollection intersectControllerApiVersions(final byte minMagic, - final Map activeControllerApiVersions) { + public static ApiVersionCollection intersectForwardableApis( + final ApiMessageType.ApiScope scope, + final RecordVersion minRecordVersion, + final Map activeControllerApiVersions + ) { ApiVersionCollection apiKeys = new ApiVersionCollection(); - for (ApiKeys apiKey : ApiKeys.brokerApis()) { - if (apiKey.minRequiredInterBrokerMagic <= minMagic) { + for (ApiKeys apiKey : ApiKeys.apisInScope(scope)) { + if (apiKey.minRequiredInterBrokerMagic <= minRecordVersion.value) { ApiVersion brokerApiVersion = toApiVersion(apiKey); final ApiVersion finalApiVersion; @@ -161,7 +196,7 @@ public static ApiVersionCollection intersectControllerApiVersions(final byte min return apiKeys; } - public static ApiVersionsResponseData createApiVersionsResponseData( + private static ApiVersionsResponseData createApiVersionsResponseData( final int throttleTimeMs, final Errors error, final ApiVersionCollection apiKeys, diff --git a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticator.java b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticator.java index 7fb5687def1d2..243495da9fe7f 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticator.java +++ b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticator.java @@ -85,6 +85,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.function.Supplier; public class SaslServerAuthenticator implements Authenticator { // GSSAPI limits requests to 64K, but we allow a bit extra for custom SASL mechanisms @@ -127,6 +128,7 @@ private enum SaslState { private final Time time; private final ReauthInfo reauthInfo; private final ChannelMetadataRegistry metadataRegistry; + private final Supplier apiVersionSupplier; // Current SASL state private SaslState saslState = SaslState.INITIAL_REQUEST; @@ -154,7 +156,8 @@ public SaslServerAuthenticator(Map configs, TransportLayer transportLayer, Map connectionsMaxReauthMsByMechanism, ChannelMetadataRegistry metadataRegistry, - Time time) { + Time time, + Supplier apiVersionSupplier) { this.callbackHandlers = callbackHandlers; this.connectionId = connectionId; this.subjects = subjects; @@ -166,6 +169,7 @@ public SaslServerAuthenticator(Map configs, this.time = time; this.reauthInfo = new ReauthInfo(); this.metadataRegistry = metadataRegistry; + this.apiVersionSupplier = apiVersionSupplier; this.configs = configs; @SuppressWarnings("unchecked") @@ -562,11 +566,6 @@ private String handleHandshakeRequest(RequestContext context, SaslHandshakeReque } } - // Visible to override for testing - protected ApiVersionsResponse apiVersionsResponse() { - return ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE; - } - // Visible to override for testing protected void enableKafkaSaslAuthenticateHeaders(boolean flag) { this.enableKafkaSaslAuthenticateHeaders = flag; @@ -583,7 +582,7 @@ else if (!apiVersionsRequest.isValid()) else { metadataRegistry.registerClientInformation(new ClientInformation(apiVersionsRequest.data().clientSoftwareName(), apiVersionsRequest.data().clientSoftwareVersion())); - sendKafkaResponse(context, apiVersionsResponse()); + sendKafkaResponse(context, apiVersionSupplier.get()); setSaslState(SaslState.HANDSHAKE_REQUEST); } } diff --git a/clients/src/main/resources/common/message/AddOffsetsToTxnRequest.json b/clients/src/main/resources/common/message/AddOffsetsToTxnRequest.json index b2bb9a78f6470..eb1c01a767f52 100644 --- a/clients/src/main/resources/common/message/AddOffsetsToTxnRequest.json +++ b/clients/src/main/resources/common/message/AddOffsetsToTxnRequest.json @@ -16,6 +16,7 @@ { "apiKey": 25, "type": "request", + "scope": ["zkBroker", "broker"], "name": "AddOffsetsToTxnRequest", // Version 1 is the same as version 0. // diff --git a/clients/src/main/resources/common/message/AddPartitionsToTxnRequest.json b/clients/src/main/resources/common/message/AddPartitionsToTxnRequest.json index 3e1d7207a51dc..8a52bb8261212 100644 --- a/clients/src/main/resources/common/message/AddPartitionsToTxnRequest.json +++ b/clients/src/main/resources/common/message/AddPartitionsToTxnRequest.json @@ -16,6 +16,7 @@ { "apiKey": 24, "type": "request", + "scope": ["zkBroker", "broker"], "name": "AddPartitionsToTxnRequest", // Version 1 is the same as version 0. // diff --git a/clients/src/main/resources/common/message/AlterClientQuotasRequest.json b/clients/src/main/resources/common/message/AlterClientQuotasRequest.json index 715a00644e9c0..746ac00873514 100644 --- a/clients/src/main/resources/common/message/AlterClientQuotasRequest.json +++ b/clients/src/main/resources/common/message/AlterClientQuotasRequest.json @@ -16,6 +16,7 @@ { "apiKey": 49, "type": "request", + "scope": ["zkBroker", "broker", "controller"], "name": "AlterClientQuotasRequest", "validVersions": "0-1", // Version 1 enables flexible versions. diff --git a/clients/src/main/resources/common/message/AlterConfigsRequest.json b/clients/src/main/resources/common/message/AlterConfigsRequest.json index a1d7d1d4ca4d4..0fb73305b24bb 100644 --- a/clients/src/main/resources/common/message/AlterConfigsRequest.json +++ b/clients/src/main/resources/common/message/AlterConfigsRequest.json @@ -16,6 +16,7 @@ { "apiKey": 33, "type": "request", + "scope": ["zkBroker", "broker", "controller"], "name": "AlterConfigsRequest", // Version 1 is the same as version 0. // Version 2 enables flexible versions. diff --git a/clients/src/main/resources/common/message/AlterIsrRequest.json b/clients/src/main/resources/common/message/AlterIsrRequest.json index 3d5084a4094ec..e766b883150ef 100644 --- a/clients/src/main/resources/common/message/AlterIsrRequest.json +++ b/clients/src/main/resources/common/message/AlterIsrRequest.json @@ -16,6 +16,7 @@ { "apiKey": 56, "type": "request", + "scope": ["zkBroker", "controller"], "name": "AlterIsrRequest", "validVersions": "0", "flexibleVersions": "0+", diff --git a/clients/src/main/resources/common/message/AlterPartitionReassignmentsRequest.json b/clients/src/main/resources/common/message/AlterPartitionReassignmentsRequest.json index ecf7eca6eae43..4ae92da434698 100644 --- a/clients/src/main/resources/common/message/AlterPartitionReassignmentsRequest.json +++ b/clients/src/main/resources/common/message/AlterPartitionReassignmentsRequest.json @@ -16,6 +16,7 @@ { "apiKey": 45, "type": "request", + "scope": ["zkBroker", "broker"], "name": "AlterPartitionReassignmentsRequest", "validVersions": "0", "flexibleVersions": "0+", diff --git a/clients/src/main/resources/common/message/AlterReplicaLogDirsRequest.json b/clients/src/main/resources/common/message/AlterReplicaLogDirsRequest.json index a6749077f4439..e74671d59c7ec 100644 --- a/clients/src/main/resources/common/message/AlterReplicaLogDirsRequest.json +++ b/clients/src/main/resources/common/message/AlterReplicaLogDirsRequest.json @@ -16,6 +16,7 @@ { "apiKey": 34, "type": "request", + "scope": ["zkBroker", "broker"], "name": "AlterReplicaLogDirsRequest", // Version 1 is the same as version 0. // Version 2 enables flexible versions. diff --git a/clients/src/main/resources/common/message/AlterUserScramCredentialsRequest.json b/clients/src/main/resources/common/message/AlterUserScramCredentialsRequest.json index 242bcb97dda79..3b9e6cbf38716 100644 --- a/clients/src/main/resources/common/message/AlterUserScramCredentialsRequest.json +++ b/clients/src/main/resources/common/message/AlterUserScramCredentialsRequest.json @@ -16,6 +16,7 @@ { "apiKey": 51, "type": "request", + "scope": ["zkBroker", "broker", "controller"], "name": "AlterUserScramCredentialsRequest", "validVersions": "0", "flexibleVersions": "0+", diff --git a/clients/src/main/resources/common/message/ApiVersionsRequest.json b/clients/src/main/resources/common/message/ApiVersionsRequest.json index 66e4511a92e93..8213e4411a3c9 100644 --- a/clients/src/main/resources/common/message/ApiVersionsRequest.json +++ b/clients/src/main/resources/common/message/ApiVersionsRequest.json @@ -16,6 +16,7 @@ { "apiKey": 18, "type": "request", + "scope": ["zkBroker", "broker", "controller", "raft"], "name": "ApiVersionsRequest", // Versions 0 through 2 of ApiVersionsRequest are the same. // diff --git a/clients/src/main/resources/common/message/BeginQuorumEpochRequest.json b/clients/src/main/resources/common/message/BeginQuorumEpochRequest.json index fd986097085f6..bae98ba15cc52 100644 --- a/clients/src/main/resources/common/message/BeginQuorumEpochRequest.json +++ b/clients/src/main/resources/common/message/BeginQuorumEpochRequest.json @@ -16,6 +16,7 @@ { "apiKey": 53, "type": "request", + "scope": ["controller", "raft"], "name": "BeginQuorumEpochRequest", "validVersions": "0", "fields": [ diff --git a/clients/src/main/resources/common/message/BrokerHeartbeatRequest.json b/clients/src/main/resources/common/message/BrokerHeartbeatRequest.json index 105a81873b519..3140c5cb9a99d 100644 --- a/clients/src/main/resources/common/message/BrokerHeartbeatRequest.json +++ b/clients/src/main/resources/common/message/BrokerHeartbeatRequest.json @@ -16,6 +16,7 @@ { "apiKey": 63, "type": "request", + "scope": ["controller"], "name": "BrokerHeartbeatRequest", "validVersions": "0", "flexibleVersions": "0+", diff --git a/clients/src/main/resources/common/message/BrokerRegistrationRequest.json b/clients/src/main/resources/common/message/BrokerRegistrationRequest.json index b123cc1f5ac75..061af50046d3d 100644 --- a/clients/src/main/resources/common/message/BrokerRegistrationRequest.json +++ b/clients/src/main/resources/common/message/BrokerRegistrationRequest.json @@ -16,6 +16,7 @@ { "apiKey":62, "type": "request", + "scope": ["controller"], "name": "BrokerRegistrationRequest", "validVersions": "0", "flexibleVersions": "0+", diff --git a/clients/src/main/resources/common/message/ControlledShutdownRequest.json b/clients/src/main/resources/common/message/ControlledShutdownRequest.json index 5756d1c16b5df..1cf371f02eda6 100644 --- a/clients/src/main/resources/common/message/ControlledShutdownRequest.json +++ b/clients/src/main/resources/common/message/ControlledShutdownRequest.json @@ -16,6 +16,7 @@ { "apiKey": 7, "type": "request", + "scope": ["zkBroker", "controller"], "name": "ControlledShutdownRequest", // Version 0 of ControlledShutdownRequest has a non-standard request header // which does not include clientId. Version 1 and later use the standard diff --git a/clients/src/main/resources/common/message/CreateAclsRequest.json b/clients/src/main/resources/common/message/CreateAclsRequest.json index a9bd9c5f60a11..7bae29c1e7447 100644 --- a/clients/src/main/resources/common/message/CreateAclsRequest.json +++ b/clients/src/main/resources/common/message/CreateAclsRequest.json @@ -16,6 +16,7 @@ { "apiKey": 30, "type": "request", + "scope": ["zkBroker", "broker", "controller"], "name": "CreateAclsRequest", // Version 1 adds resource pattern type. // Version 2 enables flexible versions. diff --git a/clients/src/main/resources/common/message/CreateDelegationTokenRequest.json b/clients/src/main/resources/common/message/CreateDelegationTokenRequest.json index 8d881355aa9d2..f73dd9fbdd797 100644 --- a/clients/src/main/resources/common/message/CreateDelegationTokenRequest.json +++ b/clients/src/main/resources/common/message/CreateDelegationTokenRequest.json @@ -16,6 +16,7 @@ { "apiKey": 38, "type": "request", + "scope": ["zkBroker", "broker", "controller"], "name": "CreateDelegationTokenRequest", // Version 1 is the same as version 0. // diff --git a/clients/src/main/resources/common/message/CreatePartitionsRequest.json b/clients/src/main/resources/common/message/CreatePartitionsRequest.json index ba1138801c1ef..4b1c995a9e768 100644 --- a/clients/src/main/resources/common/message/CreatePartitionsRequest.json +++ b/clients/src/main/resources/common/message/CreatePartitionsRequest.json @@ -16,6 +16,7 @@ { "apiKey": 37, "type": "request", + "scope": ["zkBroker", "broker", "controller"], "name": "CreatePartitionsRequest", // Version 1 is the same as version 0. // diff --git a/clients/src/main/resources/common/message/CreateTopicsRequest.json b/clients/src/main/resources/common/message/CreateTopicsRequest.json index 1a8d57a3f0bea..17769459db58d 100644 --- a/clients/src/main/resources/common/message/CreateTopicsRequest.json +++ b/clients/src/main/resources/common/message/CreateTopicsRequest.json @@ -16,6 +16,7 @@ { "apiKey": 19, "type": "request", + "scope": ["zkBroker", "broker", "controller"], "name": "CreateTopicsRequest", // Version 1 adds validateOnly. // diff --git a/clients/src/main/resources/common/message/DeleteAclsRequest.json b/clients/src/main/resources/common/message/DeleteAclsRequest.json index 664737e5810f7..532f413b1547a 100644 --- a/clients/src/main/resources/common/message/DeleteAclsRequest.json +++ b/clients/src/main/resources/common/message/DeleteAclsRequest.json @@ -16,6 +16,7 @@ { "apiKey": 31, "type": "request", + "scope": ["zkBroker", "broker", "controller"], "name": "DeleteAclsRequest", // Version 1 adds the pattern type. // Version 2 enables flexible versions. diff --git a/clients/src/main/resources/common/message/DeleteGroupsRequest.json b/clients/src/main/resources/common/message/DeleteGroupsRequest.json index 833ed7a4dc6d7..b0d068da4021c 100644 --- a/clients/src/main/resources/common/message/DeleteGroupsRequest.json +++ b/clients/src/main/resources/common/message/DeleteGroupsRequest.json @@ -16,6 +16,7 @@ { "apiKey": 42, "type": "request", + "scope": ["zkBroker", "broker"], "name": "DeleteGroupsRequest", // Version 1 is the same as version 0. // diff --git a/clients/src/main/resources/common/message/DeleteRecordsRequest.json b/clients/src/main/resources/common/message/DeleteRecordsRequest.json index 93cbd56d50a6c..16bd9c621a072 100644 --- a/clients/src/main/resources/common/message/DeleteRecordsRequest.json +++ b/clients/src/main/resources/common/message/DeleteRecordsRequest.json @@ -16,6 +16,7 @@ { "apiKey": 21, "type": "request", + "scope": ["zkBroker", "broker"], "name": "DeleteRecordsRequest", // Version 1 is the same as version 0. diff --git a/clients/src/main/resources/common/message/DeleteTopicsRequest.json b/clients/src/main/resources/common/message/DeleteTopicsRequest.json index 7e11554ad480c..73d5371be93bb 100644 --- a/clients/src/main/resources/common/message/DeleteTopicsRequest.json +++ b/clients/src/main/resources/common/message/DeleteTopicsRequest.json @@ -16,6 +16,7 @@ { "apiKey": 20, "type": "request", + "scope": ["zkBroker", "broker", "controller"], "name": "DeleteTopicsRequest", // Versions 0, 1, 2, and 3 are the same. // diff --git a/clients/src/main/resources/common/message/DescribeAclsRequest.json b/clients/src/main/resources/common/message/DescribeAclsRequest.json index a9c36768c1447..f556f77e452fb 100644 --- a/clients/src/main/resources/common/message/DescribeAclsRequest.json +++ b/clients/src/main/resources/common/message/DescribeAclsRequest.json @@ -16,6 +16,7 @@ { "apiKey": 29, "type": "request", + "scope": ["zkBroker", "broker", "controller"], "name": "DescribeAclsRequest", // Version 1 adds resource pattern type. // Version 2 enables flexible versions. diff --git a/clients/src/main/resources/common/message/DescribeClientQuotasRequest.json b/clients/src/main/resources/common/message/DescribeClientQuotasRequest.json index e35d3b0c91a01..aea0936309826 100644 --- a/clients/src/main/resources/common/message/DescribeClientQuotasRequest.json +++ b/clients/src/main/resources/common/message/DescribeClientQuotasRequest.json @@ -16,6 +16,7 @@ { "apiKey": 48, "type": "request", + "scope": ["zkBroker", "broker"], "name": "DescribeClientQuotasRequest", // Version 1 enables flexible versions. "validVersions": "0-1", diff --git a/clients/src/main/resources/common/message/DescribeClusterRequest.json b/clients/src/main/resources/common/message/DescribeClusterRequest.json index 31eb57394a438..2b2aa25a81516 100644 --- a/clients/src/main/resources/common/message/DescribeClusterRequest.json +++ b/clients/src/main/resources/common/message/DescribeClusterRequest.json @@ -16,6 +16,7 @@ { "apiKey": 60, "type": "request", + "scope": ["zkBroker", "broker"], "name": "DescribeClusterRequest", "validVersions": "0", "flexibleVersions": "0+", diff --git a/clients/src/main/resources/common/message/DescribeConfigsRequest.json b/clients/src/main/resources/common/message/DescribeConfigsRequest.json index 01a45eb2a5beb..b583fa0e6db76 100644 --- a/clients/src/main/resources/common/message/DescribeConfigsRequest.json +++ b/clients/src/main/resources/common/message/DescribeConfigsRequest.json @@ -16,6 +16,7 @@ { "apiKey": 32, "type": "request", + "scope": ["zkBroker", "broker", "controller"], "name": "DescribeConfigsRequest", // Version 1 adds IncludeSynonyms. // Version 2 is the same as version 1. diff --git a/clients/src/main/resources/common/message/DescribeDelegationTokenRequest.json b/clients/src/main/resources/common/message/DescribeDelegationTokenRequest.json index 32a4f5c5150c1..6443011714136 100644 --- a/clients/src/main/resources/common/message/DescribeDelegationTokenRequest.json +++ b/clients/src/main/resources/common/message/DescribeDelegationTokenRequest.json @@ -16,6 +16,7 @@ { "apiKey": 41, "type": "request", + "scope": ["zkBroker", "broker", "controller"], "name": "DescribeDelegationTokenRequest", // Version 1 is the same as version 0. // Version 2 adds flexible version support diff --git a/clients/src/main/resources/common/message/DescribeGroupsRequest.json b/clients/src/main/resources/common/message/DescribeGroupsRequest.json index 8a5887a5680a8..254b74d731ef6 100644 --- a/clients/src/main/resources/common/message/DescribeGroupsRequest.json +++ b/clients/src/main/resources/common/message/DescribeGroupsRequest.json @@ -16,6 +16,7 @@ { "apiKey": 15, "type": "request", + "scope": ["zkBroker", "broker"], "name": "DescribeGroupsRequest", // Versions 1 and 2 are the same as version 0. // diff --git a/clients/src/main/resources/common/message/DescribeLogDirsRequest.json b/clients/src/main/resources/common/message/DescribeLogDirsRequest.json index c498e0f22238a..b55d658bdf1d2 100644 --- a/clients/src/main/resources/common/message/DescribeLogDirsRequest.json +++ b/clients/src/main/resources/common/message/DescribeLogDirsRequest.json @@ -16,6 +16,7 @@ { "apiKey": 35, "type": "request", + "scope": ["zkBroker", "broker"], "name": "DescribeLogDirsRequest", // Version 1 is the same as version 0. "validVersions": "0-2", diff --git a/clients/src/main/resources/common/message/DescribeProducersRequest.json b/clients/src/main/resources/common/message/DescribeProducersRequest.json index bd35f91035bd4..801a12a14f996 100644 --- a/clients/src/main/resources/common/message/DescribeProducersRequest.json +++ b/clients/src/main/resources/common/message/DescribeProducersRequest.json @@ -16,6 +16,7 @@ { "apiKey": 61, "type": "request", + "scope": ["zkBroker", "broker"], "name": "DescribeProducersRequest", "validVersions": "0", "flexibleVersions": "0+", diff --git a/clients/src/main/resources/common/message/DescribeQuorumRequest.json b/clients/src/main/resources/common/message/DescribeQuorumRequest.json index f91d679db8b1b..7d606ce4229d9 100644 --- a/clients/src/main/resources/common/message/DescribeQuorumRequest.json +++ b/clients/src/main/resources/common/message/DescribeQuorumRequest.json @@ -16,6 +16,7 @@ { "apiKey": 55, "type": "request", + "scope": ["broker", "controller"], "name": "DescribeQuorumRequest", "validVersions": "0", "flexibleVersions": "0+", diff --git a/clients/src/main/resources/common/message/DescribeUserScramCredentialsRequest.json b/clients/src/main/resources/common/message/DescribeUserScramCredentialsRequest.json index f7f8c68991357..d8194cda0668a 100644 --- a/clients/src/main/resources/common/message/DescribeUserScramCredentialsRequest.json +++ b/clients/src/main/resources/common/message/DescribeUserScramCredentialsRequest.json @@ -16,6 +16,7 @@ { "apiKey": 50, "type": "request", + "scope": ["zkBroker", "broker", "controller"], "name": "DescribeUserScramCredentialsRequest", "validVersions": "0", "flexibleVersions": "0+", diff --git a/clients/src/main/resources/common/message/ElectLeadersRequest.json b/clients/src/main/resources/common/message/ElectLeadersRequest.json index a2ba2bdca735c..ca0210b815b44 100644 --- a/clients/src/main/resources/common/message/ElectLeadersRequest.json +++ b/clients/src/main/resources/common/message/ElectLeadersRequest.json @@ -16,6 +16,7 @@ { "apiKey": 43, "type": "request", + "scope": ["zkBroker", "broker", "controller"], "name": "ElectLeadersRequest", // Version 1 implements multiple leader election types, as described by KIP-460. // diff --git a/clients/src/main/resources/common/message/EndQuorumEpochRequest.json b/clients/src/main/resources/common/message/EndQuorumEpochRequest.json index 45dacde5fe99b..5ab856dee6e4c 100644 --- a/clients/src/main/resources/common/message/EndQuorumEpochRequest.json +++ b/clients/src/main/resources/common/message/EndQuorumEpochRequest.json @@ -16,6 +16,7 @@ { "apiKey": 54, "type": "request", + "scope": ["controller", "raft"], "name": "EndQuorumEpochRequest", "validVersions": "0", "fields": [ diff --git a/clients/src/main/resources/common/message/EndTxnRequest.json b/clients/src/main/resources/common/message/EndTxnRequest.json index de18b43b7d9d2..9c9ad80da07a6 100644 --- a/clients/src/main/resources/common/message/EndTxnRequest.json +++ b/clients/src/main/resources/common/message/EndTxnRequest.json @@ -16,6 +16,7 @@ { "apiKey": 26, "type": "request", + "scope": ["zkBroker", "broker"], "name": "EndTxnRequest", // Version 1 is the same as version 0. // diff --git a/clients/src/main/resources/common/message/EnvelopeRequest.json b/clients/src/main/resources/common/message/EnvelopeRequest.json index a1aa760a29b18..1265bdc3d1d94 100644 --- a/clients/src/main/resources/common/message/EnvelopeRequest.json +++ b/clients/src/main/resources/common/message/EnvelopeRequest.json @@ -16,6 +16,7 @@ { "apiKey": 58, "type": "request", + "scope": ["controller"], "name": "EnvelopeRequest", // Request struct for forwarding. "validVersions": "0", diff --git a/clients/src/main/resources/common/message/ExpireDelegationTokenRequest.json b/clients/src/main/resources/common/message/ExpireDelegationTokenRequest.json index a990862d2fc8a..1252a24e0faf9 100644 --- a/clients/src/main/resources/common/message/ExpireDelegationTokenRequest.json +++ b/clients/src/main/resources/common/message/ExpireDelegationTokenRequest.json @@ -16,6 +16,7 @@ { "apiKey": 40, "type": "request", + "scope": ["zkBroker", "broker", "controller"], "name": "ExpireDelegationTokenRequest", // Version 1 is the same as version 0. // Version 2 adds flexible version support diff --git a/clients/src/main/resources/common/message/FetchRequest.json b/clients/src/main/resources/common/message/FetchRequest.json index 0dcdd7af8bb00..561f69a13fb9d 100644 --- a/clients/src/main/resources/common/message/FetchRequest.json +++ b/clients/src/main/resources/common/message/FetchRequest.json @@ -16,6 +16,7 @@ { "apiKey": 1, "type": "request", + "scope": ["zkBroker", "broker", "controller", "raft"], "name": "FetchRequest", // // Version 1 is the same as version 0. diff --git a/clients/src/main/resources/common/message/FetchSnapshotRequest.json b/clients/src/main/resources/common/message/FetchSnapshotRequest.json index c3518f44765c7..ec39ac12fe010 100644 --- a/clients/src/main/resources/common/message/FetchSnapshotRequest.json +++ b/clients/src/main/resources/common/message/FetchSnapshotRequest.json @@ -16,6 +16,7 @@ { "apiKey": 59, "type": "request", + "scope": ["controller", "raft"], "name": "FetchSnapshotRequest", "validVersions": "0", "flexibleVersions": "0+", diff --git a/clients/src/main/resources/common/message/FindCoordinatorRequest.json b/clients/src/main/resources/common/message/FindCoordinatorRequest.json index 6a90887b1aa01..f2f8b7ae40afc 100644 --- a/clients/src/main/resources/common/message/FindCoordinatorRequest.json +++ b/clients/src/main/resources/common/message/FindCoordinatorRequest.json @@ -16,6 +16,7 @@ { "apiKey": 10, "type": "request", + "scope": ["zkBroker", "broker"], "name": "FindCoordinatorRequest", // Version 1 adds KeyType. // diff --git a/clients/src/main/resources/common/message/HeartbeatRequest.json b/clients/src/main/resources/common/message/HeartbeatRequest.json index 4d799aa3c14c1..0f8322256f409 100644 --- a/clients/src/main/resources/common/message/HeartbeatRequest.json +++ b/clients/src/main/resources/common/message/HeartbeatRequest.json @@ -16,6 +16,7 @@ { "apiKey": 12, "type": "request", + "scope": ["zkBroker", "broker"], "name": "HeartbeatRequest", // Version 1 and version 2 are the same as version 0. // diff --git a/clients/src/main/resources/common/message/IncrementalAlterConfigsRequest.json b/clients/src/main/resources/common/message/IncrementalAlterConfigsRequest.json index b1fb1e9481ac2..2eae744941828 100644 --- a/clients/src/main/resources/common/message/IncrementalAlterConfigsRequest.json +++ b/clients/src/main/resources/common/message/IncrementalAlterConfigsRequest.json @@ -16,6 +16,7 @@ { "apiKey": 44, "type": "request", + "scope": ["zkBroker", "broker", "controller"], "name": "IncrementalAlterConfigsRequest", // Version 1 is the first flexible version. "validVersions": "0-1", diff --git a/clients/src/main/resources/common/message/InitProducerIdRequest.json b/clients/src/main/resources/common/message/InitProducerIdRequest.json index dc85063c29643..5c6af0d424703 100644 --- a/clients/src/main/resources/common/message/InitProducerIdRequest.json +++ b/clients/src/main/resources/common/message/InitProducerIdRequest.json @@ -16,6 +16,7 @@ { "apiKey": 22, "type": "request", + "scope": ["zkBroker", "broker"], "name": "InitProducerIdRequest", // Version 1 is the same as version 0. // diff --git a/clients/src/main/resources/common/message/JoinGroupRequest.json b/clients/src/main/resources/common/message/JoinGroupRequest.json index 2650d89e2b4a8..a18d5fc164e59 100644 --- a/clients/src/main/resources/common/message/JoinGroupRequest.json +++ b/clients/src/main/resources/common/message/JoinGroupRequest.json @@ -16,6 +16,7 @@ { "apiKey": 11, "type": "request", + "scope": ["zkBroker", "broker"], "name": "JoinGroupRequest", // Version 1 adds RebalanceTimeoutMs. // diff --git a/clients/src/main/resources/common/message/LeaderAndIsrRequest.json b/clients/src/main/resources/common/message/LeaderAndIsrRequest.json index 129b7f77f6fac..221b73861969d 100644 --- a/clients/src/main/resources/common/message/LeaderAndIsrRequest.json +++ b/clients/src/main/resources/common/message/LeaderAndIsrRequest.json @@ -16,6 +16,7 @@ { "apiKey": 4, "type": "request", + "scope": ["zkBroker"], "name": "LeaderAndIsrRequest", // Version 1 adds IsNew. // diff --git a/clients/src/main/resources/common/message/LeaveGroupRequest.json b/clients/src/main/resources/common/message/LeaveGroupRequest.json index acc7938c387b1..664ffaf6e31eb 100644 --- a/clients/src/main/resources/common/message/LeaveGroupRequest.json +++ b/clients/src/main/resources/common/message/LeaveGroupRequest.json @@ -16,6 +16,7 @@ { "apiKey": 13, "type": "request", + "scope": ["zkBroker", "broker"], "name": "LeaveGroupRequest", // Version 1 and 2 are the same as version 0. // diff --git a/clients/src/main/resources/common/message/ListGroupsRequest.json b/clients/src/main/resources/common/message/ListGroupsRequest.json index dbe6d9b6f123a..8824255f7b5a4 100644 --- a/clients/src/main/resources/common/message/ListGroupsRequest.json +++ b/clients/src/main/resources/common/message/ListGroupsRequest.json @@ -16,6 +16,7 @@ { "apiKey": 16, "type": "request", + "scope": ["zkBroker", "broker"], "name": "ListGroupsRequest", // Version 1 and 2 are the same as version 0. // diff --git a/clients/src/main/resources/common/message/ListOffsetsRequest.json b/clients/src/main/resources/common/message/ListOffsetsRequest.json index 9855a4bf9058d..c25cb8a183a05 100644 --- a/clients/src/main/resources/common/message/ListOffsetsRequest.json +++ b/clients/src/main/resources/common/message/ListOffsetsRequest.json @@ -16,6 +16,7 @@ { "apiKey": 2, "type": "request", + "scope": ["zkBroker", "broker"], "name": "ListOffsetsRequest", // Version 1 removes MaxNumOffsets. From this version forward, only a single // offset can be returned. diff --git a/clients/src/main/resources/common/message/ListPartitionReassignmentsRequest.json b/clients/src/main/resources/common/message/ListPartitionReassignmentsRequest.json index 7322f25bc456e..0ba9963d76be9 100644 --- a/clients/src/main/resources/common/message/ListPartitionReassignmentsRequest.json +++ b/clients/src/main/resources/common/message/ListPartitionReassignmentsRequest.json @@ -16,6 +16,7 @@ { "apiKey": 46, "type": "request", + "scope": ["zkBroker", "broker"], "name": "ListPartitionReassignmentsRequest", "validVersions": "0", "flexibleVersions": "0+", diff --git a/clients/src/main/resources/common/message/MetadataRequest.json b/clients/src/main/resources/common/message/MetadataRequest.json index 02af116a1c210..edf4cd78f58cb 100644 --- a/clients/src/main/resources/common/message/MetadataRequest.json +++ b/clients/src/main/resources/common/message/MetadataRequest.json @@ -16,6 +16,7 @@ { "apiKey": 3, "type": "request", + "scope": ["zkBroker", "broker"], "name": "MetadataRequest", "validVersions": "0-11", "flexibleVersions": "9+", diff --git a/clients/src/main/resources/common/message/OffsetCommitRequest.json b/clients/src/main/resources/common/message/OffsetCommitRequest.json index 096b61917ae0f..18f065141509f 100644 --- a/clients/src/main/resources/common/message/OffsetCommitRequest.json +++ b/clients/src/main/resources/common/message/OffsetCommitRequest.json @@ -16,6 +16,7 @@ { "apiKey": 8, "type": "request", + "scope": ["zkBroker", "broker"], "name": "OffsetCommitRequest", // Version 1 adds timestamp and group membership information, as well as the commit timestamp. // diff --git a/clients/src/main/resources/common/message/OffsetDeleteRequest.json b/clients/src/main/resources/common/message/OffsetDeleteRequest.json index 108ca9f7b9eca..5e793af6b80a9 100644 --- a/clients/src/main/resources/common/message/OffsetDeleteRequest.json +++ b/clients/src/main/resources/common/message/OffsetDeleteRequest.json @@ -16,6 +16,7 @@ { "apiKey": 47, "type": "request", + "scope": ["zkBroker", "broker"], "name": "OffsetDeleteRequest", "validVersions": "0", "fields": [ diff --git a/clients/src/main/resources/common/message/OffsetFetchRequest.json b/clients/src/main/resources/common/message/OffsetFetchRequest.json index ddd53fc859600..d4fd9abaf020f 100644 --- a/clients/src/main/resources/common/message/OffsetFetchRequest.json +++ b/clients/src/main/resources/common/message/OffsetFetchRequest.json @@ -16,6 +16,7 @@ { "apiKey": 9, "type": "request", + "scope": ["zkBroker", "broker"], "name": "OffsetFetchRequest", // In version 0, the request read offsets from ZK. // diff --git a/clients/src/main/resources/common/message/OffsetForLeaderEpochRequest.json b/clients/src/main/resources/common/message/OffsetForLeaderEpochRequest.json index 75b6c8dc3227c..98ba94f771d39 100644 --- a/clients/src/main/resources/common/message/OffsetForLeaderEpochRequest.json +++ b/clients/src/main/resources/common/message/OffsetForLeaderEpochRequest.json @@ -16,6 +16,7 @@ { "apiKey": 23, "type": "request", + "scope": ["zkBroker", "broker"], "name": "OffsetForLeaderEpochRequest", // Version 1 is the same as version 0. // diff --git a/clients/src/main/resources/common/message/ProduceRequest.json b/clients/src/main/resources/common/message/ProduceRequest.json index 73e72c352f4d4..cec6cee021142 100644 --- a/clients/src/main/resources/common/message/ProduceRequest.json +++ b/clients/src/main/resources/common/message/ProduceRequest.json @@ -16,6 +16,7 @@ { "apiKey": 0, "type": "request", + "scope": ["zkBroker", "broker"], "name": "ProduceRequest", // Version 1 and 2 are the same as version 0. // diff --git a/clients/src/main/resources/common/message/RenewDelegationTokenRequest.json b/clients/src/main/resources/common/message/RenewDelegationTokenRequest.json index 9fbb0fa07a5e8..bff33ae779237 100644 --- a/clients/src/main/resources/common/message/RenewDelegationTokenRequest.json +++ b/clients/src/main/resources/common/message/RenewDelegationTokenRequest.json @@ -16,6 +16,7 @@ { "apiKey": 39, "type": "request", + "scope": ["zkBroker", "broker", "controller"], "name": "RenewDelegationTokenRequest", // Version 1 is the same as version 0. // Version 2 adds flexible version support diff --git a/clients/src/main/resources/common/message/SaslAuthenticateRequest.json b/clients/src/main/resources/common/message/SaslAuthenticateRequest.json index 122cef577076e..8a666fe1417b9 100644 --- a/clients/src/main/resources/common/message/SaslAuthenticateRequest.json +++ b/clients/src/main/resources/common/message/SaslAuthenticateRequest.json @@ -16,6 +16,7 @@ { "apiKey": 36, "type": "request", + "scope": ["zkBroker", "broker", "controller"], "name": "SaslAuthenticateRequest", // Version 1 is the same as version 0. // Version 2 adds flexible version support diff --git a/clients/src/main/resources/common/message/SaslHandshakeRequest.json b/clients/src/main/resources/common/message/SaslHandshakeRequest.json index f384f414c5c04..f9bf2ebf09e30 100644 --- a/clients/src/main/resources/common/message/SaslHandshakeRequest.json +++ b/clients/src/main/resources/common/message/SaslHandshakeRequest.json @@ -16,6 +16,7 @@ { "apiKey": 17, "type": "request", + "scope": ["zkBroker", "broker", "controller"], "name": "SaslHandshakeRequest", // Version 1 supports SASL_AUTHENTICATE. // NOTE: Version cannot be easily bumped due to incorrect diff --git a/clients/src/main/resources/common/message/StopReplicaRequest.json b/clients/src/main/resources/common/message/StopReplicaRequest.json index d43356cab89fe..28cf0122f5c7f 100644 --- a/clients/src/main/resources/common/message/StopReplicaRequest.json +++ b/clients/src/main/resources/common/message/StopReplicaRequest.json @@ -16,6 +16,7 @@ { "apiKey": 5, "type": "request", + "scope": ["zkBroker"], "name": "StopReplicaRequest", // Version 1 adds the broker epoch and reorganizes the partitions to be stored // per topic. diff --git a/clients/src/main/resources/common/message/SyncGroupRequest.json b/clients/src/main/resources/common/message/SyncGroupRequest.json index a0a599150ef53..2fa507f1cf662 100644 --- a/clients/src/main/resources/common/message/SyncGroupRequest.json +++ b/clients/src/main/resources/common/message/SyncGroupRequest.json @@ -16,6 +16,7 @@ { "apiKey": 14, "type": "request", + "scope": ["zkBroker", "broker"], "name": "SyncGroupRequest", // Versions 1 and 2 are the same as version 0. // diff --git a/clients/src/main/resources/common/message/TxnOffsetCommitRequest.json b/clients/src/main/resources/common/message/TxnOffsetCommitRequest.json index bd91df35480c6..a87865c666bc7 100644 --- a/clients/src/main/resources/common/message/TxnOffsetCommitRequest.json +++ b/clients/src/main/resources/common/message/TxnOffsetCommitRequest.json @@ -16,6 +16,7 @@ { "apiKey": 28, "type": "request", + "scope": ["zkBroker", "broker"], "name": "TxnOffsetCommitRequest", // Version 1 is the same as version 0. // diff --git a/clients/src/main/resources/common/message/UnregisterBrokerRequest.json b/clients/src/main/resources/common/message/UnregisterBrokerRequest.json index 3c43b1628cb03..35d9c9ffa7f21 100644 --- a/clients/src/main/resources/common/message/UnregisterBrokerRequest.json +++ b/clients/src/main/resources/common/message/UnregisterBrokerRequest.json @@ -16,6 +16,7 @@ { "apiKey": 64, "type": "request", + "scope": ["broker", "controller"], "name": "UnregisterBrokerRequest", "validVersions": "0", "flexibleVersions": "0+", diff --git a/clients/src/main/resources/common/message/UpdateFeaturesRequest.json b/clients/src/main/resources/common/message/UpdateFeaturesRequest.json index ab882dff1c754..faa28677ee3b1 100644 --- a/clients/src/main/resources/common/message/UpdateFeaturesRequest.json +++ b/clients/src/main/resources/common/message/UpdateFeaturesRequest.json @@ -16,6 +16,7 @@ { "apiKey": 57, "type": "request", + "scope": ["zkBroker", "broker", "controller"], "name": "UpdateFeaturesRequest", "validVersions": "0", "flexibleVersions": "0+", diff --git a/clients/src/main/resources/common/message/UpdateMetadataRequest.json b/clients/src/main/resources/common/message/UpdateMetadataRequest.json index 99d33f192ed4d..19de653686fdf 100644 --- a/clients/src/main/resources/common/message/UpdateMetadataRequest.json +++ b/clients/src/main/resources/common/message/UpdateMetadataRequest.json @@ -16,6 +16,7 @@ { "apiKey": 6, "type": "request", + "scope": ["zkBroker"], "name": "UpdateMetadataRequest", // Version 1 allows specifying multiple endpoints for each broker. // diff --git a/clients/src/main/resources/common/message/VoteRequest.json b/clients/src/main/resources/common/message/VoteRequest.json index 3926ba7c7d6af..e7395ce33728c 100644 --- a/clients/src/main/resources/common/message/VoteRequest.json +++ b/clients/src/main/resources/common/message/VoteRequest.json @@ -16,6 +16,7 @@ { "apiKey": 52, "type": "request", + "scope": ["controller", "raft"], "name": "VoteRequest", "validVersions": "0", "flexibleVersions": "0+", diff --git a/clients/src/main/resources/common/message/WriteTxnMarkersRequest.json b/clients/src/main/resources/common/message/WriteTxnMarkersRequest.json index 3fdfa05bf9773..cbeea7b731937 100644 --- a/clients/src/main/resources/common/message/WriteTxnMarkersRequest.json +++ b/clients/src/main/resources/common/message/WriteTxnMarkersRequest.json @@ -16,6 +16,7 @@ { "apiKey": 27, "type": "request", + "scope": ["zkBroker", "broker"], "name": "WriteTxnMarkersRequest", // Version 1 enables flexible versions. "validVersions": "0-1", diff --git a/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java b/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java index eb130ff934ee9..f9f0cdaf7ec22 100644 --- a/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java @@ -22,6 +22,7 @@ import org.apache.kafka.common.errors.AuthenticationException; import org.apache.kafka.common.errors.UnsupportedVersionException; import org.apache.kafka.common.internals.ClusterResourceListeners; +import org.apache.kafka.common.message.ApiMessageType; import org.apache.kafka.common.message.ApiVersionsResponseData; import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersion; import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersionCollection; @@ -247,7 +248,8 @@ private void setExpectedApiVersionsResponse(ApiVersionsResponse response) { private void awaitReady(NetworkClient client, Node node) { if (client.discoverBrokerVersions()) { - setExpectedApiVersionsResponse(ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE); + setExpectedApiVersionsResponse(ApiVersionsResponse.defaultApiVersionsResponse( + ApiMessageType.ApiScope.ZK_BROKER)); } while (!client.ready(node, time.milliseconds())) client.poll(1, time.milliseconds()); @@ -295,8 +297,7 @@ public void testApiVersionsRequest() { assertTrue(client.hasInFlightRequests(node.idString())); // prepare response - delayedApiVersionsResponse(0, ApiKeys.API_VERSIONS.latestVersion(), - ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE); + delayedApiVersionsResponse(0, ApiKeys.API_VERSIONS.latestVersion(), defaultApiVersionsResponse()); // handle completed receives client.poll(0, time.milliseconds()); @@ -367,8 +368,7 @@ public void testUnsupportedApiVersionsRequestWithVersionProvidedByTheBroker() { assertEquals(2, header.apiVersion()); // prepare response - delayedApiVersionsResponse(1, (short) 0, - ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE); + delayedApiVersionsResponse(1, (short) 0, defaultApiVersionsResponse()); // handle completed receives client.poll(0, time.milliseconds()); @@ -434,8 +434,7 @@ public void testUnsupportedApiVersionsRequestWithoutVersionProvidedByTheBroker() assertEquals(0, header.apiVersion()); // prepare response - delayedApiVersionsResponse(1, (short) 0, - ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE); + delayedApiVersionsResponse(1, (short) 0, defaultApiVersionsResponse()); // handle completed receives client.poll(0, time.milliseconds()); @@ -1079,6 +1078,10 @@ private void awaitInFlightApiVersionRequest() throws Exception { assertFalse(client.isReady(node, time.milliseconds())); } + private ApiVersionsResponse defaultApiVersionsResponse() { + return ApiVersionsResponse.defaultApiVersionsResponse(ApiMessageType.ApiScope.ZK_BROKER); + } + private static class TestCallbackHandler implements RequestCompletionHandler { public boolean executed = false; public ClientResponse response; diff --git a/clients/src/test/java/org/apache/kafka/clients/NodeApiVersionsTest.java b/clients/src/test/java/org/apache/kafka/clients/NodeApiVersionsTest.java index 7c19d9f098866..21e7404a63c2b 100644 --- a/clients/src/test/java/org/apache/kafka/clients/NodeApiVersionsTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/NodeApiVersionsTest.java @@ -16,18 +16,21 @@ */ package org.apache.kafka.clients; -import java.util.ArrayList; -import java.util.LinkedList; -import java.util.List; import org.apache.kafka.common.errors.UnsupportedVersionException; +import org.apache.kafka.common.message.ApiMessageType; import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersion; import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersionCollection; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.requests.ApiVersionsResponse; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; + +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -38,7 +41,7 @@ public void testUnsupportedVersionsToString() { NodeApiVersions versions = new NodeApiVersions(new ApiVersionCollection()); StringBuilder bld = new StringBuilder(); String prefix = "("; - for (ApiKeys apiKey : ApiKeys.brokerApis()) { + for (ApiKeys apiKey : ApiKeys.zkBrokerApis()) { bld.append(prefix).append(apiKey.name). append("(").append(apiKey.id).append("): UNSUPPORTED"); prefix = ", "; @@ -133,27 +136,26 @@ public void testLatestUsableVersionOutOfRange() { () -> apiVersions.latestUsableVersion(ApiKeys.PRODUCE)); } - @Test - public void testUsableVersionLatestVersions() { - List versionList = new LinkedList<>(ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE.data().apiKeys()); + @ParameterizedTest + @EnumSource(ApiMessageType.ApiScope.class) + public void testUsableVersionLatestVersions(ApiMessageType.ApiScope scope) { + ApiVersionsResponse defaultResponse = ApiVersionsResponse.defaultApiVersionsResponse(scope); + List versionList = new LinkedList<>(defaultResponse.data().apiKeys()); // Add an API key that we don't know about. versionList.add(new ApiVersion() .setApiKey((short) 100) .setMinVersion((short) 0) .setMaxVersion((short) 1)); NodeApiVersions versions = new NodeApiVersions(versionList); - for (ApiKeys apiKey: ApiKeys.values()) { - if (apiKey.isControllerOnlyApi) { - assertNull(versions.apiVersion(apiKey)); - } else { - assertEquals(apiKey.latestVersion(), versions.latestUsableVersion(apiKey)); - } + for (ApiKeys apiKey: ApiKeys.apisInScope(scope)) { + assertEquals(apiKey.latestVersion(), versions.latestUsableVersion(apiKey)); } } - @Test - public void testConstructionFromApiVersionsResponse() { - ApiVersionsResponse apiVersionsResponse = ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE; + @ParameterizedTest + @EnumSource(ApiMessageType.ApiScope.class) + public void testConstructionFromApiVersionsResponse(ApiMessageType.ApiScope scope) { + ApiVersionsResponse apiVersionsResponse = ApiVersionsResponse.defaultApiVersionsResponse(scope); NodeApiVersions versions = new NodeApiVersions(apiVersionsResponse.data().apiKeys()); for (ApiVersion apiVersionKey : apiVersionsResponse.data().apiKeys()) { diff --git a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java index ec05d2c002fd1..5002b71767229 100644 --- a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java @@ -74,6 +74,7 @@ import org.apache.kafka.common.message.AlterReplicaLogDirsResponseData.AlterReplicaLogDirPartitionResult; import org.apache.kafka.common.message.AlterReplicaLogDirsResponseData.AlterReplicaLogDirTopicResult; import org.apache.kafka.common.message.AlterUserScramCredentialsResponseData; +import org.apache.kafka.common.message.ApiMessageType; import org.apache.kafka.common.message.ApiVersionsResponseData; import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersion; import org.apache.kafka.common.message.CreateAclsResponseData; @@ -126,6 +127,7 @@ import org.apache.kafka.common.quota.ClientQuotaEntity; import org.apache.kafka.common.quota.ClientQuotaFilter; import org.apache.kafka.common.quota.ClientQuotaFilterComponent; +import org.apache.kafka.common.record.RecordVersion; import org.apache.kafka.common.requests.AbstractResponse; import org.apache.kafka.common.requests.AlterClientQuotasResponse; import org.apache.kafka.common.requests.AlterPartitionReassignmentsResponse; @@ -541,17 +543,17 @@ private static Features c private static ApiVersionsResponse prepareApiVersionsResponseForDescribeFeatures(Errors error) { if (error == Errors.NONE) { - return new ApiVersionsResponse(ApiVersionsResponse.createApiVersionsResponseData( - ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE.throttleTimeMs(), - error, - ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE.data().apiKeys(), + return ApiVersionsResponse.createApiVersionsResponse( + 0, + ApiVersionsResponse.filterApis(RecordVersion.current(), ApiMessageType.ApiScope.ZK_BROKER), convertSupportedFeaturesMap(defaultFeatureMetadata().supportedFeatures()), convertFinalizedFeaturesMap(defaultFeatureMetadata().finalizedFeatures()), - defaultFeatureMetadata().finalizedFeaturesEpoch().get())); + defaultFeatureMetadata().finalizedFeaturesEpoch().get() + ); } return new ApiVersionsResponse( new ApiVersionsResponseData() - .setThrottleTimeMs(ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE.throttleTimeMs()) + .setThrottleTimeMs(0) .setErrorCode(error.code())); } diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java index 9330f9eb51c55..dd785dc744278 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java @@ -48,6 +48,7 @@ import org.apache.kafka.common.header.Header; import org.apache.kafka.common.header.internals.RecordHeader; import org.apache.kafka.common.internals.ClusterResourceListeners; +import org.apache.kafka.common.message.ApiMessageType; import org.apache.kafka.common.message.ListOffsetsRequestData.ListOffsetsPartition; import org.apache.kafka.common.message.ListOffsetsRequestData.ListOffsetsTopic; import org.apache.kafka.common.message.ListOffsetsResponseData; @@ -2074,8 +2075,9 @@ public void testQuotaMetrics() { 1000, 1000, 64 * 1024, 64 * 1024, 1000, 10 * 1000, 127 * 1000, ClientDnsLookup.USE_ALL_DNS_IPS, time, true, new ApiVersions(), throttleTimeSensor, new LogContext()); - ByteBuffer buffer = RequestTestUtils.serializeResponseWithHeader(ApiVersionsResponse.createApiVersionsResponse( - 400, RecordBatch.CURRENT_MAGIC_VALUE), ApiKeys.API_VERSIONS.latestVersion(), 0); + ApiVersionsResponse apiVersionsResponse = ApiVersionsResponse.defaultApiVersionsResponse( + 400, ApiMessageType.ApiScope.ZK_BROKER); + ByteBuffer buffer = RequestTestUtils.serializeResponseWithHeader(apiVersionsResponse, ApiKeys.API_VERSIONS.latestVersion(), 0); selector.delayedReceive(new DelayedReceive(node.idString(), new NetworkReceive(node.idString(), buffer))); while (!client.ready(node, time.milliseconds())) { diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java index 439ba3b96b1a4..aff724ac6b4fc 100644 --- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java @@ -24,13 +24,6 @@ import org.apache.kafka.clients.NetworkClient; import org.apache.kafka.clients.NodeApiVersions; import org.apache.kafka.clients.producer.Callback; -import org.apache.kafka.common.errors.InvalidRequestException; -import org.apache.kafka.common.errors.TransactionAbortedException; -import org.apache.kafka.common.message.ProduceRequestData; -import org.apache.kafka.common.requests.FindCoordinatorRequest.CoordinatorType; -import org.apache.kafka.common.requests.MetadataRequest; -import org.apache.kafka.common.requests.RequestTestUtils; -import org.apache.kafka.common.utils.ProducerIdAndEpoch; import org.apache.kafka.clients.producer.RecordMetadata; import org.apache.kafka.common.Cluster; import org.apache.kafka.common.KafkaException; @@ -39,15 +32,19 @@ import org.apache.kafka.common.Node; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.errors.ClusterAuthorizationException; +import org.apache.kafka.common.errors.InvalidRequestException; import org.apache.kafka.common.errors.NetworkException; import org.apache.kafka.common.errors.RecordTooLargeException; import org.apache.kafka.common.errors.TimeoutException; import org.apache.kafka.common.errors.TopicAuthorizationException; +import org.apache.kafka.common.errors.TransactionAbortedException; import org.apache.kafka.common.errors.UnsupportedForMessageFormatException; import org.apache.kafka.common.errors.UnsupportedVersionException; import org.apache.kafka.common.internals.ClusterResourceListeners; +import org.apache.kafka.common.message.ApiMessageType; import org.apache.kafka.common.message.EndTxnResponseData; import org.apache.kafka.common.message.InitProducerIdResponseData; +import org.apache.kafka.common.message.ProduceRequestData; import org.apache.kafka.common.metrics.KafkaMetric; import org.apache.kafka.common.metrics.MetricConfig; import org.apache.kafka.common.metrics.Metrics; @@ -66,15 +63,19 @@ import org.apache.kafka.common.requests.ApiVersionsResponse; import org.apache.kafka.common.requests.EndTxnRequest; import org.apache.kafka.common.requests.EndTxnResponse; +import org.apache.kafka.common.requests.FindCoordinatorRequest.CoordinatorType; import org.apache.kafka.common.requests.FindCoordinatorResponse; import org.apache.kafka.common.requests.InitProducerIdRequest; import org.apache.kafka.common.requests.InitProducerIdResponse; +import org.apache.kafka.common.requests.MetadataRequest; import org.apache.kafka.common.requests.MetadataResponse; import org.apache.kafka.common.requests.ProduceRequest; import org.apache.kafka.common.requests.ProduceResponse; +import org.apache.kafka.common.requests.RequestTestUtils; import org.apache.kafka.common.requests.TransactionResult; import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.MockTime; +import org.apache.kafka.common.utils.ProducerIdAndEpoch; import org.apache.kafka.common.utils.Time; import org.apache.kafka.test.DelayedReceive; import org.apache.kafka.test.MockSelector; @@ -105,12 +106,12 @@ import java.util.concurrent.atomic.AtomicReference; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertSame; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; import static org.mockito.AdditionalMatchers.geq; import static org.mockito.ArgumentMatchers.any; @@ -287,9 +288,9 @@ public void testQuotaMetrics() { 1000, 1000, 64 * 1024, 64 * 1024, 1000, 10 * 1000, 127 * 1000, ClientDnsLookup.USE_ALL_DNS_IPS, time, true, new ApiVersions(), throttleTimeSensor, logContext); - ByteBuffer buffer = RequestTestUtils.serializeResponseWithHeader( - ApiVersionsResponse.createApiVersionsResponse(400, RecordBatch.CURRENT_MAGIC_VALUE), - ApiKeys.API_VERSIONS.latestVersion(), 0); + ApiVersionsResponse apiVersionsResponse = ApiVersionsResponse.defaultApiVersionsResponse( + 400, ApiMessageType.ApiScope.ZK_BROKER); + ByteBuffer buffer = RequestTestUtils.serializeResponseWithHeader(apiVersionsResponse, ApiKeys.API_VERSIONS.latestVersion(), 0); selector.delayedReceive(new DelayedReceive(node.idString(), new NetworkReceive(node.idString(), buffer))); while (!client.ready(node, time.milliseconds())) { diff --git a/clients/src/test/java/org/apache/kafka/common/network/NioEchoServer.java b/clients/src/test/java/org/apache/kafka/common/network/NioEchoServer.java index 15f8079b65fbe..5be00acc8d3a3 100644 --- a/clients/src/test/java/org/apache/kafka/common/network/NioEchoServer.java +++ b/clients/src/test/java/org/apache/kafka/common/network/NioEchoServer.java @@ -18,9 +18,11 @@ import org.apache.kafka.common.MetricName; import org.apache.kafka.common.config.AbstractConfig; +import org.apache.kafka.common.message.ApiMessageType; import org.apache.kafka.common.metrics.KafkaMetric; import org.apache.kafka.common.metrics.Metrics; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.requests.ApiVersionsResponse; import org.apache.kafka.common.security.auth.SecurityProtocol; import org.apache.kafka.common.security.authenticator.CredentialCache; import org.apache.kafka.common.security.scram.ScramCredential; @@ -117,7 +119,8 @@ public NioEchoServer(ListenerName listenerName, SecurityProtocol securityProtoco LogContext logContext = new LogContext(); if (channelBuilder == null) channelBuilder = ChannelBuilders.serverChannelBuilder(listenerName, false, - securityProtocol, config, credentialCache, tokenCache, time, logContext); + securityProtocol, config, credentialCache, tokenCache, time, logContext, + () -> ApiVersionsResponse.defaultApiVersionsResponse(ApiMessageType.ApiScope.ZK_BROKER)); this.metrics = new Metrics(); this.selector = new Selector(10000, failedAuthenticationDelayMs, metrics, time, "MetricGroup", channelBuilder, logContext); diff --git a/clients/src/test/java/org/apache/kafka/common/network/SaslChannelBuilderTest.java b/clients/src/test/java/org/apache/kafka/common/network/SaslChannelBuilderTest.java index 820500e965a15..c6f6d76dedb5c 100644 --- a/clients/src/test/java/org/apache/kafka/common/network/SaslChannelBuilderTest.java +++ b/clients/src/test/java/org/apache/kafka/common/network/SaslChannelBuilderTest.java @@ -21,6 +21,8 @@ import org.apache.kafka.common.config.SaslConfigs; import org.apache.kafka.common.config.SslConfigs; import org.apache.kafka.common.config.internals.BrokerSecurityConfigs; +import org.apache.kafka.common.message.ApiMessageType; +import org.apache.kafka.common.requests.ApiVersionsResponse; import org.apache.kafka.common.security.TestSecurityConfig; import org.apache.kafka.common.security.auth.KafkaPrincipal; import org.apache.kafka.common.security.auth.SecurityProtocol; @@ -48,6 +50,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.function.Supplier; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -160,9 +163,8 @@ public void testClientChannelBuilderWithBrokerConfigs() throws Exception { private SaslChannelBuilder createGssapiChannelBuilder(Map jaasContexts, GSSManager gssManager) { SaslChannelBuilder channelBuilder = new SaslChannelBuilder(Mode.SERVER, jaasContexts, - SecurityProtocol.SASL_PLAINTEXT, - new ListenerName("GSSAPI"), false, "GSSAPI", - true, null, null, null, Time.SYSTEM, new LogContext()) { + SecurityProtocol.SASL_PLAINTEXT, new ListenerName("GSSAPI"), false, "GSSAPI", + true, null, null, null, Time.SYSTEM, new LogContext(), defaultApiVersionsSupplier()) { @Override protected GSSManager gssManager() { @@ -174,6 +176,10 @@ protected GSSManager gssManager() { return channelBuilder; } + private Supplier defaultApiVersionsSupplier() { + return () -> ApiVersionsResponse.defaultApiVersionsResponse(ApiMessageType.ApiScope.ZK_BROKER); + } + private SaslChannelBuilder createChannelBuilder(SecurityProtocol securityProtocol, String saslMechanism) { Class loginModule = null; switch (saslMechanism) { @@ -198,7 +204,7 @@ private SaslChannelBuilder createChannelBuilder(SecurityProtocol securityProtoco Map jaasContexts = Collections.singletonMap(saslMechanism, jaasContext); return new SaslChannelBuilder(Mode.CLIENT, jaasContexts, securityProtocol, new ListenerName(saslMechanism), false, saslMechanism, true, null, - null, null, Time.SYSTEM, new LogContext()); + null, null, Time.SYSTEM, new LogContext(), defaultApiVersionsSupplier()); } public static final class TestGssapiLoginModule implements LoginModule { diff --git a/clients/src/test/java/org/apache/kafka/common/network/SslTransportLayerTest.java b/clients/src/test/java/org/apache/kafka/common/network/SslTransportLayerTest.java index 2b3bf566e3fa4..1731f72252e3f 100644 --- a/clients/src/test/java/org/apache/kafka/common/network/SslTransportLayerTest.java +++ b/clients/src/test/java/org/apache/kafka/common/network/SslTransportLayerTest.java @@ -22,7 +22,9 @@ import org.apache.kafka.common.config.types.Password; import org.apache.kafka.common.errors.InvalidConfigurationException; import org.apache.kafka.common.memory.MemoryPool; +import org.apache.kafka.common.message.ApiMessageType; import org.apache.kafka.common.metrics.Metrics; +import org.apache.kafka.common.requests.ApiVersionsResponse; import org.apache.kafka.common.security.TestSecurityConfig; import org.apache.kafka.common.security.auth.SecurityProtocol; import org.apache.kafka.common.security.ssl.DefaultSslEngineFactory; @@ -58,6 +60,7 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; import java.util.stream.Stream; import javax.net.ssl.SSLContext; @@ -1018,7 +1021,8 @@ public void testInterBrokerSslConfigValidation(Args args) throws Exception { TestSecurityConfig config = new TestSecurityConfig(args.sslServerConfigs); ListenerName listenerName = ListenerName.forSecurityProtocol(securityProtocol); ChannelBuilder serverChannelBuilder = ChannelBuilders.serverChannelBuilder(listenerName, - true, securityProtocol, config, null, null, time, new LogContext()); + true, securityProtocol, config, null, null, time, new LogContext(), + defaultApiVersionsSupplier()); server = new NioEchoServer(listenerName, securityProtocol, config, "localhost", serverChannelBuilder, null, time); server.start(); @@ -1040,8 +1044,9 @@ public void testInterBrokerSslConfigValidationFailure(Args args) { args.sslServerConfigs.put(BrokerSecurityConfigs.SSL_CLIENT_AUTH_CONFIG, "required"); TestSecurityConfig config = new TestSecurityConfig(args.sslServerConfigs); ListenerName listenerName = ListenerName.forSecurityProtocol(securityProtocol); - assertThrows(KafkaException.class, () -> ChannelBuilders.serverChannelBuilder(listenerName, true, securityProtocol, config, - null, null, time, new LogContext())); + assertThrows(KafkaException.class, () -> ChannelBuilders.serverChannelBuilder( + listenerName, true, securityProtocol, config, + null, null, time, new LogContext(), defaultApiVersionsSupplier())); } /** @@ -1055,7 +1060,8 @@ public void testServerKeystoreDynamicUpdate(Args args) throws Exception { TestSecurityConfig config = new TestSecurityConfig(args.sslServerConfigs); ListenerName listenerName = ListenerName.forSecurityProtocol(securityProtocol); ChannelBuilder serverChannelBuilder = ChannelBuilders.serverChannelBuilder(listenerName, - false, securityProtocol, config, null, null, time, new LogContext()); + false, securityProtocol, config, null, null, time, new LogContext(), + defaultApiVersionsSupplier()); server = new NioEchoServer(listenerName, securityProtocol, config, "localhost", serverChannelBuilder, null, time); server.start(); @@ -1111,7 +1117,8 @@ public void testServerKeystoreDynamicUpdateWithNewSubjectAltName(Args args) thro TestSecurityConfig config = new TestSecurityConfig(args.sslServerConfigs); ListenerName listenerName = ListenerName.forSecurityProtocol(securityProtocol); ChannelBuilder serverChannelBuilder = ChannelBuilders.serverChannelBuilder(listenerName, - false, securityProtocol, config, null, null, time, new LogContext()); + false, securityProtocol, config, null, null, time, new LogContext(), + defaultApiVersionsSupplier()); server = new NioEchoServer(listenerName, securityProtocol, config, "localhost", serverChannelBuilder, null, time); server.start(); @@ -1176,7 +1183,8 @@ public void testServerTruststoreDynamicUpdate(Args args) throws Exception { TestSecurityConfig config = new TestSecurityConfig(args.sslServerConfigs); ListenerName listenerName = ListenerName.forSecurityProtocol(securityProtocol); ChannelBuilder serverChannelBuilder = ChannelBuilders.serverChannelBuilder(listenerName, - false, securityProtocol, config, null, null, time, new LogContext()); + false, securityProtocol, config, null, null, time, new LogContext(), + defaultApiVersionsSupplier()); server = new NioEchoServer(listenerName, securityProtocol, config, "localhost", serverChannelBuilder, null, time); server.start(); @@ -1334,6 +1342,10 @@ private interface FailureAction { void run() throws IOException; } + private Supplier defaultApiVersionsSupplier() { + return () -> ApiVersionsResponse.defaultApiVersionsResponse(ApiMessageType.ApiScope.ZK_BROKER); + } + static class TestSslChannelBuilder extends SslChannelBuilder { private Integer netReadBufSizeOverride; diff --git a/clients/src/test/java/org/apache/kafka/common/protocol/ApiKeysTest.java b/clients/src/test/java/org/apache/kafka/common/protocol/ApiKeysTest.java index 52121dea81a5e..1f8e0a6ce42f4 100644 --- a/clients/src/test/java/org/apache/kafka/common/protocol/ApiKeysTest.java +++ b/clients/src/test/java/org/apache/kafka/common/protocol/ApiKeysTest.java @@ -20,9 +20,12 @@ import org.apache.kafka.common.protocol.types.Schema; import org.junit.jupiter.api.Test; +import java.util.Collections; import java.util.EnumSet; +import java.util.HashSet; import java.util.Set; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -60,11 +63,7 @@ public void testResponseThrottleTime() { Set authenticationKeys = EnumSet.of(ApiKeys.SASL_HANDSHAKE, ApiKeys.SASL_AUTHENTICATE); // Newer protocol apis include throttle time ms even for cluster actions Set clusterActionsWithThrottleTimeMs = EnumSet.of(ApiKeys.ALTER_ISR); - for (ApiKeys apiKey: ApiKeys.values()) { - // Disable broker-to-controller API throttling test - if (apiKey.isControllerOnlyApi) { - continue; - } + for (ApiKeys apiKey: ApiKeys.zkBrokerApis()) { Schema responseSchema = apiKey.messageType.responseSchemas()[apiKey.latestVersion()]; BoundField throttleTimeField = responseSchema.get("throttle_time_ms"); if ((apiKey.clusterAction && !clusterActionsWithThrottleTimeMs.contains(apiKey)) @@ -74,4 +73,17 @@ public void testResponseThrottleTime() { assertNotNull(throttleTimeField, "Throttle time field missing: " + apiKey); } } + + @Test + public void testApiScope() { + Set apisMissingScope = new HashSet<>(); + for (ApiKeys apiKey : ApiKeys.values()) { + if (apiKey.messageType.scope().isEmpty()) { + apisMissingScope.add(apiKey); + } + } + assertEquals(Collections.emptySet(), apisMissingScope, + "Found some APIs missing scope definition"); + } + } diff --git a/clients/src/test/java/org/apache/kafka/common/requests/ApiVersionsResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/ApiVersionsResponseTest.java index 38a586917624c..5ca11f86413d5 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/ApiVersionsResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/ApiVersionsResponseTest.java @@ -17,17 +17,17 @@ package org.apache.kafka.common.requests; +import org.apache.kafka.common.message.ApiMessageType; import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersion; import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersionCollection; import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.record.RecordBatch; +import org.apache.kafka.common.record.RecordVersion; import org.apache.kafka.common.utils.Utils; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; -import java.util.Collection; -import java.util.HashSet; import java.util.Map; -import java.util.Set; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -38,21 +38,15 @@ public class ApiVersionsResponseTest { - @Test - public void shouldCreateApiResponseThatHasAllApiKeysSupportedByBroker() { - assertEquals(apiKeysInResponse(ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE), new HashSet<>(ApiKeys.brokerApis())); - assertTrue(ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE.data().supportedFeatures().isEmpty()); - assertTrue(ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE.data().finalizedFeatures().isEmpty()); - assertEquals(ApiVersionsResponse.UNKNOWN_FINALIZED_FEATURES_EPOCH, ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE.data().finalizedFeaturesEpoch()); - } - - @Test - public void shouldHaveCorrectDefaultApiVersionsResponse() { - Collection apiVersions = ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE.data().apiKeys(); - assertEquals(apiVersions.size(), ApiKeys.brokerApis().size(), "API versions for all API keys must be maintained."); + @ParameterizedTest + @EnumSource(ApiMessageType.ApiScope.class) + public void shouldHaveCorrectDefaultApiVersionsResponse(ApiMessageType.ApiScope scope) { + ApiVersionsResponse defaultResponse = ApiVersionsResponse.defaultApiVersionsResponse(scope); + assertEquals(ApiKeys.apisInScope(scope).size(), defaultResponse.data().apiKeys().size(), + "API versions for all API keys must be maintained."); - for (ApiKeys key : ApiKeys.brokerApis()) { - ApiVersion version = ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE.apiVersion(key.id); + for (ApiKeys key : ApiKeys.apisInScope(scope)) { + ApiVersion version = defaultResponse.apiVersion(key.id); assertNotNull(version, "Could not find ApiVersion for API " + key.name); assertEquals(version.minVersion(), key.oldestVersion(), "Incorrect min version for Api " + key.name); assertEquals(version.maxVersion(), key.latestVersion(), "Incorrect max version for Api " + key.name); @@ -74,9 +68,9 @@ public void shouldHaveCorrectDefaultApiVersionsResponse() { } } - assertTrue(ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE.data().supportedFeatures().isEmpty()); - assertTrue(ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE.data().finalizedFeatures().isEmpty()); - assertEquals(ApiVersionsResponse.UNKNOWN_FINALIZED_FEATURES_EPOCH, ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE.data().finalizedFeaturesEpoch()); + assertTrue(defaultResponse.data().supportedFeatures().isEmpty()); + assertTrue(defaultResponse.data().finalizedFeatures().isEmpty()); + assertEquals(ApiVersionsResponse.UNKNOWN_FINALIZED_FEATURES_EPOCH, defaultResponse.data().finalizedFeaturesEpoch()); } @Test @@ -96,9 +90,11 @@ public void shouldHaveCommonlyAgreedApiVersionResponseWithControllerOnForwardabl .setMaxVersion(maxVersion)) ); - ApiVersionCollection commonResponse = ApiVersionsResponse.intersectControllerApiVersions( - RecordBatch.CURRENT_MAGIC_VALUE, - activeControllerApiVersions); + ApiVersionCollection commonResponse = ApiVersionsResponse.intersectForwardableApis( + ApiMessageType.ApiScope.ZK_BROKER, + RecordVersion.current(), + activeControllerApiVersions + ); verifyVersions(forwardableAPIKey.id, minVersion, maxVersion, commonResponse); @@ -149,11 +145,4 @@ private void verifyVersions(short forwardableAPIKey, assertEquals(expectedVersionsForForwardableAPI, commonResponse.find(forwardableAPIKey)); } - private Set apiKeysInResponse(final ApiVersionsResponse apiVersions) { - final Set apiKeys = new HashSet<>(); - for (final ApiVersion version : apiVersions.data().apiKeys()) { - apiKeys.add(ApiKeys.forId(version.apiKey())); - } - return apiKeys; - } } diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java index f87d9f91317ab..b737be9d1828f 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java @@ -44,6 +44,7 @@ import org.apache.kafka.common.message.AlterReplicaLogDirsRequestData.AlterReplicaLogDirTopic; import org.apache.kafka.common.message.AlterReplicaLogDirsRequestData.AlterReplicaLogDirTopicCollection; import org.apache.kafka.common.message.AlterReplicaLogDirsResponseData; +import org.apache.kafka.common.message.ApiMessageType; import org.apache.kafka.common.message.ApiVersionsRequestData; import org.apache.kafka.common.message.ApiVersionsResponseData; import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersion; @@ -145,11 +146,11 @@ import org.apache.kafka.common.message.OffsetDeleteResponseData.OffsetDeleteResponsePartitionCollection; import org.apache.kafka.common.message.OffsetDeleteResponseData.OffsetDeleteResponseTopic; import org.apache.kafka.common.message.OffsetDeleteResponseData.OffsetDeleteResponseTopicCollection; -import org.apache.kafka.common.message.OffsetForLeaderEpochResponseData.EpochEndOffset; import org.apache.kafka.common.message.OffsetForLeaderEpochRequestData.OffsetForLeaderPartition; import org.apache.kafka.common.message.OffsetForLeaderEpochRequestData.OffsetForLeaderTopic; import org.apache.kafka.common.message.OffsetForLeaderEpochRequestData.OffsetForLeaderTopicCollection; import org.apache.kafka.common.message.OffsetForLeaderEpochResponseData; +import org.apache.kafka.common.message.OffsetForLeaderEpochResponseData.EpochEndOffset; import org.apache.kafka.common.message.OffsetForLeaderEpochResponseData.OffsetForLeaderTopicResult; import org.apache.kafka.common.message.ProduceRequestData; import org.apache.kafka.common.message.RenewDelegationTokenRequestData; @@ -346,17 +347,6 @@ public void testSerialization() throws Exception { checkErrorResponse(createSaslAuthenticateRequest(), unknownServerException, true); checkResponse(createSaslAuthenticateResponse(), 0, true); checkResponse(createSaslAuthenticateResponse(), 1, true); - checkRequest(createApiVersionRequest(), true); - checkErrorResponse(createApiVersionRequest(), unknownServerException, true); - checkErrorResponse(createApiVersionRequest(), new UnsupportedVersionException("Not Supported"), true); - checkResponse(createApiVersionResponse(), 0, true); - checkResponse(createApiVersionResponse(), 1, true); - checkResponse(createApiVersionResponse(), 2, true); - checkResponse(createApiVersionResponse(), 3, true); - checkResponse(ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE, 0, true); - checkResponse(ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE, 1, true); - checkResponse(ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE, 2, true); - checkResponse(ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE, 3, true); for (int v = ApiKeys.CREATE_TOPICS.oldestVersion(); v <= ApiKeys.CREATE_TOPICS.latestVersion(); v++) { checkRequest(createCreateTopicRequest(v), true); @@ -521,9 +511,20 @@ public void testSerialization() throws Exception { checkResponse(createAlterClientQuotasResponse(), 0, true); } + @Test + public void testApiVersionsSerialization() { + for (short v : ApiKeys.API_VERSIONS.allVersions()) { + checkRequest(createApiVersionRequest(v), true); + checkErrorResponse(createApiVersionRequest(v), unknownServerException, true); + checkErrorResponse(createApiVersionRequest(v), new UnsupportedVersionException("Not Supported"), true); + checkResponse(createApiVersionResponse(), v, true); + checkResponse(ApiVersionsResponse.defaultApiVersionsResponse(ApiMessageType.ApiScope.ZK_BROKER), v, true); + } + } + @Test public void testBrokerHeartbeatSerialization() { - for (short v = ApiKeys.BROKER_HEARTBEAT.oldestVersion(); v <= ApiKeys.BROKER_HEARTBEAT.latestVersion(); v++) { + for (short v : ApiKeys.BROKER_HEARTBEAT.allVersions()) { checkRequest(createBrokerHeartbeatRequest(v), true); checkErrorResponse(createBrokerHeartbeatRequest(v), unknownServerException, true); checkResponse(createBrokerHeartbeatResponse(), v, true); @@ -532,7 +533,7 @@ public void testBrokerHeartbeatSerialization() { @Test public void testBrokerRegistrationSerialization() { - for (short v = ApiKeys.BROKER_REGISTRATION.oldestVersion(); v <= ApiKeys.BROKER_REGISTRATION.latestVersion(); v++) { + for (short v : ApiKeys.BROKER_REGISTRATION.allVersions()) { checkRequest(createBrokerRegistrationRequest(v), true); checkErrorResponse(createBrokerRegistrationRequest(v), unknownServerException, true); checkResponse(createBrokerRegistrationResponse(), 0, true); @@ -540,8 +541,8 @@ public void testBrokerRegistrationSerialization() { } @Test - public void testDescribeProducersSerialization() throws Exception { - for (short v = ApiKeys.DESCRIBE_PRODUCERS.oldestVersion(); v <= ApiKeys.DESCRIBE_PRODUCERS.latestVersion(); v++) { + public void testDescribeProducersSerialization() { + for (short v : ApiKeys.DESCRIBE_PRODUCERS.allVersions()) { checkRequest(createDescribeProducersRequest(v), true); checkErrorResponse(createDescribeProducersRequest(v), unknownServerException, true); checkResponse(createDescribeProducersResponse(), v, true); @@ -549,8 +550,8 @@ public void testDescribeProducersSerialization() throws Exception { } @Test - public void testDescribeClusterSerialization() throws Exception { - for (short v = ApiKeys.DESCRIBE_CLUSTER.oldestVersion(); v <= ApiKeys.DESCRIBE_CLUSTER.latestVersion(); v++) { + public void testDescribeClusterSerialization() { + for (short v : ApiKeys.DESCRIBE_CLUSTER.allVersions()) { checkRequest(createDescribeClusterRequest(v), true); checkErrorResponse(createDescribeClusterRequest(v), unknownServerException, true); checkResponse(createDescribeClusterResponse(), v, true); @@ -559,7 +560,7 @@ public void testDescribeClusterSerialization() throws Exception { @Test public void testUnregisterBrokerSerialization() { - for (short v = ApiKeys.UNREGISTER_BROKER.oldestVersion(); v <= ApiKeys.UNREGISTER_BROKER.latestVersion(); v++) { + for (short v : ApiKeys.UNREGISTER_BROKER.allVersions()) { checkRequest(createUnregisterBrokerRequest(v), true); checkErrorResponse(createUnregisterBrokerRequest(v), unknownServerException, true); checkResponse(createUnregisterBrokerResponse(), v, true); @@ -1013,47 +1014,56 @@ private void testInvalidCase(String name, String version) { @Test public void testApiVersionResponseWithUnsupportedError() { - ApiVersionsRequest request = new ApiVersionsRequest.Builder().build(); - ApiVersionsResponse response = request.getErrorResponse(0, Errors.UNSUPPORTED_VERSION.exception()); - - assertEquals(Errors.UNSUPPORTED_VERSION.code(), response.data().errorCode()); - - ApiVersion apiVersion = response.data().apiKeys().find(ApiKeys.API_VERSIONS.id); - assertNotNull(apiVersion); - assertEquals(ApiKeys.API_VERSIONS.id, apiVersion.apiKey()); - assertEquals(ApiKeys.API_VERSIONS.oldestVersion(), apiVersion.minVersion()); - assertEquals(ApiKeys.API_VERSIONS.latestVersion(), apiVersion.maxVersion()); + for (short version : ApiKeys.API_VERSIONS.allVersions()) { + ApiVersionsRequest request = new ApiVersionsRequest.Builder().build(version); + ApiVersionsResponse response = request.getErrorResponse(0, Errors.UNSUPPORTED_VERSION.exception()); + assertEquals(Errors.UNSUPPORTED_VERSION.code(), response.data().errorCode()); + + ApiVersion apiVersion = response.data().apiKeys().find(ApiKeys.API_VERSIONS.id); + assertNotNull(apiVersion); + assertEquals(ApiKeys.API_VERSIONS.id, apiVersion.apiKey()); + assertEquals(ApiKeys.API_VERSIONS.oldestVersion(), apiVersion.minVersion()); + assertEquals(ApiKeys.API_VERSIONS.latestVersion(), apiVersion.maxVersion()); + } } @Test public void testApiVersionResponseWithNotUnsupportedError() { - ApiVersionsRequest request = new ApiVersionsRequest.Builder().build(); - ApiVersionsResponse response = request.getErrorResponse(0, Errors.INVALID_REQUEST.exception()); + for (short version : ApiKeys.API_VERSIONS.allVersions()) { + ApiVersionsRequest request = new ApiVersionsRequest.Builder().build(version); + ApiVersionsResponse response = request.getErrorResponse(0, Errors.INVALID_REQUEST.exception()); + assertEquals(response.data().errorCode(), Errors.INVALID_REQUEST.code()); + assertTrue(response.data().apiKeys().isEmpty()); + } + } - assertEquals(response.data().errorCode(), Errors.INVALID_REQUEST.code()); - assertTrue(response.data().apiKeys().isEmpty()); + private ApiVersionsResponse defaultApiVersionsResponse() { + return ApiVersionsResponse.defaultApiVersionsResponse(ApiMessageType.ApiScope.ZK_BROKER); } @Test public void testApiVersionResponseParsingFallback() { - ByteBuffer buffer = ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE.serialize((short) 0); - ApiVersionsResponse response = ApiVersionsResponse.parse(buffer, ApiKeys.API_VERSIONS.latestVersion()); - - assertEquals(Errors.NONE.code(), response.data().errorCode()); + for (short version : ApiKeys.API_VERSIONS.allVersions()) { + ByteBuffer buffer = defaultApiVersionsResponse().serialize((short) 0); + ApiVersionsResponse response = ApiVersionsResponse.parse(buffer, version); + assertEquals(Errors.NONE.code(), response.data().errorCode()); + } } @Test public void testApiVersionResponseParsingFallbackException() { - short version = 0; - assertThrows(BufferUnderflowException.class, () -> ApiVersionsResponse.parse(ByteBuffer.allocate(0), version)); + for (final short version : ApiKeys.API_VERSIONS.allVersions()) { + assertThrows(BufferUnderflowException.class, () -> ApiVersionsResponse.parse(ByteBuffer.allocate(0), version)); + } } @Test public void testApiVersionResponseParsing() { - ByteBuffer buffer = ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE.serialize(ApiKeys.API_VERSIONS.latestVersion()); - ApiVersionsResponse response = ApiVersionsResponse.parse(buffer, ApiKeys.API_VERSIONS.latestVersion()); - - assertEquals(Errors.NONE.code(), response.data().errorCode()); + for (short version : ApiKeys.API_VERSIONS.allVersions()) { + ByteBuffer buffer = defaultApiVersionsResponse().serialize(version); + ApiVersionsResponse response = ApiVersionsResponse.parse(buffer, version); + assertEquals(Errors.NONE.code(), response.data().errorCode()); + } } @Test @@ -1773,8 +1783,8 @@ private SaslAuthenticateResponse createSaslAuthenticateResponse() { return new SaslAuthenticateResponse(data); } - private ApiVersionsRequest createApiVersionRequest() { - return new ApiVersionsRequest.Builder().build(); + private ApiVersionsRequest createApiVersionRequest(short version) { + return new ApiVersionsRequest.Builder().build(version); } private ApiVersionsResponse createApiVersionResponse() { diff --git a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java index cc466f7ada236..3998ab7929555 100644 --- a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java +++ b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java @@ -16,41 +16,6 @@ */ package org.apache.kafka.common.security.authenticator; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.nio.ByteBuffer; -import java.nio.channels.SelectionKey; -import java.nio.charset.StandardCharsets; -import java.security.NoSuchAlgorithmException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Base64; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.Base64.Encoder; -import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.IntStream; - -import javax.net.ssl.SSLPeerUnverifiedException; -import javax.security.auth.Subject; -import javax.security.auth.callback.Callback; -import javax.security.auth.callback.CallbackHandler; -import javax.security.auth.callback.NameCallback; -import javax.security.auth.callback.PasswordCallback; -import javax.security.auth.callback.UnsupportedCallbackException; -import javax.security.auth.login.Configuration; -import javax.security.auth.login.AppConfigurationEntry; -import javax.security.auth.login.LoginContext; -import javax.security.auth.login.LoginException; -import javax.security.sasl.SaslClient; -import javax.security.sasl.SaslException; - import org.apache.kafka.clients.NetworkClient; import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.config.SaslConfigs; @@ -60,13 +25,14 @@ import org.apache.kafka.common.config.types.Password; import org.apache.kafka.common.errors.SaslAuthenticationException; import org.apache.kafka.common.errors.SslAuthenticationException; +import org.apache.kafka.common.message.ApiMessageType; import org.apache.kafka.common.message.ApiVersionsRequestData; import org.apache.kafka.common.message.ApiVersionsResponseData; import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersion; import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersionCollection; import org.apache.kafka.common.message.ListOffsetsResponseData; -import org.apache.kafka.common.message.ListOffsetsResponseData.ListOffsetsTopicResponse; import org.apache.kafka.common.message.ListOffsetsResponseData.ListOffsetsPartitionResponse; +import org.apache.kafka.common.message.ListOffsetsResponseData.ListOffsetsTopicResponse; import org.apache.kafka.common.message.RequestHeaderData; import org.apache.kafka.common.message.SaslAuthenticateRequestData; import org.apache.kafka.common.message.SaslHandshakeRequestData; @@ -87,26 +53,28 @@ import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.types.SchemaException; -import org.apache.kafka.common.requests.ListOffsetsResponse; -import org.apache.kafka.common.requests.RequestTestUtils; -import org.apache.kafka.common.security.auth.AuthenticationContext; -import org.apache.kafka.common.security.auth.KafkaPrincipalBuilder; -import org.apache.kafka.common.security.auth.Login; -import org.apache.kafka.common.security.auth.SaslAuthenticationContext; -import org.apache.kafka.common.security.auth.SecurityProtocol; import org.apache.kafka.common.requests.AbstractRequest; import org.apache.kafka.common.requests.AbstractResponse; import org.apache.kafka.common.requests.ApiVersionsRequest; import org.apache.kafka.common.requests.ApiVersionsResponse; +import org.apache.kafka.common.requests.ListOffsetsResponse; import org.apache.kafka.common.requests.MetadataRequest; import org.apache.kafka.common.requests.RequestHeader; +import org.apache.kafka.common.requests.RequestTestUtils; import org.apache.kafka.common.requests.ResponseHeader; import org.apache.kafka.common.requests.SaslAuthenticateRequest; import org.apache.kafka.common.requests.SaslHandshakeRequest; import org.apache.kafka.common.requests.SaslHandshakeResponse; import org.apache.kafka.common.security.JaasContext; import org.apache.kafka.common.security.TestSecurityConfig; +import org.apache.kafka.common.security.auth.AuthenticateCallbackHandler; +import org.apache.kafka.common.security.auth.AuthenticationContext; import org.apache.kafka.common.security.auth.KafkaPrincipal; +import org.apache.kafka.common.security.auth.KafkaPrincipalBuilder; +import org.apache.kafka.common.security.auth.Login; +import org.apache.kafka.common.security.auth.SaslAuthenticationContext; +import org.apache.kafka.common.security.auth.SecurityProtocol; +import org.apache.kafka.common.security.authenticator.TestDigestLoginModule.DigestServerCallbackHandler; import org.apache.kafka.common.security.oauthbearer.OAuthBearerLoginModule; import org.apache.kafka.common.security.oauthbearer.OAuthBearerToken; import org.apache.kafka.common.security.oauthbearer.OAuthBearerTokenCallback; @@ -115,20 +83,17 @@ import org.apache.kafka.common.security.oauthbearer.internals.unsecured.OAuthBearerUnsecuredJws; import org.apache.kafka.common.security.oauthbearer.internals.unsecured.OAuthBearerUnsecuredLoginCallbackHandler; import org.apache.kafka.common.security.plain.PlainLoginModule; +import org.apache.kafka.common.security.plain.internals.PlainServerCallbackHandler; import org.apache.kafka.common.security.scram.ScramCredential; +import org.apache.kafka.common.security.scram.ScramLoginModule; import org.apache.kafka.common.security.scram.internals.ScramCredentialUtils; import org.apache.kafka.common.security.scram.internals.ScramFormatter; -import org.apache.kafka.common.security.scram.ScramLoginModule; import org.apache.kafka.common.security.scram.internals.ScramMechanism; import org.apache.kafka.common.security.token.delegation.TokenInformation; import org.apache.kafka.common.security.token.delegation.internals.DelegationTokenCache; import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.MockTime; import org.apache.kafka.common.utils.SecurityUtils; -import org.apache.kafka.common.security.auth.AuthenticateCallbackHandler; -import org.apache.kafka.common.security.authenticator.TestDigestLoginModule.DigestServerCallbackHandler; -import org.apache.kafka.common.security.plain.internals.PlainServerCallbackHandler; - import org.apache.kafka.common.utils.Time; import org.apache.kafka.common.utils.Utils; import org.apache.kafka.test.TestUtils; @@ -137,6 +102,41 @@ import org.junit.jupiter.api.Test; import org.opentest4j.AssertionFailedError; +import javax.net.ssl.SSLPeerUnverifiedException; +import javax.security.auth.Subject; +import javax.security.auth.callback.Callback; +import javax.security.auth.callback.CallbackHandler; +import javax.security.auth.callback.NameCallback; +import javax.security.auth.callback.PasswordCallback; +import javax.security.auth.callback.UnsupportedCallbackException; +import javax.security.auth.login.AppConfigurationEntry; +import javax.security.auth.login.Configuration; +import javax.security.auth.login.LoginContext; +import javax.security.auth.login.LoginException; +import javax.security.sasl.SaslClient; +import javax.security.sasl.SaslException; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.ByteBuffer; +import java.nio.channels.SelectionKey; +import java.nio.charset.StandardCharsets; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Base64; +import java.util.Base64.Encoder; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + import static org.apache.kafka.common.protocol.ApiKeys.LIST_OFFSETS; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -1903,31 +1903,18 @@ private NioEchoServer startServerApiVersionsUnsupportedByClient(final SecurityPr boolean isScram = ScramMechanism.isScram(saslMechanism); if (isScram) ScramCredentialUtils.createCache(credentialCache, Arrays.asList(saslMechanism)); + + Supplier apiVersionSupplier = () -> { + ApiVersionCollection versionCollection = new ApiVersionCollection(2); + versionCollection.add(new ApiVersion().setApiKey(ApiKeys.SASL_HANDSHAKE.id).setMinVersion((short) 0).setMaxVersion((short) 100)); + versionCollection.add(new ApiVersion().setApiKey(ApiKeys.SASL_AUTHENTICATE.id).setMinVersion((short) 0).setMaxVersion((short) 100)); + return new ApiVersionsResponse(new ApiVersionsResponseData().setApiKeys(versionCollection)); + }; + SaslChannelBuilder serverChannelBuilder = new SaslChannelBuilder(Mode.SERVER, jaasContexts, securityProtocol, listenerName, false, saslMechanism, true, - credentialCache, null, null, time, new LogContext()) { - - @Override - protected SaslServerAuthenticator buildServerAuthenticator(Map configs, - Map callbackHandlers, - String id, - TransportLayer transportLayer, - Map subjects, - Map connectionsMaxReauthMsByMechanism, - ChannelMetadataRegistry metadataRegistry) { - return new SaslServerAuthenticator(configs, callbackHandlers, id, subjects, null, listenerName, - securityProtocol, transportLayer, connectionsMaxReauthMsByMechanism, metadataRegistry, time) { + credentialCache, null, null, time, new LogContext(), apiVersionSupplier); - @Override - protected ApiVersionsResponse apiVersionsResponse() { - ApiVersionCollection versionCollection = new ApiVersionCollection(2); - versionCollection.add(new ApiVersion().setApiKey(ApiKeys.SASL_HANDSHAKE.id).setMinVersion((short) 0).setMaxVersion((short) 100)); - versionCollection.add(new ApiVersion().setApiKey(ApiKeys.SASL_AUTHENTICATE.id).setMinVersion((short) 0).setMaxVersion((short) 100)); - return new ApiVersionsResponse(new ApiVersionsResponseData().setApiKeys(versionCollection)); - } - }; - } - }; serverChannelBuilder.configure(saslServerConfigs); server = new NioEchoServer(listenerName, securityProtocol, new TestSecurityConfig(saslServerConfigs), "localhost", serverChannelBuilder, credentialCache, time); @@ -1945,10 +1932,29 @@ private NioEchoServer startServerWithoutSaslAuthenticateHeader(final SecurityPro boolean isScram = ScramMechanism.isScram(saslMechanism); if (isScram) ScramCredentialUtils.createCache(credentialCache, Arrays.asList(saslMechanism)); + + Supplier apiVersionSupplier = () -> { + ApiVersionsResponse defaultApiVersionResponse = ApiVersionsResponse.defaultApiVersionsResponse( + ApiMessageType.ApiScope.ZK_BROKER); + ApiVersionCollection apiVersions = new ApiVersionCollection(); + for (ApiVersion apiVersion : defaultApiVersionResponse.data().apiKeys()) { + if (apiVersion.apiKey() != ApiKeys.SASL_AUTHENTICATE.id) { + // ApiVersion can NOT be reused in second ApiVersionCollection + // due to the internal pointers it contains. + apiVersions.add(apiVersion.duplicate()); + } + + } + ApiVersionsResponseData data = new ApiVersionsResponseData() + .setErrorCode(Errors.NONE.code()) + .setThrottleTimeMs(0) + .setApiKeys(apiVersions); + return new ApiVersionsResponse(data); + }; + SaslChannelBuilder serverChannelBuilder = new SaslChannelBuilder(Mode.SERVER, jaasContexts, securityProtocol, listenerName, false, saslMechanism, true, - credentialCache, null, null, time, new LogContext()) { - + credentialCache, null, null, time, new LogContext(), apiVersionSupplier) { @Override protected SaslServerAuthenticator buildServerAuthenticator(Map configs, Map callbackHandlers, @@ -1958,27 +1964,7 @@ protected SaslServerAuthenticator buildServerAuthenticator(Map config Map connectionsMaxReauthMsByMechanism, ChannelMetadataRegistry metadataRegistry) { return new SaslServerAuthenticator(configs, callbackHandlers, id, subjects, null, listenerName, - securityProtocol, transportLayer, connectionsMaxReauthMsByMechanism, metadataRegistry, time) { - - @Override - protected ApiVersionsResponse apiVersionsResponse() { - ApiVersionsResponse defaultApiVersionResponse = ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE; - ApiVersionCollection apiVersions = new ApiVersionCollection(); - for (ApiVersion apiVersion : defaultApiVersionResponse.data().apiKeys()) { - if (apiVersion.apiKey() != ApiKeys.SASL_AUTHENTICATE.id) { - // ApiVersion can NOT be reused in second ApiVersionCollection - // due to the internal pointers it contains. - apiVersions.add(apiVersion.duplicate()); - } - - } - ApiVersionsResponseData data = new ApiVersionsResponseData() - .setErrorCode(Errors.NONE.code()) - .setThrottleTimeMs(0) - .setApiKeys(apiVersions); - return new ApiVersionsResponse(data); - } - + securityProtocol, transportLayer, connectionsMaxReauthMsByMechanism, metadataRegistry, time, apiVersionSupplier) { @Override protected void enableKafkaSaslAuthenticateHeaders(boolean flag) { // Don't enable Kafka SASL_AUTHENTICATE headers @@ -2003,7 +1989,7 @@ private void createClientConnectionWithoutSaslAuthenticateHeader(final SecurityP SaslChannelBuilder clientChannelBuilder = new SaslChannelBuilder(Mode.CLIENT, jaasContexts, securityProtocol, listenerName, false, saslMechanism, true, - null, null, null, time, new LogContext()) { + null, null, null, time, new LogContext(), null) { @Override protected SaslClientAuthenticator buildClientAuthenticator(Map configs, @@ -2545,7 +2531,8 @@ public AlternateSaslChannelBuilder(Mode mode, Map jaasConte String clientSaslMechanism, boolean handshakeRequestEnable, CredentialCache credentialCache, DelegationTokenCache tokenCache, Time time) { super(mode, jaasContexts, securityProtocol, listenerName, isInterBrokerListener, clientSaslMechanism, - handshakeRequestEnable, credentialCache, tokenCache, null, time, new LogContext()); + handshakeRequestEnable, credentialCache, tokenCache, null, time, new LogContext(), + () -> ApiVersionsResponse.defaultApiVersionsResponse(ApiMessageType.ApiScope.ZK_BROKER)); } @Override diff --git a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticatorTest.java b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticatorTest.java index e5c46c9973361..d493eef2fc54d 100644 --- a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticatorTest.java +++ b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticatorTest.java @@ -19,6 +19,7 @@ import java.net.InetAddress; import org.apache.kafka.common.config.internals.BrokerSecurityConfigs; import org.apache.kafka.common.errors.IllegalSaslStateException; +import org.apache.kafka.common.message.ApiMessageType; import org.apache.kafka.common.network.ChannelMetadataRegistry; import org.apache.kafka.common.network.ClientInformation; import org.apache.kafka.common.network.DefaultChannelMetadataRegistry; @@ -27,6 +28,7 @@ import org.apache.kafka.common.network.TransportLayer; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.requests.ApiVersionsRequest; +import org.apache.kafka.common.requests.ApiVersionsResponse; import org.apache.kafka.common.requests.RequestTestUtils; import org.apache.kafka.common.security.auth.AuthenticateCallbackHandler; import org.apache.kafka.common.security.auth.SecurityProtocol; @@ -152,15 +154,17 @@ private void testApiVersionsRequest(short version, String expectedSoftwareName, } private SaslServerAuthenticator setupAuthenticator(Map configs, TransportLayer transportLayer, - String mechanism, ChannelMetadataRegistry metadataRegistry) throws IOException { + String mechanism, ChannelMetadataRegistry metadataRegistry) { TestJaasConfig jaasConfig = new TestJaasConfig(); jaasConfig.addEntry("jaasContext", PlainLoginModule.class.getName(), new HashMap()); Map subjects = Collections.singletonMap(mechanism, new Subject()); Map callbackHandlers = Collections.singletonMap( mechanism, new SaslServerCallbackHandler()); + ApiVersionsResponse apiVersionsResponse = ApiVersionsResponse.defaultApiVersionsResponse( + ApiMessageType.ApiScope.ZK_BROKER); return new SaslServerAuthenticator(configs, callbackHandlers, "node", subjects, null, new ListenerName("ssl"), SecurityProtocol.SASL_SSL, transportLayer, Collections.emptyMap(), - metadataRegistry, Time.SYSTEM); + metadataRegistry, Time.SYSTEM, () -> apiVersionsResponse); } } diff --git a/core/src/main/scala/kafka/api/ApiVersion.scala b/core/src/main/scala/kafka/api/ApiVersion.scala index e89f9fb46a555..9e1bcdadf8c7e 100644 --- a/core/src/main/scala/kafka/api/ApiVersion.scala +++ b/core/src/main/scala/kafka/api/ApiVersion.scala @@ -21,10 +21,9 @@ import org.apache.kafka.clients.NodeApiVersions import org.apache.kafka.common.config.ConfigDef.Validator import org.apache.kafka.common.config.ConfigException import org.apache.kafka.common.feature.{Features, FinalizedVersionRange, SupportedVersionRange} -import org.apache.kafka.common.protocol.Errors -import org.apache.kafka.common.record.{RecordBatch, RecordVersion} -import org.apache.kafka.common.requests.{AbstractResponse, ApiVersionsResponse} -import org.apache.kafka.common.requests.ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE +import org.apache.kafka.common.message.ApiMessageType.ApiScope +import org.apache.kafka.common.record.RecordVersion +import org.apache.kafka.common.requests.ApiVersionsResponse /** * This class contains the different Kafka versions. @@ -147,52 +146,46 @@ object ApiVersion { } } - def apiVersionsResponse(throttleTimeMs: Int, - maxMagic: Byte, - latestSupportedFeatures: Features[SupportedVersionRange], - controllerApiVersions: Option[NodeApiVersions]): ApiVersionsResponse = { + def apiVersionsResponse( + throttleTimeMs: Int, + minRecordVersion: RecordVersion, + latestSupportedFeatures: Features[SupportedVersionRange], + controllerApiVersions: Option[NodeApiVersions], + scope: ApiScope + ): ApiVersionsResponse = { apiVersionsResponse( throttleTimeMs, - maxMagic, + minRecordVersion, latestSupportedFeatures, Features.emptyFinalizedFeatures, ApiVersionsResponse.UNKNOWN_FINALIZED_FEATURES_EPOCH, - controllerApiVersions + controllerApiVersions, + scope ) } - def apiVersionsResponse(throttleTimeMs: Int, - maxMagic: Byte, - latestSupportedFeatures: Features[SupportedVersionRange], - finalizedFeatures: Features[FinalizedVersionRange], - finalizedFeaturesEpoch: Long, - controllerApiVersions: Option[NodeApiVersions]): ApiVersionsResponse = { + def apiVersionsResponse( + throttleTimeMs: Int, + minRecordVersion: RecordVersion, + latestSupportedFeatures: Features[SupportedVersionRange], + finalizedFeatures: Features[FinalizedVersionRange], + finalizedFeaturesEpoch: Long, + controllerApiVersions: Option[NodeApiVersions], + scope: ApiScope + ): ApiVersionsResponse = { val apiKeys = controllerApiVersions match { - case None => ApiVersionsResponse.defaultApiKeys(maxMagic) - case Some(controllerApiVersion) => ApiVersionsResponse.intersectControllerApiVersions( - maxMagic, controllerApiVersion.allSupportedApiVersions()) + case None => ApiVersionsResponse.filterApis(minRecordVersion, scope) + case Some(controllerApiVersion) => ApiVersionsResponse.intersectForwardableApis( + scope, minRecordVersion, controllerApiVersion.allSupportedApiVersions()) } - if (maxMagic == RecordBatch.CURRENT_MAGIC_VALUE && - throttleTimeMs == AbstractResponse.DEFAULT_THROTTLE_TIME) { - new ApiVersionsResponse( - ApiVersionsResponse.createApiVersionsResponseData( - DEFAULT_API_VERSIONS_RESPONSE.throttleTimeMs, - Errors.forCode(DEFAULT_API_VERSIONS_RESPONSE.data.errorCode), - apiKeys, - latestSupportedFeatures, - finalizedFeatures, - finalizedFeaturesEpoch)) - } else { - new ApiVersionsResponse( - ApiVersionsResponse.createApiVersionsResponseData( - throttleTimeMs, - Errors.NONE, - apiKeys, - latestSupportedFeatures, - finalizedFeatures, - finalizedFeaturesEpoch)) - } + ApiVersionsResponse.createApiVersionsResponse( + throttleTimeMs, + apiKeys, + latestSupportedFeatures, + finalizedFeatures, + finalizedFeaturesEpoch + ) } } diff --git a/core/src/main/scala/kafka/network/RequestChannel.scala b/core/src/main/scala/kafka/network/RequestChannel.scala index 7d3112560d851..db35e75e4b033 100644 --- a/core/src/main/scala/kafka/network/RequestChannel.scala +++ b/core/src/main/scala/kafka/network/RequestChannel.scala @@ -30,6 +30,7 @@ import kafka.utils.{Logging, NotNothing, Pool} import kafka.utils.Implicits._ import org.apache.kafka.common.config.ConfigResource import org.apache.kafka.common.memory.MemoryPool +import org.apache.kafka.common.message.ApiMessageType.ApiScope import org.apache.kafka.common.message.IncrementalAlterConfigsRequestData import org.apache.kafka.common.message.IncrementalAlterConfigsRequestData._ import org.apache.kafka.common.network.Send @@ -59,16 +60,19 @@ object RequestChannel extends Logging { val sanitizedUser: String = Sanitizer.sanitize(principal.getName) } - class Metrics(allowControllerOnlyApis: Boolean = false) { + class Metrics(enabledApis: Iterable[ApiKeys]) { + def this(scope: ApiScope) = { + this(ApiKeys.apisInScope(scope).asScala) + } private val metricsMap = mutable.Map[String, RequestMetrics]() - (ApiKeys.values.toSeq.filter(!_.isControllerOnlyApi || allowControllerOnlyApis).map(_.name) ++ - Seq(RequestMetrics.consumerFetchMetricName, RequestMetrics.followFetchMetricName)).foreach { name => + (enabledApis.map(_.name) ++ + Seq(RequestMetrics.consumerFetchMetricName, RequestMetrics.followFetchMetricName)).foreach { name => metricsMap.put(name, new RequestMetrics(name)) } - def apply(metricName: String) = metricsMap(metricName) + def apply(metricName: String): RequestMetrics = metricsMap(metricName) def close(): Unit = { metricsMap.values.foreach(_.removeMetrics()) @@ -296,8 +300,6 @@ object RequestChannel extends Logging { def responseLog: Option[JsonNode] = None def onComplete: Option[Send => Unit] = None - - override def toString: String } /** responseLogValue should only be defined if request logging is enabled */ @@ -337,9 +339,8 @@ object RequestChannel extends Logging { class RequestChannel(val queueSize: Int, val metricNamePrefix: String, time: Time, - allowControllerOnlyApis: Boolean = false) extends KafkaMetricsGroup { + val metrics: RequestChannel.Metrics) extends KafkaMetricsGroup { import RequestChannel._ - val metrics = new RequestChannel.Metrics(allowControllerOnlyApis) private val requestQueue = new ArrayBlockingQueue[BaseRequest](queueSize) private val processors = new ConcurrentHashMap[Int, Processor]() val requestQueueSizeMetricName = metricNamePrefix.concat(RequestQueueSizeMetric) diff --git a/core/src/main/scala/kafka/network/SocketServer.scala b/core/src/main/scala/kafka/network/SocketServer.scala index 72c5141445f2a..9220fc06141bd 100644 --- a/core/src/main/scala/kafka/network/SocketServer.scala +++ b/core/src/main/scala/kafka/network/SocketServer.scala @@ -33,7 +33,7 @@ import kafka.network.Processor._ import kafka.network.RequestChannel.{CloseConnectionResponse, EndThrottlingResponse, NoOpResponse, SendResponse, StartThrottlingResponse} import kafka.network.SocketServer._ import kafka.security.CredentialProvider -import kafka.server.{BrokerReconfigurable, KafkaConfig} +import kafka.server.{ApiVersionManager, BrokerReconfigurable, KafkaConfig} import kafka.utils.Implicits._ import kafka.utils._ import org.apache.kafka.common.config.ConfigException @@ -78,15 +78,14 @@ class SocketServer(val config: KafkaConfig, val metrics: Metrics, val time: Time, val credentialProvider: CredentialProvider, - allowControllerOnlyApis: Boolean = false, - controllerSocketServer: Boolean = false) + val apiVersionManager: ApiVersionManager) extends Logging with KafkaMetricsGroup with BrokerReconfigurable { private val maxQueuedRequests = config.queuedMaxRequests private val nodeId = config.brokerId - private val logContext = new LogContext(s"[SocketServer ${if (controllerSocketServer) "controller" else "broker"}Id=${nodeId}] ") + private val logContext = new LogContext(s"[SocketServer listenerType=${apiVersionManager.apiScope}, nodeId=$nodeId] ") this.logIdent = logContext.logPrefix @@ -98,12 +97,12 @@ class SocketServer(val config: KafkaConfig, // data-plane private val dataPlaneProcessors = new ConcurrentHashMap[Int, Processor]() private[network] val dataPlaneAcceptors = new ConcurrentHashMap[EndPoint, Acceptor]() - val dataPlaneRequestChannel = new RequestChannel(maxQueuedRequests, DataPlaneMetricPrefix, time, allowControllerOnlyApis) + val dataPlaneRequestChannel = new RequestChannel(maxQueuedRequests, DataPlaneMetricPrefix, time, apiVersionManager.newRequestMetrics) // control-plane private var controlPlaneProcessorOpt : Option[Processor] = None private[network] var controlPlaneAcceptorOpt : Option[Acceptor] = None val controlPlaneRequestChannelOpt: Option[RequestChannel] = config.controlPlaneListenerName.map(_ => - new RequestChannel(20, ControlPlaneMetricPrefix, time, allowControllerOnlyApis)) + new RequestChannel(20, ControlPlaneMetricPrefix, time, apiVersionManager.newRequestMetrics)) private var nextProcessorId = 0 val connectionQuotas = new ConnectionQuotas(config, time, metrics) @@ -438,8 +437,9 @@ class SocketServer(val config: KafkaConfig, credentialProvider, memoryPool, logContext, - isPrivilegedListener = isPrivilegedListener, - allowControllerOnlyApis = allowControllerOnlyApis + Processor.ConnectionQueueSize, + isPrivilegedListener, + apiVersionManager ) } @@ -772,7 +772,6 @@ private[kafka] object Processor { val IdlePercentMetricName = "IdlePercent" val NetworkProcessorMetricTag = "networkProcessor" val ListenerMetricTag = "listener" - val ConnectionQueueSize = 20 } @@ -800,9 +799,9 @@ private[kafka] class Processor(val id: Int, credentialProvider: CredentialProvider, memoryPool: MemoryPool, logContext: LogContext, - connectionQueueSize: Int = ConnectionQueueSize, - isPrivilegedListener: Boolean = false, - allowControllerOnlyApis: Boolean = false) extends AbstractServerThread(connectionQuotas) with KafkaMetricsGroup { + connectionQueueSize: Int, + isPrivilegedListener: Boolean, + apiVersionManager: ApiVersionManager) extends AbstractServerThread(connectionQuotas) with KafkaMetricsGroup { private object ConnectionId { def fromString(s: String): Option[ConnectionId] = s.split("-") match { @@ -842,14 +841,19 @@ private[kafka] class Processor(val id: Int, metrics.addMetric(expiredConnectionsKilledCountMetricName, expiredConnectionsKilledCount) private val selector = createSelector( - ChannelBuilders.serverChannelBuilder(listenerName, + ChannelBuilders.serverChannelBuilder( + listenerName, listenerName == config.interBrokerListenerName, securityProtocol, config, credentialProvider.credentialCache, credentialProvider.tokenCache, time, - logContext)) + logContext, + () => apiVersionManager.apiVersionResponse(throttleTimeMs = 0) + ) + ) + // Visible to override for testing protected[network] def createSelector(channelBuilder: ChannelBuilder): KSelector = { channelBuilder match { @@ -993,10 +997,10 @@ private[kafka] class Processor(val id: Int, protected def parseRequestHeader(buffer: ByteBuffer): RequestHeader = { val header = RequestHeader.parse(buffer) - if (!header.apiKey.isControllerOnlyApi || allowControllerOnlyApis) { + if (apiVersionManager.isApiEnabled(header.apiKey)) { header } else { - throw new InvalidRequestException("Received request for KIP-500 controller-only api key " + header.apiKey) + throw new InvalidRequestException(s"Received request api key ${header.apiKey} which is not enabled") } } diff --git a/core/src/main/scala/kafka/server/ApiVersionManager.scala b/core/src/main/scala/kafka/server/ApiVersionManager.scala new file mode 100644 index 0000000000000..49a030e0780fd --- /dev/null +++ b/core/src/main/scala/kafka/server/ApiVersionManager.scala @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package kafka.server + +import kafka.api.ApiVersion +import kafka.network +import kafka.network.RequestChannel +import org.apache.kafka.common.message.ApiMessageType.ApiScope +import org.apache.kafka.common.message.ApiVersionsResponseData +import org.apache.kafka.common.protocol.ApiKeys +import org.apache.kafka.common.requests.ApiVersionsResponse + +import scala.jdk.CollectionConverters._ + +trait ApiVersionManager { + def apiScope: ApiScope + def enabledApis: collection.Set[ApiKeys] + def apiVersionResponse(throttleTimeMs: Int): ApiVersionsResponse + def isApiEnabled(apiKey: ApiKeys): Boolean = enabledApis.contains(apiKey) + def newRequestMetrics: RequestChannel.Metrics = new network.RequestChannel.Metrics(enabledApis) +} + +object ApiVersionManager { + def apply( + apiScope: ApiScope, + config: KafkaConfig, + forwardingManager: Option[ForwardingManager], + features: BrokerFeatures, + featureCache: FinalizedFeatureCache + ): ApiVersionManager = { + new DefaultApiVersionManager( + apiScope, + config.interBrokerProtocolVersion, + forwardingManager, + features, + featureCache + ) + } +} + +class SimpleApiVersionManager( + val enabledApis: collection.Set[ApiKeys] +) extends ApiVersionManager { + + def this(scope: ApiScope) = { + this(ApiKeys.apisInScope(scope).asScala) + } + + private val apiVersions = ApiVersionsResponse.collectApis(enabledApis.asJava) + + override def apiVersionResponse(requestThrottleMs: Int): ApiVersionsResponse = { + ApiVersionsResponse.createApiVersionsResponse(0, apiVersions) + } +} + +class DefaultApiVersionManager( + val apiScope: ApiScope, + interBrokerProtocolVersion: ApiVersion, + forwardingManager: Option[ForwardingManager], + features: BrokerFeatures, + featureCache: FinalizedFeatureCache +) extends ApiVersionManager { + + override def apiVersionResponse(throttleTimeMs: Int): ApiVersionsResponse = { + val supportedFeatures = features.supportedFeatures + val finalizedFeaturesOpt = featureCache.get + val controllerApiVersions = forwardingManager.flatMap(_.controllerApiVersions) + + val response = finalizedFeaturesOpt match { + case Some(finalizedFeatures) => ApiVersion.apiVersionsResponse( + throttleTimeMs, + interBrokerProtocolVersion.recordVersion, + supportedFeatures, + finalizedFeatures.features, + finalizedFeatures.epoch, + controllerApiVersions, + apiScope) + case None => ApiVersion.apiVersionsResponse( + throttleTimeMs, + interBrokerProtocolVersion.recordVersion, + supportedFeatures, + controllerApiVersions, + apiScope) + } + + // This is a temporary workaround in order to allow testing of forwarding + // in integration tests. We can remove this after the KIP-500 controller + // is available for integration testing. + if (forwardingManager.isDefined) { + response.data.apiKeys.add( + new ApiVersionsResponseData.ApiVersion() + .setApiKey(ApiKeys.ENVELOPE.id) + .setMinVersion(ApiKeys.ENVELOPE.oldestVersion) + .setMaxVersion(ApiKeys.ENVELOPE.latestVersion) + ) + } + + response + } + + override def enabledApis: collection.Set[ApiKeys] = { + forwardingManager match { + case Some(_) => ApiKeys.apisInScope(apiScope).asScala ++ Set(ApiKeys.ENVELOPE) + case None => ApiKeys.apisInScope(apiScope).asScala + } + } +} diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala index 5e8340e6b6163..5a926d4ace403 100644 --- a/core/src/main/scala/kafka/server/KafkaApis.scala +++ b/core/src/main/scala/kafka/server/KafkaApis.scala @@ -61,7 +61,7 @@ import org.apache.kafka.common.message.ListOffsetsResponseData.{ListOffsetsParti import org.apache.kafka.common.message.MetadataResponseData.{MetadataResponsePartition, MetadataResponseTopic} import org.apache.kafka.common.message.OffsetForLeaderEpochRequestData.OffsetForLeaderTopic import org.apache.kafka.common.message.OffsetForLeaderEpochResponseData.{EpochEndOffset, OffsetForLeaderTopicResult, OffsetForLeaderTopicResultCollection} -import org.apache.kafka.common.message.{AddOffsetsToTxnResponseData, AlterClientQuotasResponseData, AlterConfigsResponseData, AlterPartitionReassignmentsResponseData, AlterReplicaLogDirsResponseData, ApiVersionsResponseData, CreateAclsResponseData, CreatePartitionsResponseData, CreateTopicsResponseData, DeleteAclsResponseData, DeleteGroupsResponseData, DeleteRecordsResponseData, DeleteTopicsResponseData, DescribeAclsResponseData, DescribeClientQuotasResponseData, DescribeClusterResponseData, DescribeConfigsResponseData, DescribeGroupsResponseData, DescribeLogDirsResponseData, DescribeProducersResponseData, EndTxnResponseData, ExpireDelegationTokenResponseData, FindCoordinatorResponseData, HeartbeatResponseData, InitProducerIdResponseData, JoinGroupResponseData, LeaveGroupResponseData, ListGroupsResponseData, ListOffsetsResponseData, ListPartitionReassignmentsResponseData, OffsetCommitRequestData, OffsetCommitResponseData, OffsetDeleteResponseData, OffsetForLeaderEpochResponseData, RenewDelegationTokenResponseData, SaslAuthenticateResponseData, SaslHandshakeResponseData, StopReplicaResponseData, SyncGroupResponseData, UpdateMetadataResponseData} +import org.apache.kafka.common.message.{AddOffsetsToTxnResponseData, AlterClientQuotasResponseData, AlterConfigsResponseData, AlterPartitionReassignmentsResponseData, AlterReplicaLogDirsResponseData, CreateAclsResponseData, CreatePartitionsResponseData, CreateTopicsResponseData, DeleteAclsResponseData, DeleteGroupsResponseData, DeleteRecordsResponseData, DeleteTopicsResponseData, DescribeAclsResponseData, DescribeClientQuotasResponseData, DescribeClusterResponseData, DescribeConfigsResponseData, DescribeGroupsResponseData, DescribeLogDirsResponseData, DescribeProducersResponseData, EndTxnResponseData, ExpireDelegationTokenResponseData, FindCoordinatorResponseData, HeartbeatResponseData, InitProducerIdResponseData, JoinGroupResponseData, LeaveGroupResponseData, ListGroupsResponseData, ListOffsetsResponseData, ListPartitionReassignmentsResponseData, OffsetCommitRequestData, OffsetCommitResponseData, OffsetDeleteResponseData, OffsetForLeaderEpochResponseData, RenewDelegationTokenResponseData, SaslAuthenticateResponseData, SaslHandshakeResponseData, StopReplicaResponseData, SyncGroupResponseData, UpdateMetadataResponseData} import org.apache.kafka.common.metrics.Metrics import org.apache.kafka.common.network.{ListenerName, Send} import org.apache.kafka.common.protocol.{ApiKeys, Errors} @@ -110,8 +110,7 @@ class KafkaApis(val requestChannel: RequestChannel, val clusterId: String, time: Time, val tokenManager: DelegationTokenManager, - val brokerFeatures: BrokerFeatures, - val finalizedFeatureCache: FinalizedFeatureCache) extends ApiRequestHandler with Logging { + val apiVersionManager: ApiVersionManager) extends ApiRequestHandler with Logging { metadataSupport.ensureConsistentWith(config) @@ -158,6 +157,12 @@ class KafkaApis(val requestChannel: RequestChannel, trace(s"Handling request:${request.requestDesc(true)} from connection ${request.context.connectionId};" + s"securityProtocol:${request.context.securityProtocol},principal:${request.context.principal}") + if (!apiVersionManager.isApiEnabled(request.header.apiKey)) { + // The socket server will reject APIs which are not exposed in this scope and close the connection + // before handing them to the request handler, so this path should not be exercised in practice + throw new IllegalStateException(s"API ${request.header.apiKey} is not enabled") + } + request.header.apiKey match { case ApiKeys.PRODUCE => handleProduceRequest(request) case ApiKeys.FETCH => handleFetchRequest(request) @@ -217,17 +222,7 @@ class KafkaApis(val requestChannel: RequestChannel, case ApiKeys.DESCRIBE_CLUSTER => handleDescribeCluster(request) case ApiKeys.DESCRIBE_PRODUCERS => handleDescribeProducersRequest(request) case ApiKeys.UNREGISTER_BROKER => maybeForwardToController(request, handleUnregisterBrokerRequest) - - // Handle requests that should have been sent to the KIP-500 controller. - // Until we are ready to integrate the Raft layer, these APIs are treated as - // unexpected and we just close the connection. - case ApiKeys.VOTE => requestHelper.closeConnection(request, util.Collections.emptyMap()) - case ApiKeys.BEGIN_QUORUM_EPOCH => requestHelper.closeConnection(request, util.Collections.emptyMap()) - case ApiKeys.END_QUORUM_EPOCH => requestHelper.closeConnection(request, util.Collections.emptyMap()) - case ApiKeys.DESCRIBE_QUORUM => requestHelper.closeConnection(request, util.Collections.emptyMap()) - case ApiKeys.FETCH_SNAPSHOT => requestHelper.closeConnection(request, util.Collections.emptyMap()) - case ApiKeys.BROKER_REGISTRATION => requestHelper.closeConnection(request, util.Collections.emptyMap()) - case ApiKeys.BROKER_HEARTBEAT => requestHelper.closeConnection(request, util.Collections.emptyMap()) + case _ => throw new IllegalStateException(s"No handler for request api key ${request.header.apiKey}") } } catch { case e: FatalExitError => throw e @@ -1686,39 +1681,12 @@ class KafkaApis(val requestChannel: RequestChannel, // ApiVersionRequest is not available. def createResponseCallback(requestThrottleMs: Int): ApiVersionsResponse = { val apiVersionRequest = request.body[ApiVersionsRequest] - if (apiVersionRequest.hasUnsupportedRequestVersion) + if (apiVersionRequest.hasUnsupportedRequestVersion) { apiVersionRequest.getErrorResponse(requestThrottleMs, Errors.UNSUPPORTED_VERSION.exception) - else if (!apiVersionRequest.isValid) + } else if (!apiVersionRequest.isValid) { apiVersionRequest.getErrorResponse(requestThrottleMs, Errors.INVALID_REQUEST.exception) - else { - val supportedFeatures = brokerFeatures.supportedFeatures - val finalizedFeaturesOpt = finalizedFeatureCache.get - val controllerApiVersions = metadataSupport.forwardingManager.flatMap(_.controllerApiVersions) - - val apiVersionsResponse = - finalizedFeaturesOpt match { - case Some(finalizedFeatures) => ApiVersion.apiVersionsResponse( - requestThrottleMs, - config.interBrokerProtocolVersion.recordVersion.value, - supportedFeatures, - finalizedFeatures.features, - finalizedFeatures.epoch, - controllerApiVersions) - case None => ApiVersion.apiVersionsResponse( - requestThrottleMs, - config.interBrokerProtocolVersion.recordVersion.value, - supportedFeatures, - controllerApiVersions) - } - if (request.context.fromPrivilegedListener) { - apiVersionsResponse.data.apiKeys().add( - new ApiVersionsResponseData.ApiVersion() - .setApiKey(ApiKeys.ENVELOPE.id) - .setMinVersion(ApiKeys.ENVELOPE.oldestVersion()) - .setMaxVersion(ApiKeys.ENVELOPE.latestVersion()) - ) - } - apiVersionsResponse + } else { + apiVersionManager.apiVersionResponse(requestThrottleMs) } } requestHelper.sendResponseMaybeThrottle(request, createResponseCallback) diff --git a/core/src/main/scala/kafka/server/KafkaServer.scala b/core/src/main/scala/kafka/server/KafkaServer.scala index 3ad36874385e0..6885012251daa 100755 --- a/core/src/main/scala/kafka/server/KafkaServer.scala +++ b/core/src/main/scala/kafka/server/KafkaServer.scala @@ -37,6 +37,7 @@ import kafka.utils._ import kafka.zk.{AdminZkClient, BrokerInfo, KafkaZkClient} import org.apache.kafka.clients.{ApiVersions, ClientDnsLookup, ManualMetadataUpdater, NetworkClient, NetworkClientUtils} import org.apache.kafka.common.internals.Topic +import org.apache.kafka.common.message.ApiMessageType.ApiScope import org.apache.kafka.common.message.ControlledShutdownRequestData import org.apache.kafka.common.metrics.Metrics import org.apache.kafka.common.network._ @@ -157,7 +158,6 @@ class KafkaServer( private var _featureChangeListener: FinalizedFeatureChangeListener = null val brokerFeatures: BrokerFeatures = BrokerFeatures.createDefault() - val featureCache: FinalizedFeatureCache = new FinalizedFeatureCache(brokerFeatures) def clusterId: String = _clusterId @@ -256,14 +256,32 @@ class KafkaServer( tokenCache = new DelegationTokenCache(ScramMechanism.mechanismNames) credentialProvider = new CredentialProvider(ScramMechanism.mechanismNames, tokenCache) + if (enableForwarding) { + this.forwardingManager = Some(ForwardingManager( + config, + metadataCache, + time, + metrics, + threadNamePrefix + )) + forwardingManager.foreach(_.start()) + } + + val apiVersionManager = ApiVersionManager( + ApiScope.ZK_BROKER, + config, + forwardingManager, + brokerFeatures, + featureCache + ) + // Create and start the socket server acceptor threads so that the bound port is known. // Delay starting processors until the end of the initialization sequence to ensure // that credentials have been loaded before processing authentications. // // Note that we allow the use of KIP-500 controller APIs when forwarding is enabled // so that the Envelope request is exposed. This is only used in testing currently. - socketServer = new SocketServer(config, metrics, time, credentialProvider, - allowControllerOnlyApis = enableForwarding) + socketServer = new SocketServer(config, metrics, time, credentialProvider, apiVersionManager) socketServer.startup(startProcessingRequests = false) /* start replica manager */ @@ -300,18 +318,6 @@ class KafkaServer( kafkaController = new KafkaController(config, zkClient, time, metrics, brokerInfo, brokerEpoch, tokenManager, brokerFeatures, featureCache, threadNamePrefix) kafkaController.startup() - /* start forwarding manager */ - if (enableForwarding) { - this.forwardingManager = Some(ForwardingManager( - config, - metadataCache, - time, - metrics, - threadNamePrefix - )) - forwardingManager.foreach(_.start()) - } - adminManager = new ZkAdminManager(config, metrics, metadataCache, zkClient) /* start group coordinator */ @@ -363,7 +369,7 @@ class KafkaServer( val zkSupport = ZkSupport(adminManager, kafkaController, zkClient, forwardingManager, metadataCache) dataPlaneRequestProcessor = new KafkaApis(socketServer.dataPlaneRequestChannel, zkSupport, replicaManager, groupCoordinator, transactionCoordinator, autoTopicCreationManager, config.brokerId, config, configRepository, metadataCache, metrics, authorizer, quotaManagers, - fetchManager, brokerTopicStats, clusterId, time, tokenManager, brokerFeatures, featureCache) + fetchManager, brokerTopicStats, clusterId, time, tokenManager, apiVersionManager) dataPlaneRequestHandlerPool = new KafkaRequestHandlerPool(config.brokerId, socketServer.dataPlaneRequestChannel, dataPlaneRequestProcessor, time, config.numIoThreads, s"${SocketServer.DataPlaneMetricPrefix}RequestHandlerAvgIdlePercent", SocketServer.DataPlaneThreadPrefix) @@ -371,7 +377,7 @@ class KafkaServer( socketServer.controlPlaneRequestChannelOpt.foreach { controlPlaneRequestChannel => controlPlaneRequestProcessor = new KafkaApis(controlPlaneRequestChannel, zkSupport, replicaManager, groupCoordinator, transactionCoordinator, autoTopicCreationManager, config.brokerId, config, configRepository, metadataCache, metrics, authorizer, quotaManagers, - fetchManager, brokerTopicStats, clusterId, time, tokenManager, brokerFeatures, featureCache) + fetchManager, brokerTopicStats, clusterId, time, tokenManager, apiVersionManager) controlPlaneRequestHandlerPool = new KafkaRequestHandlerPool(config.brokerId, socketServer.controlPlaneRequestChannelOpt.get, controlPlaneRequestProcessor, time, 1, s"${SocketServer.ControlPlaneMetricPrefix}RequestHandlerAvgIdlePercent", SocketServer.ControlPlaneThreadPrefix) diff --git a/core/src/main/scala/kafka/tools/TestRaftRequestHandler.scala b/core/src/main/scala/kafka/tools/TestRaftRequestHandler.scala index ddfa1fa102f03..e4dec2ee66af3 100644 --- a/core/src/main/scala/kafka/tools/TestRaftRequestHandler.scala +++ b/core/src/main/scala/kafka/tools/TestRaftRequestHandler.scala @@ -20,7 +20,7 @@ package kafka.tools import kafka.network.RequestChannel import kafka.network.RequestConvertToJson import kafka.raft.RaftManager -import kafka.server.ApiRequestHandler +import kafka.server.{ApiRequestHandler, ApiVersionManager} import kafka.utils.Logging import org.apache.kafka.common.internals.FatalExitError import org.apache.kafka.common.message.{BeginQuorumEpochResponseData, EndQuorumEpochResponseData, FetchResponseData, FetchSnapshotResponseData, VoteResponseData} @@ -38,6 +38,7 @@ class TestRaftRequestHandler( raftManager: RaftManager[_], requestChannel: RequestChannel, time: Time, + apiVersionManager: ApiVersionManager ) extends ApiRequestHandler with Logging { override def handle(request: RequestChannel.Request): Unit = { @@ -45,6 +46,7 @@ class TestRaftRequestHandler( trace(s"Handling request:${request.requestDesc(true)} from connection ${request.context.connectionId};" + s"securityProtocol:${request.context.securityProtocol},principal:${request.context.principal}") request.header.apiKey match { + case ApiKeys.API_VERSIONS => handleApiVersions(request) case ApiKeys.VOTE => handleVote(request) case ApiKeys.BEGIN_QUORUM_EPOCH => handleBeginQuorumEpoch(request) case ApiKeys.END_QUORUM_EPOCH => handleEndQuorumEpoch(request) @@ -62,6 +64,10 @@ class TestRaftRequestHandler( } } + private def handleApiVersions(request: RequestChannel.Request): Unit = { + sendResponse(request, Some(apiVersionManager.apiVersionResponse(throttleTimeMs = 0))) + } + private def handleVote(request: RequestChannel.Request): Unit = { handle(request, response => new VoteResponse(response.asInstanceOf[VoteResponseData])) } diff --git a/core/src/main/scala/kafka/tools/TestRaftServer.scala b/core/src/main/scala/kafka/tools/TestRaftServer.scala index 2391ca4c380ed..7dee9537c9084 100644 --- a/core/src/main/scala/kafka/tools/TestRaftServer.scala +++ b/core/src/main/scala/kafka/tools/TestRaftServer.scala @@ -24,9 +24,10 @@ import joptsimple.OptionException import kafka.network.SocketServer import kafka.raft.{KafkaRaftManager, RaftManager} import kafka.security.CredentialProvider -import kafka.server.{KafkaConfig, KafkaRequestHandlerPool, MetaProperties} +import kafka.server.{KafkaConfig, KafkaRequestHandlerPool, MetaProperties, SimpleApiVersionManager} import kafka.utils.{CommandDefaultOptions, CommandLineUtils, CoreUtils, Exit, Logging, ShutdownableThread} import org.apache.kafka.common.errors.InvalidConfigurationException +import org.apache.kafka.common.message.ApiMessageType.ApiScope import org.apache.kafka.common.metrics.Metrics import org.apache.kafka.common.metrics.stats.Percentiles.BucketSizing import org.apache.kafka.common.metrics.stats.{Meter, Percentile, Percentiles} @@ -68,7 +69,8 @@ class TestRaftServer( tokenCache = new DelegationTokenCache(ScramMechanism.mechanismNames) credentialProvider = new CredentialProvider(ScramMechanism.mechanismNames, tokenCache) - socketServer = new SocketServer(config, metrics, time, credentialProvider, allowControllerOnlyApis = true) + val apiVersionManager = new SimpleApiVersionManager(ApiScope.RAFT) + socketServer = new SocketServer(config, metrics, time, credentialProvider, apiVersionManager) socketServer.startup(startProcessingRequests = false) val metaProperties = MetaProperties( @@ -96,7 +98,8 @@ class TestRaftServer( val requestHandler = new TestRaftRequestHandler( raftManager, socketServer.dataPlaneRequestChannel, - time + time, + apiVersionManager ) dataPlaneRequestHandlerPool = new KafkaRequestHandlerPool( diff --git a/core/src/test/scala/integration/kafka/admin/BrokerApiVersionsCommandTest.scala b/core/src/test/scala/integration/kafka/admin/BrokerApiVersionsCommandTest.scala index 6224591addb3e..2db694f21403e 100644 --- a/core/src/test/scala/integration/kafka/admin/BrokerApiVersionsCommandTest.scala +++ b/core/src/test/scala/integration/kafka/admin/BrokerApiVersionsCommandTest.scala @@ -55,7 +55,7 @@ class BrokerApiVersionsCommandTest extends KafkaServerTestHarness { assertTrue(lineIter.hasNext) assertEquals(s"$brokerList (id: 0 rack: null) -> (", lineIter.next()) val nodeApiVersions = NodeApiVersions.create - val enabledApis = ApiKeys.brokerApis.asScala + val enabledApis = ApiKeys.zkBrokerApis.asScala for (apiKey <- enabledApis) { val apiVersion = nodeApiVersions.apiVersion(apiKey) assertNotNull(apiVersion) diff --git a/core/src/test/scala/integration/kafka/server/GssapiAuthenticationTest.scala b/core/src/test/scala/integration/kafka/server/GssapiAuthenticationTest.scala index 46156f1c4a7c3..3bcea3f29e900 100644 --- a/core/src/test/scala/integration/kafka/server/GssapiAuthenticationTest.scala +++ b/core/src/test/scala/integration/kafka/server/GssapiAuthenticationTest.scala @@ -29,7 +29,9 @@ import org.apache.kafka.clients.CommonClientConfigs import org.apache.kafka.common.TopicPartition import org.apache.kafka.common.config.SaslConfigs import org.apache.kafka.common.errors.SaslAuthenticationException +import org.apache.kafka.common.message.ApiMessageType.ApiScope import org.apache.kafka.common.network._ +import org.apache.kafka.common.requests.ApiVersionsResponse import org.apache.kafka.common.security.{JaasContext, TestSecurityConfig} import org.apache.kafka.common.security.auth.{Login, SecurityProtocol} import org.apache.kafka.common.security.kerberos.KerberosLogin @@ -233,7 +235,8 @@ class GssapiAuthenticationTest extends IntegrationTestHarness with SaslSetup { val config = new TestSecurityConfig(clientConfig) val jaasContexts = Collections.singletonMap("GSSAPI", JaasContext.loadClientContext(config.values())) val channelBuilder = new SaslChannelBuilder(Mode.CLIENT, jaasContexts, securityProtocol, - null, false, kafkaClientSaslMechanism, true, null, null, null, time, new LogContext()) { + null, false, kafkaClientSaslMechanism, true, null, null, null, time, new LogContext(), + () => ApiVersionsResponse.defaultApiVersionsResponse(ApiScope.ZK_BROKER)) { override protected def defaultLoginClass(): Class[_ <: Login] = classOf[TestableKerberosLogin] } channelBuilder.configure(config.values()) diff --git a/core/src/test/scala/unit/kafka/api/ApiVersionTest.scala b/core/src/test/scala/unit/kafka/api/ApiVersionTest.scala index 1e86876057326..4cd56acf593bc 100644 --- a/core/src/test/scala/unit/kafka/api/ApiVersionTest.scala +++ b/core/src/test/scala/unit/kafka/api/ApiVersionTest.scala @@ -20,6 +20,7 @@ package kafka.api import java.util import org.apache.kafka.common.feature.{Features, FinalizedVersionRange, SupportedVersionRange} +import org.apache.kafka.common.message.ApiMessageType.ApiScope import org.apache.kafka.common.protocol.ApiKeys import org.apache.kafka.common.record.{RecordBatch, RecordVersion} import org.apache.kafka.common.requests.{AbstractResponse, ApiVersionsResponse} @@ -179,9 +180,10 @@ class ApiVersionTest { def shouldCreateApiResponseOnlyWithKeysSupportedByMagicValue(): Unit = { val response = ApiVersion.apiVersionsResponse( 10, - RecordBatch.MAGIC_VALUE_V1, + RecordVersion.V1, Features.emptySupportedFeatures, - None + None, + ApiScope.ZK_BROKER ) verifyApiKeysForMagic(response, RecordBatch.MAGIC_VALUE_V1) assertEquals(10, response.throttleTimeMs) @@ -194,13 +196,14 @@ class ApiVersionTest { def shouldReturnFeatureKeysWhenMagicIsCurrentValueAndThrottleMsIsDefaultThrottle(): Unit = { val response = ApiVersion.apiVersionsResponse( 10, - RecordBatch.MAGIC_VALUE_V1, + RecordVersion.V1, Features.supportedFeatures( Utils.mkMap(Utils.mkEntry("feature", new SupportedVersionRange(1.toShort, 4.toShort)))), Features.finalizedFeatures( Utils.mkMap(Utils.mkEntry("feature", new FinalizedVersionRange(2.toShort, 3.toShort)))), 10, - None + None, + ApiScope.ZK_BROKER ) verifyApiKeysForMagic(response, RecordBatch.MAGIC_VALUE_V1) @@ -228,11 +231,12 @@ class ApiVersionTest { def shouldReturnAllKeysWhenMagicIsCurrentValueAndThrottleMsIsDefaultThrottle(): Unit = { val response = ApiVersion.apiVersionsResponse( AbstractResponse.DEFAULT_THROTTLE_TIME, - RecordBatch.CURRENT_MAGIC_VALUE, + RecordVersion.current(), Features.emptySupportedFeatures, - None + None, + ApiScope.ZK_BROKER ) - assertEquals(new util.HashSet[ApiKeys](ApiKeys.brokerApis), apiKeysInResponse(response)) + assertEquals(new util.HashSet[ApiKeys](ApiKeys.zkBrokerApis), apiKeysInResponse(response)) assertEquals(AbstractResponse.DEFAULT_THROTTLE_TIME, response.throttleTimeMs) assertTrue(response.data.supportedFeatures.isEmpty) assertTrue(response.data.finalizedFeatures.isEmpty) @@ -243,9 +247,10 @@ class ApiVersionTest { def testMetadataQuorumApisAreDisabled(): Unit = { val response = ApiVersion.apiVersionsResponse( AbstractResponse.DEFAULT_THROTTLE_TIME, - RecordBatch.CURRENT_MAGIC_VALUE, + RecordVersion.current(), Features.emptySupportedFeatures, - None + None, + ApiScope.ZK_BROKER ) // Ensure that APIs needed for the internal metadata quorum (KIP-500) diff --git a/core/src/test/scala/unit/kafka/network/SocketServerTest.scala b/core/src/test/scala/unit/kafka/network/SocketServerTest.scala index dd93e43436288..5fc1c15a4f428 100644 --- a/core/src/test/scala/unit/kafka/network/SocketServerTest.scala +++ b/core/src/test/scala/unit/kafka/network/SocketServerTest.scala @@ -31,10 +31,11 @@ import com.yammer.metrics.core.{Gauge, Meter} import javax.net.ssl._ import kafka.metrics.KafkaYammerMetrics import kafka.security.CredentialProvider -import kafka.server.{KafkaConfig, ThrottledChannel} +import kafka.server.{KafkaConfig, SimpleApiVersionManager, ThrottledChannel} import kafka.utils.Implicits._ import kafka.utils.TestUtils import org.apache.kafka.common.memory.MemoryPool +import org.apache.kafka.common.message.ApiMessageType.ApiScope import org.apache.kafka.common.message.{ProduceRequestData, SaslAuthenticateRequestData, SaslHandshakeRequestData, VoteRequestData} import org.apache.kafka.common.metrics.Metrics import org.apache.kafka.common.network.KafkaChannel.ChannelMuteState @@ -73,7 +74,8 @@ class SocketServerTest { // Clean-up any metrics left around by previous tests TestUtils.clearYammerMetrics() - val server = new SocketServer(config, metrics, Time.SYSTEM, credentialProvider) + private val apiVersionManager = new SimpleApiVersionManager(ApiScope.ZK_BROKER) + val server = new SocketServer(config, metrics, Time.SYSTEM, credentialProvider, apiVersionManager) server.startup() val sockets = new ArrayBuffer[Socket] @@ -452,7 +454,8 @@ class SocketServerTest { val time = new MockTime() props.put(KafkaConfig.ConnectionsMaxIdleMsProp, idleTimeMs.toString) val serverMetrics = new Metrics - val overrideServer = new SocketServer(KafkaConfig.fromProps(props), serverMetrics, time, credentialProvider) + val overrideServer = new SocketServer(KafkaConfig.fromProps(props), serverMetrics, + time, credentialProvider, apiVersionManager) try { overrideServer.startup() @@ -504,12 +507,14 @@ class SocketServerTest { val serverMetrics = new Metrics @volatile var selector: TestableSelector = null val overrideConnectionId = "127.0.0.1:1-127.0.0.1:2-0" - val overrideServer = new SocketServer(KafkaConfig.fromProps(props), serverMetrics, time, credentialProvider) { + val overrideServer = new SocketServer( + KafkaConfig.fromProps(props), serverMetrics, time, credentialProvider, apiVersionManager + ) { override def newProcessor(id: Int, requestChannel: RequestChannel, connectionQuotas: ConnectionQuotas, listenerName: ListenerName, - protocol: SecurityProtocol, memoryPool: MemoryPool, isPrivilegedListener: Boolean = false): Processor = { + protocol: SecurityProtocol, memoryPool: MemoryPool, isPrivilegedListener: Boolean): Processor = { new Processor(id, time, config.socketRequestMaxBytes, dataPlaneRequestChannel, connectionQuotas, config.connectionsMaxIdleMs, config.failedAuthenticationDelayMs, listenerName, protocol, config, metrics, - credentialProvider, memoryPool, new LogContext(), isPrivilegedListener = isPrivilegedListener) { + credentialProvider, memoryPool, new LogContext(), Processor.ConnectionQueueSize, isPrivilegedListener, apiVersionManager) { override protected[network] def connectionId(socket: Socket): String = overrideConnectionId override protected[network] def createSelector(channelBuilder: ChannelBuilder): Selector = { val testableSelector = new TestableSelector(config, channelBuilder, time, metrics) @@ -799,7 +804,8 @@ class SocketServerTest { val newProps = TestUtils.createBrokerConfig(0, TestUtils.MockZkConnect, port = 0) newProps.setProperty(KafkaConfig.MaxConnectionsPerIpProp, "0") newProps.setProperty(KafkaConfig.MaxConnectionsPerIpOverridesProp, "%s:%s".format("127.0.0.1", "5")) - val server = new SocketServer(KafkaConfig.fromProps(newProps), new Metrics(), Time.SYSTEM, credentialProvider) + val server = new SocketServer(KafkaConfig.fromProps(newProps), new Metrics(), + Time.SYSTEM, credentialProvider, apiVersionManager) try { server.startup() // make the maximum allowable number of connections @@ -837,7 +843,8 @@ class SocketServerTest { val overrideProps = TestUtils.createBrokerConfig(0, TestUtils.MockZkConnect, port = 0) overrideProps.put(KafkaConfig.MaxConnectionsPerIpOverridesProp, s"localhost:$overrideNum") val serverMetrics = new Metrics() - val overrideServer = new SocketServer(KafkaConfig.fromProps(overrideProps), serverMetrics, Time.SYSTEM, credentialProvider) + val overrideServer = new SocketServer(KafkaConfig.fromProps(overrideProps), serverMetrics, + Time.SYSTEM, credentialProvider, apiVersionManager) try { overrideServer.startup() // make the maximum allowable number of connections @@ -866,7 +873,8 @@ class SocketServerTest { overrideProps.put(KafkaConfig.NumQuotaSamplesProp, String.valueOf(2)) val connectionRate = 5 val time = new MockTime() - val overrideServer = new SocketServer(KafkaConfig.fromProps(overrideProps), new Metrics(), time, credentialProvider) + val overrideServer = new SocketServer(KafkaConfig.fromProps(overrideProps), new Metrics(), + time, credentialProvider, apiVersionManager) // update the connection rate to 5 overrideServer.connectionQuotas.updateIpConnectionRateQuota(None, Some(connectionRate)) try { @@ -916,7 +924,8 @@ class SocketServerTest { overrideProps.put(KafkaConfig.NumQuotaSamplesProp, String.valueOf(2)) val connectionRate = 5 val time = new MockTime() - val overrideServer = new SocketServer(KafkaConfig.fromProps(overrideProps), new Metrics(), time, credentialProvider) + val overrideServer = new SocketServer(KafkaConfig.fromProps(overrideProps), new Metrics(), + time, credentialProvider, apiVersionManager) overrideServer.connectionQuotas.updateIpConnectionRateQuota(None, Some(connectionRate)) overrideServer.startup() // make the maximum allowable number of connections @@ -938,7 +947,8 @@ class SocketServerTest { @Test def testSslSocketServer(): Unit = { val serverMetrics = new Metrics - val overrideServer = new SocketServer(KafkaConfig.fromProps(sslServerProps), serverMetrics, Time.SYSTEM, credentialProvider) + val overrideServer = new SocketServer(KafkaConfig.fromProps(sslServerProps), serverMetrics, + Time.SYSTEM, credentialProvider, apiVersionManager) try { overrideServer.startup() val sslContext = SSLContext.getInstance(TestSslUtils.DEFAULT_TLS_PROTOCOL_FOR_TESTS) @@ -1078,12 +1088,14 @@ class SocketServerTest { val props = TestUtils.createBrokerConfig(0, TestUtils.MockZkConnect, port = 0) val serverMetrics = new Metrics var conn: Socket = null - val overrideServer = new SocketServer(KafkaConfig.fromProps(props), serverMetrics, Time.SYSTEM, credentialProvider) { + val overrideServer = new SocketServer( + KafkaConfig.fromProps(props), serverMetrics, Time.SYSTEM, credentialProvider, apiVersionManager + ) { override def newProcessor(id: Int, requestChannel: RequestChannel, connectionQuotas: ConnectionQuotas, listenerName: ListenerName, protocol: SecurityProtocol, memoryPool: MemoryPool, isPrivilegedListener: Boolean = false): Processor = { new Processor(id, time, config.socketRequestMaxBytes, dataPlaneRequestChannel, connectionQuotas, config.connectionsMaxIdleMs, config.failedAuthenticationDelayMs, listenerName, protocol, config, metrics, - credentialProvider, MemoryPool.NONE, new LogContext(), isPrivilegedListener = isPrivilegedListener) { + credentialProvider, MemoryPool.NONE, new LogContext(), Processor.ConnectionQueueSize, isPrivilegedListener, apiVersionManager) { override protected[network] def sendResponse(response: RequestChannel.Response, responseSend: Send): Unit = { conn.close() super.sendResponse(response, responseSend) @@ -1120,12 +1132,14 @@ class SocketServerTest { def testClientDisconnectionWithOutstandingReceivesProcessedUntilFailedSend(): Unit = { val serverMetrics = new Metrics @volatile var selector: TestableSelector = null - val overrideServer = new SocketServer(KafkaConfig.fromProps(props), serverMetrics, Time.SYSTEM, credentialProvider) { + val overrideServer = new SocketServer( + KafkaConfig.fromProps(props), serverMetrics, Time.SYSTEM, credentialProvider, apiVersionManager + ) { override def newProcessor(id: Int, requestChannel: RequestChannel, connectionQuotas: ConnectionQuotas, listenerName: ListenerName, - protocol: SecurityProtocol, memoryPool: MemoryPool, isPrivilegedListener: Boolean = false): Processor = { + protocol: SecurityProtocol, memoryPool: MemoryPool, isPrivilegedListener: Boolean): Processor = { new Processor(id, time, config.socketRequestMaxBytes, dataPlaneRequestChannel, connectionQuotas, config.connectionsMaxIdleMs, config.failedAuthenticationDelayMs, listenerName, protocol, config, metrics, - credentialProvider, memoryPool, new LogContext(), isPrivilegedListener = isPrivilegedListener) { + credentialProvider, memoryPool, new LogContext(), Processor.ConnectionQueueSize, isPrivilegedListener, apiVersionManager) { override protected[network] def createSelector(channelBuilder: ChannelBuilder): Selector = { val testableSelector = new TestableSelector(config, channelBuilder, time, metrics) selector = testableSelector @@ -1161,7 +1175,8 @@ class SocketServerTest { props.setProperty(KafkaConfig.ConnectionsMaxIdleMsProp, "110") val serverMetrics = new Metrics var conn: Socket = null - val overrideServer = new SocketServer(KafkaConfig.fromProps(props), serverMetrics, Time.SYSTEM, credentialProvider) + val overrideServer = new SocketServer(KafkaConfig.fromProps(props), serverMetrics, + Time.SYSTEM, credentialProvider, apiVersionManager) try { overrideServer.startup() conn = connect(overrideServer) @@ -1873,9 +1888,13 @@ class SocketServerTest { } } - class TestableSocketServer(config : KafkaConfig = KafkaConfig.fromProps(props), val connectionQueueSize: Int = 20, - override val time: Time = Time.SYSTEM) extends SocketServer(config, - new Metrics, time, credentialProvider) { + class TestableSocketServer( + config : KafkaConfig = KafkaConfig.fromProps(props), + connectionQueueSize: Int = 20, + time: Time = Time.SYSTEM + ) extends SocketServer( + config, new Metrics, time, credentialProvider, apiVersionManager, + ) { @volatile var selector: Option[TestableSelector] = None @volatile var uncaughtExceptions = 0 @@ -1884,7 +1903,7 @@ class SocketServerTest { protocol: SecurityProtocol, memoryPool: MemoryPool, isPrivilegedListener: Boolean = false): Processor = { new Processor(id, time, config.socketRequestMaxBytes, requestChannel, connectionQuotas, config.connectionsMaxIdleMs, config.failedAuthenticationDelayMs, listenerName, protocol, config, metrics, credentialProvider, - memoryPool, new LogContext(), connectionQueueSize, isPrivilegedListener) { + memoryPool, new LogContext(), connectionQueueSize, isPrivilegedListener, apiVersionManager) { override protected[network] def createSelector(channelBuilder: ChannelBuilder): Selector = { val testableSelector = new TestableSelector(config, channelBuilder, time, metrics, metricTags.asScala) diff --git a/core/src/test/scala/unit/kafka/server/AbstractApiVersionsRequestTest.scala b/core/src/test/scala/unit/kafka/server/AbstractApiVersionsRequestTest.scala index f7163cad10f88..8dbdf0789c888 100644 --- a/core/src/test/scala/unit/kafka/server/AbstractApiVersionsRequestTest.scala +++ b/core/src/test/scala/unit/kafka/server/AbstractApiVersionsRequestTest.scala @@ -16,15 +16,17 @@ */ package kafka.server +import java.util.Properties + import integration.kafka.server.IntegrationTestUtils import kafka.test.ClusterInstance +import org.apache.kafka.common.message.ApiMessageType.ApiScope import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersion import org.apache.kafka.common.network.ListenerName import org.apache.kafka.common.protocol.ApiKeys import org.apache.kafka.common.requests.{ApiVersionsRequest, ApiVersionsResponse} import org.junit.jupiter.api.Assertions._ -import java.util.Properties import scala.jdk.CollectionConverters._ abstract class AbstractApiVersionsRequestTest(cluster: ClusterInstance) { @@ -53,14 +55,13 @@ abstract class AbstractApiVersionsRequestTest(cluster: ClusterInstance) { } finally socket.close() } - def validateApiVersionsResponse(apiVersionsResponse: ApiVersionsResponse, listenerName: ListenerName): Unit = { - val expectedApis = ApiKeys.brokerApis() - if (listenerName == controlPlaneListenerName) { - expectedApis.add(ApiKeys.ENVELOPE) - } + def validateApiVersionsResponse(apiVersionsResponse: ApiVersionsResponse): Unit = { + val expectedApis = ApiKeys.zkBrokerApis() assertEquals(expectedApis.size(), apiVersionsResponse.data.apiKeys().size(), "API keys in ApiVersionsResponse must match API keys supported by broker.") - for (expectedApiVersion: ApiVersion <- ApiVersionsResponse.DEFAULT_API_VERSIONS_RESPONSE.data.apiKeys().asScala) { + + val defaultApiVersionsResponse = ApiVersionsResponse.defaultApiVersionsResponse(ApiScope.ZK_BROKER) + for (expectedApiVersion: ApiVersion <- defaultApiVersionsResponse.data.apiKeys().asScala) { val actualApiVersion = apiVersionsResponse.apiVersion(expectedApiVersion.apiKey) assertNotNull(actualApiVersion, s"API key ${actualApiVersion.apiKey} is supported by broker, but not received in ApiVersionsResponse.") assertEquals(expectedApiVersion.apiKey, actualApiVersion.apiKey, "API key must be supported by the broker.") diff --git a/core/src/test/scala/unit/kafka/server/ApiVersionManagerTest.scala b/core/src/test/scala/unit/kafka/server/ApiVersionManagerTest.scala new file mode 100644 index 0000000000000..504e441218240 --- /dev/null +++ b/core/src/test/scala/unit/kafka/server/ApiVersionManagerTest.scala @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package kafka.server + +import kafka.api.ApiVersion +import org.apache.kafka.clients.NodeApiVersions +import org.apache.kafka.common.message.ApiMessageType.ApiScope +import org.apache.kafka.common.protocol.ApiKeys +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.Assertions._ +import org.mockito.Mockito + +class ApiVersionManagerTest { + + @Test + def testControllerApiIntersection(): Unit = { + val controllerMinVersion: Short = 1 + val controllerMaxVersion: Short = 5 + + val forwardingManager = Mockito.mock(classOf[ForwardingManager]) + + Mockito.when(forwardingManager.controllerApiVersions).thenReturn(Some(NodeApiVersions.create( + ApiKeys.CREATE_TOPICS.id, + controllerMinVersion, + controllerMaxVersion + ))) + + val brokerFeatures = BrokerFeatures.createDefault() + val featureCache = new FinalizedFeatureCache(brokerFeatures) + + val versionManager = new DefaultApiVersionManager( + apiScope = ApiScope.ZK_BROKER, + interBrokerProtocolVersion = ApiVersion.latestVersion, + forwardingManager = Some(forwardingManager), + features = brokerFeatures, + featureCache = featureCache + ) + + val apiVersionsResponse = versionManager.apiVersionResponse(throttleTimeMs = 0) + val alterConfigVersion = apiVersionsResponse.data.apiKeys.find(ApiKeys.CREATE_TOPICS.id) + assertNotNull(alterConfigVersion) + assertEquals(controllerMinVersion, alterConfigVersion.minVersion) + assertEquals(controllerMaxVersion, alterConfigVersion.maxVersion) + } + + @Test + def testEnvelopeEnabledWhenForwardingManagerPresent(): Unit = { + val forwardingManager = Mockito.mock(classOf[ForwardingManager]) + Mockito.when(forwardingManager.controllerApiVersions).thenReturn(None) + + val brokerFeatures = BrokerFeatures.createDefault() + val featureCache = new FinalizedFeatureCache(brokerFeatures) + + val versionManager = new DefaultApiVersionManager( + apiScope = ApiScope.ZK_BROKER, + interBrokerProtocolVersion = ApiVersion.latestVersion, + forwardingManager = Some(forwardingManager), + features = brokerFeatures, + featureCache = featureCache + ) + assertTrue(versionManager.isApiEnabled(ApiKeys.ENVELOPE)) + assertTrue(versionManager.enabledApis.contains(ApiKeys.ENVELOPE)) + + val apiVersionsResponse = versionManager.apiVersionResponse(throttleTimeMs = 0) + val envelopeVersion = apiVersionsResponse.data.apiKeys.find(ApiKeys.ENVELOPE.id) + assertNotNull(envelopeVersion) + assertEquals(ApiKeys.ENVELOPE.oldestVersion, envelopeVersion.minVersion) + assertEquals(ApiKeys.ENVELOPE.latestVersion, envelopeVersion.maxVersion) + } + + @Test + def testEnvelopeDisabledWhenForwardingManagerEmpty(): Unit = { + val brokerFeatures = BrokerFeatures.createDefault() + val featureCache = new FinalizedFeatureCache(brokerFeatures) + + val versionManager = new DefaultApiVersionManager( + apiScope = ApiScope.ZK_BROKER, + interBrokerProtocolVersion = ApiVersion.latestVersion, + forwardingManager = None, + features = brokerFeatures, + featureCache = featureCache + ) + assertFalse(versionManager.isApiEnabled(ApiKeys.ENVELOPE)) + assertFalse(versionManager.enabledApis.contains(ApiKeys.ENVELOPE)) + + val apiVersionsResponse = versionManager.apiVersionResponse(throttleTimeMs = 0) + assertNull(apiVersionsResponse.data.apiKeys.find(ApiKeys.ENVELOPE.id)) + } + +} diff --git a/core/src/test/scala/unit/kafka/server/ApiVersionsRequestTest.scala b/core/src/test/scala/unit/kafka/server/ApiVersionsRequestTest.scala index dc35bae4ab09c..34ee74a0e8066 100644 --- a/core/src/test/scala/unit/kafka/server/ApiVersionsRequestTest.scala +++ b/core/src/test/scala/unit/kafka/server/ApiVersionsRequestTest.scala @@ -40,14 +40,14 @@ class ApiVersionsRequestTest(cluster: ClusterInstance) extends AbstractApiVersio def testApiVersionsRequest(): Unit = { val request = new ApiVersionsRequest.Builder().build() val apiVersionsResponse = sendApiVersionsRequest(request, cluster.clientListener()) - validateApiVersionsResponse(apiVersionsResponse, cluster.clientListener()) + validateApiVersionsResponse(apiVersionsResponse) } @ClusterTest def testApiVersionsRequestThroughControlPlaneListener(): Unit = { val request = new ApiVersionsRequest.Builder().build() val apiVersionsResponse = sendApiVersionsRequest(request, super.controlPlaneListenerName) - validateApiVersionsResponse(apiVersionsResponse, super.controlPlaneListenerName) + validateApiVersionsResponse(apiVersionsResponse) } @ClusterTest @@ -66,14 +66,14 @@ class ApiVersionsRequestTest(cluster: ClusterInstance) extends AbstractApiVersio def testApiVersionsRequestValidationV0(): Unit = { val apiVersionsRequest = new ApiVersionsRequest.Builder().build(0.asInstanceOf[Short]) val apiVersionsResponse = sendApiVersionsRequest(apiVersionsRequest, cluster.clientListener()) - validateApiVersionsResponse(apiVersionsResponse, cluster.clientListener()) + validateApiVersionsResponse(apiVersionsResponse) } @ClusterTest def testApiVersionsRequestValidationV0ThroughControlPlaneListener(): Unit = { val apiVersionsRequest = new ApiVersionsRequest.Builder().build(0.asInstanceOf[Short]) val apiVersionsResponse = sendApiVersionsRequest(apiVersionsRequest, super.controlPlaneListenerName) - validateApiVersionsResponse(apiVersionsResponse, super.controlPlaneListenerName) + validateApiVersionsResponse(apiVersionsResponse) } @ClusterTest diff --git a/core/src/test/scala/unit/kafka/server/ForwardingManagerTest.scala b/core/src/test/scala/unit/kafka/server/ForwardingManagerTest.scala index bb4f57b333d49..7e9407629f361 100644 --- a/core/src/test/scala/unit/kafka/server/ForwardingManagerTest.scala +++ b/core/src/test/scala/unit/kafka/server/ForwardingManagerTest.scala @@ -29,6 +29,7 @@ import org.apache.kafka.clients.MockClient.RequestMatcher import org.apache.kafka.common.Node import org.apache.kafka.common.config.{ConfigResource, TopicConfig} import org.apache.kafka.common.memory.MemoryPool +import org.apache.kafka.common.message.ApiMessageType.ApiScope import org.apache.kafka.common.message.{AlterConfigsResponseData, ApiVersionsResponseData} import org.apache.kafka.common.network.{ClientInformation, ListenerName} import org.apache.kafka.common.protocol.{ApiKeys, Errors} @@ -195,7 +196,7 @@ class ForwardingManagerTest { startTimeNanos = time.nanoseconds(), memoryPool = MemoryPool.NONE, buffer = requestBuffer, - metrics = new RequestChannel.Metrics(allowControllerOnlyApis = true), + metrics = new RequestChannel.Metrics(ApiScope.CONTROLLER), envelope = None ) } diff --git a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala index 5138bf67655fb..704998f8123f7 100644 --- a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala +++ b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala @@ -37,7 +37,6 @@ import kafka.server.QuotaFactory.QuotaManagers import kafka.server.metadata.{CachedConfigRepository, ConfigRepository, RaftMetadataCache} import kafka.utils.{MockTime, TestUtils} import kafka.zk.KafkaZkClient -import org.apache.kafka.clients.NodeApiVersions import org.apache.kafka.clients.admin.AlterConfigOp.OpType import org.apache.kafka.clients.admin.{AlterConfigOp, ConfigEntry} import org.apache.kafka.common.acl.AclOperation @@ -45,6 +44,7 @@ import org.apache.kafka.common.config.ConfigResource import org.apache.kafka.common.errors.UnsupportedVersionException import org.apache.kafka.common.internals.{KafkaFutureImpl, Topic} import org.apache.kafka.common.memory.MemoryPool +import org.apache.kafka.common.message.ApiMessageType.ApiScope import org.apache.kafka.common.message.CreateTopicsRequestData.{CreatableTopic, CreatableTopicCollection} import org.apache.kafka.common.message.DescribeConfigsResponseData.DescribeConfigsResult import org.apache.kafka.common.message.JoinGroupRequestData.JoinGroupRequestProtocol @@ -129,8 +129,6 @@ class KafkaApisTest { raftSupport: Boolean = false, overrideProperties: Map[String, String] = Map.empty): KafkaApis = { - val brokerFeatures = BrokerFeatures.createDefault() - val cache = new FinalizedFeatureCache(brokerFeatures) val properties = if (raftSupport) { val properties = TestUtils.createBrokerConfig(brokerId, "") properties.put(KafkaConfig.NodeIdProp, brokerId.toString) @@ -163,6 +161,14 @@ class KafkaApisTest { case _ => throw new IllegalStateException("Test must set an instance of ZkMetadataCache") } } + + val enabledApis = if (enableForwarding) { + ApiKeys.apisInScope(ApiScope.ZK_BROKER).asScala ++ Set(ApiKeys.ENVELOPE) + } else { + ApiKeys.apisInScope(ApiScope.ZK_BROKER).asScala.toSet + } + val apiVersionManager = new SimpleApiVersionManager(enabledApis) + new KafkaApis(requestChannel, metadataSupport, replicaManager, @@ -181,8 +187,7 @@ class KafkaApisTest { clusterId, time, null, - brokerFeatures, - cache) + apiVersionManager) } @Test @@ -652,75 +657,6 @@ class KafkaApisTest { } } - @Test - def testHandleApiVersionsWithControllerApiVersions(): Unit = { - val authorizer: Authorizer = EasyMock.niceMock(classOf[Authorizer]) - - val requestHeader = new RequestHeader(ApiKeys.API_VERSIONS, ApiKeys.API_VERSIONS.latestVersion, clientId, 0) - - val permittedVersion: Short = 0 - EasyMock.expect(forwardingManager.controllerApiVersions).andReturn( - Some(NodeApiVersions.create(ApiKeys.ALTER_CONFIGS.id, permittedVersion, permittedVersion))) - - val capturedResponse = expectNoThrottling() - - val apiVersionsRequest = new ApiVersionsRequest.Builder() - .build(requestHeader.apiVersion) - val request = buildRequest(apiVersionsRequest, - fromPrivilegedListener = true, requestHeader = Option(requestHeader)) - - EasyMock.replay(replicaManager, clientRequestQuotaManager, forwardingManager, - requestChannel, authorizer, adminManager, controller) - - createKafkaApis(authorizer = Some(authorizer), enableForwarding = true).handleApiVersionsRequest(request) - - val expectedVersions = new ApiVersionsResponseData.ApiVersion() - .setApiKey(ApiKeys.ALTER_CONFIGS.id) - .setMaxVersion(permittedVersion) - .setMinVersion(permittedVersion) - - val response = readResponse(apiVersionsRequest, capturedResponse) - .asInstanceOf[ApiVersionsResponse] - assertEquals(Errors.NONE, Errors.forCode(response.data().errorCode())) - - val alterConfigVersions = response.data().apiKeys().find(ApiKeys.ALTER_CONFIGS.id) - assertEquals(expectedVersions, alterConfigVersions) - - verify(authorizer, adminManager, forwardingManager) - } - - @Test - def testGetUnsupportedVersionsWhenControllerApiVersionsNotAvailable(): Unit = { - val authorizer: Authorizer = EasyMock.niceMock(classOf[Authorizer]) - - val requestHeader = new RequestHeader(ApiKeys.API_VERSIONS, ApiKeys.API_VERSIONS.latestVersion, clientId, 0) - - EasyMock.expect(forwardingManager.controllerApiVersions).andReturn(None) - - val capturedResponse = expectNoThrottling() - - val apiVersionsRequest = new ApiVersionsRequest.Builder() - .build(requestHeader.apiVersion) - val request = buildRequest(apiVersionsRequest, - fromPrivilegedListener = true, requestHeader = Option(requestHeader)) - - EasyMock.replay(replicaManager, clientRequestQuotaManager, forwardingManager, - requestChannel, authorizer, adminManager, controller) - - createKafkaApis(authorizer = Some(authorizer), enableForwarding = true).handleApiVersionsRequest(request) - - val response = readResponse(apiVersionsRequest, capturedResponse) - .asInstanceOf[ApiVersionsResponse] - assertEquals(Errors.NONE, Errors.forCode(response.data().errorCode())) - - val expectedVersions = ApiVersionsResponse.toApiVersion(ApiKeys.ALTER_CONFIGS) - - val alterConfigVersions = response.data().apiKeys().find(ApiKeys.ALTER_CONFIGS.id) - assertEquals(expectedVersions, alterConfigVersions) - - verify(authorizer, adminManager, forwardingManager) - } - @Test def testCreateTopicsWithAuthorizer(): Unit = { val authorizer: Authorizer = EasyMock.niceMock(classOf[Authorizer]) diff --git a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala index 1924034ffdf45..7706c83cca9bf 100644 --- a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala +++ b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala @@ -171,7 +171,7 @@ class RequestQuotaTest extends BaseRequestTest { def testUnauthorizedThrottle(): Unit = { RequestQuotaTest.principal = RequestQuotaTest.UnauthorizedPrincipal - for (apiKey <- ApiKeys.brokerApis.asScala) { + for (apiKey <- ApiKeys.zkBrokerApis.asScala) { submitTest(apiKey, () => checkUnauthorizedRequestThrottle(apiKey)) } @@ -754,9 +754,9 @@ class RequestQuotaTest extends BaseRequestTest { } object RequestQuotaTest { - val ClusterActions = ApiKeys.brokerApis.asScala.filter(_.clusterAction).toSet + val ClusterActions = ApiKeys.zkBrokerApis.asScala.filter(_.clusterAction).toSet val SaslActions = Set(ApiKeys.SASL_HANDSHAKE, ApiKeys.SASL_AUTHENTICATE) - val ClientActions = ApiKeys.brokerApis.asScala.toSet -- ClusterActions -- SaslActions + val ClientActions = ApiKeys.zkBrokerApis.asScala.toSet -- ClusterActions -- SaslActions val UnauthorizedPrincipal = new KafkaPrincipal(KafkaPrincipal.USER_TYPE, "Unauthorized") // Principal used for all client connections. This is modified by tests which diff --git a/core/src/test/scala/unit/kafka/server/SaslApiVersionsRequestTest.scala b/core/src/test/scala/unit/kafka/server/SaslApiVersionsRequestTest.scala index bbc71cad2679c..05c83d476d9f8 100644 --- a/core/src/test/scala/unit/kafka/server/SaslApiVersionsRequestTest.scala +++ b/core/src/test/scala/unit/kafka/server/SaslApiVersionsRequestTest.scala @@ -59,7 +59,7 @@ class SaslApiVersionsRequestTest(cluster: ClusterInstance) extends AbstractApiVe try { val apiVersionsResponse = IntegrationTestUtils.sendAndReceive[ApiVersionsResponse]( new ApiVersionsRequest.Builder().build(0), socket) - validateApiVersionsResponse(apiVersionsResponse, cluster.clientListener()) + validateApiVersionsResponse(apiVersionsResponse) sendSaslHandshakeRequestValidateResponse(socket) } finally { socket.close() @@ -88,7 +88,7 @@ class SaslApiVersionsRequestTest(cluster: ClusterInstance) extends AbstractApiVe assertEquals(Errors.UNSUPPORTED_VERSION.code, apiVersionsResponse.data.errorCode) val apiVersionsResponse2 = IntegrationTestUtils.sendAndReceive[ApiVersionsResponse]( new ApiVersionsRequest.Builder().build(0), socket) - validateApiVersionsResponse(apiVersionsResponse2, cluster.clientListener()) + validateApiVersionsResponse(apiVersionsResponse2) sendSaslHandshakeRequestValidateResponse(socket) } finally { socket.close() diff --git a/generator/src/main/java/org/apache/kafka/message/ApiMessageTypeGenerator.java b/generator/src/main/java/org/apache/kafka/message/ApiMessageTypeGenerator.java index 075ee485550b0..1d43ed5cd8ec8 100644 --- a/generator/src/main/java/org/apache/kafka/message/ApiMessageTypeGenerator.java +++ b/generator/src/main/java/org/apache/kafka/message/ApiMessageTypeGenerator.java @@ -19,14 +19,23 @@ import java.io.BufferedWriter; import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.EnumMap; +import java.util.Iterator; +import java.util.List; import java.util.Locale; import java.util.Map; import java.util.TreeMap; +import java.util.stream.Collectors; public final class ApiMessageTypeGenerator implements TypeClassGenerator { private final HeaderGenerator headerGenerator; private final CodeBuffer buffer; private final TreeMap apis; + private final EnumMap> apisByScope = new EnumMap<>(RequestApiScope.class); private static final class ApiData { short apiKey; @@ -93,6 +102,13 @@ public void registerMessageType(MessageSpec spec) { "API key " + spec.apiKey().get()); } data.requestSpec = spec; + + if (spec.scope() != null) { + for (RequestApiScope scope : spec.scope()) { + apisByScope.putIfAbsent(scope, new ArrayList<>()); + apisByScope.get(scope).add(data); + } + } break; } case RESPONSE: { @@ -140,6 +156,8 @@ private void generate() { buffer.printf("%n"); generateAccessor("highestSupportedVersion", "short"); buffer.printf("%n"); + generateAccessor("scope", "EnumSet"); + buffer.printf("%n"); generateAccessor("apiKey", "short"); buffer.printf("%n"); generateAccessor("requestSchemas", "Schema[]"); @@ -151,18 +169,48 @@ private void generate() { generateHeaderVersion("request"); buffer.printf("%n"); generateHeaderVersion("response"); + buffer.printf("%n"); + generateApiScopeEnum(); + buffer.printf("%n"); buffer.decrementIndent(); buffer.printf("}%n"); headerGenerator.generate(); } + private String generateApiScopeEnumSet(Collection values) { + if (values.isEmpty()) { + return "EnumSet.noneOf(ApiScope.class)"; + } + StringBuilder bldr = new StringBuilder("EnumSet.of("); + Iterator iter = values.iterator(); + while (iter.hasNext()) { + bldr.append("ApiScope."); + bldr.append(iter.next()); + if (iter.hasNext()) { + bldr.append(", "); + } + } + bldr.append(")"); + return bldr.toString(); + } + private void generateEnumValues() { int numProcessed = 0; for (Map.Entry entry : apis.entrySet()) { ApiData apiData = entry.getValue(); String name = apiData.name(); numProcessed++; - buffer.printf("%s(\"%s\", (short) %d, %s, %s, (short) %d, (short) %d)%s%n", + + final Collection scopes; + if (apiData.requestSpec.scope() == null) { + scopes = Collections.emptyList(); + } else { + scopes = apiData.requestSpec.scope().stream() + .map(RequestApiScope::name) + .collect(Collectors.toList()); + } + + buffer.printf("%s(\"%s\", (short) %d, %s, %s, (short) %d, (short) %d, %s)%s%n", MessageGenerator.toSnakeCase(name).toUpperCase(Locale.ROOT), MessageGenerator.capitalizeFirst(name), entry.getKey(), @@ -170,6 +218,7 @@ private void generateEnumValues() { apiData.responseSchema(), apiData.requestSpec.struct().versions().lowest(), apiData.requestSpec.struct().versions().highest(), + generateApiScopeEnumSet(scopes), (numProcessed == apis.size()) ? ";" : ","); } } @@ -181,13 +230,16 @@ private void generateInstanceVariables() { buffer.printf("private final Schema[] responseSchemas;%n"); buffer.printf("private final short lowestSupportedVersion;%n"); buffer.printf("private final short highestSupportedVersion;%n"); + buffer.printf("private final EnumSet scope;%n"); headerGenerator.addImport(MessageGenerator.SCHEMA_CLASS); + headerGenerator.addImport(MessageGenerator.ENUM_SET_CLASS); } private void generateEnumConstructor() { buffer.printf("ApiMessageType(String name, short apiKey, " + "Schema[] requestSchemas, Schema[] responseSchemas, " + - "short lowestSupportedVersion, short highestSupportedVersion) {%n"); + "short lowestSupportedVersion, short highestSupportedVersion, " + + "EnumSet scope) {%n"); buffer.incrementIndent(); buffer.printf("this.name = name;%n"); buffer.printf("this.apiKey = apiKey;%n"); @@ -195,6 +247,7 @@ private void generateEnumConstructor() { buffer.printf("this.responseSchemas = responseSchemas;%n"); buffer.printf("this.lowestSupportedVersion = lowestSupportedVersion;%n"); buffer.printf("this.highestSupportedVersion = highestSupportedVersion;%n"); + buffer.printf("this.scope = scope;%n"); buffer.decrementIndent(); buffer.printf("}%n"); } @@ -338,6 +391,18 @@ private void generateHeaderVersion(String type) { buffer.printf("}%n"); } + private void generateApiScopeEnum() { + buffer.printf("public enum ApiScope {%n"); + buffer.incrementIndent(); + Iterator scopeIterator = Arrays.stream(RequestApiScope.values()).iterator(); + while (scopeIterator.hasNext()) { + RequestApiScope scope = scopeIterator.next(); + buffer.printf("%s%s%n", scope.name(), scopeIterator.hasNext() ? "," : ";"); + } + buffer.decrementIndent(); + buffer.printf("}%n"); + } + private void write(BufferedWriter writer) throws IOException { headerGenerator.buffer().write(writer); buffer.write(writer); diff --git a/generator/src/main/java/org/apache/kafka/message/MessageGenerator.java b/generator/src/main/java/org/apache/kafka/message/MessageGenerator.java index b6fb0aa4c616d..a1b972867c0c4 100644 --- a/generator/src/main/java/org/apache/kafka/message/MessageGenerator.java +++ b/generator/src/main/java/org/apache/kafka/message/MessageGenerator.java @@ -51,6 +51,8 @@ public final class MessageGenerator { static final String API_MESSAGE_TYPE_JAVA = "ApiMessageType.java"; + static final String API_SCOPE_JAVA = "ApiScope.java"; + static final String METADATA_RECORD_TYPE_JAVA = "MetadataRecordType.java"; static final String METADATA_JSON_CONVERTERS_JAVA = "MetadataJsonConverters.java"; @@ -84,6 +86,8 @@ public final class MessageGenerator { static final String ITERATOR_CLASS = "java.util.Iterator"; + static final String ENUM_SET_CLASS = "java.util.EnumSet"; + static final String TYPE_CLASS = "org.apache.kafka.common.protocol.types.Type"; static final String FIELD_CLASS = "org.apache.kafka.common.protocol.types.Field"; @@ -94,8 +98,6 @@ public final class MessageGenerator { static final String COMPACT_ARRAYOF_CLASS = "org.apache.kafka.common.protocol.types.CompactArrayOf"; - static final String STRUCT_CLASS = "org.apache.kafka.common.protocol.types.Struct"; - static final String BYTES_CLASS = "org.apache.kafka.common.utils.Bytes"; static final String UUID_CLASS = "org.apache.kafka.common.Uuid"; diff --git a/generator/src/main/java/org/apache/kafka/message/MessageSpec.java b/generator/src/main/java/org/apache/kafka/message/MessageSpec.java index 0b53cb1db0945..7fb18de8f1690 100644 --- a/generator/src/main/java/org/apache/kafka/message/MessageSpec.java +++ b/generator/src/main/java/org/apache/kafka/message/MessageSpec.java @@ -37,6 +37,8 @@ public final class MessageSpec { private final Versions flexibleVersions; + private final List scope; + @JsonCreator public MessageSpec(@JsonProperty("name") String name, @JsonProperty("validVersions") String validVersions, @@ -44,7 +46,8 @@ public MessageSpec(@JsonProperty("name") String name, @JsonProperty("apiKey") Short apiKey, @JsonProperty("type") MessageSpecType type, @JsonProperty("commonStructs") List commonStructs, - @JsonProperty("flexibleVersions") String flexibleVersions) { + @JsonProperty("flexibleVersions") String flexibleVersions, + @JsonProperty("scope") List scope) { this.struct = new StructSpec(name, validVersions, fields); this.apiKey = apiKey == null ? Optional.empty() : Optional.of(apiKey); this.type = Objects.requireNonNull(type); @@ -57,6 +60,12 @@ public MessageSpec(@JsonProperty("name") String name, this.flexibleVersions + ", which is not open-ended. flexibleVersions must " + "be either none, or an open-ended range (that ends with a plus sign)."); } + + if (scope != null && !scope.isEmpty() && type != MessageSpecType.REQUEST) { + throw new RuntimeException("The `requestScope` property is only valid for " + + "messages with type `request`"); + } + this.scope = scope; } public StructSpec struct() { @@ -106,6 +115,11 @@ public String flexibleVersionsString() { return flexibleVersions.toString(); } + @JsonProperty("scope") + public List scope() { + return scope; + } + public String dataClassName() { switch (type) { case HEADER: diff --git a/generator/src/main/java/org/apache/kafka/message/RequestApiScope.java b/generator/src/main/java/org/apache/kafka/message/RequestApiScope.java new file mode 100644 index 0000000000000..3b2d481905228 --- /dev/null +++ b/generator/src/main/java/org/apache/kafka/message/RequestApiScope.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.message; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public enum RequestApiScope { + @JsonProperty("zkBroker") + ZK_BROKER, + + @JsonProperty("broker") + BROKER, + + @JsonProperty("controller") + CONTROLLER, + + @JsonProperty("raft") + RAFT; +} From fd268fba27f042905b94aba3df068ff76843a510 Mon Sep 17 00:00:00 2001 From: Jason Gustafson Date: Thu, 4 Feb 2021 21:15:29 -0800 Subject: [PATCH 2/7] Override `DefaultApiVersionManager.isApiEnabled` with more efficient impl --- .../apache/kafka/common/protocol/ApiKeys.java | 4 +++ .../kafka/server/ApiVersionManager.scala | 4 +++ .../kafka/server/ApiVersionManagerTest.scala | 29 +++++++++++++------ 3 files changed, 28 insertions(+), 9 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java b/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java index 038027096c807..2c304dad74883 100644 --- a/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java +++ b/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java @@ -207,6 +207,10 @@ public short responseHeaderVersion(short apiVersion) { return messageType.responseHeaderVersion(apiVersion); } + public boolean inScope(ApiMessageType.ApiScope scope) { + return messageType.scope().contains(scope); + } + private static String toHtml() { final StringBuilder b = new StringBuilder(); b.append("\n"); diff --git a/core/src/main/scala/kafka/server/ApiVersionManager.scala b/core/src/main/scala/kafka/server/ApiVersionManager.scala index 49a030e0780fd..99eb216dde671 100644 --- a/core/src/main/scala/kafka/server/ApiVersionManager.scala +++ b/core/src/main/scala/kafka/server/ApiVersionManager.scala @@ -118,4 +118,8 @@ class DefaultApiVersionManager( case None => ApiKeys.apisInScope(apiScope).asScala } } + + override def isApiEnabled(apiKey: ApiKeys): Boolean = { + apiKey.inScope(apiScope) || (apiKey == ApiKeys.ENVELOPE && forwardingManager.isDefined) + } } diff --git a/core/src/test/scala/unit/kafka/server/ApiVersionManagerTest.scala b/core/src/test/scala/unit/kafka/server/ApiVersionManagerTest.scala index 504e441218240..a39c3af1825d8 100644 --- a/core/src/test/scala/unit/kafka/server/ApiVersionManagerTest.scala +++ b/core/src/test/scala/unit/kafka/server/ApiVersionManagerTest.scala @@ -22,9 +22,29 @@ import org.apache.kafka.common.message.ApiMessageType.ApiScope import org.apache.kafka.common.protocol.ApiKeys import org.junit.jupiter.api.Test import org.junit.jupiter.api.Assertions._ +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.EnumSource import org.mockito.Mockito +import scala.jdk.CollectionConverters._ + class ApiVersionManagerTest { + private val brokerFeatures = BrokerFeatures.createDefault() + private val featureCache = new FinalizedFeatureCache(brokerFeatures) + + @ParameterizedTest + @EnumSource(classOf[ApiScope]) + def testApiScope(apiScope: ApiScope): Unit = { + val versionManager = new DefaultApiVersionManager( + apiScope = apiScope, + interBrokerProtocolVersion = ApiVersion.latestVersion, + forwardingManager = None, + features = brokerFeatures, + featureCache = featureCache + ) + assertEquals(ApiKeys.apisInScope(apiScope).asScala, versionManager.enabledApis) + assertTrue(ApiKeys.apisInScope(apiScope).asScala.forall(versionManager.isApiEnabled)) + } @Test def testControllerApiIntersection(): Unit = { @@ -39,9 +59,6 @@ class ApiVersionManagerTest { controllerMaxVersion ))) - val brokerFeatures = BrokerFeatures.createDefault() - val featureCache = new FinalizedFeatureCache(brokerFeatures) - val versionManager = new DefaultApiVersionManager( apiScope = ApiScope.ZK_BROKER, interBrokerProtocolVersion = ApiVersion.latestVersion, @@ -62,9 +79,6 @@ class ApiVersionManagerTest { val forwardingManager = Mockito.mock(classOf[ForwardingManager]) Mockito.when(forwardingManager.controllerApiVersions).thenReturn(None) - val brokerFeatures = BrokerFeatures.createDefault() - val featureCache = new FinalizedFeatureCache(brokerFeatures) - val versionManager = new DefaultApiVersionManager( apiScope = ApiScope.ZK_BROKER, interBrokerProtocolVersion = ApiVersion.latestVersion, @@ -84,9 +98,6 @@ class ApiVersionManagerTest { @Test def testEnvelopeDisabledWhenForwardingManagerEmpty(): Unit = { - val brokerFeatures = BrokerFeatures.createDefault() - val featureCache = new FinalizedFeatureCache(brokerFeatures) - val versionManager = new DefaultApiVersionManager( apiScope = ApiScope.ZK_BROKER, interBrokerProtocolVersion = ApiVersion.latestVersion, From 51d00d03d7da2cded90871cfb7cf440b4d60c796 Mon Sep 17 00:00:00 2001 From: Jason Gustafson Date: Thu, 4 Feb 2021 21:28:29 -0800 Subject: [PATCH 3/7] Fix metadata benchmark --- .../kafka/jmh/metadata/MetadataRequestBenchmark.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/metadata/MetadataRequestBenchmark.java b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/metadata/MetadataRequestBenchmark.java index c71c05878cc5c..0735866ae70eb 100644 --- a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/metadata/MetadataRequestBenchmark.java +++ b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/metadata/MetadataRequestBenchmark.java @@ -23,13 +23,11 @@ import kafka.network.RequestChannel; import kafka.network.RequestConvertToJson; import kafka.server.AutoTopicCreationManager; -import kafka.server.BrokerFeatures; import kafka.server.BrokerTopicStats; import kafka.server.ClientQuotaManager; import kafka.server.ClientRequestQuotaManager; import kafka.server.ControllerMutationQuotaManager; import kafka.server.FetchManager; -import kafka.server.FinalizedFeatureCache; import kafka.server.KafkaApis; import kafka.server.KafkaConfig; import kafka.server.KafkaConfig$; @@ -38,11 +36,13 @@ import kafka.server.QuotaFactory; import kafka.server.ReplicaManager; import kafka.server.ReplicationQuotaManager; +import kafka.server.SimpleApiVersionManager; import kafka.server.ZkAdminManager; import kafka.server.ZkSupport; import kafka.server.metadata.CachedConfigRepository; import kafka.zk.KafkaZkClient; import org.apache.kafka.common.memory.MemoryPool; +import org.apache.kafka.common.message.ApiMessageType; import org.apache.kafka.common.message.UpdateMetadataRequestData.UpdateMetadataBroker; import org.apache.kafka.common.message.UpdateMetadataRequestData.UpdateMetadataEndpoint; import org.apache.kafka.common.message.UpdateMetadataRequestData.UpdateMetadataPartitionState; @@ -172,7 +172,6 @@ private KafkaApis createKafkaApis() { Properties kafkaProps = new Properties(); kafkaProps.put(KafkaConfig$.MODULE$.ZkConnectProp(), "zk"); kafkaProps.put(KafkaConfig$.MODULE$.BrokerIdProp(), brokerId + ""); - BrokerFeatures brokerFeatures = BrokerFeatures.createDefault(); return new KafkaApis(requestChannel, new ZkSupport(adminManager, kafkaController, kafkaZkClient, Option.empty(), metadataCache), replicaManager, @@ -191,8 +190,7 @@ private KafkaApis createKafkaApis() { "clusterId", new SystemTime(), null, - brokerFeatures, - new FinalizedFeatureCache(brokerFeatures)); + new SimpleApiVersionManager(ApiMessageType.ApiScope.ZK_BROKER)); } @TearDown(Level.Trial) From e17cd66a09887fdf8e27f12fe440ec145e79b80a Mon Sep 17 00:00:00 2001 From: Jason Gustafson Date: Thu, 18 Feb 2021 07:11:01 -0800 Subject: [PATCH 4/7] Rename `scope` to `listeners` --- .../apache/kafka/common/protocol/ApiKeys.java | 10 ++-- .../common/requests/ApiVersionsResponse.java | 20 +++++--- .../message/AddOffsetsToTxnRequest.json | 2 +- .../message/AddPartitionsToTxnRequest.json | 2 +- .../message/AlterClientQuotasRequest.json | 2 +- .../common/message/AlterConfigsRequest.json | 2 +- .../common/message/AlterIsrRequest.json | 2 +- .../AlterPartitionReassignmentsRequest.json | 2 +- .../message/AlterReplicaLogDirsRequest.json | 2 +- .../AlterUserScramCredentialsRequest.json | 2 +- .../common/message/ApiVersionsRequest.json | 2 +- .../message/BeginQuorumEpochRequest.json | 2 +- .../message/BrokerHeartbeatRequest.json | 2 +- .../message/BrokerRegistrationRequest.json | 2 +- .../message/ControlledShutdownRequest.json | 2 +- .../common/message/CreateAclsRequest.json | 2 +- .../message/CreateDelegationTokenRequest.json | 2 +- .../message/CreatePartitionsRequest.json | 2 +- .../common/message/CreateTopicsRequest.json | 2 +- .../common/message/DeleteAclsRequest.json | 2 +- .../common/message/DeleteGroupsRequest.json | 2 +- .../common/message/DeleteRecordsRequest.json | 2 +- .../common/message/DeleteTopicsRequest.json | 2 +- .../common/message/DescribeAclsRequest.json | 2 +- .../message/DescribeClientQuotasRequest.json | 2 +- .../message/DescribeClusterRequest.json | 2 +- .../message/DescribeConfigsRequest.json | 2 +- .../DescribeDelegationTokenRequest.json | 2 +- .../common/message/DescribeGroupsRequest.json | 2 +- .../message/DescribeLogDirsRequest.json | 2 +- .../message/DescribeProducersRequest.json | 2 +- .../common/message/DescribeQuorumRequest.json | 2 +- .../DescribeUserScramCredentialsRequest.json | 2 +- .../common/message/ElectLeadersRequest.json | 2 +- .../common/message/EndQuorumEpochRequest.json | 2 +- .../common/message/EndTxnRequest.json | 2 +- .../common/message/EnvelopeRequest.json | 2 +- .../message/ExpireDelegationTokenRequest.json | 2 +- .../common/message/FetchRequest.json | 2 +- .../common/message/FetchSnapshotRequest.json | 2 +- .../message/FindCoordinatorRequest.json | 2 +- .../common/message/HeartbeatRequest.json | 2 +- .../IncrementalAlterConfigsRequest.json | 2 +- .../common/message/InitProducerIdRequest.json | 2 +- .../common/message/JoinGroupRequest.json | 2 +- .../common/message/LeaderAndIsrRequest.json | 2 +- .../common/message/LeaveGroupRequest.json | 2 +- .../common/message/ListGroupsRequest.json | 2 +- .../common/message/ListOffsetsRequest.json | 2 +- .../ListPartitionReassignmentsRequest.json | 2 +- .../common/message/MetadataRequest.json | 2 +- .../common/message/OffsetCommitRequest.json | 2 +- .../common/message/OffsetDeleteRequest.json | 2 +- .../common/message/OffsetFetchRequest.json | 2 +- .../message/OffsetForLeaderEpochRequest.json | 2 +- .../common/message/ProduceRequest.json | 2 +- .../message/RenewDelegationTokenRequest.json | 2 +- .../message/SaslAuthenticateRequest.json | 2 +- .../common/message/SaslHandshakeRequest.json | 2 +- .../common/message/StopReplicaRequest.json | 2 +- .../common/message/SyncGroupRequest.json | 2 +- .../message/TxnOffsetCommitRequest.json | 2 +- .../message/UnregisterBrokerRequest.json | 2 +- .../common/message/UpdateFeaturesRequest.json | 2 +- .../common/message/UpdateMetadataRequest.json | 2 +- .../resources/common/message/VoteRequest.json | 2 +- .../message/WriteTxnMarkersRequest.json | 2 +- .../kafka/clients/NetworkClientTest.java | 4 +- .../kafka/clients/NodeApiVersionsTest.java | 10 ++-- .../clients/admin/KafkaAdminClientTest.java | 2 +- .../consumer/internals/FetcherTest.java | 2 +- .../producer/internals/SenderTest.java | 2 +- .../kafka/common/network/NioEchoServer.java | 2 +- .../network/SaslChannelBuilderTest.java | 2 +- .../common/network/SslTransportLayerTest.java | 2 +- .../kafka/common/protocol/ApiKeysTest.java | 2 +- .../requests/ApiVersionsResponseTest.java | 10 ++-- .../common/requests/RequestResponseTest.java | 4 +- .../authenticator/SaslAuthenticatorTest.java | 4 +- .../SaslServerAuthenticatorTest.java | 2 +- .../src/main/scala/kafka/api/ApiVersion.scala | 12 ++--- .../scala/kafka/network/RequestChannel.scala | 6 +-- .../scala/kafka/network/SocketServer.scala | 2 +- .../kafka/server/ApiVersionManager.scala | 25 +++++----- .../main/scala/kafka/server/KafkaServer.scala | 4 +- .../scala/kafka/tools/TestRaftServer.scala | 4 +- .../server/GssapiAuthenticationTest.scala | 4 +- .../scala/unit/kafka/api/ApiVersionTest.scala | 10 ++-- .../unit/kafka/network/SocketServerTest.scala | 4 +- .../AbstractApiVersionsRequestTest.scala | 4 +- .../kafka/server/ApiVersionManagerTest.scala | 18 +++---- .../kafka/server/ForwardingManagerTest.scala | 4 +- .../unit/kafka/server/KafkaApisTest.scala | 10 ++-- .../message/ApiMessageTypeGenerator.java | 50 +++++++++---------- .../org/apache/kafka/message/MessageSpec.java | 14 +++--- ...ApiScope.java => RequestListenerType.java} | 2 +- .../metadata/MetadataRequestBenchmark.java | 2 +- 97 files changed, 196 insertions(+), 187 deletions(-) rename generator/src/main/java/org/apache/kafka/message/{RequestApiScope.java => RequestListenerType.java} (96%) diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java b/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java index 2c304dad74883..8f6422f500798 100644 --- a/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java +++ b/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java @@ -207,8 +207,8 @@ public short responseHeaderVersion(short apiVersion) { return messageType.responseHeaderVersion(apiVersion); } - public boolean inScope(ApiMessageType.ApiScope scope) { - return messageType.scope().contains(scope); + public boolean inScope(ApiMessageType.ListenerType listener) { + return messageType.listeners().contains(listener); } private static String toHtml() { @@ -251,12 +251,12 @@ public void visit(Type field) { } public static EnumSet zkBrokerApis() { - return apisInScope(ApiMessageType.ApiScope.ZK_BROKER); + return apisForListener(ApiMessageType.ListenerType.ZK_BROKER); } - public static EnumSet apisInScope(ApiMessageType.ApiScope scope) { + public static EnumSet apisForListener(ApiMessageType.ListenerType listener) { List controllerApis = Arrays.stream(ApiKeys.values()) - .filter(apiKey -> apiKey.messageType.scope().contains(scope)) + .filter(apiKey -> apiKey.messageType.listeners().contains(listener)) .collect(Collectors.toList()); return EnumSet.copyOf(controllerApis); } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsResponse.java index b4fadcdfb66f8..1190989576380 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ApiVersionsResponse.java @@ -96,16 +96,16 @@ public static ApiVersionsResponse parse(ByteBuffer buffer, short version) { } public static ApiVersionsResponse defaultApiVersionsResponse( - ApiMessageType.ApiScope scope + ApiMessageType.ListenerType listenerType ) { - return defaultApiVersionsResponse(0, scope); + return defaultApiVersionsResponse(0, listenerType); } public static ApiVersionsResponse defaultApiVersionsResponse( int throttleTimeMs, - ApiMessageType.ApiScope scope + ApiMessageType.ListenerType listenerType ) { - return createApiVersionsResponse(throttleTimeMs, filterApis(RecordVersion.current(), scope)); + return createApiVersionsResponse(throttleTimeMs, filterApis(RecordVersion.current(), listenerType)); } public static ApiVersionsResponse createApiVersionsResponse( @@ -140,9 +140,12 @@ public static ApiVersionsResponse createApiVersionsResponse( ); } - public static ApiVersionCollection filterApis(RecordVersion minRecordVersion, ApiMessageType.ApiScope scope) { + public static ApiVersionCollection filterApis( + RecordVersion minRecordVersion, + ApiMessageType.ListenerType listenerType + ) { ApiVersionCollection apiKeys = new ApiVersionCollection(); - for (ApiKeys apiKey : ApiKeys.apisInScope(scope)) { + for (ApiKeys apiKey : ApiKeys.apisForListener(listenerType)) { if (apiKey.minRequiredInterBrokerMagic <= minRecordVersion.value) { apiKeys.add(ApiVersionsResponse.toApiVersion(apiKey)); } @@ -162,17 +165,18 @@ public static ApiVersionCollection collectApis(Set apiKeys) { * Find the common range of supported API versions between the locally * known range and that of another set. * + * @param listenerType the listener type which constrains the set of exposed APIs * @param minRecordVersion min inter broker magic * @param activeControllerApiVersions controller ApiVersions * @return commonly agreed ApiVersion collection */ public static ApiVersionCollection intersectForwardableApis( - final ApiMessageType.ApiScope scope, + final ApiMessageType.ListenerType listenerType, final RecordVersion minRecordVersion, final Map activeControllerApiVersions ) { ApiVersionCollection apiKeys = new ApiVersionCollection(); - for (ApiKeys apiKey : ApiKeys.apisInScope(scope)) { + for (ApiKeys apiKey : ApiKeys.apisForListener(listenerType)) { if (apiKey.minRequiredInterBrokerMagic <= minRecordVersion.value) { ApiVersion brokerApiVersion = toApiVersion(apiKey); diff --git a/clients/src/main/resources/common/message/AddOffsetsToTxnRequest.json b/clients/src/main/resources/common/message/AddOffsetsToTxnRequest.json index eb1c01a767f52..ade3fc72c9a51 100644 --- a/clients/src/main/resources/common/message/AddOffsetsToTxnRequest.json +++ b/clients/src/main/resources/common/message/AddOffsetsToTxnRequest.json @@ -16,7 +16,7 @@ { "apiKey": 25, "type": "request", - "scope": ["zkBroker", "broker"], + "listeners": ["zkBroker", "broker"], "name": "AddOffsetsToTxnRequest", // Version 1 is the same as version 0. // diff --git a/clients/src/main/resources/common/message/AddPartitionsToTxnRequest.json b/clients/src/main/resources/common/message/AddPartitionsToTxnRequest.json index 8a52bb8261212..4920da176c723 100644 --- a/clients/src/main/resources/common/message/AddPartitionsToTxnRequest.json +++ b/clients/src/main/resources/common/message/AddPartitionsToTxnRequest.json @@ -16,7 +16,7 @@ { "apiKey": 24, "type": "request", - "scope": ["zkBroker", "broker"], + "listeners": ["zkBroker", "broker"], "name": "AddPartitionsToTxnRequest", // Version 1 is the same as version 0. // diff --git a/clients/src/main/resources/common/message/AlterClientQuotasRequest.json b/clients/src/main/resources/common/message/AlterClientQuotasRequest.json index 746ac00873514..6bfdc925c2919 100644 --- a/clients/src/main/resources/common/message/AlterClientQuotasRequest.json +++ b/clients/src/main/resources/common/message/AlterClientQuotasRequest.json @@ -16,7 +16,7 @@ { "apiKey": 49, "type": "request", - "scope": ["zkBroker", "broker", "controller"], + "listeners": ["zkBroker", "broker", "controller"], "name": "AlterClientQuotasRequest", "validVersions": "0-1", // Version 1 enables flexible versions. diff --git a/clients/src/main/resources/common/message/AlterConfigsRequest.json b/clients/src/main/resources/common/message/AlterConfigsRequest.json index 0fb73305b24bb..31057e3410aaf 100644 --- a/clients/src/main/resources/common/message/AlterConfigsRequest.json +++ b/clients/src/main/resources/common/message/AlterConfigsRequest.json @@ -16,7 +16,7 @@ { "apiKey": 33, "type": "request", - "scope": ["zkBroker", "broker", "controller"], + "listeners": ["zkBroker", "broker", "controller"], "name": "AlterConfigsRequest", // Version 1 is the same as version 0. // Version 2 enables flexible versions. diff --git a/clients/src/main/resources/common/message/AlterIsrRequest.json b/clients/src/main/resources/common/message/AlterIsrRequest.json index e766b883150ef..f950cd7005ae8 100644 --- a/clients/src/main/resources/common/message/AlterIsrRequest.json +++ b/clients/src/main/resources/common/message/AlterIsrRequest.json @@ -16,7 +16,7 @@ { "apiKey": 56, "type": "request", - "scope": ["zkBroker", "controller"], + "listeners": ["zkBroker", "controller"], "name": "AlterIsrRequest", "validVersions": "0", "flexibleVersions": "0+", diff --git a/clients/src/main/resources/common/message/AlterPartitionReassignmentsRequest.json b/clients/src/main/resources/common/message/AlterPartitionReassignmentsRequest.json index 4ae92da434698..2e124413a3bbc 100644 --- a/clients/src/main/resources/common/message/AlterPartitionReassignmentsRequest.json +++ b/clients/src/main/resources/common/message/AlterPartitionReassignmentsRequest.json @@ -16,7 +16,7 @@ { "apiKey": 45, "type": "request", - "scope": ["zkBroker", "broker"], + "listeners": ["zkBroker", "broker"], "name": "AlterPartitionReassignmentsRequest", "validVersions": "0", "flexibleVersions": "0+", diff --git a/clients/src/main/resources/common/message/AlterReplicaLogDirsRequest.json b/clients/src/main/resources/common/message/AlterReplicaLogDirsRequest.json index e74671d59c7ec..2306caaf98499 100644 --- a/clients/src/main/resources/common/message/AlterReplicaLogDirsRequest.json +++ b/clients/src/main/resources/common/message/AlterReplicaLogDirsRequest.json @@ -16,7 +16,7 @@ { "apiKey": 34, "type": "request", - "scope": ["zkBroker", "broker"], + "listeners": ["zkBroker", "broker"], "name": "AlterReplicaLogDirsRequest", // Version 1 is the same as version 0. // Version 2 enables flexible versions. diff --git a/clients/src/main/resources/common/message/AlterUserScramCredentialsRequest.json b/clients/src/main/resources/common/message/AlterUserScramCredentialsRequest.json index 3b9e6cbf38716..8937394ef6e51 100644 --- a/clients/src/main/resources/common/message/AlterUserScramCredentialsRequest.json +++ b/clients/src/main/resources/common/message/AlterUserScramCredentialsRequest.json @@ -16,7 +16,7 @@ { "apiKey": 51, "type": "request", - "scope": ["zkBroker", "broker", "controller"], + "listeners": ["zkBroker", "broker", "controller"], "name": "AlterUserScramCredentialsRequest", "validVersions": "0", "flexibleVersions": "0+", diff --git a/clients/src/main/resources/common/message/ApiVersionsRequest.json b/clients/src/main/resources/common/message/ApiVersionsRequest.json index 8213e4411a3c9..39f3cb36562f0 100644 --- a/clients/src/main/resources/common/message/ApiVersionsRequest.json +++ b/clients/src/main/resources/common/message/ApiVersionsRequest.json @@ -16,7 +16,7 @@ { "apiKey": 18, "type": "request", - "scope": ["zkBroker", "broker", "controller", "raft"], + "listeners": ["zkBroker", "broker", "controller", "raft"], "name": "ApiVersionsRequest", // Versions 0 through 2 of ApiVersionsRequest are the same. // diff --git a/clients/src/main/resources/common/message/BeginQuorumEpochRequest.json b/clients/src/main/resources/common/message/BeginQuorumEpochRequest.json index bae98ba15cc52..cad4bb2e390a2 100644 --- a/clients/src/main/resources/common/message/BeginQuorumEpochRequest.json +++ b/clients/src/main/resources/common/message/BeginQuorumEpochRequest.json @@ -16,7 +16,7 @@ { "apiKey": 53, "type": "request", - "scope": ["controller", "raft"], + "listeners": ["controller", "raft"], "name": "BeginQuorumEpochRequest", "validVersions": "0", "fields": [ diff --git a/clients/src/main/resources/common/message/BrokerHeartbeatRequest.json b/clients/src/main/resources/common/message/BrokerHeartbeatRequest.json index 3140c5cb9a99d..ce08d119b7cc2 100644 --- a/clients/src/main/resources/common/message/BrokerHeartbeatRequest.json +++ b/clients/src/main/resources/common/message/BrokerHeartbeatRequest.json @@ -16,7 +16,7 @@ { "apiKey": 63, "type": "request", - "scope": ["controller"], + "listeners": ["controller"], "name": "BrokerHeartbeatRequest", "validVersions": "0", "flexibleVersions": "0+", diff --git a/clients/src/main/resources/common/message/BrokerRegistrationRequest.json b/clients/src/main/resources/common/message/BrokerRegistrationRequest.json index 061af50046d3d..3e27cf127434f 100644 --- a/clients/src/main/resources/common/message/BrokerRegistrationRequest.json +++ b/clients/src/main/resources/common/message/BrokerRegistrationRequest.json @@ -16,7 +16,7 @@ { "apiKey":62, "type": "request", - "scope": ["controller"], + "listeners": ["controller"], "name": "BrokerRegistrationRequest", "validVersions": "0", "flexibleVersions": "0+", diff --git a/clients/src/main/resources/common/message/ControlledShutdownRequest.json b/clients/src/main/resources/common/message/ControlledShutdownRequest.json index 1cf371f02eda6..49561f7b6e7d7 100644 --- a/clients/src/main/resources/common/message/ControlledShutdownRequest.json +++ b/clients/src/main/resources/common/message/ControlledShutdownRequest.json @@ -16,7 +16,7 @@ { "apiKey": 7, "type": "request", - "scope": ["zkBroker", "controller"], + "listeners": ["zkBroker", "controller"], "name": "ControlledShutdownRequest", // Version 0 of ControlledShutdownRequest has a non-standard request header // which does not include clientId. Version 1 and later use the standard diff --git a/clients/src/main/resources/common/message/CreateAclsRequest.json b/clients/src/main/resources/common/message/CreateAclsRequest.json index 7bae29c1e7447..5b3bfed78162c 100644 --- a/clients/src/main/resources/common/message/CreateAclsRequest.json +++ b/clients/src/main/resources/common/message/CreateAclsRequest.json @@ -16,7 +16,7 @@ { "apiKey": 30, "type": "request", - "scope": ["zkBroker", "broker", "controller"], + "listeners": ["zkBroker", "broker", "controller"], "name": "CreateAclsRequest", // Version 1 adds resource pattern type. // Version 2 enables flexible versions. diff --git a/clients/src/main/resources/common/message/CreateDelegationTokenRequest.json b/clients/src/main/resources/common/message/CreateDelegationTokenRequest.json index f73dd9fbdd797..0c31d32fe56b7 100644 --- a/clients/src/main/resources/common/message/CreateDelegationTokenRequest.json +++ b/clients/src/main/resources/common/message/CreateDelegationTokenRequest.json @@ -16,7 +16,7 @@ { "apiKey": 38, "type": "request", - "scope": ["zkBroker", "broker", "controller"], + "listeners": ["zkBroker", "broker", "controller"], "name": "CreateDelegationTokenRequest", // Version 1 is the same as version 0. // diff --git a/clients/src/main/resources/common/message/CreatePartitionsRequest.json b/clients/src/main/resources/common/message/CreatePartitionsRequest.json index 4b1c995a9e768..6e249498659fa 100644 --- a/clients/src/main/resources/common/message/CreatePartitionsRequest.json +++ b/clients/src/main/resources/common/message/CreatePartitionsRequest.json @@ -16,7 +16,7 @@ { "apiKey": 37, "type": "request", - "scope": ["zkBroker", "broker", "controller"], + "listeners": ["zkBroker", "broker", "controller"], "name": "CreatePartitionsRequest", // Version 1 is the same as version 0. // diff --git a/clients/src/main/resources/common/message/CreateTopicsRequest.json b/clients/src/main/resources/common/message/CreateTopicsRequest.json index 17769459db58d..0882de9fa64d2 100644 --- a/clients/src/main/resources/common/message/CreateTopicsRequest.json +++ b/clients/src/main/resources/common/message/CreateTopicsRequest.json @@ -16,7 +16,7 @@ { "apiKey": 19, "type": "request", - "scope": ["zkBroker", "broker", "controller"], + "listeners": ["zkBroker", "broker", "controller"], "name": "CreateTopicsRequest", // Version 1 adds validateOnly. // diff --git a/clients/src/main/resources/common/message/DeleteAclsRequest.json b/clients/src/main/resources/common/message/DeleteAclsRequest.json index 532f413b1547a..fd7c1522b43bd 100644 --- a/clients/src/main/resources/common/message/DeleteAclsRequest.json +++ b/clients/src/main/resources/common/message/DeleteAclsRequest.json @@ -16,7 +16,7 @@ { "apiKey": 31, "type": "request", - "scope": ["zkBroker", "broker", "controller"], + "listeners": ["zkBroker", "broker", "controller"], "name": "DeleteAclsRequest", // Version 1 adds the pattern type. // Version 2 enables flexible versions. diff --git a/clients/src/main/resources/common/message/DeleteGroupsRequest.json b/clients/src/main/resources/common/message/DeleteGroupsRequest.json index b0d068da4021c..1ac6a053e63b3 100644 --- a/clients/src/main/resources/common/message/DeleteGroupsRequest.json +++ b/clients/src/main/resources/common/message/DeleteGroupsRequest.json @@ -16,7 +16,7 @@ { "apiKey": 42, "type": "request", - "scope": ["zkBroker", "broker"], + "listeners": ["zkBroker", "broker"], "name": "DeleteGroupsRequest", // Version 1 is the same as version 0. // diff --git a/clients/src/main/resources/common/message/DeleteRecordsRequest.json b/clients/src/main/resources/common/message/DeleteRecordsRequest.json index 16bd9c621a072..06a12d85c8bb4 100644 --- a/clients/src/main/resources/common/message/DeleteRecordsRequest.json +++ b/clients/src/main/resources/common/message/DeleteRecordsRequest.json @@ -16,7 +16,7 @@ { "apiKey": 21, "type": "request", - "scope": ["zkBroker", "broker"], + "listeners": ["zkBroker", "broker"], "name": "DeleteRecordsRequest", // Version 1 is the same as version 0. diff --git a/clients/src/main/resources/common/message/DeleteTopicsRequest.json b/clients/src/main/resources/common/message/DeleteTopicsRequest.json index 73d5371be93bb..f757ff7755453 100644 --- a/clients/src/main/resources/common/message/DeleteTopicsRequest.json +++ b/clients/src/main/resources/common/message/DeleteTopicsRequest.json @@ -16,7 +16,7 @@ { "apiKey": 20, "type": "request", - "scope": ["zkBroker", "broker", "controller"], + "listeners": ["zkBroker", "broker", "controller"], "name": "DeleteTopicsRequest", // Versions 0, 1, 2, and 3 are the same. // diff --git a/clients/src/main/resources/common/message/DescribeAclsRequest.json b/clients/src/main/resources/common/message/DescribeAclsRequest.json index f556f77e452fb..58886da654707 100644 --- a/clients/src/main/resources/common/message/DescribeAclsRequest.json +++ b/clients/src/main/resources/common/message/DescribeAclsRequest.json @@ -16,7 +16,7 @@ { "apiKey": 29, "type": "request", - "scope": ["zkBroker", "broker", "controller"], + "listeners": ["zkBroker", "broker", "controller"], "name": "DescribeAclsRequest", // Version 1 adds resource pattern type. // Version 2 enables flexible versions. diff --git a/clients/src/main/resources/common/message/DescribeClientQuotasRequest.json b/clients/src/main/resources/common/message/DescribeClientQuotasRequest.json index aea0936309826..d14cfc95733d3 100644 --- a/clients/src/main/resources/common/message/DescribeClientQuotasRequest.json +++ b/clients/src/main/resources/common/message/DescribeClientQuotasRequest.json @@ -16,7 +16,7 @@ { "apiKey": 48, "type": "request", - "scope": ["zkBroker", "broker"], + "listeners": ["zkBroker", "broker"], "name": "DescribeClientQuotasRequest", // Version 1 enables flexible versions. "validVersions": "0-1", diff --git a/clients/src/main/resources/common/message/DescribeClusterRequest.json b/clients/src/main/resources/common/message/DescribeClusterRequest.json index 2b2aa25a81516..192e4d87d4497 100644 --- a/clients/src/main/resources/common/message/DescribeClusterRequest.json +++ b/clients/src/main/resources/common/message/DescribeClusterRequest.json @@ -16,7 +16,7 @@ { "apiKey": 60, "type": "request", - "scope": ["zkBroker", "broker"], + "listeners": ["zkBroker", "broker"], "name": "DescribeClusterRequest", "validVersions": "0", "flexibleVersions": "0+", diff --git a/clients/src/main/resources/common/message/DescribeConfigsRequest.json b/clients/src/main/resources/common/message/DescribeConfigsRequest.json index b583fa0e6db76..23be19cb0e625 100644 --- a/clients/src/main/resources/common/message/DescribeConfigsRequest.json +++ b/clients/src/main/resources/common/message/DescribeConfigsRequest.json @@ -16,7 +16,7 @@ { "apiKey": 32, "type": "request", - "scope": ["zkBroker", "broker", "controller"], + "listeners": ["zkBroker", "broker", "controller"], "name": "DescribeConfigsRequest", // Version 1 adds IncludeSynonyms. // Version 2 is the same as version 1. diff --git a/clients/src/main/resources/common/message/DescribeDelegationTokenRequest.json b/clients/src/main/resources/common/message/DescribeDelegationTokenRequest.json index 6443011714136..da5bbd046d16c 100644 --- a/clients/src/main/resources/common/message/DescribeDelegationTokenRequest.json +++ b/clients/src/main/resources/common/message/DescribeDelegationTokenRequest.json @@ -16,7 +16,7 @@ { "apiKey": 41, "type": "request", - "scope": ["zkBroker", "broker", "controller"], + "listeners": ["zkBroker", "broker", "controller"], "name": "DescribeDelegationTokenRequest", // Version 1 is the same as version 0. // Version 2 adds flexible version support diff --git a/clients/src/main/resources/common/message/DescribeGroupsRequest.json b/clients/src/main/resources/common/message/DescribeGroupsRequest.json index 254b74d731ef6..6b10b0637a205 100644 --- a/clients/src/main/resources/common/message/DescribeGroupsRequest.json +++ b/clients/src/main/resources/common/message/DescribeGroupsRequest.json @@ -16,7 +16,7 @@ { "apiKey": 15, "type": "request", - "scope": ["zkBroker", "broker"], + "listeners": ["zkBroker", "broker"], "name": "DescribeGroupsRequest", // Versions 1 and 2 are the same as version 0. // diff --git a/clients/src/main/resources/common/message/DescribeLogDirsRequest.json b/clients/src/main/resources/common/message/DescribeLogDirsRequest.json index b55d658bdf1d2..cfb160f8166bf 100644 --- a/clients/src/main/resources/common/message/DescribeLogDirsRequest.json +++ b/clients/src/main/resources/common/message/DescribeLogDirsRequest.json @@ -16,7 +16,7 @@ { "apiKey": 35, "type": "request", - "scope": ["zkBroker", "broker"], + "listeners": ["zkBroker", "broker"], "name": "DescribeLogDirsRequest", // Version 1 is the same as version 0. "validVersions": "0-2", diff --git a/clients/src/main/resources/common/message/DescribeProducersRequest.json b/clients/src/main/resources/common/message/DescribeProducersRequest.json index 801a12a14f996..0ffd834e6bde8 100644 --- a/clients/src/main/resources/common/message/DescribeProducersRequest.json +++ b/clients/src/main/resources/common/message/DescribeProducersRequest.json @@ -16,7 +16,7 @@ { "apiKey": 61, "type": "request", - "scope": ["zkBroker", "broker"], + "listeners": ["zkBroker", "broker"], "name": "DescribeProducersRequest", "validVersions": "0", "flexibleVersions": "0+", diff --git a/clients/src/main/resources/common/message/DescribeQuorumRequest.json b/clients/src/main/resources/common/message/DescribeQuorumRequest.json index 7d606ce4229d9..cd4a7f1db5470 100644 --- a/clients/src/main/resources/common/message/DescribeQuorumRequest.json +++ b/clients/src/main/resources/common/message/DescribeQuorumRequest.json @@ -16,7 +16,7 @@ { "apiKey": 55, "type": "request", - "scope": ["broker", "controller"], + "listeners": ["broker", "controller"], "name": "DescribeQuorumRequest", "validVersions": "0", "flexibleVersions": "0+", diff --git a/clients/src/main/resources/common/message/DescribeUserScramCredentialsRequest.json b/clients/src/main/resources/common/message/DescribeUserScramCredentialsRequest.json index d8194cda0668a..2f7a1112c4800 100644 --- a/clients/src/main/resources/common/message/DescribeUserScramCredentialsRequest.json +++ b/clients/src/main/resources/common/message/DescribeUserScramCredentialsRequest.json @@ -16,7 +16,7 @@ { "apiKey": 50, "type": "request", - "scope": ["zkBroker", "broker", "controller"], + "listeners": ["zkBroker", "broker", "controller"], "name": "DescribeUserScramCredentialsRequest", "validVersions": "0", "flexibleVersions": "0+", diff --git a/clients/src/main/resources/common/message/ElectLeadersRequest.json b/clients/src/main/resources/common/message/ElectLeadersRequest.json index ca0210b815b44..dd9fa21641585 100644 --- a/clients/src/main/resources/common/message/ElectLeadersRequest.json +++ b/clients/src/main/resources/common/message/ElectLeadersRequest.json @@ -16,7 +16,7 @@ { "apiKey": 43, "type": "request", - "scope": ["zkBroker", "broker", "controller"], + "listeners": ["zkBroker", "broker", "controller"], "name": "ElectLeadersRequest", // Version 1 implements multiple leader election types, as described by KIP-460. // diff --git a/clients/src/main/resources/common/message/EndQuorumEpochRequest.json b/clients/src/main/resources/common/message/EndQuorumEpochRequest.json index 5ab856dee6e4c..23058328c238f 100644 --- a/clients/src/main/resources/common/message/EndQuorumEpochRequest.json +++ b/clients/src/main/resources/common/message/EndQuorumEpochRequest.json @@ -16,7 +16,7 @@ { "apiKey": 54, "type": "request", - "scope": ["controller", "raft"], + "listeners": ["controller", "raft"], "name": "EndQuorumEpochRequest", "validVersions": "0", "fields": [ diff --git a/clients/src/main/resources/common/message/EndTxnRequest.json b/clients/src/main/resources/common/message/EndTxnRequest.json index 9c9ad80da07a6..f16ef76246d35 100644 --- a/clients/src/main/resources/common/message/EndTxnRequest.json +++ b/clients/src/main/resources/common/message/EndTxnRequest.json @@ -16,7 +16,7 @@ { "apiKey": 26, "type": "request", - "scope": ["zkBroker", "broker"], + "listeners": ["zkBroker", "broker"], "name": "EndTxnRequest", // Version 1 is the same as version 0. // diff --git a/clients/src/main/resources/common/message/EnvelopeRequest.json b/clients/src/main/resources/common/message/EnvelopeRequest.json index 1265bdc3d1d94..1f6ff62de8d7f 100644 --- a/clients/src/main/resources/common/message/EnvelopeRequest.json +++ b/clients/src/main/resources/common/message/EnvelopeRequest.json @@ -16,7 +16,7 @@ { "apiKey": 58, "type": "request", - "scope": ["controller"], + "listeners": ["controller"], "name": "EnvelopeRequest", // Request struct for forwarding. "validVersions": "0", diff --git a/clients/src/main/resources/common/message/ExpireDelegationTokenRequest.json b/clients/src/main/resources/common/message/ExpireDelegationTokenRequest.json index 1252a24e0faf9..c830a93df398b 100644 --- a/clients/src/main/resources/common/message/ExpireDelegationTokenRequest.json +++ b/clients/src/main/resources/common/message/ExpireDelegationTokenRequest.json @@ -16,7 +16,7 @@ { "apiKey": 40, "type": "request", - "scope": ["zkBroker", "broker", "controller"], + "listeners": ["zkBroker", "broker", "controller"], "name": "ExpireDelegationTokenRequest", // Version 1 is the same as version 0. // Version 2 adds flexible version support diff --git a/clients/src/main/resources/common/message/FetchRequest.json b/clients/src/main/resources/common/message/FetchRequest.json index 561f69a13fb9d..768bfd5f016fd 100644 --- a/clients/src/main/resources/common/message/FetchRequest.json +++ b/clients/src/main/resources/common/message/FetchRequest.json @@ -16,7 +16,7 @@ { "apiKey": 1, "type": "request", - "scope": ["zkBroker", "broker", "controller", "raft"], + "listeners": ["zkBroker", "broker", "controller", "raft"], "name": "FetchRequest", // // Version 1 is the same as version 0. diff --git a/clients/src/main/resources/common/message/FetchSnapshotRequest.json b/clients/src/main/resources/common/message/FetchSnapshotRequest.json index ec39ac12fe010..973acb817f35e 100644 --- a/clients/src/main/resources/common/message/FetchSnapshotRequest.json +++ b/clients/src/main/resources/common/message/FetchSnapshotRequest.json @@ -16,7 +16,7 @@ { "apiKey": 59, "type": "request", - "scope": ["controller", "raft"], + "listeners": ["controller", "raft"], "name": "FetchSnapshotRequest", "validVersions": "0", "flexibleVersions": "0+", diff --git a/clients/src/main/resources/common/message/FindCoordinatorRequest.json b/clients/src/main/resources/common/message/FindCoordinatorRequest.json index f2f8b7ae40afc..cd5b77a3f83e9 100644 --- a/clients/src/main/resources/common/message/FindCoordinatorRequest.json +++ b/clients/src/main/resources/common/message/FindCoordinatorRequest.json @@ -16,7 +16,7 @@ { "apiKey": 10, "type": "request", - "scope": ["zkBroker", "broker"], + "listeners": ["zkBroker", "broker"], "name": "FindCoordinatorRequest", // Version 1 adds KeyType. // diff --git a/clients/src/main/resources/common/message/HeartbeatRequest.json b/clients/src/main/resources/common/message/HeartbeatRequest.json index 0f8322256f409..dcf776d8ec4e3 100644 --- a/clients/src/main/resources/common/message/HeartbeatRequest.json +++ b/clients/src/main/resources/common/message/HeartbeatRequest.json @@ -16,7 +16,7 @@ { "apiKey": 12, "type": "request", - "scope": ["zkBroker", "broker"], + "listeners": ["zkBroker", "broker"], "name": "HeartbeatRequest", // Version 1 and version 2 are the same as version 0. // diff --git a/clients/src/main/resources/common/message/IncrementalAlterConfigsRequest.json b/clients/src/main/resources/common/message/IncrementalAlterConfigsRequest.json index 2eae744941828..d4955c91b85a4 100644 --- a/clients/src/main/resources/common/message/IncrementalAlterConfigsRequest.json +++ b/clients/src/main/resources/common/message/IncrementalAlterConfigsRequest.json @@ -16,7 +16,7 @@ { "apiKey": 44, "type": "request", - "scope": ["zkBroker", "broker", "controller"], + "listeners": ["zkBroker", "broker", "controller"], "name": "IncrementalAlterConfigsRequest", // Version 1 is the first flexible version. "validVersions": "0-1", diff --git a/clients/src/main/resources/common/message/InitProducerIdRequest.json b/clients/src/main/resources/common/message/InitProducerIdRequest.json index 5c6af0d424703..e8795e6582169 100644 --- a/clients/src/main/resources/common/message/InitProducerIdRequest.json +++ b/clients/src/main/resources/common/message/InitProducerIdRequest.json @@ -16,7 +16,7 @@ { "apiKey": 22, "type": "request", - "scope": ["zkBroker", "broker"], + "listeners": ["zkBroker", "broker"], "name": "InitProducerIdRequest", // Version 1 is the same as version 0. // diff --git a/clients/src/main/resources/common/message/JoinGroupRequest.json b/clients/src/main/resources/common/message/JoinGroupRequest.json index a18d5fc164e59..d9113b76a3c6e 100644 --- a/clients/src/main/resources/common/message/JoinGroupRequest.json +++ b/clients/src/main/resources/common/message/JoinGroupRequest.json @@ -16,7 +16,7 @@ { "apiKey": 11, "type": "request", - "scope": ["zkBroker", "broker"], + "listeners": ["zkBroker", "broker"], "name": "JoinGroupRequest", // Version 1 adds RebalanceTimeoutMs. // diff --git a/clients/src/main/resources/common/message/LeaderAndIsrRequest.json b/clients/src/main/resources/common/message/LeaderAndIsrRequest.json index 221b73861969d..57e6f21f3cf6e 100644 --- a/clients/src/main/resources/common/message/LeaderAndIsrRequest.json +++ b/clients/src/main/resources/common/message/LeaderAndIsrRequest.json @@ -16,7 +16,7 @@ { "apiKey": 4, "type": "request", - "scope": ["zkBroker"], + "listeners": ["zkBroker"], "name": "LeaderAndIsrRequest", // Version 1 adds IsNew. // diff --git a/clients/src/main/resources/common/message/LeaveGroupRequest.json b/clients/src/main/resources/common/message/LeaveGroupRequest.json index 664ffaf6e31eb..893c945c20411 100644 --- a/clients/src/main/resources/common/message/LeaveGroupRequest.json +++ b/clients/src/main/resources/common/message/LeaveGroupRequest.json @@ -16,7 +16,7 @@ { "apiKey": 13, "type": "request", - "scope": ["zkBroker", "broker"], + "listeners": ["zkBroker", "broker"], "name": "LeaveGroupRequest", // Version 1 and 2 are the same as version 0. // diff --git a/clients/src/main/resources/common/message/ListGroupsRequest.json b/clients/src/main/resources/common/message/ListGroupsRequest.json index 8824255f7b5a4..3f62e28350956 100644 --- a/clients/src/main/resources/common/message/ListGroupsRequest.json +++ b/clients/src/main/resources/common/message/ListGroupsRequest.json @@ -16,7 +16,7 @@ { "apiKey": 16, "type": "request", - "scope": ["zkBroker", "broker"], + "listeners": ["zkBroker", "broker"], "name": "ListGroupsRequest", // Version 1 and 2 are the same as version 0. // diff --git a/clients/src/main/resources/common/message/ListOffsetsRequest.json b/clients/src/main/resources/common/message/ListOffsetsRequest.json index c25cb8a183a05..a464c9376444f 100644 --- a/clients/src/main/resources/common/message/ListOffsetsRequest.json +++ b/clients/src/main/resources/common/message/ListOffsetsRequest.json @@ -16,7 +16,7 @@ { "apiKey": 2, "type": "request", - "scope": ["zkBroker", "broker"], + "listeners": ["zkBroker", "broker"], "name": "ListOffsetsRequest", // Version 1 removes MaxNumOffsets. From this version forward, only a single // offset can be returned. diff --git a/clients/src/main/resources/common/message/ListPartitionReassignmentsRequest.json b/clients/src/main/resources/common/message/ListPartitionReassignmentsRequest.json index 0ba9963d76be9..f013e3fe9ffab 100644 --- a/clients/src/main/resources/common/message/ListPartitionReassignmentsRequest.json +++ b/clients/src/main/resources/common/message/ListPartitionReassignmentsRequest.json @@ -16,7 +16,7 @@ { "apiKey": 46, "type": "request", - "scope": ["zkBroker", "broker"], + "listeners": ["zkBroker", "broker"], "name": "ListPartitionReassignmentsRequest", "validVersions": "0", "flexibleVersions": "0+", diff --git a/clients/src/main/resources/common/message/MetadataRequest.json b/clients/src/main/resources/common/message/MetadataRequest.json index edf4cd78f58cb..66908103e99c8 100644 --- a/clients/src/main/resources/common/message/MetadataRequest.json +++ b/clients/src/main/resources/common/message/MetadataRequest.json @@ -16,7 +16,7 @@ { "apiKey": 3, "type": "request", - "scope": ["zkBroker", "broker"], + "listeners": ["zkBroker", "broker"], "name": "MetadataRequest", "validVersions": "0-11", "flexibleVersions": "9+", diff --git a/clients/src/main/resources/common/message/OffsetCommitRequest.json b/clients/src/main/resources/common/message/OffsetCommitRequest.json index 18f065141509f..cf112e1ed72cb 100644 --- a/clients/src/main/resources/common/message/OffsetCommitRequest.json +++ b/clients/src/main/resources/common/message/OffsetCommitRequest.json @@ -16,7 +16,7 @@ { "apiKey": 8, "type": "request", - "scope": ["zkBroker", "broker"], + "listeners": ["zkBroker", "broker"], "name": "OffsetCommitRequest", // Version 1 adds timestamp and group membership information, as well as the commit timestamp. // diff --git a/clients/src/main/resources/common/message/OffsetDeleteRequest.json b/clients/src/main/resources/common/message/OffsetDeleteRequest.json index 5e793af6b80a9..394d1bb64df7b 100644 --- a/clients/src/main/resources/common/message/OffsetDeleteRequest.json +++ b/clients/src/main/resources/common/message/OffsetDeleteRequest.json @@ -16,7 +16,7 @@ { "apiKey": 47, "type": "request", - "scope": ["zkBroker", "broker"], + "listeners": ["zkBroker", "broker"], "name": "OffsetDeleteRequest", "validVersions": "0", "fields": [ diff --git a/clients/src/main/resources/common/message/OffsetFetchRequest.json b/clients/src/main/resources/common/message/OffsetFetchRequest.json index d4fd9abaf020f..d4a4d5f22df94 100644 --- a/clients/src/main/resources/common/message/OffsetFetchRequest.json +++ b/clients/src/main/resources/common/message/OffsetFetchRequest.json @@ -16,7 +16,7 @@ { "apiKey": 9, "type": "request", - "scope": ["zkBroker", "broker"], + "listeners": ["zkBroker", "broker"], "name": "OffsetFetchRequest", // In version 0, the request read offsets from ZK. // diff --git a/clients/src/main/resources/common/message/OffsetForLeaderEpochRequest.json b/clients/src/main/resources/common/message/OffsetForLeaderEpochRequest.json index 98ba94f771d39..2440becd9cb05 100644 --- a/clients/src/main/resources/common/message/OffsetForLeaderEpochRequest.json +++ b/clients/src/main/resources/common/message/OffsetForLeaderEpochRequest.json @@ -16,7 +16,7 @@ { "apiKey": 23, "type": "request", - "scope": ["zkBroker", "broker"], + "listeners": ["zkBroker", "broker"], "name": "OffsetForLeaderEpochRequest", // Version 1 is the same as version 0. // diff --git a/clients/src/main/resources/common/message/ProduceRequest.json b/clients/src/main/resources/common/message/ProduceRequest.json index cec6cee021142..121cd42f44df8 100644 --- a/clients/src/main/resources/common/message/ProduceRequest.json +++ b/clients/src/main/resources/common/message/ProduceRequest.json @@ -16,7 +16,7 @@ { "apiKey": 0, "type": "request", - "scope": ["zkBroker", "broker"], + "listeners": ["zkBroker", "broker"], "name": "ProduceRequest", // Version 1 and 2 are the same as version 0. // diff --git a/clients/src/main/resources/common/message/RenewDelegationTokenRequest.json b/clients/src/main/resources/common/message/RenewDelegationTokenRequest.json index bff33ae779237..182682e4c913a 100644 --- a/clients/src/main/resources/common/message/RenewDelegationTokenRequest.json +++ b/clients/src/main/resources/common/message/RenewDelegationTokenRequest.json @@ -16,7 +16,7 @@ { "apiKey": 39, "type": "request", - "scope": ["zkBroker", "broker", "controller"], + "listeners": ["zkBroker", "broker", "controller"], "name": "RenewDelegationTokenRequest", // Version 1 is the same as version 0. // Version 2 adds flexible version support diff --git a/clients/src/main/resources/common/message/SaslAuthenticateRequest.json b/clients/src/main/resources/common/message/SaslAuthenticateRequest.json index 8a666fe1417b9..3f5558b812042 100644 --- a/clients/src/main/resources/common/message/SaslAuthenticateRequest.json +++ b/clients/src/main/resources/common/message/SaslAuthenticateRequest.json @@ -16,7 +16,7 @@ { "apiKey": 36, "type": "request", - "scope": ["zkBroker", "broker", "controller"], + "listeners": ["zkBroker", "broker", "controller"], "name": "SaslAuthenticateRequest", // Version 1 is the same as version 0. // Version 2 adds flexible version support diff --git a/clients/src/main/resources/common/message/SaslHandshakeRequest.json b/clients/src/main/resources/common/message/SaslHandshakeRequest.json index f9bf2ebf09e30..3384db862b52e 100644 --- a/clients/src/main/resources/common/message/SaslHandshakeRequest.json +++ b/clients/src/main/resources/common/message/SaslHandshakeRequest.json @@ -16,7 +16,7 @@ { "apiKey": 17, "type": "request", - "scope": ["zkBroker", "broker", "controller"], + "listeners": ["zkBroker", "broker", "controller"], "name": "SaslHandshakeRequest", // Version 1 supports SASL_AUTHENTICATE. // NOTE: Version cannot be easily bumped due to incorrect diff --git a/clients/src/main/resources/common/message/StopReplicaRequest.json b/clients/src/main/resources/common/message/StopReplicaRequest.json index 28cf0122f5c7f..b10154fe1fbf6 100644 --- a/clients/src/main/resources/common/message/StopReplicaRequest.json +++ b/clients/src/main/resources/common/message/StopReplicaRequest.json @@ -16,7 +16,7 @@ { "apiKey": 5, "type": "request", - "scope": ["zkBroker"], + "listeners": ["zkBroker"], "name": "StopReplicaRequest", // Version 1 adds the broker epoch and reorganizes the partitions to be stored // per topic. diff --git a/clients/src/main/resources/common/message/SyncGroupRequest.json b/clients/src/main/resources/common/message/SyncGroupRequest.json index 2fa507f1cf662..5525844138366 100644 --- a/clients/src/main/resources/common/message/SyncGroupRequest.json +++ b/clients/src/main/resources/common/message/SyncGroupRequest.json @@ -16,7 +16,7 @@ { "apiKey": 14, "type": "request", - "scope": ["zkBroker", "broker"], + "listeners": ["zkBroker", "broker"], "name": "SyncGroupRequest", // Versions 1 and 2 are the same as version 0. // diff --git a/clients/src/main/resources/common/message/TxnOffsetCommitRequest.json b/clients/src/main/resources/common/message/TxnOffsetCommitRequest.json index a87865c666bc7..a832ef7a96832 100644 --- a/clients/src/main/resources/common/message/TxnOffsetCommitRequest.json +++ b/clients/src/main/resources/common/message/TxnOffsetCommitRequest.json @@ -16,7 +16,7 @@ { "apiKey": 28, "type": "request", - "scope": ["zkBroker", "broker"], + "listeners": ["zkBroker", "broker"], "name": "TxnOffsetCommitRequest", // Version 1 is the same as version 0. // diff --git a/clients/src/main/resources/common/message/UnregisterBrokerRequest.json b/clients/src/main/resources/common/message/UnregisterBrokerRequest.json index 35d9c9ffa7f21..ef72bfe61feb7 100644 --- a/clients/src/main/resources/common/message/UnregisterBrokerRequest.json +++ b/clients/src/main/resources/common/message/UnregisterBrokerRequest.json @@ -16,7 +16,7 @@ { "apiKey": 64, "type": "request", - "scope": ["broker", "controller"], + "listeners": ["broker", "controller"], "name": "UnregisterBrokerRequest", "validVersions": "0", "flexibleVersions": "0+", diff --git a/clients/src/main/resources/common/message/UpdateFeaturesRequest.json b/clients/src/main/resources/common/message/UpdateFeaturesRequest.json index faa28677ee3b1..21e1bf663dda5 100644 --- a/clients/src/main/resources/common/message/UpdateFeaturesRequest.json +++ b/clients/src/main/resources/common/message/UpdateFeaturesRequest.json @@ -16,7 +16,7 @@ { "apiKey": 57, "type": "request", - "scope": ["zkBroker", "broker", "controller"], + "listeners": ["zkBroker", "broker", "controller"], "name": "UpdateFeaturesRequest", "validVersions": "0", "flexibleVersions": "0+", diff --git a/clients/src/main/resources/common/message/UpdateMetadataRequest.json b/clients/src/main/resources/common/message/UpdateMetadataRequest.json index 19de653686fdf..5f397a92c04a5 100644 --- a/clients/src/main/resources/common/message/UpdateMetadataRequest.json +++ b/clients/src/main/resources/common/message/UpdateMetadataRequest.json @@ -16,7 +16,7 @@ { "apiKey": 6, "type": "request", - "scope": ["zkBroker"], + "listeners": ["zkBroker"], "name": "UpdateMetadataRequest", // Version 1 allows specifying multiple endpoints for each broker. // diff --git a/clients/src/main/resources/common/message/VoteRequest.json b/clients/src/main/resources/common/message/VoteRequest.json index e7395ce33728c..f6a60290c073e 100644 --- a/clients/src/main/resources/common/message/VoteRequest.json +++ b/clients/src/main/resources/common/message/VoteRequest.json @@ -16,7 +16,7 @@ { "apiKey": 52, "type": "request", - "scope": ["controller", "raft"], + "listeners": ["controller", "raft"], "name": "VoteRequest", "validVersions": "0", "flexibleVersions": "0+", diff --git a/clients/src/main/resources/common/message/WriteTxnMarkersRequest.json b/clients/src/main/resources/common/message/WriteTxnMarkersRequest.json index cbeea7b731937..9e29fb39f4525 100644 --- a/clients/src/main/resources/common/message/WriteTxnMarkersRequest.json +++ b/clients/src/main/resources/common/message/WriteTxnMarkersRequest.json @@ -16,7 +16,7 @@ { "apiKey": 27, "type": "request", - "scope": ["zkBroker", "broker"], + "listeners": ["zkBroker", "broker"], "name": "WriteTxnMarkersRequest", // Version 1 enables flexible versions. "validVersions": "0-1", diff --git a/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java b/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java index f9f0cdaf7ec22..b13f85422b6db 100644 --- a/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/NetworkClientTest.java @@ -249,7 +249,7 @@ private void setExpectedApiVersionsResponse(ApiVersionsResponse response) { private void awaitReady(NetworkClient client, Node node) { if (client.discoverBrokerVersions()) { setExpectedApiVersionsResponse(ApiVersionsResponse.defaultApiVersionsResponse( - ApiMessageType.ApiScope.ZK_BROKER)); + ApiMessageType.ListenerType.ZK_BROKER)); } while (!client.ready(node, time.milliseconds())) client.poll(1, time.milliseconds()); @@ -1079,7 +1079,7 @@ private void awaitInFlightApiVersionRequest() throws Exception { } private ApiVersionsResponse defaultApiVersionsResponse() { - return ApiVersionsResponse.defaultApiVersionsResponse(ApiMessageType.ApiScope.ZK_BROKER); + return ApiVersionsResponse.defaultApiVersionsResponse(ApiMessageType.ListenerType.ZK_BROKER); } private static class TestCallbackHandler implements RequestCompletionHandler { diff --git a/clients/src/test/java/org/apache/kafka/clients/NodeApiVersionsTest.java b/clients/src/test/java/org/apache/kafka/clients/NodeApiVersionsTest.java index 21e7404a63c2b..b04d83b47df2f 100644 --- a/clients/src/test/java/org/apache/kafka/clients/NodeApiVersionsTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/NodeApiVersionsTest.java @@ -137,8 +137,8 @@ public void testLatestUsableVersionOutOfRange() { } @ParameterizedTest - @EnumSource(ApiMessageType.ApiScope.class) - public void testUsableVersionLatestVersions(ApiMessageType.ApiScope scope) { + @EnumSource(ApiMessageType.ListenerType.class) + public void testUsableVersionLatestVersions(ApiMessageType.ListenerType scope) { ApiVersionsResponse defaultResponse = ApiVersionsResponse.defaultApiVersionsResponse(scope); List versionList = new LinkedList<>(defaultResponse.data().apiKeys()); // Add an API key that we don't know about. @@ -147,14 +147,14 @@ public void testUsableVersionLatestVersions(ApiMessageType.ApiScope scope) { .setMinVersion((short) 0) .setMaxVersion((short) 1)); NodeApiVersions versions = new NodeApiVersions(versionList); - for (ApiKeys apiKey: ApiKeys.apisInScope(scope)) { + for (ApiKeys apiKey: ApiKeys.apisForListener(scope)) { assertEquals(apiKey.latestVersion(), versions.latestUsableVersion(apiKey)); } } @ParameterizedTest - @EnumSource(ApiMessageType.ApiScope.class) - public void testConstructionFromApiVersionsResponse(ApiMessageType.ApiScope scope) { + @EnumSource(ApiMessageType.ListenerType.class) + public void testConstructionFromApiVersionsResponse(ApiMessageType.ListenerType scope) { ApiVersionsResponse apiVersionsResponse = ApiVersionsResponse.defaultApiVersionsResponse(scope); NodeApiVersions versions = new NodeApiVersions(apiVersionsResponse.data().apiKeys()); diff --git a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java index 5002b71767229..f7107c886b160 100644 --- a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java @@ -545,7 +545,7 @@ private static ApiVersionsResponse prepareApiVersionsResponseForDescribeFeatures if (error == Errors.NONE) { return ApiVersionsResponse.createApiVersionsResponse( 0, - ApiVersionsResponse.filterApis(RecordVersion.current(), ApiMessageType.ApiScope.ZK_BROKER), + ApiVersionsResponse.filterApis(RecordVersion.current(), ApiMessageType.ListenerType.ZK_BROKER), convertSupportedFeaturesMap(defaultFeatureMetadata().supportedFeatures()), convertFinalizedFeaturesMap(defaultFeatureMetadata().finalizedFeatures()), defaultFeatureMetadata().finalizedFeaturesEpoch().get() diff --git a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java index dd785dc744278..2c13864fb0589 100644 --- a/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/consumer/internals/FetcherTest.java @@ -2076,7 +2076,7 @@ public void testQuotaMetrics() { time, true, new ApiVersions(), throttleTimeSensor, new LogContext()); ApiVersionsResponse apiVersionsResponse = ApiVersionsResponse.defaultApiVersionsResponse( - 400, ApiMessageType.ApiScope.ZK_BROKER); + 400, ApiMessageType.ListenerType.ZK_BROKER); ByteBuffer buffer = RequestTestUtils.serializeResponseWithHeader(apiVersionsResponse, ApiKeys.API_VERSIONS.latestVersion(), 0); selector.delayedReceive(new DelayedReceive(node.idString(), new NetworkReceive(node.idString(), buffer))); diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java index aff724ac6b4fc..e118c1110e704 100644 --- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/SenderTest.java @@ -289,7 +289,7 @@ public void testQuotaMetrics() { time, true, new ApiVersions(), throttleTimeSensor, logContext); ApiVersionsResponse apiVersionsResponse = ApiVersionsResponse.defaultApiVersionsResponse( - 400, ApiMessageType.ApiScope.ZK_BROKER); + 400, ApiMessageType.ListenerType.ZK_BROKER); ByteBuffer buffer = RequestTestUtils.serializeResponseWithHeader(apiVersionsResponse, ApiKeys.API_VERSIONS.latestVersion(), 0); selector.delayedReceive(new DelayedReceive(node.idString(), new NetworkReceive(node.idString(), buffer))); diff --git a/clients/src/test/java/org/apache/kafka/common/network/NioEchoServer.java b/clients/src/test/java/org/apache/kafka/common/network/NioEchoServer.java index 5be00acc8d3a3..53b46e7d9b650 100644 --- a/clients/src/test/java/org/apache/kafka/common/network/NioEchoServer.java +++ b/clients/src/test/java/org/apache/kafka/common/network/NioEchoServer.java @@ -120,7 +120,7 @@ public NioEchoServer(ListenerName listenerName, SecurityProtocol securityProtoco if (channelBuilder == null) channelBuilder = ChannelBuilders.serverChannelBuilder(listenerName, false, securityProtocol, config, credentialCache, tokenCache, time, logContext, - () -> ApiVersionsResponse.defaultApiVersionsResponse(ApiMessageType.ApiScope.ZK_BROKER)); + () -> ApiVersionsResponse.defaultApiVersionsResponse(ApiMessageType.ListenerType.ZK_BROKER)); this.metrics = new Metrics(); this.selector = new Selector(10000, failedAuthenticationDelayMs, metrics, time, "MetricGroup", channelBuilder, logContext); diff --git a/clients/src/test/java/org/apache/kafka/common/network/SaslChannelBuilderTest.java b/clients/src/test/java/org/apache/kafka/common/network/SaslChannelBuilderTest.java index c6f6d76dedb5c..1697c627fc499 100644 --- a/clients/src/test/java/org/apache/kafka/common/network/SaslChannelBuilderTest.java +++ b/clients/src/test/java/org/apache/kafka/common/network/SaslChannelBuilderTest.java @@ -177,7 +177,7 @@ protected GSSManager gssManager() { } private Supplier defaultApiVersionsSupplier() { - return () -> ApiVersionsResponse.defaultApiVersionsResponse(ApiMessageType.ApiScope.ZK_BROKER); + return () -> ApiVersionsResponse.defaultApiVersionsResponse(ApiMessageType.ListenerType.ZK_BROKER); } private SaslChannelBuilder createChannelBuilder(SecurityProtocol securityProtocol, String saslMechanism) { diff --git a/clients/src/test/java/org/apache/kafka/common/network/SslTransportLayerTest.java b/clients/src/test/java/org/apache/kafka/common/network/SslTransportLayerTest.java index 1731f72252e3f..13f763279f783 100644 --- a/clients/src/test/java/org/apache/kafka/common/network/SslTransportLayerTest.java +++ b/clients/src/test/java/org/apache/kafka/common/network/SslTransportLayerTest.java @@ -1343,7 +1343,7 @@ private interface FailureAction { } private Supplier defaultApiVersionsSupplier() { - return () -> ApiVersionsResponse.defaultApiVersionsResponse(ApiMessageType.ApiScope.ZK_BROKER); + return () -> ApiVersionsResponse.defaultApiVersionsResponse(ApiMessageType.ListenerType.ZK_BROKER); } static class TestSslChannelBuilder extends SslChannelBuilder { diff --git a/clients/src/test/java/org/apache/kafka/common/protocol/ApiKeysTest.java b/clients/src/test/java/org/apache/kafka/common/protocol/ApiKeysTest.java index 1f8e0a6ce42f4..17d2e1ce26e43 100644 --- a/clients/src/test/java/org/apache/kafka/common/protocol/ApiKeysTest.java +++ b/clients/src/test/java/org/apache/kafka/common/protocol/ApiKeysTest.java @@ -78,7 +78,7 @@ public void testResponseThrottleTime() { public void testApiScope() { Set apisMissingScope = new HashSet<>(); for (ApiKeys apiKey : ApiKeys.values()) { - if (apiKey.messageType.scope().isEmpty()) { + if (apiKey.messageType.listeners().isEmpty()) { apisMissingScope.add(apiKey); } } diff --git a/clients/src/test/java/org/apache/kafka/common/requests/ApiVersionsResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/ApiVersionsResponseTest.java index 5ca11f86413d5..2c9b1e8fad023 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/ApiVersionsResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/ApiVersionsResponseTest.java @@ -39,13 +39,13 @@ public class ApiVersionsResponseTest { @ParameterizedTest - @EnumSource(ApiMessageType.ApiScope.class) - public void shouldHaveCorrectDefaultApiVersionsResponse(ApiMessageType.ApiScope scope) { + @EnumSource(ApiMessageType.ListenerType.class) + public void shouldHaveCorrectDefaultApiVersionsResponse(ApiMessageType.ListenerType scope) { ApiVersionsResponse defaultResponse = ApiVersionsResponse.defaultApiVersionsResponse(scope); - assertEquals(ApiKeys.apisInScope(scope).size(), defaultResponse.data().apiKeys().size(), + assertEquals(ApiKeys.apisForListener(scope).size(), defaultResponse.data().apiKeys().size(), "API versions for all API keys must be maintained."); - for (ApiKeys key : ApiKeys.apisInScope(scope)) { + for (ApiKeys key : ApiKeys.apisForListener(scope)) { ApiVersion version = defaultResponse.apiVersion(key.id); assertNotNull(version, "Could not find ApiVersion for API " + key.name); assertEquals(version.minVersion(), key.oldestVersion(), "Incorrect min version for Api " + key.name); @@ -91,7 +91,7 @@ public void shouldHaveCommonlyAgreedApiVersionResponseWithControllerOnForwardabl ); ApiVersionCollection commonResponse = ApiVersionsResponse.intersectForwardableApis( - ApiMessageType.ApiScope.ZK_BROKER, + ApiMessageType.ListenerType.ZK_BROKER, RecordVersion.current(), activeControllerApiVersions ); diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java index b737be9d1828f..4c70771cca97c 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java @@ -518,7 +518,7 @@ public void testApiVersionsSerialization() { checkErrorResponse(createApiVersionRequest(v), unknownServerException, true); checkErrorResponse(createApiVersionRequest(v), new UnsupportedVersionException("Not Supported"), true); checkResponse(createApiVersionResponse(), v, true); - checkResponse(ApiVersionsResponse.defaultApiVersionsResponse(ApiMessageType.ApiScope.ZK_BROKER), v, true); + checkResponse(ApiVersionsResponse.defaultApiVersionsResponse(ApiMessageType.ListenerType.ZK_BROKER), v, true); } } @@ -1038,7 +1038,7 @@ public void testApiVersionResponseWithNotUnsupportedError() { } private ApiVersionsResponse defaultApiVersionsResponse() { - return ApiVersionsResponse.defaultApiVersionsResponse(ApiMessageType.ApiScope.ZK_BROKER); + return ApiVersionsResponse.defaultApiVersionsResponse(ApiMessageType.ListenerType.ZK_BROKER); } @Test diff --git a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java index 3998ab7929555..6c836f33c57bc 100644 --- a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java +++ b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java @@ -1935,7 +1935,7 @@ private NioEchoServer startServerWithoutSaslAuthenticateHeader(final SecurityPro Supplier apiVersionSupplier = () -> { ApiVersionsResponse defaultApiVersionResponse = ApiVersionsResponse.defaultApiVersionsResponse( - ApiMessageType.ApiScope.ZK_BROKER); + ApiMessageType.ListenerType.ZK_BROKER); ApiVersionCollection apiVersions = new ApiVersionCollection(); for (ApiVersion apiVersion : defaultApiVersionResponse.data().apiKeys()) { if (apiVersion.apiKey() != ApiKeys.SASL_AUTHENTICATE.id) { @@ -2532,7 +2532,7 @@ public AlternateSaslChannelBuilder(Mode mode, Map jaasConte DelegationTokenCache tokenCache, Time time) { super(mode, jaasContexts, securityProtocol, listenerName, isInterBrokerListener, clientSaslMechanism, handshakeRequestEnable, credentialCache, tokenCache, null, time, new LogContext(), - () -> ApiVersionsResponse.defaultApiVersionsResponse(ApiMessageType.ApiScope.ZK_BROKER)); + () -> ApiVersionsResponse.defaultApiVersionsResponse(ApiMessageType.ListenerType.ZK_BROKER)); } @Override diff --git a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticatorTest.java b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticatorTest.java index d493eef2fc54d..af0fedd4f5ad9 100644 --- a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticatorTest.java +++ b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticatorTest.java @@ -161,7 +161,7 @@ private SaslServerAuthenticator setupAuthenticator(Map configs, Trans Map callbackHandlers = Collections.singletonMap( mechanism, new SaslServerCallbackHandler()); ApiVersionsResponse apiVersionsResponse = ApiVersionsResponse.defaultApiVersionsResponse( - ApiMessageType.ApiScope.ZK_BROKER); + ApiMessageType.ListenerType.ZK_BROKER); return new SaslServerAuthenticator(configs, callbackHandlers, "node", subjects, null, new ListenerName("ssl"), SecurityProtocol.SASL_SSL, transportLayer, Collections.emptyMap(), metadataRegistry, Time.SYSTEM, () -> apiVersionsResponse); diff --git a/core/src/main/scala/kafka/api/ApiVersion.scala b/core/src/main/scala/kafka/api/ApiVersion.scala index 9e1bcdadf8c7e..879373787365f 100644 --- a/core/src/main/scala/kafka/api/ApiVersion.scala +++ b/core/src/main/scala/kafka/api/ApiVersion.scala @@ -21,7 +21,7 @@ import org.apache.kafka.clients.NodeApiVersions import org.apache.kafka.common.config.ConfigDef.Validator import org.apache.kafka.common.config.ConfigException import org.apache.kafka.common.feature.{Features, FinalizedVersionRange, SupportedVersionRange} -import org.apache.kafka.common.message.ApiMessageType.ApiScope +import org.apache.kafka.common.message.ApiMessageType.ListenerType import org.apache.kafka.common.record.RecordVersion import org.apache.kafka.common.requests.ApiVersionsResponse @@ -151,7 +151,7 @@ object ApiVersion { minRecordVersion: RecordVersion, latestSupportedFeatures: Features[SupportedVersionRange], controllerApiVersions: Option[NodeApiVersions], - scope: ApiScope + listenerType: ListenerType ): ApiVersionsResponse = { apiVersionsResponse( throttleTimeMs, @@ -160,7 +160,7 @@ object ApiVersion { Features.emptyFinalizedFeatures, ApiVersionsResponse.UNKNOWN_FINALIZED_FEATURES_EPOCH, controllerApiVersions, - scope + listenerType ) } @@ -171,12 +171,12 @@ object ApiVersion { finalizedFeatures: Features[FinalizedVersionRange], finalizedFeaturesEpoch: Long, controllerApiVersions: Option[NodeApiVersions], - scope: ApiScope + listenerType: ListenerType ): ApiVersionsResponse = { val apiKeys = controllerApiVersions match { - case None => ApiVersionsResponse.filterApis(minRecordVersion, scope) + case None => ApiVersionsResponse.filterApis(minRecordVersion, listenerType) case Some(controllerApiVersion) => ApiVersionsResponse.intersectForwardableApis( - scope, minRecordVersion, controllerApiVersion.allSupportedApiVersions()) + listenerType, minRecordVersion, controllerApiVersion.allSupportedApiVersions()) } ApiVersionsResponse.createApiVersionsResponse( diff --git a/core/src/main/scala/kafka/network/RequestChannel.scala b/core/src/main/scala/kafka/network/RequestChannel.scala index db35e75e4b033..48f723f4d3375 100644 --- a/core/src/main/scala/kafka/network/RequestChannel.scala +++ b/core/src/main/scala/kafka/network/RequestChannel.scala @@ -30,7 +30,7 @@ import kafka.utils.{Logging, NotNothing, Pool} import kafka.utils.Implicits._ import org.apache.kafka.common.config.ConfigResource import org.apache.kafka.common.memory.MemoryPool -import org.apache.kafka.common.message.ApiMessageType.ApiScope +import org.apache.kafka.common.message.ApiMessageType.ListenerType import org.apache.kafka.common.message.IncrementalAlterConfigsRequestData import org.apache.kafka.common.message.IncrementalAlterConfigsRequestData._ import org.apache.kafka.common.network.Send @@ -61,8 +61,8 @@ object RequestChannel extends Logging { } class Metrics(enabledApis: Iterable[ApiKeys]) { - def this(scope: ApiScope) = { - this(ApiKeys.apisInScope(scope).asScala) + def this(scope: ListenerType) = { + this(ApiKeys.apisForListener(scope).asScala) } private val metricsMap = mutable.Map[String, RequestMetrics]() diff --git a/core/src/main/scala/kafka/network/SocketServer.scala b/core/src/main/scala/kafka/network/SocketServer.scala index 9220fc06141bd..24df39f6ef2ef 100644 --- a/core/src/main/scala/kafka/network/SocketServer.scala +++ b/core/src/main/scala/kafka/network/SocketServer.scala @@ -85,7 +85,7 @@ class SocketServer(val config: KafkaConfig, private val nodeId = config.brokerId - private val logContext = new LogContext(s"[SocketServer listenerType=${apiVersionManager.apiScope}, nodeId=$nodeId] ") + private val logContext = new LogContext(s"[SocketServer listenerType=${apiVersionManager.listenerType}, nodeId=$nodeId] ") this.logIdent = logContext.logPrefix diff --git a/core/src/main/scala/kafka/server/ApiVersionManager.scala b/core/src/main/scala/kafka/server/ApiVersionManager.scala index 99eb216dde671..ebf3e74e89291 100644 --- a/core/src/main/scala/kafka/server/ApiVersionManager.scala +++ b/core/src/main/scala/kafka/server/ApiVersionManager.scala @@ -19,7 +19,7 @@ package kafka.server import kafka.api.ApiVersion import kafka.network import kafka.network.RequestChannel -import org.apache.kafka.common.message.ApiMessageType.ApiScope +import org.apache.kafka.common.message.ApiMessageType.ListenerType import org.apache.kafka.common.message.ApiVersionsResponseData import org.apache.kafka.common.protocol.ApiKeys import org.apache.kafka.common.requests.ApiVersionsResponse @@ -27,7 +27,7 @@ import org.apache.kafka.common.requests.ApiVersionsResponse import scala.jdk.CollectionConverters._ trait ApiVersionManager { - def apiScope: ApiScope + def listenerType: ListenerType def enabledApis: collection.Set[ApiKeys] def apiVersionResponse(throttleTimeMs: Int): ApiVersionsResponse def isApiEnabled(apiKey: ApiKeys): Boolean = enabledApis.contains(apiKey) @@ -36,14 +36,14 @@ trait ApiVersionManager { object ApiVersionManager { def apply( - apiScope: ApiScope, + listenerType: ListenerType, config: KafkaConfig, forwardingManager: Option[ForwardingManager], features: BrokerFeatures, featureCache: FinalizedFeatureCache ): ApiVersionManager = { new DefaultApiVersionManager( - apiScope, + listenerType, config.interBrokerProtocolVersion, forwardingManager, features, @@ -53,11 +53,12 @@ object ApiVersionManager { } class SimpleApiVersionManager( + val listenerType: ListenerType, val enabledApis: collection.Set[ApiKeys] ) extends ApiVersionManager { - def this(scope: ApiScope) = { - this(ApiKeys.apisInScope(scope).asScala) + def this(listenerType: ListenerType) = { + this(listenerType, ApiKeys.apisForListener(listenerType).asScala) } private val apiVersions = ApiVersionsResponse.collectApis(enabledApis.asJava) @@ -68,7 +69,7 @@ class SimpleApiVersionManager( } class DefaultApiVersionManager( - val apiScope: ApiScope, + val listenerType: ListenerType, interBrokerProtocolVersion: ApiVersion, forwardingManager: Option[ForwardingManager], features: BrokerFeatures, @@ -88,13 +89,13 @@ class DefaultApiVersionManager( finalizedFeatures.features, finalizedFeatures.epoch, controllerApiVersions, - apiScope) + listenerType) case None => ApiVersion.apiVersionsResponse( throttleTimeMs, interBrokerProtocolVersion.recordVersion, supportedFeatures, controllerApiVersions, - apiScope) + listenerType) } // This is a temporary workaround in order to allow testing of forwarding @@ -114,12 +115,12 @@ class DefaultApiVersionManager( override def enabledApis: collection.Set[ApiKeys] = { forwardingManager match { - case Some(_) => ApiKeys.apisInScope(apiScope).asScala ++ Set(ApiKeys.ENVELOPE) - case None => ApiKeys.apisInScope(apiScope).asScala + case Some(_) => ApiKeys.apisForListener(listenerType).asScala ++ Set(ApiKeys.ENVELOPE) + case None => ApiKeys.apisForListener(listenerType).asScala } } override def isApiEnabled(apiKey: ApiKeys): Boolean = { - apiKey.inScope(apiScope) || (apiKey == ApiKeys.ENVELOPE && forwardingManager.isDefined) + apiKey.inScope(listenerType) || (apiKey == ApiKeys.ENVELOPE && forwardingManager.isDefined) } } diff --git a/core/src/main/scala/kafka/server/KafkaServer.scala b/core/src/main/scala/kafka/server/KafkaServer.scala index 6885012251daa..4daee0866d8e9 100755 --- a/core/src/main/scala/kafka/server/KafkaServer.scala +++ b/core/src/main/scala/kafka/server/KafkaServer.scala @@ -37,7 +37,7 @@ import kafka.utils._ import kafka.zk.{AdminZkClient, BrokerInfo, KafkaZkClient} import org.apache.kafka.clients.{ApiVersions, ClientDnsLookup, ManualMetadataUpdater, NetworkClient, NetworkClientUtils} import org.apache.kafka.common.internals.Topic -import org.apache.kafka.common.message.ApiMessageType.ApiScope +import org.apache.kafka.common.message.ApiMessageType.ListenerType import org.apache.kafka.common.message.ControlledShutdownRequestData import org.apache.kafka.common.metrics.Metrics import org.apache.kafka.common.network._ @@ -268,7 +268,7 @@ class KafkaServer( } val apiVersionManager = ApiVersionManager( - ApiScope.ZK_BROKER, + ListenerType.ZK_BROKER, config, forwardingManager, brokerFeatures, diff --git a/core/src/main/scala/kafka/tools/TestRaftServer.scala b/core/src/main/scala/kafka/tools/TestRaftServer.scala index 7dee9537c9084..02806762899e4 100644 --- a/core/src/main/scala/kafka/tools/TestRaftServer.scala +++ b/core/src/main/scala/kafka/tools/TestRaftServer.scala @@ -27,7 +27,7 @@ import kafka.security.CredentialProvider import kafka.server.{KafkaConfig, KafkaRequestHandlerPool, MetaProperties, SimpleApiVersionManager} import kafka.utils.{CommandDefaultOptions, CommandLineUtils, CoreUtils, Exit, Logging, ShutdownableThread} import org.apache.kafka.common.errors.InvalidConfigurationException -import org.apache.kafka.common.message.ApiMessageType.ApiScope +import org.apache.kafka.common.message.ApiMessageType.ListenerType import org.apache.kafka.common.metrics.Metrics import org.apache.kafka.common.metrics.stats.Percentiles.BucketSizing import org.apache.kafka.common.metrics.stats.{Meter, Percentile, Percentiles} @@ -69,7 +69,7 @@ class TestRaftServer( tokenCache = new DelegationTokenCache(ScramMechanism.mechanismNames) credentialProvider = new CredentialProvider(ScramMechanism.mechanismNames, tokenCache) - val apiVersionManager = new SimpleApiVersionManager(ApiScope.RAFT) + val apiVersionManager = new SimpleApiVersionManager(ListenerType.RAFT) socketServer = new SocketServer(config, metrics, time, credentialProvider, apiVersionManager) socketServer.startup(startProcessingRequests = false) diff --git a/core/src/test/scala/integration/kafka/server/GssapiAuthenticationTest.scala b/core/src/test/scala/integration/kafka/server/GssapiAuthenticationTest.scala index 3bcea3f29e900..90454bbb3814d 100644 --- a/core/src/test/scala/integration/kafka/server/GssapiAuthenticationTest.scala +++ b/core/src/test/scala/integration/kafka/server/GssapiAuthenticationTest.scala @@ -29,7 +29,7 @@ import org.apache.kafka.clients.CommonClientConfigs import org.apache.kafka.common.TopicPartition import org.apache.kafka.common.config.SaslConfigs import org.apache.kafka.common.errors.SaslAuthenticationException -import org.apache.kafka.common.message.ApiMessageType.ApiScope +import org.apache.kafka.common.message.ApiMessageType.ListenerType import org.apache.kafka.common.network._ import org.apache.kafka.common.requests.ApiVersionsResponse import org.apache.kafka.common.security.{JaasContext, TestSecurityConfig} @@ -236,7 +236,7 @@ class GssapiAuthenticationTest extends IntegrationTestHarness with SaslSetup { val jaasContexts = Collections.singletonMap("GSSAPI", JaasContext.loadClientContext(config.values())) val channelBuilder = new SaslChannelBuilder(Mode.CLIENT, jaasContexts, securityProtocol, null, false, kafkaClientSaslMechanism, true, null, null, null, time, new LogContext(), - () => ApiVersionsResponse.defaultApiVersionsResponse(ApiScope.ZK_BROKER)) { + () => ApiVersionsResponse.defaultApiVersionsResponse(ListenerType.ZK_BROKER)) { override protected def defaultLoginClass(): Class[_ <: Login] = classOf[TestableKerberosLogin] } channelBuilder.configure(config.values()) diff --git a/core/src/test/scala/unit/kafka/api/ApiVersionTest.scala b/core/src/test/scala/unit/kafka/api/ApiVersionTest.scala index 4cd56acf593bc..6a3eb317c99cf 100644 --- a/core/src/test/scala/unit/kafka/api/ApiVersionTest.scala +++ b/core/src/test/scala/unit/kafka/api/ApiVersionTest.scala @@ -20,7 +20,7 @@ package kafka.api import java.util import org.apache.kafka.common.feature.{Features, FinalizedVersionRange, SupportedVersionRange} -import org.apache.kafka.common.message.ApiMessageType.ApiScope +import org.apache.kafka.common.message.ApiMessageType.ListenerType import org.apache.kafka.common.protocol.ApiKeys import org.apache.kafka.common.record.{RecordBatch, RecordVersion} import org.apache.kafka.common.requests.{AbstractResponse, ApiVersionsResponse} @@ -183,7 +183,7 @@ class ApiVersionTest { RecordVersion.V1, Features.emptySupportedFeatures, None, - ApiScope.ZK_BROKER + ListenerType.ZK_BROKER ) verifyApiKeysForMagic(response, RecordBatch.MAGIC_VALUE_V1) assertEquals(10, response.throttleTimeMs) @@ -203,7 +203,7 @@ class ApiVersionTest { Utils.mkMap(Utils.mkEntry("feature", new FinalizedVersionRange(2.toShort, 3.toShort)))), 10, None, - ApiScope.ZK_BROKER + ListenerType.ZK_BROKER ) verifyApiKeysForMagic(response, RecordBatch.MAGIC_VALUE_V1) @@ -234,7 +234,7 @@ class ApiVersionTest { RecordVersion.current(), Features.emptySupportedFeatures, None, - ApiScope.ZK_BROKER + ListenerType.ZK_BROKER ) assertEquals(new util.HashSet[ApiKeys](ApiKeys.zkBrokerApis), apiKeysInResponse(response)) assertEquals(AbstractResponse.DEFAULT_THROTTLE_TIME, response.throttleTimeMs) @@ -250,7 +250,7 @@ class ApiVersionTest { RecordVersion.current(), Features.emptySupportedFeatures, None, - ApiScope.ZK_BROKER + ListenerType.ZK_BROKER ) // Ensure that APIs needed for the internal metadata quorum (KIP-500) diff --git a/core/src/test/scala/unit/kafka/network/SocketServerTest.scala b/core/src/test/scala/unit/kafka/network/SocketServerTest.scala index 5fc1c15a4f428..293614432cb76 100644 --- a/core/src/test/scala/unit/kafka/network/SocketServerTest.scala +++ b/core/src/test/scala/unit/kafka/network/SocketServerTest.scala @@ -35,7 +35,7 @@ import kafka.server.{KafkaConfig, SimpleApiVersionManager, ThrottledChannel} import kafka.utils.Implicits._ import kafka.utils.TestUtils import org.apache.kafka.common.memory.MemoryPool -import org.apache.kafka.common.message.ApiMessageType.ApiScope +import org.apache.kafka.common.message.ApiMessageType.ListenerType import org.apache.kafka.common.message.{ProduceRequestData, SaslAuthenticateRequestData, SaslHandshakeRequestData, VoteRequestData} import org.apache.kafka.common.metrics.Metrics import org.apache.kafka.common.network.KafkaChannel.ChannelMuteState @@ -74,7 +74,7 @@ class SocketServerTest { // Clean-up any metrics left around by previous tests TestUtils.clearYammerMetrics() - private val apiVersionManager = new SimpleApiVersionManager(ApiScope.ZK_BROKER) + private val apiVersionManager = new SimpleApiVersionManager(ListenerType.ZK_BROKER) val server = new SocketServer(config, metrics, Time.SYSTEM, credentialProvider, apiVersionManager) server.startup() val sockets = new ArrayBuffer[Socket] diff --git a/core/src/test/scala/unit/kafka/server/AbstractApiVersionsRequestTest.scala b/core/src/test/scala/unit/kafka/server/AbstractApiVersionsRequestTest.scala index 8dbdf0789c888..9c7b4440a32b8 100644 --- a/core/src/test/scala/unit/kafka/server/AbstractApiVersionsRequestTest.scala +++ b/core/src/test/scala/unit/kafka/server/AbstractApiVersionsRequestTest.scala @@ -20,7 +20,7 @@ import java.util.Properties import integration.kafka.server.IntegrationTestUtils import kafka.test.ClusterInstance -import org.apache.kafka.common.message.ApiMessageType.ApiScope +import org.apache.kafka.common.message.ApiMessageType.ListenerType import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersion import org.apache.kafka.common.network.ListenerName import org.apache.kafka.common.protocol.ApiKeys @@ -60,7 +60,7 @@ abstract class AbstractApiVersionsRequestTest(cluster: ClusterInstance) { assertEquals(expectedApis.size(), apiVersionsResponse.data.apiKeys().size(), "API keys in ApiVersionsResponse must match API keys supported by broker.") - val defaultApiVersionsResponse = ApiVersionsResponse.defaultApiVersionsResponse(ApiScope.ZK_BROKER) + val defaultApiVersionsResponse = ApiVersionsResponse.defaultApiVersionsResponse(ListenerType.ZK_BROKER) for (expectedApiVersion: ApiVersion <- defaultApiVersionsResponse.data.apiKeys().asScala) { val actualApiVersion = apiVersionsResponse.apiVersion(expectedApiVersion.apiKey) assertNotNull(actualApiVersion, s"API key ${actualApiVersion.apiKey} is supported by broker, but not received in ApiVersionsResponse.") diff --git a/core/src/test/scala/unit/kafka/server/ApiVersionManagerTest.scala b/core/src/test/scala/unit/kafka/server/ApiVersionManagerTest.scala index a39c3af1825d8..a93cc90e3e2d6 100644 --- a/core/src/test/scala/unit/kafka/server/ApiVersionManagerTest.scala +++ b/core/src/test/scala/unit/kafka/server/ApiVersionManagerTest.scala @@ -18,7 +18,7 @@ package kafka.server import kafka.api.ApiVersion import org.apache.kafka.clients.NodeApiVersions -import org.apache.kafka.common.message.ApiMessageType.ApiScope +import org.apache.kafka.common.message.ApiMessageType.ListenerType import org.apache.kafka.common.protocol.ApiKeys import org.junit.jupiter.api.Test import org.junit.jupiter.api.Assertions._ @@ -33,17 +33,17 @@ class ApiVersionManagerTest { private val featureCache = new FinalizedFeatureCache(brokerFeatures) @ParameterizedTest - @EnumSource(classOf[ApiScope]) - def testApiScope(apiScope: ApiScope): Unit = { + @EnumSource(classOf[ListenerType]) + def testApiScope(apiScope: ListenerType): Unit = { val versionManager = new DefaultApiVersionManager( - apiScope = apiScope, + listenerType = apiScope, interBrokerProtocolVersion = ApiVersion.latestVersion, forwardingManager = None, features = brokerFeatures, featureCache = featureCache ) - assertEquals(ApiKeys.apisInScope(apiScope).asScala, versionManager.enabledApis) - assertTrue(ApiKeys.apisInScope(apiScope).asScala.forall(versionManager.isApiEnabled)) + assertEquals(ApiKeys.apisForListener(apiScope).asScala, versionManager.enabledApis) + assertTrue(ApiKeys.apisForListener(apiScope).asScala.forall(versionManager.isApiEnabled)) } @Test @@ -60,7 +60,7 @@ class ApiVersionManagerTest { ))) val versionManager = new DefaultApiVersionManager( - apiScope = ApiScope.ZK_BROKER, + listenerType = ListenerType.ZK_BROKER, interBrokerProtocolVersion = ApiVersion.latestVersion, forwardingManager = Some(forwardingManager), features = brokerFeatures, @@ -80,7 +80,7 @@ class ApiVersionManagerTest { Mockito.when(forwardingManager.controllerApiVersions).thenReturn(None) val versionManager = new DefaultApiVersionManager( - apiScope = ApiScope.ZK_BROKER, + listenerType = ListenerType.ZK_BROKER, interBrokerProtocolVersion = ApiVersion.latestVersion, forwardingManager = Some(forwardingManager), features = brokerFeatures, @@ -99,7 +99,7 @@ class ApiVersionManagerTest { @Test def testEnvelopeDisabledWhenForwardingManagerEmpty(): Unit = { val versionManager = new DefaultApiVersionManager( - apiScope = ApiScope.ZK_BROKER, + listenerType = ListenerType.ZK_BROKER, interBrokerProtocolVersion = ApiVersion.latestVersion, forwardingManager = None, features = brokerFeatures, diff --git a/core/src/test/scala/unit/kafka/server/ForwardingManagerTest.scala b/core/src/test/scala/unit/kafka/server/ForwardingManagerTest.scala index 7e9407629f361..9240af6a10857 100644 --- a/core/src/test/scala/unit/kafka/server/ForwardingManagerTest.scala +++ b/core/src/test/scala/unit/kafka/server/ForwardingManagerTest.scala @@ -29,7 +29,7 @@ import org.apache.kafka.clients.MockClient.RequestMatcher import org.apache.kafka.common.Node import org.apache.kafka.common.config.{ConfigResource, TopicConfig} import org.apache.kafka.common.memory.MemoryPool -import org.apache.kafka.common.message.ApiMessageType.ApiScope +import org.apache.kafka.common.message.ApiMessageType.ListenerType import org.apache.kafka.common.message.{AlterConfigsResponseData, ApiVersionsResponseData} import org.apache.kafka.common.network.{ClientInformation, ListenerName} import org.apache.kafka.common.protocol.{ApiKeys, Errors} @@ -196,7 +196,7 @@ class ForwardingManagerTest { startTimeNanos = time.nanoseconds(), memoryPool = MemoryPool.NONE, buffer = requestBuffer, - metrics = new RequestChannel.Metrics(ApiScope.CONTROLLER), + metrics = new RequestChannel.Metrics(ListenerType.CONTROLLER), envelope = None ) } diff --git a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala index 704998f8123f7..18203d4c306cf 100644 --- a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala +++ b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala @@ -34,7 +34,11 @@ import kafka.log.AppendOrigin import kafka.network.RequestChannel import kafka.network.RequestChannel.{CloseConnectionResponse, SendResponse} import kafka.server.QuotaFactory.QuotaManagers +<<<<<<< HEAD import kafka.server.metadata.{CachedConfigRepository, ConfigRepository, RaftMetadataCache} +======= +import kafka.server.metadata.{CachedConfigRepository, ConfigRepository} +>>>>>>> Rename `scope` to `listeners` import kafka.utils.{MockTime, TestUtils} import kafka.zk.KafkaZkClient import org.apache.kafka.clients.admin.AlterConfigOp.OpType @@ -44,7 +48,7 @@ import org.apache.kafka.common.config.ConfigResource import org.apache.kafka.common.errors.UnsupportedVersionException import org.apache.kafka.common.internals.{KafkaFutureImpl, Topic} import org.apache.kafka.common.memory.MemoryPool -import org.apache.kafka.common.message.ApiMessageType.ApiScope +import org.apache.kafka.common.message.ApiMessageType.ListenerType import org.apache.kafka.common.message.CreateTopicsRequestData.{CreatableTopic, CreatableTopicCollection} import org.apache.kafka.common.message.DescribeConfigsResponseData.DescribeConfigsResult import org.apache.kafka.common.message.JoinGroupRequestData.JoinGroupRequestProtocol @@ -163,9 +167,9 @@ class KafkaApisTest { } val enabledApis = if (enableForwarding) { - ApiKeys.apisInScope(ApiScope.ZK_BROKER).asScala ++ Set(ApiKeys.ENVELOPE) + ApiKeys.apisForListener(ListenerType.ZK_BROKER).asScala ++ Set(ApiKeys.ENVELOPE) } else { - ApiKeys.apisInScope(ApiScope.ZK_BROKER).asScala.toSet + ApiKeys.apisForListener(ListenerType.ZK_BROKER).asScala.toSet } val apiVersionManager = new SimpleApiVersionManager(enabledApis) diff --git a/generator/src/main/java/org/apache/kafka/message/ApiMessageTypeGenerator.java b/generator/src/main/java/org/apache/kafka/message/ApiMessageTypeGenerator.java index 1d43ed5cd8ec8..408e1a75ff2c8 100644 --- a/generator/src/main/java/org/apache/kafka/message/ApiMessageTypeGenerator.java +++ b/generator/src/main/java/org/apache/kafka/message/ApiMessageTypeGenerator.java @@ -35,7 +35,7 @@ public final class ApiMessageTypeGenerator implements TypeClassGenerator { private final HeaderGenerator headerGenerator; private final CodeBuffer buffer; private final TreeMap apis; - private final EnumMap> apisByScope = new EnumMap<>(RequestApiScope.class); + private final EnumMap> apisByListener = new EnumMap<>(RequestListenerType.class); private static final class ApiData { short apiKey; @@ -103,10 +103,10 @@ public void registerMessageType(MessageSpec spec) { } data.requestSpec = spec; - if (spec.scope() != null) { - for (RequestApiScope scope : spec.scope()) { - apisByScope.putIfAbsent(scope, new ArrayList<>()); - apisByScope.get(scope).add(data); + if (spec.listeners() != null) { + for (RequestListenerType listener : spec.listeners()) { + apisByListener.putIfAbsent(listener, new ArrayList<>()); + apisByListener.get(listener).add(data); } } break; @@ -156,7 +156,7 @@ private void generate() { buffer.printf("%n"); generateAccessor("highestSupportedVersion", "short"); buffer.printf("%n"); - generateAccessor("scope", "EnumSet"); + generateAccessor("listeners", "EnumSet"); buffer.printf("%n"); generateAccessor("apiKey", "short"); buffer.printf("%n"); @@ -170,21 +170,21 @@ private void generate() { buffer.printf("%n"); generateHeaderVersion("response"); buffer.printf("%n"); - generateApiScopeEnum(); + generateListenerTypesEnum(); buffer.printf("%n"); buffer.decrementIndent(); buffer.printf("}%n"); headerGenerator.generate(); } - private String generateApiScopeEnumSet(Collection values) { + private String generateListenerTypeEnumSet(Collection values) { if (values.isEmpty()) { - return "EnumSet.noneOf(ApiScope.class)"; + return "EnumSet.noneOf(ListenerType.class)"; } StringBuilder bldr = new StringBuilder("EnumSet.of("); Iterator iter = values.iterator(); while (iter.hasNext()) { - bldr.append("ApiScope."); + bldr.append("ListenerType."); bldr.append(iter.next()); if (iter.hasNext()) { bldr.append(", "); @@ -201,12 +201,12 @@ private void generateEnumValues() { String name = apiData.name(); numProcessed++; - final Collection scopes; - if (apiData.requestSpec.scope() == null) { - scopes = Collections.emptyList(); + final Collection listeners; + if (apiData.requestSpec.listeners() == null) { + listeners = Collections.emptyList(); } else { - scopes = apiData.requestSpec.scope().stream() - .map(RequestApiScope::name) + listeners = apiData.requestSpec.listeners().stream() + .map(RequestListenerType::name) .collect(Collectors.toList()); } @@ -218,7 +218,7 @@ private void generateEnumValues() { apiData.responseSchema(), apiData.requestSpec.struct().versions().lowest(), apiData.requestSpec.struct().versions().highest(), - generateApiScopeEnumSet(scopes), + generateListenerTypeEnumSet(listeners), (numProcessed == apis.size()) ? ";" : ","); } } @@ -230,7 +230,7 @@ private void generateInstanceVariables() { buffer.printf("private final Schema[] responseSchemas;%n"); buffer.printf("private final short lowestSupportedVersion;%n"); buffer.printf("private final short highestSupportedVersion;%n"); - buffer.printf("private final EnumSet scope;%n"); + buffer.printf("private final EnumSet listeners;%n"); headerGenerator.addImport(MessageGenerator.SCHEMA_CLASS); headerGenerator.addImport(MessageGenerator.ENUM_SET_CLASS); } @@ -239,7 +239,7 @@ private void generateEnumConstructor() { buffer.printf("ApiMessageType(String name, short apiKey, " + "Schema[] requestSchemas, Schema[] responseSchemas, " + "short lowestSupportedVersion, short highestSupportedVersion, " + - "EnumSet scope) {%n"); + "EnumSet listeners) {%n"); buffer.incrementIndent(); buffer.printf("this.name = name;%n"); buffer.printf("this.apiKey = apiKey;%n"); @@ -247,7 +247,7 @@ private void generateEnumConstructor() { buffer.printf("this.responseSchemas = responseSchemas;%n"); buffer.printf("this.lowestSupportedVersion = lowestSupportedVersion;%n"); buffer.printf("this.highestSupportedVersion = highestSupportedVersion;%n"); - buffer.printf("this.scope = scope;%n"); + buffer.printf("this.listeners = listeners;%n"); buffer.decrementIndent(); buffer.printf("}%n"); } @@ -391,13 +391,13 @@ private void generateHeaderVersion(String type) { buffer.printf("}%n"); } - private void generateApiScopeEnum() { - buffer.printf("public enum ApiScope {%n"); + private void generateListenerTypesEnum() { + buffer.printf("public enum ListenerType {%n"); buffer.incrementIndent(); - Iterator scopeIterator = Arrays.stream(RequestApiScope.values()).iterator(); - while (scopeIterator.hasNext()) { - RequestApiScope scope = scopeIterator.next(); - buffer.printf("%s%s%n", scope.name(), scopeIterator.hasNext() ? "," : ";"); + Iterator listenerIter = Arrays.stream(RequestListenerType.values()).iterator(); + while (listenerIter.hasNext()) { + RequestListenerType scope = listenerIter.next(); + buffer.printf("%s%s%n", scope.name(), listenerIter.hasNext() ? "," : ";"); } buffer.decrementIndent(); buffer.printf("}%n"); diff --git a/generator/src/main/java/org/apache/kafka/message/MessageSpec.java b/generator/src/main/java/org/apache/kafka/message/MessageSpec.java index 7fb18de8f1690..fdcd7cd867f83 100644 --- a/generator/src/main/java/org/apache/kafka/message/MessageSpec.java +++ b/generator/src/main/java/org/apache/kafka/message/MessageSpec.java @@ -37,7 +37,7 @@ public final class MessageSpec { private final Versions flexibleVersions; - private final List scope; + private final List listeners; @JsonCreator public MessageSpec(@JsonProperty("name") String name, @@ -47,7 +47,7 @@ public MessageSpec(@JsonProperty("name") String name, @JsonProperty("type") MessageSpecType type, @JsonProperty("commonStructs") List commonStructs, @JsonProperty("flexibleVersions") String flexibleVersions, - @JsonProperty("scope") List scope) { + @JsonProperty("listeners") List listeners) { this.struct = new StructSpec(name, validVersions, fields); this.apiKey = apiKey == null ? Optional.empty() : Optional.of(apiKey); this.type = Objects.requireNonNull(type); @@ -61,11 +61,11 @@ public MessageSpec(@JsonProperty("name") String name, "be either none, or an open-ended range (that ends with a plus sign)."); } - if (scope != null && !scope.isEmpty() && type != MessageSpecType.REQUEST) { + if (listeners != null && !listeners.isEmpty() && type != MessageSpecType.REQUEST) { throw new RuntimeException("The `requestScope` property is only valid for " + "messages with type `request`"); } - this.scope = scope; + this.listeners = listeners; } public StructSpec struct() { @@ -115,9 +115,9 @@ public String flexibleVersionsString() { return flexibleVersions.toString(); } - @JsonProperty("scope") - public List scope() { - return scope; + @JsonProperty("listeners") + public List listeners() { + return listeners; } public String dataClassName() { diff --git a/generator/src/main/java/org/apache/kafka/message/RequestApiScope.java b/generator/src/main/java/org/apache/kafka/message/RequestListenerType.java similarity index 96% rename from generator/src/main/java/org/apache/kafka/message/RequestApiScope.java rename to generator/src/main/java/org/apache/kafka/message/RequestListenerType.java index 3b2d481905228..10d0c13e29c67 100644 --- a/generator/src/main/java/org/apache/kafka/message/RequestApiScope.java +++ b/generator/src/main/java/org/apache/kafka/message/RequestListenerType.java @@ -18,7 +18,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; -public enum RequestApiScope { +public enum RequestListenerType { @JsonProperty("zkBroker") ZK_BROKER, diff --git a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/metadata/MetadataRequestBenchmark.java b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/metadata/MetadataRequestBenchmark.java index 0735866ae70eb..e59a6a14a69b3 100644 --- a/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/metadata/MetadataRequestBenchmark.java +++ b/jmh-benchmarks/src/main/java/org/apache/kafka/jmh/metadata/MetadataRequestBenchmark.java @@ -190,7 +190,7 @@ private KafkaApis createKafkaApis() { "clusterId", new SystemTime(), null, - new SimpleApiVersionManager(ApiMessageType.ApiScope.ZK_BROKER)); + new SimpleApiVersionManager(ApiMessageType.ListenerType.ZK_BROKER)); } @TearDown(Level.Trial) From b287f8bdd1800740893f20cba7a60711924a0db5 Mon Sep 17 00:00:00 2001 From: Jason Gustafson Date: Thu, 18 Feb 2021 07:16:15 -0800 Subject: [PATCH 5/7] Remove `raft` listener type --- .../main/resources/common/message/ApiVersionsRequest.json | 2 +- .../resources/common/message/BeginQuorumEpochRequest.json | 2 +- .../main/resources/common/message/EndQuorumEpochRequest.json | 2 +- clients/src/main/resources/common/message/FetchRequest.json | 2 +- .../main/resources/common/message/FetchSnapshotRequest.json | 2 +- clients/src/main/resources/common/message/VoteRequest.json | 2 +- core/src/main/scala/kafka/tools/TestRaftServer.scala | 2 +- .../java/org/apache/kafka/message/RequestListenerType.java | 5 +---- 8 files changed, 8 insertions(+), 11 deletions(-) diff --git a/clients/src/main/resources/common/message/ApiVersionsRequest.json b/clients/src/main/resources/common/message/ApiVersionsRequest.json index 39f3cb36562f0..b86edbfaaec12 100644 --- a/clients/src/main/resources/common/message/ApiVersionsRequest.json +++ b/clients/src/main/resources/common/message/ApiVersionsRequest.json @@ -16,7 +16,7 @@ { "apiKey": 18, "type": "request", - "listeners": ["zkBroker", "broker", "controller", "raft"], + "listeners": ["zkBroker", "broker", "controller"], "name": "ApiVersionsRequest", // Versions 0 through 2 of ApiVersionsRequest are the same. // diff --git a/clients/src/main/resources/common/message/BeginQuorumEpochRequest.json b/clients/src/main/resources/common/message/BeginQuorumEpochRequest.json index cad4bb2e390a2..9f7969ff889c5 100644 --- a/clients/src/main/resources/common/message/BeginQuorumEpochRequest.json +++ b/clients/src/main/resources/common/message/BeginQuorumEpochRequest.json @@ -16,7 +16,7 @@ { "apiKey": 53, "type": "request", - "listeners": ["controller", "raft"], + "listeners": ["controller"], "name": "BeginQuorumEpochRequest", "validVersions": "0", "fields": [ diff --git a/clients/src/main/resources/common/message/EndQuorumEpochRequest.json b/clients/src/main/resources/common/message/EndQuorumEpochRequest.json index 23058328c238f..3ef7f6320a915 100644 --- a/clients/src/main/resources/common/message/EndQuorumEpochRequest.json +++ b/clients/src/main/resources/common/message/EndQuorumEpochRequest.json @@ -16,7 +16,7 @@ { "apiKey": 54, "type": "request", - "listeners": ["controller", "raft"], + "listeners": ["controller"], "name": "EndQuorumEpochRequest", "validVersions": "0", "fields": [ diff --git a/clients/src/main/resources/common/message/FetchRequest.json b/clients/src/main/resources/common/message/FetchRequest.json index 768bfd5f016fd..ab4c95fba8264 100644 --- a/clients/src/main/resources/common/message/FetchRequest.json +++ b/clients/src/main/resources/common/message/FetchRequest.json @@ -16,7 +16,7 @@ { "apiKey": 1, "type": "request", - "listeners": ["zkBroker", "broker", "controller", "raft"], + "listeners": ["zkBroker", "broker", "controller"], "name": "FetchRequest", // // Version 1 is the same as version 0. diff --git a/clients/src/main/resources/common/message/FetchSnapshotRequest.json b/clients/src/main/resources/common/message/FetchSnapshotRequest.json index 973acb817f35e..accc227731209 100644 --- a/clients/src/main/resources/common/message/FetchSnapshotRequest.json +++ b/clients/src/main/resources/common/message/FetchSnapshotRequest.json @@ -16,7 +16,7 @@ { "apiKey": 59, "type": "request", - "listeners": ["controller", "raft"], + "listeners": ["controller"], "name": "FetchSnapshotRequest", "validVersions": "0", "flexibleVersions": "0+", diff --git a/clients/src/main/resources/common/message/VoteRequest.json b/clients/src/main/resources/common/message/VoteRequest.json index f6a60290c073e..fcc0017ed2574 100644 --- a/clients/src/main/resources/common/message/VoteRequest.json +++ b/clients/src/main/resources/common/message/VoteRequest.json @@ -16,7 +16,7 @@ { "apiKey": 52, "type": "request", - "listeners": ["controller", "raft"], + "listeners": ["controller"], "name": "VoteRequest", "validVersions": "0", "flexibleVersions": "0+", diff --git a/core/src/main/scala/kafka/tools/TestRaftServer.scala b/core/src/main/scala/kafka/tools/TestRaftServer.scala index 02806762899e4..ba6ab4074a73f 100644 --- a/core/src/main/scala/kafka/tools/TestRaftServer.scala +++ b/core/src/main/scala/kafka/tools/TestRaftServer.scala @@ -69,7 +69,7 @@ class TestRaftServer( tokenCache = new DelegationTokenCache(ScramMechanism.mechanismNames) credentialProvider = new CredentialProvider(ScramMechanism.mechanismNames, tokenCache) - val apiVersionManager = new SimpleApiVersionManager(ListenerType.RAFT) + val apiVersionManager = new SimpleApiVersionManager(ListenerType.CONTROLLER) socketServer = new SocketServer(config, metrics, time, credentialProvider, apiVersionManager) socketServer.startup(startProcessingRequests = false) diff --git a/generator/src/main/java/org/apache/kafka/message/RequestListenerType.java b/generator/src/main/java/org/apache/kafka/message/RequestListenerType.java index 10d0c13e29c67..cefd40db6ec4a 100644 --- a/generator/src/main/java/org/apache/kafka/message/RequestListenerType.java +++ b/generator/src/main/java/org/apache/kafka/message/RequestListenerType.java @@ -26,8 +26,5 @@ public enum RequestListenerType { BROKER, @JsonProperty("controller") - CONTROLLER, - - @JsonProperty("raft") - RAFT; + CONTROLLER; } From 49d73fdd7cec6b71dfaf75fe98503e4f83354662 Mon Sep 17 00:00:00 2001 From: Jason Gustafson Date: Thu, 18 Feb 2021 09:05:22 -0800 Subject: [PATCH 6/7] Fix breakage after merge --- .../scala/kafka/server/BrokerServer.scala | 51 ++++++++++++++----- .../scala/kafka/server/ControllerServer.scala | 8 +-- .../scala/kafka/server/KafkaRaftServer.scala | 15 ++++-- .../unit/kafka/server/KafkaApisTest.scala | 11 ++-- 4 files changed, 58 insertions(+), 27 deletions(-) diff --git a/core/src/main/scala/kafka/server/BrokerServer.scala b/core/src/main/scala/kafka/server/BrokerServer.scala index 57ceb46202fbc..19d65ab2e98bd 100644 --- a/core/src/main/scala/kafka/server/BrokerServer.scala +++ b/core/src/main/scala/kafka/server/BrokerServer.scala @@ -33,6 +33,7 @@ import kafka.server.KafkaBroker.metricsPrefix import kafka.server.metadata.{BrokerMetadataListener, CachedConfigRepository, ClientQuotaCache, ClientQuotaMetadataManager, RaftMetadataCache} import kafka.utils.{CoreUtils, KafkaScheduler} import org.apache.kafka.common.internals.Topic +import org.apache.kafka.common.message.ApiMessageType.ListenerType import org.apache.kafka.common.message.BrokerRegistrationRequestData.{Listener, ListenerCollection} import org.apache.kafka.common.metrics.Metrics import org.apache.kafka.common.network.ListenerName @@ -167,7 +168,7 @@ class BrokerServer( // Create log manager, but don't start it because we need to delay any potential unclean shutdown log recovery // until we catch up on the metadata log and have up-to-date topic and broker configs. logManager = LogManager(config, initialOfflineDirs, configRepository, kafkaScheduler, time, - brokerTopicStats, logDirFailureChannel, true) + brokerTopicStats, logDirFailureChannel, keepPartitionMetadataFile = true) metadataCache = MetadataCache.raftMetadataCache(config.nodeId) // Enable delegation token cache for all SCRAM mechanisms to simplify dynamic update. @@ -175,17 +176,44 @@ class BrokerServer( tokenCache = new DelegationTokenCache(ScramMechanism.mechanismNames) credentialProvider = new CredentialProvider(ScramMechanism.mechanismNames, tokenCache) + val controllerNodes = RaftConfig.quorumVoterStringsToNodes(controllerQuorumVotersFuture.get()).asScala + val controllerNodeProvider = RaftControllerNodeProvider(metaLogManager, config, controllerNodes) + + val forwardingChannelManager = BrokerToControllerChannelManager( + controllerNodeProvider, + time, + metrics, + config, + channelName = "forwarding", + threadNamePrefix, + retryTimeoutMs = 60000 + ) + forwardingManager = new ForwardingManagerImpl(forwardingChannelManager) + forwardingManager.start() + + val apiVersionManager = ApiVersionManager( + ListenerType.BROKER, + config, + Some(forwardingManager), + brokerFeatures, + featureCache + ) + // Create and start the socket server acceptor threads so that the bound port is known. // Delay starting processors until the end of the initialization sequence to ensure // that credentials have been loaded before processing authentications. - socketServer = new SocketServer(config, metrics, time, credentialProvider, allowControllerOnlyApis = false) + socketServer = new SocketServer(config, metrics, time, credentialProvider, apiVersionManager) socketServer.startup(startProcessingRequests = false) - val controllerNodes = - RaftConfig.quorumVoterStringsToNodes(controllerQuorumVotersFuture.get()).asScala - val controllerNodeProvider = RaftControllerNodeProvider(metaLogManager, config, controllerNodes) - val alterIsrChannelManager = BrokerToControllerChannelManager(controllerNodeProvider, - time, metrics, config, "alterisr", threadNamePrefix, 60000) + val alterIsrChannelManager = BrokerToControllerChannelManager( + controllerNodeProvider, + time, + metrics, + config, + channelName = "alterisr", + threadNamePrefix, + retryTimeoutMs = Long.MaxValue + ) alterIsrManager = new DefaultAlterIsrManager( controllerChannelManager = alterIsrChannelManager, scheduler = kafkaScheduler, @@ -200,11 +228,6 @@ class BrokerServer( brokerTopicStats, metadataCache, logDirFailureChannel, alterIsrManager, configRepository, threadNamePrefix) - val forwardingChannelManager = BrokerToControllerChannelManager(controllerNodeProvider, - time, metrics, config, "forwarding", threadNamePrefix, 60000) - forwardingManager = new ForwardingManagerImpl(forwardingChannelManager) - forwardingManager.start() - /* start token manager */ if (config.tokenAuthEnabled) { throw new UnsupportedOperationException("Delegation tokens are not supported") @@ -306,7 +329,7 @@ class BrokerServer( dataPlaneRequestProcessor = new KafkaApis(socketServer.dataPlaneRequestChannel, raftSupport, replicaManager, groupCoordinator, transactionCoordinator, autoTopicCreationManager, config.nodeId, config, configRepository, metadataCache, metrics, authorizer, quotaManagers, - fetchManager, brokerTopicStats, clusterId, time, tokenManager, brokerFeatures, featureCache) + fetchManager, brokerTopicStats, clusterId, time, tokenManager, apiVersionManager) dataPlaneRequestHandlerPool = new KafkaRequestHandlerPool(config.nodeId, socketServer.dataPlaneRequestChannel, dataPlaneRequestProcessor, time, config.numIoThreads, s"${SocketServer.DataPlaneMetricPrefix}RequestHandlerAvgIdlePercent", SocketServer.DataPlaneThreadPrefix) @@ -315,7 +338,7 @@ class BrokerServer( controlPlaneRequestProcessor = new KafkaApis(controlPlaneRequestChannel, raftSupport, replicaManager, groupCoordinator, transactionCoordinator, autoTopicCreationManager, config.nodeId, config, configRepository, metadataCache, metrics, authorizer, quotaManagers, - fetchManager, brokerTopicStats, clusterId, time, tokenManager, brokerFeatures, featureCache) + fetchManager, brokerTopicStats, clusterId, time, tokenManager, apiVersionManager) controlPlaneRequestHandlerPool = new KafkaRequestHandlerPool(config.nodeId, socketServer.controlPlaneRequestChannelOpt.get, controlPlaneRequestProcessor, time, 1, s"${SocketServer.ControlPlaneMetricPrefix}RequestHandlerAvgIdlePercent", SocketServer.ControlPlaneThreadPrefix) diff --git a/core/src/main/scala/kafka/server/ControllerServer.scala b/core/src/main/scala/kafka/server/ControllerServer.scala index efcebb491c3df..625ce5257b8e3 100644 --- a/core/src/main/scala/kafka/server/ControllerServer.scala +++ b/core/src/main/scala/kafka/server/ControllerServer.scala @@ -28,6 +28,7 @@ import kafka.raft.RaftManager import kafka.security.CredentialProvider import kafka.server.QuotaFactory.QuotaManagers import kafka.utils.{CoreUtils, Logging} +import org.apache.kafka.common.message.ApiMessageType.ListenerType import org.apache.kafka.common.metrics.Metrics import org.apache.kafka.common.security.scram.internals.ScramMechanism import org.apache.kafka.common.security.token.delegation.internals.DelegationTokenCache @@ -124,15 +125,16 @@ class ControllerServer( }.toMap } + val apiVersionManager = new SimpleApiVersionManager(ListenerType.CONTROLLER) + tokenCache = new DelegationTokenCache(ScramMechanism.mechanismNames) credentialProvider = new CredentialProvider(ScramMechanism.mechanismNames, tokenCache) socketServer = new SocketServer(config, metrics, time, credentialProvider, - allowControllerOnlyApis = true, - controllerSocketServer = true) - socketServer.startup(false, None, config.controllerListeners) + apiVersionManager) + socketServer.startup(startProcessingRequests = false, controlPlaneListener = None, config.controllerListeners) socketServerFirstBoundPortFuture.complete(socketServer.boundPort( config.controllerListeners.head.listenerName)) diff --git a/core/src/main/scala/kafka/server/KafkaRaftServer.scala b/core/src/main/scala/kafka/server/KafkaRaftServer.scala index dc3fd16fed8a6..7ec46220b91f4 100644 --- a/core/src/main/scala/kafka/server/KafkaRaftServer.scala +++ b/core/src/main/scala/kafka/server/KafkaRaftServer.scala @@ -74,8 +74,17 @@ class KafkaRaftServer( private val metaLogShim = new MetaLogRaftShim(raftManager.kafkaRaftClient, config.nodeId) private val broker: Option[BrokerServer] = if (config.processRoles.contains(BrokerRole)) { - Some(new BrokerServer(config, metaProps, metaLogShim, time, metrics, threadNamePrefix, - offlineDirs, controllerQuorumVotersFuture, Server.SUPPORTED_FEATURES)) + Some(new BrokerServer( + config, + metaProps, + metaLogShim, + time, + metrics, + threadNamePrefix, + offlineDirs, + controllerQuorumVotersFuture, + Server.SUPPORTED_FEATURES + )) } else { None } @@ -89,7 +98,7 @@ class KafkaRaftServer( time, metrics, threadNamePrefix, - CompletableFuture.completedFuture(config.quorumVoters) + controllerQuorumVotersFuture )) } else { None diff --git a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala index 18203d4c306cf..e80c6eb3a736c 100644 --- a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala +++ b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala @@ -34,11 +34,7 @@ import kafka.log.AppendOrigin import kafka.network.RequestChannel import kafka.network.RequestChannel.{CloseConnectionResponse, SendResponse} import kafka.server.QuotaFactory.QuotaManagers -<<<<<<< HEAD import kafka.server.metadata.{CachedConfigRepository, ConfigRepository, RaftMetadataCache} -======= -import kafka.server.metadata.{CachedConfigRepository, ConfigRepository} ->>>>>>> Rename `scope` to `listeners` import kafka.utils.{MockTime, TestUtils} import kafka.zk.KafkaZkClient import org.apache.kafka.clients.admin.AlterConfigOp.OpType @@ -166,12 +162,13 @@ class KafkaApisTest { } } + val listenerType = if (raftSupport) ListenerType.BROKER else ListenerType.ZK_BROKER val enabledApis = if (enableForwarding) { - ApiKeys.apisForListener(ListenerType.ZK_BROKER).asScala ++ Set(ApiKeys.ENVELOPE) + ApiKeys.apisForListener(listenerType).asScala ++ Set(ApiKeys.ENVELOPE) } else { - ApiKeys.apisForListener(ListenerType.ZK_BROKER).asScala.toSet + ApiKeys.apisForListener(listenerType).asScala.toSet } - val apiVersionManager = new SimpleApiVersionManager(enabledApis) + val apiVersionManager = new SimpleApiVersionManager(listenerType, enabledApis) new KafkaApis(requestChannel, metadataSupport, From 882c36db16429a383e9689513ce124b2f22e1b7f Mon Sep 17 00:00:00 2001 From: Jason Gustafson Date: Thu, 18 Feb 2021 12:28:25 -0800 Subject: [PATCH 7/7] Store static map of apis by listener type --- .../org/apache/kafka/common/protocol/ApiKeys.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java b/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java index 8f6422f500798..475fc84c7355c 100644 --- a/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java +++ b/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java @@ -23,6 +23,7 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.EnumMap; import java.util.EnumSet; import java.util.List; import java.util.Map; @@ -106,6 +107,15 @@ public enum ApiKeys { BROKER_HEARTBEAT(ApiMessageType.BROKER_HEARTBEAT, true, RecordBatch.MAGIC_VALUE_V0, false), UNREGISTER_BROKER(ApiMessageType.UNREGISTER_BROKER, false, RecordBatch.MAGIC_VALUE_V0, true); + private static final Map> APIS_BY_LISTENER = + new EnumMap<>(ApiMessageType.ListenerType.class); + + static { + for (ApiMessageType.ListenerType listenerType : ApiMessageType.ListenerType.values()) { + APIS_BY_LISTENER.put(listenerType, filterApisForListener(listenerType)); + } + } + // The generator ensures every `ApiMessageType` has a unique id private static final Map ID_TO_TYPE = Arrays.stream(ApiKeys.values()) .collect(Collectors.toMap(key -> (int) key.id, Function.identity())); @@ -255,6 +265,10 @@ public static EnumSet zkBrokerApis() { } public static EnumSet apisForListener(ApiMessageType.ListenerType listener) { + return APIS_BY_LISTENER.get(listener); + } + + private static EnumSet filterApisForListener(ApiMessageType.ListenerType listener) { List controllerApis = Arrays.stream(ApiKeys.values()) .filter(apiKey -> apiKey.messageType.listeners().contains(listener)) .collect(Collectors.toList());