From ca6508233725aa718c5342f2c2adbc9ebacece72 Mon Sep 17 00:00:00 2001 From: Brian Byrne Date: Mon, 10 Feb 2020 14:15:42 -0800 Subject: [PATCH 1/6] KIP-546 (1/2): Implements describeClientQuotas() and alterClientQuotas(). Implements describeClientQuotas() and alterClientQuotas() to match current functionality, i.e. extensible entity types and other new quotas features like resolving quotas is not yet supported. This is the minimal functionality necessary for KIP-500, and converts the ConfigCommand to use the client quotas APIs. --- checkstyle/import-control.xml | 3 +- checkstyle/suppressions.xml | 2 +- .../org/apache/kafka/clients/admin/Admin.java | 82 +++- .../admin/AlterClientQuotasOptions.java | 46 ++ .../admin/AlterClientQuotasResult.java | 58 +++ .../admin/DescribeClientQuotasOptions.java | 46 ++ .../admin/DescribeClientQuotasResult.java | 52 ++ .../kafka/clients/admin/KafkaAdminClient.java | 68 ++- .../apache/kafka/common/protocol/ApiKeys.java | 8 +- .../kafka/common/quota/QuotaAlteration.java | 101 ++++ .../kafka/common/quota/QuotaEntity.java | 76 +++ .../kafka/common/quota/QuotaFilter.java | 110 +++++ .../common/requests/AbstractRequest.java | 4 + .../common/requests/AbstractResponse.java | 4 + .../requests/AlterClientQuotasRequest.java | 133 ++++++ .../requests/AlterClientQuotasResponse.java | 124 +++++ .../requests/DescribeClientQuotasRequest.java | 113 +++++ .../DescribeClientQuotasResponse.java | 119 +++++ .../message/AlterClientQuotasRequest.json | 45 ++ .../message/AlterClientQuotasResponse.json | 40 ++ .../message/DescribeClientQuotasRequest.json | 35 ++ .../message/DescribeClientQuotasResponse.json | 47 ++ .../clients/admin/KafkaAdminClientTest.java | 95 ++++ .../kafka/clients/admin/MockAdminClient.java | 12 + .../scala/kafka/admin/ConfigCommand.scala | 130 +++-- .../scala/kafka/server/AdminManager.scala | 178 ++++++- .../scala/kafka/server/DynamicConfig.scala | 4 + .../main/scala/kafka/server/KafkaApis.scala | 30 ++ .../unit/kafka/admin/ConfigCommandTest.scala | 145 +++++- .../server/ClientQuotasRequestTest.scala | 444 ++++++++++++++++++ 30 files changed, 2306 insertions(+), 48 deletions(-) create mode 100644 clients/src/main/java/org/apache/kafka/clients/admin/AlterClientQuotasOptions.java create mode 100644 clients/src/main/java/org/apache/kafka/clients/admin/AlterClientQuotasResult.java create mode 100644 clients/src/main/java/org/apache/kafka/clients/admin/DescribeClientQuotasOptions.java create mode 100644 clients/src/main/java/org/apache/kafka/clients/admin/DescribeClientQuotasResult.java create mode 100644 clients/src/main/java/org/apache/kafka/common/quota/QuotaAlteration.java create mode 100644 clients/src/main/java/org/apache/kafka/common/quota/QuotaEntity.java create mode 100644 clients/src/main/java/org/apache/kafka/common/quota/QuotaFilter.java create mode 100644 clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasRequest.java create mode 100644 clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasResponse.java create mode 100644 clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasRequest.java create mode 100644 clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasResponse.java create mode 100644 clients/src/main/resources/common/message/AlterClientQuotasRequest.json create mode 100644 clients/src/main/resources/common/message/AlterClientQuotasResponse.json create mode 100644 clients/src/main/resources/common/message/DescribeClientQuotasRequest.json create mode 100644 clients/src/main/resources/common/message/DescribeClientQuotasResponse.json create mode 100644 core/src/test/scala/unit/kafka/server/ClientQuotasRequestTest.scala diff --git a/checkstyle/import-control.xml b/checkstyle/import-control.xml index e3975a30c8c70..491adc0deddf3 100644 --- a/checkstyle/import-control.xml +++ b/checkstyle/import-control.xml @@ -145,6 +145,7 @@ + @@ -241,7 +242,7 @@ - + diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml index f533179c74013..aa3a0868f982f 100644 --- a/checkstyle/suppressions.xml +++ b/checkstyle/suppressions.xml @@ -62,7 +62,7 @@ files="(ConsumerCoordinator|Fetcher|Sender|KafkaProducer|BufferPool|ConfigDef|RecordAccumulator|KerberosLogin|AbstractRequest|AbstractResponse|Selector|SslFactory|SslTransportLayer|SaslClientAuthenticator|SaslClientCallbackHandler|SaslServerAuthenticator|AbstractCoordinator|TransactionManager).java"/> + files="(AbstractRequest|AbstractResponse|KerberosLogin|WorkerSinkTaskTest|TransactionManagerTest|SenderTest|KafkaAdminClient|ConsumerCoordinatorTest).java"/> diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/Admin.java b/clients/src/main/java/org/apache/kafka/clients/admin/Admin.java index fd70e978e419a..6a607f3ce614f 100644 --- a/clients/src/main/java/org/apache/kafka/clients/admin/Admin.java +++ b/clients/src/main/java/org/apache/kafka/clients/admin/Admin.java @@ -36,6 +36,8 @@ import org.apache.kafka.common.acl.AclBindingFilter; import org.apache.kafka.common.annotation.InterfaceStability; import org.apache.kafka.common.config.ConfigResource; +import org.apache.kafka.common.quota.QuotaAlteration; +import org.apache.kafka.common.quota.QuotaFilter; import org.apache.kafka.common.requests.LeaveGroupResponse; /** @@ -408,7 +410,6 @@ default AlterConfigsResult incrementalAlterConfigs(Map @@ -1132,6 +1133,85 @@ default ListOffsetsResult listOffsets(Map topicParti */ ListOffsetsResult listOffsets(Map topicPartitionOffsets, ListOffsetsOptions options); + /** + * Describes all entities matching all provided filters (logical AND) that have at least one quota + * configuration value defined. + *

+ * This is a convenience method for {@link #describeClientQuotas(Collection, DescribeClientQuotasOptions)} + * with default options. See the overload for more details. + *

+ * This operation is supported by brokers with version 2.6.0 or higher. + * + * @param filters filtering rules to apply to matching entities + * @return the DescribeClientQuotasResult containing the result + */ + default DescribeClientQuotasResult describeClientQuotas(Collection filters) { + return describeClientQuotas(filters, new DescribeClientQuotasOptions()); + } + + /** + * Describes all entities matching all provided filters (logical AND) that have at least one quota + * configuration value defined. + *

+ * The following exceptions can be anticipated when calling {@code get()} on the future from the + * returned {@link DescribeClientQuotasResult}: + *

    + *
  • {@link org.apache.kafka.common.errors.ClusterAuthorizationException} + * If the authenticated user didn't have describe access to the cluster.
    • + *
  • {@link org.apache.kafka.common.errors.InvalidRequestException} + * If the request details are invalid. e.g., an invalid entity type was specified.
    • + *
  • {@link org.apache.kafka.common.errors.TimeoutException} + * If the request timed out before the describe could finish.
    • + *
+ *

+ * This operation is supported by brokers with version 2.6.0 or higher. + * + * @param filters filtering rules to apply to matching entities + * @param options the options to use + * @return the DescribeClientQuotasResult containing the result + */ + DescribeClientQuotasResult describeClientQuotas(Collection filters, DescribeClientQuotasOptions options); + + /** + * Alters client quota configurations with the specified alterations. + *

+ * This is a convenience method for {@link #alterClientQuotas(Collection, AlterClientQuotasOptions)} + * with default options. See the overload for more details. + *

+ * This operation is supported by brokers with version 2.6.0 or higher. + * + * @param entries the alterations to perform + * @return the AlterClientQuotasResult containing the result + */ + default AlterClientQuotasResult alterClientQuotas(Collection entries) { + return alterClientQuotas(entries, new AlterClientQuotasOptions()); + } + + /** + * Alters client quota configurations with the specified alterations. + *

+ * Alterations for a single entity are atomic, but across entities is not guaranteed. The resulting + * per-entity error code should be evaluated to resolve the success or failure of all updates. + *

+ * The following exceptions can be anticipated when calling {@code get()} on the futures obtained from + * the returned {@link AlterClientQuotasResult}: + *

    + *
  • {@link org.apache.kafka.common.errors.ClusterAuthorizationException} + * If the authenticated user didn't have alter access to the cluster.
    • + *
  • {@link org.apache.kafka.common.errors.InvalidRequestException} + * If the request details are invalid. e.g., a configuration key was specified more than once for an entity.
    • + *
  • {@link org.apache.kafka.common.errors.TimeoutException} + * If the request timed out before the alterations could finish. It cannot be guaranteed whether the update + * succeed or not.
    • + *
+ *

+ * This operation is supported by brokers with version 2.6.0 or higher. + * + * @param entries the alterations to perform + * @return the AlterClientQuotasResult containing the result + */ + AlterClientQuotasResult alterClientQuotas(Collection entries, AlterClientQuotasOptions options); + /** * Get the metrics kept by the adminClient */ diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/AlterClientQuotasOptions.java b/clients/src/main/java/org/apache/kafka/clients/admin/AlterClientQuotasOptions.java new file mode 100644 index 0000000000000..3cdaa97762a4d --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/clients/admin/AlterClientQuotasOptions.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.clients.admin; + +import org.apache.kafka.common.annotation.InterfaceStability; + +/** + * Options for {@link Admin#alterClientQuotas(Collection, AlterClientQuotasOptions)}. + * + * The API of this class is evolving, see {@link Admin} for details. + */ +@InterfaceStability.Evolving +public class AlterClientQuotasOptions extends AbstractOptions { + + private boolean validateOnly = false; + + /** + * Returns whether the request should be validated without altering the configs. + */ + public boolean validateOnly() { + return this.validateOnly; + } + + /** + * Sets whether the request should be validated without altering the configs. + */ + public AlterClientQuotasOptions validateOnly(boolean validateOnly) { + this.validateOnly = validateOnly; + return this; + } +} diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/AlterClientQuotasResult.java b/clients/src/main/java/org/apache/kafka/clients/admin/AlterClientQuotasResult.java new file mode 100644 index 0000000000000..fd7f3a9d7ba19 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/clients/admin/AlterClientQuotasResult.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.clients.admin; + +import org.apache.kafka.common.KafkaFuture; +import org.apache.kafka.common.annotation.InterfaceStability; +import org.apache.kafka.common.quota.QuotaEntity; + +import java.util.Map; + +/** + * The result of the {@link Admin#alterClientQuotas(Collection, AlterClientQuotasOptions)} call. + * + * The API of this class is evolving, see {@link Admin} for details. + */ +@InterfaceStability.Evolving +public class AlterClientQuotasResult { + + private final Map> futures; + + /** + * Maps an entity to its alteration result. + * + * @param futures maps entity to its alteration result + */ + public AlterClientQuotasResult(Map> futures) { + this.futures = futures; + } + + /** + * Returns a map from quota entity to a future which can be used to check the status of the operation. + */ + public Map> values() { + return futures; + } + + /** + * Returns a future which succeeds only if all quota alterations succeed. + */ + public KafkaFuture all() { + return KafkaFuture.allOf(futures.values().toArray(new KafkaFuture[0])); + } +} diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/DescribeClientQuotasOptions.java b/clients/src/main/java/org/apache/kafka/clients/admin/DescribeClientQuotasOptions.java new file mode 100644 index 0000000000000..c0b49113bc47d --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/clients/admin/DescribeClientQuotasOptions.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.clients.admin; + +import org.apache.kafka.common.annotation.InterfaceStability; + +/** + * Options for {@link Admin#describeClientQuotas(Collection, DescribeClientQuotasOptions)}. + * + * The API of this class is evolving, see {@link Admin} for details. + */ +@InterfaceStability.Evolving +public class DescribeClientQuotasOptions extends AbstractOptions { + + private boolean includeUnspecifiedTypes = false; + + /** + * Returns whether the result should include unspecified entity types. + */ + public boolean includeUnspecifiedTypes() { + return this.includeUnspecifiedTypes; + } + + /** + * Sets whether the result should include unspecified entity types. + */ + public DescribeClientQuotasOptions includeUnspecifiedTypes(boolean includeUnspecifiedTypes) { + this.includeUnspecifiedTypes = includeUnspecifiedTypes; + return this; + } +} diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/DescribeClientQuotasResult.java b/clients/src/main/java/org/apache/kafka/clients/admin/DescribeClientQuotasResult.java new file mode 100644 index 0000000000000..27677787e8a29 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/clients/admin/DescribeClientQuotasResult.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.clients.admin; + +import org.apache.kafka.common.KafkaFuture; +import org.apache.kafka.common.annotation.InterfaceStability; +import org.apache.kafka.common.quota.QuotaEntity; + +import java.util.Map; + +/** + * The result of the {@link Admin#describeClientQuotas(Collection, DescribeClientQuotasOptions)} call. + * + * The API of this class is evolving, see {@link Admin} for details. + */ +@InterfaceStability.Evolving +public class DescribeClientQuotasResult { + + private final KafkaFuture>> entities; + + /** + * Maps an entity to its configured quota value(s). Note if no value is defined for a quota + * type for that entity's config, then it is not included in the resulting value map. + * + * @param entities future for the collection of entities that matched the filter + */ + public DescribeClientQuotasResult(KafkaFuture>> entities) { + this.entities = entities; + } + + /** + * Returns a map from quota entity to a future which can be used to check the status of the operation. + */ + public KafkaFuture>> entities() { + return entities; + } +} diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java index 56276af08d369..151660c513c31 100644 --- a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java +++ b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java @@ -124,8 +124,13 @@ import org.apache.kafka.common.network.ChannelBuilder; import org.apache.kafka.common.network.Selector; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.quota.QuotaAlteration; +import org.apache.kafka.common.quota.QuotaEntity; +import org.apache.kafka.common.quota.QuotaFilter; import org.apache.kafka.common.requests.AbstractRequest; import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.AlterClientQuotasRequest; +import org.apache.kafka.common.requests.AlterClientQuotasResponse; import org.apache.kafka.common.requests.AlterConfigsRequest; import org.apache.kafka.common.requests.AlterConfigsResponse; import org.apache.kafka.common.requests.AlterPartitionReassignmentsRequest; @@ -151,6 +156,8 @@ import org.apache.kafka.common.requests.DeleteTopicsResponse; import org.apache.kafka.common.requests.DescribeAclsRequest; import org.apache.kafka.common.requests.DescribeAclsResponse; +import org.apache.kafka.common.requests.DescribeClientQuotasRequest; +import org.apache.kafka.common.requests.DescribeClientQuotasResponse; import org.apache.kafka.common.requests.DescribeConfigsRequest; import org.apache.kafka.common.requests.DescribeConfigsResponse; import org.apache.kafka.common.requests.DescribeDelegationTokenRequest; @@ -3726,7 +3733,7 @@ public ListOffsetsResult listOffsets(Map topicPartit MetadataOperationContext context = new MetadataOperationContext<>(topics, options, deadline, futures); - Call metadataCall = getMetadataCall(context, + Call metadataCall = getMetadataCall(context, () -> KafkaAdminClient.this.getListOffsetsCalls(context, topicPartitionOffsets, futures)); runnable.call(metadataCall, nowMetadata); @@ -3823,6 +3830,65 @@ void handleFailure(Throwable throwable) { return calls; } + @Override + public DescribeClientQuotasResult describeClientQuotas(Collection filters, DescribeClientQuotasOptions options) { + KafkaFutureImpl>> future = new KafkaFutureImpl<>(); + + final long now = time.milliseconds(); + runnable.call(new Call("describeClientQuotas", calcDeadlineMs(now, options.timeoutMs()), + new LeastLoadedNodeProvider()) { + + @Override + DescribeClientQuotasRequest.Builder createRequest(int timeoutMs) { + return new DescribeClientQuotasRequest.Builder(filters, options.includeUnspecifiedTypes()); + } + + @Override + void handleResponse(AbstractResponse abstractResponse) { + DescribeClientQuotasResponse response = (DescribeClientQuotasResponse) abstractResponse; + response.complete(future); + } + + @Override + void handleFailure(Throwable throwable) { + future.completeExceptionally(throwable); + } + }, now); + + return new DescribeClientQuotasResult(future); + } + + @Override + public AlterClientQuotasResult alterClientQuotas(Collection entries, AlterClientQuotasOptions options) { + Map> futures = new HashMap<>(entries.size()); + for (QuotaAlteration entry : entries) { + futures.put(entry.entity(), new KafkaFutureImpl<>()); + } + + final long now = time.milliseconds(); + runnable.call(new Call("alterClientQuotas", calcDeadlineMs(now, options.timeoutMs()), + new LeastLoadedNodeProvider()) { + + @Override + AlterClientQuotasRequest.Builder createRequest(int timeoutMs) { + return new AlterClientQuotasRequest.Builder(entries, options.validateOnly()); + } + + @Override + void handleResponse(AbstractResponse abstractResponse) { + AlterClientQuotasResponse response = (AlterClientQuotasResponse) abstractResponse; + response.complete(futures); + } + + @Override + void handleFailure(Throwable throwable) { + completeAllExceptionally(futures.values(), throwable); + } + }, now); + + return new AlterClientQuotasResult(Collections.unmodifiableMap(futures)); + } + /** * Get a sub level error when the request is in batch. If given key was not found, * return an {@link IllegalArgumentException}. diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java b/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java index f48f11216ee39..25e75523f10f7 100644 --- a/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java +++ b/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java @@ -19,6 +19,8 @@ import org.apache.kafka.common.message.ApiMessageType; import org.apache.kafka.common.message.ApiVersionsRequestData; import org.apache.kafka.common.message.ApiVersionsResponseData; +import org.apache.kafka.common.message.AlterClientQuotasRequestData; +import org.apache.kafka.common.message.AlterClientQuotasResponseData; import org.apache.kafka.common.message.ControlledShutdownRequestData; import org.apache.kafka.common.message.ControlledShutdownResponseData; import org.apache.kafka.common.message.CreateAclsRequestData; @@ -37,6 +39,8 @@ import org.apache.kafka.common.message.DeleteTopicsResponseData; import org.apache.kafka.common.message.DescribeAclsRequestData; import org.apache.kafka.common.message.DescribeAclsResponseData; +import org.apache.kafka.common.message.DescribeClientQuotasRequestData; +import org.apache.kafka.common.message.DescribeClientQuotasResponseData; import org.apache.kafka.common.message.DescribeDelegationTokenRequestData; import org.apache.kafka.common.message.DescribeDelegationTokenResponseData; import org.apache.kafka.common.message.DescribeGroupsRequestData; @@ -205,7 +209,9 @@ public Struct parseResponse(short version, ByteBuffer buffer) { AlterPartitionReassignmentsResponseData.SCHEMAS), LIST_PARTITION_REASSIGNMENTS(46, "ListPartitionReassignments", ListPartitionReassignmentsRequestData.SCHEMAS, ListPartitionReassignmentsResponseData.SCHEMAS), - OFFSET_DELETE(47, "OffsetDelete", OffsetDeleteRequestData.SCHEMAS, OffsetDeleteResponseData.SCHEMAS); + OFFSET_DELETE(47, "OffsetDelete", OffsetDeleteRequestData.SCHEMAS, OffsetDeleteResponseData.SCHEMAS), + DESCRIBE_CLIENT_QUOTAS(48, "DescribeClientQuotas", DescribeClientQuotasRequestData.SCHEMAS, DescribeClientQuotasResponseData.SCHEMAS), + ALTER_CLIENT_QUOTAS(49, "AlterClientQuotas", AlterClientQuotasRequestData.SCHEMAS, AlterClientQuotasResponseData.SCHEMAS); private static final ApiKeys[] ID_TO_TYPE; private static final int MIN_API_KEY = 0; diff --git a/clients/src/main/java/org/apache/kafka/common/quota/QuotaAlteration.java b/clients/src/main/java/org/apache/kafka/common/quota/QuotaAlteration.java new file mode 100644 index 0000000000000..f992681f0b679 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/common/quota/QuotaAlteration.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.common.quota; + +import java.util.Collection; +import java.util.Objects; + +/** + * Describes a configuration alteration to be made to a quota entity. + */ +public class QuotaAlteration { + + public static class Op { + private final String key; + private final Double value; + + /** + * @param key the quota type to alter + * @param value if set then the existing value is updated, + * otherwise if null, the existing value is cleared + */ + public Op(String key, Double value) { + this.key = key; + this.value = value; + } + + /** + * @return the quota type to alter + */ + public String key() { + return this.key; + } + + /** + * @return if set then the existing value is updated, + * otherwise if null, the existing value is cleared + */ + public Double value() { + return this.value; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Op that = (Op) o; + return Objects.equals(key, that.key) && Objects.equals(value, that.value); + } + + @Override + public int hashCode() { + return Objects.hash(key, value); + } + + @Override + public String toString() { + return "QuotaAlteration.Op(key=" + key + ", value=" + value + ")"; + } + } + + private final QuotaEntity entity; + private final Collection ops; + + /** + * @param entity the entity whose config will be modified + * @param ops the alteration to perform + */ + public QuotaAlteration(QuotaEntity entity, Collection ops) { + this.entity = entity; + this.ops = ops; + } + + /** + * @return the entity whose config will be modified + */ + public QuotaEntity entity() { + return this.entity; + } + + /** + * @return the alteration to perform + */ + public Collection ops() { + return this.ops; + } +} diff --git a/clients/src/main/java/org/apache/kafka/common/quota/QuotaEntity.java b/clients/src/main/java/org/apache/kafka/common/quota/QuotaEntity.java new file mode 100644 index 0000000000000..8293f1aa3c766 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/common/quota/QuotaEntity.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.common.quota; + +import java.util.Map; +import java.util.Objects; + +/** + * Describes a quota entity, which is a mapping of entity types to their names. + */ +public class QuotaEntity { + + private final Map entries; + + /** + * Type of an entity entry. + */ + public static final String USER = "user"; + public static final String CLIENT_ID = "client-id"; + + /** + * Represents the default name for an user/client ID, i.e. the name that's resolved + * when an exact match isn't found. + */ + public final static String USER_DEFAULT = ""; + public final static String CLIENT_ID_DEFAULT = ""; + + /** + * Constructs a quota entity for the given types and names. + * + * @param entries maps entity type to its name + */ + public QuotaEntity(Map entries) { + this.entries = entries; + } + + /** + * @return map of entity type to its name + */ + public Map entries() { + return this.entries; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + QuotaEntity that = (QuotaEntity) o; + return Objects.equals(entries, that.entries); + } + + @Override + public int hashCode() { + return Objects.hash(entries); + } + + @Override + public String toString() { + return "QuotaEntity(entries=" + entries + ")"; + } +} diff --git a/clients/src/main/java/org/apache/kafka/common/quota/QuotaFilter.java b/clients/src/main/java/org/apache/kafka/common/quota/QuotaFilter.java new file mode 100644 index 0000000000000..825584612c163 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/common/quota/QuotaFilter.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.common.quota; + +import java.util.Objects; + +/** + * Describes a quota entity filter. + */ +public class QuotaFilter { + + // Matches all entities with the entity type specified. + private static final String MATCH_SPECIFIED = ""; + + private final String entityType; + private final String match; + + /** + * A filter to be applied. + * + * @param entityType the entity type the filter applies to + * @param match the string that's matched exactly + */ + private QuotaFilter(String entityType, String match) { + this.entityType = Objects.requireNonNull(entityType); + this.match = Objects.requireNonNull(match); + } + + /** + * Constructs and returns a quota filter that matches the provided entity + * name for the entity type exactly. + * + * @param entityType the entity type the filter applies to + * @param match the string that's matched exactly + */ + public static QuotaFilter matchExact(String entityType, String entityName) { + return new QuotaFilter(entityType, entityName); + } + + /** + * Constructs and returns a quota filter that matches the any entity name + * that's specified for the entity type. + * + * @param entityType the entity type the filter applies to + */ + public static QuotaFilter matchSpecified(String entityType) { + return new QuotaFilter(entityType, MATCH_SPECIFIED); + } + + /** + * @return the entity type the filter applies to + */ + public String entityType() { + return this.entityType; + } + + /** + * @return the string that's matched exactly + */ + public String match() { + return this.match; + } + + /** + * @return whether to match the exact entity name + */ + public boolean isMatchExact() { + return !isMatchSpecified(); + } + + /** + * @return whether to match all entities with the entity type specified + */ + public boolean isMatchSpecified() { + return this.match == MATCH_SPECIFIED; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + QuotaFilter that = (QuotaFilter) o; + return Objects.equals(entityType, that.entityType) && Objects.equals(match, that.match); + } + + @Override + public int hashCode() { + return Objects.hash(entityType, match); + } + + @Override + public String toString() { + return "QuotaFilter(entityType=" + entityType + ", match=" + match + ")"; + } +} diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java index 97bc72852e39f..f4085b833b21c 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java @@ -239,6 +239,10 @@ public static AbstractRequest parseRequest(ApiKeys apiKey, short apiVersion, Str return new ListPartitionReassignmentsRequest(struct, apiVersion); case OFFSET_DELETE: return new OffsetDeleteRequest(struct, apiVersion); + case DESCRIBE_CLIENT_QUOTAS: + return new DescribeClientQuotasRequest(struct, apiVersion); + case ALTER_CLIENT_QUOTAS: + return new AlterClientQuotasRequest(struct, apiVersion); default: throw new AssertionError(String.format("ApiKey %s is not currently handled in `parseRequest`, the " + "code should be updated to do so.", apiKey)); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java index 621b6a1829458..22175d625f0ce 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java @@ -176,6 +176,10 @@ public static AbstractResponse parseResponse(ApiKeys apiKey, Struct struct, shor return new ListPartitionReassignmentsResponse(struct, version); case OFFSET_DELETE: return new OffsetDeleteResponse(struct, version); + case DESCRIBE_CLIENT_QUOTAS: + return new DescribeClientQuotasResponse(struct, version); + case ALTER_CLIENT_QUOTAS: + return new AlterClientQuotasResponse(struct, version); default: throw new AssertionError(String.format("ApiKey %s is not currently handled in `parseResponse`, the " + "code should be updated to do so.", apiKey)); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasRequest.java new file mode 100644 index 0000000000000..a110c5d8b6152 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasRequest.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.common.requests; + +import org.apache.kafka.common.message.AlterClientQuotasRequestData; +import org.apache.kafka.common.message.AlterClientQuotasRequestData.EntityData; +import org.apache.kafka.common.message.AlterClientQuotasRequestData.EntryData; +import org.apache.kafka.common.message.AlterClientQuotasRequestData.OpData; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.quota.QuotaAlteration; +import org.apache.kafka.common.quota.QuotaEntity; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class AlterClientQuotasRequest extends AbstractRequest { + + public static class Builder extends AbstractRequest.Builder { + + private final AlterClientQuotasRequestData data; + + public Builder(Collection entries, boolean validateOnly) { + super(ApiKeys.ALTER_CLIENT_QUOTAS); + + List entryData = new ArrayList<>(entries.size()); + for (QuotaAlteration entry : entries) { + List entityData = new ArrayList<>(entry.entity().entries().size()); + for (Map.Entry entityEntries : entry.entity().entries().entrySet()) { + entityData.add(new EntityData() + .setEntityType(entityEntries.getKey()) + .setEntityName(entityEntries.getValue())); + } + + List opData = new ArrayList<>(entry.ops().size()); + for (QuotaAlteration.Op op : entry.ops()) { + opData.add(new OpData() + .setKey(op.key()) + .setValue(op.value() == null ? 0.0 : op.value()) + .setRemove(op.value() == null)); + } + + entryData.add(new EntryData() + .setEntity(entityData) + .setOps(opData)); + } + + this.data = new AlterClientQuotasRequestData() + .setEntries(entryData) + .setValidateOnly(validateOnly); + } + + @Override + public AlterClientQuotasRequest build(short version) { + return new AlterClientQuotasRequest(data, version); + } + + @Override + public String toString() { + return data.toString(); + } + } + + private final AlterClientQuotasRequestData data; + + public AlterClientQuotasRequest(AlterClientQuotasRequestData data, short version) { + super(ApiKeys.ALTER_CLIENT_QUOTAS, version); + this.data = data; + } + + public AlterClientQuotasRequest(Struct struct, short version) { + super(ApiKeys.ALTER_CLIENT_QUOTAS, version); + this.data = new AlterClientQuotasRequestData(struct, version); + } + + public Collection entries() { + List entries = new ArrayList<>(data.entries().size()); + for (EntryData entryData : data.entries()) { + Map entity = new HashMap<>(entryData.entity().size()); + for (EntityData entityData : entryData.entity()) { + entity.put(entityData.entityType(), entityData.entityName()); + } + + List ops = new ArrayList<>(entryData.ops().size()); + for (OpData opData : entryData.ops()) { + Double value = opData.remove() ? null : opData.value(); + ops.add(new QuotaAlteration.Op(opData.key(), value)); + } + + entries.add(new QuotaAlteration(new QuotaEntity(entity), ops)); + } + return entries; + } + + public boolean validateOnly() { + return data.validateOnly(); + } + + @Override + public AlterClientQuotasResponse getErrorResponse(int throttleTimeMs, Throwable e) { + ArrayList entities = new ArrayList<>(data.entries().size()); + for (EntryData entryData : data.entries()) { + Map entity = new HashMap<>(entryData.entity().size()); + for (EntityData entityData : entryData.entity()) { + entity.put(entityData.entityType(), entityData.entityName()); + } + entities.add(new QuotaEntity(entity)); + } + return new AlterClientQuotasResponse(entities, throttleTimeMs, e); + } + + @Override + protected Struct toStruct() { + return data.toStruct(version()); + } +} diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasResponse.java new file mode 100644 index 0000000000000..aca941df3276d --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasResponse.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.common.requests; + +import org.apache.kafka.common.internals.KafkaFutureImpl; +import org.apache.kafka.common.message.AlterClientQuotasResponseData; +import org.apache.kafka.common.message.AlterClientQuotasResponseData.EntityData; +import org.apache.kafka.common.message.AlterClientQuotasResponseData.EntryData; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.quota.QuotaEntity; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class AlterClientQuotasResponse extends AbstractResponse { + + private final AlterClientQuotasResponseData data; + + public AlterClientQuotasResponse(Map result, int throttleTimeMs) { + List entries = new ArrayList<>(result.size()); + for (Map.Entry entry : result.entrySet()) { + ApiError e = entry.getValue(); + entries.add(new EntryData() + .setErrorCode(e.error().code()) + .setErrorMessage(e.message()) + .setEntity(toEntityData(entry.getKey()))); + } + + this.data = new AlterClientQuotasResponseData() + .setThrottleTimeMs(throttleTimeMs) + .setEntries(entries); + } + + public AlterClientQuotasResponse(Collection entities, int throttleTimeMs, Throwable e) { + short errorCode = Errors.forException(e).code(); + String errorMessage = e.getMessage(); + + List entries = new ArrayList<>(entities.size()); + for (QuotaEntity entity : entities) { + entries.add(new EntryData() + .setErrorCode(errorCode) + .setErrorMessage(errorMessage) + .setEntity(toEntityData(entity))); + } + + this.data = new AlterClientQuotasResponseData() + .setThrottleTimeMs(throttleTimeMs) + .setEntries(entries); + } + + public AlterClientQuotasResponse(Struct struct, short version) { + this.data = new AlterClientQuotasResponseData(struct, version); + } + + public void complete(Map> futures) { + for (EntryData entryData : data.entries()) { + Map entityEntries = new HashMap<>(entryData.entity().size()); + for (EntityData entityData : entryData.entity()) { + entityEntries.put(entityData.entityType(), entityData.entityName()); + } + QuotaEntity entity = new QuotaEntity(entityEntries); + + KafkaFutureImpl future = futures.get(entity); + if (future == null) { + throw new IllegalArgumentException("Future map must contain entity " + entity); + } + + Errors error = Errors.forCode(entryData.errorCode()); + if (error == Errors.NONE) { + future.complete(null); + } else { + future.completeExceptionally(error.exception(entryData.errorMessage())); + } + } + } + + @Override + public int throttleTimeMs() { + return data.throttleTimeMs(); + } + + @Override + public Map errorCounts() { + Map counts = new HashMap<>(); + for (EntryData entry : data.entries()) { + Errors error = Errors.forCode(entry.errorCode()); + counts.put(error, counts.getOrDefault(error, 0) + 1); + } + return counts; + } + + @Override + protected Struct toStruct(short version) { + return data.toStruct(version); + } + + private static List toEntityData(QuotaEntity entity) { + List entityData = new ArrayList<>(entity.entries().size()); + for (Map.Entry entry : entity.entries().entrySet()) { + entityData.add(new AlterClientQuotasResponseData.EntityData() + .setEntityType(entry.getKey()) + .setEntityName(entry.getValue())); + } + return entityData; + } +} diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasRequest.java new file mode 100644 index 0000000000000..ef9e366744b3b --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasRequest.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.common.requests; + +import org.apache.kafka.common.message.DescribeClientQuotasRequestData; +import org.apache.kafka.common.message.DescribeClientQuotasRequestData.FilterData; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.quota.QuotaFilter; + +import java.util.ArrayList; +import java.util.List; +import java.util.Collection; + +public class DescribeClientQuotasRequest extends AbstractRequest { + // These values must not change. + private static final byte MATCH_TYPE_EXACT = 0; + private static final byte MATCH_TYPE_SPECIFIED = 1; + + public static class Builder extends AbstractRequest.Builder { + + private final DescribeClientQuotasRequestData data; + + public Builder(Collection filters, boolean includeUnspecifiedTypes) { + super(ApiKeys.DESCRIBE_CLIENT_QUOTAS); + + List filterData = new ArrayList<>(filters.size()); + for (QuotaFilter filter : filters) { + FilterData fd = new FilterData().setEntityType(filter.entityType()); + if (filter.isMatchSpecified()) { + fd.setMatchType(MATCH_TYPE_SPECIFIED); + fd.setMatch(null); + } else { + fd.setMatchType(MATCH_TYPE_EXACT); + fd.setMatch(filter.match()); + } + filterData.add(fd); + } + this.data = new DescribeClientQuotasRequestData() + .setFilters(filterData) + .setIncludeUnspecifiedTypes(includeUnspecifiedTypes); + } + + @Override + public DescribeClientQuotasRequest build(short version) { + return new DescribeClientQuotasRequest(data, version); + } + + @Override + public String toString() { + return data.toString(); + } + } + + private final DescribeClientQuotasRequestData data; + + public DescribeClientQuotasRequest(DescribeClientQuotasRequestData data, short version) { + super(ApiKeys.DESCRIBE_CLIENT_QUOTAS, version); + this.data = data; + } + + public DescribeClientQuotasRequest(Struct struct, short version) { + super(ApiKeys.DESCRIBE_CLIENT_QUOTAS, version); + this.data = new DescribeClientQuotasRequestData(struct, version); + } + + public Collection filters() { + List filters = new ArrayList<>(data.filters().size()); + for (FilterData filterData : data.filters()) { + QuotaFilter filter; + switch (filterData.matchType()) { + case MATCH_TYPE_EXACT: + filter = QuotaFilter.matchExact(filterData.entityType(), filterData.match()); + break; + case MATCH_TYPE_SPECIFIED: + filter = QuotaFilter.matchSpecified(filterData.entityType()); + break; + default: + throw new IllegalArgumentException("Unexpected match type: " + filterData.matchType()); + } + filters.add(filter); + } + return filters; + } + + public boolean includeUnspecifiedTypes() { + return data.includeUnspecifiedTypes(); + } + + @Override + public DescribeClientQuotasResponse getErrorResponse(int throttleTimeMs, Throwable e) { + return new DescribeClientQuotasResponse(throttleTimeMs, e); + } + + @Override + protected Struct toStruct() { + return data.toStruct(version()); + } +} diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasResponse.java new file mode 100644 index 0000000000000..63538c5ffbaf9 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasResponse.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.common.requests; + +import org.apache.kafka.common.internals.KafkaFutureImpl; +import org.apache.kafka.common.message.DescribeClientQuotasResponseData; +import org.apache.kafka.common.message.DescribeClientQuotasResponseData.EntityData; +import org.apache.kafka.common.message.DescribeClientQuotasResponseData.EntryData; +import org.apache.kafka.common.message.DescribeClientQuotasResponseData.ValueData; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.quota.QuotaEntity; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class DescribeClientQuotasResponse extends AbstractResponse { + + private final DescribeClientQuotasResponseData data; + + public DescribeClientQuotasResponse(Map> entities, int throttleTimeMs) { + List entries = new ArrayList<>(entities.size()); + for (Map.Entry> entry : entities.entrySet()) { + QuotaEntity quotaEntity = entry.getKey(); + List entityData = new ArrayList<>(quotaEntity.entries().size()); + for (Map.Entry entityEntry : quotaEntity.entries().entrySet()) { + entityData.add(new EntityData() + .setEntityType(entityEntry.getKey()) + .setEntityName(entityEntry.getValue())); + } + + Map quotaValues = entry.getValue(); + List valueData = new ArrayList<>(quotaValues.size()); + for (Map.Entry valuesEntry : entry.getValue().entrySet()) { + valueData.add(new ValueData() + .setKey(valuesEntry.getKey()) + .setValue(valuesEntry.getValue())); + } + + entries.add(new EntryData() + .setEntity(entityData) + .setValues(valueData)); + } + + this.data = new DescribeClientQuotasResponseData() + .setThrottleTimeMs(throttleTimeMs) + .setErrorCode((short) 0) + .setErrorMessage(null) + .setEntries(entries); + } + + public DescribeClientQuotasResponse(int throttleTimeMs, Throwable e) { + this.data = new DescribeClientQuotasResponseData() + .setThrottleTimeMs(throttleTimeMs) + .setErrorCode(Errors.forException(e).code()) + .setErrorMessage(e.getMessage()) + .setEntries(null); + } + + public DescribeClientQuotasResponse(Struct struct, short version) { + this.data = new DescribeClientQuotasResponseData(struct, version); + } + + public void complete(KafkaFutureImpl>> future) { + Errors error = Errors.forCode(data.errorCode()); + if (error != Errors.NONE) { + future.completeExceptionally(error.exception(data.errorMessage())); + return; + } + + Map> result = new HashMap<>(data.entries().size()); + for (EntryData entries : data.entries()) { + Map entity = new HashMap<>(entries.entity().size()); + for (EntityData entityData : entries.entity()) { + entity.put(entityData.entityType(), entityData.entityName()); + } + + Map values = new HashMap<>(entries.values().size()); + for (ValueData valueData : entries.values()) { + values.put(valueData.key(), valueData.value()); + } + + result.put(new QuotaEntity(entity), values); + } + future.complete(result); + } + + @Override + public int throttleTimeMs() { + return data.throttleTimeMs(); + } + + @Override + public Map errorCounts() { + return Collections.singletonMap(Errors.forCode(data.errorCode()), 1); + } + + @Override + protected Struct toStruct(short version) { + return data.toStruct(version); + } +} diff --git a/clients/src/main/resources/common/message/AlterClientQuotasRequest.json b/clients/src/main/resources/common/message/AlterClientQuotasRequest.json new file mode 100644 index 0000000000000..0d979b5dc4706 --- /dev/null +++ b/clients/src/main/resources/common/message/AlterClientQuotasRequest.json @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 49, + "type": "request", + "name": "AlterClientQuotasRequest", + "validVersions": "0", + "flexibleVersions": "none", + "fields": [ + { "name": "Entries", "type": "[]EntryData", "versions": "0+", + "about": "The quota configuration entries to alter.", "fields": [ + { "name": "Entity", "type": "[]EntityData", "versions": "0+", + "about": "The quota entity to alter.", "fields": [ + { "name": "EntityType", "type": "string", "versions": "0+", + "about": "The entity type." }, + { "name": "EntityName", "type": "string", "versions": "0+", + "about": "The name of the entity." } + ]}, + { "name": "Ops", "type": "[]OpData", "versions": "0+", + "about": "An individual quota configuration entry to alter.", "fields": [ + { "name": "Key", "type": "string", "versions": "0+", + "about": "The quota configuration key." }, + { "name": "Value", "type": "float64", "versions": "0+", + "about": "The value to set, otherwise ignored if the value is to be removed." }, + { "name": "Remove", "type": "bool", "versions": "0+", + "about": "Whether the quota configuration value should be removed, otherwise set." } + ]} + ]}, + { "name": "ValidateOnly", "type": "bool", "versions": "0+", + "about": "Whether the alteration should be validated, but not performed." } + ] +} diff --git a/clients/src/main/resources/common/message/AlterClientQuotasResponse.json b/clients/src/main/resources/common/message/AlterClientQuotasResponse.json new file mode 100644 index 0000000000000..48f1cf16b1e05 --- /dev/null +++ b/clients/src/main/resources/common/message/AlterClientQuotasResponse.json @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 49, + "type": "response", + "name": "AlterClientQuotasResponse", + "validVersions": "0", + "flexibleVersions": "none", + "fields": [ + { "name": "ThrottleTimeMs", "type": "int32", "versions": "0+", + "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." }, + { "name": "Entries", "type": "[]EntryData", "versions": "0+", + "about": "The quota configuration entries to alter.", "fields": [ + { "name": "ErrorCode", "type": "int16", "versions": "0+", + "about": "The error code, or `0` if the quota alteration succeeded." }, + { "name": "ErrorMessage", "type": "string", "versions": "0+", "nullableVersions": "0+", + "about": "The error message, or `null` if the quota alteration succeeded." }, + { "name": "Entity", "type": "[]EntityData", "versions": "0+", + "about": "The quota entity to alter.", "fields": [ + { "name": "EntityType", "type": "string", "versions": "0+", + "about": "The entity type." }, + { "name": "EntityName", "type": "string", "versions": "0+", + "about": "The name of the entity." } + ]} + ]} + ] +} diff --git a/clients/src/main/resources/common/message/DescribeClientQuotasRequest.json b/clients/src/main/resources/common/message/DescribeClientQuotasRequest.json new file mode 100644 index 0000000000000..3617b5714e5ab --- /dev/null +++ b/clients/src/main/resources/common/message/DescribeClientQuotasRequest.json @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 48, + "type": "request", + "name": "DescribeClientQuotasRequest", + "validVersions": "0", + "flexibleVersions": "none", + "fields": [ + { "name": "Filters", "type": "[]FilterData", "versions": "0+", + "about": "Filters to apply to quota entities.", "fields": [ + { "name": "EntityType", "type": "string", "versions": "0+", + "about": "The entity type that the filter applies to." }, + { "name": "MatchType", "type": "int8", "versions": "0+", + "about": "How to match the entity {0 = exact, 1 = specified}." }, + { "name": "Match", "type": "string", "versions": "0+", "nullableVersions": "0+", + "about": "The string to match against, or null if unused for the match type." } + ]}, + { "name": "IncludeUnspecifiedTypes", "type": "bool", "versions": "0+", + "about": "Whether to include entity types which are not specified in the filters." } + ] +} diff --git a/clients/src/main/resources/common/message/DescribeClientQuotasResponse.json b/clients/src/main/resources/common/message/DescribeClientQuotasResponse.json new file mode 100644 index 0000000000000..6ce1bb5fd4216 --- /dev/null +++ b/clients/src/main/resources/common/message/DescribeClientQuotasResponse.json @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +{ + "apiKey": 48, + "type": "response", + "name": "DescribeClientQuotasResponse", + "validVersions": "0", + "flexibleVersions": "none", + "fields": [ + { "name": "ThrottleTimeMs", "type": "int32", "versions": "0+", + "about": "The duration in milliseconds for which the request was throttled due to a quota violation, or zero if the request did not violate any quota." }, + { "name": "ErrorCode", "type": "int16", "versions": "0+", + "about": "The error code, or `0` if the quota description succeeded." }, + { "name": "ErrorMessage", "type": "string", "versions": "0+", "nullableVersions": "0+", + "about": "The error message, or `null` if the quota description succeeded." }, + { "name": "Entries", "type": "[]EntryData", "versions": "0+", "nullableVersions": "0+", + "about": "A result entry.", "fields": [ + { "name": "Entity", "type": "[]EntityData", "versions": "0+", + "about": "The quota entity description.", "fields": [ + { "name": "EntityType", "type": "string", "versions": "0+", + "about": "The entity type." }, + { "name": "EntityName", "type": "string", "versions": "0+", + "about": "The entity name." } + ]}, + { "name": "Values", "type": "[]ValueData", "versions": "0+", + "about": "The quota values for the entity.", "fields": [ + { "name": "Key", "type": "string", "versions": "0+", + "about": "The quota configuration key." }, + { "name": "Value", "type": "float64", "versions": "0+", + "about": "The quota configuration value." } + ]} + ]} + ] +} diff --git a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java index 28b46401e0b96..0e1eea144e574 100644 --- a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java @@ -88,6 +88,10 @@ import org.apache.kafka.common.message.OffsetDeleteResponseData.OffsetDeleteResponseTopic; import org.apache.kafka.common.message.OffsetDeleteResponseData.OffsetDeleteResponseTopicCollection; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.quota.QuotaAlteration; +import org.apache.kafka.common.quota.QuotaEntity; +import org.apache.kafka.common.quota.QuotaFilter; +import org.apache.kafka.common.requests.AlterClientQuotasResponse; import org.apache.kafka.common.requests.AlterPartitionReassignmentsResponse; import org.apache.kafka.common.requests.ApiError; import org.apache.kafka.common.requests.CreateAclsResponse; @@ -100,6 +104,7 @@ import org.apache.kafka.common.requests.DeleteTopicsRequest; import org.apache.kafka.common.requests.DeleteTopicsResponse; import org.apache.kafka.common.requests.DescribeAclsResponse; +import org.apache.kafka.common.requests.DescribeClientQuotasResponse; import org.apache.kafka.common.requests.DescribeConfigsResponse; import org.apache.kafka.common.requests.DescribeGroupsResponse; import org.apache.kafka.common.requests.ElectLeadersResponse; @@ -637,6 +642,7 @@ public void testAdminClientApisAuthenticationFailure() throws Exception { env.kafkaClient().createPendingAuthenticationError(cluster.nodes().get(0), TimeUnit.DAYS.toMillis(1)); callAdminClientApisAndExpectAnAuthenticationError(env); + callClientQuotasApisAndExpectAnAuthenticationError(env); } } @@ -695,6 +701,26 @@ private void callAdminClientApisAndExpectAnAuthenticationError(AdminClientUnitTe } } + private void callClientQuotasApisAndExpectAnAuthenticationError(AdminClientUnitTestEnv env) throws InterruptedException { + try { + env.adminClient().describeClientQuotas(Collections.emptyList()).entities().get(); + fail("Expected an authentication error."); + } catch (ExecutionException e) { + assertTrue("Expected an authentication error, but got " + Utils.stackTrace(e), + e.getCause() instanceof AuthenticationException); + } + + try { + QuotaEntity entity = new QuotaEntity(Collections.singletonMap(QuotaEntity.USER, "user")); + QuotaAlteration alteration = new QuotaAlteration(entity, asList(new QuotaAlteration.Op("consumer_byte_rate", 1000.0))); + env.adminClient().alterClientQuotas(asList(alteration)).all().get(); + fail("Expected an authentication error."); + } catch (ExecutionException e) { + assertTrue("Expected an authentication error, but got " + Utils.stackTrace(e), + e.getCause() instanceof AuthenticationException); + } + } + private static final AclBinding ACL1 = new AclBinding(new ResourcePattern(ResourceType.TOPIC, "mytopic3", PatternType.LITERAL), new AccessControlEntry("User:ANONYMOUS", "*", AclOperation.DESCRIBE, AclPermissionType.ALLOW)); private static final AclBinding ACL2 = new AclBinding(new ResourcePattern(ResourceType.TOPIC, "mytopic4", PatternType.LITERAL), @@ -2794,6 +2820,75 @@ public void testRequestTimeoutExceedingDefaultApiTimeout() throws Exception { } } + private QuotaEntity newQuotaEntity(String... args) { + assertTrue(args.length % 2 == 0); + + Map entityMap = new HashMap<>(args.length / 2); + for (int index = 0; index < args.length; index += 2) { + entityMap.put(args[index], args[index + 1]); + } + return new QuotaEntity(entityMap); + } + + @Test + public void testDescribeClientQuotas() throws Exception { + try (AdminClientUnitTestEnv env = mockClientEnv()) { + env.kafkaClient().setNodeApiVersions(NodeApiVersions.create()); + + final String customKey = "custom"; + final String customValue = "custom-value"; + + Map> responseData = new HashMap<>(); + QuotaEntity entity1 = newQuotaEntity(QuotaEntity.USER, "user-1", customKey, customValue); + QuotaEntity entity2 = newQuotaEntity(QuotaEntity.USER, "user-2", customKey, customValue); + responseData.put(entity1, Collections.singletonMap("consumer_byte_rate", 10000.0)); + responseData.put(entity2, Collections.singletonMap("producer_byte_rate", 20000.0)); + + env.kafkaClient().prepareResponse(new DescribeClientQuotasResponse(responseData, 0)); + + Collection filters = asList(QuotaFilter.matchExact(customKey, customValue)); + + DescribeClientQuotasResult result = env.adminClient().describeClientQuotas(filters); + Map> resultData = result.entities().get(); + assertEquals(resultData.size(), 2); + assertTrue(resultData.containsKey(entity1)); + Map config1 = resultData.get(entity1); + assertEquals(config1.size(), 1); + assertEquals(config1.get("consumer_byte_rate"), 10000.0, 1e-6); + assertTrue(resultData.containsKey(entity2)); + Map config2 = resultData.get(entity2); + assertEquals(config2.size(), 1); + assertEquals(config2.get("producer_byte_rate"), 20000.0, 1e-6); + } + } + + public void testAlterClientQuotas() throws Exception { + try (AdminClientUnitTestEnv env = mockClientEnv()) { + env.kafkaClient().setNodeApiVersions(NodeApiVersions.create()); + + QuotaEntity goodEntity = newQuotaEntity(QuotaEntity.USER, "user-1"); + QuotaEntity unauthorizedEntity = newQuotaEntity(QuotaEntity.USER, "user-0"); + QuotaEntity invalidEntity = newQuotaEntity("", "user-0"); + + Map responseData = new HashMap<>(2); + responseData.put(goodEntity, new ApiError(Errors.CLUSTER_AUTHORIZATION_FAILED, "Authorization failed")); + responseData.put(unauthorizedEntity, new ApiError(Errors.CLUSTER_AUTHORIZATION_FAILED, "Authorization failed")); + responseData.put(invalidEntity, new ApiError(Errors.INVALID_REQUEST, "Invalid quota entity")); + + env.kafkaClient().prepareResponse(new AlterClientQuotasResponse(responseData, 0)); + + List entries = new ArrayList<>(3); + entries.add(new QuotaAlteration(goodEntity, Collections.singleton(new QuotaAlteration.Op("consumer_byte_rate", 10000.0)))); + entries.add(new QuotaAlteration(unauthorizedEntity, Collections.singleton(new QuotaAlteration.Op("producer_byte_rate", 10000.0)))); + entries.add(new QuotaAlteration(invalidEntity, Collections.singleton(new QuotaAlteration.Op("producer_byte_rate", 100.0)))); + + AlterClientQuotasResult result = env.adminClient().alterClientQuotas(entries); + result.values().get(goodEntity); + TestUtils.assertFutureError(result.values().get(unauthorizedEntity), ClusterAuthorizationException.class); + TestUtils.assertFutureError(result.values().get(invalidEntity), InvalidRequestException.class); + } + } + private static MemberDescription convertToMemberDescriptions(DescribedGroupMember member, MemberAssignment assignment) { return new MemberDescription(member.memberId(), diff --git a/clients/src/test/java/org/apache/kafka/clients/admin/MockAdminClient.java b/clients/src/test/java/org/apache/kafka/clients/admin/MockAdminClient.java index 7cfd0510cc767..f14d7390e7e0f 100644 --- a/clients/src/test/java/org/apache/kafka/clients/admin/MockAdminClient.java +++ b/clients/src/test/java/org/apache/kafka/clients/admin/MockAdminClient.java @@ -33,6 +33,8 @@ import org.apache.kafka.common.errors.TopicExistsException; import org.apache.kafka.common.errors.UnknownTopicOrPartitionException; import org.apache.kafka.common.internals.KafkaFutureImpl; +import org.apache.kafka.common.quota.QuotaAlteration; +import org.apache.kafka.common.quota.QuotaFilter; import java.time.Duration; import java.util.ArrayList; @@ -458,6 +460,16 @@ public ListOffsetsResult listOffsets(Map topicPartit throw new UnsupportedOperationException("Not implement yet"); } + @Override + public DescribeClientQuotasResult describeClientQuotas(Collection filters, DescribeClientQuotasOptions options) { + throw new UnsupportedOperationException("Not implemented yet"); + } + + @Override + public AlterClientQuotasResult alterClientQuotas(Collection entries, AlterClientQuotasOptions options) { + throw new UnsupportedOperationException("Not implemented yet"); + } + @Override public void close(Duration timeout) {} diff --git a/core/src/main/scala/kafka/admin/ConfigCommand.scala b/core/src/main/scala/kafka/admin/ConfigCommand.scala index 562a91a45a46d..652aa387a5fa2 100644 --- a/core/src/main/scala/kafka/admin/ConfigCommand.scala +++ b/core/src/main/scala/kafka/admin/ConfigCommand.scala @@ -28,11 +28,12 @@ import kafka.utils.{CommandDefaultOptions, CommandLineUtils, Exit, PasswordEncod import kafka.utils.Implicits._ import kafka.zk.{AdminZkClient, KafkaZkClient} import org.apache.kafka.clients.CommonClientConfigs -import org.apache.kafka.clients.admin.{Admin, AlterConfigOp, AlterConfigsOptions, ConfigEntry, DescribeClusterOptions, DescribeConfigsOptions, ListTopicsOptions, Config => JConfig} +import org.apache.kafka.clients.admin.{Admin, AlterClientQuotasOptions, AlterConfigOp, AlterConfigsOptions, ConfigEntry, DescribeClusterOptions, DescribeConfigsOptions, ListTopicsOptions, Config => JConfig} import org.apache.kafka.common.config.ConfigResource import org.apache.kafka.common.config.types.Password import org.apache.kafka.common.errors.InvalidConfigurationException import org.apache.kafka.common.internals.Topic +import org.apache.kafka.common.quota.{QuotaAlteration, QuotaEntity, QuotaFilter} import org.apache.kafka.common.security.JaasUtils import org.apache.kafka.common.security.scram.internals.{ScramCredentialUtils, ScramFormatter, ScramMechanism} import org.apache.kafka.common.utils.{Sanitizer, Time, Utils} @@ -61,7 +62,8 @@ import scala.collection._ object ConfigCommand extends Config { val BrokerLoggerConfigType = "broker-loggers" - val BrokerSupportedConfigTypes = Seq(ConfigType.Topic, ConfigType.Broker, BrokerLoggerConfigType) + val BrokerSupportedConfigTypes = ConfigType.all :+ BrokerLoggerConfigType + val ZkSupportedConfigTypes = ConfigType.all val DefaultScramIterations = 4096 // Dynamic broker configs can only be updated using the new AdminClient once brokers have started // so that configs may be fully validated. Prior to starting brokers, updates may be performed using @@ -283,14 +285,8 @@ object ConfigCommand extends Config { props.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, opts.options.valueOf(opts.bootstrapServerOpt)) val adminClient = Admin.create(props) - if (opts.entityTypes.size != 1) - throw new IllegalArgumentException(s"Exactly one entity type (out of ${BrokerSupportedConfigTypes.mkString(",")}) must be specified with --bootstrap-server") - - val entityNames = opts.entityNames - if (entityNames.size > 1) - throw new IllegalArgumentException(s"At most one entity name must be specified with --bootstrap-server") - else if (opts.options.has(opts.alterOpt) && entityNames.size != 1) - throw new IllegalArgumentException(s"Exactly one entity name must be specified with --bootstrap-server for --alter") + if (opts.options.has(opts.alterOpt) && opts.entityTypes.size != opts.entityNames.size) + throw new IllegalArgumentException(s"An entity name must be specified for every entity type") try { if (opts.options.has(opts.alterOpt)) @@ -303,14 +299,17 @@ object ConfigCommand extends Config { } private[admin] def alterConfig(adminClient: Admin, opts: ConfigCommandOptions): Unit = { - val entityType = opts.entityTypes.head - val entityName = opts.entityNames.head - val configsToBeAdded = parseConfigsToBeAdded(opts).asScala.map { case (k, v) => (k, new ConfigEntry(k, v)) } + val entityTypes = opts.entityTypes + val entityNames = opts.entityNames + val entityTypeHead = entityTypes.head + val entityNameHead = entityNames.head + val configsToBeAddedMap = parseConfigsToBeAdded(opts).asScala + val configsToBeAdded = configsToBeAddedMap.map { case (k, v) => (k, new ConfigEntry(k, v)) } val configsToBeDeleted = parseConfigsToBeDeleted(opts) - entityType match { + entityTypeHead match { case ConfigType.Topic => - val oldConfig = getConfig(adminClient, entityType, entityName, includeSynonyms = false, describeAll = false) + val oldConfig = getGeneralConfig(adminClient, entityTypeHead, entityNameHead, includeSynonyms = false, describeAll = false) .map { entry => (entry.name, entry) }.toMap // fail the command if any of the configs to be deleted does not exist @@ -318,7 +317,7 @@ object ConfigCommand extends Config { if (invalidConfigs.nonEmpty) throw new InvalidConfigurationException(s"Invalid config(s): ${invalidConfigs.mkString(",")}") - val configResource = new ConfigResource(ConfigResource.Type.TOPIC, entityName) + val configResource = new ConfigResource(ConfigResource.Type.TOPIC, entityNameHead) val alterOptions = new AlterConfigsOptions().timeoutMs(30000).validateOnly(false) val alterEntries = (configsToBeAdded.values.map(new AlterConfigOp(_, AlterConfigOp.OpType.SET)) ++ configsToBeDeleted.map { k => new AlterConfigOp(new ConfigEntry(k, ""), AlterConfigOp.OpType.DELETE) } @@ -326,7 +325,7 @@ object ConfigCommand extends Config { adminClient.incrementalAlterConfigs(Map(configResource -> alterEntries).asJava, alterOptions).all().get(60, TimeUnit.SECONDS) case ConfigType.Broker => - val oldConfig = getConfig(adminClient, entityType, entityName, includeSynonyms = false, describeAll = false) + val oldConfig = getGeneralConfig(adminClient, entityTypeHead, entityNameHead, includeSynonyms = false, describeAll = false) .map { entry => (entry.name, entry) }.toMap // fail the command if any of the configs to be deleted does not exist @@ -340,38 +339,81 @@ object ConfigCommand extends Config { throw new InvalidConfigurationException(s"All sensitive broker config entries must be specified for --alter, missing entries: ${sensitiveEntries.keySet}") val newConfig = new JConfig(newEntries.asJava.values) - val configResource = new ConfigResource(ConfigResource.Type.BROKER, entityName) + val configResource = new ConfigResource(ConfigResource.Type.BROKER, entityNameHead) val alterOptions = new AlterConfigsOptions().timeoutMs(30000).validateOnly(false) adminClient.alterConfigs(Map(configResource -> newConfig).asJava, alterOptions).all().get(60, TimeUnit.SECONDS) case BrokerLoggerConfigType => - val validLoggers = getConfig(adminClient, entityType, entityName, includeSynonyms = true, describeAll = false).map(_.name) + val validLoggers = getGeneralConfig(adminClient, entityTypeHead, entityNameHead, includeSynonyms = true, describeAll = false).map(_.name) // fail the command if any of the configured broker loggers do not exist val invalidBrokerLoggers = configsToBeDeleted.filterNot(validLoggers.contains) ++ configsToBeAdded.keys.filterNot(validLoggers.contains) if (invalidBrokerLoggers.nonEmpty) throw new InvalidConfigurationException(s"Invalid broker logger(s): ${invalidBrokerLoggers.mkString(",")}") - val configResource = new ConfigResource(ConfigResource.Type.BROKER_LOGGER, entityName) + val configResource = new ConfigResource(ConfigResource.Type.BROKER_LOGGER, entityNameHead) val alterOptions = new AlterConfigsOptions().timeoutMs(30000).validateOnly(false) val alterLogLevelEntries = (configsToBeAdded.values.map(new AlterConfigOp(_, AlterConfigOp.OpType.SET)) ++ configsToBeDeleted.map { k => new AlterConfigOp(new ConfigEntry(k, ""), AlterConfigOp.OpType.DELETE) } ).asJavaCollection adminClient.incrementalAlterConfigs(Map(configResource -> alterLogLevelEntries).asJava, alterOptions).all().get(60, TimeUnit.SECONDS) - case _ => throw new IllegalArgumentException(s"Unsupported entity type: $entityType") + case ConfigType.User => + case ConfigType.Client => + val oldConfig: Map[String, java.lang.Double] = getClientQuotasConfig(adminClient, entityTypes, entityNames) + + val invalidConfigs = configsToBeDeleted.filterNot(oldConfig.contains) + if (invalidConfigs.nonEmpty) + throw new InvalidConfigurationException(s"Invalid config(s): ${invalidConfigs.mkString(",")}") + + val entity = new QuotaEntity(opts.entityTypes.map { entType => + entType match { + case ConfigType.User => QuotaEntity.USER + case ConfigType.Client => QuotaEntity.CLIENT_ID + case _ => throw new IllegalArgumentException(s"Unexpected entity type: ${entType}") + } + }.zip(opts.entityNames).toMap.asJava) + + val alterOptions = new AlterClientQuotasOptions().validateOnly(false) + val alterOps = (configsToBeAddedMap.map { case (key, value) => + val doubleValue = try value.toDouble catch { + case _: NumberFormatException => + throw new IllegalArgumentException(s"Cannot parse quota configuration value for ${key}: ${value}") + } + new QuotaAlteration.Op(key, doubleValue) + } ++ configsToBeDeleted.map(key => new QuotaAlteration.Op(key, null))).asJavaCollection + + adminClient.alterClientQuotas(Collections.singleton(new QuotaAlteration(entity, alterOps)), alterOptions) + .all().get(60, TimeUnit.SECONDS) + + case _ => throw new IllegalArgumentException(s"Unsupported entity type: $entityTypeHead") } - if (entityName.nonEmpty) - println(s"Completed updating config for ${entityType.dropRight(1)} $entityName.") + if (entityNameHead.nonEmpty) + println(s"Completed updating config for ${entityTypeHead.dropRight(1)} $entityNameHead.") else - println(s"Completed updating default config for $entityType in the cluster.") + println(s"Completed updating default config for $entityTypeHead in the cluster.") } +<<<<<<< HEAD private[admin] def describeConfig(adminClient: Admin, opts: ConfigCommandOptions): Unit = { val entityType = opts.entityTypes.head val entityName = opts.entityNames.headOption +======= + private def describeConfig(adminClient: Admin, opts: ConfigCommandOptions): Unit = { + val entityTypes = opts.entityTypes + val entityNames = opts.entityNames +>>>>>>> KIP-546 (1/2): Implements describeClientQuotas() and alterClientQuotas(). val describeAll = opts.options.has(opts.allOpt) + entityTypes.head match { + case ConfigType.Topic | ConfigType.Broker | BrokerLoggerConfigType => + describeGeneralConfig(adminClient, entityTypes.head, entityNames.headOption, describeAll) + case ConfigType.User | ConfigType.Client => + describeClientQuotasConfig(adminClient, entityTypes, entityNames) + } + } + + private def describeGeneralConfig(adminClient: Admin, entityType: String, entityName: Option[String], describeAll: Boolean): Unit = { val entities = entityName .map(name => List(name)) .getOrElse(entityType match { @@ -389,14 +431,14 @@ object ConfigCommand extends Config { val configSourceStr = if (describeAll) "All" else "Dynamic" println(s"$configSourceStr configs for ${entityType.dropRight(1)} $entity are:") } - getConfig(adminClient, entityType, entity, includeSynonyms = true, describeAll).foreach { entry => + getGeneralConfig(adminClient, entityType, entity, includeSynonyms = true, describeAll).foreach { entry => val synonyms = entry.synonyms.asScala.map(synonym => s"${synonym.source}:${synonym.name}=${synonym.value}").mkString(", ") println(s" ${entry.name}=${entry.value} sensitive=${entry.isSensitive} synonyms={$synonyms}") } } } - private def getConfig(adminClient: Admin, entityType: String, entityName: String, includeSynonyms: Boolean, describeAll: Boolean) = { + private def getGeneralConfig(adminClient: Admin, entityType: String, entityName: String, includeSynonyms: Boolean, describeAll: Boolean) = { def validateBrokerId(): Unit = try entityName.toInt catch { case _: NumberFormatException => throw new IllegalArgumentException(s"The entity name for $entityType must be a valid integer broker id, found: $entityName") @@ -436,6 +478,37 @@ object ConfigCommand extends Config { }).toSeq } + private def describeClientQuotasConfig(adminClient: Admin, entityTypes: List[String], entityNames: List[String]) = { + getAllClientQuotasConfigs(adminClient, entityTypes, entityNames).foreach { case (entity, entries) => + val entityEntries = entity.entries.asScala + val entityStr = (entityEntries.get(QuotaEntity.USER).map(u => s"user-principal '${u}'") ++ + entityEntries.get(QuotaEntity.CLIENT_ID).map(c => s"client-id '${c}'")).mkString(", ") + val entriesStr = entries.asScala.map(e => s"${e._1}=${e._2}").mkString(", ") + println(s"Configs for ${entityStr} are ${entriesStr}") + } + } + + private def getClientQuotasConfig(adminClient: Admin, entityTypes: List[String], entityNames: List[String]): Map[String, java.lang.Double] = { + if (entityTypes.size != entityNames.size) + throw new IllegalArgumentException("Exactly one entity name must be specified for every entity type") + getAllClientQuotasConfigs(adminClient, entityTypes, entityNames).headOption.map(_._2.asScala).getOrElse(Map.empty) + } + + private def getAllClientQuotasConfigs(adminClient: Admin, entityTypes: List[String], entityNames: List[String]) = { + val filters = entityTypes.map(Some(_)).zipAll(entityNames.map(Some(_)), None, None).map { case (entityTypeOpt, entityNameOpt) => + val entityType = entityTypeOpt match { + case Some(ConfigType.User) => QuotaEntity.USER + case Some(ConfigType.Client) => QuotaEntity.CLIENT_ID + case Some(_) => throw new IllegalArgumentException(s"Unexpected entity type ${entityTypeOpt.get}") + case None => throw new IllegalArgumentException("More entity names specified than entity types") + } + entityNameOpt.map(QuotaFilter.matchExact(entityType, _)).getOrElse(QuotaFilter.matchSpecified(entityType)) + } + + val options = new DescribeClientQuotasOptions().includeUnspecifiedTypes(false) + adminClient.describeClientQuotas(filters.asJavaCollection, options).entities.get(30, TimeUnit.SECONDS).asScala + } + case class Entity(entityType: String, sanitizedName: Option[String]) { val entityPath = sanitizedName match { case Some(n) => entityType + "/" + n @@ -651,7 +724,7 @@ object ConfigCommand extends Config { val (allowedEntityTypes, connectOptString) = if (options.has(bootstrapServerOpt)) (BrokerSupportedConfigTypes, "--bootstrap-server") else - (ConfigType.all, "--zookeeper") + (ZkSupportedConfigTypes, "--zookeeper") entityTypeVals.foreach(entityTypeVal => if (!allowedEntityTypes.contains(entityTypeVal)) @@ -687,9 +760,6 @@ object ConfigCommand extends Config { } } - if (entityTypeVals.contains(ConfigType.Client) || entityTypeVals.contains(ConfigType.User)) - CommandLineUtils.checkRequiredArgs(parser, options, zkConnectOpt) - if (options.has(describeOpt) && entityTypeVals.contains(BrokerLoggerConfigType) && !hasEntityName) throw new IllegalArgumentException(s"an entity name must be specified with --describe of ${entityTypeVals.mkString(",")}") diff --git a/core/src/main/scala/kafka/server/AdminManager.scala b/core/src/main/scala/kafka/server/AdminManager.scala index 43f6e466b515c..46f37b749b3f9 100644 --- a/core/src/main/scala/kafka/server/AdminManager.scala +++ b/core/src/main/scala/kafka/server/AdminManager.scala @@ -36,12 +36,14 @@ import org.apache.kafka.common.message.CreateTopicsRequestData.CreatableTopic import org.apache.kafka.common.message.CreateTopicsResponseData.{CreatableTopicConfigs, CreatableTopicResult} import org.apache.kafka.common.metrics.Metrics import org.apache.kafka.common.network.ListenerName +import org.apache.kafka.server.policy.{AlterConfigPolicy, CreateTopicPolicy} +import org.apache.kafka.server.policy.CreateTopicPolicy.RequestMetadata import org.apache.kafka.common.protocol.Errors +import org.apache.kafka.common.quota.{QuotaAlteration, QuotaEntity, QuotaFilter} import org.apache.kafka.common.requests.CreateTopicsRequest._ import org.apache.kafka.common.requests.DescribeConfigsResponse.ConfigSource import org.apache.kafka.common.requests.{AlterConfigsRequest, ApiError, DescribeConfigsResponse} -import org.apache.kafka.server.policy.{AlterConfigPolicy, CreateTopicPolicy} -import org.apache.kafka.server.policy.CreateTopicPolicy.RequestMetadata +import org.apache.kafka.common.utils.Sanitizer import scala.collection.{Map, mutable, _} import scala.collection.JavaConverters._ @@ -700,4 +702,176 @@ class AdminManager(val config: KafkaConfig, val readOnly = !DynamicBrokerConfig.AllDynamicConfigs.contains(name) new DescribeConfigsResponse.ConfigEntry(name, valueAsString, source, isSensitive, readOnly, synonyms.asJava) } + + private def entityToSanitizedUserClientId(entity: QuotaEntity): (Option[String], Option[String]) = { + if (entity.entries.isEmpty) + throw new InvalidRequestException("Invalid empty quota entity") + + var user: Option[String] = None + var clientId: Option[String] = None + entity.entries().asScala.foreach { case (entityType, entityName) => + val sanitizedEntityName = Some(Sanitizer.sanitize(entityName)) + entityType match { + case QuotaEntity.USER => user = sanitizedEntityName + case QuotaEntity.CLIENT_ID => clientId = sanitizedEntityName + case _ => throw new InvalidRequestException(s"Unhandled quota entity type: ${entityType}") + } + if (entityName == "") + throw new InvalidRequestException(s"Empty ${entityType} not supported") + } + (user, clientId) + } + + private def userClientIdToEntity(user: Option[String], clientId: Option[String]): QuotaEntity = { + new QuotaEntity((user.map(u => QuotaEntity.USER -> u) ++ clientId.map(c => QuotaEntity.CLIENT_ID -> c)).toMap.asJava) + } + + def describeClientQuotas(filters: Seq[QuotaFilter], includeUnspecifiedTypes: Boolean): Map[QuotaEntity, Map[String, Double]] = { + var userFilter: Option[QuotaFilter] = None + var clientIdFilter: Option[QuotaFilter] = None + var otherFilter: Boolean = false + filters.foreach { filter => + filter.entityType() match { + case QuotaEntity.USER => + if (userFilter.isDefined) + throw new InvalidRequestException(s"Duplicate user filter entity type"); + userFilter = Some(filter) + case QuotaEntity.CLIENT_ID => + if (clientIdFilter.isDefined) + throw new InvalidRequestException(s"Duplicate client filter entity type"); + clientIdFilter = Some(filter) + case "" => + throw new InvalidRequestException(s"Unexpected empty filter entity type") + case _ => + // Supplying other entity types is not yet supported. + otherFilter = true + } + } + if (otherFilter) + Map.empty + else + handleDescribeClientQuotas(userFilter, clientIdFilter, includeUnspecifiedTypes) + } + + def handleDescribeClientQuotas(userFilter: Option[QuotaFilter], clientIdFilter: Option[QuotaFilter], includeUnspecifiedTypes: Boolean) = { + def wantExact(filter: Option[QuotaFilter]): Boolean = filter.map(_.isMatchExact).getOrElse(false) + def wantExcluded(filter: Option[QuotaFilter]): Boolean = filter.map(_ => false).getOrElse(!includeUnspecifiedTypes) + def sanitized(filter: Option[QuotaFilter]): String = filter.map(f => Sanitizer.sanitize(f.`match`)).getOrElse("") + + val sanitizedUser = sanitized(userFilter) + val exactUser = wantExact(userFilter) + val excludeUser = wantExcluded(userFilter) + + val sanitizedClientId = sanitized(clientIdFilter) + val exactClientId = wantExact(clientIdFilter) + val excludeClientId = wantExcluded(clientIdFilter) + + val userEntries = if (exactUser && excludeClientId) + Map(((Some(userFilter.get.`match`), None) -> adminZkClient.fetchEntityConfig(ConfigType.User, sanitizedUser))) + else if (!excludeUser && !exactClientId) + adminZkClient.fetchAllEntityConfigs(ConfigType.User).map { case (name, props) => + ((Some(Sanitizer.desanitize(name)), None) -> props) + } + else + Map.empty + + val clientIdEntries = if (excludeUser && exactClientId) + Map(((None, Some(clientIdFilter.get.`match`)) -> adminZkClient.fetchEntityConfig(ConfigType.Client, sanitizedClientId))) + else if (!exactUser && !excludeClientId) + adminZkClient.fetchAllEntityConfigs(ConfigType.Client).map { case (name, props) => + ((None, Some(Sanitizer.desanitize(name))) -> props) + } + else + Map.empty + + val bothEntries = if (exactUser && exactClientId) + Map(((Some(userFilter.get.`match`), Some(clientIdFilter.get.`match`)) -> + adminZkClient.fetchEntityConfig(ConfigType.User, s"${sanitizedUser}/clients/${sanitizedClientId}"))) + else if (!excludeUser && !excludeClientId) + adminZkClient.fetchAllChildEntityConfigs(ConfigType.User, ConfigType.Client).map { case (name, props) => + val components = name.split("/") + if (components.size != 3 || components(1) != "clients") + throw new IllegalArgumentException(s"Unexpected config path: ${name}") + ((Some(Sanitizer.desanitize(components(0))), Some(Sanitizer.desanitize(components(2)))) -> props) + } + else + Map.empty + + def matches(nameFilter: Option[QuotaFilter], name: Option[String]): Boolean = nameFilter match { + case Some(filter) => + if (filter.isMatchExact()) + name.map(_ == filter.`match`).getOrElse(false) + else if (filter.isMatchSpecified()) + name.isDefined + else + throw new IllegalStateException(s"Unexected quota filter type") + case None => + includeUnspecifiedTypes || !name.isDefined + } + + def fromProps(props: Properties): Map[String, Double] = { + props.asScala.map { case (key, value) => + val doubleValue = try value.toDouble catch { + case _: NumberFormatException => + throw new IllegalStateException(s"Unexpected quota configuration value: ${key} -> ${value}") + } + (key -> doubleValue) + } + } + + (userEntries ++ clientIdEntries ++ bothEntries).map { case ((u, c), p) => + if (!p.isEmpty && matches(userFilter, u) && matches(clientIdFilter, c)) + Some((userClientIdToEntity(u, c) -> fromProps(p))) + else + None + }.flatten.toMap + } + + def alterClientQuotas(entries: Seq[QuotaAlteration], validateOnly: Boolean): Map[QuotaEntity, ApiError] = { + def alterEntityQuotas(entity: QuotaEntity, ops: Iterable[QuotaAlteration.Op]): Unit = { + val (path, configType, configKeys) = entityToSanitizedUserClientId(entity) match { + case (Some(user), Some(clientId)) => (user + "/clients/" + clientId, ConfigType.User, DynamicConfig.User.configKeys) + case (Some(user), None) => (user, ConfigType.User, DynamicConfig.User.configKeys) + case (None, Some(clientId)) => (clientId, ConfigType.Client, DynamicConfig.Client.configKeys) + case _ => throw new InvalidRequestException("Invalid empty quota entity") + } + + val props = adminZkClient.fetchEntityConfig(configType, path) + ops.foreach { op => + op.value match { + case null => + props.remove(op.key) + case value => configKeys.get(op.key) match { + case null => + throw new InvalidRequestException(s"Invalid configuration key ${op.key}") + case key => key.`type` match { + case ConfigDef.Type.DOUBLE => + props.setProperty(op.key, value.toString) + case ConfigDef.Type.LONG => + val epsilon = 1e-6 + val longValue = (value + epsilon).toLong + if ((longValue.toDouble - value).abs > epsilon) + throw new InvalidRequestException(s"Configuration ${op.key} must be a Long value") + props.setProperty(op.key, longValue.toString) + case _ => + throw new IllegalStateException(s"Unexpected config type ${key.`type`}") + } + } + } + } + if (!validateOnly) + adminZkClient.changeConfigs(configType, path, props) + } + entries.map { entry => + val apiError = try { + alterEntityQuotas(entry.entity, entry.ops.asScala) + ApiError.NONE + } catch { + case e: Throwable => + info(s"Error encountered while updating client quotas", e) + ApiError.fromThrowable(e) + } + (entry.entity -> apiError) + }.toMap + } } diff --git a/core/src/main/scala/kafka/server/DynamicConfig.scala b/core/src/main/scala/kafka/server/DynamicConfig.scala index e5974d3642dc8..abea5dee821e5 100644 --- a/core/src/main/scala/kafka/server/DynamicConfig.scala +++ b/core/src/main/scala/kafka/server/DynamicConfig.scala @@ -89,6 +89,8 @@ object DynamicConfig { .define(ConsumerByteRateOverrideProp, LONG, DefaultConsumerOverride, MEDIUM, ConsumerOverrideDoc) .define(RequestPercentageOverrideProp, DOUBLE, DefaultRequestOverride, MEDIUM, RequestOverrideDoc) + def configKeys = clientConfigs.configKeys + def names = clientConfigs.names def validate(props: Properties) = DynamicConfig.validate(clientConfigs, props, customPropsAllowed = false) @@ -102,6 +104,8 @@ object DynamicConfig { .define(Client.ConsumerByteRateOverrideProp, LONG, Client.DefaultConsumerOverride, MEDIUM, Client.ConsumerOverrideDoc) .define(Client.RequestPercentageOverrideProp, DOUBLE, Client.DefaultRequestOverride, MEDIUM, Client.RequestOverrideDoc) + def configKeys = userConfigs.configKeys + def names = userConfigs.names def validate(props: Properties) = DynamicConfig.validate(userConfigs, props, customPropsAllowed = false) diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala index 8a14cd79c9f1b..47ca09c60bb0b 100644 --- a/core/src/main/scala/kafka/server/KafkaApis.scala +++ b/core/src/main/scala/kafka/server/KafkaApis.scala @@ -173,6 +173,8 @@ class KafkaApis(val requestChannel: RequestChannel, case ApiKeys.ALTER_PARTITION_REASSIGNMENTS => handleAlterPartitionReassignmentsRequest(request) case ApiKeys.LIST_PARTITION_REASSIGNMENTS => handleListPartitionReassignmentsRequest(request) case ApiKeys.OFFSET_DELETE => handleOffsetDeleteRequest(request) + case ApiKeys.DESCRIBE_CLIENT_QUOTAS => handleDescribeClientQuotasRequest(request) + case ApiKeys.ALTER_CLIENT_QUOTAS => handleAlterClientQuotasRequest(request) } } catch { case e: FatalExitError => throw e @@ -2785,6 +2787,34 @@ class KafkaApis(val requestChannel: RequestChannel, } } + def handleDescribeClientQuotasRequest(request: RequestChannel.Request): Unit = { + val describeClientQuotasRequest = request.body[DescribeClientQuotasRequest] + + if (authorize(request, DESCRIBE_CONFIGS, CLUSTER, CLUSTER_NAME)) { + val result = adminManager.describeClientQuotas(describeClientQuotasRequest.filters().asScala.toSeq, + describeClientQuotasRequest.includeUnspecifiedTypes()).mapValues(_.mapValues(Double.box).asJava).asJava + sendResponseMaybeThrottle(request, requestThrottleMs => + new DescribeClientQuotasResponse(result, requestThrottleMs)) + } else { + sendResponseMaybeThrottle(request, requestThrottleMs => + describeClientQuotasRequest.getErrorResponse(requestThrottleMs, Errors.CLUSTER_AUTHORIZATION_FAILED.exception)) + } + } + + def handleAlterClientQuotasRequest(request: RequestChannel.Request): Unit = { + val alterClientQuotasRequest = request.body[AlterClientQuotasRequest] + + if (authorize(request, ALTER_CONFIGS, CLUSTER, CLUSTER_NAME)) { + val result = adminManager.alterClientQuotas(alterClientQuotasRequest.entries().asScala.toSeq, + alterClientQuotasRequest.validateOnly()).asJava + sendResponseMaybeThrottle(request, requestThrottleMs => + new AlterClientQuotasResponse(result, requestThrottleMs)) + } else { + sendResponseMaybeThrottle(request, requestThrottleMs => + alterClientQuotasRequest.getErrorResponse(requestThrottleMs, Errors.CLUSTER_AUTHORIZATION_FAILED.exception)) + } + } + private def authorize(request: RequestChannel.Request, operation: AclOperation, resourceType: ResourceType, diff --git a/core/src/test/scala/unit/kafka/admin/ConfigCommandTest.scala b/core/src/test/scala/unit/kafka/admin/ConfigCommandTest.scala index e938a6d859cb9..e2dc0170a3028 100644 --- a/core/src/test/scala/unit/kafka/admin/ConfigCommandTest.scala +++ b/core/src/test/scala/unit/kafka/admin/ConfigCommandTest.scala @@ -31,6 +31,7 @@ import org.apache.kafka.common.internals.KafkaFutureImpl import org.apache.kafka.common.Node import org.apache.kafka.common.errors.InvalidConfigurationException import org.apache.kafka.common.network.ListenerName +import org.apache.kafka.common.quota.{QuotaAlteration, QuotaEntity, QuotaFilter} import org.apache.kafka.common.security.auth.SecurityProtocol import org.apache.kafka.common.security.scram.internals.ScramCredentialUtils import org.apache.kafka.common.utils.Sanitizer @@ -91,9 +92,19 @@ class ConfigCommandTest extends ZooKeeperTestHarness with Logging { testArgumentParse("clients", zkConfig = true) } + @Test + def shouldParseArgumentsForClientsEntityType(): Unit = { + testArgumentParse("clients", zkConfig = false) + } + @Test def shouldParseArgumentsForUsersEntityTypeUsingZookeeper(): Unit = { - testArgumentParse("clients", zkConfig = true) + testArgumentParse("users", zkConfig = true) + } + + @Test + def shouldParseArgumentsForUsersEntityType(): Unit = { + testArgumentParse("users", zkConfig = false) } @Test @@ -215,10 +226,14 @@ class ConfigCommandTest extends ZooKeeperTestHarness with Logging { assertTrue(addedProps2.getProperty("f").isEmpty) } - @Test - def testOptionEntityTypeNames(): Unit = { + def doTestOptionEntityTypeNames(zkConfig: Boolean): Unit = { + val connectOpts = if (zkConfig) + ("--zookeeper", zkConnect) + else + ("--bootstrap-server", "localhost:9092") + def testExpectedEntityTypeNames(expectedTypes: List[String], expectedNames: List[String], args: String*): Unit = { - val createOpts = new ConfigCommandOptions(Array("--zookeeper", zkConnect, "--describe") ++ args) + val createOpts = new ConfigCommandOptions(Array(connectOpts._1, connectOpts._2, "--describe") ++ args) createOpts.checkArgs() assertEquals(createOpts.entityTypes, expectedTypes) assertEquals(createOpts.entityNames, expectedNames) @@ -243,6 +258,16 @@ class ConfigCommandTest extends ZooKeeperTestHarness with Logging { testExpectedEntityTypeNames(List(ConfigType.Broker), List.empty, "--entity-type", "brokers") } + @Test + def testOptionEntityTypeNamesUsingZookeeper(): Unit = { + doTestOptionEntityTypeNames(zkConfig = true) + } + + @Test + def testOptionEntityTypeNames(): Unit = { + doTestOptionEntityTypeNames(zkConfig = false) + } + @Test(expected = classOf[IllegalArgumentException]) def shouldFailIfUnrecognisedEntityTypeUsingZookeeper(): Unit = { val createOpts = new ConfigCommandOptions(Array("--zookeeper", zkConnect, @@ -292,6 +317,13 @@ class ConfigCommandTest extends ZooKeeperTestHarness with Logging { createOpts.checkArgs() } + @Test(expected = classOf[IllegalArgumentException]) + def shouldFailIfMixedEntityTypeFlags(): Unit = { + val createOpts = new ConfigCommandOptions(Array("--bootstrap-server", "localhost:9092", + "--entity-name", "A", "--entity-type", "users", "--client", "B", "--describe")) + createOpts.checkArgs() + } + @Test def shouldAddClientConfigUsingZookeeper(): Unit = { val createOpts = new ConfigCommandOptions(Array("--zookeeper", zkConnect, @@ -311,6 +343,66 @@ class ConfigCommandTest extends ZooKeeperTestHarness with Logging { ConfigCommand.alterConfigWithZk(null, createOpts, new TestAdminZkClient(zkClient)) } + @Test + def shouldAddClientConfig(): Unit = { + val createOpts = new ConfigCommandOptions(Array("--bootstrap-server", "localhost:9092", + "--entity-name", "my-client-id", + "--entity-type", "clients", + "--alter", + "--add-config", "consumer_byte_rate=20000,producer_byte_rate=10000", + "--delete-config", "request_percentage")) + + val entity = new QuotaEntity(Map((QuotaEntity.CLIENT_ID -> "my-client-id")).asJava) + + var describedConfigs = false + val describeFuture = new KafkaFutureImpl[util.Map[QuotaEntity, util.Map[String, java.lang.Double]]] + describeFuture.complete(Map((entity -> Map(("request_percentage" -> Double.box(50.0))).asJava)).asJava) + val describeResult: DescribeClientQuotasResult = EasyMock.createNiceMock(classOf[DescribeClientQuotasResult]) + EasyMock.expect(describeResult.entities()).andReturn(describeFuture) + + var alteredConfigs = false + val alterFuture = new KafkaFutureImpl[Void] + alterFuture.complete(null) + val alterResult: AlterClientQuotasResult = EasyMock.createNiceMock(classOf[AlterClientQuotasResult]) + EasyMock.expect(alterResult.all()).andReturn(alterFuture) + + val node = new Node(1, "localhost", 9092) + val mockAdminClient = new MockAdminClient(util.Collections.singletonList(node), node) { + override def describeClientQuotas(filters: util.Collection[QuotaFilter], options: DescribeClientQuotasOptions): DescribeClientQuotasResult = { + assertEquals(1, filters.size) + val filter = filters.asScala.head + assertTrue(filter.isMatchExact) + assertEquals(QuotaEntity.CLIENT_ID, filter.entityType) + assertEquals("my-client-id", filter.`match`) + assertFalse(options.includeUnspecifiedTypes) + describedConfigs = true + describeResult + } + + override def alterClientQuotas(entries: util.Collection[QuotaAlteration], options: AlterClientQuotasOptions): AlterClientQuotasResult = { + assertFalse(options.validateOnly) + assertEquals(1, entries.size) + val alteration = entries.asScala.head + assertEquals(entity, alteration.entity) + val ops = alteration.ops.asScala + assertEquals(3, ops.size) + val expectedOps = Set( + new QuotaAlteration.Op("consumer_byte_rate", Double.box(20000)), + new QuotaAlteration.Op("producer_byte_rate", Double.box(10000)), + new QuotaAlteration.Op("request_percentage", null) + ) + assertEquals(expectedOps, ops.toSet) + alteredConfigs = true + alterResult + } + } + EasyMock.replay(alterResult, describeResult) + ConfigCommand.alterConfig(mockAdminClient, createOpts) + assertTrue(describedConfigs) + assertTrue(alteredConfigs) + EasyMock.reset(alterResult, describeResult) + } + @Test def shouldAddTopicConfigUsingZookeeper(): Unit = { val createOpts = new ConfigCommandOptions(Array("--zookeeper", zkConnect, @@ -935,12 +1027,14 @@ class ConfigCommandTest extends ZooKeeperTestHarness with Logging { ConfigCommand.alterConfigWithZk(null, del512, CredentialChange("userB", Set(), 4096)) } - @Test - def testQuotaConfigEntity(): Unit = { + def doTestQuotaConfigEntity(zkConfig: Boolean): Unit = { + val connectOpts = if (zkConfig) + ("--zookeeper", zkConnect) + else + ("--bootstrap-server", "localhost:9092") def createOpts(entityType: String, entityName: Option[String], otherArgs: Array[String]) : ConfigCommandOptions = { - val optArray = Array("--zookeeper", zkConnect, - "--entity-type", entityType) + val optArray = Array(connectOpts._1, connectOpts._2, "--entity-type", entityType) val nameArray = entityName match { case Some(name) => Array("--entity-name", name) case None => Array[String]() @@ -1008,9 +1102,23 @@ class ConfigCommandTest extends ZooKeeperTestHarness with Logging { } @Test - def testUserClientQuotaOpts(): Unit = { + def testQuotaConfigEntityUsingZookeeper(): Unit = { + doTestQuotaConfigEntity(zkConfig = true) + } + + @Test + def testQuotaConfigEntity(): Unit = { + doTestQuotaConfigEntity(zkConfig = false) + } + + def doTestUserClientQuotaOpts(zkConfig: Boolean): Unit = { + val connectOpts = if (zkConfig) + ("--zookeeper", zkConnect) + else + ("--bootstrap-server", "localhost:9092") + def checkEntity(expectedEntityType: String, expectedEntityName: String, args: String*): Unit = { - val opts = new ConfigCommandOptions(Array("--zookeeper", zkConnect) ++ args) + val opts = new ConfigCommandOptions(Array(connectOpts._1, connectOpts._2) ++ args) opts.checkArgs() val entity = ConfigCommand.parseEntity(opts) assertEquals(expectedEntityType, entity.root.entityType) @@ -1025,7 +1133,6 @@ class ConfigCommandTest extends ZooKeeperTestHarness with Logging { "--entity-type", "clients", "--entity-name", "", "--alter", "--add-config", "a=b,c=d") - checkEntity("users", Sanitizer.sanitize("CN=user1") + "/clients/client1", "--entity-type", "users", "--entity-name", "CN=user1", "--entity-type", "clients", "--entity-name", "client1", "--alter", "--add-config", "a=b,c=d") @@ -1049,6 +1156,16 @@ class ConfigCommandTest extends ZooKeeperTestHarness with Logging { "--alter", "--add-config", "a=b,c=d") } + @Test + def testUserClientQuotaOptsUsingZookeeper(): Unit = { + doTestUserClientQuotaOpts(zkConfig = true) + } + + @Test + def testUserClientQuotaOpts(): Unit = { + doTestUserClientQuotaOpts(zkConfig = false) + } + @Test def testQuotaDescribeEntities(): Unit = { val zkClient: KafkaZkClient = EasyMock.createNiceMock(classOf[KafkaZkClient]) @@ -1134,5 +1251,11 @@ class ConfigCommandTest extends ZooKeeperTestHarness with Logging { options: AlterConfigsOptions): AlterConfigsResult = EasyMock.createNiceMock(classOf[AlterConfigsResult]) override def alterConfigs(configs: util.Map[ConfigResource, Config], options: AlterConfigsOptions): AlterConfigsResult = EasyMock.createNiceMock(classOf[AlterConfigsResult]) + override def describeClientQuotas(filters: util.Collection[QuotaFilter], + options: DescribeClientQuotasOptions): DescribeClientQuotasResult = + EasyMock.createNiceMock(classOf[DescribeClientQuotasResult]) + override def alterClientQuotas(entries: util.Collection[QuotaAlteration], + options: AlterClientQuotasOptions): AlterClientQuotasResult = + EasyMock.createNiceMock(classOf[AlterClientQuotasResult]) } } diff --git a/core/src/test/scala/unit/kafka/server/ClientQuotasRequestTest.scala b/core/src/test/scala/unit/kafka/server/ClientQuotasRequestTest.scala new file mode 100644 index 0000000000000..04a978a865ee5 --- /dev/null +++ b/core/src/test/scala/unit/kafka/server/ClientQuotasRequestTest.scala @@ -0,0 +1,444 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kafka.server + +import org.apache.kafka.common.errors.InvalidRequestException +import org.apache.kafka.common.internals.KafkaFutureImpl +import org.apache.kafka.common.quota.{QuotaAlteration, QuotaEntity, QuotaFilter} +import org.apache.kafka.common.requests.{AlterClientQuotasRequest, AlterClientQuotasResponse, DescribeClientQuotasRequest, DescribeClientQuotasResponse} +import org.junit.Assert._ +import org.junit.Test + +import java.util.concurrent.{ExecutionException, TimeUnit} + +import scala.collection.JavaConverters._ + +class ClientQuotasRequestTest extends BaseRequestTest { + private val ConsumerByteRateProp = DynamicConfig.Client.ConsumerByteRateOverrideProp + private val ProducerByteRateProp = DynamicConfig.Client.ProducerByteRateOverrideProp + private val RequestPercentageProp = DynamicConfig.Client.RequestPercentageOverrideProp + + override val brokerCount = 1 + + @Test + def testAlterClientQuotasRequest(): Unit = { + val entity = new QuotaEntity(Map((QuotaEntity.USER -> "user"), (QuotaEntity.CLIENT_ID -> "client-id")).asJava) + + // Expect an empty configuration. + verifyDescribeEntityQuotas(entity, Map.empty) + + // Add two configuration entries. + alterEntityQuotas(entity, Map( + (ProducerByteRateProp -> Some(10000.0)), + (ConsumerByteRateProp -> Some(20000.0)) + ), validateOnly = false) + + verifyDescribeEntityQuotas(entity, Map( + (ProducerByteRateProp -> 10000.0), + (ConsumerByteRateProp -> 20000.0) + )) + + // Update an existing entry. + alterEntityQuotas(entity, Map( + (ProducerByteRateProp -> Some(15000.0)) + ), validateOnly = false) + + verifyDescribeEntityQuotas(entity, Map( + (ProducerByteRateProp -> 15000.0), + (ConsumerByteRateProp -> 20000.0) + )) + + // Remove an existing configuration entry. + alterEntityQuotas(entity, Map( + (ProducerByteRateProp -> None) + ), validateOnly = false) + + verifyDescribeEntityQuotas(entity, Map( + (ConsumerByteRateProp -> 20000.0) + )) + + // Remove a non-existent configuration entry. This should make no changes. + alterEntityQuotas(entity, Map( + (RequestPercentageProp -> None) + ), validateOnly = false) + + verifyDescribeEntityQuotas(entity, Map( + (ConsumerByteRateProp -> 20000.0) + )) + + // Add back a deleted configuration entry. + alterEntityQuotas(entity, Map( + (ProducerByteRateProp -> Some(5000.0)) + ), validateOnly = false) + + verifyDescribeEntityQuotas(entity, Map( + (ProducerByteRateProp -> 5000.0), + (ConsumerByteRateProp -> 20000.0) + )) + + // Perform a mixed update. + alterEntityQuotas(entity, Map( + (ProducerByteRateProp -> Some(20000.0)), + (ConsumerByteRateProp -> None), + (RequestPercentageProp -> Some(12.3)) + ), validateOnly = false) + + verifyDescribeEntityQuotas(entity, Map( + (ProducerByteRateProp -> 20000.0), + (RequestPercentageProp -> 12.3) + )) + } + + @Test + def testAlterClientQuotasRequestValidateOnly(): Unit = { + val entity = new QuotaEntity(Map((QuotaEntity.USER -> "user")).asJava) + + // Set up a configuration. + alterEntityQuotas(entity, Map( + (ProducerByteRateProp -> Some(20000.0)), + (RequestPercentageProp -> Some(23.45)) + ), validateOnly = false) + + verifyDescribeEntityQuotas(entity, Map( + (ProducerByteRateProp -> 20000.0), + (RequestPercentageProp -> 23.45) + )) + + // Validate-only addition. + alterEntityQuotas(entity, Map( + (ConsumerByteRateProp -> Some(50000.0)) + ), validateOnly = true) + + verifyDescribeEntityQuotas(entity, Map( + (ProducerByteRateProp -> 20000.0), + (RequestPercentageProp -> 23.45) + )) + + // Validate-only modification. + alterEntityQuotas(entity, Map( + (ProducerByteRateProp -> Some(10000.0)) + ), validateOnly = true) + + verifyDescribeEntityQuotas(entity, Map( + (ProducerByteRateProp -> 20000.0), + (RequestPercentageProp -> 23.45) + )) + + // Validate-only removal. + alterEntityQuotas(entity, Map( + (RequestPercentageProp -> None) + ), validateOnly = true) + + verifyDescribeEntityQuotas(entity, Map( + (ProducerByteRateProp -> 20000.0), + (RequestPercentageProp -> 23.45) + )) + + // Validate-only mixed update. + alterEntityQuotas(entity, Map( + (ProducerByteRateProp -> Some(10000.0)), + (ConsumerByteRateProp -> Some(50000.0)), + (RequestPercentageProp -> None) + ), validateOnly = true) + + verifyDescribeEntityQuotas(entity, Map( + (ProducerByteRateProp -> 20000.0), + (RequestPercentageProp -> 23.45) + )) + } + + @Test(expected = classOf[InvalidRequestException]) + def testAlterClientQuotasBadUser(): Unit = { + val entity = new QuotaEntity(Map((QuotaEntity.USER -> "")).asJava) + alterEntityQuotas(entity, Map((RequestPercentageProp -> Some(12.34))), validateOnly = true) + } + + @Test(expected = classOf[InvalidRequestException]) + def testAlterClientQuotasBadClientId(): Unit = { + val entity = new QuotaEntity(Map((QuotaEntity.CLIENT_ID -> "")).asJava) + alterEntityQuotas(entity, Map((RequestPercentageProp -> Some(12.34))), validateOnly = true) + } + + @Test(expected = classOf[InvalidRequestException]) + def testAlterClientQuotasBadEntityType(): Unit = { + val entity = new QuotaEntity(Map(("" -> "name")).asJava) + alterEntityQuotas(entity, Map((RequestPercentageProp -> Some(12.34))), validateOnly = true) + } + + @Test(expected = classOf[InvalidRequestException]) + def testAlterClientQuotasEmptyEntity(): Unit = { + val entity = new QuotaEntity(Map.empty.asJava) + alterEntityQuotas(entity, Map((ProducerByteRateProp -> Some(10000.5))), validateOnly = true) + } + + @Test(expected = classOf[InvalidRequestException]) + def testAlterClientQuotasBadConfigKey(): Unit = { + val entity = new QuotaEntity(Map((QuotaEntity.USER -> "user")).asJava) + alterEntityQuotas(entity, Map(("bad" -> Some(1.0))), validateOnly = true) + } + + @Test(expected = classOf[InvalidRequestException]) + def testAlterClientQuotasBadConfigValue(): Unit = { + val entity = new QuotaEntity(Map((QuotaEntity.USER -> "user")).asJava) + alterEntityQuotas(entity, Map((ProducerByteRateProp -> Some(10000.5))), validateOnly = true) + } + + // Entities to be matched against. + private val matchEntities = List( + (Some("user-1"), Some("client-id-1"), 50.50), + (Some("user-2"), Some("client-id-1"), 51.51), + (Some("user-3"), Some("client-id-2"), 52.52), + (Some(QuotaEntity.USER_DEFAULT), Some("client-id-1"), 53.53), + (Some("user-1"), Some(QuotaEntity.CLIENT_ID_DEFAULT), 54.54), + (Some("user-3"), Some(QuotaEntity.CLIENT_ID_DEFAULT), 55.55), + (Some("user-1"), None, 56.56), + (Some("user-2"), None, 57.57), + (Some("user-3"), None, 58.58), + (Some(QuotaEntity.USER_DEFAULT), None, 59.59), + (None, Some("client-id-2"), 60.60) + ).map { case (u, c, v) => (toEntity(u, c), v) } + + private def setupDescribeClientQuotasMatchTest() = { + val result = alterClientQuotas(matchEntities.map { case (e, v) => + (e -> Map((RequestPercentageProp, Some(v)))) + }.toMap, validateOnly = false) + matchEntities.foreach(e => result.get(e._1).get.get(10, TimeUnit.SECONDS)) + + // Allow time for watch callbacks to be triggered. + Thread.sleep(500) + } + + @Test + def testDescribeClientQuotasMatchExact(): Unit = { + setupDescribeClientQuotasMatchTest() + + def matchEntity(entity: QuotaEntity) = { + val filters = entity.entries.asScala.map(e => QuotaFilter.matchExact(e._1, e._2)) + describeClientQuotas(filters, includeUnspecifiedTypes = false) + } + + // Test exact matches. + matchEntities.foreach { case (e, v) => + val result = matchEntity(e) + assertEquals(1, result.size) + assertTrue(result.get(e) != null) + val value = result.get(e).get(RequestPercentageProp) + assertTrue(value != null) + assertEquals(value, v, 1e-6) + } + + // Entities not contained in `matchEntityList`. + val notMatchEntities = List( + (Some("user-1"), Some("client-id-1"), Some("user-1")), + (Some("user-1"), Some("client-id-2"), None), + (Some("user-3"), Some("client-id-1"), None), + (Some("user-2"), Some(QuotaEntity.CLIENT_ID_DEFAULT), None), + (Some("user-4"), None, None), + (Some(QuotaEntity.USER_DEFAULT), Some("client-id-2"), None), + (None, Some("client-id-1"), None), + (None, Some("client-id-3"), None), + (None, None, Some("user-1")) + ).map { case (u, c, o) => + new QuotaEntity((u.map((QuotaEntity.USER, _)) ++ + c.map((QuotaEntity.CLIENT_ID, _)) ++ + o.map(("other", _))).toMap.asJava) + } + + // Verify exact matches of the non-matches returns empty. + notMatchEntities.foreach { e => + val result = matchEntity(e) + assertEquals(0, result.size) + } + } + + @Test + def testDescribeClientQuotasMatchPartial(): Unit = { + setupDescribeClientQuotasMatchTest() + + def testMatchEntities(filters: Iterable[QuotaFilter], includeUnspecifiedTypes: Boolean, expectedMatchSize: Int, partition: QuotaEntity => Boolean) { + val result = describeClientQuotas(filters, includeUnspecifiedTypes) + val (expectedMatches, expectedNonMatches) = matchEntities.partition(e => partition(e._1)) + assertEquals(expectedMatchSize, expectedMatches.size) // for test verification + assertEquals(expectedMatchSize, result.size) + val expectedMatchesMap = expectedMatches.toMap + matchEntities.foreach { case (entity, expectedValue) => + if (expectedMatchesMap.contains(entity)) { + val config = result.get(entity) + assertTrue(config != null) + val value = config.get(RequestPercentageProp) + assertTrue(value != null) + assertEquals(expectedValue, value, 1e-6) + } else { + assertTrue(result.get(entity) == null) + } + } + } + + // Match open-ended existing user. + testMatchEntities( + List(QuotaFilter.matchExact(QuotaEntity.USER, "user-1")), true, 3, + entity => entity.entries.get(QuotaEntity.USER) == "user-1" + ) + + // Match open-ended non-existent user. + testMatchEntities( + List(QuotaFilter.matchExact(QuotaEntity.USER, "unknown")), true, 0, + entity => false + ) + + // Match open-ended existing client ID. + testMatchEntities( + List(QuotaFilter.matchExact(QuotaEntity.CLIENT_ID, "client-id-2")), true, 2, + entity => entity.entries.get(QuotaEntity.CLIENT_ID) == "client-id-2" + ) + + // Match open-ended default user. + testMatchEntities( + List(QuotaFilter.matchExact(QuotaEntity.USER, QuotaEntity.USER_DEFAULT)), true, 2, + entity => entity.entries.get(QuotaEntity.USER) == QuotaEntity.USER_DEFAULT + ) + + // Match close-ended existing user. + testMatchEntities( + List(QuotaFilter.matchExact(QuotaEntity.USER, "user-2")), false, 1, + entity => entity.entries.get(QuotaEntity.USER) == "user-2" && entity.entries.get(QuotaEntity.CLIENT_ID) == null + ) + + // Match close-ended existing client ID that has no matching entity. + testMatchEntities( + List(QuotaFilter.matchExact(QuotaEntity.CLIENT_ID, "client-id-1")), false, 0, + entity => false + ) + + // Match an "other" entity type. This shouldn't match anything (but also shouldn't produce an error) because support for + // setting configurations of other entity types is not implemented yet. + testMatchEntities( + List(QuotaFilter.matchExact("other", "user-1")), true, 0, + entity => false, + ) + + // Match against all entities with the user type in a close-ended match. + testMatchEntities( + List(QuotaFilter.matchSpecified(QuotaEntity.USER)), false, 4, + entity => entity.entries.get(QuotaEntity.USER) != null && entity.entries.get(QuotaEntity.CLIENT_ID) == null + ) + + // Match against all entities with the user type in an open-ended match. + testMatchEntities( + List(QuotaFilter.matchSpecified(QuotaEntity.USER)), true, 10, + entity => entity.entries.get(QuotaEntity.USER) != null + ) + + // Match against all entities with the client ID type in a close-ended match. + testMatchEntities( + List(QuotaFilter.matchSpecified(QuotaEntity.CLIENT_ID)), false, 1, + entity => entity.entries.get(QuotaEntity.CLIENT_ID) != null && entity.entries.get(QuotaEntity.USER) == null + ) + + // Match against all entities with the client ID type in an open-ended match. + testMatchEntities( + List(QuotaFilter.matchSpecified(QuotaEntity.CLIENT_ID)), true, 7, + entity => entity.entries.get(QuotaEntity.CLIENT_ID) != null + ) + + // Match against all entities with an unexpected type in an open-ended match. + testMatchEntities( + List(QuotaFilter.matchSpecified("other")), true, 0, + entity => false + ) + + // Match open-ended empty filter list. This should match all entities. + testMatchEntities(List.empty, true, 11, entity => true) + + // Match close-ended empty filter list. This should match no entities. + testMatchEntities(List.empty, false, 0, entity => false) + } + + @Test + def testClientQuotasSanitized(): Unit = { + // An entity with name that must be sanitized when writing to Zookeeper. + val entity = new QuotaEntity(Map((QuotaEntity.USER -> "user with spaces")).asJava) + + alterEntityQuotas(entity, Map( + (ProducerByteRateProp -> Some(20000.0)), + ), validateOnly = false) + + verifyDescribeEntityQuotas(entity, Map( + (ProducerByteRateProp -> 20000.0), + )) + } + + private def verifyDescribeEntityQuotas(entity: QuotaEntity, quotas: Map[String, Double]) = { + val filters = entity.entries.asScala.map(e => QuotaFilter.matchExact(e._1, e._2)) + val describe = describeClientQuotas(filters, includeUnspecifiedTypes = false) + if (quotas.isEmpty) { + assertEquals(0, describe.size) + } else { + assertEquals(1, describe.size) + val configs = describe.get(entity) + assertTrue(configs != null) + assertEquals(quotas.size, configs.size) + quotas.foreach { case (k, v) => + val value = configs.get(k) + assertTrue(value != null) + assertEquals(v, value, 1e-6) + } + } + } + + private def toEntity(user: Option[String], clientId: Option[String]) = + new QuotaEntity((user.map((QuotaEntity.USER, _)) ++ clientId.map((QuotaEntity.CLIENT_ID, _))).toMap.asJava) + + private def describeClientQuotas(filters: Iterable[QuotaFilter], includeUnspecifiedTypes: Boolean) = { + val result = new KafkaFutureImpl[java.util.Map[QuotaEntity, java.util.Map[String, java.lang.Double]]] + sendDescribeClientQuotasRequest(filters, includeUnspecifiedTypes).complete(result) + result.get + } + + private def sendDescribeClientQuotasRequest(filters: Iterable[QuotaFilter], includeUnspecifiedTypes: Boolean): DescribeClientQuotasResponse = { + val request = new DescribeClientQuotasRequest.Builder(filters.asJavaCollection, includeUnspecifiedTypes).build() + connectAndReceive[DescribeClientQuotasResponse](request, destination = controllerSocketServer) + } + + private def alterEntityQuotas(entity: QuotaEntity, alter: Map[String, Option[Double]], validateOnly: Boolean) = + try alterClientQuotas(Map(entity -> alter), validateOnly).get(entity).get.get(10, TimeUnit.SECONDS) catch { + case e: ExecutionException => throw e.getCause + } + + private def alterClientQuotas(request: Map[QuotaEntity, Map[String, Option[Double]]], validateOnly: Boolean) = { + val entries = request.map { case (entity, alter) => + val ops = alter.map { case (key, value) => + new QuotaAlteration.Op(key, value.map(Double.box).getOrElse(null)) + }.asJavaCollection + new QuotaAlteration(entity, ops) + } + + val response = request.map(e => (e._1 -> new KafkaFutureImpl[Void])).asJava + sendAlterClientQuotasRequest(entries, validateOnly).complete(response) + val result = response.asScala + assertEquals(request.size, result.size) + request.foreach(e => assertTrue(result.get(e._1).isDefined)) + result + } + + private def sendAlterClientQuotasRequest(entries: Iterable[QuotaAlteration], validateOnly: Boolean): AlterClientQuotasResponse = { + val request = new AlterClientQuotasRequest.Builder(entries.asJavaCollection, validateOnly).build() + connectAndReceive[AlterClientQuotasResponse](request, destination = controllerSocketServer) + } + +} From be1836987e449bcbcaa3e0af83f9d1a1f3309c14 Mon Sep 17 00:00:00 2001 From: Brian Byrne Date: Mon, 10 Feb 2020 14:30:10 -0800 Subject: [PATCH 2/6] (minor function renaming) --- .../src/main/scala/kafka/admin/ConfigCommand.scala | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/core/src/main/scala/kafka/admin/ConfigCommand.scala b/core/src/main/scala/kafka/admin/ConfigCommand.scala index 652aa387a5fa2..5f075d5e13fea 100644 --- a/core/src/main/scala/kafka/admin/ConfigCommand.scala +++ b/core/src/main/scala/kafka/admin/ConfigCommand.scala @@ -309,7 +309,7 @@ object ConfigCommand extends Config { entityTypeHead match { case ConfigType.Topic => - val oldConfig = getGeneralConfig(adminClient, entityTypeHead, entityNameHead, includeSynonyms = false, describeAll = false) + val oldConfig = getResourceConfig(adminClient, entityTypeHead, entityNameHead, includeSynonyms = false, describeAll = false) .map { entry => (entry.name, entry) }.toMap // fail the command if any of the configs to be deleted does not exist @@ -325,7 +325,7 @@ object ConfigCommand extends Config { adminClient.incrementalAlterConfigs(Map(configResource -> alterEntries).asJava, alterOptions).all().get(60, TimeUnit.SECONDS) case ConfigType.Broker => - val oldConfig = getGeneralConfig(adminClient, entityTypeHead, entityNameHead, includeSynonyms = false, describeAll = false) + val oldConfig = getResourceConfig(adminClient, entityTypeHead, entityNameHead, includeSynonyms = false, describeAll = false) .map { entry => (entry.name, entry) }.toMap // fail the command if any of the configs to be deleted does not exist @@ -344,7 +344,7 @@ object ConfigCommand extends Config { adminClient.alterConfigs(Map(configResource -> newConfig).asJava, alterOptions).all().get(60, TimeUnit.SECONDS) case BrokerLoggerConfigType => - val validLoggers = getGeneralConfig(adminClient, entityTypeHead, entityNameHead, includeSynonyms = true, describeAll = false).map(_.name) + val validLoggers = getResourceConfig(adminClient, entityTypeHead, entityNameHead, includeSynonyms = true, describeAll = false).map(_.name) // fail the command if any of the configured broker loggers do not exist val invalidBrokerLoggers = configsToBeDeleted.filterNot(validLoggers.contains) ++ configsToBeAdded.keys.filterNot(validLoggers.contains) if (invalidBrokerLoggers.nonEmpty) @@ -407,13 +407,13 @@ object ConfigCommand extends Config { entityTypes.head match { case ConfigType.Topic | ConfigType.Broker | BrokerLoggerConfigType => - describeGeneralConfig(adminClient, entityTypes.head, entityNames.headOption, describeAll) + describeResourceConfig(adminClient, entityTypes.head, entityNames.headOption, describeAll) case ConfigType.User | ConfigType.Client => describeClientQuotasConfig(adminClient, entityTypes, entityNames) } } - private def describeGeneralConfig(adminClient: Admin, entityType: String, entityName: Option[String], describeAll: Boolean): Unit = { + private def describeResourceConfig(adminClient: Admin, entityType: String, entityName: Option[String], describeAll: Boolean): Unit = { val entities = entityName .map(name => List(name)) .getOrElse(entityType match { @@ -431,14 +431,14 @@ object ConfigCommand extends Config { val configSourceStr = if (describeAll) "All" else "Dynamic" println(s"$configSourceStr configs for ${entityType.dropRight(1)} $entity are:") } - getGeneralConfig(adminClient, entityType, entity, includeSynonyms = true, describeAll).foreach { entry => + getResourceConfig(adminClient, entityType, entity, includeSynonyms = true, describeAll).foreach { entry => val synonyms = entry.synonyms.asScala.map(synonym => s"${synonym.source}:${synonym.name}=${synonym.value}").mkString(", ") println(s" ${entry.name}=${entry.value} sensitive=${entry.isSensitive} synonyms={$synonyms}") } } } - private def getGeneralConfig(adminClient: Admin, entityType: String, entityName: String, includeSynonyms: Boolean, describeAll: Boolean) = { + private def getResourceConfig(adminClient: Admin, entityType: String, entityName: String, includeSynonyms: Boolean, describeAll: Boolean) = { def validateBrokerId(): Unit = try entityName.toInt catch { case _: NumberFormatException => throw new IllegalArgumentException(s"The entity name for $entityType must be a valid integer broker id, found: $entityName") From 4db7d0c81e3253135ad37a7a9b4918b4dfde26af Mon Sep 17 00:00:00 2001 From: Brian Byrne Date: Mon, 2 Mar 2020 09:56:10 -0800 Subject: [PATCH 3/6] (address review comments) --- checkstyle/import-control.xml | 4 ++++ .../main/java/org/apache/kafka/common/quota/QuotaFilter.java | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/checkstyle/import-control.xml b/checkstyle/import-control.xml index 491adc0deddf3..c4a7662d94ce3 100644 --- a/checkstyle/import-control.xml +++ b/checkstyle/import-control.xml @@ -163,6 +163,10 @@ + + + + diff --git a/clients/src/main/java/org/apache/kafka/common/quota/QuotaFilter.java b/clients/src/main/java/org/apache/kafka/common/quota/QuotaFilter.java index 825584612c163..6904dd2f7fa3b 100644 --- a/clients/src/main/java/org/apache/kafka/common/quota/QuotaFilter.java +++ b/clients/src/main/java/org/apache/kafka/common/quota/QuotaFilter.java @@ -25,7 +25,7 @@ public class QuotaFilter { // Matches all entities with the entity type specified. - private static final String MATCH_SPECIFIED = ""; + private static final String MATCH_SPECIFIED = ""; private final String entityType; private final String match; From 4a5c7e3eb73be700455ecdb52e126caac03c51f7 Mon Sep 17 00:00:00 2001 From: Brian Byrne Date: Tue, 10 Mar 2020 17:17:32 -0700 Subject: [PATCH 4/6] (address review comments, tests still need updating) --- .../kafka/common/quota/QuotaFilter.java | 53 ++++++++++++------- .../requests/DescribeClientQuotasRequest.java | 6 ++- .../message/DescribeClientQuotasRequest.json | 2 +- .../scala/kafka/server/AdminManager.scala | 26 +++++---- .../unit/kafka/admin/ConfigCommandTest.scala | 2 +- 5 files changed, 54 insertions(+), 35 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/common/quota/QuotaFilter.java b/clients/src/main/java/org/apache/kafka/common/quota/QuotaFilter.java index 6904dd2f7fa3b..556772343a3e3 100644 --- a/clients/src/main/java/org/apache/kafka/common/quota/QuotaFilter.java +++ b/clients/src/main/java/org/apache/kafka/common/quota/QuotaFilter.java @@ -18,27 +18,27 @@ package org.apache.kafka.common.quota; import java.util.Objects; +import java.util.Optional; /** * Describes a quota entity filter. */ public class QuotaFilter { - // Matches all entities with the entity type specified. - private static final String MATCH_SPECIFIED = ""; - private final String entityType; - private final String match; + private final Optional match; /** * A filter to be applied. * * @param entityType the entity type the filter applies to - * @param match the string that's matched exactly + * @param if specified, match the name that's matched exactly + * if empty, matches any specified name + * if null, matches no specified name */ - private QuotaFilter(String entityType, String match) { + private QuotaFilter(String entityType, Optional match) { this.entityType = Objects.requireNonNull(entityType); - this.match = Objects.requireNonNull(match); + this.match = match; } /** @@ -46,20 +46,30 @@ private QuotaFilter(String entityType, String match) { * name for the entity type exactly. * * @param entityType the entity type the filter applies to - * @param match the string that's matched exactly + * @param entityName the entity name that's matched exactly */ public static QuotaFilter matchExact(String entityType, String entityName) { - return new QuotaFilter(entityType, entityName); + return new QuotaFilter(entityType, Optional.of(Objects.requireNonNull(entityName))); } /** - * Constructs and returns a quota filter that matches the any entity name - * that's specified for the entity type. + * Constructs and returns a quota filter that matches any specified name for + * the entity type. * * @param entityType the entity type the filter applies to */ public static QuotaFilter matchSpecified(String entityType) { - return new QuotaFilter(entityType, MATCH_SPECIFIED); + return new QuotaFilter(entityType, Optional.empty()); + } + + /** + * Constructs and returns a quota filter that matches no entity name for the + * entity type exactly. + * + * @param entityType the entity type the filter applies to + */ + public static QuotaFilter matchNone(String entityType) { + return new QuotaFilter(entityType, null); } /** @@ -69,25 +79,32 @@ public String entityType() { return this.entityType; } + /** + * @return whether to match the exact entity name + */ + public boolean isMatchExact() { + return this.match != null && this.match.isPresent(); + } + /** * @return the string that's matched exactly */ - public String match() { - return this.match; + public String matchExact() { + return this.match.get(); } /** * @return whether to match the exact entity name */ - public boolean isMatchExact() { - return !isMatchSpecified(); + public boolean isMatchSpecified() { + return this.match != null && !this.match.isPresent(); } /** * @return whether to match all entities with the entity type specified */ - public boolean isMatchSpecified() { - return this.match == MATCH_SPECIFIED; + public boolean isMatchNone() { + return this.match == null; } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasRequest.java index ef9e366744b3b..3e0c80c4e7545 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasRequest.java @@ -30,6 +30,7 @@ public class DescribeClientQuotasRequest extends AbstractRequest { // These values must not change. private static final byte MATCH_TYPE_EXACT = 0; private static final byte MATCH_TYPE_SPECIFIED = 1; + private static final byte MATCH_TYPE_NONE = 2; public static class Builder extends AbstractRequest.Builder { @@ -44,9 +45,12 @@ public Builder(Collection filters, boolean includeUnspecifiedTypes) if (filter.isMatchSpecified()) { fd.setMatchType(MATCH_TYPE_SPECIFIED); fd.setMatch(null); + } else if (filter.isMatchNone()) { + fd.setMatchType(MATCH_TYPE_NONE); + fd.setMatch(null); } else { fd.setMatchType(MATCH_TYPE_EXACT); - fd.setMatch(filter.match()); + fd.setMatch(filter.matchExact()); } filterData.add(fd); } diff --git a/clients/src/main/resources/common/message/DescribeClientQuotasRequest.json b/clients/src/main/resources/common/message/DescribeClientQuotasRequest.json index 3617b5714e5ab..30d59876d48c6 100644 --- a/clients/src/main/resources/common/message/DescribeClientQuotasRequest.json +++ b/clients/src/main/resources/common/message/DescribeClientQuotasRequest.json @@ -25,7 +25,7 @@ { "name": "EntityType", "type": "string", "versions": "0+", "about": "The entity type that the filter applies to." }, { "name": "MatchType", "type": "int8", "versions": "0+", - "about": "How to match the entity {0 = exact, 1 = specified}." }, + "about": "How to match the entity {0 = exact, 1 = specified, 2 = none}." }, { "name": "Match", "type": "string", "versions": "0+", "nullableVersions": "0+", "about": "The string to match against, or null if unused for the match type." } ]}, diff --git a/core/src/main/scala/kafka/server/AdminManager.scala b/core/src/main/scala/kafka/server/AdminManager.scala index 46f37b749b3f9..4f3373b0fd42a 100644 --- a/core/src/main/scala/kafka/server/AdminManager.scala +++ b/core/src/main/scala/kafka/server/AdminManager.scala @@ -29,7 +29,7 @@ import org.apache.kafka.clients.admin.AlterConfigOp import org.apache.kafka.clients.admin.AlterConfigOp.OpType import org.apache.kafka.common.config.ConfigDef.ConfigKey import org.apache.kafka.common.config.{AbstractConfig, ConfigDef, ConfigException, ConfigResource, LogLevelConfig} -import org.apache.kafka.common.errors.{ApiException, InvalidConfigurationException, InvalidPartitionsException, InvalidReplicaAssignmentException, InvalidRequestException, ReassignmentInProgressException, TopicExistsException, UnknownTopicOrPartitionException} +import org.apache.kafka.common.errors.{ApiException, InvalidConfigurationException, InvalidPartitionsException, InvalidReplicaAssignmentException, InvalidRequestException, ReassignmentInProgressException, TopicExistsException, UnknownTopicOrPartitionException, UnsupportedVersionException} import org.apache.kafka.common.internals.Topic import org.apache.kafka.common.message.CreatePartitionsRequestData.CreatePartitionsTopic import org.apache.kafka.common.message.CreateTopicsRequestData.CreatableTopic @@ -716,7 +716,7 @@ class AdminManager(val config: KafkaConfig, case QuotaEntity.CLIENT_ID => clientId = sanitizedEntityName case _ => throw new InvalidRequestException(s"Unhandled quota entity type: ${entityType}") } - if (entityName == "") + if (entityName.isEmpty) throw new InvalidRequestException(s"Empty ${entityType} not supported") } (user, clientId) @@ -729,7 +729,6 @@ class AdminManager(val config: KafkaConfig, def describeClientQuotas(filters: Seq[QuotaFilter], includeUnspecifiedTypes: Boolean): Map[QuotaEntity, Map[String, Double]] = { var userFilter: Option[QuotaFilter] = None var clientIdFilter: Option[QuotaFilter] = None - var otherFilter: Boolean = false filters.foreach { filter => filter.entityType() match { case QuotaEntity.USER => @@ -742,21 +741,18 @@ class AdminManager(val config: KafkaConfig, clientIdFilter = Some(filter) case "" => throw new InvalidRequestException(s"Unexpected empty filter entity type") - case _ => + case et => // Supplying other entity types is not yet supported. - otherFilter = true + throw new UnsupportedVersionException(s"Custom entity type '${et}' not supported") } } - if (otherFilter) - Map.empty - else - handleDescribeClientQuotas(userFilter, clientIdFilter, includeUnspecifiedTypes) + handleDescribeClientQuotas(userFilter, clientIdFilter, includeUnspecifiedTypes) } def handleDescribeClientQuotas(userFilter: Option[QuotaFilter], clientIdFilter: Option[QuotaFilter], includeUnspecifiedTypes: Boolean) = { def wantExact(filter: Option[QuotaFilter]): Boolean = filter.map(_.isMatchExact).getOrElse(false) def wantExcluded(filter: Option[QuotaFilter]): Boolean = filter.map(_ => false).getOrElse(!includeUnspecifiedTypes) - def sanitized(filter: Option[QuotaFilter]): String = filter.map(f => Sanitizer.sanitize(f.`match`)).getOrElse("") + def sanitized(filter: Option[QuotaFilter]): String = filter.map(f => Sanitizer.sanitize(f.matchExact)).getOrElse("") val sanitizedUser = sanitized(userFilter) val exactUser = wantExact(userFilter) @@ -767,7 +763,7 @@ class AdminManager(val config: KafkaConfig, val excludeClientId = wantExcluded(clientIdFilter) val userEntries = if (exactUser && excludeClientId) - Map(((Some(userFilter.get.`match`), None) -> adminZkClient.fetchEntityConfig(ConfigType.User, sanitizedUser))) + Map(((Some(userFilter.get.matchExact()), None) -> adminZkClient.fetchEntityConfig(ConfigType.User, sanitizedUser))) else if (!excludeUser && !exactClientId) adminZkClient.fetchAllEntityConfigs(ConfigType.User).map { case (name, props) => ((Some(Sanitizer.desanitize(name)), None) -> props) @@ -776,7 +772,7 @@ class AdminManager(val config: KafkaConfig, Map.empty val clientIdEntries = if (excludeUser && exactClientId) - Map(((None, Some(clientIdFilter.get.`match`)) -> adminZkClient.fetchEntityConfig(ConfigType.Client, sanitizedClientId))) + Map(((None, Some(clientIdFilter.get.matchExact())) -> adminZkClient.fetchEntityConfig(ConfigType.Client, sanitizedClientId))) else if (!exactUser && !excludeClientId) adminZkClient.fetchAllEntityConfigs(ConfigType.Client).map { case (name, props) => ((None, Some(Sanitizer.desanitize(name))) -> props) @@ -785,7 +781,7 @@ class AdminManager(val config: KafkaConfig, Map.empty val bothEntries = if (exactUser && exactClientId) - Map(((Some(userFilter.get.`match`), Some(clientIdFilter.get.`match`)) -> + Map(((Some(userFilter.get.matchExact()), Some(clientIdFilter.get.matchExact())) -> adminZkClient.fetchEntityConfig(ConfigType.User, s"${sanitizedUser}/clients/${sanitizedClientId}"))) else if (!excludeUser && !excludeClientId) adminZkClient.fetchAllChildEntityConfigs(ConfigType.User, ConfigType.Client).map { case (name, props) => @@ -800,9 +796,11 @@ class AdminManager(val config: KafkaConfig, def matches(nameFilter: Option[QuotaFilter], name: Option[String]): Boolean = nameFilter match { case Some(filter) => if (filter.isMatchExact()) - name.map(_ == filter.`match`).getOrElse(false) + name.exists(_ == filter.matchExact()) else if (filter.isMatchSpecified()) name.isDefined + else if (filter.isMatchNone()) + !name.isDefined else throw new IllegalStateException(s"Unexected quota filter type") case None => diff --git a/core/src/test/scala/unit/kafka/admin/ConfigCommandTest.scala b/core/src/test/scala/unit/kafka/admin/ConfigCommandTest.scala index e2dc0170a3028..a0a1408b66b87 100644 --- a/core/src/test/scala/unit/kafka/admin/ConfigCommandTest.scala +++ b/core/src/test/scala/unit/kafka/admin/ConfigCommandTest.scala @@ -373,7 +373,7 @@ class ConfigCommandTest extends ZooKeeperTestHarness with Logging { val filter = filters.asScala.head assertTrue(filter.isMatchExact) assertEquals(QuotaEntity.CLIENT_ID, filter.entityType) - assertEquals("my-client-id", filter.`match`) + assertEquals("my-client-id", filter.matchExact()) assertFalse(options.includeUnspecifiedTypes) describedConfigs = true describeResult From 717d937c474cd9ec157f36e3c17c0db771238dc9 Mon Sep 17 00:00:00 2001 From: Brian Byrne Date: Wed, 11 Mar 2020 10:53:19 -0700 Subject: [PATCH 5/6] (remove includeUnspecified, matchSpecified -> matchSome, update tests) --- .../admin/DescribeClientQuotasOptions.java | 17 --- .../kafka/clients/admin/KafkaAdminClient.java | 2 +- .../kafka/common/quota/QuotaFilter.java | 6 +- .../requests/DescribeClientQuotasRequest.java | 22 ++-- .../message/DescribeClientQuotasRequest.json | 6 +- .../scala/kafka/admin/ConfigCommand.scala | 27 ++-- .../scala/kafka/server/AdminManager.scala | 18 +-- .../main/scala/kafka/server/KafkaApis.scala | 4 +- .../unit/kafka/admin/ConfigCommandTest.scala | 16 ++- .../server/ClientQuotasRequestTest.scala | 122 ++++++++---------- 10 files changed, 104 insertions(+), 136 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/DescribeClientQuotasOptions.java b/clients/src/main/java/org/apache/kafka/clients/admin/DescribeClientQuotasOptions.java index c0b49113bc47d..1692c730d3f55 100644 --- a/clients/src/main/java/org/apache/kafka/clients/admin/DescribeClientQuotasOptions.java +++ b/clients/src/main/java/org/apache/kafka/clients/admin/DescribeClientQuotasOptions.java @@ -26,21 +26,4 @@ */ @InterfaceStability.Evolving public class DescribeClientQuotasOptions extends AbstractOptions { - - private boolean includeUnspecifiedTypes = false; - - /** - * Returns whether the result should include unspecified entity types. - */ - public boolean includeUnspecifiedTypes() { - return this.includeUnspecifiedTypes; - } - - /** - * Sets whether the result should include unspecified entity types. - */ - public DescribeClientQuotasOptions includeUnspecifiedTypes(boolean includeUnspecifiedTypes) { - this.includeUnspecifiedTypes = includeUnspecifiedTypes; - return this; - } } diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java index 151660c513c31..ef78cd9cc27a8 100644 --- a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java +++ b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java @@ -3840,7 +3840,7 @@ public DescribeClientQuotasResult describeClientQuotas(Collection f @Override DescribeClientQuotasRequest.Builder createRequest(int timeoutMs) { - return new DescribeClientQuotasRequest.Builder(filters, options.includeUnspecifiedTypes()); + return new DescribeClientQuotasRequest.Builder(filters); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/quota/QuotaFilter.java b/clients/src/main/java/org/apache/kafka/common/quota/QuotaFilter.java index 556772343a3e3..822a128e056b7 100644 --- a/clients/src/main/java/org/apache/kafka/common/quota/QuotaFilter.java +++ b/clients/src/main/java/org/apache/kafka/common/quota/QuotaFilter.java @@ -58,7 +58,7 @@ public static QuotaFilter matchExact(String entityType, String entityName) { * * @param entityType the entity type the filter applies to */ - public static QuotaFilter matchSpecified(String entityType) { + public static QuotaFilter matchSome(String entityType) { return new QuotaFilter(entityType, Optional.empty()); } @@ -96,12 +96,12 @@ public String matchExact() { /** * @return whether to match the exact entity name */ - public boolean isMatchSpecified() { + public boolean isMatchSome() { return this.match != null && !this.match.isPresent(); } /** - * @return whether to match all entities with the entity type specified + * @return whether to match no entities with the entity type specified */ public boolean isMatchNone() { return this.match == null; diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasRequest.java index 3e0c80c4e7545..b7653a1cbf5ac 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasRequest.java @@ -29,21 +29,21 @@ public class DescribeClientQuotasRequest extends AbstractRequest { // These values must not change. private static final byte MATCH_TYPE_EXACT = 0; - private static final byte MATCH_TYPE_SPECIFIED = 1; + private static final byte MATCH_TYPE_SOME = 1; private static final byte MATCH_TYPE_NONE = 2; public static class Builder extends AbstractRequest.Builder { private final DescribeClientQuotasRequestData data; - public Builder(Collection filters, boolean includeUnspecifiedTypes) { + public Builder(Collection filters) { super(ApiKeys.DESCRIBE_CLIENT_QUOTAS); List filterData = new ArrayList<>(filters.size()); for (QuotaFilter filter : filters) { FilterData fd = new FilterData().setEntityType(filter.entityType()); - if (filter.isMatchSpecified()) { - fd.setMatchType(MATCH_TYPE_SPECIFIED); + if (filter.isMatchSome()) { + fd.setMatchType(MATCH_TYPE_SOME); fd.setMatch(null); } else if (filter.isMatchNone()) { fd.setMatchType(MATCH_TYPE_NONE); @@ -55,8 +55,7 @@ public Builder(Collection filters, boolean includeUnspecifiedTypes) filterData.add(fd); } this.data = new DescribeClientQuotasRequestData() - .setFilters(filterData) - .setIncludeUnspecifiedTypes(includeUnspecifiedTypes); + .setFilters(filterData); } @Override @@ -90,8 +89,11 @@ public Collection filters() { case MATCH_TYPE_EXACT: filter = QuotaFilter.matchExact(filterData.entityType(), filterData.match()); break; - case MATCH_TYPE_SPECIFIED: - filter = QuotaFilter.matchSpecified(filterData.entityType()); + case MATCH_TYPE_SOME: + filter = QuotaFilter.matchSome(filterData.entityType()); + break; + case MATCH_TYPE_NONE: + filter = QuotaFilter.matchNone(filterData.entityType()); break; default: throw new IllegalArgumentException("Unexpected match type: " + filterData.matchType()); @@ -101,10 +103,6 @@ public Collection filters() { return filters; } - public boolean includeUnspecifiedTypes() { - return data.includeUnspecifiedTypes(); - } - @Override public DescribeClientQuotasResponse getErrorResponse(int throttleTimeMs, Throwable e) { return new DescribeClientQuotasResponse(throttleTimeMs, e); diff --git a/clients/src/main/resources/common/message/DescribeClientQuotasRequest.json b/clients/src/main/resources/common/message/DescribeClientQuotasRequest.json index 30d59876d48c6..06c77483494bd 100644 --- a/clients/src/main/resources/common/message/DescribeClientQuotasRequest.json +++ b/clients/src/main/resources/common/message/DescribeClientQuotasRequest.json @@ -25,11 +25,9 @@ { "name": "EntityType", "type": "string", "versions": "0+", "about": "The entity type that the filter applies to." }, { "name": "MatchType", "type": "int8", "versions": "0+", - "about": "How to match the entity {0 = exact, 1 = specified, 2 = none}." }, + "about": "How to match the entity {0 = exact, 1 = some, 2 = none}." }, { "name": "Match", "type": "string", "versions": "0+", "nullableVersions": "0+", "about": "The string to match against, or null if unused for the match type." } - ]}, - { "name": "IncludeUnspecifiedTypes", "type": "bool", "versions": "0+", - "about": "Whether to include entity types which are not specified in the filters." } + ]} ] } diff --git a/core/src/main/scala/kafka/admin/ConfigCommand.scala b/core/src/main/scala/kafka/admin/ConfigCommand.scala index 5f075d5e13fea..2e4ecd11eed8e 100644 --- a/core/src/main/scala/kafka/admin/ConfigCommand.scala +++ b/core/src/main/scala/kafka/admin/ConfigCommand.scala @@ -394,15 +394,9 @@ object ConfigCommand extends Config { println(s"Completed updating default config for $entityTypeHead in the cluster.") } -<<<<<<< HEAD private[admin] def describeConfig(adminClient: Admin, opts: ConfigCommandOptions): Unit = { - val entityType = opts.entityTypes.head - val entityName = opts.entityNames.headOption -======= - private def describeConfig(adminClient: Admin, opts: ConfigCommandOptions): Unit = { val entityTypes = opts.entityTypes val entityNames = opts.entityNames ->>>>>>> KIP-546 (1/2): Implements describeClientQuotas() and alterClientQuotas(). val describeAll = opts.options.has(opts.allOpt) entityTypes.head match { @@ -495,18 +489,25 @@ object ConfigCommand extends Config { } private def getAllClientQuotasConfigs(adminClient: Admin, entityTypes: List[String], entityNames: List[String]) = { - val filters = entityTypes.map(Some(_)).zipAll(entityNames.map(Some(_)), None, None).map { case (entityTypeOpt, entityNameOpt) => - val entityType = entityTypeOpt match { - case Some(ConfigType.User) => QuotaEntity.USER - case Some(ConfigType.Client) => QuotaEntity.CLIENT_ID + var userFilter: Option[QuotaFilter] = None + var clientIdFilter: Option[QuotaFilter] = None + + def toFilter(entityType: String, entityName: Option[String]): QuotaFilter = + entityName.map(QuotaFilter.matchExact(entityType, _)).getOrElse(QuotaFilter.matchSome(entityType)) + + entityTypes.map(Some(_)).zipAll(entityNames.map(Some(_)), None, None).foreach { case (entityTypeOpt, entityNameOpt) => + entityTypeOpt match { + case Some(ConfigType.User) => userFilter = Some(toFilter(QuotaEntity.USER, entityNameOpt)) + case Some(ConfigType.Client) => clientIdFilter = Some(toFilter(QuotaEntity.CLIENT_ID, entityNameOpt)) case Some(_) => throw new IllegalArgumentException(s"Unexpected entity type ${entityTypeOpt.get}") case None => throw new IllegalArgumentException("More entity names specified than entity types") } - entityNameOpt.map(QuotaFilter.matchExact(entityType, _)).getOrElse(QuotaFilter.matchSpecified(entityType)) } - val options = new DescribeClientQuotasOptions().includeUnspecifiedTypes(false) - adminClient.describeClientQuotas(filters.asJavaCollection, options).entities.get(30, TimeUnit.SECONDS).asScala + val filters = List(userFilter.getOrElse(QuotaFilter.matchNone(QuotaEntity.USER)), + clientIdFilter.getOrElse(QuotaFilter.matchNone(QuotaEntity.CLIENT_ID))) + + adminClient.describeClientQuotas(filters.asJavaCollection).entities.get(30, TimeUnit.SECONDS).asScala } case class Entity(entityType: String, sanitizedName: Option[String]) { diff --git a/core/src/main/scala/kafka/server/AdminManager.scala b/core/src/main/scala/kafka/server/AdminManager.scala index 4f3373b0fd42a..30c872ab34680 100644 --- a/core/src/main/scala/kafka/server/AdminManager.scala +++ b/core/src/main/scala/kafka/server/AdminManager.scala @@ -726,7 +726,7 @@ class AdminManager(val config: KafkaConfig, new QuotaEntity((user.map(u => QuotaEntity.USER -> u) ++ clientId.map(c => QuotaEntity.CLIENT_ID -> c)).toMap.asJava) } - def describeClientQuotas(filters: Seq[QuotaFilter], includeUnspecifiedTypes: Boolean): Map[QuotaEntity, Map[String, Double]] = { + def describeClientQuotas(filters: Seq[QuotaFilter]): Map[QuotaEntity, Map[String, Double]] = { var userFilter: Option[QuotaFilter] = None var clientIdFilter: Option[QuotaFilter] = None filters.foreach { filter => @@ -746,13 +746,15 @@ class AdminManager(val config: KafkaConfig, throw new UnsupportedVersionException(s"Custom entity type '${et}' not supported") } } - handleDescribeClientQuotas(userFilter, clientIdFilter, includeUnspecifiedTypes) + handleDescribeClientQuotas(userFilter, clientIdFilter) } - def handleDescribeClientQuotas(userFilter: Option[QuotaFilter], clientIdFilter: Option[QuotaFilter], includeUnspecifiedTypes: Boolean) = { - def wantExact(filter: Option[QuotaFilter]): Boolean = filter.map(_.isMatchExact).getOrElse(false) - def wantExcluded(filter: Option[QuotaFilter]): Boolean = filter.map(_ => false).getOrElse(!includeUnspecifiedTypes) - def sanitized(filter: Option[QuotaFilter]): String = filter.map(f => Sanitizer.sanitize(f.matchExact)).getOrElse("") + def handleDescribeClientQuotas(userFilter: Option[QuotaFilter], clientIdFilter: Option[QuotaFilter]) = { + def wantExact(filter: Option[QuotaFilter]): Boolean = filter.exists(_.isMatchExact) + def wantExcluded(filter: Option[QuotaFilter]): Boolean = filter.exists(_.isMatchNone) + def sanitized(filter: Option[QuotaFilter]): String = { + filter.map(f => if (f.isMatchExact) Sanitizer.sanitize(f.matchExact) else "").getOrElse("") + } val sanitizedUser = sanitized(userFilter) val exactUser = wantExact(userFilter) @@ -797,14 +799,14 @@ class AdminManager(val config: KafkaConfig, case Some(filter) => if (filter.isMatchExact()) name.exists(_ == filter.matchExact()) - else if (filter.isMatchSpecified()) + else if (filter.isMatchSome()) name.isDefined else if (filter.isMatchNone()) !name.isDefined else throw new IllegalStateException(s"Unexected quota filter type") case None => - includeUnspecifiedTypes || !name.isDefined + true } def fromProps(props: Properties): Map[String, Double] = { diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala index 47ca09c60bb0b..7aaa179398d9c 100644 --- a/core/src/main/scala/kafka/server/KafkaApis.scala +++ b/core/src/main/scala/kafka/server/KafkaApis.scala @@ -2791,8 +2791,8 @@ class KafkaApis(val requestChannel: RequestChannel, val describeClientQuotasRequest = request.body[DescribeClientQuotasRequest] if (authorize(request, DESCRIBE_CONFIGS, CLUSTER, CLUSTER_NAME)) { - val result = adminManager.describeClientQuotas(describeClientQuotasRequest.filters().asScala.toSeq, - describeClientQuotasRequest.includeUnspecifiedTypes()).mapValues(_.mapValues(Double.box).asJava).asJava + val result = adminManager.describeClientQuotas( + describeClientQuotasRequest.filters().asScala.toSeq).mapValues(_.mapValues(Double.box).asJava).asJava sendResponseMaybeThrottle(request, requestThrottleMs => new DescribeClientQuotasResponse(result, requestThrottleMs)) } else { diff --git a/core/src/test/scala/unit/kafka/admin/ConfigCommandTest.scala b/core/src/test/scala/unit/kafka/admin/ConfigCommandTest.scala index a0a1408b66b87..961da196e87ff 100644 --- a/core/src/test/scala/unit/kafka/admin/ConfigCommandTest.scala +++ b/core/src/test/scala/unit/kafka/admin/ConfigCommandTest.scala @@ -369,12 +369,16 @@ class ConfigCommandTest extends ZooKeeperTestHarness with Logging { val node = new Node(1, "localhost", 9092) val mockAdminClient = new MockAdminClient(util.Collections.singletonList(node), node) { override def describeClientQuotas(filters: util.Collection[QuotaFilter], options: DescribeClientQuotasOptions): DescribeClientQuotasResult = { - assertEquals(1, filters.size) - val filter = filters.asScala.head - assertTrue(filter.isMatchExact) - assertEquals(QuotaEntity.CLIENT_ID, filter.entityType) - assertEquals("my-client-id", filter.matchExact()) - assertFalse(options.includeUnspecifiedTypes) + assertEquals(2, filters.size) + filters.asScala.foreach { filter => + if (filter.entityType == QuotaEntity.USER) { + assertTrue(filter.isMatchNone) + } else { + assertEquals(QuotaEntity.CLIENT_ID, filter.entityType) + assertTrue(filter.isMatchExact) + assertEquals("my-client-id", filter.matchExact) + } + } describedConfigs = true describeResult } diff --git a/core/src/test/scala/unit/kafka/server/ClientQuotasRequestTest.scala b/core/src/test/scala/unit/kafka/server/ClientQuotasRequestTest.scala index 04a978a865ee5..2520f919c8e17 100644 --- a/core/src/test/scala/unit/kafka/server/ClientQuotasRequestTest.scala +++ b/core/src/test/scala/unit/kafka/server/ClientQuotasRequestTest.scala @@ -17,7 +17,7 @@ package kafka.server -import org.apache.kafka.common.errors.InvalidRequestException +import org.apache.kafka.common.errors.{InvalidRequestException, UnsupportedVersionException} import org.apache.kafka.common.internals.KafkaFutureImpl import org.apache.kafka.common.quota.{QuotaAlteration, QuotaEntity, QuotaFilter} import org.apache.kafka.common.requests.{AlterClientQuotasRequest, AlterClientQuotasResponse, DescribeClientQuotasRequest, DescribeClientQuotasResponse} @@ -228,8 +228,11 @@ class ClientQuotasRequestTest extends BaseRequestTest { setupDescribeClientQuotasMatchTest() def matchEntity(entity: QuotaEntity) = { - val filters = entity.entries.asScala.map(e => QuotaFilter.matchExact(e._1, e._2)) - describeClientQuotas(filters, includeUnspecifiedTypes = false) + val entries = entity.entries.asScala + def toFilter(entityType: String) = + entries.get(entityType).map(entityName => QuotaFilter.matchExact(entityType, entityName)).getOrElse(QuotaFilter.matchNone(entityType)) + val filters = List(toFilter(QuotaEntity.USER), toFilter(QuotaEntity.CLIENT_ID)) + describeClientQuotas(filters) } // Test exact matches. @@ -244,19 +247,16 @@ class ClientQuotasRequestTest extends BaseRequestTest { // Entities not contained in `matchEntityList`. val notMatchEntities = List( - (Some("user-1"), Some("client-id-1"), Some("user-1")), - (Some("user-1"), Some("client-id-2"), None), - (Some("user-3"), Some("client-id-1"), None), - (Some("user-2"), Some(QuotaEntity.CLIENT_ID_DEFAULT), None), - (Some("user-4"), None, None), - (Some(QuotaEntity.USER_DEFAULT), Some("client-id-2"), None), - (None, Some("client-id-1"), None), - (None, Some("client-id-3"), None), - (None, None, Some("user-1")) - ).map { case (u, c, o) => + (Some("user-1"), Some("client-id-2")), + (Some("user-3"), Some("client-id-1")), + (Some("user-2"), Some(QuotaEntity.CLIENT_ID_DEFAULT)), + (Some("user-4"), None), + (Some(QuotaEntity.USER_DEFAULT), Some("client-id-2")), + (None, Some("client-id-1")), + (None, Some("client-id-3")), + ).map { case (u, c) => new QuotaEntity((u.map((QuotaEntity.USER, _)) ++ - c.map((QuotaEntity.CLIENT_ID, _)) ++ - o.map(("other", _))).toMap.asJava) + c.map((QuotaEntity.CLIENT_ID, _))).toMap.asJava) } // Verify exact matches of the non-matches returns empty. @@ -270,8 +270,8 @@ class ClientQuotasRequestTest extends BaseRequestTest { def testDescribeClientQuotasMatchPartial(): Unit = { setupDescribeClientQuotasMatchTest() - def testMatchEntities(filters: Iterable[QuotaFilter], includeUnspecifiedTypes: Boolean, expectedMatchSize: Int, partition: QuotaEntity => Boolean) { - val result = describeClientQuotas(filters, includeUnspecifiedTypes) + def testMatchEntities(filters: Iterable[QuotaFilter], expectedMatchSize: Int, partition: QuotaEntity => Boolean) { + val result = describeClientQuotas(filters) val (expectedMatches, expectedNonMatches) = matchEntities.partition(e => partition(e._1)) assertEquals(expectedMatchSize, expectedMatches.size) // for test verification assertEquals(expectedMatchSize, result.size) @@ -289,84 +289,66 @@ class ClientQuotasRequestTest extends BaseRequestTest { } } - // Match open-ended existing user. + // Match existing user. testMatchEntities( - List(QuotaFilter.matchExact(QuotaEntity.USER, "user-1")), true, 3, + List(QuotaFilter.matchExact(QuotaEntity.USER, "user-1")), 3, entity => entity.entries.get(QuotaEntity.USER) == "user-1" ) - // Match open-ended non-existent user. + // Match non-existent user. testMatchEntities( - List(QuotaFilter.matchExact(QuotaEntity.USER, "unknown")), true, 0, + List(QuotaFilter.matchExact(QuotaEntity.USER, "unknown")), 0, entity => false ) - // Match open-ended existing client ID. + // Match existing client ID. testMatchEntities( - List(QuotaFilter.matchExact(QuotaEntity.CLIENT_ID, "client-id-2")), true, 2, + List(QuotaFilter.matchExact(QuotaEntity.CLIENT_ID, "client-id-2")), 2, entity => entity.entries.get(QuotaEntity.CLIENT_ID) == "client-id-2" ) - // Match open-ended default user. + // Match default user. testMatchEntities( - List(QuotaFilter.matchExact(QuotaEntity.USER, QuotaEntity.USER_DEFAULT)), true, 2, + List(QuotaFilter.matchExact(QuotaEntity.USER, QuotaEntity.USER_DEFAULT)), 2, entity => entity.entries.get(QuotaEntity.USER) == QuotaEntity.USER_DEFAULT ) - // Match close-ended existing user. + // Match against all entities with the user type in an match. testMatchEntities( - List(QuotaFilter.matchExact(QuotaEntity.USER, "user-2")), false, 1, - entity => entity.entries.get(QuotaEntity.USER) == "user-2" && entity.entries.get(QuotaEntity.CLIENT_ID) == null - ) - - // Match close-ended existing client ID that has no matching entity. - testMatchEntities( - List(QuotaFilter.matchExact(QuotaEntity.CLIENT_ID, "client-id-1")), false, 0, - entity => false - ) - - // Match an "other" entity type. This shouldn't match anything (but also shouldn't produce an error) because support for - // setting configurations of other entity types is not implemented yet. - testMatchEntities( - List(QuotaFilter.matchExact("other", "user-1")), true, 0, - entity => false, - ) - - // Match against all entities with the user type in a close-ended match. - testMatchEntities( - List(QuotaFilter.matchSpecified(QuotaEntity.USER)), false, 4, - entity => entity.entries.get(QuotaEntity.USER) != null && entity.entries.get(QuotaEntity.CLIENT_ID) == null - ) - - // Match against all entities with the user type in an open-ended match. - testMatchEntities( - List(QuotaFilter.matchSpecified(QuotaEntity.USER)), true, 10, + List(QuotaFilter.matchSome(QuotaEntity.USER)), 10, entity => entity.entries.get(QuotaEntity.USER) != null ) - // Match against all entities with the client ID type in a close-ended match. + // Match against all entities with the client ID type in an match. testMatchEntities( - List(QuotaFilter.matchSpecified(QuotaEntity.CLIENT_ID)), false, 1, - entity => entity.entries.get(QuotaEntity.CLIENT_ID) != null && entity.entries.get(QuotaEntity.USER) == null + List(QuotaFilter.matchSome(QuotaEntity.CLIENT_ID)), 7, + entity => entity.entries.get(QuotaEntity.CLIENT_ID) != null ) - // Match against all entities with the client ID type in an open-ended match. + // Match against no entities with the user type. testMatchEntities( - List(QuotaFilter.matchSpecified(QuotaEntity.CLIENT_ID)), true, 7, - entity => entity.entries.get(QuotaEntity.CLIENT_ID) != null + List(QuotaFilter.matchNone(QuotaEntity.USER)), 1, + entity => entity.entries.get(QuotaEntity.USER) == null ) - // Match against all entities with an unexpected type in an open-ended match. + // Match against no entities with the client ID type. testMatchEntities( - List(QuotaFilter.matchSpecified("other")), true, 0, - entity => false + List(QuotaFilter.matchNone(QuotaEntity.CLIENT_ID)), 4, + entity => entity.entries.get(QuotaEntity.CLIENT_ID) == null ) - // Match open-ended empty filter list. This should match all entities. - testMatchEntities(List.empty, true, 11, entity => true) + // Match empty filter list. This should match all entities. + testMatchEntities(List.empty, 11, entity => true) + } - // Match close-ended empty filter list. This should match no entities. - testMatchEntities(List.empty, false, 0, entity => false) + @Test + def testClientQuotasUnsupportedEntityTypes() { + val entity = new QuotaEntity(Map(("other" -> "name")).asJava) + try { + verifyDescribeEntityQuotas(entity, Map()) + } catch { + case e: ExecutionException => assertTrue(e.getCause.isInstanceOf[UnsupportedVersionException]) + } } @Test @@ -385,7 +367,7 @@ class ClientQuotasRequestTest extends BaseRequestTest { private def verifyDescribeEntityQuotas(entity: QuotaEntity, quotas: Map[String, Double]) = { val filters = entity.entries.asScala.map(e => QuotaFilter.matchExact(e._1, e._2)) - val describe = describeClientQuotas(filters, includeUnspecifiedTypes = false) + val describe = describeClientQuotas(filters) if (quotas.isEmpty) { assertEquals(0, describe.size) } else { @@ -404,14 +386,14 @@ class ClientQuotasRequestTest extends BaseRequestTest { private def toEntity(user: Option[String], clientId: Option[String]) = new QuotaEntity((user.map((QuotaEntity.USER, _)) ++ clientId.map((QuotaEntity.CLIENT_ID, _))).toMap.asJava) - private def describeClientQuotas(filters: Iterable[QuotaFilter], includeUnspecifiedTypes: Boolean) = { + private def describeClientQuotas(filters: Iterable[QuotaFilter]) = { val result = new KafkaFutureImpl[java.util.Map[QuotaEntity, java.util.Map[String, java.lang.Double]]] - sendDescribeClientQuotasRequest(filters, includeUnspecifiedTypes).complete(result) + sendDescribeClientQuotasRequest(filters).complete(result) result.get } - private def sendDescribeClientQuotasRequest(filters: Iterable[QuotaFilter], includeUnspecifiedTypes: Boolean): DescribeClientQuotasResponse = { - val request = new DescribeClientQuotasRequest.Builder(filters.asJavaCollection, includeUnspecifiedTypes).build() + private def sendDescribeClientQuotasRequest(filters: Iterable[QuotaFilter]): DescribeClientQuotasResponse = { + val request = new DescribeClientQuotasRequest.Builder(filters.asJavaCollection).build() connectAndReceive[DescribeClientQuotasResponse](request, destination = controllerSocketServer) } From 5fb5904d862a544d2c6bb718d4584f8723ec0ce4 Mon Sep 17 00:00:00 2001 From: Brian Byrne Date: Fri, 13 Mar 2020 17:19:20 -0700 Subject: [PATCH 6/6] (update filter API) --- build.gradle | 2 +- .../org/apache/kafka/clients/admin/Admin.java | 28 ++-- .../admin/AlterClientQuotasResult.java | 8 +- .../admin/DescribeClientQuotasOptions.java | 2 +- .../admin/DescribeClientQuotasResult.java | 10 +- .../kafka/clients/admin/KafkaAdminClient.java | 18 +-- ...ration.java => ClientQuotaAlteration.java} | 17 +- ...uotaEntity.java => ClientQuotaEntity.java} | 22 +-- .../kafka/common/quota/ClientQuotaFilter.java | 101 ++++++++++++ .../quota/ClientQuotaFilterComponent.java | 109 +++++++++++++ .../kafka/common/quota/QuotaFilter.java | 127 --------------- .../requests/AlterClientQuotasRequest.java | 24 +-- .../requests/AlterClientQuotasResponse.java | 16 +- .../requests/DescribeClientQuotasRequest.java | 66 ++++---- .../DescribeClientQuotasResponse.java | 14 +- .../message/AlterClientQuotasRequest.json | 4 +- .../message/AlterClientQuotasResponse.json | 4 +- .../message/DescribeClientQuotasRequest.json | 12 +- .../message/DescribeClientQuotasResponse.json | 4 +- .../clients/admin/KafkaAdminClientTest.java | 48 +++--- .../kafka/clients/admin/MockAdminClient.java | 8 +- .../scala/kafka/admin/ConfigCommand.scala | 42 ++--- .../scala/kafka/server/AdminManager.scala | 139 +++++++++------- .../main/scala/kafka/server/KafkaApis.scala | 2 +- .../unit/kafka/admin/ConfigCommandTest.scala | 37 ++--- .../server/ClientQuotasRequestTest.scala | 149 ++++++++++-------- 26 files changed, 563 insertions(+), 450 deletions(-) rename clients/src/main/java/org/apache/kafka/common/quota/{QuotaAlteration.java => ClientQuotaAlteration.java} (84%) rename clients/src/main/java/org/apache/kafka/common/quota/{QuotaEntity.java => ClientQuotaEntity.java} (72%) create mode 100644 clients/src/main/java/org/apache/kafka/common/quota/ClientQuotaFilter.java create mode 100644 clients/src/main/java/org/apache/kafka/common/quota/ClientQuotaFilterComponent.java delete mode 100644 clients/src/main/java/org/apache/kafka/common/quota/QuotaFilter.java diff --git a/build.gradle b/build.gradle index 975d56b4881ed..05fc20f78db4e 100644 --- a/build.gradle +++ b/build.gradle @@ -457,7 +457,7 @@ subprojects { // See https://www.lightbend.com/blog/scala-inliner-optimizer for more information about the optimizer. scalaCompileOptions.additionalParameters += ["-opt:l:inline"] scalaCompileOptions.additionalParameters += inlineFrom - + // these options are valid for Scala versions < 2.13 only // Scala 2.13 removes them, see https://github.com/scala/scala/pull/6502 and https://github.com/scala/scala/pull/5969 if (versions.baseScala == '2.12') { diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/Admin.java b/clients/src/main/java/org/apache/kafka/clients/admin/Admin.java index 6a607f3ce614f..b99fd5ff58c39 100644 --- a/clients/src/main/java/org/apache/kafka/clients/admin/Admin.java +++ b/clients/src/main/java/org/apache/kafka/clients/admin/Admin.java @@ -36,8 +36,8 @@ import org.apache.kafka.common.acl.AclBindingFilter; import org.apache.kafka.common.annotation.InterfaceStability; import org.apache.kafka.common.config.ConfigResource; -import org.apache.kafka.common.quota.QuotaAlteration; -import org.apache.kafka.common.quota.QuotaFilter; +import org.apache.kafka.common.quota.ClientQuotaAlteration; +import org.apache.kafka.common.quota.ClientQuotaFilter; import org.apache.kafka.common.requests.LeaveGroupResponse; /** @@ -1134,24 +1134,24 @@ default ListOffsetsResult listOffsets(Map topicParti ListOffsetsResult listOffsets(Map topicPartitionOffsets, ListOffsetsOptions options); /** - * Describes all entities matching all provided filters (logical AND) that have at least one quota - * configuration value defined. + * Describes all entities matching the provided filter that have at least one client quota configuration + * value defined. *

- * This is a convenience method for {@link #describeClientQuotas(Collection, DescribeClientQuotasOptions)} + * This is a convenience method for {@link #describeClientQuotas(ClientQuotaFilter, DescribeClientQuotasOptions)} * with default options. See the overload for more details. *

* This operation is supported by brokers with version 2.6.0 or higher. * - * @param filters filtering rules to apply to matching entities + * @param filter the filter to apply to match entities * @return the DescribeClientQuotasResult containing the result */ - default DescribeClientQuotasResult describeClientQuotas(Collection filters) { - return describeClientQuotas(filters, new DescribeClientQuotasOptions()); + default DescribeClientQuotasResult describeClientQuotas(ClientQuotaFilter filter) { + return describeClientQuotas(filter, new DescribeClientQuotasOptions()); } /** - * Describes all entities matching all provided filters (logical AND) that have at least one quota - * configuration value defined. + * Describes all entities matching the provided filter that have at least one client quota configuration + * value defined. *

* The following exceptions can be anticipated when calling {@code get()} on the future from the * returned {@link DescribeClientQuotasResult}: @@ -1166,11 +1166,11 @@ default DescribeClientQuotasResult describeClientQuotas(Collection *

* This operation is supported by brokers with version 2.6.0 or higher. * - * @param filters filtering rules to apply to matching entities + * @param filter the filter to apply to match entities * @param options the options to use * @return the DescribeClientQuotasResult containing the result */ - DescribeClientQuotasResult describeClientQuotas(Collection filters, DescribeClientQuotasOptions options); + DescribeClientQuotasResult describeClientQuotas(ClientQuotaFilter filter, DescribeClientQuotasOptions options); /** * Alters client quota configurations with the specified alterations. @@ -1183,7 +1183,7 @@ default DescribeClientQuotasResult describeClientQuotas(Collection * @param entries the alterations to perform * @return the AlterClientQuotasResult containing the result */ - default AlterClientQuotasResult alterClientQuotas(Collection entries) { + default AlterClientQuotasResult alterClientQuotas(Collection entries) { return alterClientQuotas(entries, new AlterClientQuotasOptions()); } @@ -1210,7 +1210,7 @@ default AlterClientQuotasResult alterClientQuotas(Collection en * @param entries the alterations to perform * @return the AlterClientQuotasResult containing the result */ - AlterClientQuotasResult alterClientQuotas(Collection entries, AlterClientQuotasOptions options); + AlterClientQuotasResult alterClientQuotas(Collection entries, AlterClientQuotasOptions options); /** * Get the metrics kept by the adminClient diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/AlterClientQuotasResult.java b/clients/src/main/java/org/apache/kafka/clients/admin/AlterClientQuotasResult.java index fd7f3a9d7ba19..63c6b3e714043 100644 --- a/clients/src/main/java/org/apache/kafka/clients/admin/AlterClientQuotasResult.java +++ b/clients/src/main/java/org/apache/kafka/clients/admin/AlterClientQuotasResult.java @@ -19,7 +19,7 @@ import org.apache.kafka.common.KafkaFuture; import org.apache.kafka.common.annotation.InterfaceStability; -import org.apache.kafka.common.quota.QuotaEntity; +import org.apache.kafka.common.quota.ClientQuotaEntity; import java.util.Map; @@ -31,21 +31,21 @@ @InterfaceStability.Evolving public class AlterClientQuotasResult { - private final Map> futures; + private final Map> futures; /** * Maps an entity to its alteration result. * * @param futures maps entity to its alteration result */ - public AlterClientQuotasResult(Map> futures) { + public AlterClientQuotasResult(Map> futures) { this.futures = futures; } /** * Returns a map from quota entity to a future which can be used to check the status of the operation. */ - public Map> values() { + public Map> values() { return futures; } diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/DescribeClientQuotasOptions.java b/clients/src/main/java/org/apache/kafka/clients/admin/DescribeClientQuotasOptions.java index 1692c730d3f55..14e7e451219e6 100644 --- a/clients/src/main/java/org/apache/kafka/clients/admin/DescribeClientQuotasOptions.java +++ b/clients/src/main/java/org/apache/kafka/clients/admin/DescribeClientQuotasOptions.java @@ -20,7 +20,7 @@ import org.apache.kafka.common.annotation.InterfaceStability; /** - * Options for {@link Admin#describeClientQuotas(Collection, DescribeClientQuotasOptions)}. + * Options for {@link Admin#describeClientQuotas(ClientQuotaFilter, DescribeClientQuotasOptions)}. * * The API of this class is evolving, see {@link Admin} for details. */ diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/DescribeClientQuotasResult.java b/clients/src/main/java/org/apache/kafka/clients/admin/DescribeClientQuotasResult.java index 27677787e8a29..b4855901a03de 100644 --- a/clients/src/main/java/org/apache/kafka/clients/admin/DescribeClientQuotasResult.java +++ b/clients/src/main/java/org/apache/kafka/clients/admin/DescribeClientQuotasResult.java @@ -19,19 +19,19 @@ import org.apache.kafka.common.KafkaFuture; import org.apache.kafka.common.annotation.InterfaceStability; -import org.apache.kafka.common.quota.QuotaEntity; +import org.apache.kafka.common.quota.ClientQuotaEntity; import java.util.Map; /** - * The result of the {@link Admin#describeClientQuotas(Collection, DescribeClientQuotasOptions)} call. + * The result of the {@link Admin#describeClientQuotas(ClientQuotaFilter, DescribeClientQuotasOptions)} call. * * The API of this class is evolving, see {@link Admin} for details. */ @InterfaceStability.Evolving public class DescribeClientQuotasResult { - private final KafkaFuture>> entities; + private final KafkaFuture>> entities; /** * Maps an entity to its configured quota value(s). Note if no value is defined for a quota @@ -39,14 +39,14 @@ public class DescribeClientQuotasResult { * * @param entities future for the collection of entities that matched the filter */ - public DescribeClientQuotasResult(KafkaFuture>> entities) { + public DescribeClientQuotasResult(KafkaFuture>> entities) { this.entities = entities; } /** * Returns a map from quota entity to a future which can be used to check the status of the operation. */ - public KafkaFuture>> entities() { + public KafkaFuture>> entities() { return entities; } } diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java index ef78cd9cc27a8..c35144c790280 100644 --- a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java +++ b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java @@ -124,9 +124,9 @@ import org.apache.kafka.common.network.ChannelBuilder; import org.apache.kafka.common.network.Selector; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.quota.QuotaAlteration; -import org.apache.kafka.common.quota.QuotaEntity; -import org.apache.kafka.common.quota.QuotaFilter; +import org.apache.kafka.common.quota.ClientQuotaAlteration; +import org.apache.kafka.common.quota.ClientQuotaEntity; +import org.apache.kafka.common.quota.ClientQuotaFilter; import org.apache.kafka.common.requests.AbstractRequest; import org.apache.kafka.common.requests.AbstractResponse; import org.apache.kafka.common.requests.AlterClientQuotasRequest; @@ -3831,8 +3831,8 @@ void handleFailure(Throwable throwable) { } @Override - public DescribeClientQuotasResult describeClientQuotas(Collection filters, DescribeClientQuotasOptions options) { - KafkaFutureImpl>> future = new KafkaFutureImpl<>(); + public DescribeClientQuotasResult describeClientQuotas(ClientQuotaFilter filter, DescribeClientQuotasOptions options) { + KafkaFutureImpl>> future = new KafkaFutureImpl<>(); final long now = time.milliseconds(); runnable.call(new Call("describeClientQuotas", calcDeadlineMs(now, options.timeoutMs()), @@ -3840,7 +3840,7 @@ public DescribeClientQuotasResult describeClientQuotas(Collection f @Override DescribeClientQuotasRequest.Builder createRequest(int timeoutMs) { - return new DescribeClientQuotasRequest.Builder(filters); + return new DescribeClientQuotasRequest.Builder(filter); } @Override @@ -3859,9 +3859,9 @@ void handleFailure(Throwable throwable) { } @Override - public AlterClientQuotasResult alterClientQuotas(Collection entries, AlterClientQuotasOptions options) { - Map> futures = new HashMap<>(entries.size()); - for (QuotaAlteration entry : entries) { + public AlterClientQuotasResult alterClientQuotas(Collection entries, AlterClientQuotasOptions options) { + Map> futures = new HashMap<>(entries.size()); + for (ClientQuotaAlteration entry : entries) { futures.put(entry.entity(), new KafkaFutureImpl<>()); } diff --git a/clients/src/main/java/org/apache/kafka/common/quota/QuotaAlteration.java b/clients/src/main/java/org/apache/kafka/common/quota/ClientQuotaAlteration.java similarity index 84% rename from clients/src/main/java/org/apache/kafka/common/quota/QuotaAlteration.java rename to clients/src/main/java/org/apache/kafka/common/quota/ClientQuotaAlteration.java index f992681f0b679..88670ce8ddc05 100644 --- a/clients/src/main/java/org/apache/kafka/common/quota/QuotaAlteration.java +++ b/clients/src/main/java/org/apache/kafka/common/quota/ClientQuotaAlteration.java @@ -21,9 +21,9 @@ import java.util.Objects; /** - * Describes a configuration alteration to be made to a quota entity. + * Describes a configuration alteration to be made to a client quota entity. */ -public class QuotaAlteration { +public class ClientQuotaAlteration { public static class Op { private final String key; @@ -69,18 +69,18 @@ public int hashCode() { @Override public String toString() { - return "QuotaAlteration.Op(key=" + key + ", value=" + value + ")"; + return "ClientQuotaAlteration.Op(key=" + key + ", value=" + value + ")"; } } - private final QuotaEntity entity; + private final ClientQuotaEntity entity; private final Collection ops; /** * @param entity the entity whose config will be modified * @param ops the alteration to perform */ - public QuotaAlteration(QuotaEntity entity, Collection ops) { + public ClientQuotaAlteration(ClientQuotaEntity entity, Collection ops) { this.entity = entity; this.ops = ops; } @@ -88,7 +88,7 @@ public QuotaAlteration(QuotaEntity entity, Collection ops) { /** * @return the entity whose config will be modified */ - public QuotaEntity entity() { + public ClientQuotaEntity entity() { return this.entity; } @@ -98,4 +98,9 @@ public QuotaEntity entity() { public Collection ops() { return this.ops; } + + @Override + public String toString() { + return "ClientQuotaAlteration(entity=" + entity + ", ops=" + ops + ")"; + } } diff --git a/clients/src/main/java/org/apache/kafka/common/quota/QuotaEntity.java b/clients/src/main/java/org/apache/kafka/common/quota/ClientQuotaEntity.java similarity index 72% rename from clients/src/main/java/org/apache/kafka/common/quota/QuotaEntity.java rename to clients/src/main/java/org/apache/kafka/common/quota/ClientQuotaEntity.java index 8293f1aa3c766..0fee3d3bed106 100644 --- a/clients/src/main/java/org/apache/kafka/common/quota/QuotaEntity.java +++ b/clients/src/main/java/org/apache/kafka/common/quota/ClientQuotaEntity.java @@ -21,31 +21,25 @@ import java.util.Objects; /** - * Describes a quota entity, which is a mapping of entity types to their names. + * Describes a client quota entity, which is a mapping of entity types to their names. */ -public class QuotaEntity { +public class ClientQuotaEntity { private final Map entries; /** - * Type of an entity entry. + * The type of an entity entry. */ public static final String USER = "user"; public static final String CLIENT_ID = "client-id"; /** - * Represents the default name for an user/client ID, i.e. the name that's resolved - * when an exact match isn't found. - */ - public final static String USER_DEFAULT = ""; - public final static String CLIENT_ID_DEFAULT = ""; - - /** - * Constructs a quota entity for the given types and names. + * Constructs a quota entity for the given types and names. If a name is null, + * then it is mapped to the built-in default entity name. * * @param entries maps entity type to its name */ - public QuotaEntity(Map entries) { + public ClientQuotaEntity(Map entries) { this.entries = entries; } @@ -60,7 +54,7 @@ public Map entries() { public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - QuotaEntity that = (QuotaEntity) o; + ClientQuotaEntity that = (ClientQuotaEntity) o; return Objects.equals(entries, that.entries); } @@ -71,6 +65,6 @@ public int hashCode() { @Override public String toString() { - return "QuotaEntity(entries=" + entries + ")"; + return "ClientQuotaEntity(entries=" + entries + ")"; } } diff --git a/clients/src/main/java/org/apache/kafka/common/quota/ClientQuotaFilter.java b/clients/src/main/java/org/apache/kafka/common/quota/ClientQuotaFilter.java new file mode 100644 index 0000000000000..e8a6a7290d480 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/common/quota/ClientQuotaFilter.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.common.quota; + +import java.util.Collection; +import java.util.Collections; +import java.util.Objects; + +/** + * Describes a client quota entity filter. + */ +public class ClientQuotaFilter { + + private final Collection components; + private final boolean strict; + + /** + * A filter to be applied to matching client quotas. + * + * @param components the components to filter on + * @param strict whether the filter only includes specified components + */ + private ClientQuotaFilter(Collection components, boolean strict) { + this.components = components; + this.strict = strict; + } + + /** + * Constructs and returns a quota filter that matches all provided components. Matching entities + * with entity types that are not specified by a component will also be included in the result. + * + * @param components the components for the filter + */ + public static ClientQuotaFilter contains(Collection components) { + return new ClientQuotaFilter(components, false); + } + + /** + * Constructs and returns a quota filter that matches all provided components. Matching entities + * with entity types that are not specified by a component will *not* be included in the result. + * + * @param components the components for the filter + */ + public static ClientQuotaFilter containsOnly(Collection components) { + return new ClientQuotaFilter(components, true); + } + + /** + * Constructs and returns a quota filter that matches all configured entities. + */ + public static ClientQuotaFilter all() { + return new ClientQuotaFilter(Collections.emptyList(), false); + } + + /** + * @return the filter's components + */ + public Collection components() { + return this.components; + } + + /** + * @return whether the filter is strict, i.e. only includes specified components + */ + public boolean strict() { + return this.strict; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ClientQuotaFilter that = (ClientQuotaFilter) o; + return Objects.equals(components, that.components) && Objects.equals(strict, that.strict); + } + + @Override + public int hashCode() { + return Objects.hash(components, strict); + } + + @Override + public String toString() { + return "ClientQuotaFilter(components=" + components + ", strict=" + strict + ")"; + } +} diff --git a/clients/src/main/java/org/apache/kafka/common/quota/ClientQuotaFilterComponent.java b/clients/src/main/java/org/apache/kafka/common/quota/ClientQuotaFilterComponent.java new file mode 100644 index 0000000000000..b981ead63fbc3 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/common/quota/ClientQuotaFilterComponent.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.common.quota; + +import java.util.Objects; +import java.util.Optional; + +/** + * Describes a component for applying a client quota filter. + */ +public class ClientQuotaFilterComponent { + + private final String entityType; + private final Optional match; + + /** + * A filter to be applied. + * + * @param entityType the entity type the filter component applies to + * @param match if present, the name that's matched exactly + * if empty, matches the default name + * if null, matches any specified name + */ + private ClientQuotaFilterComponent(String entityType, Optional match) { + this.entityType = Objects.requireNonNull(entityType); + this.match = match; + } + + /** + * Constructs and returns a filter component that exactly matches the provided entity + * name for the entity type. + * + * @param entityType the entity type the filter component applies to + * @param entityName the entity name that's matched exactly + */ + public static ClientQuotaFilterComponent ofEntity(String entityType, String entityName) { + return new ClientQuotaFilterComponent(entityType, Optional.of(Objects.requireNonNull(entityName))); + } + + /** + * Constructs and returns a filter component that matches the built-in default entity name + * for the entity type. + * + * @param entityType the entity type the filter component applies to + */ + public static ClientQuotaFilterComponent ofDefaultEntity(String entityType) { + return new ClientQuotaFilterComponent(entityType, Optional.empty()); + } + + /** + * Constructs and returns a filter component that matches any specified name for the + * entity type. + * + * @param entityType the entity type the filter component applies to + */ + public static ClientQuotaFilterComponent ofEntityType(String entityType) { + return new ClientQuotaFilterComponent(entityType, null); + } + + /** + * @return the component's entity type + */ + public String entityType() { + return this.entityType; + } + + /** + * @return the optional match string, where: + * if present, the name that's matched exactly + * if empty, matches the default name + * if null, matches any specified name + */ + public Optional match() { + return this.match; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ClientQuotaFilterComponent that = (ClientQuotaFilterComponent) o; + return Objects.equals(entityType, match); + } + + @Override + public int hashCode() { + return Objects.hash(entityType, match); + } + + @Override + public String toString() { + return "ClientQuotaFilterComponent(entityType=" + entityType + ", match=" + match + ")"; + } +} diff --git a/clients/src/main/java/org/apache/kafka/common/quota/QuotaFilter.java b/clients/src/main/java/org/apache/kafka/common/quota/QuotaFilter.java deleted file mode 100644 index 822a128e056b7..0000000000000 --- a/clients/src/main/java/org/apache/kafka/common/quota/QuotaFilter.java +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kafka.common.quota; - -import java.util.Objects; -import java.util.Optional; - -/** - * Describes a quota entity filter. - */ -public class QuotaFilter { - - private final String entityType; - private final Optional match; - - /** - * A filter to be applied. - * - * @param entityType the entity type the filter applies to - * @param if specified, match the name that's matched exactly - * if empty, matches any specified name - * if null, matches no specified name - */ - private QuotaFilter(String entityType, Optional match) { - this.entityType = Objects.requireNonNull(entityType); - this.match = match; - } - - /** - * Constructs and returns a quota filter that matches the provided entity - * name for the entity type exactly. - * - * @param entityType the entity type the filter applies to - * @param entityName the entity name that's matched exactly - */ - public static QuotaFilter matchExact(String entityType, String entityName) { - return new QuotaFilter(entityType, Optional.of(Objects.requireNonNull(entityName))); - } - - /** - * Constructs and returns a quota filter that matches any specified name for - * the entity type. - * - * @param entityType the entity type the filter applies to - */ - public static QuotaFilter matchSome(String entityType) { - return new QuotaFilter(entityType, Optional.empty()); - } - - /** - * Constructs and returns a quota filter that matches no entity name for the - * entity type exactly. - * - * @param entityType the entity type the filter applies to - */ - public static QuotaFilter matchNone(String entityType) { - return new QuotaFilter(entityType, null); - } - - /** - * @return the entity type the filter applies to - */ - public String entityType() { - return this.entityType; - } - - /** - * @return whether to match the exact entity name - */ - public boolean isMatchExact() { - return this.match != null && this.match.isPresent(); - } - - /** - * @return the string that's matched exactly - */ - public String matchExact() { - return this.match.get(); - } - - /** - * @return whether to match the exact entity name - */ - public boolean isMatchSome() { - return this.match != null && !this.match.isPresent(); - } - - /** - * @return whether to match no entities with the entity type specified - */ - public boolean isMatchNone() { - return this.match == null; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - QuotaFilter that = (QuotaFilter) o; - return Objects.equals(entityType, that.entityType) && Objects.equals(match, that.match); - } - - @Override - public int hashCode() { - return Objects.hash(entityType, match); - } - - @Override - public String toString() { - return "QuotaFilter(entityType=" + entityType + ", match=" + match + ")"; - } -} diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasRequest.java index a110c5d8b6152..6d4c2a1f357c3 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasRequest.java @@ -22,8 +22,8 @@ import org.apache.kafka.common.message.AlterClientQuotasRequestData.OpData; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.types.Struct; -import org.apache.kafka.common.quota.QuotaAlteration; -import org.apache.kafka.common.quota.QuotaEntity; +import org.apache.kafka.common.quota.ClientQuotaAlteration; +import org.apache.kafka.common.quota.ClientQuotaEntity; import java.util.ArrayList; import java.util.Collection; @@ -37,11 +37,11 @@ public static class Builder extends AbstractRequest.Builder entries, boolean validateOnly) { + public Builder(Collection entries, boolean validateOnly) { super(ApiKeys.ALTER_CLIENT_QUOTAS); List entryData = new ArrayList<>(entries.size()); - for (QuotaAlteration entry : entries) { + for (ClientQuotaAlteration entry : entries) { List entityData = new ArrayList<>(entry.entity().entries().size()); for (Map.Entry entityEntries : entry.entity().entries().entrySet()) { entityData.add(new EntityData() @@ -50,7 +50,7 @@ public Builder(Collection entries, boolean validateOnly) { } List opData = new ArrayList<>(entry.ops().size()); - for (QuotaAlteration.Op op : entry.ops()) { + for (ClientQuotaAlteration.Op op : entry.ops()) { opData.add(new OpData() .setKey(op.key()) .setValue(op.value() == null ? 0.0 : op.value()) @@ -90,21 +90,21 @@ public AlterClientQuotasRequest(Struct struct, short version) { this.data = new AlterClientQuotasRequestData(struct, version); } - public Collection entries() { - List entries = new ArrayList<>(data.entries().size()); + public Collection entries() { + List entries = new ArrayList<>(data.entries().size()); for (EntryData entryData : data.entries()) { Map entity = new HashMap<>(entryData.entity().size()); for (EntityData entityData : entryData.entity()) { entity.put(entityData.entityType(), entityData.entityName()); } - List ops = new ArrayList<>(entryData.ops().size()); + List ops = new ArrayList<>(entryData.ops().size()); for (OpData opData : entryData.ops()) { Double value = opData.remove() ? null : opData.value(); - ops.add(new QuotaAlteration.Op(opData.key(), value)); + ops.add(new ClientQuotaAlteration.Op(opData.key(), value)); } - entries.add(new QuotaAlteration(new QuotaEntity(entity), ops)); + entries.add(new ClientQuotaAlteration(new ClientQuotaEntity(entity), ops)); } return entries; } @@ -115,13 +115,13 @@ public boolean validateOnly() { @Override public AlterClientQuotasResponse getErrorResponse(int throttleTimeMs, Throwable e) { - ArrayList entities = new ArrayList<>(data.entries().size()); + ArrayList entities = new ArrayList<>(data.entries().size()); for (EntryData entryData : data.entries()) { Map entity = new HashMap<>(entryData.entity().size()); for (EntityData entityData : entryData.entity()) { entity.put(entityData.entityType(), entityData.entityName()); } - entities.add(new QuotaEntity(entity)); + entities.add(new ClientQuotaEntity(entity)); } return new AlterClientQuotasResponse(entities, throttleTimeMs, e); } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasResponse.java index aca941df3276d..7e4e8917a8896 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AlterClientQuotasResponse.java @@ -22,7 +22,7 @@ import org.apache.kafka.common.message.AlterClientQuotasResponseData.EntryData; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.types.Struct; -import org.apache.kafka.common.quota.QuotaEntity; +import org.apache.kafka.common.quota.ClientQuotaEntity; import java.util.ArrayList; import java.util.Collection; @@ -34,9 +34,9 @@ public class AlterClientQuotasResponse extends AbstractResponse { private final AlterClientQuotasResponseData data; - public AlterClientQuotasResponse(Map result, int throttleTimeMs) { + public AlterClientQuotasResponse(Map result, int throttleTimeMs) { List entries = new ArrayList<>(result.size()); - for (Map.Entry entry : result.entrySet()) { + for (Map.Entry entry : result.entrySet()) { ApiError e = entry.getValue(); entries.add(new EntryData() .setErrorCode(e.error().code()) @@ -49,12 +49,12 @@ public AlterClientQuotasResponse(Map result, int throttle .setEntries(entries); } - public AlterClientQuotasResponse(Collection entities, int throttleTimeMs, Throwable e) { + public AlterClientQuotasResponse(Collection entities, int throttleTimeMs, Throwable e) { short errorCode = Errors.forException(e).code(); String errorMessage = e.getMessage(); List entries = new ArrayList<>(entities.size()); - for (QuotaEntity entity : entities) { + for (ClientQuotaEntity entity : entities) { entries.add(new EntryData() .setErrorCode(errorCode) .setErrorMessage(errorMessage) @@ -70,13 +70,13 @@ public AlterClientQuotasResponse(Struct struct, short version) { this.data = new AlterClientQuotasResponseData(struct, version); } - public void complete(Map> futures) { + public void complete(Map> futures) { for (EntryData entryData : data.entries()) { Map entityEntries = new HashMap<>(entryData.entity().size()); for (EntityData entityData : entryData.entity()) { entityEntries.put(entityData.entityType(), entityData.entityName()); } - QuotaEntity entity = new QuotaEntity(entityEntries); + ClientQuotaEntity entity = new ClientQuotaEntity(entityEntries); KafkaFutureImpl future = futures.get(entity); if (future == null) { @@ -112,7 +112,7 @@ protected Struct toStruct(short version) { return data.toStruct(version); } - private static List toEntityData(QuotaEntity entity) { + private static List toEntityData(ClientQuotaEntity entity) { List entityData = new ArrayList<>(entity.entries().size()); for (Map.Entry entry : entity.entries().entrySet()) { entityData.add(new AlterClientQuotasResponseData.EntityData() diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasRequest.java index b7653a1cbf5ac..a5496ef0f619b 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasRequest.java @@ -17,10 +17,11 @@ package org.apache.kafka.common.requests; import org.apache.kafka.common.message.DescribeClientQuotasRequestData; -import org.apache.kafka.common.message.DescribeClientQuotasRequestData.FilterData; +import org.apache.kafka.common.message.DescribeClientQuotasRequestData.ComponentData; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.types.Struct; -import org.apache.kafka.common.quota.QuotaFilter; +import org.apache.kafka.common.quota.ClientQuotaFilter; +import org.apache.kafka.common.quota.ClientQuotaFilterComponent; import java.util.ArrayList; import java.util.List; @@ -29,33 +30,34 @@ public class DescribeClientQuotasRequest extends AbstractRequest { // These values must not change. private static final byte MATCH_TYPE_EXACT = 0; - private static final byte MATCH_TYPE_SOME = 1; - private static final byte MATCH_TYPE_NONE = 2; + private static final byte MATCH_TYPE_DEFAULT = 1; + private static final byte MATCH_TYPE_SPECIFIED = 2; public static class Builder extends AbstractRequest.Builder { private final DescribeClientQuotasRequestData data; - public Builder(Collection filters) { + public Builder(ClientQuotaFilter filter) { super(ApiKeys.DESCRIBE_CLIENT_QUOTAS); - List filterData = new ArrayList<>(filters.size()); - for (QuotaFilter filter : filters) { - FilterData fd = new FilterData().setEntityType(filter.entityType()); - if (filter.isMatchSome()) { - fd.setMatchType(MATCH_TYPE_SOME); + List componentData = new ArrayList<>(filter.components().size()); + for (ClientQuotaFilterComponent component : filter.components()) { + ComponentData fd = new ComponentData().setEntityType(component.entityType()); + if (component.match() == null) { + fd.setMatchType(MATCH_TYPE_SPECIFIED); fd.setMatch(null); - } else if (filter.isMatchNone()) { - fd.setMatchType(MATCH_TYPE_NONE); - fd.setMatch(null); - } else { + } else if (component.match().isPresent()) { fd.setMatchType(MATCH_TYPE_EXACT); - fd.setMatch(filter.matchExact()); + fd.setMatch(component.match().get()); + } else { + fd.setMatchType(MATCH_TYPE_DEFAULT); + fd.setMatch(null); } - filterData.add(fd); + componentData.add(fd); } this.data = new DescribeClientQuotasRequestData() - .setFilters(filterData); + .setComponents(componentData) + .setStrict(filter.strict()); } @Override @@ -81,26 +83,30 @@ public DescribeClientQuotasRequest(Struct struct, short version) { this.data = new DescribeClientQuotasRequestData(struct, version); } - public Collection filters() { - List filters = new ArrayList<>(data.filters().size()); - for (FilterData filterData : data.filters()) { - QuotaFilter filter; - switch (filterData.matchType()) { + public ClientQuotaFilter filter() { + List components = new ArrayList<>(data.components().size()); + for (ComponentData componentData : data.components()) { + ClientQuotaFilterComponent component; + switch (componentData.matchType()) { case MATCH_TYPE_EXACT: - filter = QuotaFilter.matchExact(filterData.entityType(), filterData.match()); + component = ClientQuotaFilterComponent.ofEntity(componentData.entityType(), componentData.match()); break; - case MATCH_TYPE_SOME: - filter = QuotaFilter.matchSome(filterData.entityType()); + case MATCH_TYPE_DEFAULT: + component = ClientQuotaFilterComponent.ofDefaultEntity(componentData.entityType()); break; - case MATCH_TYPE_NONE: - filter = QuotaFilter.matchNone(filterData.entityType()); + case MATCH_TYPE_SPECIFIED: + component = ClientQuotaFilterComponent.ofEntityType(componentData.entityType()); break; default: - throw new IllegalArgumentException("Unexpected match type: " + filterData.matchType()); + throw new IllegalArgumentException("Unexpected match type: " + componentData.matchType()); } - filters.add(filter); + components.add(component); + } + if (data.strict()) { + return ClientQuotaFilter.containsOnly(components); + } else { + return ClientQuotaFilter.contains(components); } - return filters; } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasResponse.java index 63538c5ffbaf9..6ed5b1b54e59d 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeClientQuotasResponse.java @@ -23,7 +23,7 @@ import org.apache.kafka.common.message.DescribeClientQuotasResponseData.ValueData; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.types.Struct; -import org.apache.kafka.common.quota.QuotaEntity; +import org.apache.kafka.common.quota.ClientQuotaEntity; import java.util.ArrayList; import java.util.Collections; @@ -35,10 +35,10 @@ public class DescribeClientQuotasResponse extends AbstractResponse { private final DescribeClientQuotasResponseData data; - public DescribeClientQuotasResponse(Map> entities, int throttleTimeMs) { + public DescribeClientQuotasResponse(Map> entities, int throttleTimeMs) { List entries = new ArrayList<>(entities.size()); - for (Map.Entry> entry : entities.entrySet()) { - QuotaEntity quotaEntity = entry.getKey(); + for (Map.Entry> entry : entities.entrySet()) { + ClientQuotaEntity quotaEntity = entry.getKey(); List entityData = new ArrayList<>(quotaEntity.entries().size()); for (Map.Entry entityEntry : quotaEntity.entries().entrySet()) { entityData.add(new EntityData() @@ -78,14 +78,14 @@ public DescribeClientQuotasResponse(Struct struct, short version) { this.data = new DescribeClientQuotasResponseData(struct, version); } - public void complete(KafkaFutureImpl>> future) { + public void complete(KafkaFutureImpl>> future) { Errors error = Errors.forCode(data.errorCode()); if (error != Errors.NONE) { future.completeExceptionally(error.exception(data.errorMessage())); return; } - Map> result = new HashMap<>(data.entries().size()); + Map> result = new HashMap<>(data.entries().size()); for (EntryData entries : data.entries()) { Map entity = new HashMap<>(entries.entity().size()); for (EntityData entityData : entries.entity()) { @@ -97,7 +97,7 @@ public void complete(KafkaFutureImpl>> futu values.put(valueData.key(), valueData.value()); } - result.put(new QuotaEntity(entity), values); + result.put(new ClientQuotaEntity(entity), values); } future.complete(result); } diff --git a/clients/src/main/resources/common/message/AlterClientQuotasRequest.json b/clients/src/main/resources/common/message/AlterClientQuotasRequest.json index 0d979b5dc4706..7e74d44980750 100644 --- a/clients/src/main/resources/common/message/AlterClientQuotasRequest.json +++ b/clients/src/main/resources/common/message/AlterClientQuotasRequest.json @@ -26,8 +26,8 @@ "about": "The quota entity to alter.", "fields": [ { "name": "EntityType", "type": "string", "versions": "0+", "about": "The entity type." }, - { "name": "EntityName", "type": "string", "versions": "0+", - "about": "The name of the entity." } + { "name": "EntityName", "type": "string", "versions": "0+", "nullableVersions": "0+", + "about": "The name of the entity, or null if the default." } ]}, { "name": "Ops", "type": "[]OpData", "versions": "0+", "about": "An individual quota configuration entry to alter.", "fields": [ diff --git a/clients/src/main/resources/common/message/AlterClientQuotasResponse.json b/clients/src/main/resources/common/message/AlterClientQuotasResponse.json index 48f1cf16b1e05..2f8fb88620bb4 100644 --- a/clients/src/main/resources/common/message/AlterClientQuotasResponse.json +++ b/clients/src/main/resources/common/message/AlterClientQuotasResponse.json @@ -32,8 +32,8 @@ "about": "The quota entity to alter.", "fields": [ { "name": "EntityType", "type": "string", "versions": "0+", "about": "The entity type." }, - { "name": "EntityName", "type": "string", "versions": "0+", - "about": "The name of the entity." } + { "name": "EntityName", "type": "string", "versions": "0+", "nullableVersions": "0+", + "about": "The name of the entity, or null if the default." } ]} ]} ] diff --git a/clients/src/main/resources/common/message/DescribeClientQuotasRequest.json b/clients/src/main/resources/common/message/DescribeClientQuotasRequest.json index 06c77483494bd..7abfd3cb1348e 100644 --- a/clients/src/main/resources/common/message/DescribeClientQuotasRequest.json +++ b/clients/src/main/resources/common/message/DescribeClientQuotasRequest.json @@ -20,14 +20,16 @@ "validVersions": "0", "flexibleVersions": "none", "fields": [ - { "name": "Filters", "type": "[]FilterData", "versions": "0+", - "about": "Filters to apply to quota entities.", "fields": [ + { "name": "Components", "type": "[]ComponentData", "versions": "0+", + "about": "Filter components to apply to quota entities.", "fields": [ { "name": "EntityType", "type": "string", "versions": "0+", - "about": "The entity type that the filter applies to." }, + "about": "The entity type that the filter component applies to." }, { "name": "MatchType", "type": "int8", "versions": "0+", - "about": "How to match the entity {0 = exact, 1 = some, 2 = none}." }, + "about": "How to match the entity {0 = exact name, 1 = default name, 2 = any specified name}." }, { "name": "Match", "type": "string", "versions": "0+", "nullableVersions": "0+", "about": "The string to match against, or null if unused for the match type." } - ]} + ]}, + { "name": "Strict", "type": "bool", "versions": "0+", + "about": "Whether the match is strict, i.e. should exclude entities with unspecified entity types." } ] } diff --git a/clients/src/main/resources/common/message/DescribeClientQuotasResponse.json b/clients/src/main/resources/common/message/DescribeClientQuotasResponse.json index 6ce1bb5fd4216..5f5c784336034 100644 --- a/clients/src/main/resources/common/message/DescribeClientQuotasResponse.json +++ b/clients/src/main/resources/common/message/DescribeClientQuotasResponse.json @@ -32,8 +32,8 @@ "about": "The quota entity description.", "fields": [ { "name": "EntityType", "type": "string", "versions": "0+", "about": "The entity type." }, - { "name": "EntityName", "type": "string", "versions": "0+", - "about": "The entity name." } + { "name": "EntityName", "type": "string", "versions": "0+", "nullableVersions": "0+", + "about": "The entity name, or null if the default." } ]}, { "name": "Values", "type": "[]ValueData", "versions": "0+", "about": "The quota values for the entity.", "fields": [ diff --git a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java index 0e1eea144e574..e563472109b05 100644 --- a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java @@ -88,9 +88,10 @@ import org.apache.kafka.common.message.OffsetDeleteResponseData.OffsetDeleteResponseTopic; import org.apache.kafka.common.message.OffsetDeleteResponseData.OffsetDeleteResponseTopicCollection; import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.quota.QuotaAlteration; -import org.apache.kafka.common.quota.QuotaEntity; -import org.apache.kafka.common.quota.QuotaFilter; +import org.apache.kafka.common.quota.ClientQuotaAlteration; +import org.apache.kafka.common.quota.ClientQuotaEntity; +import org.apache.kafka.common.quota.ClientQuotaFilter; +import org.apache.kafka.common.quota.ClientQuotaFilterComponent; import org.apache.kafka.common.requests.AlterClientQuotasResponse; import org.apache.kafka.common.requests.AlterPartitionReassignmentsResponse; import org.apache.kafka.common.requests.ApiError; @@ -703,7 +704,7 @@ private void callAdminClientApisAndExpectAnAuthenticationError(AdminClientUnitTe private void callClientQuotasApisAndExpectAnAuthenticationError(AdminClientUnitTestEnv env) throws InterruptedException { try { - env.adminClient().describeClientQuotas(Collections.emptyList()).entities().get(); + env.adminClient().describeClientQuotas(ClientQuotaFilter.all()).entities().get(); fail("Expected an authentication error."); } catch (ExecutionException e) { assertTrue("Expected an authentication error, but got " + Utils.stackTrace(e), @@ -711,8 +712,8 @@ private void callClientQuotasApisAndExpectAnAuthenticationError(AdminClientUnitT } try { - QuotaEntity entity = new QuotaEntity(Collections.singletonMap(QuotaEntity.USER, "user")); - QuotaAlteration alteration = new QuotaAlteration(entity, asList(new QuotaAlteration.Op("consumer_byte_rate", 1000.0))); + ClientQuotaEntity entity = new ClientQuotaEntity(Collections.singletonMap(ClientQuotaEntity.USER, "user")); + ClientQuotaAlteration alteration = new ClientQuotaAlteration(entity, asList(new ClientQuotaAlteration.Op("consumer_byte_rate", 1000.0))); env.adminClient().alterClientQuotas(asList(alteration)).all().get(); fail("Expected an authentication error."); } catch (ExecutionException e) { @@ -2820,14 +2821,14 @@ public void testRequestTimeoutExceedingDefaultApiTimeout() throws Exception { } } - private QuotaEntity newQuotaEntity(String... args) { + private ClientQuotaEntity newClientQuotaEntity(String... args) { assertTrue(args.length % 2 == 0); Map entityMap = new HashMap<>(args.length / 2); for (int index = 0; index < args.length; index += 2) { entityMap.put(args[index], args[index + 1]); } - return new QuotaEntity(entityMap); + return new ClientQuotaEntity(entityMap); } @Test @@ -2835,21 +2836,20 @@ public void testDescribeClientQuotas() throws Exception { try (AdminClientUnitTestEnv env = mockClientEnv()) { env.kafkaClient().setNodeApiVersions(NodeApiVersions.create()); - final String customKey = "custom"; - final String customValue = "custom-value"; + final String value = "value"; - Map> responseData = new HashMap<>(); - QuotaEntity entity1 = newQuotaEntity(QuotaEntity.USER, "user-1", customKey, customValue); - QuotaEntity entity2 = newQuotaEntity(QuotaEntity.USER, "user-2", customKey, customValue); + Map> responseData = new HashMap<>(); + ClientQuotaEntity entity1 = newClientQuotaEntity(ClientQuotaEntity.USER, "user-1", ClientQuotaEntity.CLIENT_ID, value); + ClientQuotaEntity entity2 = newClientQuotaEntity(ClientQuotaEntity.USER, "user-2", ClientQuotaEntity.CLIENT_ID, value); responseData.put(entity1, Collections.singletonMap("consumer_byte_rate", 10000.0)); responseData.put(entity2, Collections.singletonMap("producer_byte_rate", 20000.0)); env.kafkaClient().prepareResponse(new DescribeClientQuotasResponse(responseData, 0)); - Collection filters = asList(QuotaFilter.matchExact(customKey, customValue)); + ClientQuotaFilter filter = ClientQuotaFilter.contains(asList(ClientQuotaFilterComponent.ofEntity(ClientQuotaEntity.USER, value))); - DescribeClientQuotasResult result = env.adminClient().describeClientQuotas(filters); - Map> resultData = result.entities().get(); + DescribeClientQuotasResult result = env.adminClient().describeClientQuotas(filter); + Map> resultData = result.entities().get(); assertEquals(resultData.size(), 2); assertTrue(resultData.containsKey(entity1)); Map config1 = resultData.get(entity1); @@ -2866,21 +2866,21 @@ public void testAlterClientQuotas() throws Exception { try (AdminClientUnitTestEnv env = mockClientEnv()) { env.kafkaClient().setNodeApiVersions(NodeApiVersions.create()); - QuotaEntity goodEntity = newQuotaEntity(QuotaEntity.USER, "user-1"); - QuotaEntity unauthorizedEntity = newQuotaEntity(QuotaEntity.USER, "user-0"); - QuotaEntity invalidEntity = newQuotaEntity("", "user-0"); + ClientQuotaEntity goodEntity = newClientQuotaEntity(ClientQuotaEntity.USER, "user-1"); + ClientQuotaEntity unauthorizedEntity = newClientQuotaEntity(ClientQuotaEntity.USER, "user-0"); + ClientQuotaEntity invalidEntity = newClientQuotaEntity("", "user-0"); - Map responseData = new HashMap<>(2); + Map responseData = new HashMap<>(2); responseData.put(goodEntity, new ApiError(Errors.CLUSTER_AUTHORIZATION_FAILED, "Authorization failed")); responseData.put(unauthorizedEntity, new ApiError(Errors.CLUSTER_AUTHORIZATION_FAILED, "Authorization failed")); responseData.put(invalidEntity, new ApiError(Errors.INVALID_REQUEST, "Invalid quota entity")); env.kafkaClient().prepareResponse(new AlterClientQuotasResponse(responseData, 0)); - List entries = new ArrayList<>(3); - entries.add(new QuotaAlteration(goodEntity, Collections.singleton(new QuotaAlteration.Op("consumer_byte_rate", 10000.0)))); - entries.add(new QuotaAlteration(unauthorizedEntity, Collections.singleton(new QuotaAlteration.Op("producer_byte_rate", 10000.0)))); - entries.add(new QuotaAlteration(invalidEntity, Collections.singleton(new QuotaAlteration.Op("producer_byte_rate", 100.0)))); + List entries = new ArrayList<>(3); + entries.add(new ClientQuotaAlteration(goodEntity, Collections.singleton(new ClientQuotaAlteration.Op("consumer_byte_rate", 10000.0)))); + entries.add(new ClientQuotaAlteration(unauthorizedEntity, Collections.singleton(new ClientQuotaAlteration.Op("producer_byte_rate", 10000.0)))); + entries.add(new ClientQuotaAlteration(invalidEntity, Collections.singleton(new ClientQuotaAlteration.Op("producer_byte_rate", 100.0)))); AlterClientQuotasResult result = env.adminClient().alterClientQuotas(entries); result.values().get(goodEntity); diff --git a/clients/src/test/java/org/apache/kafka/clients/admin/MockAdminClient.java b/clients/src/test/java/org/apache/kafka/clients/admin/MockAdminClient.java index f14d7390e7e0f..c6d9d14fde73c 100644 --- a/clients/src/test/java/org/apache/kafka/clients/admin/MockAdminClient.java +++ b/clients/src/test/java/org/apache/kafka/clients/admin/MockAdminClient.java @@ -33,8 +33,8 @@ import org.apache.kafka.common.errors.TopicExistsException; import org.apache.kafka.common.errors.UnknownTopicOrPartitionException; import org.apache.kafka.common.internals.KafkaFutureImpl; -import org.apache.kafka.common.quota.QuotaAlteration; -import org.apache.kafka.common.quota.QuotaFilter; +import org.apache.kafka.common.quota.ClientQuotaAlteration; +import org.apache.kafka.common.quota.ClientQuotaFilter; import java.time.Duration; import java.util.ArrayList; @@ -461,12 +461,12 @@ public ListOffsetsResult listOffsets(Map topicPartit } @Override - public DescribeClientQuotasResult describeClientQuotas(Collection filters, DescribeClientQuotasOptions options) { + public DescribeClientQuotasResult describeClientQuotas(ClientQuotaFilter filter, DescribeClientQuotasOptions options) { throw new UnsupportedOperationException("Not implemented yet"); } @Override - public AlterClientQuotasResult alterClientQuotas(Collection entries, AlterClientQuotasOptions options) { + public AlterClientQuotasResult alterClientQuotas(Collection entries, AlterClientQuotasOptions options) { throw new UnsupportedOperationException("Not implemented yet"); } diff --git a/core/src/main/scala/kafka/admin/ConfigCommand.scala b/core/src/main/scala/kafka/admin/ConfigCommand.scala index 2e4ecd11eed8e..de2b593883fc2 100644 --- a/core/src/main/scala/kafka/admin/ConfigCommand.scala +++ b/core/src/main/scala/kafka/admin/ConfigCommand.scala @@ -33,7 +33,7 @@ import org.apache.kafka.common.config.ConfigResource import org.apache.kafka.common.config.types.Password import org.apache.kafka.common.errors.InvalidConfigurationException import org.apache.kafka.common.internals.Topic -import org.apache.kafka.common.quota.{QuotaAlteration, QuotaEntity, QuotaFilter} +import org.apache.kafka.common.quota.{ClientQuotaAlteration, ClientQuotaEntity, ClientQuotaFilter, ClientQuotaFilterComponent} import org.apache.kafka.common.security.JaasUtils import org.apache.kafka.common.security.scram.internals.{ScramCredentialUtils, ScramFormatter, ScramMechanism} import org.apache.kafka.common.utils.{Sanitizer, Time, Utils} @@ -365,10 +365,10 @@ object ConfigCommand extends Config { if (invalidConfigs.nonEmpty) throw new InvalidConfigurationException(s"Invalid config(s): ${invalidConfigs.mkString(",")}") - val entity = new QuotaEntity(opts.entityTypes.map { entType => + val entity = new ClientQuotaEntity(opts.entityTypes.map { entType => entType match { - case ConfigType.User => QuotaEntity.USER - case ConfigType.Client => QuotaEntity.CLIENT_ID + case ConfigType.User => ClientQuotaEntity.USER + case ConfigType.Client => ClientQuotaEntity.CLIENT_ID case _ => throw new IllegalArgumentException(s"Unexpected entity type: ${entType}") } }.zip(opts.entityNames).toMap.asJava) @@ -379,10 +379,10 @@ object ConfigCommand extends Config { case _: NumberFormatException => throw new IllegalArgumentException(s"Cannot parse quota configuration value for ${key}: ${value}") } - new QuotaAlteration.Op(key, doubleValue) - } ++ configsToBeDeleted.map(key => new QuotaAlteration.Op(key, null))).asJavaCollection + new ClientQuotaAlteration.Op(key, doubleValue) + } ++ configsToBeDeleted.map(key => new ClientQuotaAlteration.Op(key, null))).asJavaCollection - adminClient.alterClientQuotas(Collections.singleton(new QuotaAlteration(entity, alterOps)), alterOptions) + adminClient.alterClientQuotas(Collections.singleton(new ClientQuotaAlteration(entity, alterOps)), alterOptions) .all().get(60, TimeUnit.SECONDS) case _ => throw new IllegalArgumentException(s"Unsupported entity type: $entityTypeHead") @@ -475,8 +475,8 @@ object ConfigCommand extends Config { private def describeClientQuotasConfig(adminClient: Admin, entityTypes: List[String], entityNames: List[String]) = { getAllClientQuotasConfigs(adminClient, entityTypes, entityNames).foreach { case (entity, entries) => val entityEntries = entity.entries.asScala - val entityStr = (entityEntries.get(QuotaEntity.USER).map(u => s"user-principal '${u}'") ++ - entityEntries.get(QuotaEntity.CLIENT_ID).map(c => s"client-id '${c}'")).mkString(", ") + val entityStr = (entityEntries.get(ClientQuotaEntity.USER).map(u => s"user-principal '${u}'") ++ + entityEntries.get(ClientQuotaEntity.CLIENT_ID).map(c => s"client-id '${c}'")).mkString(", ") val entriesStr = entries.asScala.map(e => s"${e._1}=${e._2}").mkString(", ") println(s"Configs for ${entityStr} are ${entriesStr}") } @@ -489,25 +489,17 @@ object ConfigCommand extends Config { } private def getAllClientQuotasConfigs(adminClient: Admin, entityTypes: List[String], entityNames: List[String]) = { - var userFilter: Option[QuotaFilter] = None - var clientIdFilter: Option[QuotaFilter] = None - - def toFilter(entityType: String, entityName: Option[String]): QuotaFilter = - entityName.map(QuotaFilter.matchExact(entityType, _)).getOrElse(QuotaFilter.matchSome(entityType)) - - entityTypes.map(Some(_)).zipAll(entityNames.map(Some(_)), None, None).foreach { case (entityTypeOpt, entityNameOpt) => - entityTypeOpt match { - case Some(ConfigType.User) => userFilter = Some(toFilter(QuotaEntity.USER, entityNameOpt)) - case Some(ConfigType.Client) => clientIdFilter = Some(toFilter(QuotaEntity.CLIENT_ID, entityNameOpt)) + val components = entityTypes.map(Some(_)).zipAll(entityNames.map(Some(_)), None, None).map { case (entityTypeOpt, entityNameOpt) => + val entityType = entityTypeOpt match { + case Some(ConfigType.User) => ClientQuotaEntity.USER + case Some(ConfigType.Client) => ClientQuotaEntity.CLIENT_ID case Some(_) => throw new IllegalArgumentException(s"Unexpected entity type ${entityTypeOpt.get}") case None => throw new IllegalArgumentException("More entity names specified than entity types") } + entityNameOpt.map(ClientQuotaFilterComponent.ofEntity(entityType, _)).getOrElse(ClientQuotaFilterComponent.ofEntityType(entityType)) } - val filters = List(userFilter.getOrElse(QuotaFilter.matchNone(QuotaEntity.USER)), - clientIdFilter.getOrElse(QuotaFilter.matchNone(QuotaEntity.CLIENT_ID))) - - adminClient.describeClientQuotas(filters.asJavaCollection).entities.get(30, TimeUnit.SECONDS).asScala + adminClient.describeClientQuotas(ClientQuotaFilter.containsOnly(components.asJava)).entities.get(30, TimeUnit.SECONDS).asScala } case class Entity(entityType: String, sanitizedName: Option[String]) { @@ -580,7 +572,7 @@ object ConfigCommand extends Config { val entityTypes = opts.entityTypes val entityNames = opts.entityNames if (entityTypes.head == ConfigType.User || entityTypes.head == ConfigType.Client) - parseQuotaEntity(opts, entityTypes, entityNames) + parseClientQuotaEntity(opts, entityTypes, entityNames) else { // Exactly one entity type and at-most one entity name expected for other entities val name = entityNames.headOption match { @@ -591,7 +583,7 @@ object ConfigCommand extends Config { } } - private def parseQuotaEntity(opts: ConfigCommandOptions, types: List[String], names: List[String]): ConfigEntity = { + private def parseClientQuotaEntity(opts: ConfigCommandOptions, types: List[String], names: List[String]): ConfigEntity = { if (opts.options.has(opts.alterOpt) && names.size != types.size) throw new IllegalArgumentException("--entity-name or --entity-default must be specified with each --entity-type for --alter") diff --git a/core/src/main/scala/kafka/server/AdminManager.scala b/core/src/main/scala/kafka/server/AdminManager.scala index 30c872ab34680..dd60760d352fe 100644 --- a/core/src/main/scala/kafka/server/AdminManager.scala +++ b/core/src/main/scala/kafka/server/AdminManager.scala @@ -39,7 +39,7 @@ import org.apache.kafka.common.network.ListenerName import org.apache.kafka.server.policy.{AlterConfigPolicy, CreateTopicPolicy} import org.apache.kafka.server.policy.CreateTopicPolicy.RequestMetadata import org.apache.kafka.common.protocol.Errors -import org.apache.kafka.common.quota.{QuotaAlteration, QuotaEntity, QuotaFilter} +import org.apache.kafka.common.quota.{ClientQuotaAlteration, ClientQuotaEntity, ClientQuotaFilter, ClientQuotaFilterComponent} import org.apache.kafka.common.requests.CreateTopicsRequest._ import org.apache.kafka.common.requests.DescribeConfigsResponse.ConfigSource import org.apache.kafka.common.requests.{AlterConfigsRequest, ApiError, DescribeConfigsResponse} @@ -703,137 +703,156 @@ class AdminManager(val config: KafkaConfig, new DescribeConfigsResponse.ConfigEntry(name, valueAsString, source, isSensitive, readOnly, synonyms.asJava) } - private def entityToSanitizedUserClientId(entity: QuotaEntity): (Option[String], Option[String]) = { + private def sanitizeEntityName(entityName: String): String = { + if (entityName == ConfigEntityName.Default) + throw new InvalidRequestException(s"Entity name '${ConfigEntityName.Default}' is reserved") + Sanitizer.sanitize(Option(entityName).getOrElse(ConfigEntityName.Default)) + } + + private def desanitizeEntityName(sanitizedEntityName: String): String = + Sanitizer.desanitize(sanitizedEntityName) match { + case ConfigEntityName.Default => null + case name => name + } + + private def entityToSanitizedUserClientId(entity: ClientQuotaEntity): (Option[String], Option[String]) = { if (entity.entries.isEmpty) - throw new InvalidRequestException("Invalid empty quota entity") + throw new InvalidRequestException("Invalid empty client quota entity") var user: Option[String] = None var clientId: Option[String] = None - entity.entries().asScala.foreach { case (entityType, entityName) => - val sanitizedEntityName = Some(Sanitizer.sanitize(entityName)) + entity.entries.asScala.foreach { case (entityType, entityName) => + val sanitizedEntityName = Some(sanitizeEntityName(entityName)) entityType match { - case QuotaEntity.USER => user = sanitizedEntityName - case QuotaEntity.CLIENT_ID => clientId = sanitizedEntityName - case _ => throw new InvalidRequestException(s"Unhandled quota entity type: ${entityType}") + case ClientQuotaEntity.USER => user = sanitizedEntityName + case ClientQuotaEntity.CLIENT_ID => clientId = sanitizedEntityName + case _ => throw new InvalidRequestException(s"Unhandled client quota entity type: ${entityType}") } - if (entityName.isEmpty) + if (entityName != null && entityName.isEmpty) throw new InvalidRequestException(s"Empty ${entityType} not supported") } (user, clientId) } - private def userClientIdToEntity(user: Option[String], clientId: Option[String]): QuotaEntity = { - new QuotaEntity((user.map(u => QuotaEntity.USER -> u) ++ clientId.map(c => QuotaEntity.CLIENT_ID -> c)).toMap.asJava) + private def userClientIdToEntity(user: Option[String], clientId: Option[String]): ClientQuotaEntity = { + new ClientQuotaEntity((user.map(u => ClientQuotaEntity.USER -> u) ++ clientId.map(c => ClientQuotaEntity.CLIENT_ID -> c)).toMap.asJava) } - def describeClientQuotas(filters: Seq[QuotaFilter]): Map[QuotaEntity, Map[String, Double]] = { - var userFilter: Option[QuotaFilter] = None - var clientIdFilter: Option[QuotaFilter] = None - filters.foreach { filter => - filter.entityType() match { - case QuotaEntity.USER => - if (userFilter.isDefined) - throw new InvalidRequestException(s"Duplicate user filter entity type"); - userFilter = Some(filter) - case QuotaEntity.CLIENT_ID => - if (clientIdFilter.isDefined) - throw new InvalidRequestException(s"Duplicate client filter entity type"); - clientIdFilter = Some(filter) + def describeClientQuotas(filter: ClientQuotaFilter): Map[ClientQuotaEntity, Map[String, Double]] = { + var userComponent: Option[ClientQuotaFilterComponent] = None + var clientIdComponent: Option[ClientQuotaFilterComponent] = None + filter.components.asScala.foreach { component => + component.entityType match { + case ClientQuotaEntity.USER => + if (userComponent.isDefined) + throw new InvalidRequestException(s"Duplicate user filter component entity type"); + userComponent = Some(component) + case ClientQuotaEntity.CLIENT_ID => + if (clientIdComponent.isDefined) + throw new InvalidRequestException(s"Duplicate client filter component entity type"); + clientIdComponent = Some(component) case "" => - throw new InvalidRequestException(s"Unexpected empty filter entity type") + throw new InvalidRequestException(s"Unexpected empty filter component entity type") case et => // Supplying other entity types is not yet supported. throw new UnsupportedVersionException(s"Custom entity type '${et}' not supported") } } - handleDescribeClientQuotas(userFilter, clientIdFilter) + handleDescribeClientQuotas(userComponent, clientIdComponent, filter.strict) } - def handleDescribeClientQuotas(userFilter: Option[QuotaFilter], clientIdFilter: Option[QuotaFilter]) = { - def wantExact(filter: Option[QuotaFilter]): Boolean = filter.exists(_.isMatchExact) - def wantExcluded(filter: Option[QuotaFilter]): Boolean = filter.exists(_.isMatchNone) - def sanitized(filter: Option[QuotaFilter]): String = { - filter.map(f => if (f.isMatchExact) Sanitizer.sanitize(f.matchExact) else "").getOrElse("") - } + def handleDescribeClientQuotas(userComponent: Option[ClientQuotaFilterComponent], + clientIdComponent: Option[ClientQuotaFilterComponent], strict: Boolean) = { - val sanitizedUser = sanitized(userFilter) - val exactUser = wantExact(userFilter) - val excludeUser = wantExcluded(userFilter) + def toOption(opt: java.util.Optional[String]): Option[String] = + if (opt == null) + None + else if (opt.isPresent) + Some(opt.get) + else + Some(null) + + val user = userComponent.flatMap(c => toOption(c.`match`)) + val clientId = clientIdComponent.flatMap(c => toOption(c.`match`)) + + def sanitized(name: Option[String]): String = name.map(n => sanitizeEntityName(n)).getOrElse("") + val sanitizedUser = sanitized(user) + val sanitizedClientId = sanitized(clientId) - val sanitizedClientId = sanitized(clientIdFilter) - val exactClientId = wantExact(clientIdFilter) - val excludeClientId = wantExcluded(clientIdFilter) + def wantExact(component: Option[ClientQuotaFilterComponent]): Boolean = component.exists(_.`match` != null) + val exactUser = wantExact(userComponent) + val exactClientId = wantExact(clientIdComponent) + + def wantExcluded(component: Option[ClientQuotaFilterComponent]): Boolean = strict && !component.isDefined + val excludeUser = wantExcluded(userComponent) + val excludeClientId = wantExcluded(clientIdComponent) val userEntries = if (exactUser && excludeClientId) - Map(((Some(userFilter.get.matchExact()), None) -> adminZkClient.fetchEntityConfig(ConfigType.User, sanitizedUser))) + Map(((Some(user.get), None) -> adminZkClient.fetchEntityConfig(ConfigType.User, sanitizedUser))) else if (!excludeUser && !exactClientId) adminZkClient.fetchAllEntityConfigs(ConfigType.User).map { case (name, props) => - ((Some(Sanitizer.desanitize(name)), None) -> props) + ((Some(desanitizeEntityName(name)), None) -> props) } else Map.empty val clientIdEntries = if (excludeUser && exactClientId) - Map(((None, Some(clientIdFilter.get.matchExact())) -> adminZkClient.fetchEntityConfig(ConfigType.Client, sanitizedClientId))) + Map(((None, Some(clientId.get)) -> adminZkClient.fetchEntityConfig(ConfigType.Client, sanitizedClientId))) else if (!exactUser && !excludeClientId) adminZkClient.fetchAllEntityConfigs(ConfigType.Client).map { case (name, props) => - ((None, Some(Sanitizer.desanitize(name))) -> props) + ((None, Some(desanitizeEntityName(name))) -> props) } else Map.empty val bothEntries = if (exactUser && exactClientId) - Map(((Some(userFilter.get.matchExact()), Some(clientIdFilter.get.matchExact())) -> + Map(((Some(user.get), Some(clientId.get)) -> adminZkClient.fetchEntityConfig(ConfigType.User, s"${sanitizedUser}/clients/${sanitizedClientId}"))) else if (!excludeUser && !excludeClientId) adminZkClient.fetchAllChildEntityConfigs(ConfigType.User, ConfigType.Client).map { case (name, props) => val components = name.split("/") if (components.size != 3 || components(1) != "clients") throw new IllegalArgumentException(s"Unexpected config path: ${name}") - ((Some(Sanitizer.desanitize(components(0))), Some(Sanitizer.desanitize(components(2)))) -> props) + ((Some(desanitizeEntityName(components(0))), Some(desanitizeEntityName(components(2)))) -> props) } else Map.empty - def matches(nameFilter: Option[QuotaFilter], name: Option[String]): Boolean = nameFilter match { - case Some(filter) => - if (filter.isMatchExact()) - name.exists(_ == filter.matchExact()) - else if (filter.isMatchSome()) - name.isDefined - else if (filter.isMatchNone()) - !name.isDefined - else - throw new IllegalStateException(s"Unexected quota filter type") + def matches(nameComponent: Option[ClientQuotaFilterComponent], name: Option[String]): Boolean = nameComponent match { + case Some(component) => + toOption(component.`match`) match { + case Some(n) => name.exists(_ == n) + case None => name.isDefined + } case None => - true + !name.isDefined || !strict } def fromProps(props: Properties): Map[String, Double] = { props.asScala.map { case (key, value) => val doubleValue = try value.toDouble catch { case _: NumberFormatException => - throw new IllegalStateException(s"Unexpected quota configuration value: ${key} -> ${value}") + throw new IllegalStateException(s"Unexpected client quota configuration value: ${key} -> ${value}") } (key -> doubleValue) } } (userEntries ++ clientIdEntries ++ bothEntries).map { case ((u, c), p) => - if (!p.isEmpty && matches(userFilter, u) && matches(clientIdFilter, c)) + if (!p.isEmpty && matches(userComponent, u) && matches(clientIdComponent, c)) Some((userClientIdToEntity(u, c) -> fromProps(p))) else None }.flatten.toMap } - def alterClientQuotas(entries: Seq[QuotaAlteration], validateOnly: Boolean): Map[QuotaEntity, ApiError] = { - def alterEntityQuotas(entity: QuotaEntity, ops: Iterable[QuotaAlteration.Op]): Unit = { + def alterClientQuotas(entries: Seq[ClientQuotaAlteration], validateOnly: Boolean): Map[ClientQuotaEntity, ApiError] = { + def alterEntityQuotas(entity: ClientQuotaEntity, ops: Iterable[ClientQuotaAlteration.Op]): Unit = { val (path, configType, configKeys) = entityToSanitizedUserClientId(entity) match { case (Some(user), Some(clientId)) => (user + "/clients/" + clientId, ConfigType.User, DynamicConfig.User.configKeys) case (Some(user), None) => (user, ConfigType.User, DynamicConfig.User.configKeys) case (None, Some(clientId)) => (clientId, ConfigType.Client, DynamicConfig.Client.configKeys) - case _ => throw new InvalidRequestException("Invalid empty quota entity") + case _ => throw new InvalidRequestException("Invalid empty client quota entity") } val props = adminZkClient.fetchEntityConfig(configType, path) diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala index 7aaa179398d9c..0fe34a74d1a5c 100644 --- a/core/src/main/scala/kafka/server/KafkaApis.scala +++ b/core/src/main/scala/kafka/server/KafkaApis.scala @@ -2792,7 +2792,7 @@ class KafkaApis(val requestChannel: RequestChannel, if (authorize(request, DESCRIBE_CONFIGS, CLUSTER, CLUSTER_NAME)) { val result = adminManager.describeClientQuotas( - describeClientQuotasRequest.filters().asScala.toSeq).mapValues(_.mapValues(Double.box).asJava).asJava + describeClientQuotasRequest.filter).mapValues(_.mapValues(Double.box).asJava).asJava sendResponseMaybeThrottle(request, requestThrottleMs => new DescribeClientQuotasResponse(result, requestThrottleMs)) } else { diff --git a/core/src/test/scala/unit/kafka/admin/ConfigCommandTest.scala b/core/src/test/scala/unit/kafka/admin/ConfigCommandTest.scala index 961da196e87ff..5a76f75c60a24 100644 --- a/core/src/test/scala/unit/kafka/admin/ConfigCommandTest.scala +++ b/core/src/test/scala/unit/kafka/admin/ConfigCommandTest.scala @@ -31,7 +31,7 @@ import org.apache.kafka.common.internals.KafkaFutureImpl import org.apache.kafka.common.Node import org.apache.kafka.common.errors.InvalidConfigurationException import org.apache.kafka.common.network.ListenerName -import org.apache.kafka.common.quota.{QuotaAlteration, QuotaEntity, QuotaFilter} +import org.apache.kafka.common.quota.{ClientQuotaAlteration, ClientQuotaEntity, ClientQuotaFilter} import org.apache.kafka.common.security.auth.SecurityProtocol import org.apache.kafka.common.security.scram.internals.ScramCredentialUtils import org.apache.kafka.common.utils.Sanitizer @@ -352,10 +352,10 @@ class ConfigCommandTest extends ZooKeeperTestHarness with Logging { "--add-config", "consumer_byte_rate=20000,producer_byte_rate=10000", "--delete-config", "request_percentage")) - val entity = new QuotaEntity(Map((QuotaEntity.CLIENT_ID -> "my-client-id")).asJava) + val entity = new ClientQuotaEntity(Map((ClientQuotaEntity.CLIENT_ID -> "my-client-id")).asJava) var describedConfigs = false - val describeFuture = new KafkaFutureImpl[util.Map[QuotaEntity, util.Map[String, java.lang.Double]]] + val describeFuture = new KafkaFutureImpl[util.Map[ClientQuotaEntity, util.Map[String, java.lang.Double]]] describeFuture.complete(Map((entity -> Map(("request_percentage" -> Double.box(50.0))).asJava)).asJava) val describeResult: DescribeClientQuotasResult = EasyMock.createNiceMock(classOf[DescribeClientQuotasResult]) EasyMock.expect(describeResult.entities()).andReturn(describeFuture) @@ -368,22 +368,18 @@ class ConfigCommandTest extends ZooKeeperTestHarness with Logging { val node = new Node(1, "localhost", 9092) val mockAdminClient = new MockAdminClient(util.Collections.singletonList(node), node) { - override def describeClientQuotas(filters: util.Collection[QuotaFilter], options: DescribeClientQuotasOptions): DescribeClientQuotasResult = { - assertEquals(2, filters.size) - filters.asScala.foreach { filter => - if (filter.entityType == QuotaEntity.USER) { - assertTrue(filter.isMatchNone) - } else { - assertEquals(QuotaEntity.CLIENT_ID, filter.entityType) - assertTrue(filter.isMatchExact) - assertEquals("my-client-id", filter.matchExact) - } - } + override def describeClientQuotas(filter: ClientQuotaFilter, options: DescribeClientQuotasOptions): DescribeClientQuotasResult = { + assertEquals(1, filter.components.size) + assertTrue(filter.strict) + val component = filter.components.asScala.head + assertEquals(ClientQuotaEntity.CLIENT_ID, component.entityType) + assertTrue(component.`match`.isPresent) + assertEquals("my-client-id", component.`match`.get) describedConfigs = true describeResult } - override def alterClientQuotas(entries: util.Collection[QuotaAlteration], options: AlterClientQuotasOptions): AlterClientQuotasResult = { + override def alterClientQuotas(entries: util.Collection[ClientQuotaAlteration], options: AlterClientQuotasOptions): AlterClientQuotasResult = { assertFalse(options.validateOnly) assertEquals(1, entries.size) val alteration = entries.asScala.head @@ -391,9 +387,9 @@ class ConfigCommandTest extends ZooKeeperTestHarness with Logging { val ops = alteration.ops.asScala assertEquals(3, ops.size) val expectedOps = Set( - new QuotaAlteration.Op("consumer_byte_rate", Double.box(20000)), - new QuotaAlteration.Op("producer_byte_rate", Double.box(10000)), - new QuotaAlteration.Op("request_percentage", null) + new ClientQuotaAlteration.Op("consumer_byte_rate", Double.box(20000)), + new ClientQuotaAlteration.Op("producer_byte_rate", Double.box(10000)), + new ClientQuotaAlteration.Op("request_percentage", null) ) assertEquals(expectedOps, ops.toSet) alteredConfigs = true @@ -1255,10 +1251,9 @@ class ConfigCommandTest extends ZooKeeperTestHarness with Logging { options: AlterConfigsOptions): AlterConfigsResult = EasyMock.createNiceMock(classOf[AlterConfigsResult]) override def alterConfigs(configs: util.Map[ConfigResource, Config], options: AlterConfigsOptions): AlterConfigsResult = EasyMock.createNiceMock(classOf[AlterConfigsResult]) - override def describeClientQuotas(filters: util.Collection[QuotaFilter], - options: DescribeClientQuotasOptions): DescribeClientQuotasResult = + override def describeClientQuotas(filter: ClientQuotaFilter, options: DescribeClientQuotasOptions): DescribeClientQuotasResult = EasyMock.createNiceMock(classOf[DescribeClientQuotasResult]) - override def alterClientQuotas(entries: util.Collection[QuotaAlteration], + override def alterClientQuotas(entries: util.Collection[ClientQuotaAlteration], options: AlterClientQuotasOptions): AlterClientQuotasResult = EasyMock.createNiceMock(classOf[AlterClientQuotasResult]) } diff --git a/core/src/test/scala/unit/kafka/server/ClientQuotasRequestTest.scala b/core/src/test/scala/unit/kafka/server/ClientQuotasRequestTest.scala index 2520f919c8e17..047ff7105239b 100644 --- a/core/src/test/scala/unit/kafka/server/ClientQuotasRequestTest.scala +++ b/core/src/test/scala/unit/kafka/server/ClientQuotasRequestTest.scala @@ -19,7 +19,7 @@ package kafka.server import org.apache.kafka.common.errors.{InvalidRequestException, UnsupportedVersionException} import org.apache.kafka.common.internals.KafkaFutureImpl -import org.apache.kafka.common.quota.{QuotaAlteration, QuotaEntity, QuotaFilter} +import org.apache.kafka.common.quota.{ClientQuotaAlteration, ClientQuotaEntity, ClientQuotaFilter, ClientQuotaFilterComponent} import org.apache.kafka.common.requests.{AlterClientQuotasRequest, AlterClientQuotasResponse, DescribeClientQuotasRequest, DescribeClientQuotasResponse} import org.junit.Assert._ import org.junit.Test @@ -37,7 +37,7 @@ class ClientQuotasRequestTest extends BaseRequestTest { @Test def testAlterClientQuotasRequest(): Unit = { - val entity = new QuotaEntity(Map((QuotaEntity.USER -> "user"), (QuotaEntity.CLIENT_ID -> "client-id")).asJava) + val entity = new ClientQuotaEntity(Map((ClientQuotaEntity.USER -> "user"), (ClientQuotaEntity.CLIENT_ID -> "client-id")).asJava) // Expect an empty configuration. verifyDescribeEntityQuotas(entity, Map.empty) @@ -106,7 +106,7 @@ class ClientQuotasRequestTest extends BaseRequestTest { @Test def testAlterClientQuotasRequestValidateOnly(): Unit = { - val entity = new QuotaEntity(Map((QuotaEntity.USER -> "user")).asJava) + val entity = new ClientQuotaEntity(Map((ClientQuotaEntity.USER -> "user")).asJava) // Set up a configuration. alterEntityQuotas(entity, Map( @@ -164,37 +164,37 @@ class ClientQuotasRequestTest extends BaseRequestTest { @Test(expected = classOf[InvalidRequestException]) def testAlterClientQuotasBadUser(): Unit = { - val entity = new QuotaEntity(Map((QuotaEntity.USER -> "")).asJava) + val entity = new ClientQuotaEntity(Map((ClientQuotaEntity.USER -> "")).asJava) alterEntityQuotas(entity, Map((RequestPercentageProp -> Some(12.34))), validateOnly = true) } @Test(expected = classOf[InvalidRequestException]) def testAlterClientQuotasBadClientId(): Unit = { - val entity = new QuotaEntity(Map((QuotaEntity.CLIENT_ID -> "")).asJava) + val entity = new ClientQuotaEntity(Map((ClientQuotaEntity.CLIENT_ID -> "")).asJava) alterEntityQuotas(entity, Map((RequestPercentageProp -> Some(12.34))), validateOnly = true) } @Test(expected = classOf[InvalidRequestException]) def testAlterClientQuotasBadEntityType(): Unit = { - val entity = new QuotaEntity(Map(("" -> "name")).asJava) + val entity = new ClientQuotaEntity(Map(("" -> "name")).asJava) alterEntityQuotas(entity, Map((RequestPercentageProp -> Some(12.34))), validateOnly = true) } @Test(expected = classOf[InvalidRequestException]) def testAlterClientQuotasEmptyEntity(): Unit = { - val entity = new QuotaEntity(Map.empty.asJava) + val entity = new ClientQuotaEntity(Map.empty.asJava) alterEntityQuotas(entity, Map((ProducerByteRateProp -> Some(10000.5))), validateOnly = true) } @Test(expected = classOf[InvalidRequestException]) def testAlterClientQuotasBadConfigKey(): Unit = { - val entity = new QuotaEntity(Map((QuotaEntity.USER -> "user")).asJava) + val entity = new ClientQuotaEntity(Map((ClientQuotaEntity.USER -> "user")).asJava) alterEntityQuotas(entity, Map(("bad" -> Some(1.0))), validateOnly = true) } @Test(expected = classOf[InvalidRequestException]) def testAlterClientQuotasBadConfigValue(): Unit = { - val entity = new QuotaEntity(Map((QuotaEntity.USER -> "user")).asJava) + val entity = new ClientQuotaEntity(Map((ClientQuotaEntity.USER -> "user")).asJava) alterEntityQuotas(entity, Map((ProducerByteRateProp -> Some(10000.5))), validateOnly = true) } @@ -203,13 +203,13 @@ class ClientQuotasRequestTest extends BaseRequestTest { (Some("user-1"), Some("client-id-1"), 50.50), (Some("user-2"), Some("client-id-1"), 51.51), (Some("user-3"), Some("client-id-2"), 52.52), - (Some(QuotaEntity.USER_DEFAULT), Some("client-id-1"), 53.53), - (Some("user-1"), Some(QuotaEntity.CLIENT_ID_DEFAULT), 54.54), - (Some("user-3"), Some(QuotaEntity.CLIENT_ID_DEFAULT), 55.55), + (Some(null), Some("client-id-1"), 53.53), + (Some("user-1"), Some(null), 54.54), + (Some("user-3"), Some(null), 55.55), (Some("user-1"), None, 56.56), (Some("user-2"), None, 57.57), (Some("user-3"), None, 58.58), - (Some(QuotaEntity.USER_DEFAULT), None, 59.59), + (Some(null), None, 59.59), (None, Some("client-id-2"), 60.60) ).map { case (u, c, v) => (toEntity(u, c), v) } @@ -227,12 +227,14 @@ class ClientQuotasRequestTest extends BaseRequestTest { def testDescribeClientQuotasMatchExact(): Unit = { setupDescribeClientQuotasMatchTest() - def matchEntity(entity: QuotaEntity) = { - val entries = entity.entries.asScala - def toFilter(entityType: String) = - entries.get(entityType).map(entityName => QuotaFilter.matchExact(entityType, entityName)).getOrElse(QuotaFilter.matchNone(entityType)) - val filters = List(toFilter(QuotaEntity.USER), toFilter(QuotaEntity.CLIENT_ID)) - describeClientQuotas(filters) + def matchEntity(entity: ClientQuotaEntity) = { + val components = entity.entries.asScala.map { case (entityType, entityName) => + entityName match { + case null => ClientQuotaFilterComponent.ofDefaultEntity(entityType) + case name => ClientQuotaFilterComponent.ofEntity(entityType, name) + } + } + describeClientQuotas(ClientQuotaFilter.containsOnly(components.toList.asJava)) } // Test exact matches. @@ -249,14 +251,14 @@ class ClientQuotasRequestTest extends BaseRequestTest { val notMatchEntities = List( (Some("user-1"), Some("client-id-2")), (Some("user-3"), Some("client-id-1")), - (Some("user-2"), Some(QuotaEntity.CLIENT_ID_DEFAULT)), + (Some("user-2"), Some(null)), (Some("user-4"), None), - (Some(QuotaEntity.USER_DEFAULT), Some("client-id-2")), + (Some(null), Some("client-id-2")), (None, Some("client-id-1")), (None, Some("client-id-3")), ).map { case (u, c) => - new QuotaEntity((u.map((QuotaEntity.USER, _)) ++ - c.map((QuotaEntity.CLIENT_ID, _))).toMap.asJava) + new ClientQuotaEntity((u.map((ClientQuotaEntity.USER, _)) ++ + c.map((ClientQuotaEntity.CLIENT_ID, _))).toMap.asJava) } // Verify exact matches of the non-matches returns empty. @@ -270,8 +272,8 @@ class ClientQuotasRequestTest extends BaseRequestTest { def testDescribeClientQuotasMatchPartial(): Unit = { setupDescribeClientQuotasMatchTest() - def testMatchEntities(filters: Iterable[QuotaFilter], expectedMatchSize: Int, partition: QuotaEntity => Boolean) { - val result = describeClientQuotas(filters) + def testMatchEntities(filter: ClientQuotaFilter, expectedMatchSize: Int, partition: ClientQuotaEntity => Boolean) { + val result = describeClientQuotas(filter) val (expectedMatches, expectedNonMatches) = matchEntities.partition(e => partition(e._1)) assertEquals(expectedMatchSize, expectedMatches.size) // for test verification assertEquals(expectedMatchSize, result.size) @@ -289,61 +291,76 @@ class ClientQuotasRequestTest extends BaseRequestTest { } } - // Match existing user. + // Match open-ended existing user. testMatchEntities( - List(QuotaFilter.matchExact(QuotaEntity.USER, "user-1")), 3, - entity => entity.entries.get(QuotaEntity.USER) == "user-1" + ClientQuotaFilter.contains(List(ClientQuotaFilterComponent.ofEntity(ClientQuotaEntity.USER, "user-1")).asJava), 3, + entity => entity.entries.get(ClientQuotaEntity.USER) == "user-1" ) - // Match non-existent user. + // Match open-ended non-existent user. testMatchEntities( - List(QuotaFilter.matchExact(QuotaEntity.USER, "unknown")), 0, + ClientQuotaFilter.contains(List(ClientQuotaFilterComponent.ofEntity(ClientQuotaEntity.USER, "unknown")).asJava), 0, entity => false ) - // Match existing client ID. + // Match open-ended existing client ID. + testMatchEntities( + ClientQuotaFilter.contains(List(ClientQuotaFilterComponent.ofEntity(ClientQuotaEntity.CLIENT_ID, "client-id-2")).asJava), 2, + entity => entity.entries.get(ClientQuotaEntity.CLIENT_ID) == "client-id-2" + ) + + // Match open-ended default user. testMatchEntities( - List(QuotaFilter.matchExact(QuotaEntity.CLIENT_ID, "client-id-2")), 2, - entity => entity.entries.get(QuotaEntity.CLIENT_ID) == "client-id-2" + ClientQuotaFilter.contains(List(ClientQuotaFilterComponent.ofDefaultEntity(ClientQuotaEntity.USER)).asJava), 2, + entity => entity.entries.containsKey(ClientQuotaEntity.USER) && entity.entries.get(ClientQuotaEntity.USER) == null ) - // Match default user. + // Match close-ended existing user. testMatchEntities( - List(QuotaFilter.matchExact(QuotaEntity.USER, QuotaEntity.USER_DEFAULT)), 2, - entity => entity.entries.get(QuotaEntity.USER) == QuotaEntity.USER_DEFAULT + ClientQuotaFilter.containsOnly(List(ClientQuotaFilterComponent.ofEntity(ClientQuotaEntity.USER, "user-2")).asJava), 1, + entity => entity.entries.get(ClientQuotaEntity.USER) == "user-2" && !entity.entries.containsKey(ClientQuotaEntity.CLIENT_ID) ) - // Match against all entities with the user type in an match. + // Match close-ended existing client ID that has no matching entity. testMatchEntities( - List(QuotaFilter.matchSome(QuotaEntity.USER)), 10, - entity => entity.entries.get(QuotaEntity.USER) != null + ClientQuotaFilter.containsOnly(List(ClientQuotaFilterComponent.ofEntity(ClientQuotaEntity.CLIENT_ID, "client-id-1")).asJava), 0, + entity => false ) - // Match against all entities with the client ID type in an match. + // Match against all entities with the user type in a close-ended match. testMatchEntities( - List(QuotaFilter.matchSome(QuotaEntity.CLIENT_ID)), 7, - entity => entity.entries.get(QuotaEntity.CLIENT_ID) != null + ClientQuotaFilter.containsOnly(List(ClientQuotaFilterComponent.ofEntityType(ClientQuotaEntity.USER)).asJava), 4, + entity => entity.entries.containsKey(ClientQuotaEntity.USER) && !entity.entries.containsKey(ClientQuotaEntity.CLIENT_ID) ) - // Match against no entities with the user type. + // Match against all entities with the user type in an open-ended match. testMatchEntities( - List(QuotaFilter.matchNone(QuotaEntity.USER)), 1, - entity => entity.entries.get(QuotaEntity.USER) == null + ClientQuotaFilter.contains(List(ClientQuotaFilterComponent.ofEntityType(ClientQuotaEntity.USER)).asJava), 10, + entity => entity.entries.containsKey(ClientQuotaEntity.USER) ) - // Match against no entities with the client ID type. + // Match against all entities with the client ID type in a close-ended match. testMatchEntities( - List(QuotaFilter.matchNone(QuotaEntity.CLIENT_ID)), 4, - entity => entity.entries.get(QuotaEntity.CLIENT_ID) == null + ClientQuotaFilter.containsOnly(List(ClientQuotaFilterComponent.ofEntityType(ClientQuotaEntity.CLIENT_ID)).asJava), 1, + entity => entity.entries.containsKey(ClientQuotaEntity.CLIENT_ID) && !entity.entries.containsKey(ClientQuotaEntity.USER) ) - // Match empty filter list. This should match all entities. - testMatchEntities(List.empty, 11, entity => true) + // Match against all entities with the client ID type in an open-ended match. + testMatchEntities( + ClientQuotaFilter.contains(List(ClientQuotaFilterComponent.ofEntityType(ClientQuotaEntity.CLIENT_ID)).asJava), 7, + entity => entity.entries.containsKey(ClientQuotaEntity.CLIENT_ID) + ) + + // Match open-ended empty filter list. This should match all entities. + testMatchEntities(ClientQuotaFilter.contains(List.empty.asJava), 11, entity => true) + + // Match close-ended empty filter list. This should match no entities. + testMatchEntities(ClientQuotaFilter.containsOnly(List.empty.asJava), 0, entity => false) } @Test def testClientQuotasUnsupportedEntityTypes() { - val entity = new QuotaEntity(Map(("other" -> "name")).asJava) + val entity = new ClientQuotaEntity(Map(("other" -> "name")).asJava) try { verifyDescribeEntityQuotas(entity, Map()) } catch { @@ -354,7 +371,7 @@ class ClientQuotasRequestTest extends BaseRequestTest { @Test def testClientQuotasSanitized(): Unit = { // An entity with name that must be sanitized when writing to Zookeeper. - val entity = new QuotaEntity(Map((QuotaEntity.USER -> "user with spaces")).asJava) + val entity = new ClientQuotaEntity(Map((ClientQuotaEntity.USER -> "user with spaces")).asJava) alterEntityQuotas(entity, Map( (ProducerByteRateProp -> Some(20000.0)), @@ -365,9 +382,9 @@ class ClientQuotasRequestTest extends BaseRequestTest { )) } - private def verifyDescribeEntityQuotas(entity: QuotaEntity, quotas: Map[String, Double]) = { - val filters = entity.entries.asScala.map(e => QuotaFilter.matchExact(e._1, e._2)) - val describe = describeClientQuotas(filters) + private def verifyDescribeEntityQuotas(entity: ClientQuotaEntity, quotas: Map[String, Double]) = { + val components = entity.entries.asScala.map(e => ClientQuotaFilterComponent.ofEntity(e._1, e._2)) + val describe = describeClientQuotas(ClientQuotaFilter.containsOnly(components.toList.asJava)) if (quotas.isEmpty) { assertEquals(0, describe.size) } else { @@ -384,30 +401,30 @@ class ClientQuotasRequestTest extends BaseRequestTest { } private def toEntity(user: Option[String], clientId: Option[String]) = - new QuotaEntity((user.map((QuotaEntity.USER, _)) ++ clientId.map((QuotaEntity.CLIENT_ID, _))).toMap.asJava) + new ClientQuotaEntity((user.map((ClientQuotaEntity.USER -> _)) ++ clientId.map((ClientQuotaEntity.CLIENT_ID -> _))).toMap.asJava) - private def describeClientQuotas(filters: Iterable[QuotaFilter]) = { - val result = new KafkaFutureImpl[java.util.Map[QuotaEntity, java.util.Map[String, java.lang.Double]]] - sendDescribeClientQuotasRequest(filters).complete(result) + private def describeClientQuotas(filter: ClientQuotaFilter) = { + val result = new KafkaFutureImpl[java.util.Map[ClientQuotaEntity, java.util.Map[String, java.lang.Double]]] + sendDescribeClientQuotasRequest(filter).complete(result) result.get } - private def sendDescribeClientQuotasRequest(filters: Iterable[QuotaFilter]): DescribeClientQuotasResponse = { - val request = new DescribeClientQuotasRequest.Builder(filters.asJavaCollection).build() + private def sendDescribeClientQuotasRequest(filter: ClientQuotaFilter): DescribeClientQuotasResponse = { + val request = new DescribeClientQuotasRequest.Builder(filter).build() connectAndReceive[DescribeClientQuotasResponse](request, destination = controllerSocketServer) } - private def alterEntityQuotas(entity: QuotaEntity, alter: Map[String, Option[Double]], validateOnly: Boolean) = + private def alterEntityQuotas(entity: ClientQuotaEntity, alter: Map[String, Option[Double]], validateOnly: Boolean) = try alterClientQuotas(Map(entity -> alter), validateOnly).get(entity).get.get(10, TimeUnit.SECONDS) catch { case e: ExecutionException => throw e.getCause } - private def alterClientQuotas(request: Map[QuotaEntity, Map[String, Option[Double]]], validateOnly: Boolean) = { + private def alterClientQuotas(request: Map[ClientQuotaEntity, Map[String, Option[Double]]], validateOnly: Boolean) = { val entries = request.map { case (entity, alter) => val ops = alter.map { case (key, value) => - new QuotaAlteration.Op(key, value.map(Double.box).getOrElse(null)) + new ClientQuotaAlteration.Op(key, value.map(Double.box).getOrElse(null)) }.asJavaCollection - new QuotaAlteration(entity, ops) + new ClientQuotaAlteration(entity, ops) } val response = request.map(e => (e._1 -> new KafkaFutureImpl[Void])).asJava @@ -418,7 +435,7 @@ class ClientQuotasRequestTest extends BaseRequestTest { result } - private def sendAlterClientQuotasRequest(entries: Iterable[QuotaAlteration], validateOnly: Boolean): AlterClientQuotasResponse = { + private def sendAlterClientQuotasRequest(entries: Iterable[ClientQuotaAlteration], validateOnly: Boolean): AlterClientQuotasResponse = { val request = new AlterClientQuotasRequest.Builder(entries.asJavaCollection, validateOnly).build() connectAndReceive[AlterClientQuotasResponse](request, destination = controllerSocketServer) }