diff --git a/bin/kafka-run-class.sh b/bin/kafka-run-class.sh index bb786da43ca82..4dd092323b043 100755 --- a/bin/kafka-run-class.sh +++ b/bin/kafka-run-class.sh @@ -69,28 +69,50 @@ do fi done -for file in "$base_dir"/clients/build/libs/kafka-clients*.jar; -do - if should_include_file "$file"; then - CLASSPATH="$CLASSPATH":"$file" - fi -done +if [ -z "$UPGRADE_KAFKA_STREAMS_TEST_VERSION" ]; then + clients_lib_dir=$(dirname $0)/../clients/build/libs + streams_lib_dir=$(dirname $0)/../streams/build/libs + rocksdb_lib_dir=$(dirname $0)/../streams/build/dependant-libs-${SCALA_VERSION} +else + clients_lib_dir=/opt/kafka-$UPGRADE_KAFKA_STREAMS_TEST_VERSION/libs + streams_lib_dir=$clients_lib_dir + rocksdb_lib_dir=$streams_lib_dir +fi + -for file in "$base_dir"/streams/build/libs/kafka-streams*.jar; +for file in "$clients_lib_dir"/kafka-clients*.jar; do if should_include_file "$file"; then CLASSPATH="$CLASSPATH":"$file" fi done -for file in "$base_dir"/streams/examples/build/libs/kafka-streams-examples*.jar; +for file in "$streams_lib_dir"/kafka-streams*.jar; do if should_include_file "$file"; then CLASSPATH="$CLASSPATH":"$file" fi done -for file in "$base_dir"/streams/build/dependant-libs-${SCALA_VERSION}/rocksdb*.jar; +if [ -z "$UPGRADE_KAFKA_STREAMS_TEST_VERSION" ]; then + for file in "$base_dir"/streams/examples/build/libs/kafka-streams-examples*.jar; + do + if should_include_file "$file"; then + CLASSPATH="$CLASSPATH":"$file" + fi + done +else + VERSION_NO_DOTS=`echo $UPGRADE_KAFKA_STREAMS_TEST_VERSION | sed 's/\.//g'` + SHORT_VERSION_NO_DOTS=${VERSION_NO_DOTS:0:((${#VERSION_NO_DOTS} - 1))} # remove last char, ie, bug-fix number + for file in "$base_dir"/streams/upgrade-system-tests-$SHORT_VERSION_NO_DOTS/build/libs/kafka-streams-upgrade-system-tests*.jar; + do + if should_include_file "$file"; then + CLASSPATH="$CLASSPATH":"$file" + fi + done +fi + +for file in "$rocksdb_lib_dir"/rocksdb*.jar; do CLASSPATH="$CLASSPATH":"$file" done diff --git a/build.gradle b/build.gradle index 22c4c2991b089..69f560ec38fef 100644 --- a/build.gradle +++ b/build.gradle @@ -41,7 +41,6 @@ allprojects { } apply plugin: 'idea' - apply plugin: "jacoco" apply plugin: 'org.owasp.dependencycheck' apply plugin: 'com.github.ben-manes.versions' @@ -384,6 +383,7 @@ subprojects { // Ignore core since its a scala project if (it.path != ':core') { + apply plugin: "jacoco" jacoco { toolVersion = "0.8.1" @@ -591,8 +591,7 @@ project(':core') { scoverage libs.scoveragePlugin scoverage libs.scoverageRuntime } - - jacocoTestReport.enabled = false + scoverage { reportDir = file("${rootProject.buildDir}/scoverage") highlighting = false @@ -859,6 +858,7 @@ project(':clients') { include "**/org/apache/kafka/common/config/*" include "**/org/apache/kafka/common/security/auth/*" include "**/org/apache/kafka/server/policy/*" + include "**/org/apache/kafka/common/security/token/delegation/*" } } @@ -920,10 +920,11 @@ project(':streams') { compile libs.slf4jApi compile libs.rocksDBJni + // testCompileOnly prevents streams from exporting a dependency on test-utils, which would cause a dependency cycle + testCompileOnly project(':streams:test-utils') testCompile project(':clients').sourceSets.test.output testCompile project(':core') testCompile project(':core').sourceSets.test.output - testCompile project(':streams:test-utils').sourceSets.main.output testCompile libs.junit testCompile libs.easymock testCompile libs.bcpkix @@ -968,21 +969,11 @@ project(':streams:test-utils') { archivesBaseName = "kafka-streams-test-utils" dependencies { - // streams and streams-test-utils have a circular dependency that needs to be broken. Making streams a regular - // compile dependency here is a circular dependency. Making the streams project's main sourceSet output a dependency - // incorrectly includes raw class files as dependencies and ends up including them in output like the release tgz. - // Making the main sourceSet's output a compileOnly (provided) dependency resolves the circular dependency in gradle - // and also leaves the raw class files out of the output. This does require that users of this jar specify the - // streams dependency separately, but they wouldn't be using this module in tests unless they already had a - // compile dependency on streams. For this module, however, we need to also include streams as a separate test - // dependency since it is compileOnly/provided for the compile phase. - compileOnly project(':streams').sourceSets.main.output + compile project(':streams') compile project(':clients') - testCompile project(':streams') testCompile project(':clients').sourceSets.test.output testCompile libs.junit - testCompile libs.rocksDBJni testRuntime libs.slf4jlog4j } @@ -1014,8 +1005,8 @@ project(':streams:examples') { compile project(':connect:json') // this dependency should be removed after we unify data API compile libs.slf4jlog4j - testCompile project(':clients').sourceSets.test.output // for org.apache.kafka.test.IntegrationTest testCompile project(':streams:test-utils') + testCompile project(':clients').sourceSets.test.output // for org.apache.kafka.test.IntegrationTest testCompile libs.junit } @@ -1036,6 +1027,66 @@ project(':streams:examples') { } } +project(':streams:upgrade-system-tests-0100') { + archivesBaseName = "kafka-streams-upgrade-system-tests-0100" + + dependencies { + testCompile libs.kafkaStreams_0100 + } + + systemTestLibs { + dependsOn testJar + } +} + +project(':streams:upgrade-system-tests-0101') { + archivesBaseName = "kafka-streams-upgrade-system-tests-0101" + + dependencies { + testCompile libs.kafkaStreams_0101 + } + + systemTestLibs { + dependsOn testJar + } +} + +project(':streams:upgrade-system-tests-0102') { + archivesBaseName = "kafka-streams-upgrade-system-tests-0102" + + dependencies { + testCompile libs.kafkaStreams_0102 + } + + systemTestLibs { + dependsOn testJar + } +} + +project(':streams:upgrade-system-tests-0110') { + archivesBaseName = "kafka-streams-upgrade-system-tests-0110" + + dependencies { + testCompile libs.kafkaStreams_0110 + } + + systemTestLibs { + dependsOn testJar + } +} + +project(':streams:upgrade-system-tests-10') { + archivesBaseName = "kafka-streams-upgrade-system-tests-10" + + dependencies { + testCompile libs.kafkaStreams_10 + } + + systemTestLibs { + dependsOn testJar + } +} + project(':jmh-benchmarks') { apply plugin: 'com.github.johnrengelman.shadow' diff --git a/checkstyle/checkstyle.xml b/checkstyle/checkstyle.xml index ccab85ce4fecb..ad8545027a000 100644 --- a/checkstyle/checkstyle.xml +++ b/checkstyle/checkstyle.xml @@ -105,7 +105,7 @@ - + @@ -114,7 +114,7 @@ - + diff --git a/checkstyle/import-control.xml b/checkstyle/import-control.xml index 000acc38463b4..192d73557f70c 100644 --- a/checkstyle/import-control.xml +++ b/checkstyle/import-control.xml @@ -164,6 +164,8 @@ + + diff --git a/checkstyle/suppressions.xml b/checkstyle/suppressions.xml index e3bf15102bd61..2767132886dfe 100644 --- a/checkstyle/suppressions.xml +++ b/checkstyle/suppressions.xml @@ -10,7 +10,7 @@ + files="(Fetcher|Sender|SenderTest|ConsumerCoordinator|KafkaConsumer|KafkaProducer|Utils|TransactionManagerTest|KafkaAdminClient|NetworkClient|AdminClient).java"/> + files="KStreamKStreamJoinTest.java|SmokeTestDriver.java"/> diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/AdminClient.java b/clients/src/main/java/org/apache/kafka/clients/admin/AdminClient.java index 897e127d5577b..0171b617e7ca2 100644 --- a/clients/src/main/java/org/apache/kafka/clients/admin/AdminClient.java +++ b/clients/src/main/java/org/apache/kafka/clients/admin/AdminClient.java @@ -535,4 +535,237 @@ public DeleteRecordsResult deleteRecords(Map re */ public abstract DeleteRecordsResult deleteRecords(Map recordsToDelete, DeleteRecordsOptions options); + + /** + *

Create a Delegation Token.

+ * + *

This is a convenience method for {@link #createDelegationToken(CreateDelegationTokenOptions)} with default options. + * See the overload for more details.

+ * + * @return The CreateDelegationTokenResult. + */ + public CreateDelegationTokenResult createDelegationToken() { + return createDelegationToken(new CreateDelegationTokenOptions()); + } + + + /** + *

Create a Delegation Token.

+ * + *

This operation is supported by brokers with version 1.1.0 or higher.

+ * + *

The following exceptions can be anticipated when calling {@code get()} on the futures obtained from the + * {@link CreateDelegationTokenResult#delegationToken() delegationToken()} method of the returned {@code CreateDelegationTokenResult}

+ *
    + *
  • {@link org.apache.kafka.common.errors.UnsupportedByAuthenticationException} + * If the request sent on PLAINTEXT/1-way SSL channels or delegation token authenticated channels.
    • + *
  • {@link org.apache.kafka.common.errors.InvalidPrincipalTypeException} + * if the renewers principal type is not supported.
    • + *
  • {@link org.apache.kafka.common.errors.DelegationTokenDisabledException} + * if the delegation token feature is disabled.
    • + *
  • {@link org.apache.kafka.common.errors.TimeoutException} + * if the request was not completed in within the given {@link CreateDelegationTokenOptions#timeoutMs()}.
    • + *
+ * + * @param options The options to use when creating delegation token. + * @return The DeleteRecordsResult. + */ + public abstract CreateDelegationTokenResult createDelegationToken(CreateDelegationTokenOptions options); + + + /** + *

Renew a Delegation Token.

+ * + *

This is a convenience method for {@link #renewDelegationToken(byte[], RenewDelegationTokenOptions)} with default options. + * See the overload for more details.

+ * + * + * @param hmac HMAC of the Delegation token + * @return The RenewDelegationTokenResult. + */ + public RenewDelegationTokenResult renewDelegationToken(byte[] hmac) { + return renewDelegationToken(hmac, new RenewDelegationTokenOptions()); + } + + /** + *

Renew a Delegation Token.

+ * + *

This operation is supported by brokers with version 1.1.0 or higher.

+ * + *

The following exceptions can be anticipated when calling {@code get()} on the futures obtained from the + * {@link RenewDelegationTokenResult#expiryTimestamp() expiryTimestamp()} method of the returned {@code RenewDelegationTokenResult}

+ *
    + *
  • {@link org.apache.kafka.common.errors.UnsupportedByAuthenticationException} + * If the request sent on PLAINTEXT/1-way SSL channels or delegation token authenticated channels.
    • + *
  • {@link org.apache.kafka.common.errors.DelegationTokenDisabledException} + * if the delegation token feature is disabled.
    • + *
  • {@link org.apache.kafka.common.errors.DelegationTokenNotFoundException} + * if the delegation token is not found on server.
    • + *
  • {@link org.apache.kafka.common.errors.DelegationTokenOwnerMismatchException} + * if the authenticated user is not owner/renewer of the token.
    • + *
  • {@link org.apache.kafka.common.errors.DelegationTokenExpiredException} + * if the delegation token is expired.
    • + *
  • {@link org.apache.kafka.common.errors.TimeoutException} + * if the request was not completed in within the given {@link RenewDelegationTokenOptions#timeoutMs()}.
    • + *
+ * + * @param hmac HMAC of the Delegation token + * @param options The options to use when renewing delegation token. + * @return The RenewDelegationTokenResult. + */ + public abstract RenewDelegationTokenResult renewDelegationToken(byte[] hmac, RenewDelegationTokenOptions options); + + /** + *

Expire a Delegation Token.

+ * + *

This is a convenience method for {@link #expireDelegationToken(byte[], ExpireDelegationTokenOptions)} with default options. + * This will expire the token immediately. See the overload for more details.

+ * + * @param hmac HMAC of the Delegation token + * @return The ExpireDelegationTokenResult. + */ + public ExpireDelegationTokenResult expireDelegationToken(byte[] hmac) { + return expireDelegationToken(hmac, new ExpireDelegationTokenOptions()); + } + + /** + *

Expire a Delegation Token.

+ * + *

This operation is supported by brokers with version 1.1.0 or higher.

+ * + *

The following exceptions can be anticipated when calling {@code get()} on the futures obtained from the + * {@link ExpireDelegationTokenResult#expiryTimestamp() expiryTimestamp()} method of the returned {@code ExpireDelegationTokenResult}

+ *
    + *
  • {@link org.apache.kafka.common.errors.UnsupportedByAuthenticationException} + * If the request sent on PLAINTEXT/1-way SSL channels or delegation token authenticated channels.
    • + *
  • {@link org.apache.kafka.common.errors.DelegationTokenDisabledException} + * if the delegation token feature is disabled.
    • + *
  • {@link org.apache.kafka.common.errors.DelegationTokenNotFoundException} + * if the delegation token is not found on server.
    • + *
  • {@link org.apache.kafka.common.errors.DelegationTokenOwnerMismatchException} + * if the authenticated user is not owner/renewer of the requested token.
    • + *
  • {@link org.apache.kafka.common.errors.DelegationTokenExpiredException} + * if the delegation token is expired.
    • + *
  • {@link org.apache.kafka.common.errors.TimeoutException} + * if the request was not completed in within the given {@link ExpireDelegationTokenOptions#timeoutMs()}.
    • + *
+ * + * @param hmac HMAC of the Delegation token + * @param options The options to use when expiring delegation token. + * @return The ExpireDelegationTokenResult. + */ + public abstract ExpireDelegationTokenResult expireDelegationToken(byte[] hmac, ExpireDelegationTokenOptions options); + + /** + *

Describe the Delegation Tokens.

+ * + *

This is a convenience method for {@link #describeDelegationToken(DescribeDelegationTokenOptions)} with default options. + * This will return all the user owned tokens and tokens where user have Describe permission. See the overload for more details.

+ * + * @return The DescribeDelegationTokenResult. + */ + public DescribeDelegationTokenResult describeDelegationToken() { + return describeDelegationToken(new DescribeDelegationTokenOptions()); + } + + /** + *

Describe the Delegation Tokens.

+ * + *

This operation is supported by brokers with version 1.1.0 or higher.

+ * + *

The following exceptions can be anticipated when calling {@code get()} on the futures obtained from the + * {@link DescribeDelegationTokenResult#delegationTokens() delegationTokens()} method of the returned {@code DescribeDelegationTokenResult}

+ *
    + *
  • {@link org.apache.kafka.common.errors.UnsupportedByAuthenticationException} + * If the request sent on PLAINTEXT/1-way SSL channels or delegation token authenticated channels.
    • + *
  • {@link org.apache.kafka.common.errors.DelegationTokenDisabledException} + * if the delegation token feature is disabled.
    • + *
  • {@link org.apache.kafka.common.errors.TimeoutException} + * if the request was not completed in within the given {@link DescribeDelegationTokenOptions#timeoutMs()}.
    • + *
+ * + * @param options The options to use when describing delegation tokens. + * @return The DescribeDelegationTokenResult. + */ + public abstract DescribeDelegationTokenResult describeDelegationToken(DescribeDelegationTokenOptions options); + + /** + * Describe some group IDs in the cluster. + * + * @param groupIds The IDs of the groups to describe. + * @param options The options to use when describing the groups. + * @return The DescribeConsumerGroupResult. + */ + public abstract DescribeConsumerGroupsResult describeConsumerGroups(Collection groupIds, + DescribeConsumerGroupsOptions options); + + /** + * Describe some group IDs in the cluster, with the default options. + *

+ * This is a convenience method for + * #{@link AdminClient#describeConsumerGroups(Collection, DescribeConsumerGroupsOptions)} with + * default options. See the overload for more details. + * + * @param groupIds The IDs of the groups to describe. + * @return The DescribeConsumerGroupResult. + */ + public DescribeConsumerGroupsResult describeConsumerGroups(Collection groupIds) { + return describeConsumerGroups(groupIds, new DescribeConsumerGroupsOptions()); + } + + /** + * List the consumer groups available in the cluster. + * + * @param options The options to use when listing the consumer groups. + * @return The ListGroupsResult. + */ + public abstract ListConsumerGroupsResult listConsumerGroups(ListConsumerGroupsOptions options); + + /** + * List the consumer groups available in the cluster with the default options. + * + * This is a convenience method for #{@link AdminClient#listConsumerGroups(ListConsumerGroupsOptions)} with default options. + * See the overload for more details. + * + * @return The ListGroupsResult. + */ + public ListConsumerGroupsResult listConsumerGroups() { + return listConsumerGroups(new ListConsumerGroupsOptions()); + } + + /** + * List the consumer group offsets available in the cluster. + * + * @param options The options to use when listing the consumer group offsets. + * @return The ListGroupOffsetsResult + */ + public abstract ListConsumerGroupOffsetsResult listConsumerGroupOffsets(String groupId, ListConsumerGroupOffsetsOptions options); + + /** + * List the consumer group offsets available in the cluster with the default options. + * + * This is a convenience method for #{@link AdminClient#listConsumerGroupOffsets(String, ListConsumerGroupOffsetsOptions)} with default options. + * + * @return The ListGroupOffsetsResult. + */ + public ListConsumerGroupOffsetsResult listConsumerGroupOffsets(String groupId) { + return listConsumerGroupOffsets(groupId, new ListConsumerGroupOffsetsOptions()); + } + + /** + * Delete consumer groups from the cluster. + * + * @param options The options to use when deleting a consumer group. + * @return The DeletConsumerGroupResult. + */ + public abstract DeleteConsumerGroupsResult deleteConsumerGroups(Collection groupIds, DeleteConsumerGroupsOptions options); + + /** + * Delete consumer groups from the cluster with the default options. + * + * @return The DeleteConsumerGroupResult. + */ + public DeleteConsumerGroupsResult deleteConsumerGroups(Collection groupIds) { + return deleteConsumerGroups(groupIds, new DeleteConsumerGroupsOptions()); + } } diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/ConsumerGroupDescription.java b/clients/src/main/java/org/apache/kafka/clients/admin/ConsumerGroupDescription.java new file mode 100644 index 0000000000000..0bfa8a782d5ac --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/clients/admin/ConsumerGroupDescription.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.clients.admin; + +import org.apache.kafka.common.utils.Utils; + +import java.util.List; + +/** + * A detailed description of a single consumer group in the cluster. + */ +public class ConsumerGroupDescription { + + private final String groupId; + private final boolean isSimpleConsumerGroup; + private final List members; + private final String partitionAssignor; + + /** + * Creates an instance with the specified parameters. + * + * @param groupId The consumer group id + * @param isSimpleConsumerGroup If Consumer Group is simple + * @param members The consumer group members + * @param partitionAssignor The consumer group partition assignor + */ + public ConsumerGroupDescription(String groupId, boolean isSimpleConsumerGroup, List members, String partitionAssignor) { + this.groupId = groupId; + this.isSimpleConsumerGroup = isSimpleConsumerGroup; + this.members = members; + this.partitionAssignor = partitionAssignor; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + ConsumerGroupDescription that = (ConsumerGroupDescription) o; + + if (isSimpleConsumerGroup != that.isSimpleConsumerGroup) return false; + if (groupId != null ? !groupId.equals(that.groupId) : that.groupId != null) return false; + if (members != null ? !members.equals(that.members) : that.members != null) return false; + return partitionAssignor != null ? partitionAssignor.equals(that.partitionAssignor) : that.partitionAssignor == null; + } + + @Override + public int hashCode() { + int result = groupId != null ? groupId.hashCode() : 0; + result = 31 * result + (isSimpleConsumerGroup ? 1 : 0); + result = 31 * result + (members != null ? members.hashCode() : 0); + result = 31 * result + (partitionAssignor != null ? partitionAssignor.hashCode() : 0); + return result; + } + + /** + * The id of the consumer group. + */ + public String groupId() { + return groupId; + } + + /** + * If consumer group is simple or not. + */ + public boolean isSimpleConsumerGroup() { + return isSimpleConsumerGroup; + } + + /** + * A list of the members of the consumer group. + */ + public List members() { + return members; + } + + /** + * The consumer group partition assignor. + */ + public String partitionAssignor() { + return partitionAssignor; + } + + @Override + public String toString() { + return "(groupId=" + groupId + ", isSimpleConsumerGroup=" + isSimpleConsumerGroup + ", members=" + + Utils.join(members, ",") + ", partitionAssignor=" + partitionAssignor + ")"; + } +} diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/ConsumerGroupListing.java b/clients/src/main/java/org/apache/kafka/clients/admin/ConsumerGroupListing.java new file mode 100644 index 0000000000000..46da9628010b6 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/clients/admin/ConsumerGroupListing.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.clients.admin; + +/** + * A listing of a consumer group in the cluster. + */ +public class ConsumerGroupListing { + private final String groupId; + private final boolean isSimpleConsumerGroup; + + /** + * Create an instance with the specified parameters. + * + * @param groupId Group Id + * @param isSimpleConsumerGroup If consumer group is simple or not. + */ + public ConsumerGroupListing(String groupId, boolean isSimpleConsumerGroup) { + this.groupId = groupId; + this.isSimpleConsumerGroup = isSimpleConsumerGroup; + } + + /** + * Consumer Group Id + */ + public String groupId() { + return groupId; + } + + /** + * If Consumer Group is simple or not. + */ + public boolean isSimpleConsumerGroup() { + return isSimpleConsumerGroup; + } + + @Override + public String toString() { + return "(" + + "groupId='" + groupId + '\'' + + ", isSimpleConsumerGroup=" + isSimpleConsumerGroup + + ')'; + } +} diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/CreateDelegationTokenOptions.java b/clients/src/main/java/org/apache/kafka/clients/admin/CreateDelegationTokenOptions.java new file mode 100644 index 0000000000000..1b77b943800ec --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/clients/admin/CreateDelegationTokenOptions.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.clients.admin; + +import java.util.LinkedList; +import java.util.List; + +import org.apache.kafka.common.annotation.InterfaceStability; +import org.apache.kafka.common.security.auth.KafkaPrincipal; + +/** + * Options for {@link AdminClient#createDelegationToken(CreateDelegationTokenOptions)}. + * + * The API of this class is evolving, see {@link AdminClient} for details. + */ +@InterfaceStability.Evolving +public class CreateDelegationTokenOptions extends AbstractOptions { + private long maxLifeTimeMs = -1; + private List renewers = new LinkedList<>(); + + public CreateDelegationTokenOptions renewers(List renewers) { + this.renewers = renewers; + return this; + } + + public List renewers() { + return renewers; + } + + public CreateDelegationTokenOptions maxlifeTimeMs(long maxLifeTimeMs) { + this.maxLifeTimeMs = maxLifeTimeMs; + return this; + } + + public long maxlifeTimeMs() { + return maxLifeTimeMs; + } +} diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/CreateDelegationTokenResult.java b/clients/src/main/java/org/apache/kafka/clients/admin/CreateDelegationTokenResult.java new file mode 100644 index 0000000000000..043cbe87fef5c --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/clients/admin/CreateDelegationTokenResult.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.clients.admin; + +import org.apache.kafka.common.KafkaFuture; +import org.apache.kafka.common.annotation.InterfaceStability; +import org.apache.kafka.common.security.token.delegation.DelegationToken; + +/** + * The result of the {@link KafkaAdminClient#createDelegationToken(CreateDelegationTokenOptions)} call. + * + * The API of this class is evolving, see {@link AdminClient} for details. + */ +@InterfaceStability.Evolving +public class CreateDelegationTokenResult { + private final KafkaFuture delegationToken; + + CreateDelegationTokenResult(KafkaFuture delegationToken) { + this.delegationToken = delegationToken; + } + + /** + * Returns a future which yields a delegation token + */ + public KafkaFuture delegationToken() { + return delegationToken; + } +} diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/DeleteConsumerGroupsOptions.java b/clients/src/main/java/org/apache/kafka/clients/admin/DeleteConsumerGroupsOptions.java new file mode 100644 index 0000000000000..cd505f4c11138 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/clients/admin/DeleteConsumerGroupsOptions.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.clients.admin; + +import org.apache.kafka.common.annotation.InterfaceStability; + +import java.util.Collection; + +/** + * Options for the {@link AdminClient#deleteConsumerGroups(Collection)} call. + * + * The API of this class is evolving, see {@link AdminClient} for details. + */ +@InterfaceStability.Evolving +public class DeleteConsumerGroupsOptions extends AbstractOptions { + +} diff --git a/clients/src/main/java/org/apache/kafka/common/security/authenticator/AuthCallbackHandler.java b/clients/src/main/java/org/apache/kafka/clients/admin/DeleteConsumerGroupsResult.java similarity index 52% rename from clients/src/main/java/org/apache/kafka/common/security/authenticator/AuthCallbackHandler.java rename to clients/src/main/java/org/apache/kafka/clients/admin/DeleteConsumerGroupsResult.java index d517162a1762c..b4bce26440547 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/authenticator/AuthCallbackHandler.java +++ b/clients/src/main/java/org/apache/kafka/clients/admin/DeleteConsumerGroupsResult.java @@ -14,32 +14,28 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.kafka.common.security.authenticator; +package org.apache.kafka.clients.admin; -import java.util.Map; - -import org.apache.kafka.common.network.Mode; +import org.apache.kafka.common.KafkaFuture; +import org.apache.kafka.common.annotation.InterfaceStability; -import javax.security.auth.Subject; -import javax.security.auth.callback.CallbackHandler; +import java.util.Collection; +import java.util.Map; -/* - * Callback handler for SASL-based authentication +/** + * The result of the {@link AdminClient#deleteConsumerGroups(Collection)} call. + * + * The API of this class is evolving, see {@link AdminClient} for details. */ -public interface AuthCallbackHandler extends CallbackHandler { +@InterfaceStability.Evolving +public class DeleteConsumerGroupsResult { + final KafkaFuture>> futures; - /** - * Configures this callback handler. - * - * @param configs Configuration - * @param mode The mode that indicates if this is a client or server connection - * @param subject Subject from login context - * @param saslMechanism Negotiated SASL mechanism - */ - void configure(Map configs, Mode mode, Subject subject, String saslMechanism); + DeleteConsumerGroupsResult(KafkaFuture>> futures) { + this.futures = futures; + } - /** - * Closes this instance. - */ - void close(); + public KafkaFuture>> deletedGroups() { + return futures; + } } diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/DescribeConsumerGroupsOptions.java b/clients/src/main/java/org/apache/kafka/clients/admin/DescribeConsumerGroupsOptions.java new file mode 100644 index 0000000000000..7daff1a483be8 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/clients/admin/DescribeConsumerGroupsOptions.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.clients.admin; + +import org.apache.kafka.common.annotation.InterfaceStability; + +import java.util.Collection; + +/** + * Options for {@link AdminClient#describeConsumerGroups(Collection, DescribeConsumerGroupsOptions)}. + *

+ * The API of this class is evolving, see {@link AdminClient} for details. + */ +@InterfaceStability.Evolving +public class DescribeConsumerGroupsOptions extends AbstractOptions { +} \ No newline at end of file diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/DescribeConsumerGroupsResult.java b/clients/src/main/java/org/apache/kafka/clients/admin/DescribeConsumerGroupsResult.java new file mode 100644 index 0000000000000..adde031b6788d --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/clients/admin/DescribeConsumerGroupsResult.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.clients.admin; + +import org.apache.kafka.common.KafkaFuture; +import org.apache.kafka.common.annotation.InterfaceStability; + +import java.util.Collection; +import java.util.Map; + + +/** + * The result of the {@link KafkaAdminClient#describeConsumerGroups(Collection, DescribeConsumerGroupsOptions)}} call. + * + * The API of this class is evolving, see {@link AdminClient} for details. + */ +@InterfaceStability.Evolving +public class DescribeConsumerGroupsResult { + + private final KafkaFuture>> futures; + + public DescribeConsumerGroupsResult(KafkaFuture>> futures) { + this.futures = futures; + } + + /** + * Return a map from group name to futures which can be used to check the description of a consumer group. + */ + public KafkaFuture>> describedGroups() { + return futures; + } +} diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/DescribeDelegationTokenOptions.java b/clients/src/main/java/org/apache/kafka/clients/admin/DescribeDelegationTokenOptions.java new file mode 100644 index 0000000000000..60b99354e3552 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/clients/admin/DescribeDelegationTokenOptions.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.clients.admin; + +import java.util.List; + +import org.apache.kafka.common.annotation.InterfaceStability; +import org.apache.kafka.common.security.auth.KafkaPrincipal; + +/** + * Options for {@link AdminClient#describeDelegationToken(DescribeDelegationTokenOptions)}. + * + * The API of this class is evolving, see {@link AdminClient} for details. + */ +@InterfaceStability.Evolving +public class DescribeDelegationTokenOptions extends AbstractOptions { + private List owners; + + /** + * if owners is null, all the user owned tokens and tokens where user have Describe permission + * will be returned. + * @param owners + * @return this instance + */ + public DescribeDelegationTokenOptions owners(List owners) { + this.owners = owners; + return this; + } + + public List owners() { + return owners; + } +} diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/DescribeDelegationTokenResult.java b/clients/src/main/java/org/apache/kafka/clients/admin/DescribeDelegationTokenResult.java new file mode 100644 index 0000000000000..7a9d4b9dd9737 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/clients/admin/DescribeDelegationTokenResult.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.clients.admin; + +import java.util.List; + +import org.apache.kafka.common.KafkaFuture; +import org.apache.kafka.common.annotation.InterfaceStability; +import org.apache.kafka.common.security.token.delegation.DelegationToken; + +/** + * The result of the {@link KafkaAdminClient#describeDelegationToken(DescribeDelegationTokenOptions)} call. + * + * The API of this class is evolving, see {@link AdminClient} for details. + */ +@InterfaceStability.Evolving +public class DescribeDelegationTokenResult { + private final KafkaFuture> delegationTokens; + + DescribeDelegationTokenResult(KafkaFuture> delegationTokens) { + this.delegationTokens = delegationTokens; + } + + /** + * Returns a future which yields list of delegation tokens + */ + public KafkaFuture> delegationTokens() { + return delegationTokens; + } +} diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/ExpireDelegationTokenOptions.java b/clients/src/main/java/org/apache/kafka/clients/admin/ExpireDelegationTokenOptions.java new file mode 100644 index 0000000000000..138cd4e69e477 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/clients/admin/ExpireDelegationTokenOptions.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.clients.admin; + +import org.apache.kafka.common.annotation.InterfaceStability; + +/** + * Options for {@link AdminClient#expireDelegationToken(byte[], ExpireDelegationTokenOptions)}. + * + * The API of this class is evolving, see {@link AdminClient} for details. + */ +@InterfaceStability.Evolving +public class ExpireDelegationTokenOptions extends AbstractOptions { + private long expiryTimePeriodMs = -1L; + + public ExpireDelegationTokenOptions expiryTimePeriodMs(long expiryTimePeriodMs) { + this.expiryTimePeriodMs = expiryTimePeriodMs; + return this; + } + + public long expiryTimePeriodMs() { + return expiryTimePeriodMs; + } +} diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/ExpireDelegationTokenResult.java b/clients/src/main/java/org/apache/kafka/clients/admin/ExpireDelegationTokenResult.java new file mode 100644 index 0000000000000..41782bdcb5ce7 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/clients/admin/ExpireDelegationTokenResult.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.clients.admin; + +import org.apache.kafka.common.KafkaFuture; +import org.apache.kafka.common.annotation.InterfaceStability; + +/** + * The result of the {@link KafkaAdminClient#expireDelegationToken(byte[], ExpireDelegationTokenOptions)} call. + * + * The API of this class is evolving, see {@link AdminClient} for details. + */ +@InterfaceStability.Evolving +public class ExpireDelegationTokenResult { + private final KafkaFuture expiryTimestamp; + + ExpireDelegationTokenResult(KafkaFuture expiryTimestamp) { + this.expiryTimestamp = expiryTimestamp; + } + + /** + * Returns a future which yields expiry timestamp + */ + public KafkaFuture expiryTimestamp() { + return expiryTimestamp; + } +} diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java index e455b9ce81c63..50bcfd388569c 100644 --- a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java +++ b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java @@ -27,6 +27,9 @@ import org.apache.kafka.clients.admin.DeleteAclsResult.FilterResult; import org.apache.kafka.clients.admin.DeleteAclsResult.FilterResults; import org.apache.kafka.clients.admin.DescribeReplicaLogDirsResult.ReplicaLogDirInfo; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; +import org.apache.kafka.clients.consumer.internals.ConsumerProtocol; +import org.apache.kafka.clients.consumer.internals.PartitionAssignor; import org.apache.kafka.common.Cluster; import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.KafkaFuture; @@ -69,6 +72,8 @@ import org.apache.kafka.common.requests.CreateAclsRequest.AclCreation; import org.apache.kafka.common.requests.CreateAclsResponse; import org.apache.kafka.common.requests.CreateAclsResponse.AclCreationResponse; +import org.apache.kafka.common.requests.CreateDelegationTokenRequest; +import org.apache.kafka.common.requests.CreateDelegationTokenResponse; import org.apache.kafka.common.requests.CreatePartitionsRequest; import org.apache.kafka.common.requests.CreatePartitionsResponse; import org.apache.kafka.common.requests.CreateTopicsRequest; @@ -77,6 +82,8 @@ import org.apache.kafka.common.requests.DeleteAclsResponse; import org.apache.kafka.common.requests.DeleteAclsResponse.AclDeletionResult; import org.apache.kafka.common.requests.DeleteAclsResponse.AclFilterResponse; +import org.apache.kafka.common.requests.DeleteGroupsRequest; +import org.apache.kafka.common.requests.DeleteGroupsResponse; import org.apache.kafka.common.requests.DeleteRecordsRequest; import org.apache.kafka.common.requests.DeleteRecordsResponse; import org.apache.kafka.common.requests.DeleteTopicsRequest; @@ -85,19 +92,37 @@ import org.apache.kafka.common.requests.DescribeAclsResponse; import org.apache.kafka.common.requests.DescribeConfigsRequest; import org.apache.kafka.common.requests.DescribeConfigsResponse; +import org.apache.kafka.common.requests.DescribeDelegationTokenRequest; +import org.apache.kafka.common.requests.DescribeDelegationTokenResponse; import org.apache.kafka.common.requests.DescribeLogDirsRequest; import org.apache.kafka.common.requests.DescribeLogDirsResponse; +import org.apache.kafka.common.requests.ExpireDelegationTokenRequest; +import org.apache.kafka.common.requests.ExpireDelegationTokenResponse; import org.apache.kafka.common.requests.MetadataRequest; import org.apache.kafka.common.requests.MetadataResponse; +import org.apache.kafka.common.requests.RenewDelegationTokenRequest; +import org.apache.kafka.common.requests.RenewDelegationTokenResponse; +import org.apache.kafka.common.requests.DescribeGroupsRequest; +import org.apache.kafka.common.requests.DescribeGroupsResponse; +import org.apache.kafka.common.requests.FindCoordinatorRequest; +import org.apache.kafka.common.requests.FindCoordinatorResponse; +import org.apache.kafka.common.requests.ListGroupsRequest; +import org.apache.kafka.common.requests.ListGroupsResponse; +import org.apache.kafka.common.requests.OffsetFetchRequest; +import org.apache.kafka.common.requests.OffsetFetchResponse; import org.apache.kafka.common.requests.Resource; import org.apache.kafka.common.requests.ResourceType; +import org.apache.kafka.common.security.token.delegation.DelegationToken; +import org.apache.kafka.common.security.token.delegation.TokenInformation; import org.apache.kafka.common.utils.AppInfoParser; import org.apache.kafka.common.utils.KafkaThread; import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.Time; +import org.apache.kafka.common.utils.Utils; import org.slf4j.Logger; import java.net.InetSocketAddress; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -1150,9 +1175,10 @@ void handleFailure(Throwable throwable) { } @Override - public DeleteTopicsResult deleteTopics(final Collection topicNames, + public DeleteTopicsResult deleteTopics(Collection topicNames, DeleteTopicsOptions options) { final Map> topicFutures = new HashMap<>(topicNames.size()); + final List validTopicNames = new ArrayList<>(topicNames.size()); for (String topicName : topicNames) { if (topicNameIsUnrepresentable(topicName)) { KafkaFutureImpl future = new KafkaFutureImpl<>(); @@ -1161,6 +1187,7 @@ public DeleteTopicsResult deleteTopics(final Collection topicNames, topicFutures.put(topicName, future); } else if (!topicFutures.containsKey(topicName)) { topicFutures.put(topicName, new KafkaFutureImpl()); + validTopicNames.add(topicName); } } final long now = time.milliseconds(); @@ -1169,7 +1196,7 @@ public DeleteTopicsResult deleteTopics(final Collection topicNames, @Override AbstractRequest.Builder createRequest(int timeoutMs) { - return new DeleteTopicsRequest.Builder(new HashSet<>(topicNames), timeoutMs); + return new DeleteTopicsRequest.Builder(new HashSet<>(validTopicNames), timeoutMs); } @Override @@ -1204,7 +1231,7 @@ void handleFailure(Throwable throwable) { completeAllExceptionally(topicFutures.values(), throwable); } }; - if (!topicNames.isEmpty()) { + if (!validTopicNames.isEmpty()) { runnable.call(call, now); } return new DeleteTopicsResult(new HashMap>(topicFutures)); @@ -2070,4 +2097,452 @@ void handleFailure(Throwable throwable) { return new DeleteRecordsResult(new HashMap>(futures)); } + + @Override + public CreateDelegationTokenResult createDelegationToken(final CreateDelegationTokenOptions options) { + final KafkaFutureImpl delegationTokenFuture = new KafkaFutureImpl<>(); + final long now = time.milliseconds(); + runnable.call(new Call("createDelegationToken", calcDeadlineMs(now, options.timeoutMs()), + new LeastLoadedNodeProvider()) { + + @Override + AbstractRequest.Builder createRequest(int timeoutMs) { + return new CreateDelegationTokenRequest.Builder(options.renewers(), options.maxlifeTimeMs()); + } + + @Override + void handleResponse(AbstractResponse abstractResponse) { + CreateDelegationTokenResponse response = (CreateDelegationTokenResponse) abstractResponse; + if (response.hasError()) { + delegationTokenFuture.completeExceptionally(response.error().exception()); + } else { + TokenInformation tokenInfo = new TokenInformation(response.tokenId(), response.owner(), + options.renewers(), response.issueTimestamp(), response.maxTimestamp(), response.expiryTimestamp()); + DelegationToken token = new DelegationToken(tokenInfo, response.hmacBytes()); + delegationTokenFuture.complete(token); + } + } + + @Override + void handleFailure(Throwable throwable) { + delegationTokenFuture.completeExceptionally(throwable); + } + }, now); + + return new CreateDelegationTokenResult(delegationTokenFuture); + } + + @Override + public RenewDelegationTokenResult renewDelegationToken(final byte[] hmac, final RenewDelegationTokenOptions options) { + final KafkaFutureImpl expiryTimeFuture = new KafkaFutureImpl<>(); + final long now = time.milliseconds(); + runnable.call(new Call("renewDelegationToken", calcDeadlineMs(now, options.timeoutMs()), + new LeastLoadedNodeProvider()) { + + @Override + AbstractRequest.Builder createRequest(int timeoutMs) { + return new RenewDelegationTokenRequest.Builder(hmac, options.renewTimePeriodMs()); + } + + @Override + void handleResponse(AbstractResponse abstractResponse) { + RenewDelegationTokenResponse response = (RenewDelegationTokenResponse) abstractResponse; + if (response.hasError()) { + expiryTimeFuture.completeExceptionally(response.error().exception()); + } else { + expiryTimeFuture.complete(response.expiryTimestamp()); + } + } + + @Override + void handleFailure(Throwable throwable) { + expiryTimeFuture.completeExceptionally(throwable); + } + }, now); + + return new RenewDelegationTokenResult(expiryTimeFuture); + } + + @Override + public ExpireDelegationTokenResult expireDelegationToken(final byte[] hmac, final ExpireDelegationTokenOptions options) { + final KafkaFutureImpl expiryTimeFuture = new KafkaFutureImpl<>(); + final long now = time.milliseconds(); + runnable.call(new Call("expireDelegationToken", calcDeadlineMs(now, options.timeoutMs()), + new LeastLoadedNodeProvider()) { + + @Override + AbstractRequest.Builder createRequest(int timeoutMs) { + return new ExpireDelegationTokenRequest.Builder(hmac, options.expiryTimePeriodMs()); + } + + @Override + void handleResponse(AbstractResponse abstractResponse) { + ExpireDelegationTokenResponse response = (ExpireDelegationTokenResponse) abstractResponse; + if (response.hasError()) { + expiryTimeFuture.completeExceptionally(response.error().exception()); + } else { + expiryTimeFuture.complete(response.expiryTimestamp()); + } + } + + @Override + void handleFailure(Throwable throwable) { + expiryTimeFuture.completeExceptionally(throwable); + } + }, now); + + return new ExpireDelegationTokenResult(expiryTimeFuture); + } + + @Override + public DescribeDelegationTokenResult describeDelegationToken(final DescribeDelegationTokenOptions options) { + final KafkaFutureImpl> tokensFuture = new KafkaFutureImpl<>(); + final long now = time.milliseconds(); + runnable.call(new Call("describeDelegationToken", calcDeadlineMs(now, options.timeoutMs()), + new LeastLoadedNodeProvider()) { + + @Override + AbstractRequest.Builder createRequest(int timeoutMs) { + return new DescribeDelegationTokenRequest.Builder(options.owners()); + } + + @Override + void handleResponse(AbstractResponse abstractResponse) { + DescribeDelegationTokenResponse response = (DescribeDelegationTokenResponse) abstractResponse; + if (response.hasError()) { + tokensFuture.completeExceptionally(response.error().exception()); + } else { + tokensFuture.complete(response.tokens()); + } + } + + @Override + void handleFailure(Throwable throwable) { + tokensFuture.completeExceptionally(throwable); + } + }, now); + + return new DescribeDelegationTokenResult(tokensFuture); + } + + @Override + public DescribeConsumerGroupsResult describeConsumerGroups(final Collection groupIds, + final DescribeConsumerGroupsOptions options) { + final KafkaFutureImpl>> resultFutures = new KafkaFutureImpl<>(); + final Map> consumerGroupFutures = new HashMap<>(groupIds.size()); + final ArrayList groupIdList = new ArrayList<>(); + for (String groupId : groupIds) { + if (!consumerGroupFutures.containsKey(groupId)) { + consumerGroupFutures.put(groupId, new KafkaFutureImpl()); + groupIdList.add(groupId); + } + } + + for (final String groupId : groupIdList) { + + final long nowFindCoordinator = time.milliseconds(); + final long deadline = calcDeadlineMs(nowFindCoordinator, options.timeoutMs()); + + runnable.call(new Call("findCoordinator", deadline, new LeastLoadedNodeProvider()) { + @Override + AbstractRequest.Builder createRequest(int timeoutMs) { + return new FindCoordinatorRequest.Builder(FindCoordinatorRequest.CoordinatorType.GROUP, groupId); + } + + @Override + void handleResponse(AbstractResponse abstractResponse) { + final FindCoordinatorResponse response = (FindCoordinatorResponse) abstractResponse; + + final long nowDescribeConsumerGroups = time.milliseconds(); + + final int nodeId = response.node().id(); + + runnable.call(new Call("describeConsumerGroups", deadline, new ConstantNodeIdProvider(nodeId)) { + + @Override + AbstractRequest.Builder createRequest(int timeoutMs) { + return new DescribeGroupsRequest.Builder(groupIdList); + } + + @Override + void handleResponse(AbstractResponse abstractResponse) { + final DescribeGroupsResponse response = (DescribeGroupsResponse) abstractResponse; + // Handle server responses for particular groupId. + for (Map.Entry> entry : consumerGroupFutures.entrySet()) { + final String groupId = entry.getKey(); + final KafkaFutureImpl future = entry.getValue(); + final DescribeGroupsResponse.GroupMetadata groupMetadata = response.groups().get(groupId); + final Errors groupError = groupMetadata.error(); + if (groupError != Errors.NONE) { + future.completeExceptionally(groupError.exception()); + continue; + } + + final String protocolType = groupMetadata.protocolType(); + if (protocolType.equals(ConsumerProtocol.PROTOCOL_TYPE) || protocolType.isEmpty()) { + final List members = groupMetadata.members(); + final List consumers = new ArrayList<>(members.size()); + + for (DescribeGroupsResponse.GroupMember groupMember : members) { + final PartitionAssignor.Assignment assignment = + ConsumerProtocol.deserializeAssignment( + ByteBuffer.wrap(Utils.readBytes(groupMember.memberAssignment()))); + + final MemberDescription memberDescription = + new MemberDescription( + groupMember.memberId(), + groupMember.clientId(), + groupMember.clientHost(), + new MemberAssignment(assignment.partitions())); + consumers.add(memberDescription); + } + final String protocol = groupMetadata.protocol(); + final ConsumerGroupDescription consumerGroupDescription = + new ConsumerGroupDescription(groupId, protocolType.isEmpty(), consumers, protocol); + future.complete(consumerGroupDescription); + } + } + } + + @Override + void handleFailure(Throwable throwable) { + completeAllExceptionally(consumerGroupFutures.values(), throwable); + } + }, nowDescribeConsumerGroups); + + resultFutures.complete(new HashMap>(consumerGroupFutures)); + } + + @Override + void handleFailure(Throwable throwable) { + resultFutures.completeExceptionally(throwable); + } + }, nowFindCoordinator); + } + + return new DescribeConsumerGroupsResult(resultFutures); + } + + @Override + public ListConsumerGroupsResult listConsumerGroups(ListConsumerGroupsOptions options) { + //final KafkaFutureImpl>>> nodeAndConsumerGroupListing = new KafkaFutureImpl<>(); + final KafkaFutureImpl> future = new KafkaFutureImpl>(); + + final long nowMetadata = time.milliseconds(); + final long deadline = calcDeadlineMs(nowMetadata, options.timeoutMs()); + + runnable.call(new Call("listNodes", deadline, new LeastLoadedNodeProvider()) { + @Override + AbstractRequest.Builder createRequest(int timeoutMs) { + return new MetadataRequest.Builder(Collections.emptyList(), true); + } + + @Override + void handleResponse(AbstractResponse abstractResponse) { + MetadataResponse metadataResponse = (MetadataResponse) abstractResponse; + + final Map>> futures = new HashMap<>(); + + for (final Node node : metadataResponse.brokers()) { + futures.put(node, new KafkaFutureImpl>()); + } + + future.combine(futures.values().toArray(new KafkaFuture[0])).thenApply( + new KafkaFuture.BaseFunction, Collection>() { + @Override + public Collection apply(Collection v) { + List listings = new ArrayList<>(); + for (Map.Entry>> entry : futures.entrySet()) { + Collection results; + try { + results = entry.getValue().get(); + } catch (Throwable e) { + // This should be unreachable, since the future returned by KafkaFuture#allOf should + // have failed if any Future failed. + throw new KafkaException("ListConsumerGroupsResult#listings(): internal error", e); + } + listings.addAll(results); + } + return listings; + } + }); + + + for (final Map.Entry>> entry : futures.entrySet()) { + final long nowList = time.milliseconds(); + + final int brokerId = entry.getKey().id(); + + runnable.call(new Call("listConsumerGroups", deadline, new ConstantNodeIdProvider(brokerId)) { + + private final KafkaFutureImpl> future = entry.getValue(); + + @Override + AbstractRequest.Builder createRequest(int timeoutMs) { + return new ListGroupsRequest.Builder(); + } + + @Override + void handleResponse(AbstractResponse abstractResponse) { + final ListGroupsResponse response = (ListGroupsResponse) abstractResponse; + final List groupsListing = new ArrayList<>(); + for (ListGroupsResponse.Group group : response.groups()) { + if (group.protocolType().equals(ConsumerProtocol.PROTOCOL_TYPE) || group.protocolType().isEmpty()) { + final String groupId = group.groupId(); + final String protocolType = group.protocolType(); + final ConsumerGroupListing groupListing = new ConsumerGroupListing(groupId, protocolType.isEmpty()); + groupsListing.add(groupListing); + } + } + future.complete(groupsListing); + } + + @Override + void handleFailure(Throwable throwable) { + completeAllExceptionally(futures.values(), throwable); + } + }, nowList); + + } + } + + @Override + void handleFailure(Throwable throwable) { + future.completeExceptionally(throwable); + } + }, nowMetadata); + + return new ListConsumerGroupsResult(future); + } + + @Override + public ListConsumerGroupOffsetsResult listConsumerGroupOffsets(final String groupId, final ListConsumerGroupOffsetsOptions options) { + final KafkaFutureImpl> groupOffsetListingFuture = new KafkaFutureImpl<>(); + + final long nowFindCoordinator = time.milliseconds(); + final long deadline = calcDeadlineMs(nowFindCoordinator, options.timeoutMs()); + + runnable.call(new Call("findCoordinator", deadline, new LeastLoadedNodeProvider()) { + @Override + AbstractRequest.Builder createRequest(int timeoutMs) { + return new FindCoordinatorRequest.Builder(FindCoordinatorRequest.CoordinatorType.GROUP, groupId); + } + + @Override + void handleResponse(AbstractResponse abstractResponse) { + final FindCoordinatorResponse response = (FindCoordinatorResponse) abstractResponse; + + final long nowListConsumerGroupOffsets = time.milliseconds(); + + final int nodeId = response.node().id(); + + runnable.call(new Call("listConsumerGroupOffsets", deadline, new ConstantNodeIdProvider(nodeId)) { + @Override + AbstractRequest.Builder createRequest(int timeoutMs) { + return new OffsetFetchRequest.Builder(groupId, options.topicPartitions()); + } + + @Override + void handleResponse(AbstractResponse abstractResponse) { + final OffsetFetchResponse response = (OffsetFetchResponse) abstractResponse; + final Map groupOffsetsListing = new HashMap<>(); + for (Map.Entry entry : + response.responseData().entrySet()) { + final TopicPartition topicPartition = entry.getKey(); + final Long offset = entry.getValue().offset; + final String metadata = entry.getValue().metadata; + groupOffsetsListing.put(topicPartition, new OffsetAndMetadata(offset, metadata)); + } + groupOffsetListingFuture.complete(groupOffsetsListing); + } + + @Override + void handleFailure(Throwable throwable) { + groupOffsetListingFuture.completeExceptionally(throwable); + } + }, nowListConsumerGroupOffsets); + } + + @Override + void handleFailure(Throwable throwable) { + groupOffsetListingFuture.completeExceptionally(throwable); + } + }, nowFindCoordinator); + + return new ListConsumerGroupOffsetsResult(groupOffsetListingFuture); + } + + @Override + public DeleteConsumerGroupsResult deleteConsumerGroups(Collection groupIds, DeleteConsumerGroupsOptions options) { + final KafkaFutureImpl>> deleteConsumerGroupsFuture = new KafkaFutureImpl<>(); + final Map> deleteConsumerGroupFutures = new HashMap<>(groupIds.size()); + final Set groupIdList = new HashSet<>(); + for (String groupId : groupIds) { + if (!deleteConsumerGroupFutures.containsKey(groupId)) { + deleteConsumerGroupFutures.put(groupId, new KafkaFutureImpl()); + groupIdList.add(groupId); + } + } + + for (final String groupId : groupIdList) { + + final long nowFindCoordinator = time.milliseconds(); + final long deadline = calcDeadlineMs(nowFindCoordinator, options.timeoutMs()); + + runnable.call(new Call("findCoordinator", deadline, new LeastLoadedNodeProvider()) { + @Override + AbstractRequest.Builder createRequest(int timeoutMs) { + return new FindCoordinatorRequest.Builder(FindCoordinatorRequest.CoordinatorType.GROUP, groupId); + } + + @Override + void handleResponse(AbstractResponse abstractResponse) { + final FindCoordinatorResponse response = (FindCoordinatorResponse) abstractResponse; + + final long nowDeleteConsumerGroups = time.milliseconds(); + + final int nodeId = response.node().id(); + + runnable.call(new Call("deleteConsumerGroups", deadline, new ConstantNodeIdProvider(nodeId)) { + + @Override + AbstractRequest.Builder createRequest(int timeoutMs) { + return new DeleteGroupsRequest.Builder(Collections.singleton(groupId)); + } + + @Override + void handleResponse(AbstractResponse abstractResponse) { + final DeleteGroupsResponse response = (DeleteGroupsResponse) abstractResponse; + // Handle server responses for particular groupId. + for (Map.Entry> entry : deleteConsumerGroupFutures.entrySet()) { + final String groupId = entry.getKey(); + final KafkaFutureImpl future = entry.getValue(); + final Errors groupError = response.get(groupId); + if (groupError != Errors.NONE) { + future.completeExceptionally(groupError.exception()); + continue; + } + + future.complete(null); + } + } + + @Override + void handleFailure(Throwable throwable) { + completeAllExceptionally(deleteConsumerGroupFutures.values(), throwable); + } + }, nowDeleteConsumerGroups); + + deleteConsumerGroupsFuture.complete(new HashMap>(deleteConsumerGroupFutures)); + } + + @Override + void handleFailure(Throwable throwable) { + deleteConsumerGroupsFuture.completeExceptionally(throwable); + } + }, nowFindCoordinator); + } + + return new DeleteConsumerGroupsResult(deleteConsumerGroupsFuture); + } } diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/ListConsumerGroupOffsetsOptions.java b/clients/src/main/java/org/apache/kafka/clients/admin/ListConsumerGroupOffsetsOptions.java new file mode 100644 index 0000000000000..c6434ebb15c13 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/clients/admin/ListConsumerGroupOffsetsOptions.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.clients.admin; + +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.annotation.InterfaceStability; + +import java.util.List; + +/** + * Options for {@link AdminClient#listConsumerGroupOffsets(String)}. + *

+ * The API of this class is evolving, see {@link AdminClient} for details. + */ +@InterfaceStability.Evolving +public class ListConsumerGroupOffsetsOptions extends AbstractOptions { + + private List topicPartitions = null; + + /** + * Set the topic partitions to list as part of the result. + * {@code null} includes all topic partitions. + * + * @param topicPartitions List of topic partitions to include + * @return This ListGroupOffsetsOptions + */ + public ListConsumerGroupOffsetsOptions topicPartitions(List topicPartitions) { + this.topicPartitions = topicPartitions; + return this; + } + + /** + * Returns a list of topic partitions to add as part of the result. + */ + public List topicPartitions() { + return topicPartitions; + } +} diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/ListConsumerGroupOffsetsResult.java b/clients/src/main/java/org/apache/kafka/clients/admin/ListConsumerGroupOffsetsResult.java new file mode 100644 index 0000000000000..23657b5ad3cf7 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/clients/admin/ListConsumerGroupOffsetsResult.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.clients.admin; + +import org.apache.kafka.clients.consumer.OffsetAndMetadata; +import org.apache.kafka.common.KafkaFuture; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.annotation.InterfaceStability; + +import java.util.Map; + +/** + * The result of the {@link AdminClient#listConsumerGroupOffsets(String)} call. + *

+ * The API of this class is evolving, see {@link AdminClient} for details. + */ +@InterfaceStability.Evolving +public class ListConsumerGroupOffsetsResult { + + final KafkaFuture> future; + + ListConsumerGroupOffsetsResult(KafkaFuture> future) { + this.future = future; + } + + /** + * Return a future which yields a map of topic partitions to OffsetAndMetadata objects. + */ + public KafkaFuture> partitionsToOffsetAndMetadata() { + return future; + } + +} diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/ListConsumerGroupsOptions.java b/clients/src/main/java/org/apache/kafka/clients/admin/ListConsumerGroupsOptions.java new file mode 100644 index 0000000000000..86ca171b726c0 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/clients/admin/ListConsumerGroupsOptions.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.clients.admin; + +import org.apache.kafka.common.annotation.InterfaceStability; + +/** + * Options for {@link AdminClient#listConsumerGroups()}. + * + * The API of this class is evolving, see {@link AdminClient} for details. + */ +@InterfaceStability.Evolving +public class ListConsumerGroupsOptions extends AbstractOptions { +} diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/ListConsumerGroupsResult.java b/clients/src/main/java/org/apache/kafka/clients/admin/ListConsumerGroupsResult.java new file mode 100644 index 0000000000000..c72537104409d --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/clients/admin/ListConsumerGroupsResult.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.clients.admin; + +import org.apache.kafka.common.KafkaFuture; +import org.apache.kafka.common.annotation.InterfaceStability; + +import java.util.Collection; + +/** + * The result of the {@link AdminClient#listConsumerGroups()} call. + *

+ * The API of this class is evolving, see {@link AdminClient} for details. + */ +@InterfaceStability.Evolving +public class ListConsumerGroupsResult { + private final KafkaFuture> future; + + ListConsumerGroupsResult(KafkaFuture> future) { + this.future = future; + } + + /** + * Return a future which yields a collection of ConsumerGroupListing objects. + */ + public KafkaFuture> listings() { + return future; + } +} diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/MemberAssignment.java b/clients/src/main/java/org/apache/kafka/clients/admin/MemberAssignment.java new file mode 100644 index 0000000000000..bd958132b7c89 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/clients/admin/MemberAssignment.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.clients.admin; + +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.utils.Utils; + +import java.util.List; + +/** + * A description of the assignments of a specific group member. + */ +public class MemberAssignment { + private final List topicPartitions; + + /** + * Creates an instance with the specified parameters. + * + * @param topicPartitions List of topic partitions + */ + public MemberAssignment(List topicPartitions) { + this.topicPartitions = topicPartitions; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + MemberAssignment that = (MemberAssignment) o; + + return topicPartitions != null ? topicPartitions.equals(that.topicPartitions) : that.topicPartitions == null; + } + + @Override + public int hashCode() { + return topicPartitions != null ? topicPartitions.hashCode() : 0; + } + + /** + * The topic partitions assigned to a group member. + */ + public List topicPartitions() { + return topicPartitions; + } + + @Override + public String toString() { + return "(topicPartitions=" + Utils.join(topicPartitions, ",") + ")"; + } +} diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/MemberDescription.java b/clients/src/main/java/org/apache/kafka/clients/admin/MemberDescription.java new file mode 100644 index 0000000000000..2ba19634208e1 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/clients/admin/MemberDescription.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.clients.admin; + +/** + * A detailed description of a single group instance in the cluster. + */ +public class MemberDescription { + + private final String memberId; + private final String clientId; + private final String host; + private final MemberAssignment assignment; + + /** + * Creates an instance with the specified parameters. + * + * @param memberId The consumer id + * @param clientId The client id + * @param host The host + * @param assignment The assignment + */ + public MemberDescription(String memberId, String clientId, String host, MemberAssignment assignment) { + this.memberId = memberId; + this.clientId = clientId; + this.host = host; + this.assignment = assignment; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + MemberDescription that = (MemberDescription) o; + + if (memberId != null ? !memberId.equals(that.memberId) : that.memberId != null) return false; + if (clientId != null ? !clientId.equals(that.clientId) : that.clientId != null) return false; + return assignment != null ? assignment.equals(that.assignment) : that.assignment == null; + } + + @Override + public int hashCode() { + int result = memberId != null ? memberId.hashCode() : 0; + result = 31 * result + (clientId != null ? clientId.hashCode() : 0); + result = 31 * result + (assignment != null ? assignment.hashCode() : 0); + return result; + } + + /** + * The consumer id of the group member. + */ + public String consumerId() { + return memberId; + } + + /** + * The client id of the group member. + */ + public String clientId() { + return clientId; + } + + /** + * The host where the group member is running. + */ + public String host() { + return host; + } + + /** + * The assignment of the group member. + */ + public MemberAssignment assignment() { + return assignment; + } + + @Override + public String toString() { + return "(memberId=" + memberId + ", clientId=" + clientId + ", host=" + host + ", assignment=" + + assignment + ")"; + } +} diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/RenewDelegationTokenOptions.java b/clients/src/main/java/org/apache/kafka/clients/admin/RenewDelegationTokenOptions.java new file mode 100644 index 0000000000000..238dc4a3494a8 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/clients/admin/RenewDelegationTokenOptions.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.clients.admin; + +import org.apache.kafka.common.annotation.InterfaceStability; + +/** + * Options for {@link AdminClient#renewDelegationToken(byte[], RenewDelegationTokenOptions)}. + * + * The API of this class is evolving, see {@link AdminClient} for details. + */ +@InterfaceStability.Evolving +public class RenewDelegationTokenOptions extends AbstractOptions { + private long renewTimePeriodMs = -1; + + public RenewDelegationTokenOptions renewTimePeriodMs(long renewTimePeriodMs) { + this.renewTimePeriodMs = renewTimePeriodMs; + return this; + } + + public long renewTimePeriodMs() { + return renewTimePeriodMs; + } +} diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/RenewDelegationTokenResult.java b/clients/src/main/java/org/apache/kafka/clients/admin/RenewDelegationTokenResult.java new file mode 100644 index 0000000000000..38cdf1ae1b241 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/clients/admin/RenewDelegationTokenResult.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.clients.admin; + +import org.apache.kafka.common.KafkaFuture; +import org.apache.kafka.common.annotation.InterfaceStability; + +/** + * The result of the {@link KafkaAdminClient#expireDelegationToken(byte[], ExpireDelegationTokenOptions)} call. + * + * The API of this class is evolving, see {@link AdminClient} for details. + */ +@InterfaceStability.Evolving +public class RenewDelegationTokenResult { + private final KafkaFuture expiryTimestamp; + + RenewDelegationTokenResult(KafkaFuture expiryTimestamp) { + this.expiryTimestamp = expiryTimestamp; + } + + /** + * Returns a future which yields expiry timestamp + */ + public KafkaFuture expiryTimestamp() { + return expiryTimestamp; + } +} diff --git a/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java b/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java index 426b273b88538..0514c99563529 100644 --- a/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java +++ b/clients/src/main/java/org/apache/kafka/clients/producer/internals/Sender.java @@ -329,7 +329,7 @@ private boolean maybeSendTransactionalRequest(long now) { return false; AbstractRequest.Builder requestBuilder = nextRequestHandler.requestBuilder(); - while (running) { + while (!forceClose) { Node targetNode = null; try { if (nextRequestHandler.needsCoordinator()) { diff --git a/clients/src/main/java/org/apache/kafka/common/KafkaFuture.java b/clients/src/main/java/org/apache/kafka/common/KafkaFuture.java index 9cd2e01dc42f6..4af996cb6b8c7 100644 --- a/clients/src/main/java/org/apache/kafka/common/KafkaFuture.java +++ b/clients/src/main/java/org/apache/kafka/common/KafkaFuture.java @@ -106,6 +106,15 @@ public static KafkaFuture allOf(KafkaFuture... futures) { return allOfFuture; } + public KafkaFuture combine(KafkaFuture... futures) { + AllOfAdapter allOfWaiter = new AllOfAdapter<>(futures.length, this); + for (KafkaFuture future : futures) { + future.addWaiter(allOfWaiter); + } + + return this; + } + /** * Returns a new KafkaFuture that, when this future completes normally, is executed with this * futures's result as the argument to the supplied function. diff --git a/clients/src/main/java/org/apache/kafka/common/config/ConfigResource.java b/clients/src/main/java/org/apache/kafka/common/config/ConfigResource.java index 4402c260202e4..da718f54d547e 100644 --- a/clients/src/main/java/org/apache/kafka/common/config/ConfigResource.java +++ b/clients/src/main/java/org/apache/kafka/common/config/ConfigResource.java @@ -90,6 +90,6 @@ public int hashCode() { @Override public String toString() { - return "ConfigResource{type=" + type + ", name='" + name + "'}"; + return "ConfigResource(type=" + type + ", name='" + name + "')"; } } diff --git a/clients/src/main/java/org/apache/kafka/common/config/SaslConfigs.java b/clients/src/main/java/org/apache/kafka/common/config/SaslConfigs.java index f61b7dd5e6c66..148ab15dfadcb 100644 --- a/clients/src/main/java/org/apache/kafka/common/config/SaslConfigs.java +++ b/clients/src/main/java/org/apache/kafka/common/config/SaslConfigs.java @@ -49,7 +49,24 @@ public class SaslConfigs { public static final String SASL_JAAS_CONFIG = "sasl.jaas.config"; public static final String SASL_JAAS_CONFIG_DOC = "JAAS login context parameters for SASL connections in the format used by JAAS configuration files. " + "JAAS configuration file format is described here. " - + "The format for the value is: ' (=)*;'"; + + "The format for the value is: ' (=)*;'. For brokers, " + + "the config must be prefixed with listener prefix and SASL mechanism name in lower-case. For example, " + + "listener.name.sasl_ssl.scram-sha-256.sasl.jaas.config=com.example.ScramLoginModule required;"; + + public static final String SASL_CLIENT_CALLBACK_HANDLER_CLASS = "sasl.client.callback.handler.class"; + public static final String SASL_CLIENT_CALLBACK_HANDLER_CLASS_DOC = "The fully qualified name of a SASL client callback handler class " + + "that implements the AuthenticateCallbackHandler interface."; + + public static final String SASL_LOGIN_CALLBACK_HANDLER_CLASS = "sasl.login.callback.handler.class"; + public static final String SASL_LOGIN_CALLBACK_HANDLER_CLASS_DOC = "The fully qualified name of a SASL login callback handler class " + + "that implements the AuthenticateCallbackHandler interface. For brokers, login callback handler config must be prefixed with " + + "listener prefix and SASL mechanism name in lower-case. For example, " + + "listener.name.sasl_ssl.scram-sha-256.sasl.login.callback.handler.class=com.example.CustomScramLoginCallbackHandler"; + + public static final String SASL_LOGIN_CLASS = "sasl.login.class"; + public static final String SASL_LOGIN_CLASS_DOC = "The fully qualified name of a class that implements the Login interface. " + + "For brokers, login config must be prefixed with listener prefix and SASL mechanism name in lower-case. For example, " + + "listener.name.sasl_ssl.scram-sha-256.sasl.login.class=com.example.CustomScramLogin"; public static final String SASL_KERBEROS_SERVICE_NAME = "sasl.kerberos.service.name"; public static final String SASL_KERBEROS_SERVICE_NAME_DOC = "The Kerberos principal name that Kafka runs as. " @@ -95,6 +112,9 @@ public static void addClientSaslSupport(ConfigDef config) { .define(SaslConfigs.SASL_KERBEROS_TICKET_RENEW_JITTER, ConfigDef.Type.DOUBLE, SaslConfigs.DEFAULT_KERBEROS_TICKET_RENEW_JITTER, ConfigDef.Importance.LOW, SaslConfigs.SASL_KERBEROS_TICKET_RENEW_JITTER_DOC) .define(SaslConfigs.SASL_KERBEROS_MIN_TIME_BEFORE_RELOGIN, ConfigDef.Type.LONG, SaslConfigs.DEFAULT_KERBEROS_MIN_TIME_BEFORE_RELOGIN, ConfigDef.Importance.LOW, SaslConfigs.SASL_KERBEROS_MIN_TIME_BEFORE_RELOGIN_DOC) .define(SaslConfigs.SASL_MECHANISM, ConfigDef.Type.STRING, SaslConfigs.DEFAULT_SASL_MECHANISM, ConfigDef.Importance.MEDIUM, SaslConfigs.SASL_MECHANISM_DOC) - .define(SaslConfigs.SASL_JAAS_CONFIG, ConfigDef.Type.PASSWORD, null, ConfigDef.Importance.MEDIUM, SaslConfigs.SASL_JAAS_CONFIG_DOC); + .define(SaslConfigs.SASL_JAAS_CONFIG, ConfigDef.Type.PASSWORD, null, ConfigDef.Importance.MEDIUM, SaslConfigs.SASL_JAAS_CONFIG_DOC) + .define(SaslConfigs.SASL_CLIENT_CALLBACK_HANDLER_CLASS, ConfigDef.Type.CLASS, null, ConfigDef.Importance.MEDIUM, SaslConfigs.SASL_CLIENT_CALLBACK_HANDLER_CLASS_DOC) + .define(SaslConfigs.SASL_LOGIN_CALLBACK_HANDLER_CLASS, ConfigDef.Type.CLASS, null, ConfigDef.Importance.MEDIUM, SaslConfigs.SASL_LOGIN_CALLBACK_HANDLER_CLASS_DOC) + .define(SaslConfigs.SASL_LOGIN_CLASS, ConfigDef.Type.CLASS, null, ConfigDef.Importance.MEDIUM, SaslConfigs.SASL_LOGIN_CLASS_DOC); } } diff --git a/clients/src/main/java/org/apache/kafka/common/config/internals/BrokerSecurityConfigs.java b/clients/src/main/java/org/apache/kafka/common/config/internals/BrokerSecurityConfigs.java index 18616ec74e57e..a29d8069b9999 100644 --- a/clients/src/main/java/org/apache/kafka/common/config/internals/BrokerSecurityConfigs.java +++ b/clients/src/main/java/org/apache/kafka/common/config/internals/BrokerSecurityConfigs.java @@ -33,6 +33,7 @@ public class BrokerSecurityConfigs { public static final String SASL_KERBEROS_PRINCIPAL_TO_LOCAL_RULES_CONFIG = "sasl.kerberos.principal.to.local.rules"; public static final String SSL_CLIENT_AUTH_CONFIG = "ssl.client.auth"; public static final String SASL_ENABLED_MECHANISMS_CONFIG = "sasl.enabled.mechanisms"; + public static final String SASL_SERVER_CALLBACK_HANDLER_CLASS = "sasl.server.callback.handler.class"; public static final String PRINCIPAL_BUILDER_CLASS_DOC = "The fully qualified name of a class that implements the " + "KafkaPrincipalBuilder interface, which is used to build the KafkaPrincipal object used during " + @@ -67,4 +68,9 @@ public class BrokerSecurityConfigs { + "Only GSSAPI is enabled by default."; public static final List DEFAULT_SASL_ENABLED_MECHANISMS = Collections.singletonList(SaslConfigs.GSSAPI_MECHANISM); + public static final String SASL_SERVER_CALLBACK_HANDLER_CLASS_DOC = "The fully qualified name of a SASL server callback handler " + + "class that implements the AuthenticateCallbackHandler interface. Server callback handlers must be prefixed with " + + "listener prefix and SASL mechanism name in lower-case. For example, " + + "listener.name.sasl_ssl.plain.sasl.server.callback.handler.class=com.example.CustomPlainCallbackHandler."; + } diff --git a/clients/src/main/java/org/apache/kafka/common/network/ChannelBuilders.java b/clients/src/main/java/org/apache/kafka/common/network/ChannelBuilders.java index 80ccb7e1382a2..078d844e85fce 100644 --- a/clients/src/main/java/org/apache/kafka/common/network/ChannelBuilders.java +++ b/clients/src/main/java/org/apache/kafka/common/network/ChannelBuilders.java @@ -26,7 +26,7 @@ import org.apache.kafka.common.security.auth.KafkaPrincipalBuilder; import org.apache.kafka.common.security.authenticator.CredentialCache; import org.apache.kafka.common.security.kerberos.KerberosShortNamer; -import org.apache.kafka.common.security.token.delegation.DelegationTokenCache; +import org.apache.kafka.common.security.token.delegation.internal.DelegationTokenCache; import org.apache.kafka.common.utils.Utils; import java.util.Collections; diff --git a/clients/src/main/java/org/apache/kafka/common/network/ListenerName.java b/clients/src/main/java/org/apache/kafka/common/network/ListenerName.java index fc0cb14925b01..2decccbc50649 100644 --- a/clients/src/main/java/org/apache/kafka/common/network/ListenerName.java +++ b/clients/src/main/java/org/apache/kafka/common/network/ListenerName.java @@ -73,6 +73,10 @@ public String configPrefix() { } public String saslMechanismConfigPrefix(String saslMechanism) { - return configPrefix() + saslMechanism.toLowerCase(Locale.ROOT) + "."; + return configPrefix() + saslMechanismPrefix(saslMechanism); + } + + public static String saslMechanismPrefix(String saslMechanism) { + return saslMechanism.toLowerCase(Locale.ROOT) + "."; } } diff --git a/clients/src/main/java/org/apache/kafka/common/network/SaslChannelBuilder.java b/clients/src/main/java/org/apache/kafka/common/network/SaslChannelBuilder.java index 095f826ccba5a..3985c7e97fe56 100644 --- a/clients/src/main/java/org/apache/kafka/common/network/SaslChannelBuilder.java +++ b/clients/src/main/java/org/apache/kafka/common/network/SaslChannelBuilder.java @@ -21,22 +21,35 @@ import org.apache.kafka.common.config.SslConfigs; import org.apache.kafka.common.config.internals.BrokerSecurityConfigs; import org.apache.kafka.common.memory.MemoryPool; -import org.apache.kafka.common.security.auth.SecurityProtocol; import org.apache.kafka.common.security.JaasContext; -import org.apache.kafka.common.security.token.delegation.DelegationTokenCache; -import org.apache.kafka.common.security.kerberos.KerberosShortNamer; +import org.apache.kafka.common.security.auth.AuthenticateCallbackHandler; +import org.apache.kafka.common.security.auth.Login; +import org.apache.kafka.common.security.auth.SecurityProtocol; import org.apache.kafka.common.security.authenticator.CredentialCache; +import org.apache.kafka.common.security.authenticator.DefaultLogin; import org.apache.kafka.common.security.authenticator.LoginManager; import org.apache.kafka.common.security.authenticator.SaslClientAuthenticator; +import org.apache.kafka.common.security.authenticator.SaslClientCallbackHandler; import org.apache.kafka.common.security.authenticator.SaslServerAuthenticator; +import org.apache.kafka.common.security.authenticator.SaslServerCallbackHandler; +import org.apache.kafka.common.security.kerberos.KerberosClientCallbackHandler; +import org.apache.kafka.common.security.kerberos.KerberosLogin; +import org.apache.kafka.common.security.kerberos.KerberosShortNamer; +import org.apache.kafka.common.security.plain.internal.PlainSaslServer; +import org.apache.kafka.common.security.plain.internal.PlainServerCallbackHandler; +import org.apache.kafka.common.security.scram.ScramCredential; +import org.apache.kafka.common.security.scram.internal.ScramMechanism; +import org.apache.kafka.common.security.scram.internal.ScramServerCallbackHandler; import org.apache.kafka.common.security.ssl.SslFactory; +import org.apache.kafka.common.security.token.delegation.internal.DelegationTokenCache; import org.apache.kafka.common.utils.Java; +import org.apache.kafka.common.utils.Utils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; +import java.io.IOException; import java.net.Socket; import java.nio.channels.SelectionKey; import java.nio.channels.SocketChannel; @@ -66,6 +79,7 @@ public class SaslChannelBuilder implements ChannelBuilder, ListenerReconfigurabl private SslFactory sslFactory; private Map configs; private KerberosShortNamer kerberosShortNamer; + private Map saslCallbackHandlers; public SaslChannelBuilder(Mode mode, Map jaasContexts, @@ -87,15 +101,25 @@ public SaslChannelBuilder(Mode mode, this.clientSaslMechanism = clientSaslMechanism; this.credentialCache = credentialCache; this.tokenCache = tokenCache; + this.saslCallbackHandlers = new HashMap<>(); } + @SuppressWarnings("unchecked") @Override public void configure(Map configs) throws KafkaException { try { this.configs = configs; - boolean hasKerberos = jaasContexts.containsKey(SaslConfigs.GSSAPI_MECHANISM); + if (mode == Mode.SERVER) + createServerCallbackHandlers(configs); + else + createClientCallbackHandler(configs); + for (Map.Entry entry : saslCallbackHandlers.entrySet()) { + String mechanism = entry.getKey(); + entry.getValue().configure(configs, mechanism, jaasContexts.get(mechanism).configurationEntries()); + } - if (hasKerberos) { + Class defaultLoginClass = DefaultLogin.class; + if (jaasContexts.containsKey(SaslConfigs.GSSAPI_MECHANISM)) { String defaultRealm; try { defaultRealm = defaultKerberosRealm(); @@ -106,12 +130,13 @@ public void configure(Map configs) throws KafkaException { List principalToLocalRules = (List) configs.get(BrokerSecurityConfigs.SASL_KERBEROS_PRINCIPAL_TO_LOCAL_RULES_CONFIG); if (principalToLocalRules != null) kerberosShortNamer = KerberosShortNamer.fromUnparsedRules(defaultRealm, principalToLocalRules); + defaultLoginClass = KerberosLogin.class; } for (Map.Entry entry : jaasContexts.entrySet()) { String mechanism = entry.getKey(); // With static JAAS configuration, use KerberosLogin if Kerberos is enabled. With dynamic JAAS configuration, // use KerberosLogin only for the LoginContext corresponding to GSSAPI - LoginManager loginManager = LoginManager.acquireLoginManager(entry.getValue(), mechanism, hasKerberos, configs); + LoginManager loginManager = LoginManager.acquireLoginManager(entry.getValue(), mechanism, defaultLoginClass, configs); loginManagers.put(mechanism, loginManager); subjects.put(mechanism, loginManager.subject()); } @@ -120,7 +145,7 @@ public void configure(Map configs) throws KafkaException { this.sslFactory = new SslFactory(mode, "none", isInterBrokerListener); this.sslFactory.configure(configs); } - } catch (Exception e) { + } catch (Throwable e) { close(); throw new KafkaException(e); } @@ -156,11 +181,20 @@ public KafkaChannel buildChannel(String id, SelectionKey key, int maxReceiveSize TransportLayer transportLayer = buildTransportLayer(id, key, socketChannel); Authenticator authenticator; if (mode == Mode.SERVER) { - authenticator = buildServerAuthenticator(configs, id, transportLayer, subjects); + authenticator = buildServerAuthenticator(configs, + saslCallbackHandlers, + id, + transportLayer, + subjects); } else { LoginManager loginManager = loginManagers.get(clientSaslMechanism); - authenticator = buildClientAuthenticator(configs, id, socket.getInetAddress().getHostName(), - loginManager.serviceName(), transportLayer, loginManager.subject()); + authenticator = buildClientAuthenticator(configs, + saslCallbackHandlers.get(clientSaslMechanism), + id, + socket.getInetAddress().getHostName(), + loginManager.serviceName(), + transportLayer, + subjects.get(clientSaslMechanism)); } return new KafkaChannel(id, transportLayer, authenticator, maxReceiveSize, memoryPool != null ? memoryPool : MemoryPool.NONE); } catch (Exception e) { @@ -174,6 +208,8 @@ public void close() { for (LoginManager loginManager : loginManagers.values()) loginManager.release(); loginManagers.clear(); + for (AuthenticateCallbackHandler handler : saslCallbackHandlers.values()) + handler.close(); } private TransportLayer buildTransportLayer(String id, SelectionKey key, SocketChannel socketChannel) throws IOException { @@ -186,16 +222,23 @@ private TransportLayer buildTransportLayer(String id, SelectionKey key, SocketCh } // Visible to override for testing - protected SaslServerAuthenticator buildServerAuthenticator(Map configs, String id, - TransportLayer transportLayer, Map subjects) throws IOException { - return new SaslServerAuthenticator(configs, id, jaasContexts, subjects, - kerberosShortNamer, credentialCache, listenerName, securityProtocol, transportLayer, tokenCache); + protected SaslServerAuthenticator buildServerAuthenticator(Map configs, + Map callbackHandlers, + String id, + TransportLayer transportLayer, + Map subjects) throws IOException { + return new SaslServerAuthenticator(configs, callbackHandlers, id, subjects, + kerberosShortNamer, listenerName, securityProtocol, transportLayer); } // Visible to override for testing - protected SaslClientAuthenticator buildClientAuthenticator(Map configs, String id, - String serverHost, String servicePrincipal, TransportLayer transportLayer, Subject subject) throws IOException { - return new SaslClientAuthenticator(configs, id, subject, servicePrincipal, + protected SaslClientAuthenticator buildClientAuthenticator(Map configs, + AuthenticateCallbackHandler callbackHandler, + String id, + String serverHost, + String servicePrincipal, + TransportLayer transportLayer, Subject subject) throws IOException { + return new SaslClientAuthenticator(configs, callbackHandler, id, subject, servicePrincipal, serverHost, clientSaslMechanism, handshakeRequestEnable, transportLayer); } @@ -224,4 +267,30 @@ private static String defaultKerberosRealm() throws ClassNotFoundException, NoSu getDefaultRealmMethod = classRef.getDeclaredMethod("getDefaultRealm", new Class[0]); return (String) getDefaultRealmMethod.invoke(kerbConf, new Object[0]); } + + private void createClientCallbackHandler(Map configs) { + Class clazz = (Class) configs.get(SaslConfigs.SASL_CLIENT_CALLBACK_HANDLER_CLASS); + if (clazz == null) + clazz = clientSaslMechanism.equals(SaslConfigs.GSSAPI_MECHANISM) ? KerberosClientCallbackHandler.class : SaslClientCallbackHandler.class; + AuthenticateCallbackHandler callbackHandler = Utils.newInstance(clazz); + saslCallbackHandlers.put(clientSaslMechanism, callbackHandler); + } + + private void createServerCallbackHandlers(Map configs) throws ClassNotFoundException { + for (String mechanism : jaasContexts.keySet()) { + AuthenticateCallbackHandler callbackHandler; + String prefix = ListenerName.saslMechanismPrefix(mechanism); + Class clazz = + (Class) configs.get(prefix + BrokerSecurityConfigs.SASL_SERVER_CALLBACK_HANDLER_CLASS); + if (clazz != null) + callbackHandler = Utils.newInstance(clazz); + else if (mechanism.equals(PlainSaslServer.PLAIN_MECHANISM)) + callbackHandler = new PlainServerCallbackHandler(); + else if (ScramMechanism.isScram(mechanism)) + callbackHandler = new ScramServerCallbackHandler(credentialCache.cache(mechanism, ScramCredential.class), tokenCache); + else + callbackHandler = new SaslServerCallbackHandler(); + saslCallbackHandlers.put(mechanism, callbackHandler); + } + } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenResponse.java index dba29eafe9937..7ba270a61530b 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/DescribeDelegationTokenResponse.java @@ -184,4 +184,8 @@ public Errors error() { public List tokens() { return tokens; } + + public boolean hasError() { + return this.error != Errors.NONE; + } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenRequest.java index 0d43440d32986..40f0aadc0bb85 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ExpireDelegationTokenRequest.java @@ -88,9 +88,9 @@ public static class Builder extends AbstractRequest.Builderkey from this context. * If login module name is specified, return option value only from that module. */ - public String configEntryOption(String key, String loginModuleName) { + public static String configEntryOption(List configurationEntries, String key, String loginModuleName) { for (AppConfigurationEntry entry : configurationEntries) { if (loginModuleName != null && !loginModuleName.equals(entry.getLoginModuleName())) continue; diff --git a/clients/src/main/java/org/apache/kafka/common/security/auth/AuthenticateCallbackHandler.java b/clients/src/main/java/org/apache/kafka/common/security/auth/AuthenticateCallbackHandler.java new file mode 100644 index 0000000000000..8951d3a589367 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/common/security/auth/AuthenticateCallbackHandler.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.common.security.auth; + +import java.util.List; +import java.util.Map; + +import javax.security.auth.callback.CallbackHandler; +import javax.security.auth.login.AppConfigurationEntry; + +/* + * Callback handler for SASL-based authentication + */ +public interface AuthenticateCallbackHandler extends CallbackHandler { + + /** + * Configures this callback handler for the specified SASL mechanism. + * + * @param configs Key-value pairs containing the parsed configuration options of + * the client or broker. Note that these are the Kafka configuration options + * and not the JAAS configuration options. JAAS config options may be obtained + * from `jaasConfigEntries` for callbacks which obtain some configs from the + * JAAS configuration. For configs that may be specified as both Kafka config + * as well as JAAS config (e.g. sasl.kerberos.service.name), the configuration + * is treated as invalid if conflicting values are provided. + * @param saslMechanism Negotiated SASL mechanism. For clients, this is the SASL + * mechanism configured for the client. For brokers, this is the mechanism + * negotiated with the client and is one of the mechanisms enabled on the broker. + * @param jaasConfigEntries JAAS configuration entries from the JAAS login context. + * This list contains a single entry for clients and may contain more than + * one entry for brokers if multiple mechanisms are enabled on a listener using + * static JAAS configuration where there is no mapping between mechanisms and + * login module entries. In this case, callback handlers can use the login module in + * `jaasConfigEntries` to identify the entry corresponding to `saslMechanism`. + * Alternatively, dynamic JAAS configuration option + * {@link org.apache.kafka.common.config.SaslConfigs#SASL_JAAS_CONFIG} may be + * configured on brokers with listener and mechanism prefix, in which case + * only the configuration entry corresponding to `saslMechanism` will be provided + * in `jaasConfigEntries`. + */ + void configure(Map configs, String saslMechanism, List jaasConfigEntries); + + /** + * Closes this instance. + */ + void close(); +} diff --git a/clients/src/main/java/org/apache/kafka/common/security/authenticator/Login.java b/clients/src/main/java/org/apache/kafka/common/security/auth/Login.java similarity index 55% rename from clients/src/main/java/org/apache/kafka/common/security/authenticator/Login.java rename to clients/src/main/java/org/apache/kafka/common/security/auth/Login.java index b41d1b2572fcc..eda5e7a225af1 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/authenticator/Login.java +++ b/clients/src/main/java/org/apache/kafka/common/security/auth/Login.java @@ -14,13 +14,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.kafka.common.security.authenticator; - -import org.apache.kafka.common.security.JaasContext; +package org.apache.kafka.common.security.auth; import java.util.Map; import javax.security.auth.Subject; +import javax.security.auth.login.Configuration; import javax.security.auth.login.LoginContext; import javax.security.auth.login.LoginException; @@ -31,8 +30,21 @@ public interface Login { /** * Configures this login instance. + * @param configs Key-value pairs containing the parsed configuration options of + * the client or broker. Note that these are the Kafka configuration options + * and not the JAAS configuration options. The JAAS options may be obtained + * from `jaasConfiguration`. + * @param contextName JAAS context name for this login which may be used to obtain + * the login context from `jaasConfiguration`. + * @param jaasConfiguration JAAS configuration containing the login context named + * `contextName`. If static JAAS configuration is used, this `Configuration` + * may also contain other login contexts. + * @param loginCallbackHandler Login callback handler instance to use for this Login. + * Login callback handler class may be configured using + * {@link org.apache.kafka.common.config.SaslConfigs#SASL_LOGIN_CALLBACK_HANDLER_CLASS}. */ - void configure(Map configs, JaasContext jaasContext); + void configure(Map configs, String contextName, Configuration jaasConfiguration, + AuthenticateCallbackHandler loginCallbackHandler); /** * Performs login for each login module specified for the login context of this instance. @@ -54,4 +66,3 @@ public interface Login { */ void close(); } - diff --git a/clients/src/main/java/org/apache/kafka/common/security/authenticator/AbstractLogin.java b/clients/src/main/java/org/apache/kafka/common/security/authenticator/AbstractLogin.java index 643f859e8298c..7e1350864e438 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/authenticator/AbstractLogin.java +++ b/clients/src/main/java/org/apache/kafka/common/security/authenticator/AbstractLogin.java @@ -16,20 +16,23 @@ */ package org.apache.kafka.common.security.authenticator; +import javax.security.auth.login.AppConfigurationEntry; +import javax.security.auth.login.Configuration; import javax.security.auth.login.LoginContext; import javax.security.auth.login.LoginException; import javax.security.sasl.RealmCallback; import javax.security.auth.callback.Callback; -import javax.security.auth.callback.CallbackHandler; import javax.security.auth.callback.NameCallback; import javax.security.auth.callback.PasswordCallback; import javax.security.auth.callback.UnsupportedCallbackException; import javax.security.auth.Subject; -import org.apache.kafka.common.security.JaasContext; +import org.apache.kafka.common.security.auth.AuthenticateCallbackHandler; +import org.apache.kafka.common.security.auth.Login; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.List; import java.util.Map; /** @@ -38,17 +41,22 @@ public abstract class AbstractLogin implements Login { private static final Logger log = LoggerFactory.getLogger(AbstractLogin.class); - private JaasContext jaasContext; + private String contextName; + private Configuration configuration; private LoginContext loginContext; + private AuthenticateCallbackHandler loginCallbackHandler; @Override - public void configure(Map configs, JaasContext jaasContext) { - this.jaasContext = jaasContext; + public void configure(Map configs, String contextName, Configuration configuration, + AuthenticateCallbackHandler loginCallbackHandler) { + this.contextName = contextName; + this.configuration = configuration; + this.loginCallbackHandler = loginCallbackHandler; } @Override public LoginContext login() throws LoginException { - loginContext = new LoginContext(jaasContext.name(), null, new LoginCallbackHandler(), jaasContext.configuration()); + loginContext = new LoginContext(contextName, null, loginCallbackHandler, configuration); loginContext.login(); log.info("Successfully logged in."); return loginContext; @@ -59,8 +67,12 @@ public Subject subject() { return loginContext.getSubject(); } - protected JaasContext jaasContext() { - return jaasContext; + protected String contextName() { + return contextName; + } + + protected Configuration configuration() { + return configuration; } /** @@ -70,7 +82,11 @@ protected JaasContext jaasContext() { * callback handlers which require additional user input. * */ - public static class LoginCallbackHandler implements CallbackHandler { + public static class DefaultLoginCallbackHandler implements AuthenticateCallbackHandler { + + @Override + public void configure(Map configs, String saslMechanism, List jaasConfigEntries) { + } @Override public void handle(Callback[] callbacks) throws UnsupportedCallbackException { @@ -90,6 +106,10 @@ public void handle(Callback[] callbacks) throws UnsupportedCallbackException { } } } + + @Override + public void close() { + } } } diff --git a/clients/src/main/java/org/apache/kafka/common/security/authenticator/LoginManager.java b/clients/src/main/java/org/apache/kafka/common/security/authenticator/LoginManager.java index 81dc063c7433f..4ae798d7d293b 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/authenticator/LoginManager.java +++ b/clients/src/main/java/org/apache/kafka/common/security/authenticator/LoginManager.java @@ -23,11 +23,17 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.Map; +import java.util.Objects; + +import org.apache.kafka.common.config.ConfigException; import org.apache.kafka.common.config.SaslConfigs; import org.apache.kafka.common.config.types.Password; +import org.apache.kafka.common.network.ListenerName; import org.apache.kafka.common.security.JaasContext; -import org.apache.kafka.common.security.kerberos.KerberosLogin; +import org.apache.kafka.common.security.auth.AuthenticateCallbackHandler; +import org.apache.kafka.common.security.auth.Login; +import org.apache.kafka.common.utils.Utils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -36,20 +42,23 @@ public class LoginManager { private static final Logger LOGGER = LoggerFactory.getLogger(LoginManager.class); // static configs (broker or client) - private static final Map STATIC_INSTANCES = new HashMap<>(); + private static final Map, LoginManager> STATIC_INSTANCES = new HashMap<>(); - // dynamic configs (client-only) - private static final Map DYNAMIC_INSTANCES = new HashMap<>(); + // dynamic configs (broker or client) + private static final Map, LoginManager> DYNAMIC_INSTANCES = new HashMap<>(); private final Login login; - private final Object cacheKey; + private final LoginMetadata loginMetadata; + private final AuthenticateCallbackHandler loginCallbackHandler; private int refCount; - private LoginManager(JaasContext jaasContext, boolean hasKerberos, Map configs, - Object cacheKey) throws IOException, LoginException { - this.cacheKey = cacheKey; - login = hasKerberos ? new KerberosLogin() : new DefaultLogin(); - login.configure(configs, jaasContext); + private LoginManager(JaasContext jaasContext, String saslMechanism, Map configs, + LoginMetadata loginMetadata) throws IOException, LoginException { + this.loginMetadata = loginMetadata; + this.login = Utils.newInstance(loginMetadata.loginClass); + loginCallbackHandler = Utils.newInstance(loginMetadata.loginCallbackClass); + loginCallbackHandler.configure(configs, saslMechanism, jaasContext.configurationEntries()); + login.configure(configs, jaasContext.name(), jaasContext.configuration(), loginCallbackHandler); login.login(); } @@ -72,28 +81,34 @@ private LoginManager(JaasContext jaasContext, boolean hasKerberos, Map defaultLoginClass, Map configs) throws IOException, LoginException { + Class loginClass = configuredClassOrDefault(configs, jaasContext, + saslMechanism, SaslConfigs.SASL_LOGIN_CLASS, defaultLoginClass); + Class loginCallbackClass = configuredClassOrDefault(configs, + jaasContext, saslMechanism, SaslConfigs.SASL_LOGIN_CALLBACK_HANDLER_CLASS, + AbstractLogin.DefaultLoginCallbackHandler.class); synchronized (LoginManager.class) { LoginManager loginManager; Password jaasConfigValue = jaasContext.dynamicJaasConfig(); if (jaasConfigValue != null) { - loginManager = DYNAMIC_INSTANCES.get(jaasConfigValue); + LoginMetadata loginMetadata = new LoginMetadata<>(jaasConfigValue, loginClass, loginCallbackClass); + loginManager = DYNAMIC_INSTANCES.get(loginMetadata); if (loginManager == null) { - loginManager = new LoginManager(jaasContext, saslMechanism.equals(SaslConfigs.GSSAPI_MECHANISM), configs, jaasConfigValue); - DYNAMIC_INSTANCES.put(jaasConfigValue, loginManager); + loginManager = new LoginManager(jaasContext, saslMechanism, configs, loginMetadata); + DYNAMIC_INSTANCES.put(loginMetadata, loginManager); } } else { - loginManager = STATIC_INSTANCES.get(jaasContext.name()); + LoginMetadata loginMetadata = new LoginMetadata<>(jaasContext.name(), loginClass, loginCallbackClass); + loginManager = STATIC_INSTANCES.get(loginMetadata); if (loginManager == null) { - loginManager = new LoginManager(jaasContext, hasKerberos, configs, jaasContext.name()); - STATIC_INSTANCES.put(jaasContext.name(), loginManager); + loginManager = new LoginManager(jaasContext, saslMechanism, configs, loginMetadata); + STATIC_INSTANCES.put(loginMetadata, loginManager); } } return loginManager.acquire(); @@ -110,7 +125,7 @@ public String serviceName() { // Only for testing Object cacheKey() { - return cacheKey; + return loginMetadata.configInfo; } private LoginManager acquire() { @@ -127,12 +142,13 @@ public void release() { if (refCount == 0) throw new IllegalStateException("release() called on disposed " + this); else if (refCount == 1) { - if (cacheKey instanceof Password) { - DYNAMIC_INSTANCES.remove(cacheKey); + if (loginMetadata.configInfo instanceof Password) { + DYNAMIC_INSTANCES.remove(loginMetadata); } else { - STATIC_INSTANCES.remove(cacheKey); + STATIC_INSTANCES.remove(loginMetadata); } login.close(); + loginCallbackHandler.close(); } --refCount; LOGGER.trace("{} released", this); @@ -150,10 +166,56 @@ public String toString() { /* Should only be used in tests. */ public static void closeAll() { synchronized (LoginManager.class) { - for (String key : new ArrayList<>(STATIC_INSTANCES.keySet())) + for (LoginMetadata key : new ArrayList<>(STATIC_INSTANCES.keySet())) STATIC_INSTANCES.remove(key).login.close(); - for (Password key : new ArrayList<>(DYNAMIC_INSTANCES.keySet())) + for (LoginMetadata key : new ArrayList<>(DYNAMIC_INSTANCES.keySet())) DYNAMIC_INSTANCES.remove(key).login.close(); } } + + private static Class configuredClassOrDefault(Map configs, + JaasContext jaasContext, + String saslMechanism, + String configName, + Class defaultClass) { + String prefix = jaasContext.type() == JaasContext.Type.SERVER ? ListenerName.saslMechanismPrefix(saslMechanism) : ""; + Class clazz = (Class) configs.get(prefix + configName); + if (clazz != null && jaasContext.configurationEntries().size() != 1) { + String errorMessage = configName + " cannot be specified with multiple login modules in the JAAS context. " + + SaslConfigs.SASL_JAAS_CONFIG + " must be configured to override mechanism-specific configs."; + throw new ConfigException(errorMessage); + } + if (clazz == null) + clazz = defaultClass; + return clazz; + } + + private static class LoginMetadata { + final T configInfo; + final Class loginClass; + final Class loginCallbackClass; + + LoginMetadata(T configInfo, Class loginClass, + Class loginCallbackClass) { + this.configInfo = configInfo; + this.loginClass = loginClass; + this.loginCallbackClass = loginCallbackClass; + } + + @Override + public int hashCode() { + return Objects.hash(configInfo, loginClass, loginCallbackClass); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + LoginMetadata loginMetadata = (LoginMetadata) o; + return Objects.equals(configInfo, loginMetadata.configInfo) && + Objects.equals(loginClass, loginMetadata.loginClass) && + Objects.equals(loginCallbackClass, loginMetadata.loginCallbackClass); + } + } } diff --git a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java index 8b0116563d8be..2ef6d77f13f36 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java +++ b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientAuthenticator.java @@ -24,9 +24,8 @@ import org.apache.kafka.common.errors.SaslAuthenticationException; import org.apache.kafka.common.errors.UnsupportedSaslMechanismException; import org.apache.kafka.common.network.Authenticator; -import org.apache.kafka.common.network.Mode; -import org.apache.kafka.common.network.NetworkReceive; import org.apache.kafka.common.network.NetworkSend; +import org.apache.kafka.common.network.NetworkReceive; import org.apache.kafka.common.network.Send; import org.apache.kafka.common.network.TransportLayer; import org.apache.kafka.common.protocol.ApiKeys; @@ -41,6 +40,7 @@ import org.apache.kafka.common.requests.SaslAuthenticateResponse; import org.apache.kafka.common.requests.SaslHandshakeRequest; import org.apache.kafka.common.requests.SaslHandshakeResponse; +import org.apache.kafka.common.security.auth.AuthenticateCallbackHandler; import org.apache.kafka.common.security.auth.KafkaPrincipal; import org.apache.kafka.common.utils.Utils; import org.slf4j.Logger; @@ -87,7 +87,7 @@ public enum SaslState { private final SaslClient saslClient; private final Map configs; private final String clientPrincipalName; - private final AuthCallbackHandler callbackHandler; + private final AuthenticateCallbackHandler callbackHandler; // buffers used in `authenticate` private NetworkReceive netInBuffer; @@ -105,6 +105,7 @@ public enum SaslState { private short saslAuthenticateVersion; public SaslClientAuthenticator(Map configs, + AuthenticateCallbackHandler callbackHandler, String node, Subject subject, String servicePrincipal, @@ -114,6 +115,7 @@ public SaslClientAuthenticator(Map configs, TransportLayer transportLayer) throws IOException { this.node = node; this.subject = subject; + this.callbackHandler = callbackHandler; this.host = host; this.servicePrincipal = servicePrincipal; this.mechanism = mechanism; @@ -133,9 +135,6 @@ public SaslClientAuthenticator(Map configs, else this.clientPrincipalName = null; - callbackHandler = new SaslClientCallbackHandler(); - callbackHandler.configure(configs, Mode.CLIENT, subject, mechanism); - saslClient = createSaslClient(); } catch (Exception e) { throw new SaslAuthenticationException("Failed to configure SaslClientAuthenticator", e); @@ -325,8 +324,6 @@ public boolean complete() { public void close() throws IOException { if (saslClient != null) saslClient.dispose(); - if (callbackHandler != null) - callbackHandler.close(); } private byte[] receiveToken() throws IOException { diff --git a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientCallbackHandler.java b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientCallbackHandler.java index 31c51c22ca38c..5b2a28181cd58 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientCallbackHandler.java +++ b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslClientCallbackHandler.java @@ -16,6 +16,8 @@ */ package org.apache.kafka.common.security.authenticator; +import java.security.AccessController; +import java.util.List; import java.util.Map; import javax.security.auth.Subject; @@ -23,52 +25,46 @@ import javax.security.auth.callback.NameCallback; import javax.security.auth.callback.PasswordCallback; import javax.security.auth.callback.UnsupportedCallbackException; +import javax.security.auth.login.AppConfigurationEntry; import javax.security.sasl.AuthorizeCallback; import javax.security.sasl.RealmCallback; import org.apache.kafka.common.config.SaslConfigs; -import org.apache.kafka.common.network.Mode; import org.apache.kafka.common.security.scram.ScramExtensionsCallback; +import org.apache.kafka.common.security.auth.AuthenticateCallbackHandler; /** - * Callback handler for Sasl clients. The callbacks required for the SASL mechanism + * Default callback handler for Sasl clients. The callbacks required for the SASL mechanism * configured for the client should be supported by this callback handler. See * Java SASL API * for the list of SASL callback handlers required for each SASL mechanism. */ -public class SaslClientCallbackHandler implements AuthCallbackHandler { +public class SaslClientCallbackHandler implements AuthenticateCallbackHandler { - private boolean isKerberos; - private Subject subject; + private String mechanism; @Override - public void configure(Map configs, Mode mode, Subject subject, String mechanism) { - this.isKerberos = mechanism.equals(SaslConfigs.GSSAPI_MECHANISM); - this.subject = subject; + public void configure(Map configs, String saslMechanism, List jaasConfigEntries) { + this.mechanism = saslMechanism; } @Override public void handle(Callback[] callbacks) throws UnsupportedCallbackException { + Subject subject = Subject.getSubject(AccessController.getContext()); for (Callback callback : callbacks) { if (callback instanceof NameCallback) { NameCallback nc = (NameCallback) callback; - if (!isKerberos && subject != null && !subject.getPublicCredentials(String.class).isEmpty()) { + if (subject != null && !subject.getPublicCredentials(String.class).isEmpty()) { nc.setName(subject.getPublicCredentials(String.class).iterator().next()); } else nc.setName(nc.getDefaultName()); } else if (callback instanceof PasswordCallback) { - if (!isKerberos && subject != null && !subject.getPrivateCredentials(String.class).isEmpty()) { + if (subject != null && !subject.getPrivateCredentials(String.class).isEmpty()) { char[] password = subject.getPrivateCredentials(String.class).iterator().next().toCharArray(); ((PasswordCallback) callback).setPassword(password); } else { String errorMessage = "Could not login: the client is being asked for a password, but the Kafka" + " client code does not currently support obtaining a password from the user."; - if (isKerberos) { - errorMessage += " Make sure -Djava.security.auth.login.config property passed to JVM and" + - " the client is configured to use a ticket cache (using" + - " the JAAS configuration setting 'useTicketCache=true)'. Make sure you are using" + - " FQDN of the Kafka broker you are trying to connect to."; - } throw new UnsupportedCallbackException(callback, errorMessage); } } else if (callback instanceof RealmCallback) { @@ -83,7 +79,7 @@ public void handle(Callback[] callbacks) throws UnsupportedCallbackException { ac.setAuthorizedID(authzId); } else if (callback instanceof ScramExtensionsCallback) { ScramExtensionsCallback sc = (ScramExtensionsCallback) callback; - if (!isKerberos && subject != null && !subject.getPublicCredentials(Map.class).isEmpty()) { + if (!SaslConfigs.GSSAPI_MECHANISM.equals(mechanism) && subject != null && !subject.getPublicCredentials(Map.class).isEmpty()) { sc.extensions((Map) subject.getPublicCredentials(Map.class).iterator().next()); } } else { diff --git a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticator.java b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticator.java index 2a80e5bc0e4c0..5140afb196d2c 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticator.java +++ b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticator.java @@ -28,7 +28,6 @@ import org.apache.kafka.common.network.Authenticator; import org.apache.kafka.common.network.ChannelBuilders; import org.apache.kafka.common.network.ListenerName; -import org.apache.kafka.common.network.Mode; import org.apache.kafka.common.network.NetworkReceive; import org.apache.kafka.common.network.NetworkSend; import org.apache.kafka.common.network.Send; @@ -46,18 +45,15 @@ import org.apache.kafka.common.requests.SaslAuthenticateResponse; import org.apache.kafka.common.requests.SaslHandshakeRequest; import org.apache.kafka.common.requests.SaslHandshakeResponse; -import org.apache.kafka.common.security.JaasContext; +import org.apache.kafka.common.security.auth.AuthenticateCallbackHandler; import org.apache.kafka.common.security.auth.KafkaPrincipal; import org.apache.kafka.common.security.auth.KafkaPrincipalBuilder; import org.apache.kafka.common.security.auth.SaslAuthenticationContext; import org.apache.kafka.common.security.kerberos.KerberosName; import org.apache.kafka.common.security.kerberos.KerberosShortNamer; -import org.apache.kafka.common.security.scram.ScramCredential; import org.apache.kafka.common.security.scram.ScramLoginModule; -import org.apache.kafka.common.security.scram.ScramMechanism; -import org.apache.kafka.common.security.scram.ScramServerCallbackHandler; +import org.apache.kafka.common.security.scram.internal.ScramMechanism; import org.apache.kafka.common.utils.Utils; -import org.apache.kafka.common.security.token.delegation.DelegationTokenCache; import org.ietf.jgss.GSSContext; import org.ietf.jgss.GSSCredential; import org.ietf.jgss.GSSException; @@ -102,14 +98,12 @@ private enum SaslState { private final SecurityProtocol securityProtocol; private final ListenerName listenerName; private final String connectionId; - private final Map jaasContexts; private final Map subjects; - private final CredentialCache credentialCache; private final TransportLayer transportLayer; private final Set enabledMechanisms; private final Map configs; private final KafkaPrincipalBuilder principalBuilder; - private final DelegationTokenCache tokenCache; + private final Map callbackHandlers; // Current SASL state private SaslState saslState = SaslState.INITIAL_REQUEST; @@ -119,7 +113,6 @@ private enum SaslState { private AuthenticationException pendingException = null; private SaslServer saslServer; private String saslMechanism; - private AuthCallbackHandler callbackHandler; // buffers used in `authenticate` private NetworkReceive netInBuffer; @@ -128,23 +121,19 @@ private enum SaslState { private boolean enableKafkaSaslAuthenticateHeaders; public SaslServerAuthenticator(Map configs, + Map callbackHandlers, String connectionId, - Map jaasContexts, Map subjects, KerberosShortNamer kerberosNameParser, - CredentialCache credentialCache, ListenerName listenerName, SecurityProtocol securityProtocol, - TransportLayer transportLayer, - DelegationTokenCache tokenCache) throws IOException { + TransportLayer transportLayer) throws IOException { + this.callbackHandlers = callbackHandlers; this.connectionId = connectionId; - this.jaasContexts = jaasContexts; this.subjects = subjects; - this.credentialCache = credentialCache; this.listenerName = listenerName; this.securityProtocol = securityProtocol; this.enableKafkaSaslAuthenticateHeaders = false; - this.tokenCache = tokenCache; this.transportLayer = transportLayer; this.configs = configs; @@ -154,8 +143,8 @@ public SaslServerAuthenticator(Map configs, throw new IllegalArgumentException("No SASL mechanisms are enabled"); this.enabledMechanisms = new HashSet<>(enabledMechanisms); for (String mechanism : enabledMechanisms) { - if (!jaasContexts.containsKey(mechanism)) - throw new IllegalArgumentException("Jaas context not specified for SASL mechanism " + mechanism); + if (!callbackHandlers.containsKey(mechanism)) + throw new IllegalArgumentException("Callback handler not specified for SASL mechanism " + mechanism); if (!subjects.containsKey(mechanism)) throw new IllegalArgumentException("Subject cannot be null for SASL mechanism " + mechanism); } @@ -168,11 +157,7 @@ public SaslServerAuthenticator(Map configs, private void createSaslServer(String mechanism) throws IOException { this.saslMechanism = mechanism; Subject subject = subjects.get(mechanism); - if (!ScramMechanism.isScram(mechanism)) - callbackHandler = new SaslServerCallbackHandler(jaasContexts.get(mechanism)); - else - callbackHandler = new ScramServerCallbackHandler(credentialCache.cache(mechanism, ScramCredential.class), tokenCache); - callbackHandler.configure(configs, Mode.SERVER, subject, saslMechanism); + final AuthenticateCallbackHandler callbackHandler = callbackHandlers.get(mechanism); if (mechanism.equals(SaslConfigs.GSSAPI_MECHANISM)) { saslServer = createSaslKerberosServer(callbackHandler, configs, subject); } else { @@ -189,7 +174,7 @@ public SaslServer run() throws SaslException { } } - private SaslServer createSaslKerberosServer(final AuthCallbackHandler saslServerCallbackHandler, final Map configs, Subject subject) throws IOException { + private SaslServer createSaslKerberosServer(final AuthenticateCallbackHandler saslServerCallbackHandler, final Map configs, Subject subject) throws IOException { // server is using a JAAS-authenticated subject: determine service principal name and hostname from kafka server's subject. final String servicePrincipal = SaslClientAuthenticator.firstPrincipal(subject); KerberosName kerberosName; @@ -316,8 +301,6 @@ public void close() throws IOException { Utils.closeQuietly((Closeable) principalBuilder, "principal builder"); if (saslServer != null) saslServer.dispose(); - if (callbackHandler != null) - callbackHandler.close(); } private void setSaslState(SaslState saslState) throws IOException { diff --git a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslServerCallbackHandler.java b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslServerCallbackHandler.java index 7d5372db9771d..d3d43cbfa26d2 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslServerCallbackHandler.java +++ b/clients/src/main/java/org/apache/kafka/common/security/authenticator/SaslServerCallbackHandler.java @@ -16,51 +16,46 @@ */ package org.apache.kafka.common.security.authenticator; -import java.io.IOException; +import java.util.List; import java.util.Map; -import org.apache.kafka.common.security.JaasContext; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.security.auth.Subject; import javax.security.auth.callback.Callback; import javax.security.auth.callback.UnsupportedCallbackException; +import javax.security.auth.login.AppConfigurationEntry; import javax.security.sasl.AuthorizeCallback; import javax.security.sasl.RealmCallback; -import org.apache.kafka.common.network.Mode; +import org.apache.kafka.common.config.SaslConfigs; +import org.apache.kafka.common.security.auth.AuthenticateCallbackHandler; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** - * Callback handler for Sasl servers. The callbacks required for all the SASL + * Default callback handler for Sasl servers. The callbacks required for all the SASL * mechanisms enabled in the server should be supported by this callback handler. See * Java SASL API * for the list of SASL callback handlers required for each SASL mechanism. */ -public class SaslServerCallbackHandler implements AuthCallbackHandler { +public class SaslServerCallbackHandler implements AuthenticateCallbackHandler { private static final Logger LOG = LoggerFactory.getLogger(SaslServerCallbackHandler.class); - private final JaasContext jaasContext; - public SaslServerCallbackHandler(JaasContext jaasContext) throws IOException { - this.jaasContext = jaasContext; - } + private String mechanism; @Override - public void configure(Map configs, Mode mode, Subject subject, String saslMechanism) { - } - - public JaasContext jaasContext() { - return jaasContext; + public void configure(Map configs, String mechanism, List jaasConfigEntries) { + this.mechanism = mechanism; } @Override public void handle(Callback[] callbacks) throws UnsupportedCallbackException { for (Callback callback : callbacks) { - if (callback instanceof RealmCallback) { + if (callback instanceof RealmCallback) handleRealmCallback((RealmCallback) callback); - } else if (callback instanceof AuthorizeCallback) { + else if (callback instanceof AuthorizeCallback && mechanism.equals(SaslConfigs.GSSAPI_MECHANISM)) handleAuthorizeCallback((AuthorizeCallback) callback); - } + else + throw new UnsupportedCallbackException(callback); } } diff --git a/clients/src/main/java/org/apache/kafka/common/security/kerberos/KerberosClientCallbackHandler.java b/clients/src/main/java/org/apache/kafka/common/security/kerberos/KerberosClientCallbackHandler.java new file mode 100644 index 0000000000000..fa9cad261c589 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/common/security/kerberos/KerberosClientCallbackHandler.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.common.security.kerberos; + +import org.apache.kafka.common.config.SaslConfigs; +import org.apache.kafka.common.security.auth.AuthenticateCallbackHandler; + +import javax.security.auth.callback.Callback; +import javax.security.auth.callback.NameCallback; +import javax.security.auth.callback.PasswordCallback; +import javax.security.auth.callback.UnsupportedCallbackException; +import javax.security.auth.login.AppConfigurationEntry; +import javax.security.sasl.AuthorizeCallback; +import javax.security.sasl.RealmCallback; +import java.util.List; +import java.util.Map; + +/** + * Callback handler for SASL/GSSAPI clients. + */ +public class KerberosClientCallbackHandler implements AuthenticateCallbackHandler { + + @Override + public void configure(Map configs, String saslMechanism, List jaasConfigEntries) { + if (!saslMechanism.equals(SaslConfigs.GSSAPI_MECHANISM)) + throw new IllegalStateException("Kerberos callback handler should only be used with GSSAPI"); + } + + @Override + public void handle(Callback[] callbacks) throws UnsupportedCallbackException { + for (Callback callback : callbacks) { + if (callback instanceof NameCallback) { + NameCallback nc = (NameCallback) callback; + nc.setName(nc.getDefaultName()); + } else if (callback instanceof PasswordCallback) { + String errorMessage = "Could not login: the client is being asked for a password, but the Kafka" + + " client code does not currently support obtaining a password from the user."; + errorMessage += " Make sure -Djava.security.auth.login.config property passed to JVM and" + + " the client is configured to use a ticket cache (using" + + " the JAAS configuration setting 'useTicketCache=true)'. Make sure you are using" + + " FQDN of the Kafka broker you are trying to connect to."; + throw new UnsupportedCallbackException(callback, errorMessage); + } else if (callback instanceof RealmCallback) { + RealmCallback rc = (RealmCallback) callback; + rc.setText(rc.getDefaultText()); + } else if (callback instanceof AuthorizeCallback) { + AuthorizeCallback ac = (AuthorizeCallback) callback; + String authId = ac.getAuthenticationID(); + String authzId = ac.getAuthorizationID(); + ac.setAuthorized(authId.equals(authzId)); + if (ac.isAuthorized()) + ac.setAuthorizedID(authzId); + } else { + throw new UnsupportedCallbackException(callback, "Unrecognized SASL ClientCallback"); + } + } + } + + @Override + public void close() { + } +} diff --git a/clients/src/main/java/org/apache/kafka/common/security/kerberos/KerberosLogin.java b/clients/src/main/java/org/apache/kafka/common/security/kerberos/KerberosLogin.java index 65c3b1cbe444c..ec996a8ccc6c0 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/kerberos/KerberosLogin.java +++ b/clients/src/main/java/org/apache/kafka/common/security/kerberos/KerberosLogin.java @@ -18,6 +18,7 @@ import javax.security.auth.kerberos.KerberosPrincipal; import javax.security.auth.login.AppConfigurationEntry; +import javax.security.auth.login.Configuration; import javax.security.auth.login.LoginContext; import javax.security.auth.login.LoginException; import javax.security.auth.kerberos.KerberosTicket; @@ -25,6 +26,7 @@ import org.apache.kafka.common.security.JaasContext; import org.apache.kafka.common.security.JaasUtils; +import org.apache.kafka.common.security.auth.AuthenticateCallbackHandler; import org.apache.kafka.common.security.authenticator.AbstractLogin; import org.apache.kafka.common.config.SaslConfigs; import org.apache.kafka.common.utils.KafkaThread; @@ -33,11 +35,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Arrays; import java.util.Date; import java.util.List; +import java.util.Map; import java.util.Random; import java.util.Set; -import java.util.Map; /** * This class is responsible for refreshing Kerberos credentials for @@ -78,13 +81,15 @@ public class KerberosLogin extends AbstractLogin { private String serviceName; private long lastLogin; - public void configure(Map configs, JaasContext jaasContext) { - super.configure(configs, jaasContext); + @Override + public void configure(Map configs, String contextName, Configuration configuration, + AuthenticateCallbackHandler callbackHandler) { + super.configure(configs, contextName, configuration, callbackHandler); this.ticketRenewWindowFactor = (Double) configs.get(SaslConfigs.SASL_KERBEROS_TICKET_RENEW_WINDOW_FACTOR); this.ticketRenewJitter = (Double) configs.get(SaslConfigs.SASL_KERBEROS_TICKET_RENEW_JITTER); this.minTimeBeforeRelogin = (Long) configs.get(SaslConfigs.SASL_KERBEROS_MIN_TIME_BEFORE_RELOGIN); this.kinitCmd = (String) configs.get(SaslConfigs.SASL_KERBEROS_KINIT_CMD); - this.serviceName = getServiceName(configs, jaasContext); + this.serviceName = getServiceName(configs, contextName, configuration); } /** @@ -99,13 +104,13 @@ public LoginContext login() throws LoginException { subject = loginContext.getSubject(); isKrbTicket = !subject.getPrivateCredentials(KerberosTicket.class).isEmpty(); - List entries = jaasContext().configurationEntries(); - if (entries.isEmpty()) { + AppConfigurationEntry[] entries = configuration().getAppConfigurationEntry(contextName()); + if (entries.length == 0) { isUsingTicketCache = false; principal = null; } else { // there will only be a single entry - AppConfigurationEntry entry = entries.get(0); + AppConfigurationEntry entry = entries[0]; if (entry.getOptions().get("useTicketCache") != null) { String val = (String) entry.getOptions().get("useTicketCache"); isUsingTicketCache = val.equals("true"); @@ -280,8 +285,9 @@ public String serviceName() { return serviceName; } - private static String getServiceName(Map configs, JaasContext jaasContext) { - String jaasServiceName = jaasContext.configEntryOption(JaasUtils.SERVICE_NAME, null); + private static String getServiceName(Map configs, String contextName, Configuration configuration) { + List configEntries = Arrays.asList(configuration.getAppConfigurationEntry(contextName)); + String jaasServiceName = JaasContext.configEntryOption(configEntries, JaasUtils.SERVICE_NAME, null); String configServiceName = (String) configs.get(SaslConfigs.SASL_KERBEROS_SERVICE_NAME); if (jaasServiceName != null && configServiceName != null && !jaasServiceName.equals(configServiceName)) { String message = String.format("Conflicting serviceName values found in JAAS and Kafka configs " + @@ -360,7 +366,7 @@ private void reLogin() throws LoginException { loginContext.logout(); //login and also update the subject field of this instance to //have the new credentials (pass it to the LoginContext constructor) - loginContext = new LoginContext(jaasContext().name(), subject, null, jaasContext().configuration()); + loginContext = new LoginContext(contextName(), subject, null, configuration()); log.info("Initiating re-login for {}", principal); loginContext.login(); } diff --git a/clients/src/main/java/org/apache/kafka/common/security/plain/PlainAuthenticateCallback.java b/clients/src/main/java/org/apache/kafka/common/security/plain/PlainAuthenticateCallback.java new file mode 100644 index 0000000000000..7f42645e487fb --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/common/security/plain/PlainAuthenticateCallback.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.common.security.plain; + +import javax.security.auth.callback.Callback; + +/* + * Authentication callback for SASL/PLAIN authentication. Callback handler must + * set authenticated flag to true if the client provided password in the callback + * matches the expected password. + */ +public class PlainAuthenticateCallback implements Callback { + private final char[] password; + private boolean authenticated; + + /** + * Creates a callback with the password provided by the client + * @param password The password provided by the client during SASL/PLAIN authentication + */ + public PlainAuthenticateCallback(char[] password) { + this.password = password; + } + + /** + * Returns the password provided by the client during SASL/PLAIN authentication + */ + public char[] password() { + return password; + } + + /** + * Returns true if client password matches expected password, false otherwise. + * This state is set the server-side callback handler. + */ + public boolean authenticated() { + return this.authenticated; + } + + /** + * Sets the authenticated state. This is set by the server-side callback handler + * by matching the client provided password with the expected password. + * + * @param authenticated true indicates successful authentication + */ + public void authenticated(boolean authenticated) { + this.authenticated = authenticated; + } +} diff --git a/clients/src/main/java/org/apache/kafka/common/security/plain/PlainLoginModule.java b/clients/src/main/java/org/apache/kafka/common/security/plain/PlainLoginModule.java index c8b29fc69a9cc..f0a5971ebebbf 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/plain/PlainLoginModule.java +++ b/clients/src/main/java/org/apache/kafka/common/security/plain/PlainLoginModule.java @@ -16,6 +16,8 @@ */ package org.apache.kafka.common.security.plain; +import org.apache.kafka.common.security.plain.internal.PlainSaslServerProvider; + import java.util.Map; import javax.security.auth.Subject; diff --git a/clients/src/main/java/org/apache/kafka/common/security/plain/PlainSaslServer.java b/clients/src/main/java/org/apache/kafka/common/security/plain/internal/PlainSaslServer.java similarity index 87% rename from clients/src/main/java/org/apache/kafka/common/security/plain/PlainSaslServer.java rename to clients/src/main/java/org/apache/kafka/common/security/plain/internal/PlainSaslServer.java index e54887f652ffb..811d9e94acafe 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/plain/PlainSaslServer.java +++ b/clients/src/main/java/org/apache/kafka/common/security/plain/internal/PlainSaslServer.java @@ -14,21 +14,22 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.kafka.common.security.plain; +package org.apache.kafka.common.security.plain.internal; import java.io.UnsupportedEncodingException; import java.util.Arrays; import java.util.Map; +import javax.security.auth.callback.Callback; import javax.security.auth.callback.CallbackHandler; +import javax.security.auth.callback.NameCallback; import javax.security.sasl.Sasl; import javax.security.sasl.SaslException; import javax.security.sasl.SaslServer; import javax.security.sasl.SaslServerFactory; import org.apache.kafka.common.errors.SaslAuthenticationException; -import org.apache.kafka.common.security.JaasContext; -import org.apache.kafka.common.security.authenticator.SaslServerCallbackHandler; +import org.apache.kafka.common.security.plain.PlainAuthenticateCallback; /** * Simple SaslServer implementation for SASL/PLAIN. In order to make this implementation @@ -46,15 +47,13 @@ public class PlainSaslServer implements SaslServer { public static final String PLAIN_MECHANISM = "PLAIN"; - private static final String JAAS_USER_PREFIX = "user_"; - - private final JaasContext jaasContext; + private final CallbackHandler callbackHandler; private boolean complete; private String authorizationId; - public PlainSaslServer(JaasContext jaasContext) { - this.jaasContext = jaasContext; + public PlainSaslServer(CallbackHandler callbackHandler) { + this.callbackHandler = callbackHandler; } /** @@ -101,12 +100,15 @@ public byte[] evaluateResponse(byte[] response) throws SaslException, SaslAuthen throw new SaslException("Authentication failed: password not specified"); } - String expectedPassword = jaasContext.configEntryOption(JAAS_USER_PREFIX + username, - PlainLoginModule.class.getName()); - if (!password.equals(expectedPassword)) { - throw new SaslAuthenticationException("Authentication failed: Invalid username or password"); + NameCallback nameCallback = new NameCallback("username", username); + PlainAuthenticateCallback authenticateCallback = new PlainAuthenticateCallback(password.toCharArray()); + try { + callbackHandler.handle(new Callback[]{nameCallback, authenticateCallback}); + } catch (Throwable e) { + throw new SaslAuthenticationException("Authentication failed: credentials for user could not be verified", e); } - + if (!authenticateCallback.authenticated()) + throw new SaslAuthenticationException("Authentication failed: Invalid username or password"); if (!authorizationIdFromClient.isEmpty() && !authorizationIdFromClient.equals(username)) throw new SaslAuthenticationException("Authentication failed: Client requested an authorization id that is different from username"); @@ -167,10 +169,7 @@ public SaslServer createSaslServer(String mechanism, String protocol, String ser if (!PLAIN_MECHANISM.equals(mechanism)) throw new SaslException(String.format("Mechanism \'%s\' is not supported. Only PLAIN is supported.", mechanism)); - if (!(cbh instanceof SaslServerCallbackHandler)) - throw new SaslException("CallbackHandler must be of type SaslServerCallbackHandler, but it is: " + cbh.getClass()); - - return new PlainSaslServer(((SaslServerCallbackHandler) cbh).jaasContext()); + return new PlainSaslServer(cbh); } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/security/plain/PlainSaslServerProvider.java b/clients/src/main/java/org/apache/kafka/common/security/plain/internal/PlainSaslServerProvider.java similarity index 90% rename from clients/src/main/java/org/apache/kafka/common/security/plain/PlainSaslServerProvider.java rename to clients/src/main/java/org/apache/kafka/common/security/plain/internal/PlainSaslServerProvider.java index ae1424417f8fb..c2229532449e5 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/plain/PlainSaslServerProvider.java +++ b/clients/src/main/java/org/apache/kafka/common/security/plain/internal/PlainSaslServerProvider.java @@ -14,12 +14,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.kafka.common.security.plain; +package org.apache.kafka.common.security.plain.internal; import java.security.Provider; import java.security.Security; -import org.apache.kafka.common.security.plain.PlainSaslServer.PlainSaslServerFactory; +import org.apache.kafka.common.security.plain.internal.PlainSaslServer.PlainSaslServerFactory; public class PlainSaslServerProvider extends Provider { diff --git a/clients/src/main/java/org/apache/kafka/common/security/plain/internal/PlainServerCallbackHandler.java b/clients/src/main/java/org/apache/kafka/common/security/plain/internal/PlainServerCallbackHandler.java new file mode 100644 index 0000000000000..84fbdfd4c62f5 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/common/security/plain/internal/PlainServerCallbackHandler.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.common.security.plain.internal; + +import org.apache.kafka.common.security.JaasContext; +import org.apache.kafka.common.security.auth.AuthenticateCallbackHandler; +import org.apache.kafka.common.KafkaException; +import org.apache.kafka.common.security.plain.PlainAuthenticateCallback; +import org.apache.kafka.common.security.plain.PlainLoginModule; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import javax.security.auth.callback.Callback; +import javax.security.auth.callback.NameCallback; +import javax.security.auth.callback.UnsupportedCallbackException; +import javax.security.auth.login.AppConfigurationEntry; + +public class PlainServerCallbackHandler implements AuthenticateCallbackHandler { + + private static final String JAAS_USER_PREFIX = "user_"; + private List jaasConfigEntries; + + @Override + public void configure(Map configs, String mechanism, List jaasConfigEntries) { + this.jaasConfigEntries = jaasConfigEntries; + } + + @Override + public void handle(Callback[] callbacks) throws IOException, UnsupportedCallbackException { + String username = null; + for (Callback callback: callbacks) { + if (callback instanceof NameCallback) + username = ((NameCallback) callback).getDefaultName(); + else if (callback instanceof PlainAuthenticateCallback) { + PlainAuthenticateCallback plainCallback = (PlainAuthenticateCallback) callback; + boolean authenticated = authenticate(username, plainCallback.password()); + plainCallback.authenticated(authenticated); + } else + throw new UnsupportedCallbackException(callback); + } + } + + protected boolean authenticate(String username, char[] password) throws IOException { + if (username == null) + return false; + else { + String expectedPassword = JaasContext.configEntryOption(jaasConfigEntries, + JAAS_USER_PREFIX + username, + PlainLoginModule.class.getName()); + return expectedPassword != null && Arrays.equals(password, expectedPassword.toCharArray()); + } + } + + @Override + public void close() throws KafkaException { + } + +} diff --git a/clients/src/main/java/org/apache/kafka/common/security/scram/ScramCredential.java b/clients/src/main/java/org/apache/kafka/common/security/scram/ScramCredential.java index 09ff0aacd3c84..dfbfef15b4f74 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/scram/ScramCredential.java +++ b/clients/src/main/java/org/apache/kafka/common/security/scram/ScramCredential.java @@ -16,6 +16,11 @@ */ package org.apache.kafka.common.security.scram; +/** + * SCRAM credential class that encapsulates the credential data persisted for each user that is + * accessible to the server. See RFC rfc5802 + * for details. + */ public class ScramCredential { private final byte[] salt; @@ -23,6 +28,9 @@ public class ScramCredential { private final byte[] storedKey; private final int iterations; + /** + * Constructs a new credential. + */ public ScramCredential(byte[] salt, byte[] storedKey, byte[] serverKey, int iterations) { this.salt = salt; this.serverKey = serverKey; @@ -30,18 +38,30 @@ public ScramCredential(byte[] salt, byte[] storedKey, byte[] serverKey, int iter this.iterations = iterations; } + /** + * Returns the salt used to process this credential using the SCRAM algorithm. + */ public byte[] salt() { return salt; } + /** + * Server key computed from the client password using the SCRAM algorithm. + */ public byte[] serverKey() { return serverKey; } + /** + * Stored key computed from the client password using the SCRAM algorithm. + */ public byte[] storedKey() { return storedKey; } + /** + * Number of iterations used to process this credential using the SCRAM algorithm. + */ public int iterations() { return iterations; } diff --git a/clients/src/main/java/org/apache/kafka/common/security/scram/ScramCredentialCallback.java b/clients/src/main/java/org/apache/kafka/common/security/scram/ScramCredentialCallback.java index 931210a0e0702..d5988cbeb8905 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/scram/ScramCredentialCallback.java +++ b/clients/src/main/java/org/apache/kafka/common/security/scram/ScramCredentialCallback.java @@ -18,14 +18,23 @@ import javax.security.auth.callback.Callback; +/** + * Callback used for SCRAM mechanisms. + */ public class ScramCredentialCallback implements Callback { private ScramCredential scramCredential; - public ScramCredential scramCredential() { - return scramCredential; - } - + /** + * Sets the SCRAM credential for this instance. + */ public void scramCredential(ScramCredential scramCredential) { this.scramCredential = scramCredential; } -} \ No newline at end of file + + /** + * Returns the SCRAM credential if set on this instance. + */ + public ScramCredential scramCredential() { + return scramCredential; + } +} diff --git a/clients/src/main/java/org/apache/kafka/common/security/scram/ScramExtensionsCallback.java b/clients/src/main/java/org/apache/kafka/common/security/scram/ScramExtensionsCallback.java index b40468bd650ef..debe163e36b2c 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/scram/ScramExtensionsCallback.java +++ b/clients/src/main/java/org/apache/kafka/common/security/scram/ScramExtensionsCallback.java @@ -21,13 +21,25 @@ import java.util.Collections; import java.util.Map; +/** + * Optional callback used for SCRAM mechanisms if any extensions need to be set + * in the SASL/SCRAM exchange. + */ public class ScramExtensionsCallback implements Callback { private Map extensions = Collections.emptyMap(); + /** + * Returns the extension names and values that are sent by the client to + * the server in the initial client SCRAM authentication message. + * Default is an empty map. + */ public Map extensions() { return extensions; } + /** + * Sets the SCRAM extensions on this callback. + */ public void extensions(Map extensions) { this.extensions = extensions; } diff --git a/clients/src/main/java/org/apache/kafka/common/security/scram/ScramLoginModule.java b/clients/src/main/java/org/apache/kafka/common/security/scram/ScramLoginModule.java index 43df515256fc9..20d1f221b1c05 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/scram/ScramLoginModule.java +++ b/clients/src/main/java/org/apache/kafka/common/security/scram/ScramLoginModule.java @@ -16,6 +16,9 @@ */ package org.apache.kafka.common.security.scram; +import org.apache.kafka.common.security.scram.internal.ScramSaslClientProvider; +import org.apache.kafka.common.security.scram.internal.ScramSaslServerProvider; + import java.util.Collections; import java.util.Map; diff --git a/clients/src/main/java/org/apache/kafka/common/security/scram/ScramCredentialUtils.java b/clients/src/main/java/org/apache/kafka/common/security/scram/internal/ScramCredentialUtils.java similarity index 96% rename from clients/src/main/java/org/apache/kafka/common/security/scram/ScramCredentialUtils.java rename to clients/src/main/java/org/apache/kafka/common/security/scram/internal/ScramCredentialUtils.java index b4875d6f57ee4..91e28a62e1d34 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/scram/ScramCredentialUtils.java +++ b/clients/src/main/java/org/apache/kafka/common/security/scram/internal/ScramCredentialUtils.java @@ -14,12 +14,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.kafka.common.security.scram; +package org.apache.kafka.common.security.scram.internal; import java.util.Collection; import java.util.Properties; import org.apache.kafka.common.security.authenticator.CredentialCache; +import org.apache.kafka.common.security.scram.ScramCredential; import org.apache.kafka.common.utils.Base64; /** diff --git a/clients/src/main/java/org/apache/kafka/common/security/scram/ScramExtensions.java b/clients/src/main/java/org/apache/kafka/common/security/scram/internal/ScramExtensions.java similarity index 95% rename from clients/src/main/java/org/apache/kafka/common/security/scram/ScramExtensions.java rename to clients/src/main/java/org/apache/kafka/common/security/scram/internal/ScramExtensions.java index 0f461c0e3c2df..66d9362a3e6cc 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/scram/ScramExtensions.java +++ b/clients/src/main/java/org/apache/kafka/common/security/scram/internal/ScramExtensions.java @@ -14,7 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.kafka.common.security.scram; +package org.apache.kafka.common.security.scram.internal; + +import org.apache.kafka.common.security.scram.ScramLoginModule; import java.util.Collections; import java.util.HashMap; diff --git a/clients/src/main/java/org/apache/kafka/common/security/scram/ScramFormatter.java b/clients/src/main/java/org/apache/kafka/common/security/scram/internal/ScramFormatter.java similarity index 94% rename from clients/src/main/java/org/apache/kafka/common/security/scram/ScramFormatter.java rename to clients/src/main/java/org/apache/kafka/common/security/scram/internal/ScramFormatter.java index 406c28599b3bb..6fcb7a152dbaa 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/scram/ScramFormatter.java +++ b/clients/src/main/java/org/apache/kafka/common/security/scram/internal/ScramFormatter.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.kafka.common.security.scram; +package org.apache.kafka.common.security.scram.internal; import java.math.BigInteger; import java.nio.charset.StandardCharsets; @@ -27,9 +27,10 @@ import javax.crypto.spec.SecretKeySpec; import org.apache.kafka.common.KafkaException; -import org.apache.kafka.common.security.scram.ScramMessages.ClientFinalMessage; -import org.apache.kafka.common.security.scram.ScramMessages.ClientFirstMessage; -import org.apache.kafka.common.security.scram.ScramMessages.ServerFirstMessage; +import org.apache.kafka.common.security.scram.ScramCredential; +import org.apache.kafka.common.security.scram.internal.ScramMessages.ClientFinalMessage; +import org.apache.kafka.common.security.scram.internal.ScramMessages.ClientFirstMessage; +import org.apache.kafka.common.security.scram.internal.ScramMessages.ServerFirstMessage; /** * Scram message salt and hash functions defined in RFC 5802. diff --git a/clients/src/main/java/org/apache/kafka/common/security/scram/ScramMechanism.java b/clients/src/main/java/org/apache/kafka/common/security/scram/internal/ScramMechanism.java similarity index 97% rename from clients/src/main/java/org/apache/kafka/common/security/scram/ScramMechanism.java rename to clients/src/main/java/org/apache/kafka/common/security/scram/internal/ScramMechanism.java index d8c0c6d3d603e..73be4cf24fb81 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/scram/ScramMechanism.java +++ b/clients/src/main/java/org/apache/kafka/common/security/scram/internal/ScramMechanism.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.kafka.common.security.scram; +package org.apache.kafka.common.security.scram.internal; import java.util.Collection; import java.util.Collections; diff --git a/clients/src/main/java/org/apache/kafka/common/security/scram/ScramMessages.java b/clients/src/main/java/org/apache/kafka/common/security/scram/internal/ScramMessages.java similarity index 99% rename from clients/src/main/java/org/apache/kafka/common/security/scram/ScramMessages.java rename to clients/src/main/java/org/apache/kafka/common/security/scram/internal/ScramMessages.java index 05b3d7755404e..439b274f561d2 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/scram/ScramMessages.java +++ b/clients/src/main/java/org/apache/kafka/common/security/scram/internal/ScramMessages.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.kafka.common.security.scram; +package org.apache.kafka.common.security.scram.internal; import org.apache.kafka.common.utils.Base64; diff --git a/clients/src/main/java/org/apache/kafka/common/security/scram/ScramSaslClient.java b/clients/src/main/java/org/apache/kafka/common/security/scram/internal/ScramSaslClient.java similarity index 90% rename from clients/src/main/java/org/apache/kafka/common/security/scram/ScramSaslClient.java rename to clients/src/main/java/org/apache/kafka/common/security/scram/internal/ScramSaslClient.java index 71109df2ddd1f..a98a86d61cb1a 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/scram/ScramSaslClient.java +++ b/clients/src/main/java/org/apache/kafka/common/security/scram/internal/ScramSaslClient.java @@ -14,9 +14,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.kafka.common.security.scram; +package org.apache.kafka.common.security.scram.internal; -import java.io.IOException; import java.nio.charset.StandardCharsets; import java.security.InvalidKeyException; import java.security.NoSuchAlgorithmException; @@ -34,9 +33,10 @@ import javax.security.sasl.SaslException; import org.apache.kafka.common.errors.IllegalSaslStateException; -import org.apache.kafka.common.security.scram.ScramMessages.ClientFinalMessage; -import org.apache.kafka.common.security.scram.ScramMessages.ServerFinalMessage; -import org.apache.kafka.common.security.scram.ScramMessages.ServerFirstMessage; +import org.apache.kafka.common.security.scram.ScramExtensionsCallback; +import org.apache.kafka.common.security.scram.internal.ScramMessages.ClientFinalMessage; +import org.apache.kafka.common.security.scram.internal.ScramMessages.ServerFinalMessage; +import org.apache.kafka.common.security.scram.internal.ScramMessages.ServerFirstMessage; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -100,9 +100,15 @@ public byte[] evaluateChallenge(byte[] challenge) throws SaslException { ScramExtensionsCallback extensionsCallback = new ScramExtensionsCallback(); try { - callbackHandler.handle(new Callback[]{nameCallback, extensionsCallback}); - } catch (IOException | UnsupportedCallbackException e) { - throw new SaslException("User name could not be obtained", e); + callbackHandler.handle(new Callback[]{nameCallback}); + try { + callbackHandler.handle(new Callback[]{extensionsCallback}); + } catch (UnsupportedCallbackException e) { + log.debug("Extensions callback is not supported by client callback handler {}, no extensions will be added", + callbackHandler); + } + } catch (Throwable e) { + throw new SaslException("User name or extensions could not be obtained", e); } String username = nameCallback.getName(); @@ -121,7 +127,7 @@ public byte[] evaluateChallenge(byte[] challenge) throws SaslException { PasswordCallback passwordCallback = new PasswordCallback("Password:", false); try { callbackHandler.handle(new Callback[]{passwordCallback}); - } catch (IOException | UnsupportedCallbackException e) { + } catch (Throwable e) { throw new SaslException("User name could not be obtained", e); } this.clientFinalMessage = handleServerFirstMessage(passwordCallback.getPassword()); diff --git a/clients/src/main/java/org/apache/kafka/common/security/scram/ScramSaslClientProvider.java b/clients/src/main/java/org/apache/kafka/common/security/scram/internal/ScramSaslClientProvider.java similarity index 90% rename from clients/src/main/java/org/apache/kafka/common/security/scram/ScramSaslClientProvider.java rename to clients/src/main/java/org/apache/kafka/common/security/scram/internal/ScramSaslClientProvider.java index d389f044a9967..4d9ff81309b79 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/scram/ScramSaslClientProvider.java +++ b/clients/src/main/java/org/apache/kafka/common/security/scram/internal/ScramSaslClientProvider.java @@ -14,12 +14,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.kafka.common.security.scram; +package org.apache.kafka.common.security.scram.internal; import java.security.Provider; import java.security.Security; -import org.apache.kafka.common.security.scram.ScramSaslClient.ScramSaslClientFactory; +import org.apache.kafka.common.security.scram.internal.ScramSaslClient.ScramSaslClientFactory; public class ScramSaslClientProvider extends Provider { diff --git a/clients/src/main/java/org/apache/kafka/common/security/scram/ScramSaslServer.java b/clients/src/main/java/org/apache/kafka/common/security/scram/internal/ScramSaslServer.java similarity index 92% rename from clients/src/main/java/org/apache/kafka/common/security/scram/ScramSaslServer.java rename to clients/src/main/java/org/apache/kafka/common/security/scram/internal/ScramSaslServer.java index 314c1d413ba52..60857279f59ad 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/scram/ScramSaslServer.java +++ b/clients/src/main/java/org/apache/kafka/common/security/scram/internal/ScramSaslServer.java @@ -14,9 +14,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.kafka.common.security.scram; +package org.apache.kafka.common.security.scram.internal; -import java.io.IOException; import java.security.InvalidKeyException; import java.security.NoSuchAlgorithmException; import java.util.Arrays; @@ -27,18 +26,21 @@ import javax.security.auth.callback.Callback; import javax.security.auth.callback.CallbackHandler; import javax.security.auth.callback.NameCallback; -import javax.security.auth.callback.UnsupportedCallbackException; import javax.security.sasl.SaslException; import javax.security.sasl.SaslServer; import javax.security.sasl.SaslServerFactory; +import org.apache.kafka.common.errors.AuthenticationException; import org.apache.kafka.common.errors.IllegalSaslStateException; import org.apache.kafka.common.errors.SaslAuthenticationException; -import org.apache.kafka.common.security.scram.ScramMessages.ClientFinalMessage; -import org.apache.kafka.common.security.scram.ScramMessages.ClientFirstMessage; -import org.apache.kafka.common.security.scram.ScramMessages.ServerFinalMessage; -import org.apache.kafka.common.security.scram.ScramMessages.ServerFirstMessage; -import org.apache.kafka.common.security.token.delegation.DelegationTokenCredentialCallback; +import org.apache.kafka.common.security.scram.ScramCredential; +import org.apache.kafka.common.security.scram.ScramCredentialCallback; +import org.apache.kafka.common.security.scram.ScramLoginModule; +import org.apache.kafka.common.security.scram.internal.ScramMessages.ClientFinalMessage; +import org.apache.kafka.common.security.scram.internal.ScramMessages.ClientFirstMessage; +import org.apache.kafka.common.security.scram.internal.ScramMessages.ServerFinalMessage; +import org.apache.kafka.common.security.scram.internal.ScramMessages.ServerFirstMessage; +import org.apache.kafka.common.security.token.delegation.internal.DelegationTokenCredentialCallback; import org.apache.kafka.common.utils.Utils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -133,7 +135,9 @@ public byte[] evaluateResponse(byte[] response) throws SaslException, SaslAuthen scramCredential.iterations()); setState(State.RECEIVE_CLIENT_FINAL_MESSAGE); return serverFirstMessage.toBytes(); - } catch (IOException | NumberFormatException | UnsupportedCallbackException e) { + } catch (SaslException | AuthenticationException e) { + throw e; + } catch (Throwable e) { throw new SaslException("Authentication failed: Credentials could not be obtained", e); } @@ -154,7 +158,7 @@ public byte[] evaluateResponse(byte[] response) throws SaslException, SaslAuthen default: throw new IllegalSaslStateException("Unexpected challenge in Sasl server state " + state); } - } catch (SaslException e) { + } catch (SaslException | AuthenticationException e) { clearCredentials(); setState(State.FAILED); throw e; diff --git a/clients/src/main/java/org/apache/kafka/common/security/scram/ScramSaslServerProvider.java b/clients/src/main/java/org/apache/kafka/common/security/scram/internal/ScramSaslServerProvider.java similarity index 90% rename from clients/src/main/java/org/apache/kafka/common/security/scram/ScramSaslServerProvider.java rename to clients/src/main/java/org/apache/kafka/common/security/scram/internal/ScramSaslServerProvider.java index 9f2a6b3d25621..099e50e19ddab 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/scram/ScramSaslServerProvider.java +++ b/clients/src/main/java/org/apache/kafka/common/security/scram/internal/ScramSaslServerProvider.java @@ -14,12 +14,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.kafka.common.security.scram; +package org.apache.kafka.common.security.scram.internal; import java.security.Provider; import java.security.Security; -import org.apache.kafka.common.security.scram.ScramSaslServer.ScramSaslServerFactory; +import org.apache.kafka.common.security.scram.internal.ScramSaslServer.ScramSaslServerFactory; public class ScramSaslServerProvider extends Provider { diff --git a/clients/src/main/java/org/apache/kafka/common/security/scram/ScramServerCallbackHandler.java b/clients/src/main/java/org/apache/kafka/common/security/scram/internal/ScramServerCallbackHandler.java similarity index 76% rename from clients/src/main/java/org/apache/kafka/common/security/scram/ScramServerCallbackHandler.java rename to clients/src/main/java/org/apache/kafka/common/security/scram/internal/ScramServerCallbackHandler.java index 5e37eae9d4fb6..9a3f0dc66b764 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/scram/ScramServerCallbackHandler.java +++ b/clients/src/main/java/org/apache/kafka/common/security/scram/internal/ScramServerCallbackHandler.java @@ -14,23 +14,25 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.kafka.common.security.scram; +package org.apache.kafka.common.security.scram.internal; import java.io.IOException; +import java.util.List; import java.util.Map; -import javax.security.auth.Subject; import javax.security.auth.callback.Callback; import javax.security.auth.callback.NameCallback; import javax.security.auth.callback.UnsupportedCallbackException; +import javax.security.auth.login.AppConfigurationEntry; -import org.apache.kafka.common.network.Mode; -import org.apache.kafka.common.security.authenticator.AuthCallbackHandler; +import org.apache.kafka.common.security.auth.AuthenticateCallbackHandler; import org.apache.kafka.common.security.authenticator.CredentialCache; -import org.apache.kafka.common.security.token.delegation.DelegationTokenCache; -import org.apache.kafka.common.security.token.delegation.DelegationTokenCredentialCallback; +import org.apache.kafka.common.security.scram.ScramCredential; +import org.apache.kafka.common.security.scram.ScramCredentialCallback; +import org.apache.kafka.common.security.token.delegation.internal.DelegationTokenCache; +import org.apache.kafka.common.security.token.delegation.internal.DelegationTokenCredentialCallback; -public class ScramServerCallbackHandler implements AuthCallbackHandler { +public class ScramServerCallbackHandler implements AuthenticateCallbackHandler { private final CredentialCache.Cache credentialCache; private final DelegationTokenCache tokenCache; @@ -42,6 +44,11 @@ public ScramServerCallbackHandler(CredentialCache.Cache credent this.tokenCache = tokenCache; } + @Override + public void configure(Map configs, String mechanism, List jaasConfigEntries) { + this.saslMechanism = mechanism; + } + @Override public void handle(Callback[] callbacks) throws IOException, UnsupportedCallbackException { String username = null; @@ -60,11 +67,6 @@ else if (callback instanceof DelegationTokenCredentialCallback) { } } - @Override - public void configure(Map configs, Mode mode, Subject subject, String saslMechanism) { - this.saslMechanism = saslMechanism; - } - @Override public void close() { } diff --git a/clients/src/main/java/org/apache/kafka/common/security/token/delegation/DelegationToken.java b/clients/src/main/java/org/apache/kafka/common/security/token/delegation/DelegationToken.java index 05ccbda2fe6ee..e1f97c1b72d77 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/token/delegation/DelegationToken.java +++ b/clients/src/main/java/org/apache/kafka/common/security/token/delegation/DelegationToken.java @@ -16,11 +16,16 @@ */ package org.apache.kafka.common.security.token.delegation; +import org.apache.kafka.common.annotation.InterfaceStability; import org.apache.kafka.common.utils.Base64; -import java.nio.ByteBuffer; import java.util.Arrays; +/** + * A class representing a delegation token. + * + */ +@InterfaceStability.Evolving public class DelegationToken { private TokenInformation tokenInformation; private byte[] hmac; @@ -42,10 +47,6 @@ public String hmacAsBase64String() { return Base64.encoder().encodeToString(hmac); } - public ByteBuffer hmacBuffer() { - return ByteBuffer.wrap(hmac); - } - @Override public boolean equals(Object o) { if (this == o) { diff --git a/clients/src/main/java/org/apache/kafka/common/security/token/delegation/TokenInformation.java b/clients/src/main/java/org/apache/kafka/common/security/token/delegation/TokenInformation.java index 1d500d21eeff6..ffd2af3f1c231 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/token/delegation/TokenInformation.java +++ b/clients/src/main/java/org/apache/kafka/common/security/token/delegation/TokenInformation.java @@ -16,11 +16,17 @@ */ package org.apache.kafka.common.security.token.delegation; +import org.apache.kafka.common.annotation.InterfaceStability; import org.apache.kafka.common.security.auth.KafkaPrincipal; import java.util.ArrayList; import java.util.Collection; +/** + * A class representing a delegation token details. + * + */ +@InterfaceStability.Evolving public class TokenInformation { private KafkaPrincipal owner; diff --git a/clients/src/main/java/org/apache/kafka/common/security/token/delegation/DelegationTokenCache.java b/clients/src/main/java/org/apache/kafka/common/security/token/delegation/internal/DelegationTokenCache.java similarity index 91% rename from clients/src/main/java/org/apache/kafka/common/security/token/delegation/DelegationTokenCache.java rename to clients/src/main/java/org/apache/kafka/common/security/token/delegation/internal/DelegationTokenCache.java index 78575b81bb6e3..c05b7350b4174 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/token/delegation/DelegationTokenCache.java +++ b/clients/src/main/java/org/apache/kafka/common/security/token/delegation/internal/DelegationTokenCache.java @@ -15,12 +15,14 @@ * limitations under the License. */ -package org.apache.kafka.common.security.token.delegation; +package org.apache.kafka.common.security.token.delegation.internal; import org.apache.kafka.common.security.authenticator.CredentialCache; import org.apache.kafka.common.security.scram.ScramCredential; -import org.apache.kafka.common.security.scram.ScramCredentialUtils; -import org.apache.kafka.common.security.scram.ScramMechanism; +import org.apache.kafka.common.security.scram.internal.ScramCredentialUtils; +import org.apache.kafka.common.security.scram.internal.ScramMechanism; +import org.apache.kafka.common.security.token.delegation.DelegationToken; +import org.apache.kafka.common.security.token.delegation.TokenInformation; import java.util.Collection; import java.util.HashMap; diff --git a/clients/src/main/java/org/apache/kafka/common/security/token/delegation/DelegationTokenCredentialCallback.java b/clients/src/main/java/org/apache/kafka/common/security/token/delegation/internal/DelegationTokenCredentialCallback.java similarity index 94% rename from clients/src/main/java/org/apache/kafka/common/security/token/delegation/DelegationTokenCredentialCallback.java rename to clients/src/main/java/org/apache/kafka/common/security/token/delegation/internal/DelegationTokenCredentialCallback.java index 7490a3e91b1ca..294d7b1d3a015 100644 --- a/clients/src/main/java/org/apache/kafka/common/security/token/delegation/DelegationTokenCredentialCallback.java +++ b/clients/src/main/java/org/apache/kafka/common/security/token/delegation/internal/DelegationTokenCredentialCallback.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.kafka.common.security.token.delegation; +package org.apache.kafka.common.security.token.delegation.internal; import org.apache.kafka.common.security.scram.ScramCredentialCallback; diff --git a/clients/src/main/java/org/apache/kafka/server/quota/ClientQuotaCallback.java b/clients/src/main/java/org/apache/kafka/server/quota/ClientQuotaCallback.java new file mode 100644 index 0000000000000..210e9f45840e2 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/server/quota/ClientQuotaCallback.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.server.quota; + +import org.apache.kafka.common.Cluster; +import org.apache.kafka.common.Configurable; +import org.apache.kafka.common.security.auth.KafkaPrincipal; + +import java.util.Map; + +/** + * Quota callback interface for brokers that enables customization of client quota computation. + */ +public interface ClientQuotaCallback extends Configurable { + + /** + * Quota callback invoked to determine the quota metric tags to be applied for a request. + * Quota limits are associated with quota metrics and all clients which use the same + * metric tags share the quota limit. + * + * @param quotaType Type of quota requested + * @param principal The user principal of the connection for which quota is requested + * @param clientId The client id associated with the request + * @return quota metric tags that indicate which other clients share this quota + */ + Map quotaMetricTags(ClientQuotaType quotaType, KafkaPrincipal principal, String clientId); + + /** + * Returns the quota limit associated with the provided metric tags. These tags were returned from + * a previous call to {@link #quotaMetricTags(ClientQuotaType, KafkaPrincipal, String)}. This method is + * invoked by quota managers to obtain the current quota limit applied to a metric when the first request + * using these tags is processed. It is also invoked after a quota update or cluster metadata change. + * If the tags are no longer in use after the update, (e.g. this is a {user, client-id} quota metric + * and the quota now in use is a {user} quota), null is returned. + * + * @param quotaType Type of quota requested + * @param metricTags Metric tags for a quota metric of type `quotaType` + * @return the quota limit for the provided metric tags or null if the metric tags are no longer in use + */ + Double quotaLimit(ClientQuotaType quotaType, Map metricTags); + + /** + * Quota configuration update callback that is invoked when quota configuration for an entity is + * updated in ZooKeeper. This is useful to track configured quotas if built-in quota configuration + * tools are used for quota management. + * + * @param quotaType Type of quota being updated + * @param quotaEntity The quota entity for which quota is being updated + * @param newValue The new quota value + */ + void updateQuota(ClientQuotaType quotaType, ClientQuotaEntity quotaEntity, double newValue); + + /** + * Quota configuration removal callback that is invoked when quota configuration for an entity is + * removed in ZooKeeper. This is useful to track configured quotas if built-in quota configuration + * tools are used for quota management. + * + * @param quotaType Type of quota being updated + * @param quotaEntity The quota entity for which quota is being updated + */ + void removeQuota(ClientQuotaType quotaType, ClientQuotaEntity quotaEntity); + + /** + * Returns true if any of the existing quota configs may have been updated since the last call + * to this method for the provided quota type. Quota updates as a result of calls to + * {@link #updateClusterMetadata(Cluster)}, {@link #updateQuota(ClientQuotaType, ClientQuotaEntity, double)} + * and {@link #removeQuota(ClientQuotaType, ClientQuotaEntity)} are automatically processed. + * So callbacks that rely only on built-in quota configuration tools always return false. Quota callbacks + * with external quota configuration or custom reconfigurable quota configs that affect quota limits must + * return true if existing metric configs may need to be updated. This method is invoked on every request + * and hence is expected to be handled by callbacks as a simple flag that is updated when quotas change. + * + * @param quotaType Type of quota + */ + boolean quotaResetRequired(ClientQuotaType quotaType); + + /** + * Metadata update callback that is invoked whenever UpdateMetadata request is received from + * the controller. This is useful if quota computation takes partitions into account. + * Topics that are being deleted will not be included in `cluster`. + * + * @param cluster Cluster metadata including partitions and their leaders if known + * @return true if quotas have changed and metric configs may need to be updated + */ + boolean updateClusterMetadata(Cluster cluster); + + /** + * Closes this instance. + */ + void close(); +} + diff --git a/clients/src/main/java/org/apache/kafka/server/quota/ClientQuotaEntity.java b/clients/src/main/java/org/apache/kafka/server/quota/ClientQuotaEntity.java new file mode 100644 index 0000000000000..a5ff082dfef7e --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/server/quota/ClientQuotaEntity.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.server.quota; + +import java.util.List; + +/** + * The metadata for an entity for which quota is configured. Quotas may be defined at + * different levels and `configEntities` gives the list of config entities that define + * the level of this quota entity. + */ +public interface ClientQuotaEntity { + + /** + * Entity type of a {@link ConfigEntity} + */ + public enum ConfigEntityType { + USER, + CLIENT_ID, + DEFAULT_USER, + DEFAULT_CLIENT_ID + } + + /** + * Interface representing a quota configuration entity. Quota may be + * configured at levels that include one or more configuration entities. + * For example, {user, client-id} quota is represented using two + * instances of ConfigEntity with entity types USER and CLIENT_ID. + */ + public interface ConfigEntity { + /** + * Returns the name of this entity. For default quotas, an empty string is returned. + */ + String name(); + + /** + * Returns the type of this entity. + */ + ConfigEntityType entityType(); + } + + /** + * Returns the list of configuration entities that this quota entity is comprised of. + * For {user} or {clientId} quota, this is a single entity and for {user, clientId} + * quota, this is a list of two entities. + */ + List configEntities(); +} diff --git a/clients/src/main/java/org/apache/kafka/server/quota/ClientQuotaType.java b/clients/src/main/java/org/apache/kafka/server/quota/ClientQuotaType.java new file mode 100644 index 0000000000000..4dd67d3125d37 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/server/quota/ClientQuotaType.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.server.quota; + +/** + * Types of quotas that may be configured on brokers for client requests. + */ +public enum ClientQuotaType { + PRODUCE, + FETCH, + REQUEST +} diff --git a/clients/src/test/java/org/apache/kafka/clients/MockClient.java b/clients/src/test/java/org/apache/kafka/clients/MockClient.java index 60af9bcc84937..a73175c995430 100644 --- a/clients/src/test/java/org/apache/kafka/clients/MockClient.java +++ b/clients/src/test/java/org/apache/kafka/clients/MockClient.java @@ -16,6 +16,7 @@ */ package org.apache.kafka.clients; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.kafka.common.Cluster; import org.apache.kafka.common.Node; import org.apache.kafka.common.errors.AuthenticationException; @@ -88,7 +89,7 @@ public FutureResponse(Node node, private final Queue metadataUpdates = new ArrayDeque<>(); private volatile NodeApiVersions nodeApiVersions = NodeApiVersions.create(); private volatile int numBlockingWakeups = 0; - + private final AtomicInteger totalRequestCount = new AtomicInteger(0); public MockClient(Time time) { this(time, null); } @@ -394,6 +395,7 @@ public void reset() { futureResponses.clear(); metadataUpdates.clear(); authenticationErrors.clear(); + totalRequestCount.set(0); } public boolean hasPendingMetadataUpdates() { @@ -461,6 +463,7 @@ public ClientRequest newClientRequest(String nodeId, AbstractRequest.Builder @Override public ClientRequest newClientRequest(String nodeId, AbstractRequest.Builder requestBuilder, long createdTimeMs, boolean expectResponse, RequestCompletionHandler callback) { + totalRequestCount.incrementAndGet(); return new ClientRequest(nodeId, requestBuilder, 0, "mockClientId", createdTimeMs, expectResponse, callback); } @@ -503,4 +506,9 @@ private static class MetadataUpdate { this.expectMatchRefreshTopics = expectMatchRefreshTopics; } } + + // visible for testing + public int totalRequestCount() { + return totalRequestCount.get(); + } } diff --git a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java index f08a99b6ddc4b..a242ed660609b 100644 --- a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java @@ -18,6 +18,9 @@ import org.apache.kafka.clients.NodeApiVersions; import org.apache.kafka.clients.admin.DeleteAclsResult.FilterResults; +import org.apache.kafka.clients.consumer.OffsetAndMetadata; +import org.apache.kafka.clients.consumer.internals.ConsumerProtocol; +import org.apache.kafka.clients.consumer.internals.PartitionAssignor; import org.apache.kafka.common.Cluster; import org.apache.kafka.common.KafkaFuture; import org.apache.kafka.common.Node; @@ -47,10 +50,15 @@ import org.apache.kafka.common.requests.DeleteAclsResponse; import org.apache.kafka.common.requests.DeleteAclsResponse.AclDeletionResult; import org.apache.kafka.common.requests.DeleteAclsResponse.AclFilterResponse; +import org.apache.kafka.common.requests.DeleteGroupsResponse; import org.apache.kafka.common.requests.DeleteRecordsResponse; import org.apache.kafka.common.requests.DescribeAclsResponse; import org.apache.kafka.common.requests.DescribeConfigsResponse; +import org.apache.kafka.common.requests.DescribeGroupsResponse; +import org.apache.kafka.common.requests.FindCoordinatorResponse; +import org.apache.kafka.common.requests.ListGroupsResponse; import org.apache.kafka.common.requests.MetadataResponse; +import org.apache.kafka.common.requests.OffsetFetchResponse; import org.apache.kafka.common.resource.Resource; import org.apache.kafka.common.resource.ResourceFilter; import org.apache.kafka.common.resource.ResourceType; @@ -64,6 +72,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -81,6 +90,7 @@ import static org.apache.kafka.common.requests.ResourceType.TOPIC; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -230,22 +240,26 @@ public void testInvalidTopicNames() throws Exception { for (String sillyTopicName : sillyTopicNames) { assertFutureError(deleteFutures.get(sillyTopicName), InvalidTopicException.class); } + assertEquals(0, env.kafkaClient().totalRequestCount()); Map> describeFutures = env.adminClient().describeTopics(sillyTopicNames).values(); for (String sillyTopicName : sillyTopicNames) { assertFutureError(describeFutures.get(sillyTopicName), InvalidTopicException.class); } + assertEquals(0, env.kafkaClient().totalRequestCount()); List newTopics = new ArrayList<>(); for (String sillyTopicName : sillyTopicNames) { newTopics.add(new NewTopic(sillyTopicName, 1, (short) 1)); } + Map> createFutures = env.adminClient().createTopics(newTopics).values(); for (String sillyTopicName : sillyTopicNames) { assertFutureError(createFutures .get(sillyTopicName), InvalidTopicException.class); } + assertEquals(0, env.kafkaClient().totalRequestCount()); } } @@ -632,6 +646,188 @@ public void testDeleteRecords() throws Exception { } } + //Ignoring test to be fixed on follow-up PR + @Ignore + @Test + public void testListConsumerGroups() throws Exception { + final HashMap nodes = new HashMap<>(); + nodes.put(0, new Node(0, "localhost", 8121)); + + final Cluster cluster = + new Cluster( + "mockClusterId", + nodes.values(), + Collections.emptyList(), + Collections.emptySet(), + Collections.emptySet(), nodes.get(0)); + + try (AdminClientUnitTestEnv env = new AdminClientUnitTestEnv(cluster)) { + env.kafkaClient().setNodeApiVersions(NodeApiVersions.create()); + env.kafkaClient().prepareMetadataUpdate(env.cluster(), Collections.emptySet()); + env.kafkaClient().setNode(env.cluster().controller()); + + env.kafkaClient().prepareResponse( + new MetadataResponse( + env.cluster().nodes(), + env.cluster().clusterResource().clusterId(), + env.cluster().controller().id(), + new ArrayList())); + + env.kafkaClient().prepareResponse( + new ListGroupsResponse( + Errors.NONE, + Arrays.asList( + new ListGroupsResponse.Group("group-1", ConsumerProtocol.PROTOCOL_TYPE), + new ListGroupsResponse.Group("group-connect-1", "connector") + ))); + + final ListConsumerGroupsResult result = env.adminClient().listConsumerGroups(); + final List consumerGroups = new ArrayList<>(); + + final KafkaFuture> listings = result.listings(); + consumerGroups.addAll(listings.get()); + assertEquals(1, consumerGroups.size()); + } + } + + @Test + public void testDescribeConsumerGroups() throws Exception { + final HashMap nodes = new HashMap<>(); + nodes.put(0, new Node(0, "localhost", 8121)); + + final Cluster cluster = + new Cluster( + "mockClusterId", + nodes.values(), + Collections.emptyList(), + Collections.emptySet(), + Collections.emptySet(), nodes.get(0)); + + try (AdminClientUnitTestEnv env = new AdminClientUnitTestEnv(cluster)) { + env.kafkaClient().setNodeApiVersions(NodeApiVersions.create()); + env.kafkaClient().prepareMetadataUpdate(env.cluster(), Collections.emptySet()); + env.kafkaClient().setNode(env.cluster().controller()); + + env.kafkaClient().prepareResponse(new FindCoordinatorResponse(Errors.NONE, env.cluster().controller())); + + final Map groupMetadataMap = new HashMap<>(); + TopicPartition myTopicPartition0 = new TopicPartition("my_topic", 0); + TopicPartition myTopicPartition1 = new TopicPartition("my_topic", 1); + TopicPartition myTopicPartition2 = new TopicPartition("my_topic", 2); + + final List topicPartitions = new ArrayList<>(); + topicPartitions.add(0, myTopicPartition0); + topicPartitions.add(1, myTopicPartition1); + topicPartitions.add(2, myTopicPartition2); + + final ByteBuffer memberAssignment = ConsumerProtocol.serializeAssignment(new PartitionAssignor.Assignment(topicPartitions)); + + groupMetadataMap.put( + "group-0", + new DescribeGroupsResponse.GroupMetadata( + Errors.NONE, + "", + ConsumerProtocol.PROTOCOL_TYPE, + "", + Arrays.asList( + new DescribeGroupsResponse.GroupMember("0", "clientId0", "clientHost", null, memberAssignment), + new DescribeGroupsResponse.GroupMember("1", "clientId1", "clientHost", null, memberAssignment)))); + groupMetadataMap.put( + "group-connect-0", + new DescribeGroupsResponse.GroupMetadata( + Errors.NONE, + "", + "connect", + "", + Arrays.asList( + new DescribeGroupsResponse.GroupMember("0", "clientId0", "clientHost", null, memberAssignment), + new DescribeGroupsResponse.GroupMember("1", "clientId1", "clientHost", null, memberAssignment)))); + + env.kafkaClient().prepareResponse(new DescribeGroupsResponse(groupMetadataMap)); + + final DescribeConsumerGroupsResult result = env.adminClient().describeConsumerGroups(Collections.singletonList("group-0")); + final KafkaFuture groupDescriptionFuture = result.describedGroups().get().get("group-0"); + final ConsumerGroupDescription groupDescription = groupDescriptionFuture.get(); + + assertEquals(1, result.describedGroups().get().size()); + assertEquals("group-0", groupDescription.groupId()); + assertEquals(2, groupDescription.members().size()); + } + } + + @Test + public void testDescribeConsumerGroupOffsets() throws Exception { + final HashMap nodes = new HashMap<>(); + nodes.put(0, new Node(0, "localhost", 8121)); + + final Cluster cluster = + new Cluster( + "mockClusterId", + nodes.values(), + Collections.emptyList(), + Collections.emptySet(), + Collections.emptySet(), nodes.get(0)); + + try (AdminClientUnitTestEnv env = new AdminClientUnitTestEnv(cluster)) { + env.kafkaClient().setNodeApiVersions(NodeApiVersions.create()); + env.kafkaClient().prepareMetadataUpdate(env.cluster(), Collections.emptySet()); + env.kafkaClient().setNode(env.cluster().controller()); + + env.kafkaClient().prepareResponse(new FindCoordinatorResponse(Errors.NONE, env.cluster().controller())); + + TopicPartition myTopicPartition0 = new TopicPartition("my_topic", 0); + TopicPartition myTopicPartition1 = new TopicPartition("my_topic", 1); + TopicPartition myTopicPartition2 = new TopicPartition("my_topic", 2); + + final Map responseData = new HashMap<>(); + responseData.put(myTopicPartition0, new OffsetFetchResponse.PartitionData(10, "", Errors.NONE)); + responseData.put(myTopicPartition1, new OffsetFetchResponse.PartitionData(0, "", Errors.NONE)); + responseData.put(myTopicPartition2, new OffsetFetchResponse.PartitionData(20, "", Errors.NONE)); + env.kafkaClient().prepareResponse(new OffsetFetchResponse(Errors.NONE, responseData)); + + final ListConsumerGroupOffsetsResult result = env.adminClient().listConsumerGroupOffsets("group-0"); + + assertEquals(3, result.partitionsToOffsetAndMetadata().get().size()); + final TopicPartition topicPartition = result.partitionsToOffsetAndMetadata().get().keySet().iterator().next(); + assertEquals("my_topic", topicPartition.topic()); + final OffsetAndMetadata offsetAndMetadata = result.partitionsToOffsetAndMetadata().get().values().iterator().next(); + assertEquals(10, offsetAndMetadata.offset()); + } + } + + @Test + public void testDeleteConsumerGroups() throws Exception { + final HashMap nodes = new HashMap<>(); + nodes.put(0, new Node(0, "localhost", 8121)); + + final Cluster cluster = + new Cluster( + "mockClusterId", + nodes.values(), + Collections.emptyList(), + Collections.emptySet(), + Collections.emptySet(), nodes.get(0)); + + final List groupIds = Collections.singletonList("group-0"); + + try (AdminClientUnitTestEnv env = new AdminClientUnitTestEnv(cluster)) { + env.kafkaClient().setNodeApiVersions(NodeApiVersions.create()); + env.kafkaClient().prepareMetadataUpdate(env.cluster(), Collections.emptySet()); + env.kafkaClient().setNode(env.cluster().controller()); + + env.kafkaClient().prepareResponse(new FindCoordinatorResponse(Errors.NONE, env.cluster().controller())); + + final Map response = new HashMap<>(); + response.put("group-0", Errors.NONE); + env.kafkaClient().prepareResponse(new DeleteGroupsResponse(response)); + + final DeleteConsumerGroupsResult result = env.adminClient().deleteConsumerGroups(groupIds); + + final Map> results = result.deletedGroups().get(); + assertNull(results.get("group-0").get()); + } + } + private static void assertCollectionIs(Collection collection, T... elements) { for (T element : elements) { assertTrue("Did not find " + element, collection.contains(element)); diff --git a/clients/src/test/java/org/apache/kafka/clients/admin/MockAdminClient.java b/clients/src/test/java/org/apache/kafka/clients/admin/MockAdminClient.java index c141a8acac9e9..2fc7048b759aa 100644 --- a/clients/src/test/java/org/apache/kafka/clients/admin/MockAdminClient.java +++ b/clients/src/test/java/org/apache/kafka/clients/admin/MockAdminClient.java @@ -276,6 +276,46 @@ public DeleteRecordsResult deleteRecords(Map re } } + @Override + public CreateDelegationTokenResult createDelegationToken(CreateDelegationTokenOptions options) { + throw new UnsupportedOperationException("Not implemented yet"); + } + + @Override + public RenewDelegationTokenResult renewDelegationToken(byte[] hmac, RenewDelegationTokenOptions options) { + throw new UnsupportedOperationException("Not implemented yet"); + } + + @Override + public ExpireDelegationTokenResult expireDelegationToken(byte[] hmac, ExpireDelegationTokenOptions options) { + throw new UnsupportedOperationException("Not implemented yet"); + } + + @Override + public DescribeDelegationTokenResult describeDelegationToken(DescribeDelegationTokenOptions options) { + throw new UnsupportedOperationException("Not implemented yet"); + } + + @Override + public DescribeConsumerGroupsResult describeConsumerGroups(Collection groupIds, DescribeConsumerGroupsOptions options) { + throw new UnsupportedOperationException("Not implemented yet"); + } + + @Override + public ListConsumerGroupsResult listConsumerGroups(ListConsumerGroupsOptions options) { + throw new UnsupportedOperationException("Not implemented yet"); + } + + @Override + public ListConsumerGroupOffsetsResult listConsumerGroupOffsets(String groupId, ListConsumerGroupOffsetsOptions options) { + throw new UnsupportedOperationException("Not implemented yet"); + } + + @Override + public DeleteConsumerGroupsResult deleteConsumerGroups(Collection groupIds, DeleteConsumerGroupsOptions options) { + throw new UnsupportedOperationException("Not implemented yet"); + } + @Override public CreateAclsResult createAcls(Collection acls, CreateAclsOptions options) { throw new UnsupportedOperationException("Not implemented yet"); diff --git a/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java b/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java index 6fcf48059672a..558ec72109695 100644 --- a/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/producer/internals/TransactionManagerTest.java @@ -132,6 +132,26 @@ public void setup() { client.setNode(brokerNode); } + @Test + public void testSenderShutdownWithPendingAddPartitions() throws Exception { + long pid = 13131L; + short epoch = 1; + doInitTransactions(pid, epoch); + transactionManager.beginTransaction(); + + transactionManager.maybeAddPartitionToTransaction(tp0); + FutureRecordMetadata sendFuture = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), + "value".getBytes(), Record.EMPTY_HEADERS, null, MAX_BLOCK_TIMEOUT).future; + + prepareAddPartitionsToTxn(tp0, Errors.NONE); + prepareProduceResponse(Errors.NONE, pid, epoch); + + sender.initiateClose(); + sender.run(); + + assertTrue(sendFuture.isDone()); + } + @Test public void testEndTxnNotSentIfIncompleteBatches() { long pid = 13131L; diff --git a/clients/src/test/java/org/apache/kafka/common/network/NioEchoServer.java b/clients/src/test/java/org/apache/kafka/common/network/NioEchoServer.java index 0352adec72de7..68979a1e01ff9 100644 --- a/clients/src/test/java/org/apache/kafka/common/network/NioEchoServer.java +++ b/clients/src/test/java/org/apache/kafka/common/network/NioEchoServer.java @@ -24,8 +24,8 @@ import org.apache.kafka.common.metrics.Metrics; import org.apache.kafka.common.security.auth.SecurityProtocol; import org.apache.kafka.common.security.authenticator.CredentialCache; -import org.apache.kafka.common.security.scram.ScramCredentialUtils; -import org.apache.kafka.common.security.scram.ScramMechanism; +import org.apache.kafka.common.security.scram.ScramCredential; +import org.apache.kafka.common.security.scram.internal.ScramMechanism; import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.MockTime; import org.apache.kafka.test.TestCondition; @@ -44,7 +44,7 @@ import java.util.List; import java.util.Map; -import org.apache.kafka.common.security.token.delegation.DelegationTokenCache; +import org.apache.kafka.common.security.token.delegation.internal.DelegationTokenCache; /** * Non-blocking EchoServer implementation that uses ChannelBuilder to create channels @@ -79,8 +79,12 @@ public NioEchoServer(ListenerName listenerName, SecurityProtocol securityProtoco this.newChannels = Collections.synchronizedList(new ArrayList()); this.credentialCache = credentialCache; this.tokenCache = new DelegationTokenCache(ScramMechanism.mechanismNames()); - if (securityProtocol == SecurityProtocol.SASL_PLAINTEXT || securityProtocol == SecurityProtocol.SASL_SSL) - ScramCredentialUtils.createCache(credentialCache, ScramMechanism.mechanismNames()); + if (securityProtocol == SecurityProtocol.SASL_PLAINTEXT || securityProtocol == SecurityProtocol.SASL_SSL) { + for (String mechanism : ScramMechanism.mechanismNames()) { + if (credentialCache.cache(mechanism, ScramCredential.class) == null) + credentialCache.createCache(mechanism, ScramCredential.class); + } + } if (channelBuilder == null) channelBuilder = ChannelBuilders.serverChannelBuilder(listenerName, false, securityProtocol, config, credentialCache, tokenCache); this.metrics = new Metrics(); diff --git a/clients/src/test/java/org/apache/kafka/common/network/SslTransportLayerTest.java b/clients/src/test/java/org/apache/kafka/common/network/SslTransportLayerTest.java index f54f490ec0a88..d6b4bac48d8e3 100644 --- a/clients/src/test/java/org/apache/kafka/common/network/SslTransportLayerTest.java +++ b/clients/src/test/java/org/apache/kafka/common/network/SslTransportLayerTest.java @@ -635,7 +635,7 @@ public void testApplicationBufferResize() throws Exception { @Test public void testNetworkThreadTimeRecorded() throws Exception { selector.close(); - this.selector = new Selector(NetworkReceive.UNLIMITED, 5000, new Metrics(), Time.SYSTEM, + this.selector = new Selector(NetworkReceive.UNLIMITED, Selector.NO_IDLE_TIMEOUT_MS, new Metrics(), Time.SYSTEM, "MetricGroup", new HashMap(), false, true, channelBuilder, MemoryPool.NONE, new LogContext()); String node = "0"; diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java index bdbd10626856a..c63cecdda2813 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java @@ -1223,7 +1223,7 @@ private CreateDelegationTokenResponse createCreateTokenResponse() { } private RenewDelegationTokenRequest createRenewTokenRequest() { - return new RenewDelegationTokenRequest.Builder(ByteBuffer.wrap("test".getBytes()), System.currentTimeMillis()).build(); + return new RenewDelegationTokenRequest.Builder("test".getBytes(), System.currentTimeMillis()).build(); } private RenewDelegationTokenResponse createRenewTokenResponse() { @@ -1231,7 +1231,7 @@ private RenewDelegationTokenResponse createRenewTokenResponse() { } private ExpireDelegationTokenRequest createExpireTokenRequest() { - return new ExpireDelegationTokenRequest.Builder(ByteBuffer.wrap("test".getBytes()), System.currentTimeMillis()).build(); + return new ExpireDelegationTokenRequest.Builder("test".getBytes(), System.currentTimeMillis()).build(); } private ExpireDelegationTokenResponse createExpireTokenResponse() { diff --git a/clients/src/test/java/org/apache/kafka/common/security/JaasContextTest.java b/clients/src/test/java/org/apache/kafka/common/security/JaasContextTest.java index e8535d2c02c94..d96b359c1d3bf 100644 --- a/clients/src/test/java/org/apache/kafka/common/security/JaasContextTest.java +++ b/clients/src/test/java/org/apache/kafka/common/security/JaasContextTest.java @@ -174,6 +174,11 @@ public void testNumericOptionWithoutQuotes() throws Exception { checkInvalidConfiguration("test.testNumericOptionWithoutQuotes required option1=3;"); } + @Test + public void testInvalidControlFlag() throws Exception { + checkInvalidConfiguration("test.testInvalidControlFlag { option1=3;"); + } + @Test public void testNumericOptionWithQuotes() throws Exception { Map options = new HashMap<>(); diff --git a/clients/src/test/java/org/apache/kafka/common/security/TestSecurityConfig.java b/clients/src/test/java/org/apache/kafka/common/security/TestSecurityConfig.java index 05294cfc5c268..81a883cebf98f 100644 --- a/clients/src/test/java/org/apache/kafka/common/security/TestSecurityConfig.java +++ b/clients/src/test/java/org/apache/kafka/common/security/TestSecurityConfig.java @@ -31,6 +31,9 @@ public class TestSecurityConfig extends AbstractConfig { .define(BrokerSecurityConfigs.SASL_ENABLED_MECHANISMS_CONFIG, Type.LIST, BrokerSecurityConfigs.DEFAULT_SASL_ENABLED_MECHANISMS, Importance.MEDIUM, BrokerSecurityConfigs.SASL_ENABLED_MECHANISMS_DOC) + .define(BrokerSecurityConfigs.SASL_SERVER_CALLBACK_HANDLER_CLASS, Type.CLASS, + null, + Importance.MEDIUM, BrokerSecurityConfigs.SASL_SERVER_CALLBACK_HANDLER_CLASS_DOC) .define(BrokerSecurityConfigs.PRINCIPAL_BUILDER_CLASS_CONFIG, Type.CLASS, null, Importance.MEDIUM, BrokerSecurityConfigs.PRINCIPAL_BUILDER_CLASS_DOC) .withClientSslSupport() diff --git a/clients/src/test/java/org/apache/kafka/common/security/auth/DefaultKafkaPrincipalBuilderTest.java b/clients/src/test/java/org/apache/kafka/common/security/auth/DefaultKafkaPrincipalBuilderTest.java index a30c09ff3167e..fdf368788c02e 100644 --- a/clients/src/test/java/org/apache/kafka/common/security/auth/DefaultKafkaPrincipalBuilderTest.java +++ b/clients/src/test/java/org/apache/kafka/common/security/auth/DefaultKafkaPrincipalBuilderTest.java @@ -22,7 +22,7 @@ import org.apache.kafka.common.security.authenticator.DefaultKafkaPrincipalBuilder; import org.apache.kafka.common.security.kerberos.KerberosName; import org.apache.kafka.common.security.kerberos.KerberosShortNamer; -import org.apache.kafka.common.security.scram.ScramMechanism; +import org.apache.kafka.common.security.scram.internal.ScramMechanism; import org.easymock.EasyMock; import org.easymock.EasyMockSupport; import org.junit.Test; diff --git a/clients/src/test/java/org/apache/kafka/common/security/authenticator/LoginManagerTest.java b/clients/src/test/java/org/apache/kafka/common/security/authenticator/LoginManagerTest.java index 8be72fb5b8c93..5436b2a1dfc37 100644 --- a/clients/src/test/java/org/apache/kafka/common/security/authenticator/LoginManagerTest.java +++ b/clients/src/test/java/org/apache/kafka/common/security/authenticator/LoginManagerTest.java @@ -59,17 +59,17 @@ public void testClientLoginManager() throws Exception { JaasContext staticContext = JaasContext.loadClientContext(Collections.emptyMap()); LoginManager dynamicLogin = LoginManager.acquireLoginManager(dynamicContext, "PLAIN", - false, configs); + DefaultLogin.class, configs); assertEquals(dynamicPlainContext, dynamicLogin.cacheKey()); LoginManager staticLogin = LoginManager.acquireLoginManager(staticContext, "SCRAM-SHA-256", - false, configs); + DefaultLogin.class, configs); assertNotSame(dynamicLogin, staticLogin); assertEquals("KafkaClient", staticLogin.cacheKey()); assertSame(dynamicLogin, LoginManager.acquireLoginManager(dynamicContext, "PLAIN", - false, configs)); + DefaultLogin.class, configs)); assertSame(staticLogin, LoginManager.acquireLoginManager(staticContext, "SCRAM-SHA-256", - false, configs)); + DefaultLogin.class, configs)); verifyLoginManagerRelease(dynamicLogin, 2, dynamicContext, configs); verifyLoginManagerRelease(staticLogin, 2, staticContext, configs); @@ -86,23 +86,23 @@ public void testServerLoginManager() throws Exception { JaasContext scramJaasContext = JaasContext.loadServerContext(listenerName, "SCRAM-SHA-256", configs); LoginManager dynamicPlainLogin = LoginManager.acquireLoginManager(plainJaasContext, "PLAIN", - false, configs); + DefaultLogin.class, configs); assertEquals(dynamicPlainContext, dynamicPlainLogin.cacheKey()); LoginManager dynamicDigestLogin = LoginManager.acquireLoginManager(digestJaasContext, "DIGEST-MD5", - false, configs); + DefaultLogin.class, configs); assertNotSame(dynamicPlainLogin, dynamicDigestLogin); assertEquals(dynamicDigestContext, dynamicDigestLogin.cacheKey()); LoginManager staticScramLogin = LoginManager.acquireLoginManager(scramJaasContext, "SCRAM-SHA-256", - false, configs); + DefaultLogin.class, configs); assertNotSame(dynamicPlainLogin, staticScramLogin); assertEquals("KafkaServer", staticScramLogin.cacheKey()); assertSame(dynamicPlainLogin, LoginManager.acquireLoginManager(plainJaasContext, "PLAIN", - false, configs)); + DefaultLogin.class, configs)); assertSame(dynamicDigestLogin, LoginManager.acquireLoginManager(digestJaasContext, "DIGEST-MD5", - false, configs)); + DefaultLogin.class, configs)); assertSame(staticScramLogin, LoginManager.acquireLoginManager(scramJaasContext, "SCRAM-SHA-256", - false, configs)); + DefaultLogin.class, configs)); verifyLoginManagerRelease(dynamicPlainLogin, 2, plainJaasContext, configs); verifyLoginManagerRelease(dynamicDigestLogin, 2, digestJaasContext, configs); @@ -116,13 +116,13 @@ private void verifyLoginManagerRelease(LoginManager loginManager, int acquireCou for (int i = 0; i < acquireCount - 1; i++) loginManager.release(); assertSame(loginManager, LoginManager.acquireLoginManager(jaasContext, "PLAIN", - false, configs)); + DefaultLogin.class, configs)); // Release all references and verify that new LoginManager is created on next acquire for (int i = 0; i < 2; i++) // release all references loginManager.release(); LoginManager newLoginManager = LoginManager.acquireLoginManager(jaasContext, "PLAIN", - false, configs); + DefaultLogin.class, configs); assertNotSame(loginManager, newLoginManager); newLoginManager.release(); } diff --git a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java index b8edc612f6975..bfd1d976d22ba 100644 --- a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java +++ b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslAuthenticatorTest.java @@ -16,6 +16,32 @@ */ package org.apache.kafka.common.security.authenticator; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.ByteBuffer; +import java.nio.channels.SelectionKey; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.atomic.AtomicInteger; + +import javax.security.auth.Subject; +import javax.security.auth.callback.Callback; +import javax.security.auth.callback.CallbackHandler; +import javax.security.auth.callback.NameCallback; +import javax.security.auth.callback.PasswordCallback; +import javax.security.auth.callback.UnsupportedCallbackException; +import javax.security.auth.login.Configuration; +import javax.security.auth.login.AppConfigurationEntry; +import javax.security.auth.login.LoginContext; +import javax.security.auth.login.LoginException; + import org.apache.kafka.clients.NetworkClient; import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.config.SaslConfigs; @@ -37,6 +63,7 @@ import org.apache.kafka.common.network.TransportLayer; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.security.auth.Login; import org.apache.kafka.common.security.auth.SecurityProtocol; import org.apache.kafka.common.requests.AbstractRequest; import org.apache.kafka.common.requests.AbstractResponse; @@ -54,32 +81,20 @@ import org.apache.kafka.common.security.auth.KafkaPrincipal; import org.apache.kafka.common.security.plain.PlainLoginModule; import org.apache.kafka.common.security.scram.ScramCredential; -import org.apache.kafka.common.security.scram.ScramCredentialUtils; -import org.apache.kafka.common.security.scram.ScramFormatter; +import org.apache.kafka.common.security.scram.internal.ScramCredentialUtils; +import org.apache.kafka.common.security.scram.internal.ScramFormatter; import org.apache.kafka.common.security.scram.ScramLoginModule; -import org.apache.kafka.common.security.scram.ScramMechanism; +import org.apache.kafka.common.security.scram.internal.ScramMechanism; import org.apache.kafka.common.security.token.delegation.TokenInformation; import org.apache.kafka.common.utils.SecurityUtils; +import org.apache.kafka.common.security.auth.AuthenticateCallbackHandler; +import org.apache.kafka.common.security.authenticator.TestDigestLoginModule.DigestServerCallbackHandler; +import org.apache.kafka.common.security.plain.internal.PlainServerCallbackHandler; + import org.junit.After; import org.junit.Before; import org.junit.Test; -import javax.security.auth.Subject; -import javax.security.auth.login.Configuration; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.nio.ByteBuffer; -import java.nio.channels.SelectionKey; -import java.security.NoSuchAlgorithmException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Random; - import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -110,6 +125,7 @@ public void setup() throws Exception { saslServerConfigs = serverCertStores.getTrustingConfig(clientCertStores); saslClientConfigs = clientCertStores.getTrustingConfig(serverCertStores); credentialCache = new CredentialCache(); + TestLogin.loginCount.set(0); } @After @@ -234,6 +250,7 @@ public void testMechanismPluggability() throws Exception { String node = "0"; SecurityProtocol securityProtocol = SecurityProtocol.SASL_SSL; configureMechanisms("DIGEST-MD5", Arrays.asList("DIGEST-MD5")); + configureDigestMd5ServerCallback(securityProtocol); server = createEchoServer(securityProtocol); createAndCheckClientConnection(securityProtocol, node); @@ -247,6 +264,7 @@ public void testMechanismPluggability() throws Exception { public void testMultipleServerMechanisms() throws Exception { SecurityProtocol securityProtocol = SecurityProtocol.SASL_SSL; configureMechanisms("DIGEST-MD5", Arrays.asList("DIGEST-MD5", "PLAIN", "SCRAM-SHA-256")); + configureDigestMd5ServerCallback(securityProtocol); server = createEchoServer(securityProtocol); updateScramCredentialCache(TestJaasConfig.USERNAME, TestJaasConfig.PASSWORD); @@ -680,6 +698,207 @@ public void testInvalidLoginModule() throws Exception { } } + /** + * Tests SASL client authentication callback handler override. + */ + @Test + public void testClientAuthenticateCallbackHandler() throws Exception { + SecurityProtocol securityProtocol = SecurityProtocol.SASL_PLAINTEXT; + TestJaasConfig jaasConfig = configureMechanisms("PLAIN", Collections.singletonList("PLAIN")); + saslClientConfigs.put(SaslConfigs.SASL_CLIENT_CALLBACK_HANDLER_CLASS, TestClientCallbackHandler.class.getName()); + jaasConfig.setClientOptions("PLAIN", "", ""); // remove username, password in login context + + Map options = new HashMap<>(); + options.put("user_" + TestClientCallbackHandler.USERNAME, TestClientCallbackHandler.PASSWORD); + jaasConfig.createOrUpdateEntry(TestJaasConfig.LOGIN_CONTEXT_SERVER, PlainLoginModule.class.getName(), options); + server = createEchoServer(securityProtocol); + createAndCheckClientConnection(securityProtocol, "good"); + + options.clear(); + options.put("user_" + TestClientCallbackHandler.USERNAME, "invalid-password"); + jaasConfig.createOrUpdateEntry(TestJaasConfig.LOGIN_CONTEXT_SERVER, PlainLoginModule.class.getName(), options); + createAndCheckClientConnectionFailure(securityProtocol, "invalid"); + } + + /** + * Tests SASL server authentication callback handler override. + */ + @Test + public void testServerAuthenticateCallbackHandler() throws Exception { + SecurityProtocol securityProtocol = SecurityProtocol.SASL_PLAINTEXT; + TestJaasConfig jaasConfig = configureMechanisms("PLAIN", Collections.singletonList("PLAIN")); + jaasConfig.createOrUpdateEntry(TestJaasConfig.LOGIN_CONTEXT_SERVER, PlainLoginModule.class.getName(), new HashMap()); + String callbackPrefix = ListenerName.forSecurityProtocol(securityProtocol).saslMechanismConfigPrefix("PLAIN"); + saslServerConfigs.put(callbackPrefix + BrokerSecurityConfigs.SASL_SERVER_CALLBACK_HANDLER_CLASS, + TestServerCallbackHandler.class.getName()); + server = createEchoServer(securityProtocol); + + // Set client username/password to the values used by `TestServerCallbackHandler` + jaasConfig.setClientOptions("PLAIN", TestServerCallbackHandler.USERNAME, TestServerCallbackHandler.PASSWORD); + createAndCheckClientConnection(securityProtocol, "good"); + + // Set client username/password to the invalid values + jaasConfig.setClientOptions("PLAIN", TestJaasConfig.USERNAME, "invalid-password"); + createAndCheckClientConnectionFailure(securityProtocol, "invalid"); + } + + /** + * Test that callback handlers are only applied to connections for the mechanisms + * configured for the handler. Test enables two mechanisms 'PLAIN` and `DIGEST-MD5` + * on the servers with different callback handlers for the two mechanisms. Verifies + * that clients using both mechanisms authenticate successfully. + */ + @Test + public void testAuthenticateCallbackHandlerMechanisms() throws Exception { + SecurityProtocol securityProtocol = SecurityProtocol.SASL_PLAINTEXT; + TestJaasConfig jaasConfig = configureMechanisms("DIGEST-MD5", Arrays.asList("DIGEST-MD5", "PLAIN")); + + // Connections should fail using the digest callback handler if listener.mechanism prefix not specified + saslServerConfigs.put("plain." + BrokerSecurityConfigs.SASL_SERVER_CALLBACK_HANDLER_CLASS, + TestServerCallbackHandler.class); + saslServerConfigs.put("digest-md5." + BrokerSecurityConfigs.SASL_SERVER_CALLBACK_HANDLER_CLASS, + DigestServerCallbackHandler.class); + server = createEchoServer(securityProtocol); + createAndCheckClientConnectionFailure(securityProtocol, "invalid"); + + // Connections should succeed using the server callback handler associated with the listener + ListenerName listener = ListenerName.forSecurityProtocol(securityProtocol); + saslServerConfigs.remove("plain." + BrokerSecurityConfigs.SASL_SERVER_CALLBACK_HANDLER_CLASS); + saslServerConfigs.remove("digest-md5." + BrokerSecurityConfigs.SASL_SERVER_CALLBACK_HANDLER_CLASS); + saslServerConfigs.put(listener.saslMechanismConfigPrefix("plain") + BrokerSecurityConfigs.SASL_SERVER_CALLBACK_HANDLER_CLASS, + TestServerCallbackHandler.class); + saslServerConfigs.put(listener.saslMechanismConfigPrefix("digest-md5") + BrokerSecurityConfigs.SASL_SERVER_CALLBACK_HANDLER_CLASS, + DigestServerCallbackHandler.class); + server = createEchoServer(securityProtocol); + + // Verify that DIGEST-MD5 (currently configured for client) works with `DigestServerCallbackHandler` + createAndCheckClientConnection(securityProtocol, "good-digest-md5"); + + // Verify that PLAIN works with `TestServerCallbackHandler` + jaasConfig.setClientOptions("PLAIN", TestServerCallbackHandler.USERNAME, TestServerCallbackHandler.PASSWORD); + saslClientConfigs.put(SaslConfigs.SASL_MECHANISM, "PLAIN"); + createAndCheckClientConnection(securityProtocol, "good-plain"); + } + + /** + * Tests SASL login class override. + */ + @Test + public void testClientLoginOverride() throws Exception { + SecurityProtocol securityProtocol = SecurityProtocol.SASL_PLAINTEXT; + TestJaasConfig jaasConfig = configureMechanisms("PLAIN", Collections.singletonList("PLAIN")); + jaasConfig.setClientOptions("PLAIN", "invaliduser", "invalidpassword"); + server = createEchoServer(securityProtocol); + + // Connection should succeed using login override that sets correct username/password in Subject + saslClientConfigs.put(SaslConfigs.SASL_LOGIN_CLASS, TestLogin.class.getName()); + createAndCheckClientConnection(securityProtocol, "1"); + assertEquals(1, TestLogin.loginCount.get()); + + // Connection should fail without login override since username/password in jaas config is invalid + saslClientConfigs.remove(SaslConfigs.SASL_LOGIN_CLASS); + createAndCheckClientConnectionFailure(securityProtocol, "invalid"); + assertEquals(1, TestLogin.loginCount.get()); + } + + /** + * Tests SASL server login class override. + */ + @Test + public void testServerLoginOverride() throws Exception { + SecurityProtocol securityProtocol = SecurityProtocol.SASL_PLAINTEXT; + configureMechanisms("PLAIN", Collections.singletonList("PLAIN")); + String prefix = ListenerName.forSecurityProtocol(securityProtocol).saslMechanismConfigPrefix("PLAIN"); + saslServerConfigs.put(prefix + SaslConfigs.SASL_LOGIN_CLASS, TestLogin.class.getName()); + server = createEchoServer(securityProtocol); + + // Login is performed when server channel builder is created (before any connections are made on the server) + assertEquals(1, TestLogin.loginCount.get()); + + createAndCheckClientConnection(securityProtocol, "1"); + assertEquals(1, TestLogin.loginCount.get()); + } + + /** + * Tests SASL login callback class override. + */ + @Test + public void testClientLoginCallbackOverride() throws Exception { + SecurityProtocol securityProtocol = SecurityProtocol.SASL_PLAINTEXT; + TestJaasConfig jaasConfig = configureMechanisms("PLAIN", Collections.singletonList("PLAIN")); + jaasConfig.createOrUpdateEntry(TestJaasConfig.LOGIN_CONTEXT_CLIENT, TestPlainLoginModule.class.getName(), + Collections.emptyMap()); + server = createEchoServer(securityProtocol); + + // Connection should succeed using login callback override that sets correct username/password + saslClientConfigs.put(SaslConfigs.SASL_LOGIN_CALLBACK_HANDLER_CLASS, TestLoginCallbackHandler.class.getName()); + createAndCheckClientConnection(securityProtocol, "1"); + + // Connection should fail without login callback override since username/password in jaas config is invalid + saslClientConfigs.remove(SaslConfigs.SASL_LOGIN_CALLBACK_HANDLER_CLASS); + try { + createClientConnection(securityProtocol, "invalid"); + } catch (Exception e) { + assertTrue("Unexpected exception " + e.getCause(), e.getCause() instanceof LoginException); + } + } + + /** + * Tests SASL server login callback class override. + */ + @Test + public void testServerLoginCallbackOverride() throws Exception { + SecurityProtocol securityProtocol = SecurityProtocol.SASL_PLAINTEXT; + TestJaasConfig jaasConfig = configureMechanisms("PLAIN", Collections.singletonList("PLAIN")); + jaasConfig.createOrUpdateEntry(TestJaasConfig.LOGIN_CONTEXT_SERVER, TestPlainLoginModule.class.getName(), + Collections.emptyMap()); + jaasConfig.setClientOptions("PLAIN", TestServerCallbackHandler.USERNAME, TestServerCallbackHandler.PASSWORD); + ListenerName listenerName = ListenerName.forSecurityProtocol(securityProtocol); + String prefix = listenerName.saslMechanismConfigPrefix("PLAIN"); + saslServerConfigs.put(prefix + BrokerSecurityConfigs.SASL_SERVER_CALLBACK_HANDLER_CLASS, + TestServerCallbackHandler.class); + Class loginCallback = TestLoginCallbackHandler.class; + + try { + createEchoServer(securityProtocol); + fail("Should have failed to create server with default login handler"); + } catch (KafkaException e) { + // Expected exception + } + + try { + saslServerConfigs.put(SaslConfigs.SASL_LOGIN_CALLBACK_HANDLER_CLASS, loginCallback); + createEchoServer(securityProtocol); + fail("Should have failed to create server with login handler config without listener+mechanism prefix"); + } catch (KafkaException e) { + // Expected exception + saslServerConfigs.remove(SaslConfigs.SASL_LOGIN_CALLBACK_HANDLER_CLASS); + } + + try { + saslServerConfigs.put("plain." + SaslConfigs.SASL_LOGIN_CALLBACK_HANDLER_CLASS, loginCallback); + createEchoServer(securityProtocol); + fail("Should have failed to create server with login handler config without listener prefix"); + } catch (KafkaException e) { + // Expected exception + saslServerConfigs.remove("plain." + SaslConfigs.SASL_LOGIN_CALLBACK_HANDLER_CLASS); + } + + try { + saslServerConfigs.put(listenerName.configPrefix() + SaslConfigs.SASL_LOGIN_CALLBACK_HANDLER_CLASS, loginCallback); + createEchoServer(securityProtocol); + fail("Should have failed to create server with login handler config without mechanism prefix"); + } catch (KafkaException e) { + // Expected exception + saslServerConfigs.remove("plain." + SaslConfigs.SASL_LOGIN_CALLBACK_HANDLER_CLASS); + } + + // Connection should succeed using login callback override for mechanism + saslServerConfigs.put(prefix + SaslConfigs.SASL_LOGIN_CALLBACK_HANDLER_CLASS, loginCallback); + server = createEchoServer(securityProtocol); + createAndCheckClientConnection(securityProtocol, "1"); + } + /** * Tests that mechanisms with default implementation in Kafka may be disabled in * the Kafka server by removing from the enabled mechanism list. @@ -1028,10 +1247,12 @@ private NioEchoServer startServerWithoutSaslAuthenticateHeader(final SecurityPro securityProtocol, listenerName, false, saslMechanism, true, credentialCache, null) { @Override - protected SaslServerAuthenticator buildServerAuthenticator(Map configs, String id, - TransportLayer transportLayer, Map subjects) throws IOException { - return new SaslServerAuthenticator(configs, id, jaasContexts, subjects, null, - credentialCache, listenerName, securityProtocol, transportLayer, null) { + protected SaslServerAuthenticator buildServerAuthenticator(Map configs, + Map callbackHandlers, + String id, + TransportLayer transportLayer, + Map subjects) throws IOException { + return new SaslServerAuthenticator(configs, callbackHandlers, id, subjects, null, listenerName, securityProtocol, transportLayer) { @Override protected ApiVersionsResponse apiVersionsResponse() { @@ -1072,11 +1293,15 @@ private void createClientConnectionWithoutSaslAuthenticateHeader(final SecurityP securityProtocol, listenerName, false, saslMechanism, true, null, null) { @Override - protected SaslClientAuthenticator buildClientAuthenticator(Map configs, String id, - String serverHost, String servicePrincipal, - TransportLayer transportLayer, Subject subject) throws IOException { - - return new SaslClientAuthenticator(configs, id, subject, + protected SaslClientAuthenticator buildClientAuthenticator(Map configs, + AuthenticateCallbackHandler callbackHandler, + String id, + String serverHost, + String servicePrincipal, + TransportLayer transportLayer, + Subject subject) throws IOException { + + return new SaslClientAuthenticator(configs, callbackHandler, id, subject, servicePrincipal, serverHost, saslMechanism, true, transportLayer) { @Override protected SaslHandshakeRequest createSaslHandshakeRequest(short version) { @@ -1173,9 +1398,19 @@ private void authenticateUsingSaslPlainAndCheckConnection(String node, boolean e private TestJaasConfig configureMechanisms(String clientMechanism, List serverMechanisms) { saslClientConfigs.put(SaslConfigs.SASL_MECHANISM, clientMechanism); saslServerConfigs.put(BrokerSecurityConfigs.SASL_ENABLED_MECHANISMS_CONFIG, serverMechanisms); + if (serverMechanisms.contains("DIGEST-MD5")) { + saslServerConfigs.put("digest-md5." + BrokerSecurityConfigs.SASL_SERVER_CALLBACK_HANDLER_CLASS, + TestDigestLoginModule.DigestServerCallbackHandler.class.getName()); + } return TestJaasConfig.createConfiguration(clientMechanism, serverMechanisms); } + private void configureDigestMd5ServerCallback(SecurityProtocol securityProtocol) { + String callbackPrefix = ListenerName.forSecurityProtocol(securityProtocol).saslMechanismConfigPrefix("DIGEST-MD5"); + saslServerConfigs.put(callbackPrefix + BrokerSecurityConfigs.SASL_SERVER_CALLBACK_HANDLER_CLASS, + TestDigestLoginModule.DigestServerCallbackHandler.class); + } + private void createSelector(SecurityProtocol securityProtocol, Map clientConfigs) { if (selector != null) { selector.close(); @@ -1261,6 +1496,28 @@ private ByteBuffer waitForResponse() throws IOException { return selector.completedReceives().get(0).payload(); } + public static class TestServerCallbackHandler extends PlainServerCallbackHandler { + + static final String USERNAME = "TestServerCallbackHandler-user"; + static final String PASSWORD = "TestServerCallbackHandler-password"; + private volatile boolean configured; + + @Override + public void configure(Map configs, String mechanism, List jaasConfigEntries) { + if (configured) + throw new IllegalStateException("Server callback handler configured twice"); + configured = true; + super.configure(configs, mechanism, jaasConfigEntries); + } + + @Override + protected boolean authenticate(String username, char[] password) throws IOException { + if (!configured) + throw new IllegalStateException("Server callback handler not configured"); + return USERNAME.equals(username) && new String(password).equals(PASSWORD); + } + } + @SuppressWarnings("unchecked") private void updateScramCredentialCache(String username, String password) throws NoSuchAlgorithmException { for (String mechanism : (List) saslServerConfigs.get(BrokerSecurityConfigs.SASL_ENABLED_MECHANISMS_CONFIG)) { @@ -1289,4 +1546,121 @@ private void updateTokenCredentialCache(String username, String password) throws } } } + + public static class TestClientCallbackHandler implements AuthenticateCallbackHandler { + + static final String USERNAME = "TestClientCallbackHandler-user"; + static final String PASSWORD = "TestClientCallbackHandler-password"; + private volatile boolean configured; + + @Override + public void configure(Map configs, String mechanism, List jaasConfigEntries) { + if (configured) + throw new IllegalStateException("Client callback handler configured twice"); + configured = true; + } + + @Override + public void handle(Callback[] callbacks) throws IOException, UnsupportedCallbackException { + if (!configured) + throw new IllegalStateException("Client callback handler not configured"); + for (Callback callback : callbacks) { + if (callback instanceof NameCallback) + ((NameCallback) callback).setName(USERNAME); + else if (callback instanceof PasswordCallback) + ((PasswordCallback) callback).setPassword(PASSWORD.toCharArray()); + else + throw new UnsupportedCallbackException(callback); + } + } + + @Override + public void close() { + } + } + + public static class TestLogin implements Login { + + static AtomicInteger loginCount = new AtomicInteger(); + + private String contextName; + private Configuration configuration; + private Subject subject; + @Override + public void configure(Map configs, String contextName, Configuration configuration, + AuthenticateCallbackHandler callbackHandler) { + assertEquals(1, configuration.getAppConfigurationEntry(contextName).length); + this.contextName = contextName; + this.configuration = configuration; + } + + @Override + public LoginContext login() throws LoginException { + LoginContext context = new LoginContext(contextName, null, new AbstractLogin.DefaultLoginCallbackHandler(), configuration); + context.login(); + subject = context.getSubject(); + subject.getPublicCredentials().clear(); + subject.getPrivateCredentials().clear(); + subject.getPublicCredentials().add(TestJaasConfig.USERNAME); + subject.getPrivateCredentials().add(TestJaasConfig.PASSWORD); + loginCount.incrementAndGet(); + return context; + } + + @Override + public Subject subject() { + return subject; + } + + @Override + public String serviceName() { + return "kafka"; + } + + @Override + public void close() { + } + } + + public static class TestLoginCallbackHandler implements AuthenticateCallbackHandler { + private volatile boolean configured = false; + @Override + public void configure(Map configs, String saslMechanism, List jaasConfigEntries) { + if (configured) + throw new IllegalStateException("Login callback handler configured twice"); + configured = true; + } + + @Override + public void handle(Callback[] callbacks) throws IOException, UnsupportedCallbackException { + if (!configured) + throw new IllegalStateException("Login callback handler not configured"); + + for (Callback callback : callbacks) { + if (callback instanceof NameCallback) + ((NameCallback) callback).setName(TestJaasConfig.USERNAME); + else if (callback instanceof PasswordCallback) + ((PasswordCallback) callback).setPassword(TestJaasConfig.PASSWORD.toCharArray()); + } + } + + @Override + public void close() { + } + } + + public static final class TestPlainLoginModule extends PlainLoginModule { + @Override + public void initialize(Subject subject, CallbackHandler callbackHandler, Map sharedState, Map options) { + try { + NameCallback nameCallback = new NameCallback("name:"); + PasswordCallback passwordCallback = new PasswordCallback("password:", false); + callbackHandler.handle(new Callback[]{nameCallback, passwordCallback}); + subject.getPublicCredentials().add(nameCallback.getName()); + subject.getPrivateCredentials().add(new String(passwordCallback.getPassword())); + } catch (Exception e) { + throw new SaslAuthenticationException("Login initialization failed", e); + } + } + } } diff --git a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticatorTest.java b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticatorTest.java index 17d31bda72ccb..3ec30317f5ef2 100644 --- a/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticatorTest.java +++ b/clients/src/test/java/org/apache/kafka/common/security/authenticator/SaslServerAuthenticatorTest.java @@ -22,13 +22,12 @@ import org.apache.kafka.common.network.ListenerName; import org.apache.kafka.common.network.TransportLayer; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.security.auth.AuthenticateCallbackHandler; import org.apache.kafka.common.security.auth.SecurityProtocol; import org.apache.kafka.common.protocol.types.Struct; import org.apache.kafka.common.requests.RequestHeader; import org.apache.kafka.common.security.JaasContext; import org.apache.kafka.common.security.plain.PlainLoginModule; -import org.apache.kafka.common.security.scram.ScramMechanism; -import org.apache.kafka.common.security.token.delegation.DelegationTokenCache; import org.easymock.Capture; import org.easymock.EasyMock; import org.easymock.IAnswer; @@ -41,7 +40,7 @@ import java.util.HashMap; import java.util.Map; -import static org.apache.kafka.common.security.scram.ScramMechanism.SCRAM_SHA_256; +import static org.apache.kafka.common.security.scram.internal.ScramMechanism.SCRAM_SHA_256; import static org.junit.Assert.fail; public class SaslServerAuthenticatorTest { @@ -112,8 +111,10 @@ private SaslServerAuthenticator setupAuthenticator(Map configs, Trans Map jaasContexts = Collections.singletonMap(mechanism, new JaasContext("jaasContext", JaasContext.Type.SERVER, jaasConfig, null)); Map subjects = Collections.singletonMap(mechanism, new Subject()); - return new SaslServerAuthenticator(configs, "node", jaasContexts, subjects, null, new CredentialCache(), - new ListenerName("ssl"), SecurityProtocol.SASL_SSL, transportLayer, new DelegationTokenCache(ScramMechanism.mechanismNames())); + Map callbackHandlers = Collections.singletonMap( + mechanism, new SaslServerCallbackHandler()); + return new SaslServerAuthenticator(configs, callbackHandlers, "node", subjects, null, + new ListenerName("ssl"), SecurityProtocol.SASL_SSL, transportLayer); } } diff --git a/clients/src/test/java/org/apache/kafka/common/security/authenticator/TestDigestLoginModule.java b/clients/src/test/java/org/apache/kafka/common/security/authenticator/TestDigestLoginModule.java index f1ef740e5dbe3..97b0b2715b230 100644 --- a/clients/src/test/java/org/apache/kafka/common/security/authenticator/TestDigestLoginModule.java +++ b/clients/src/test/java/org/apache/kafka/common/security/authenticator/TestDigestLoginModule.java @@ -17,62 +17,47 @@ package org.apache.kafka.common.security.authenticator; import java.io.IOException; -import java.security.Provider; -import java.security.Security; -import java.util.Arrays; -import java.util.Enumeration; -import java.util.HashMap; +import java.util.List; import java.util.Map; import javax.security.auth.callback.Callback; -import javax.security.auth.callback.CallbackHandler; import javax.security.auth.callback.NameCallback; import javax.security.auth.callback.PasswordCallback; import javax.security.auth.callback.UnsupportedCallbackException; +import javax.security.auth.login.AppConfigurationEntry; import javax.security.sasl.AuthorizeCallback; import javax.security.sasl.RealmCallback; -import javax.security.sasl.Sasl; -import javax.security.sasl.SaslException; -import javax.security.sasl.SaslServer; -import javax.security.sasl.SaslServerFactory; +import org.apache.kafka.common.security.auth.AuthenticateCallbackHandler; import org.apache.kafka.common.security.plain.PlainLoginModule; /** - * Digest-MD5 login module for multi-mechanism tests. Since callback handlers are not configurable in Kafka - * yet, this replaces the standard Digest-MD5 SASL server provider with one that invokes the test callback handler. + * Digest-MD5 login module for multi-mechanism tests. * This login module uses the same format as PlainLoginModule and hence simply reuses the same methods. * */ public class TestDigestLoginModule extends PlainLoginModule { - private static final SaslServerFactory STANDARD_DIGEST_SASL_SERVER_FACTORY; - static { - SaslServerFactory digestSaslServerFactory = null; - Enumeration factories = Sasl.getSaslServerFactories(); - Map emptyProps = new HashMap<>(); - while (factories.hasMoreElements()) { - SaslServerFactory factory = factories.nextElement(); - if (Arrays.asList(factory.getMechanismNames(emptyProps)).contains("DIGEST-MD5")) { - digestSaslServerFactory = factory; - break; - } - } - STANDARD_DIGEST_SASL_SERVER_FACTORY = digestSaslServerFactory; - Security.insertProviderAt(new DigestSaslServerProvider(), 1); - } + public static class DigestServerCallbackHandler implements AuthenticateCallbackHandler { - public static class DigestServerCallbackHandler implements CallbackHandler { + @Override + public void configure(Map configs, String saslMechanism, List jaasConfigEntries) { + } @Override public void handle(Callback[] callbacks) throws IOException, UnsupportedCallbackException { + String username = null; for (Callback callback : callbacks) { if (callback instanceof NameCallback) { NameCallback nameCallback = (NameCallback) callback; - nameCallback.setName(nameCallback.getDefaultName()); + if (TestJaasConfig.USERNAME.equals(nameCallback.getDefaultName())) { + nameCallback.setName(nameCallback.getDefaultName()); + username = TestJaasConfig.USERNAME; + } } else if (callback instanceof PasswordCallback) { PasswordCallback passwordCallback = (PasswordCallback) callback; - passwordCallback.setPassword(TestJaasConfig.PASSWORD.toCharArray()); + if (TestJaasConfig.USERNAME.equals(username)) + passwordCallback.setPassword(TestJaasConfig.PASSWORD.toCharArray()); } else if (callback instanceof RealmCallback) { RealmCallback realmCallback = (RealmCallback) callback; realmCallback.setText(realmCallback.getDefaultText()); @@ -85,30 +70,9 @@ public void handle(Callback[] callbacks) throws IOException, UnsupportedCallback } } } - } - - public static class DigestSaslServerFactory implements SaslServerFactory { - - @Override - public SaslServer createSaslServer(String mechanism, String protocol, String serverName, Map props, CallbackHandler cbh) - throws SaslException { - return STANDARD_DIGEST_SASL_SERVER_FACTORY.createSaslServer(mechanism, protocol, serverName, props, new DigestServerCallbackHandler()); - } @Override - public String[] getMechanismNames(Map props) { - return new String[] {"DIGEST-MD5"}; - } - } - - public static class DigestSaslServerProvider extends Provider { - - private static final long serialVersionUID = 1L; - - @SuppressWarnings("deprecation") - protected DigestSaslServerProvider() { - super("Test SASL/Digest-MD5 Server Provider", 1.0, "Test SASL/Digest-MD5 Server Provider for Kafka"); - put("SaslServerFactory.DIGEST-MD5", TestDigestLoginModule.DigestSaslServerFactory.class.getName()); + public void close() { } } } diff --git a/clients/src/test/java/org/apache/kafka/common/security/authenticator/TestJaasConfig.java b/clients/src/test/java/org/apache/kafka/common/security/authenticator/TestJaasConfig.java index dafa79db98f70..3ee7c2ca4d7b7 100644 --- a/clients/src/test/java/org/apache/kafka/common/security/authenticator/TestJaasConfig.java +++ b/clients/src/test/java/org/apache/kafka/common/security/authenticator/TestJaasConfig.java @@ -28,7 +28,7 @@ import org.apache.kafka.common.config.types.Password; import org.apache.kafka.common.security.plain.PlainLoginModule; import org.apache.kafka.common.security.scram.ScramLoginModule; -import org.apache.kafka.common.security.scram.ScramMechanism; +import org.apache.kafka.common.security.scram.internal.ScramMechanism; public class TestJaasConfig extends Configuration { diff --git a/clients/src/test/java/org/apache/kafka/common/security/plain/PlainSaslServerTest.java b/clients/src/test/java/org/apache/kafka/common/security/plain/internal/PlainSaslServerTest.java similarity index 89% rename from clients/src/test/java/org/apache/kafka/common/security/plain/PlainSaslServerTest.java rename to clients/src/test/java/org/apache/kafka/common/security/plain/internal/PlainSaslServerTest.java index 86baf3e07af23..1410c8accbc7e 100644 --- a/clients/src/test/java/org/apache/kafka/common/security/plain/PlainSaslServerTest.java +++ b/clients/src/test/java/org/apache/kafka/common/security/plain/internal/PlainSaslServerTest.java @@ -14,8 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.kafka.common.security.plain; +package org.apache.kafka.common.security.plain.internal; +import org.apache.kafka.common.security.plain.PlainLoginModule; import org.junit.Before; import org.junit.Test; @@ -46,7 +47,9 @@ public void setUp() throws Exception { options.put("user_" + USER_B, PASSWORD_B); jaasConfig.addEntry("jaasContext", PlainLoginModule.class.getName(), options); JaasContext jaasContext = new JaasContext("jaasContext", JaasContext.Type.SERVER, jaasConfig, null); - saslServer = new PlainSaslServer(jaasContext); + PlainServerCallbackHandler callbackHandler = new PlainServerCallbackHandler(); + callbackHandler.configure(null, "PLAIN", jaasContext.configurationEntries()); + saslServer = new PlainSaslServer(callbackHandler); } @Test diff --git a/clients/src/test/java/org/apache/kafka/common/security/scram/ScramCredentialUtilsTest.java b/clients/src/test/java/org/apache/kafka/common/security/scram/internal/ScramCredentialUtilsTest.java similarity index 97% rename from clients/src/test/java/org/apache/kafka/common/security/scram/ScramCredentialUtilsTest.java rename to clients/src/test/java/org/apache/kafka/common/security/scram/internal/ScramCredentialUtilsTest.java index e9dd285f54e62..a1a1d20d63e52 100644 --- a/clients/src/test/java/org/apache/kafka/common/security/scram/ScramCredentialUtilsTest.java +++ b/clients/src/test/java/org/apache/kafka/common/security/scram/internal/ScramCredentialUtilsTest.java @@ -14,23 +14,23 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.kafka.common.security.scram; +package org.apache.kafka.common.security.scram.internal; -import org.junit.Test; +import java.security.NoSuchAlgorithmException; +import java.util.Arrays; +import org.apache.kafka.common.security.authenticator.CredentialCache; +import org.apache.kafka.common.security.scram.ScramCredential; + +import org.junit.Before; +import org.junit.Test; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; - -import java.security.NoSuchAlgorithmException; -import java.util.Arrays; - import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertTrue; -import org.apache.kafka.common.security.authenticator.CredentialCache; -import org.junit.Before; public class ScramCredentialUtilsTest { diff --git a/clients/src/test/java/org/apache/kafka/common/security/scram/ScramFormatterTest.java b/clients/src/test/java/org/apache/kafka/common/security/scram/internal/ScramFormatterTest.java similarity index 91% rename from clients/src/test/java/org/apache/kafka/common/security/scram/ScramFormatterTest.java rename to clients/src/test/java/org/apache/kafka/common/security/scram/internal/ScramFormatterTest.java index a86e0ddb80079..b06b039768a01 100644 --- a/clients/src/test/java/org/apache/kafka/common/security/scram/ScramFormatterTest.java +++ b/clients/src/test/java/org/apache/kafka/common/security/scram/internal/ScramFormatterTest.java @@ -14,19 +14,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.kafka.common.security.scram; +package org.apache.kafka.common.security.scram.internal; import org.apache.kafka.common.utils.Base64; -import org.junit.Test; +import org.apache.kafka.common.security.scram.internal.ScramMessages.ClientFinalMessage; +import org.apache.kafka.common.security.scram.internal.ScramMessages.ClientFirstMessage; +import org.apache.kafka.common.security.scram.internal.ScramMessages.ServerFinalMessage; +import org.apache.kafka.common.security.scram.internal.ScramMessages.ServerFirstMessage; +import org.junit.Test; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; -import org.apache.kafka.common.security.scram.ScramMessages.ClientFinalMessage; -import org.apache.kafka.common.security.scram.ScramMessages.ClientFirstMessage; -import org.apache.kafka.common.security.scram.ScramMessages.ServerFinalMessage; -import org.apache.kafka.common.security.scram.ScramMessages.ServerFirstMessage; - public class ScramFormatterTest { /** diff --git a/clients/src/test/java/org/apache/kafka/common/security/scram/ScramMessagesTest.java b/clients/src/test/java/org/apache/kafka/common/security/scram/internal/ScramMessagesTest.java similarity index 96% rename from clients/src/test/java/org/apache/kafka/common/security/scram/ScramMessagesTest.java rename to clients/src/test/java/org/apache/kafka/common/security/scram/internal/ScramMessagesTest.java index 7b04ede6e786a..d856f373a7862 100644 --- a/clients/src/test/java/org/apache/kafka/common/security/scram/ScramMessagesTest.java +++ b/clients/src/test/java/org/apache/kafka/common/security/scram/internal/ScramMessagesTest.java @@ -14,29 +14,28 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.kafka.common.security.scram; - -import org.apache.kafka.common.utils.Base64; -import org.junit.Before; -import org.junit.Test; +package org.apache.kafka.common.security.scram.internal; import java.nio.charset.StandardCharsets; import java.util.Collections; import javax.security.sasl.SaslException; +import org.apache.kafka.common.security.scram.internal.ScramMessages.AbstractScramMessage; +import org.apache.kafka.common.security.scram.internal.ScramMessages.ClientFinalMessage; +import org.apache.kafka.common.security.scram.internal.ScramMessages.ClientFirstMessage; +import org.apache.kafka.common.security.scram.internal.ScramMessages.ServerFinalMessage; +import org.apache.kafka.common.security.scram.internal.ScramMessages.ServerFirstMessage; +import org.apache.kafka.common.utils.Base64; + +import org.junit.Before; +import org.junit.Test; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -import org.apache.kafka.common.security.scram.ScramMessages.AbstractScramMessage; -import org.apache.kafka.common.security.scram.ScramMessages.ClientFinalMessage; -import org.apache.kafka.common.security.scram.ScramMessages.ClientFirstMessage; -import org.apache.kafka.common.security.scram.ScramMessages.ServerFinalMessage; -import org.apache.kafka.common.security.scram.ScramMessages.ServerFirstMessage; - public class ScramMessagesTest { private static final String[] VALID_EXTENSIONS = { diff --git a/clients/src/test/java/org/apache/kafka/common/security/scram/ScramSaslServerTest.java b/clients/src/test/java/org/apache/kafka/common/security/scram/internal/ScramSaslServerTest.java similarity index 93% rename from clients/src/test/java/org/apache/kafka/common/security/scram/ScramSaslServerTest.java rename to clients/src/test/java/org/apache/kafka/common/security/scram/internal/ScramSaslServerTest.java index 82ad91415be48..f6e43f9edcff5 100644 --- a/clients/src/test/java/org/apache/kafka/common/security/scram/ScramSaslServerTest.java +++ b/clients/src/test/java/org/apache/kafka/common/security/scram/internal/ScramSaslServerTest.java @@ -14,19 +14,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.kafka.common.security.scram; +package org.apache.kafka.common.security.scram.internal; -import org.apache.kafka.common.security.token.delegation.DelegationTokenCache; -import org.junit.Before; -import org.junit.Test; import java.nio.charset.StandardCharsets; import java.util.HashMap; -import static org.junit.Assert.assertTrue; - import org.apache.kafka.common.errors.SaslAuthenticationException; import org.apache.kafka.common.security.authenticator.CredentialCache; +import org.apache.kafka.common.security.scram.ScramCredential; +import org.apache.kafka.common.security.token.delegation.internal.DelegationTokenCache; + +import org.junit.Before; +import org.junit.Test; +import static org.junit.Assert.assertTrue; public class ScramSaslServerTest { diff --git a/connect/api/src/main/java/org/apache/kafka/connect/data/ConnectSchema.java b/connect/api/src/main/java/org/apache/kafka/connect/data/ConnectSchema.java index 85357fef3c226..ff8271635f304 100644 --- a/connect/api/src/main/java/org/apache/kafka/connect/data/ConnectSchema.java +++ b/connect/api/src/main/java/org/apache/kafka/connect/data/ConnectSchema.java @@ -180,6 +180,7 @@ public List fields() { return fields; } + @Override public Field field(String fieldName) { if (type != Type.STRUCT) throw new DataException("Cannot look up fields on non-struct type"); diff --git a/connect/api/src/main/java/org/apache/kafka/connect/data/SchemaBuilder.java b/connect/api/src/main/java/org/apache/kafka/connect/data/SchemaBuilder.java index a9064b841fabf..fdcf05a7ac055 100644 --- a/connect/api/src/main/java/org/apache/kafka/connect/data/SchemaBuilder.java +++ b/connect/api/src/main/java/org/apache/kafka/connect/data/SchemaBuilder.java @@ -336,12 +336,14 @@ public SchemaBuilder field(String fieldName, Schema fieldSchema) { * Get the list of fields for this Schema. Throws a DataException if this schema is not a struct. * @return the list of fields for this Schema */ + @Override public List fields() { if (type != Type.STRUCT) throw new DataException("Cannot list fields on non-struct type"); return new ArrayList<>(fields.values()); } + @Override public Field field(String fieldName) { if (type != Type.STRUCT) throw new DataException("Cannot look up fields on non-struct type"); diff --git a/connect/api/src/main/java/org/apache/kafka/connect/header/ConnectHeaders.java b/connect/api/src/main/java/org/apache/kafka/connect/header/ConnectHeaders.java index e3b5e72f99e39..185ba65bf0fb6 100644 --- a/connect/api/src/main/java/org/apache/kafka/connect/header/ConnectHeaders.java +++ b/connect/api/src/main/java/org/apache/kafka/connect/header/ConnectHeaders.java @@ -505,6 +505,7 @@ private FilterByKeyIterator(Iterator
original, String key) { this.key = key; } + @Override protected Header makeNext() { while (original.hasNext()) { Header header = original.next(); diff --git a/connect/api/src/main/java/org/apache/kafka/connect/sink/SinkTask.java b/connect/api/src/main/java/org/apache/kafka/connect/sink/SinkTask.java index 1406b30ca8d52..5d308c439625a 100644 --- a/connect/api/src/main/java/org/apache/kafka/connect/sink/SinkTask.java +++ b/connect/api/src/main/java/org/apache/kafka/connect/sink/SinkTask.java @@ -169,5 +169,6 @@ public void onPartitionsRevoked(Collection partitions) { * commit has completed. Implementations of this method should only need to perform final cleanup operations, such * as closing network connections to the sink system. */ + @Override public abstract void stop(); } diff --git a/connect/api/src/main/java/org/apache/kafka/connect/source/SourceTask.java b/connect/api/src/main/java/org/apache/kafka/connect/source/SourceTask.java index 5507f56210f36..8767a62045e4e 100644 --- a/connect/api/src/main/java/org/apache/kafka/connect/source/SourceTask.java +++ b/connect/api/src/main/java/org/apache/kafka/connect/source/SourceTask.java @@ -83,6 +83,7 @@ public void commit() throws InterruptedException { * could set a flag that will force {@link #poll()} to exit immediately and invoke * {@link java.nio.channels.Selector#wakeup() wakeup()} to interrupt any ongoing requests. */ + @Override public abstract void stop(); /** diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/ConnectorConfig.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/ConnectorConfig.java index fd05af57a648c..6a90310df5a82 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/ConnectorConfig.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/ConnectorConfig.java @@ -97,6 +97,7 @@ private static class EnrichedConnectorConfig extends AbstractConfig { super(configDef, props); } + @Override public Object get(String key) { return super.get(key); } diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSourceTask.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSourceTask.java index 6ef5ae3201cb0..f2cef5a63f787 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSourceTask.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/WorkerSourceTask.java @@ -135,6 +135,7 @@ public void initialize(TaskConfig taskConfig) { } } + @Override protected void close() { producer.close(30, TimeUnit.SECONDS); transformationChain.close(); diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedHerder.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedHerder.java index c6e6a4760fbac..5e9707aa29e9c 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedHerder.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/DistributedHerder.java @@ -109,6 +109,8 @@ public class DistributedHerder extends AbstractHerder implements Runnable { private static final Logger log = LoggerFactory.getLogger(DistributedHerder.class); + private static final long FORWARD_REQUEST_SHUTDOWN_TIMEOUT_MS = TimeUnit.SECONDS.toMillis(10); + private static final long START_AND_STOP_SHUTDOWN_TIMEOUT_MS = TimeUnit.SECONDS.toMillis(1); private static final long RECONFIGURE_CONNECTOR_TASKS_BACKOFF_MS = 250; private static final int START_STOP_THREAD_POOL_SIZE = 8; @@ -404,9 +406,9 @@ public void stop() { forwardRequestExecutor.shutdown(); startAndStopExecutor.shutdown(); - if (!forwardRequestExecutor.awaitTermination(10000L, TimeUnit.MILLISECONDS)) + if (!forwardRequestExecutor.awaitTermination(FORWARD_REQUEST_SHUTDOWN_TIMEOUT_MS, TimeUnit.MILLISECONDS)) forwardRequestExecutor.shutdownNow(); - if (!startAndStopExecutor.awaitTermination(1000L, TimeUnit.MILLISECONDS)) + if (!startAndStopExecutor.awaitTermination(START_AND_STOP_SHUTDOWN_TIMEOUT_MS, TimeUnit.MILLISECONDS)) startAndStopExecutor.shutdownNow(); } catch (InterruptedException e) { // ignore diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinator.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinator.java index 600772851f0f4..60407c1d0dcba 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinator.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinator.java @@ -93,6 +93,7 @@ public WorkerCoordinator(LogContext logContext, this.rejoinRequested = false; } + @Override public void requestRejoin() { rejoinRequested = true; } @@ -315,12 +316,14 @@ public WorkerCoordinatorMetrics(Metrics metrics, String metricGrpPrefix) { this.metricGrpName = metricGrpPrefix + "-coordinator-metrics"; Measurable numConnectors = new Measurable() { + @Override public double measure(MetricConfig config, long now) { return assignmentSnapshot.connectors().size(); } }; Measurable numTasks = new Measurable() { + @Override public double measure(MetricConfig config, long now) { return assignmentSnapshot.tasks().size(); } diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/isolation/DelegatingClassLoader.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/isolation/DelegatingClassLoader.java index 37b795137e100..c67dfb5f2de5c 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/isolation/DelegatingClassLoader.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/isolation/DelegatingClassLoader.java @@ -238,6 +238,7 @@ private void loadJdbcDrivers(final ClassLoader loader) { // implementing the java.sql.Driver interface. AccessController.doPrivileged( new PrivilegedAction() { + @Override public Void run() { ServiceLoader loadedDrivers = ServiceLoader.load( Driver.class, diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/RestServer.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/RestServer.java index 77f6cdf59114d..8d7803edb39e2 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/RestServer.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/rest/RestServer.java @@ -62,6 +62,7 @@ public class RestServer { private static final Logger log = LoggerFactory.getLogger(RestServer.class); + private static final Pattern LISTENER_PATTERN = Pattern.compile("^(.*)://\\[?([0-9a-zA-Z\\-%._:]*)\\]?:(-?[0-9]+)"); private static final long GRACEFUL_SHUTDOWN_TIMEOUT_MS = 60 * 1000; private static final String PROTOCOL_HTTP = "http"; @@ -118,8 +119,7 @@ public void createConnectors(List listeners) { * Creates Jetty connector according to configuration */ public Connector createConnector(String listener) { - Pattern listenerPattern = Pattern.compile("^(.*)://\\[?([0-9a-zA-Z\\-%._:]*)\\]?:(-?[0-9]+)"); - Matcher listenerMatcher = listenerPattern.matcher(listener); + Matcher listenerMatcher = LISTENER_PATTERN.matcher(listener); if (!listenerMatcher.matches()) throw new ConfigException("Listener doesn't have the right format (protocol://hostname:port)."); diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/standalone/StandaloneHerder.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/standalone/StandaloneHerder.java index 41609cb054dff..96f8e8767bb7c 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/standalone/StandaloneHerder.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/standalone/StandaloneHerder.java @@ -67,12 +67,14 @@ public StandaloneHerder(Worker worker, String kafkaClusterId) { configBackingStore.setUpdateListener(new ConfigUpdateListener()); } + @Override public synchronized void start() { log.info("Herder starting"); startServices(); log.info("Herder started"); } + @Override public synchronized void stop() { log.info("Herder stopping"); diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/FileOffsetBackingStore.java b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/FileOffsetBackingStore.java index 9961a7cce6172..35e485528a354 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/storage/FileOffsetBackingStore.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/storage/FileOffsetBackingStore.java @@ -88,6 +88,7 @@ private void load() { } } + @Override protected void save() { try (ObjectOutputStream os = new ObjectOutputStream(new FileOutputStream(file))) { Map raw = new HashMap<>(); diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/tools/SchemaSourceTask.java b/connect/runtime/src/main/java/org/apache/kafka/connect/tools/SchemaSourceTask.java index e87851cf291cc..95b49af6155c7 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/tools/SchemaSourceTask.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/tools/SchemaSourceTask.java @@ -81,6 +81,7 @@ public class SchemaSourceTask extends SourceTask { .field("seqno", Schema.INT64_SCHEMA) .build(); + @Override public String version() { return new SchemaSourceConnector().version(); } diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/util/ReflectionsUtil.java b/connect/runtime/src/main/java/org/apache/kafka/connect/util/ReflectionsUtil.java index 61d0e35c6288c..f7dfe7be83123 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/util/ReflectionsUtil.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/util/ReflectionsUtil.java @@ -52,6 +52,7 @@ private EmptyUrlType(final List endings) { this.endings = endings; } + @Override public boolean matches(URL url) { final String protocol = url.getProtocol(); final String externalForm = url.toExternalForm(); @@ -66,6 +67,7 @@ public boolean matches(URL url) { return false; } + @Override public Dir createDir(final URL url) throws Exception { return emptyVfsDir(url); } diff --git a/core/src/main/scala/kafka/admin/AdminClient.scala b/core/src/main/scala/kafka/admin/AdminClient.scala index c010ba0a4ac78..bcc11fd49176c 100644 --- a/core/src/main/scala/kafka/admin/AdminClient.scala +++ b/core/src/main/scala/kafka/admin/AdminClient.scala @@ -35,8 +35,6 @@ import org.apache.kafka.common.requests.ApiVersionsResponse.ApiVersion import org.apache.kafka.common.requests.DescribeGroupsResponse.GroupMetadata import org.apache.kafka.common.requests.OffsetFetchResponse import org.apache.kafka.common.utils.LogContext -import org.apache.kafka.common.security.auth.KafkaPrincipal -import org.apache.kafka.common.security.token.delegation.{DelegationToken, TokenInformation} import org.apache.kafka.common.utils.{KafkaThread, Time, Utils} import org.apache.kafka.common.{Cluster, Node, TopicPartition} @@ -342,33 +340,6 @@ class AdminClient(val time: Time, ConsumerGroupSummary(metadata.state, metadata.protocol, Some(consumers), coordinator) } - def createToken(renewers: List[KafkaPrincipal], maxTimePeriodMs: Long = -1): (Errors, DelegationToken) = { - val responseBody = sendAnyNode(ApiKeys.CREATE_DELEGATION_TOKEN, new CreateDelegationTokenRequest.Builder(renewers.asJava, maxTimePeriodMs)) - val response = responseBody.asInstanceOf[CreateDelegationTokenResponse] - val tokenInfo = new TokenInformation(response.tokenId, response.owner, renewers.asJava, - response.issueTimestamp, response.maxTimestamp, response.expiryTimestamp) - (response.error, new DelegationToken(tokenInfo, response.hmacBytes)) - } - - def renewToken(hmac: ByteBuffer, renewTimePeriod: Long = -1): (Errors, Long) = { - val responseBody = sendAnyNode(ApiKeys.RENEW_DELEGATION_TOKEN, new RenewDelegationTokenRequest.Builder(hmac, renewTimePeriod)) - val response = responseBody.asInstanceOf[RenewDelegationTokenResponse] - (response.error, response.expiryTimestamp) - } - - def expireToken(hmac: ByteBuffer, expiryTimeStamp: Long = -1): (Errors, Long) = { - val responseBody = sendAnyNode(ApiKeys.EXPIRE_DELEGATION_TOKEN, new ExpireDelegationTokenRequest.Builder(hmac, expiryTimeStamp)) - val response = responseBody.asInstanceOf[ExpireDelegationTokenResponse] - (response.error, response.expiryTimestamp) - } - - def describeToken(owners: List[KafkaPrincipal]): (Errors, List[DelegationToken]) = { - val ownersList = if (owners == null) null else owners.asJava - val responseBody = sendAnyNode(ApiKeys.RENEW_DELEGATION_TOKEN, new DescribeDelegationTokenRequest.Builder(ownersList)) - val response = responseBody.asInstanceOf[DescribeDelegationTokenResponse] - (response.error, response.tokens().asScala.toList) - } - def deleteConsumerGroups(groups: List[String]): Map[String, Errors] = { def coordinatorLookup(group: String): Either[Node, Errors] = { diff --git a/core/src/main/scala/kafka/admin/ConfigCommand.scala b/core/src/main/scala/kafka/admin/ConfigCommand.scala index 3563448feb616..c19599d3cc0ef 100644 --- a/core/src/main/scala/kafka/admin/ConfigCommand.scala +++ b/core/src/main/scala/kafka/admin/ConfigCommand.scala @@ -32,7 +32,7 @@ import org.apache.kafka.clients.CommonClientConfigs import org.apache.kafka.clients.admin.{AlterConfigsOptions, ConfigEntry, DescribeConfigsOptions, AdminClient => JAdminClient, Config => JConfig} import org.apache.kafka.common.config.ConfigResource import org.apache.kafka.common.security.JaasUtils -import org.apache.kafka.common.security.scram._ +import org.apache.kafka.common.security.scram.internal.{ScramCredentialUtils, ScramFormatter, ScramMechanism} import org.apache.kafka.common.utils.{Sanitizer, Time, Utils} import scala.collection._ diff --git a/core/src/main/scala/kafka/admin/DelegationTokenCommand.scala b/core/src/main/scala/kafka/admin/DelegationTokenCommand.scala index 6c5d1ce925df5..0e6ea86034b28 100644 --- a/core/src/main/scala/kafka/admin/DelegationTokenCommand.scala +++ b/core/src/main/scala/kafka/admin/DelegationTokenCommand.scala @@ -17,12 +17,13 @@ package kafka.admin -import java.nio.ByteBuffer +import java.text.SimpleDateFormat +import java.util -import joptsimple._ +import joptsimple.{ArgumentAcceptingOptionSpec, OptionParser} import kafka.utils.{CommandLineUtils, Exit, Logging} import org.apache.kafka.clients.CommonClientConfigs -import org.apache.kafka.common.protocol.Errors +import org.apache.kafka.clients.admin.{CreateDelegationTokenOptions, DescribeDelegationTokenOptions, ExpireDelegationTokenOptions, RenewDelegationTokenOptions, AdminClient => JAdminClient} import org.apache.kafka.common.security.auth.KafkaPrincipal import org.apache.kafka.common.security.token.delegation.DelegationToken import org.apache.kafka.common.utils.{Base64, SecurityUtils, Utils} @@ -71,19 +72,20 @@ object DelegationTokenCommand extends Logging { } } - def createToken(adminClient: AdminClient, opts: DelegationTokenCommandOptions) = { - val renewerPrincipals = getPrincipals(opts, opts.renewPrincipalsOpt) + def createToken(adminClient: JAdminClient, opts: DelegationTokenCommandOptions): DelegationToken = { + val renewerPrincipals = getPrincipals(opts, opts.renewPrincipalsOpt).getOrElse(new util.LinkedList[KafkaPrincipal]()) val maxLifeTimeMs = opts.options.valueOf(opts.maxLifeTimeOpt).longValue println("Calling create token operation with renewers :" + renewerPrincipals +" , max-life-time-period :"+ maxLifeTimeMs) - val response = adminClient.createToken(renewerPrincipals, maxLifeTimeMs) - response match { - case (Errors.NONE, token) => println("Created delegation token with tokenId : %s".format(token.tokenInfo.tokenId)); printToken(List(token)) - case (e, _) => throw new AdminOperationException(e.message) - } + val createDelegationTokenOptions = new CreateDelegationTokenOptions().maxlifeTimeMs(maxLifeTimeMs).renewers(renewerPrincipals) + val createResult = adminClient.createDelegationToken(createDelegationTokenOptions) + val token = createResult.delegationToken().get() + println("Created delegation token with tokenId : %s".format(token.tokenInfo.tokenId)); printToken(List(token)) + token } def printToken(tokens: List[DelegationToken]): Unit = { + val dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm") print("\n%-15s %-30s %-15s %-25s %-15s %-15s %-15s".format("TOKENID", "HMAC", "OWNER", "RENEWERS", "ISSUEDATE", "EXPIRYDATE", "MAXDATE")) for (token <- tokens) { val tokenInfo = token.tokenInfo @@ -92,56 +94,59 @@ object DelegationTokenCommand extends Logging { token.hmacAsBase64String, tokenInfo.owner, tokenInfo.renewersAsString, - tokenInfo.issueTimestamp, - tokenInfo.expiryTimestamp, - tokenInfo.maxTimestamp)) + dateFormat.format(tokenInfo.issueTimestamp), + dateFormat.format(tokenInfo.expiryTimestamp), + dateFormat.format(tokenInfo.maxTimestamp))) println() } } - private def getPrincipals(opts: DelegationTokenCommandOptions, principalOptionSpec: ArgumentAcceptingOptionSpec[String]): List[KafkaPrincipal] = { + private def getPrincipals(opts: DelegationTokenCommandOptions, principalOptionSpec: ArgumentAcceptingOptionSpec[String]): Option[util.List[KafkaPrincipal]] = { if (opts.options.has(principalOptionSpec)) - opts.options.valuesOf(principalOptionSpec).asScala.map(s => SecurityUtils.parseKafkaPrincipal(s.trim)).toList + Some(opts.options.valuesOf(principalOptionSpec).asScala.map(s => SecurityUtils.parseKafkaPrincipal(s.trim)).toList.asJava) else - List.empty[KafkaPrincipal] + None } - def renewToken(adminClient: AdminClient, opts: DelegationTokenCommandOptions) = { + def renewToken(adminClient: JAdminClient, opts: DelegationTokenCommandOptions): Long = { val hmac = opts.options.valueOf(opts.hmacOpt) val renewTimePeriodMs = opts.options.valueOf(opts.renewTimePeriodOpt).longValue() println("Calling renew token operation with hmac :" + hmac +" , renew-time-period :"+ renewTimePeriodMs) - val response = adminClient.renewToken(ByteBuffer.wrap(Base64.decoder.decode(hmac)), renewTimePeriodMs) - response match { - case (Errors.NONE, expiryTimeStamp) => println("Completed renew operation. New expiry timestamp : %s".format(expiryTimeStamp)) - case (e, expiryTimeStamp) => throw new AdminOperationException(e.message) - } + val renewResult = adminClient.renewDelegationToken(Base64.decoder.decode(hmac), new RenewDelegationTokenOptions().renewTimePeriodMs(renewTimePeriodMs)) + val expiryTimeStamp = renewResult.expiryTimestamp().get() + val dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm") + println("Completed renew operation. New expiry date : %s".format(dateFormat.format(expiryTimeStamp))) + expiryTimeStamp } - def expireToken(adminClient: AdminClient, opts: DelegationTokenCommandOptions) = { + def expireToken(adminClient: JAdminClient, opts: DelegationTokenCommandOptions): Long = { val hmac = opts.options.valueOf(opts.hmacOpt) val expiryTimePeriodMs = opts.options.valueOf(opts.expiryTimePeriodOpt).longValue() println("Calling expire token operation with hmac :" + hmac +" , expire-time-period : "+ expiryTimePeriodMs) - val response = adminClient.expireToken(ByteBuffer.wrap(Base64.decoder.decode(hmac)), expiryTimePeriodMs) - response match { - case (Errors.NONE, expiryTimeStamp) => println("Completed expire operation. New expiry timestamp : %s".format(expiryTimeStamp)) - case (e, expiryTimeStamp) => throw new AdminOperationException(e.message) - } + val expireResult = adminClient.expireDelegationToken(Base64.decoder.decode(hmac), new ExpireDelegationTokenOptions().expiryTimePeriodMs(expiryTimePeriodMs)) + val expiryTimeStamp = expireResult.expiryTimestamp().get() + val dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm") + println("Completed expire operation. New expiry date : %s".format(dateFormat.format(expiryTimeStamp))) + expiryTimeStamp } - def describeToken(adminClient: AdminClient, opts: DelegationTokenCommandOptions) = { + def describeToken(adminClient: JAdminClient, opts: DelegationTokenCommandOptions): List[DelegationToken] = { val ownerPrincipals = getPrincipals(opts, opts.ownerPrincipalsOpt) - println("Calling describe token operation for owners :" + ownerPrincipals) - val response = adminClient.describeToken(ownerPrincipals) - response match { - case (Errors.NONE, tokens) => println("Total Number of tokens : %s".format(tokens.size)); printToken(tokens) - case (e, tokens) => throw new AdminOperationException(e.message) - } + if (ownerPrincipals.isEmpty) + println("Calling describe token operation for current user.") + else + println("Calling describe token operation for owners :" + ownerPrincipals.get) + + val describeResult = adminClient.describeDelegationToken(new DescribeDelegationTokenOptions().owners(ownerPrincipals.orNull)) + val tokens = describeResult.delegationTokens().get().asScala.toList + println("Total number of tokens : %s".format(tokens.size)); printToken(tokens) + tokens } - private def createAdminClient(opts: DelegationTokenCommandOptions): AdminClient = { + private def createAdminClient(opts: DelegationTokenCommandOptions): JAdminClient = { val props = Utils.loadProps(opts.options.valueOf(opts.commandConfigOpt)) props.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, opts.options.valueOf(opts.bootstrapServerOpt)) - AdminClient.create(props) + JAdminClient.create(props) } class DelegationTokenCommandOptions(args: Array[String]) { @@ -157,10 +162,11 @@ object DelegationTokenCommand extends Logging { .withRequiredArg .ofType(classOf[String]) - val createOpt = parser.accepts("create", "Create a new delegation token.") - val renewOpt = parser.accepts("renew", "Renew delegation token.") - val expiryOpt = parser.accepts("expire", "Expire delegation token.") - val describeOpt = parser.accepts("describe", "describe delegation tokens.") + val createOpt = parser.accepts("create", "Create a new delegation token. Use --renewer-principal option to pass renewers principals.") + val renewOpt = parser.accepts("renew", "Renew delegation token. Use --renew-time-period option to set renew time period.") + val expiryOpt = parser.accepts("expire", "Expire delegation token. Use --expiry-time-period option to expire the token.") + val describeOpt = parser.accepts("describe", "Describe delegation tokens for the given principals. Use --owner-principal to pass owner/renewer principals." + + " If --owner-principal option is not supplied, all the user owned tokens and tokens where user have Describe permission will be returned.") val ownerPrincipalsOpt = parser.accepts("owner-principal", "owner is a kafka principal. It is should be in principalType:name format.") .withOptionalArg() diff --git a/core/src/main/scala/kafka/cluster/Partition.scala b/core/src/main/scala/kafka/cluster/Partition.scala index 6eb9611c805a4..93377bad00ede 100755 --- a/core/src/main/scala/kafka/cluster/Partition.scala +++ b/core/src/main/scala/kafka/cluster/Partition.scala @@ -261,7 +261,8 @@ class Partition(val topic: String, (assignedReplicas.map(_.brokerId) -- newAssignedReplicas).foreach(removeReplica) inSyncReplicas = newInSyncReplicas - info(s"$topicPartition starts at Leader Epoch ${partitionStateInfo.basePartitionState.leaderEpoch} from offset ${getReplica().get.logEndOffset.messageOffset}. Previous Leader Epoch was: $leaderEpoch") + info(s"$topicPartition starts at Leader Epoch ${partitionStateInfo.basePartitionState.leaderEpoch} from " + + s"offset ${getReplica().get.logEndOffset.messageOffset}. Previous Leader Epoch was: $leaderEpoch") //We cache the leader epoch here, persisting it only if it's local (hence having a log dir) leaderEpoch = partitionStateInfo.basePartitionState.leaderEpoch diff --git a/core/src/main/scala/kafka/controller/KafkaController.scala b/core/src/main/scala/kafka/controller/KafkaController.scala index 4778a7a6e891e..eee625ef66317 100644 --- a/core/src/main/scala/kafka/controller/KafkaController.scala +++ b/core/src/main/scala/kafka/controller/KafkaController.scala @@ -873,7 +873,7 @@ class KafkaController(val config: KafkaConfig, zkClient: KafkaZkClient, time: Ti } private def removePartitionsFromPreferredReplicaElection(partitionsToBeRemoved: Set[TopicPartition], - isTriggeredByAutoRebalance : Boolean) { + isTriggeredByAutoRebalance : Boolean) { for (partition <- partitionsToBeRemoved) { // check the status val currentLeader = controllerContext.partitionLeadershipInfo(partition).leaderAndIsr.leader @@ -881,7 +881,8 @@ class KafkaController(val config: KafkaConfig, zkClient: KafkaZkClient, time: Ti if (currentLeader == preferredReplica) { info(s"Partition $partition completed preferred replica leader election. New leader is $preferredReplica") } else { - warn(s"Partition $partition failed to complete preferred replica leader election. Leader is $currentLeader") + warn(s"Partition $partition failed to complete preferred replica leader election to $preferredReplica. " + + s"Leader is still $currentLeader") } } if (!isTriggeredByAutoRebalance) { diff --git a/core/src/main/scala/kafka/controller/PartitionStateMachine.scala b/core/src/main/scala/kafka/controller/PartitionStateMachine.scala index 74bc59faee2a5..6805e32139398 100755 --- a/core/src/main/scala/kafka/controller/PartitionStateMachine.scala +++ b/core/src/main/scala/kafka/controller/PartitionStateMachine.scala @@ -362,7 +362,7 @@ class PartitionStateMachine(config: KafkaConfig, if (leaderIsrAndControllerEpochOpt.nonEmpty) { val leaderIsrAndControllerEpoch = leaderIsrAndControllerEpochOpt.get val isr = leaderIsrAndControllerEpoch.leaderAndIsr.isr - val leaderOpt = PartitionLeaderElectionAlgorithms.offlinePartitionLeaderElection(assignment, isr, liveReplicas.toSet, uncleanLeaderElectionEnabled) + val leaderOpt = PartitionLeaderElectionAlgorithms.offlinePartitionLeaderElection(assignment, isr, liveReplicas.toSet, uncleanLeaderElectionEnabled, controllerContext) val newLeaderAndIsrOpt = leaderOpt.map { leader => val newIsr = if (isr.contains(leader)) isr.filter(replica => controllerContext.isReplicaOnline(replica, partition)) else List(leader) @@ -435,10 +435,13 @@ class PartitionStateMachine(config: KafkaConfig, } object PartitionLeaderElectionAlgorithms { - def offlinePartitionLeaderElection(assignment: Seq[Int], isr: Seq[Int], liveReplicas: Set[Int], uncleanLeaderElectionEnabled: Boolean): Option[Int] = { + def offlinePartitionLeaderElection(assignment: Seq[Int], isr: Seq[Int], liveReplicas: Set[Int], uncleanLeaderElectionEnabled: Boolean, controllerContext: ControllerContext): Option[Int] = { assignment.find(id => liveReplicas.contains(id) && isr.contains(id)).orElse { if (uncleanLeaderElectionEnabled) { - assignment.find(liveReplicas.contains) + val leaderOpt = assignment.find(liveReplicas.contains) + if (!leaderOpt.isEmpty) + controllerContext.stats.uncleanLeaderElectionRate.mark() + leaderOpt } else { None } diff --git a/core/src/main/scala/kafka/security/CredentialProvider.scala b/core/src/main/scala/kafka/security/CredentialProvider.scala index 0e7ebb672b153..f20808791a211 100644 --- a/core/src/main/scala/kafka/security/CredentialProvider.scala +++ b/core/src/main/scala/kafka/security/CredentialProvider.scala @@ -20,10 +20,11 @@ package kafka.security import java.util.{Collection, Properties} import org.apache.kafka.common.security.authenticator.CredentialCache -import org.apache.kafka.common.security.scram.{ScramCredential, ScramCredentialUtils, ScramMechanism} +import org.apache.kafka.common.security.scram.ScramCredential import org.apache.kafka.common.config.ConfigDef import org.apache.kafka.common.config.ConfigDef._ -import org.apache.kafka.common.security.token.delegation.DelegationTokenCache +import org.apache.kafka.common.security.scram.internal.{ScramCredentialUtils, ScramMechanism} +import org.apache.kafka.common.security.token.delegation.internal.DelegationTokenCache class CredentialProvider(scramMechanisms: Collection[String], val tokenCache: DelegationTokenCache) { diff --git a/core/src/main/scala/kafka/server/ClientQuotaManager.scala b/core/src/main/scala/kafka/server/ClientQuotaManager.scala index 8ec27a3a4c03b..0f8690fd8b9a9 100644 --- a/core/src/main/scala/kafka/server/ClientQuotaManager.scala +++ b/core/src/main/scala/kafka/server/ClientQuotaManager.scala @@ -16,24 +16,29 @@ */ package kafka.server +import java.{lang, util} import java.util.concurrent.{ConcurrentHashMap, DelayQueue, TimeUnit} import java.util.concurrent.locks.ReentrantReadWriteLock +import kafka.network.RequestChannel.Session +import kafka.server.ClientQuotaManager._ import kafka.utils.{Logging, ShutdownableThread} -import org.apache.kafka.common.MetricName +import org.apache.kafka.common.{Cluster, MetricName} import org.apache.kafka.common.metrics._ import org.apache.kafka.common.metrics.stats.{Avg, Rate, Total} +import org.apache.kafka.common.security.auth.KafkaPrincipal import org.apache.kafka.common.utils.{Sanitizer, Time} +import org.apache.kafka.server.quota.{ClientQuotaCallback, ClientQuotaEntity, ClientQuotaType} import scala.collection.JavaConverters._ /** * Represents the sensors aggregated per client - * @param quotaEntity Quota entity representing , or + * @param metricTags Quota metric tags for the client * @param quotaSensor @Sensor that tracks the quota * @param throttleTimeSensor @Sensor that tracks the throttle time */ -case class ClientSensors(quotaEntity: QuotaEntity, quotaSensor: Sensor, throttleTimeSensor: Sensor) +case class ClientSensors(metricTags: Map[String, String], quotaSensor: Sensor, throttleTimeSensor: Sensor) /** * Configuration settings for quota management @@ -61,9 +66,6 @@ object ClientQuotaManagerConfig { val NanosToPercentagePerSecond = 100.0 / TimeUnit.SECONDS.toNanos(1) val UnlimitedQuota = Quota.upperBound(Long.MaxValue) - val DefaultClientIdQuotaId = QuotaId(None, Some(ConfigEntityName.Default), Some(ConfigEntityName.Default)) - val DefaultUserQuotaId = QuotaId(Some(ConfigEntityName.Default), None, None) - val DefaultUserClientIdQuotaId = QuotaId(Some(ConfigEntityName.Default), Some(ConfigEntityName.Default), Some(ConfigEntityName.Default)) } object QuotaTypes { @@ -71,11 +73,60 @@ object QuotaTypes { val ClientIdQuotaEnabled = 1 val UserQuotaEnabled = 2 val UserClientIdQuotaEnabled = 4 + val CustomQuotas = 8 // No metric update optimizations are used with custom quotas } -case class QuotaId(sanitizedUser: Option[String], clientId: Option[String], sanitizedClientId: Option[String]) +object ClientQuotaManager { + val DefaultClientIdQuotaEntity = KafkaQuotaEntity(None, Some(DefaultClientIdEntity)) + val DefaultUserQuotaEntity = KafkaQuotaEntity(Some(DefaultUserEntity), None) + val DefaultUserClientIdQuotaEntity = KafkaQuotaEntity(Some(DefaultUserEntity), Some(DefaultClientIdEntity)) -case class QuotaEntity(quotaId: QuotaId, sanitizedUser: String, clientId: String, sanitizedClientId: String, quota: Quota) + case class UserEntity(sanitizedUser: String) extends ClientQuotaEntity.ConfigEntity { + override def entityType: ClientQuotaEntity.ConfigEntityType = ClientQuotaEntity.ConfigEntityType.USER + override def name: String = Sanitizer.desanitize(sanitizedUser) + override def toString: String = s"user $sanitizedUser" + } + + case class ClientIdEntity(clientId: String) extends ClientQuotaEntity.ConfigEntity { + override def entityType: ClientQuotaEntity.ConfigEntityType = ClientQuotaEntity.ConfigEntityType.CLIENT_ID + override def name: String = clientId + override def toString: String = s"client-id $clientId" + } + + case object DefaultUserEntity extends ClientQuotaEntity.ConfigEntity { + override def entityType: ClientQuotaEntity.ConfigEntityType = ClientQuotaEntity.ConfigEntityType.DEFAULT_USER + override def name: String = ConfigEntityName.Default + override def toString: String = "default user" + } + + case object DefaultClientIdEntity extends ClientQuotaEntity.ConfigEntity { + override def entityType: ClientQuotaEntity.ConfigEntityType = ClientQuotaEntity.ConfigEntityType.DEFAULT_CLIENT_ID + override def name: String = ConfigEntityName.Default + override def toString: String = "default client-id" + } + + case class KafkaQuotaEntity(userEntity: Option[ClientQuotaEntity.ConfigEntity], + clientIdEntity: Option[ClientQuotaEntity.ConfigEntity]) extends ClientQuotaEntity { + override def configEntities: util.List[ClientQuotaEntity.ConfigEntity] = + (userEntity.toList ++ clientIdEntity.toList).asJava + def sanitizedUser: String = userEntity.map { + case entity: UserEntity => entity.sanitizedUser + case DefaultUserEntity => ConfigEntityName.Default + }.getOrElse("") + def clientId: String = clientIdEntity.map(_.name).getOrElse("") + + override def toString: String = { + val user = userEntity.map(_.toString).getOrElse("") + val clientId = clientIdEntity.map(_.toString).getOrElse("") + s"$user $clientId".trim + } + } + + object DefaultTags { + val User = "user" + val ClientId = "client-id" + } +} /** * Helper class that records per-client metrics. It is also responsible for maintaining Quota usage statistics @@ -107,21 +158,26 @@ class ClientQuotaManager(private val config: ClientQuotaManagerConfig, private val metrics: Metrics, private val quotaType: QuotaType, private val time: Time, - threadNamePrefix: String) extends Logging { - private val overriddenQuota = new ConcurrentHashMap[QuotaId, Quota]() + threadNamePrefix: String, + clientQuotaCallback: Option[ClientQuotaCallback] = None) extends Logging { private val staticConfigClientIdQuota = Quota.upperBound(config.quotaBytesPerSecondDefault) - @volatile private var quotaTypesEnabled = - if (config.quotaBytesPerSecondDefault == Long.MaxValue) QuotaTypes.NoQuotas - else QuotaTypes.ClientIdQuotaEnabled + private val clientQuotaType = quotaTypeToClientQuotaType(quotaType) + @volatile private var quotaTypesEnabled = clientQuotaCallback match { + case Some(_) => QuotaTypes.CustomQuotas + case None => + if (config.quotaBytesPerSecondDefault == Long.MaxValue) QuotaTypes.NoQuotas + else QuotaTypes.ClientIdQuotaEnabled + } private val lock = new ReentrantReadWriteLock() private val delayQueue = new DelayQueue[ThrottledResponse]() private val sensorAccessor = new SensorAccess(lock, metrics) private[server] val throttledRequestReaper = new ThrottledRequestReaper(delayQueue, threadNamePrefix) + private val quotaCallback = clientQuotaCallback.getOrElse(new DefaultQuotaCallback) private val delayQueueSensor = metrics.sensor(quotaType + "-delayQueue") delayQueueSensor.add(metrics.metricName("queue-size", - quotaType.toString, - "Tracks the size of the delay queue"), new Total()) + quotaType.toString, + "Tracks the size of the delay queue"), new Total()) start() // Use start method to keep findbugs happy private def start() { throttledRequestReaper.start() @@ -132,7 +188,7 @@ class ClientQuotaManager(private val config: ClientQuotaManagerConfig, * @param delayQueue DelayQueue to dequeue from */ class ThrottledRequestReaper(delayQueue: DelayQueue[ThrottledResponse], prefix: String) extends ShutdownableThread( - s"${prefix}ThrottledRequestReaper-${quotaType}", false) { + s"${prefix}ThrottledRequestReaper-$quotaType", false) { override def doWork(): Unit = { val response: ThrottledResponse = delayQueue.poll(1, TimeUnit.SECONDS) @@ -158,17 +214,18 @@ class ClientQuotaManager(private val config: ClientQuotaManagerConfig, * Records that a user/clientId changed some metric being throttled (produced/consumed bytes, request processing time etc.) * If quota has been violated, callback is invoked after a delay, otherwise the callback is invoked immediately. * Throttle time calculation may be overridden by sub-classes. - * @param sanitizedUser user principal of client + * + * @param session the session associated with this request * @param clientId clientId that produced/fetched the data - * @param value amount of data in bytes or request processing time as a percentage + * @param value amount of data in bytes or request processing time as a percentage * @param callback Callback function. This will be triggered immediately if quota is not violated. * If there is a quota violation, this callback will be triggered after a delay * @return Number of milliseconds to delay the response in case of Quota violation. * Zero otherwise */ - def maybeRecordAndThrottle(sanitizedUser: String, clientId: String, value: Double, callback: Int => Unit): Int = { + def maybeRecordAndThrottle(session: Session, clientId: String, value: Double, callback: Int => Unit): Int = { if (quotasEnabled) { - val clientSensors = getOrCreateQuotaSensors(sanitizedUser, clientId) + val clientSensors = getOrCreateQuotaSensors(session, clientId) recordAndThrottleOnQuotaViolation(clientSensors, value, callback) } else { // Don't record any metrics if quotas are not enabled at any level @@ -187,9 +244,8 @@ class ClientQuotaManager(private val config: ClientQuotaManagerConfig, } catch { case _: QuotaViolationException => // Compute the delay - val clientQuotaEntity = clientSensors.quotaEntity - val clientMetric = metrics.metrics().get(clientRateMetricName(clientQuotaEntity.sanitizedUser, clientQuotaEntity.clientId)) - throttleTimeMs = throttleTime(clientMetric, getQuotaMetricConfig(clientQuotaEntity.quota)).toInt + val clientMetric = metrics.metrics().get(clientRateMetricName(clientSensors.metricTags)) + throttleTimeMs = throttleTime(clientMetric).toInt clientSensors.throttleTimeSensor.record(throttleTimeMs) // If delayed, add the element to the delayQueue delayQueue.add(new ThrottledResponse(time, throttleTimeMs, callback)) @@ -209,126 +265,27 @@ class ClientQuotaManager(private val config: ClientQuotaManagerConfig, } /** - * Determines the quota-id for the client with the specified user principal - * and client-id and returns the quota entity that encapsulates the quota-id - * and the associated quota override or default quota. + * Returns the quota for the client with the specified (non-encoded) user principal and client-id. * + * Note: this method is expensive, it is meant to be used by tests only */ - private def quotaEntity(sanitizedUser: String, clientId: String, sanitizedClientId: String) : QuotaEntity = { - quotaTypesEnabled match { - case QuotaTypes.NoQuotas | QuotaTypes.ClientIdQuotaEnabled => - val quotaId = QuotaId(None, Some(clientId), Some(sanitizedClientId)) - var quota = overriddenQuota.get(quotaId) - if (quota == null) { - quota = overriddenQuota.get(ClientQuotaManagerConfig.DefaultClientIdQuotaId) - if (quota == null) - quota = staticConfigClientIdQuota - } - QuotaEntity(quotaId, "", clientId, sanitizedClientId, quota) - case QuotaTypes.UserQuotaEnabled => - val quotaId = QuotaId(Some(sanitizedUser), None, None) - var quota = overriddenQuota.get(quotaId) - if (quota == null) { - quota = overriddenQuota.get(ClientQuotaManagerConfig.DefaultUserQuotaId) - if (quota == null) - quota = ClientQuotaManagerConfig.UnlimitedQuota - } - QuotaEntity(quotaId, sanitizedUser, "", "", quota) - case QuotaTypes.UserClientIdQuotaEnabled => - val quotaId = QuotaId(Some(sanitizedUser), Some(clientId), Some(sanitizedClientId)) - var quota = overriddenQuota.get(quotaId) - if (quota == null) { - quota = overriddenQuota.get(QuotaId(Some(sanitizedUser), Some(ConfigEntityName.Default), Some(ConfigEntityName.Default))) - if (quota == null) { - quota = overriddenQuota.get(QuotaId(Some(ConfigEntityName.Default), Some(clientId), Some(sanitizedClientId))) - if (quota == null) { - quota = overriddenQuota.get(ClientQuotaManagerConfig.DefaultUserClientIdQuotaId) - if (quota == null) - quota = ClientQuotaManagerConfig.UnlimitedQuota - } - } - } - QuotaEntity(quotaId, sanitizedUser, clientId, sanitizedClientId, quota) - case _ => - quotaEntityWithMultipleQuotaLevels(sanitizedUser, clientId, sanitizedClientId) - } - } - - private def quotaEntityWithMultipleQuotaLevels(sanitizedUser: String, clientId: String, sanitizerClientId: String) : QuotaEntity = { - val userClientQuotaId = QuotaId(Some(sanitizedUser), Some(clientId), Some(sanitizerClientId)) - - val userQuotaId = QuotaId(Some(sanitizedUser), None, None) - val clientQuotaId = QuotaId(None, Some(clientId), Some(sanitizerClientId)) - var quotaId = userClientQuotaId - var quotaConfigId = userClientQuotaId - // 1) /config/users//clients/ - var quota = overriddenQuota.get(quotaConfigId) - if (quota == null) { - // 2) /config/users//clients/ - quotaId = userClientQuotaId - quotaConfigId = QuotaId(Some(sanitizedUser), Some(ConfigEntityName.Default), Some(ConfigEntityName.Default)) - quota = overriddenQuota.get(quotaConfigId) - - if (quota == null) { - // 3) /config/users/ - quotaId = userQuotaId - quotaConfigId = quotaId - quota = overriddenQuota.get(quotaConfigId) - - if (quota == null) { - // 4) /config/users//clients/ - quotaId = userClientQuotaId - quotaConfigId = QuotaId(Some(ConfigEntityName.Default), Some(clientId), Some(sanitizerClientId)) - quota = overriddenQuota.get(quotaConfigId) - - if (quota == null) { - // 5) /config/users//clients/ - quotaId = userClientQuotaId - quotaConfigId = QuotaId(Some(ConfigEntityName.Default), Some(ConfigEntityName.Default), Some(ConfigEntityName.Default)) - quota = overriddenQuota.get(quotaConfigId) - - if (quota == null) { - // 6) /config/users/ - quotaId = userQuotaId - quotaConfigId = QuotaId(Some(ConfigEntityName.Default), None, None) - quota = overriddenQuota.get(quotaConfigId) - - if (quota == null) { - // 7) /config/clients/ - quotaId = clientQuotaId - quotaConfigId = QuotaId(None, Some(clientId), Some(sanitizerClientId)) - quota = overriddenQuota.get(quotaConfigId) - - if (quota == null) { - // 8) /config/clients/ - quotaId = clientQuotaId - quotaConfigId = QuotaId(None, Some(ConfigEntityName.Default), Some(ConfigEntityName.Default)) - quota = overriddenQuota.get(quotaConfigId) - - if (quota == null) { - quotaId = clientQuotaId - quotaConfigId = null - quota = staticConfigClientIdQuota - } - } - } - } - } - } - } - } - val quotaUser = if (quotaId == clientQuotaId) "" else sanitizedUser - val quotaClientId = if (quotaId == userQuotaId) "" else clientId - QuotaEntity(quotaId, quotaUser, quotaClientId, sanitizerClientId, quota) + def quota(user: String, clientId: String): Quota = { + val userPrincipal = new KafkaPrincipal(KafkaPrincipal.USER_TYPE, user) + quota(userPrincipal, clientId) } /** - * Returns the quota for the client with the specified (non-encoded) user principal and client-id. - * + * Returns the quota for the client with the specified user principal and client-id. + * * Note: this method is expensive, it is meant to be used by tests only */ - def quota(user: String, clientId: String) = { - quotaEntity(Sanitizer.sanitize(user), clientId, Sanitizer.sanitize(clientId)).quota + def quota(userPrincipal: KafkaPrincipal, clientId: String): Quota = { + val metricTags = quotaCallback.quotaMetricTags(clientQuotaType, userPrincipal, clientId) + Quota.upperBound(quotaLimit(metricTags)) + } + + private def quotaLimit(metricTags: util.Map[String, String]): Double = { + Option(quotaCallback.quotaLimit(clientQuotaType, metricTags)).map(_.toDouble)getOrElse(Long.MaxValue) } /* @@ -339,10 +296,11 @@ class ClientQuotaManager(private val config: ClientQuotaManagerConfig, * we need to add a delay of X to W such that O * W / (W + X) = T. * Solving for X, we get X = (O - T)/T * W. */ - protected def throttleTime(clientMetric: KafkaMetric, config: MetricConfig): Long = { + protected def throttleTime(clientMetric: KafkaMetric): Long = { + val config = clientMetric.config val rateMetric: Rate = measurableAsRate(clientMetric.metricName(), clientMetric.measurable()) val quota = config.quota() - val difference = clientMetric.value() - quota.bound + val difference = clientMetric.metricValue.asInstanceOf[Double] - quota.bound // Use the precise window used by the rate calculation val throttleTimeMs = difference / quota.bound * rateMetric.windowSize(config, time.milliseconds()) throttleTimeMs.round @@ -360,56 +318,72 @@ class ClientQuotaManager(private val config: ClientQuotaManagerConfig, * This function either returns the sensors for a given client id or creates them if they don't exist * First sensor of the tuple is the quota enforcement sensor. Second one is the throttle time sensor */ - def getOrCreateQuotaSensors(sanitizedUser: String, clientId: String): ClientSensors = { - val sanitizedClientId = Sanitizer.sanitize(clientId) - val clientQuotaEntity = quotaEntity(sanitizedUser, clientId, sanitizedClientId) + def getOrCreateQuotaSensors(session: Session, clientId: String): ClientSensors = { + // Use cached sanitized principal if using default callback + val metricTags = quotaCallback match { + case callback: DefaultQuotaCallback => callback.quotaMetricTags(session.sanitizedUser, clientId) + case _ => quotaCallback.quotaMetricTags(clientQuotaType, session.principal, clientId).asScala.toMap + } // Names of the sensors to access - ClientSensors( - clientQuotaEntity, + val sensors = ClientSensors( + metricTags, sensorAccessor.getOrCreate( - getQuotaSensorName(clientQuotaEntity.quotaId), + getQuotaSensorName(metricTags), ClientQuotaManagerConfig.InactiveSensorExpirationTimeSeconds, - clientRateMetricName(clientQuotaEntity.sanitizedUser, clientQuotaEntity.clientId), - Some(getQuotaMetricConfig(clientQuotaEntity.quota)), + clientRateMetricName(metricTags), + Some(getQuotaMetricConfig(metricTags)), new Rate ), - sensorAccessor.getOrCreate(getThrottleTimeSensorName(clientQuotaEntity.quotaId), + sensorAccessor.getOrCreate(getThrottleTimeSensorName(metricTags), ClientQuotaManagerConfig.InactiveSensorExpirationTimeSeconds, - throttleMetricName(clientQuotaEntity), + throttleMetricName(metricTags), None, new Avg ) ) + if (quotaCallback.quotaResetRequired(clientQuotaType)) + updateQuotaMetricConfigs() + sensors } - private def getThrottleTimeSensorName(quotaId: QuotaId): String = quotaType + "ThrottleTime-" + quotaId.sanitizedUser.getOrElse("") + ':' + quotaId.clientId.getOrElse("") + private def metricTagsToSensorSuffix(metricTags: Map[String, String]): String = + metricTags.values.mkString(":") - private def getQuotaSensorName(quotaId: QuotaId): String = quotaType + "-" + quotaId.sanitizedUser.getOrElse("") + ':' + quotaId.clientId.getOrElse("") + private def getThrottleTimeSensorName(metricTags: Map[String, String]): String = + s"${quotaType}ThrottleTime-${metricTagsToSensorSuffix(metricTags)}" - protected def getQuotaMetricConfig(quota: Quota): MetricConfig = { + private def getQuotaSensorName(metricTags: Map[String, String]): String = + s"$quotaType-${metricTagsToSensorSuffix(metricTags)}" + + private def getQuotaMetricConfig(metricTags: Map[String, String]): MetricConfig = { + getQuotaMetricConfig(quotaLimit(metricTags.asJava)) + } + + private def getQuotaMetricConfig(quotaLimit: Double): MetricConfig = { new MetricConfig() - .timeWindow(config.quotaWindowSizeSeconds, TimeUnit.SECONDS) - .samples(config.numQuotaSamples) - .quota(quota) + .timeWindow(config.quotaWindowSizeSeconds, TimeUnit.SECONDS) + .samples(config.numQuotaSamples) + .quota(new Quota(quotaLimit, true)) } protected def getOrCreateSensor(sensorName: String, metricName: MetricName): Sensor = { sensorAccessor.getOrCreate( - sensorName, - ClientQuotaManagerConfig.InactiveSensorExpirationTimeSeconds, - metricName, - None, - new Rate - ) + sensorName, + ClientQuotaManagerConfig.InactiveSensorExpirationTimeSeconds, + metricName, + None, + new Rate + ) } /** * Overrides quotas for , or or the dynamic defaults * for any of these levels. - * @param sanitizedUser user to override if quota applies to or - * @param clientId client to override if quota applies to or + * + * @param sanitizedUser user to override if quota applies to or + * @param clientId client to override if quota applies to or * @param sanitizedClientId sanitized client ID to override if quota applies to or - * @param quota custom quota to apply or None if quota override is being removed + * @param quota custom quota to apply or None if quota override is being removed */ def updateQuota(sanitizedUser: Option[String], clientId: Option[String], sanitizedClientId: Option[String], quota: Option[Quota]) { /* @@ -421,86 +395,233 @@ class ClientQuotaManager(private val config: ClientQuotaManagerConfig, */ lock.writeLock().lock() try { - val quotaId = QuotaId(sanitizedUser, clientId, sanitizedClientId) - val userInfo = sanitizedUser match { - case Some(ConfigEntityName.Default) => "default user " - case Some(user) => "user " + user + " " - case None => "" + val userEntity = sanitizedUser.map { + case ConfigEntityName.Default => DefaultUserEntity + case user => UserEntity(user) } - val clientIdInfo = clientId match { - case Some(ConfigEntityName.Default) => "default client-id" - case Some(id) => "client-id " + id - case None => "" + val clientIdEntity = sanitizedClientId.map { + case ConfigEntityName.Default => DefaultClientIdEntity + case _ => ClientIdEntity(clientId.getOrElse(throw new IllegalStateException("Client-id not provided"))) } + val quotaEntity = KafkaQuotaEntity(userEntity, clientIdEntity) + + if (userEntity.nonEmpty) { + if (quotaEntity.clientIdEntity.nonEmpty) + quotaTypesEnabled |= QuotaTypes.UserClientIdQuotaEnabled + else + quotaTypesEnabled |= QuotaTypes.UserQuotaEnabled + } else if (clientIdEntity.nonEmpty) + quotaTypesEnabled |= QuotaTypes.ClientIdQuotaEnabled + quota match { - case Some(newQuota) => - info(s"Changing ${quotaType} quota for ${userInfo}${clientIdInfo} to $newQuota.bound}") - overriddenQuota.put(quotaId, newQuota) - (sanitizedUser, clientId) match { - case (Some(_), Some(_)) => quotaTypesEnabled |= QuotaTypes.UserClientIdQuotaEnabled - case (Some(_), None) => quotaTypesEnabled |= QuotaTypes.UserQuotaEnabled - case (None, Some(_)) => quotaTypesEnabled |= QuotaTypes.ClientIdQuotaEnabled - case (None, None) => - } - case None => - info(s"Removing ${quotaType} quota for ${userInfo}${clientIdInfo}") - overriddenQuota.remove(quotaId) + case Some(newQuota) => quotaCallback.updateQuota(clientQuotaType, quotaEntity, newQuota.bound) + case None => quotaCallback.removeQuota(clientQuotaType, quotaEntity) } + val updatedEntity = if (userEntity.contains(DefaultUserEntity) || clientIdEntity.contains(DefaultClientIdEntity)) + None // more than one entity may need updating, so `updateQuotaMetricConfigs` will go through all metrics + else + Some(quotaEntity) + updateQuotaMetricConfigs(updatedEntity) - val quotaMetricName = clientRateMetricName(sanitizedUser.getOrElse(""), clientId.getOrElse("")) - val allMetrics = metrics.metrics() + } finally { + lock.writeLock().unlock() + } + } - // If multiple-levels of quotas are defined or if this is a default quota update, traverse metrics - // to find all affected values. Otherwise, update just the single matching one. - val singleUpdate = quotaTypesEnabled match { - case QuotaTypes.NoQuotas | QuotaTypes.ClientIdQuotaEnabled | QuotaTypes.UserQuotaEnabled | QuotaTypes.UserClientIdQuotaEnabled => - !sanitizedUser.filter(_ == ConfigEntityName.Default).isDefined && !clientId.filter(_ == ConfigEntityName.Default).isDefined - case _ => false + /** + * Updates metrics configs. This is invoked when quota configs are updated in ZooKeeper + * or when partitions leaders change and custom callbacks that implement partition-based quotas + * have updated quotas. + * @param updatedQuotaEntity If set to one entity and quotas have only been enabled at one + * level, then an optimized update is performed with a single metric update. If None is provided, + * or if custom callbacks are used or if multi-level quotas have been enabled, all metric configs + * are checked and updated if required. + */ + def updateQuotaMetricConfigs(updatedQuotaEntity: Option[KafkaQuotaEntity] = None): Unit = { + val allMetrics = metrics.metrics() + + // If using custom quota callbacks or if multiple-levels of quotas are defined or + // if this is a default quota update, traverse metrics to find all affected values. + // Otherwise, update just the single matching one. + val singleUpdate = quotaTypesEnabled match { + case QuotaTypes.NoQuotas | QuotaTypes.ClientIdQuotaEnabled | QuotaTypes.UserQuotaEnabled | QuotaTypes.UserClientIdQuotaEnabled => + updatedQuotaEntity.nonEmpty + case _ => false + } + if (singleUpdate) { + val quotaEntity = updatedQuotaEntity.getOrElse(throw new IllegalStateException("Quota entity not specified")) + val user = quotaEntity.sanitizedUser + val clientId = quotaEntity.clientId + val metricTags = Map(DefaultTags.User -> user, DefaultTags.ClientId -> clientId) + + val quotaMetricName = clientRateMetricName(metricTags) + // Change the underlying metric config if the sensor has been created + val metric = allMetrics.get(quotaMetricName) + if (metric != null) { + Option(quotaCallback.quotaLimit(clientQuotaType, metricTags.asJava)).foreach { newQuota => + info(s"Sensor for $quotaEntity already exists. Changing quota to $newQuota in MetricConfig") + metric.config(getQuotaMetricConfig(newQuota)) + } } - if (singleUpdate) { - // Change the underlying metric config if the sensor has been created - val metric = allMetrics.get(quotaMetricName) - if (metric != null) { - val metricConfigEntity = quotaEntity(sanitizedUser.getOrElse(""), clientId.getOrElse(""), sanitizedClientId.getOrElse("")) - val newQuota = metricConfigEntity.quota - info(s"Sensor for ${userInfo}${clientIdInfo} already exists. Changing quota to ${newQuota.bound()} in MetricConfig") - metric.config(getQuotaMetricConfig(newQuota)) - } - } else { - allMetrics.asScala.filterKeys(n => n.name == quotaMetricName.name && n.group == quotaMetricName.group).foreach { - case (metricName, metric) => - val userTag = if (metricName.tags.containsKey("user")) metricName.tags.get("user") else "" - val clientIdTag = if (metricName.tags.containsKey("client-id")) metricName.tags.get("client-id") else "" - val metricConfigEntity = quotaEntity(userTag, clientIdTag, Sanitizer.sanitize(clientIdTag)) - if (metricConfigEntity.quota != metric.config.quota) { - val newQuota = metricConfigEntity.quota - info(s"Sensor for quota-id ${metricConfigEntity.quotaId} already exists. Setting quota to ${newQuota.bound} in MetricConfig") - metric.config(getQuotaMetricConfig(newQuota)) - } + } else { + val quotaMetricName = clientRateMetricName(Map.empty) + allMetrics.asScala.filterKeys(n => n.name == quotaMetricName.name && n.group == quotaMetricName.group).foreach { + case (metricName, metric) => + val metricTags = metricName.tags + Option(quotaCallback.quotaLimit(clientQuotaType, metricTags)).foreach { quota => + val newQuota = quota.asInstanceOf[Double] + if (newQuota != metric.config.quota.bound) { + info(s"Sensor for quota-id $metricTags already exists. Setting quota to $newQuota in MetricConfig") + metric.config(getQuotaMetricConfig(newQuota)) + } } } - - } finally { - lock.writeLock().unlock() } } - protected def clientRateMetricName(sanitizedUser: String, clientId: String): MetricName = { + protected def clientRateMetricName(quotaMetricTags: Map[String, String]): MetricName = { metrics.metricName("byte-rate", quotaType.toString, - "Tracking byte-rate per user/client-id", - "user", sanitizedUser, - "client-id", clientId) + "Tracking byte-rate per user/client-id", + quotaMetricTags.asJava) } - private def throttleMetricName(quotaEntity: QuotaEntity): MetricName = { + private def throttleMetricName(quotaMetricTags: Map[String, String]): MetricName = { metrics.metricName("throttle-time", - quotaType.toString, - "Tracking average throttle-time per user/client-id", - "user", quotaEntity.sanitizedUser, - "client-id", quotaEntity.clientId) + quotaType.toString, + "Tracking average throttle-time per user/client-id", + quotaMetricTags.asJava) } - def shutdown() = { + private def quotaTypeToClientQuotaType(quotaType: QuotaType): ClientQuotaType = { + quotaType match { + case QuotaType.Fetch => ClientQuotaType.FETCH + case QuotaType.Produce => ClientQuotaType.PRODUCE + case QuotaType.Request => ClientQuotaType.REQUEST + case _ => throw new IllegalArgumentException(s"Not a client quota type: $quotaType") + } + } + + def shutdown(): Unit = { throttledRequestReaper.shutdown() } + + class DefaultQuotaCallback extends ClientQuotaCallback { + private val overriddenQuotas = new ConcurrentHashMap[ClientQuotaEntity, Quota]() + + override def configure(configs: util.Map[String, _]): Unit = {} + + override def quotaMetricTags(quotaType: ClientQuotaType, principal: KafkaPrincipal, clientId: String): util.Map[String, String] = { + quotaMetricTags(Sanitizer.sanitize(principal.getName), clientId).asJava + } + + override def quotaLimit(quotaType: ClientQuotaType, metricTags: util.Map[String, String]): lang.Double = { + val sanitizedUser = metricTags.get(DefaultTags.User) + val clientId = metricTags.get(DefaultTags.ClientId) + var quota: Quota = null + + if (sanitizedUser != null && clientId != null) { + val userEntity = Some(UserEntity(sanitizedUser)) + val clientIdEntity = Some(ClientIdEntity(clientId)) + if (!sanitizedUser.isEmpty && !clientId.isEmpty) { + // /config/users//clients/ + quota = overriddenQuotas.get(KafkaQuotaEntity(userEntity, clientIdEntity)) + if (quota == null) { + // /config/users//clients/ + quota = overriddenQuotas.get(KafkaQuotaEntity(userEntity, Some(DefaultClientIdEntity))) + } + if (quota == null) { + // /config/users//clients/ + quota = overriddenQuotas.get(KafkaQuotaEntity(Some(DefaultUserEntity), clientIdEntity)) + } + if (quota == null) { + // /config/users//clients/ + quota = overriddenQuotas.get(DefaultUserClientIdQuotaEntity) + } + } else if (!sanitizedUser.isEmpty) { + // /config/users/ + quota = overriddenQuotas.get(KafkaQuotaEntity(userEntity, None)) + if (quota == null) { + // /config/users/ + quota = overriddenQuotas.get(DefaultUserQuotaEntity) + } + } else if (!clientId.isEmpty) { + // /config/clients/ + quota = overriddenQuotas.get(KafkaQuotaEntity(None, clientIdEntity)) + if (quota == null) { + // /config/clients/ + quota = overriddenQuotas.get(DefaultClientIdQuotaEntity) + } + if (quota == null) + quota = staticConfigClientIdQuota + } + } + if (quota == null) null else quota.bound + } + + override def updateClusterMetadata(cluster: Cluster): Boolean = { + // Default quota callback does not use any cluster metadata + false + } + + override def updateQuota(quotaType: ClientQuotaType, entity: ClientQuotaEntity, newValue: Double): Unit = { + val quotaEntity = entity.asInstanceOf[KafkaQuotaEntity] + info(s"Changing $quotaType quota for $quotaEntity to $newValue") + overriddenQuotas.put(quotaEntity, new Quota(newValue, true)) + } + + override def removeQuota(quotaType: ClientQuotaType, entity: ClientQuotaEntity): Unit = { + val quotaEntity = entity.asInstanceOf[KafkaQuotaEntity] + info(s"Removing $quotaType quota for $quotaEntity") + overriddenQuotas.remove(quotaEntity) + } + + override def quotaResetRequired(quotaType: ClientQuotaType): Boolean = false + + def quotaMetricTags(sanitizedUser: String, clientId: String) : Map[String, String] = { + val (userTag, clientIdTag) = quotaTypesEnabled match { + case QuotaTypes.NoQuotas | QuotaTypes.ClientIdQuotaEnabled => + ("", clientId) + case QuotaTypes.UserQuotaEnabled => + (sanitizedUser, "") + case QuotaTypes.UserClientIdQuotaEnabled => + (sanitizedUser, clientId) + case _ => + val userEntity = Some(UserEntity(sanitizedUser)) + val clientIdEntity = Some(ClientIdEntity(clientId)) + + var metricTags = (sanitizedUser, clientId) + // 1) /config/users//clients/ + if (!overriddenQuotas.containsKey(KafkaQuotaEntity(userEntity, clientIdEntity))) { + // 2) /config/users//clients/ + metricTags = (sanitizedUser, clientId) + if (!overriddenQuotas.containsKey(KafkaQuotaEntity(userEntity, Some(DefaultClientIdEntity)))) { + // 3) /config/users/ + metricTags = (sanitizedUser, "") + if (!overriddenQuotas.containsKey(KafkaQuotaEntity(userEntity, None))) { + // 4) /config/users//clients/ + metricTags = (sanitizedUser, clientId) + if (!overriddenQuotas.containsKey(KafkaQuotaEntity(Some(DefaultUserEntity), clientIdEntity))) { + // 5) /config/users//clients/ + metricTags = (sanitizedUser, clientId) + if (!overriddenQuotas.containsKey(DefaultUserClientIdQuotaEntity)) { + // 6) /config/users/ + metricTags = (sanitizedUser, "") + if (!overriddenQuotas.containsKey(DefaultUserQuotaEntity)) { + // 7) /config/clients/ + // 8) /config/clients/ + // 9) static client-id quota + metricTags = ("", clientId) + } + } + } + } + } + } + metricTags + } + Map(DefaultTags.User -> userTag, DefaultTags.ClientId -> clientIdTag) + } + + override def close(): Unit = {} + } } diff --git a/core/src/main/scala/kafka/server/ClientRequestQuotaManager.scala b/core/src/main/scala/kafka/server/ClientRequestQuotaManager.scala index 59fa4218acb43..3078a62175e23 100644 --- a/core/src/main/scala/kafka/server/ClientRequestQuotaManager.scala +++ b/core/src/main/scala/kafka/server/ClientRequestQuotaManager.scala @@ -22,13 +22,17 @@ import kafka.network.RequestChannel import org.apache.kafka.common.MetricName import org.apache.kafka.common.metrics._ import org.apache.kafka.common.utils.Time +import org.apache.kafka.server.quota.ClientQuotaCallback + +import scala.collection.JavaConverters._ class ClientRequestQuotaManager(private val config: ClientQuotaManagerConfig, private val metrics: Metrics, private val time: Time, - threadNamePrefix: String) - extends ClientQuotaManager(config, metrics, QuotaType.Request, time, threadNamePrefix) { + threadNamePrefix: String, + quotaCallback: Option[ClientQuotaCallback]) + extends ClientQuotaManager(config, metrics, QuotaType.Request, time, threadNamePrefix, quotaCallback) { val maxThrottleTimeMs = TimeUnit.SECONDS.toMillis(this.config.quotaWindowSizeSeconds) def exemptSensor = getOrCreateSensor(exemptSensorName, exemptMetricName) @@ -43,7 +47,7 @@ class ClientRequestQuotaManager(private val config: ClientQuotaManagerConfig, } if (quotasEnabled) { - val quotaSensors = getOrCreateQuotaSensors(request.session.sanitizedUser, request.header.clientId) + val quotaSensors = getOrCreateQuotaSensors(request.session, request.header.clientId) request.recordNetworkThreadTimeCallback = Some(timeNanos => recordNoThrottle(quotaSensors, nanosToPercentage(timeNanos))) recordAndThrottleOnQuotaViolation( @@ -62,15 +66,14 @@ class ClientRequestQuotaManager(private val config: ClientQuotaManagerConfig, } } - override protected def throttleTime(clientMetric: KafkaMetric, config: MetricConfig): Long = { - math.min(super.throttleTime(clientMetric, config), maxThrottleTimeMs) + override protected def throttleTime(clientMetric: KafkaMetric): Long = { + math.min(super.throttleTime(clientMetric), maxThrottleTimeMs) } - override protected def clientRateMetricName(sanitizedUser: String, clientId: String): MetricName = { + override protected def clientRateMetricName(quotaMetricTags: Map[String, String]): MetricName = { metrics.metricName("request-time", QuotaType.Request.toString, - "Tracking request-time per user/client-id", - "user", sanitizedUser, - "client-id", clientId) + "Tracking request-time per user/client-id", + quotaMetricTags.asJava) } private def exemptMetricName: MetricName = { diff --git a/core/src/main/scala/kafka/server/DelegationTokenManager.scala b/core/src/main/scala/kafka/server/DelegationTokenManager.scala index 008dc323d7a48..62a5e20322b66 100644 --- a/core/src/main/scala/kafka/server/DelegationTokenManager.scala +++ b/core/src/main/scala/kafka/server/DelegationTokenManager.scala @@ -29,8 +29,10 @@ import kafka.utils.{CoreUtils, Json, Logging} import kafka.zk.{DelegationTokenChangeNotificationSequenceZNode, DelegationTokenChangeNotificationZNode, DelegationTokensZNode, KafkaZkClient} import org.apache.kafka.common.protocol.Errors import org.apache.kafka.common.security.auth.KafkaPrincipal -import org.apache.kafka.common.security.scram.{ScramCredential, ScramFormatter, ScramMechanism} -import org.apache.kafka.common.security.token.delegation.{DelegationToken, DelegationTokenCache, TokenInformation} +import org.apache.kafka.common.security.scram.internal.{ScramFormatter, ScramMechanism} +import org.apache.kafka.common.security.scram.ScramCredential +import org.apache.kafka.common.security.token.delegation.internal.DelegationTokenCache +import org.apache.kafka.common.security.token.delegation.{DelegationToken, TokenInformation} import org.apache.kafka.common.utils.{Base64, Sanitizer, SecurityUtils, Time} import scala.collection.JavaConverters._ diff --git a/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala b/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala index 766907a7de29d..be0ed6b19997a 100755 --- a/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala +++ b/core/src/main/scala/kafka/server/DynamicBrokerConfig.scala @@ -75,13 +75,12 @@ object DynamicBrokerConfig { private[server] val DynamicSecurityConfigs = SslConfigs.RECONFIGURABLE_CONFIGS.asScala - val AllDynamicConfigs = mutable.Set[String]() - AllDynamicConfigs ++= DynamicSecurityConfigs - AllDynamicConfigs ++= LogCleaner.ReconfigurableConfigs - AllDynamicConfigs ++= DynamicLogConfig.ReconfigurableConfigs - AllDynamicConfigs ++= DynamicThreadPool.ReconfigurableConfigs - AllDynamicConfigs ++= Set(KafkaConfig.MetricReporterClassesProp) - AllDynamicConfigs ++= DynamicListenerConfig.ReconfigurableConfigs + val AllDynamicConfigs = DynamicSecurityConfigs ++ + LogCleaner.ReconfigurableConfigs ++ + DynamicLogConfig.ReconfigurableConfigs ++ + DynamicThreadPool.ReconfigurableConfigs ++ + Set(KafkaConfig.MetricReporterClassesProp) ++ + DynamicListenerConfig.ReconfigurableConfigs private val PerBrokerConfigs = DynamicSecurityConfigs ++ DynamicListenerConfig.ReconfigurableConfigs @@ -159,16 +158,17 @@ class DynamicBrokerConfig(private val kafkaConfig: KafkaConfig) extends Logging addBrokerReconfigurable(kafkaServer.logManager.cleaner) addReconfigurable(new DynamicLogConfig(kafkaServer.logManager)) addReconfigurable(new DynamicMetricsReporters(kafkaConfig.brokerId, kafkaServer)) + addReconfigurable(new DynamicClientQuotaCallback(kafkaConfig.brokerId, kafkaServer)) addBrokerReconfigurable(new DynamicListenerConfig(kafkaServer)) } def addReconfigurable(reconfigurable: Reconfigurable): Unit = CoreUtils.inWriteLock(lock) { - require(reconfigurable.reconfigurableConfigs.asScala.forall(AllDynamicConfigs.contains)) + verifyReconfigurableConfigs(reconfigurable.reconfigurableConfigs.asScala) reconfigurables += reconfigurable } def addBrokerReconfigurable(reconfigurable: BrokerReconfigurable): Unit = CoreUtils.inWriteLock(lock) { - require(reconfigurable.reconfigurableConfigs.forall(AllDynamicConfigs.contains)) + verifyReconfigurableConfigs(reconfigurable.reconfigurableConfigs) brokerReconfigurables += reconfigurable } @@ -176,6 +176,11 @@ class DynamicBrokerConfig(private val kafkaConfig: KafkaConfig) extends Logging reconfigurables -= reconfigurable } + private def verifyReconfigurableConfigs(configNames: Set[String]): Unit = CoreUtils.inWriteLock(lock) { + val nonDynamic = configNames.filter(DynamicConfig.Broker.nonDynamicProps.contains) + require(nonDynamic.isEmpty, s"Reconfigurable contains non-dynamic configs $nonDynamic") + } + // Visibility for testing private[server] def currentKafkaConfig: KafkaConfig = CoreUtils.inReadLock(lock) { currentConfig @@ -705,6 +710,37 @@ object DynamicListenerConfig { ) } +class DynamicClientQuotaCallback(brokerId: Int, server: KafkaServer) extends Reconfigurable { + + override def configure(configs: util.Map[String, _]): Unit = {} + + override def reconfigurableConfigs(): util.Set[String] = { + val configs = new util.HashSet[String]() + server.quotaManagers.clientQuotaCallback.foreach { + case callback: Reconfigurable => configs.addAll(callback.reconfigurableConfigs) + case _ => + } + configs + } + + override def validateReconfiguration(configs: util.Map[String, _]): Unit = { + server.quotaManagers.clientQuotaCallback.foreach { + case callback: Reconfigurable => callback.validateReconfiguration(configs) + case _ => + } + } + + override def reconfigure(configs: util.Map[String, _]): Unit = { + val config = server.config + server.quotaManagers.clientQuotaCallback.foreach { + case callback: Reconfigurable => + config.dynamicConfig.maybeReconfigure(callback, config.dynamicConfig.currentKafkaConfig, configs) + true + case _ => false + } + } +} + class DynamicListenerConfig(server: KafkaServer) extends BrokerReconfigurable with Logging { override def reconfigurableConfigs: Set[String] = { diff --git a/core/src/main/scala/kafka/server/DynamicConfigManager.scala b/core/src/main/scala/kafka/server/DynamicConfigManager.scala index 728f88ca8f7cf..d56b46fe9926d 100644 --- a/core/src/main/scala/kafka/server/DynamicConfigManager.scala +++ b/core/src/main/scala/kafka/server/DynamicConfigManager.scala @@ -22,9 +22,9 @@ import java.nio.charset.StandardCharsets import kafka.common.{NotificationHandler, ZkNodeChangeNotificationListener} import kafka.utils.{Json, Logging} import kafka.utils.json.JsonObject -import kafka.zk.{KafkaZkClient, AdminZkClient, ConfigEntityChangeNotificationZNode, ConfigEntityChangeNotificationSequenceZNode} +import kafka.zk.{AdminZkClient, ConfigEntityChangeNotificationSequenceZNode, ConfigEntityChangeNotificationZNode, KafkaZkClient} import org.apache.kafka.common.config.types.Password -import org.apache.kafka.common.security.scram.ScramMechanism +import org.apache.kafka.common.security.scram.internal.ScramMechanism import org.apache.kafka.common.utils.Time import scala.collection.JavaConverters._ diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala index 9e79afa2a5beb..a0caa4a53c0a2 100644 --- a/core/src/main/scala/kafka/server/KafkaApis.scala +++ b/core/src/main/scala/kafka/server/KafkaApis.scala @@ -231,6 +231,13 @@ class KafkaApis(val requestChannel: RequestChannel, adminManager.tryCompleteDelayedTopicOperations(topic) } } + quotas.clientQuotaCallback.foreach { callback => + if (callback.updateClusterMetadata(metadataCache.getClusterMetadata(clusterId, request.context.listenerName))) { + quotas.fetch.updateQuotaMetricConfigs() + quotas.produce.updateQuotaMetricConfigs() + quotas.request.updateQuotaMetricConfigs() + } + } sendResponseExemptThrottle(request, new UpdateMetadataResponse(Errors.NONE)) } else { sendResponseMaybeThrottle(request, _ => new UpdateMetadataResponse(Errors.CLUSTER_AUTHORIZATION_FAILED)) @@ -445,7 +452,7 @@ class KafkaApis(val requestChannel: RequestChannel, request.apiRemoteCompleteTimeNanos = time.nanoseconds quotas.produce.maybeRecordAndThrottle( - request.session.sanitizedUser, + request.session, request.header.clientId, numBytesAppended, produceResponseCallback) @@ -610,7 +617,7 @@ class KafkaApis(val requestChannel: RequestChannel, // This may be slightly different from the actual response size. But since down conversions // result in data being loaded into memory, it is better to do this after throttling to avoid OOM. val responseStruct = unconvertedFetchResponse.toStruct(versionId) - quotas.fetch.maybeRecordAndThrottle(request.session.sanitizedUser, clientId, responseStruct.sizeOf, + quotas.fetch.maybeRecordAndThrottle(request.session, clientId, responseStruct.sizeOf, fetchResponseCallback) } } diff --git a/core/src/main/scala/kafka/server/KafkaConfig.scala b/core/src/main/scala/kafka/server/KafkaConfig.scala index 5a1dca395bb8f..4834791f995eb 100755 --- a/core/src/main/scala/kafka/server/KafkaConfig.scala +++ b/core/src/main/scala/kafka/server/KafkaConfig.scala @@ -37,6 +37,7 @@ import org.apache.kafka.common.metrics.Sensor import org.apache.kafka.common.network.ListenerName import org.apache.kafka.common.record.TimestampType import org.apache.kafka.common.security.auth.SecurityProtocol +import org.apache.kafka.server.quota.ClientQuotaCallback import scala.collection.JavaConverters._ import scala.collection.Map @@ -390,6 +391,7 @@ object KafkaConfig { val QuotaWindowSizeSecondsProp = "quota.window.size.seconds" val ReplicationQuotaWindowSizeSecondsProp = "replication.quota.window.size.seconds" val AlterLogDirsReplicationQuotaWindowSizeSecondsProp = "alter.log.dirs.replication.quota.window.size.seconds" + val ClientQuotaCallbackClassProp = "client.quota.callback.class" val DeleteTopicEnableProp = "delete.topic.enable" val CompressionTypeProp = "compression.type" @@ -425,6 +427,10 @@ object KafkaConfig { val SaslMechanismInterBrokerProtocolProp = "sasl.mechanism.inter.broker.protocol" val SaslJaasConfigProp = SaslConfigs.SASL_JAAS_CONFIG val SaslEnabledMechanismsProp = BrokerSecurityConfigs.SASL_ENABLED_MECHANISMS_CONFIG + val SaslServerCallbackHandlerClassProp = BrokerSecurityConfigs.SASL_SERVER_CALLBACK_HANDLER_CLASS + val SaslClientCallbackHandlerClassProp = SaslConfigs.SASL_CLIENT_CALLBACK_HANDLER_CLASS + val SaslLoginClassProp = SaslConfigs.SASL_LOGIN_CLASS + val SaslLoginCallbackHandlerClassProp = SaslConfigs.SASL_LOGIN_CALLBACK_HANDLER_CLASS val SaslKerberosServiceNameProp = SaslConfigs.SASL_KERBEROS_SERVICE_NAME val SaslKerberosKinitCmdProp = SaslConfigs.SASL_KERBEROS_KINIT_CMD val SaslKerberosTicketRenewWindowFactorProp = SaslConfigs.SASL_KERBEROS_TICKET_RENEW_WINDOW_FACTOR @@ -513,8 +519,9 @@ object KafkaConfig { val SocketSendBufferBytesDoc = "The SO_SNDBUF buffer of the socket sever sockets. If the value is -1, the OS default will be used." val SocketReceiveBufferBytesDoc = "The SO_RCVBUF buffer of the socket sever sockets. If the value is -1, the OS default will be used." val SocketRequestMaxBytesDoc = "The maximum number of bytes in a socket request" - val MaxConnectionsPerIpDoc = "The maximum number of connections we allow from each ip address" - val MaxConnectionsPerIpOverridesDoc = "Per-ip or hostname overrides to the default maximum number of connections" + val MaxConnectionsPerIpDoc = "The maximum number of connections we allow from each ip address. This can be set to 0 if there are overrides " + + "configured using " + MaxConnectionsPerIpOverridesProp + " property" + val MaxConnectionsPerIpOverridesDoc = "A comma-separated list of per-ip or hostname overrides to the default maximum number of connections. An example value is \"hostName:100,127.0.0.1:200\"" val ConnectionsMaxIdleMsDoc = "Idle connections timeout: the server socket processor threads close the connections that idle more than this" /************* Rack Configuration **************/ val RackDoc = "Rack of the broker. This will be used in rack aware replication assignment for fault tolerance. Examples: `RACK1`, `us-east-1d`" @@ -672,6 +679,10 @@ object KafkaConfig { val QuotaWindowSizeSecondsDoc = "The time span of each sample for client quotas" val ReplicationQuotaWindowSizeSecondsDoc = "The time span of each sample for replication quotas" val AlterLogDirsReplicationQuotaWindowSizeSecondsDoc = "The time span of each sample for alter log dirs replication quotas" + val ClientQuotaCallbackClassDoc = "The fully qualified name of a class that implements the ClientQuotaCallback interface, " + + "which is used to determine quota limits applied to client requests. By default, , or " + + "quotas stored in ZooKeeper are applied. For any given request, the most specific quota that matches the user principal " + + "of the session and the client-id of the request is applied." /** ********* Transaction Configuration ***********/ val TransactionIdExpirationMsDoc = "The maximum time of inactivity before a transactional id is expired by the " + "transaction coordinator. Note that this also influences producer id expiration: Producer ids are guaranteed to expire " + @@ -713,7 +724,11 @@ object KafkaConfig { /** ********* Sasl Configuration ****************/ val SaslMechanismInterBrokerProtocolDoc = "SASL mechanism used for inter-broker communication. Default is GSSAPI." val SaslJaasConfigDoc = SaslConfigs.SASL_JAAS_CONFIG_DOC - val SaslEnabledMechanismsDoc = SaslConfigs.SASL_ENABLED_MECHANISMS_DOC + val SaslEnabledMechanismsDoc = BrokerSecurityConfigs.SASL_ENABLED_MECHANISMS_DOC + val SaslServerCallbackHandlerClassDoc = BrokerSecurityConfigs.SASL_SERVER_CALLBACK_HANDLER_CLASS_DOC + val SaslClientCallbackHandlerClassDoc = SaslConfigs.SASL_CLIENT_CALLBACK_HANDLER_CLASS_DOC + val SaslLoginClassDoc = SaslConfigs.SASL_LOGIN_CLASS_DOC + val SaslLoginCallbackHandlerClassDoc = SaslConfigs.SASL_LOGIN_CALLBACK_HANDLER_CLASS_DOC val SaslKerberosServiceNameDoc = SaslConfigs.SASL_KERBEROS_SERVICE_NAME_DOC val SaslKerberosKinitCmdDoc = SaslConfigs.SASL_KERBEROS_KINIT_CMD_DOC val SaslKerberosTicketRenewWindowFactorDoc = SaslConfigs.SASL_KERBEROS_TICKET_RENEW_WINDOW_FACTOR_DOC @@ -913,6 +928,7 @@ object KafkaConfig { .define(QuotaWindowSizeSecondsProp, INT, Defaults.QuotaWindowSizeSeconds, atLeast(1), LOW, QuotaWindowSizeSecondsDoc) .define(ReplicationQuotaWindowSizeSecondsProp, INT, Defaults.ReplicationQuotaWindowSizeSeconds, atLeast(1), LOW, ReplicationQuotaWindowSizeSecondsDoc) .define(AlterLogDirsReplicationQuotaWindowSizeSecondsProp, INT, Defaults.AlterLogDirsReplicationQuotaWindowSizeSeconds, atLeast(1), LOW, AlterLogDirsReplicationQuotaWindowSizeSecondsDoc) + .define(ClientQuotaCallbackClassProp, CLASS, null, LOW, ClientQuotaCallbackClassDoc) /** ********* SSL Configuration ****************/ .define(PrincipalBuilderClassProp, CLASS, null, MEDIUM, PrincipalBuilderClassDoc) @@ -937,6 +953,10 @@ object KafkaConfig { .define(SaslMechanismInterBrokerProtocolProp, STRING, Defaults.SaslMechanismInterBrokerProtocol, MEDIUM, SaslMechanismInterBrokerProtocolDoc) .define(SaslJaasConfigProp, PASSWORD, null, MEDIUM, SaslJaasConfigDoc) .define(SaslEnabledMechanismsProp, LIST, Defaults.SaslEnabledMechanisms, MEDIUM, SaslEnabledMechanismsDoc) + .define(SaslServerCallbackHandlerClassProp, CLASS, null, MEDIUM, SaslServerCallbackHandlerClassDoc) + .define(SaslClientCallbackHandlerClassProp, CLASS, null, MEDIUM, SaslClientCallbackHandlerClassDoc) + .define(SaslLoginClassProp, CLASS, null, MEDIUM, SaslLoginClassDoc) + .define(SaslLoginCallbackHandlerClassProp, CLASS, null, MEDIUM, SaslLoginCallbackHandlerClassDoc) .define(SaslKerberosServiceNameProp, STRING, null, MEDIUM, SaslKerberosServiceNameDoc) .define(SaslKerberosKinitCmdProp, STRING, Defaults.SaslKerberosKinitCmd, MEDIUM, SaslKerberosKinitCmdDoc) .define(SaslKerberosTicketRenewWindowFactorProp, DOUBLE, Defaults.SaslKerberosTicketRenewWindowFactor, MEDIUM, SaslKerberosTicketRenewWindowFactorDoc) @@ -1344,5 +1364,9 @@ class KafkaConfig(val props: java.util.Map[_, _], doLog: Boolean, dynamicConfigO s"${KafkaConfig.SaslMechanismInterBrokerProtocolProp} must be included in ${KafkaConfig.SaslEnabledMechanismsProp} when SASL is used for inter-broker communication") require(queuedMaxBytes <= 0 || queuedMaxBytes >= socketRequestMaxBytes, s"${KafkaConfig.QueuedMaxBytesProp} must be larger or equal to ${KafkaConfig.SocketRequestMaxBytesProp}") + + if (maxConnectionsPerIp == 0) + require(!maxConnectionsPerIpOverrides.isEmpty, s"${KafkaConfig.MaxConnectionsPerIpProp} can be set to zero only if" + + s" ${KafkaConfig.MaxConnectionsPerIpOverridesProp} property is set.") } } diff --git a/core/src/main/scala/kafka/server/KafkaServer.scala b/core/src/main/scala/kafka/server/KafkaServer.scala index d7ca65658f696..a0d2c799e6ae5 100755 --- a/core/src/main/scala/kafka/server/KafkaServer.scala +++ b/core/src/main/scala/kafka/server/KafkaServer.scala @@ -44,8 +44,8 @@ import org.apache.kafka.common.network._ import org.apache.kafka.common.protocol.Errors import org.apache.kafka.common.requests.{ControlledShutdownRequest, ControlledShutdownResponse} import org.apache.kafka.common.security.auth.SecurityProtocol -import org.apache.kafka.common.security.scram.ScramMechanism -import org.apache.kafka.common.security.token.delegation.DelegationTokenCache +import org.apache.kafka.common.security.scram.internal.ScramMechanism +import org.apache.kafka.common.security.token.delegation.internal.DelegationTokenCache import org.apache.kafka.common.security.{JaasContext, JaasUtils} import org.apache.kafka.common.utils.{AppInfoParser, LogContext, Time} import org.apache.kafka.common.{ClusterResource, Node} @@ -595,6 +595,7 @@ class KafkaServer(val config: KafkaConfig, time: Time = Time.SYSTEM, threadNameP if (quotaManagers != null) CoreUtils.swallow(quotaManagers.shutdown(), this) + // Even though socket server is stopped much earlier, controller can generate // response for controlled shutdown request. Shutdown server at the end to // avoid any failures (e.g. when metrics are recorded) diff --git a/core/src/main/scala/kafka/server/MetadataCache.scala b/core/src/main/scala/kafka/server/MetadataCache.scala index 7cdb8f1a1a3c2..43fe35287d384 100755 --- a/core/src/main/scala/kafka/server/MetadataCache.scala +++ b/core/src/main/scala/kafka/server/MetadataCache.scala @@ -17,6 +17,7 @@ package kafka.server +import java.util.Collections import java.util.concurrent.locks.ReentrantReadWriteLock import scala.collection.{Seq, Set, mutable} @@ -28,7 +29,7 @@ import kafka.controller.StateChangeLogger import kafka.utils.CoreUtils._ import kafka.utils.Logging import org.apache.kafka.common.internals.Topic -import org.apache.kafka.common.{Node, TopicPartition} +import org.apache.kafka.common.{Cluster, Node, PartitionInfo, TopicPartition} import org.apache.kafka.common.network.ListenerName import org.apache.kafka.common.protocol.Errors import org.apache.kafka.common.requests.{MetadataResponse, UpdateMetadataRequest} @@ -127,6 +128,14 @@ class MetadataCache(brokerId: Int) extends Logging { } } + def getAllPartitions(): Map[TopicPartition, UpdateMetadataRequest.PartitionState] = { + inReadLock(partitionMetadataLock) { + cache.flatMap { case (topic, partitionStates) => + partitionStates.map { case (partition, state ) => (new TopicPartition(topic, partition), state) } + }.toMap + } + } + def getNonExistingTopics(topics: Set[String]): Set[String] = { inReadLock(partitionMetadataLock) { topics -- cache.keySet @@ -180,6 +189,27 @@ class MetadataCache(brokerId: Int) extends Logging { def getControllerId: Option[Int] = controllerId + def getClusterMetadata(clusterId: String, listenerName: ListenerName): Cluster = { + inReadLock(partitionMetadataLock) { + val nodes = aliveNodes.map { case (id, nodes) => (id, nodes.get(listenerName).orNull) } + def node(id: Integer): Node = nodes.get(id).orNull + val partitions = getAllPartitions() + .filter { case (_, state) => state.basePartitionState.leader != LeaderAndIsr.LeaderDuringDelete } + .map { case (tp, state) => + new PartitionInfo(tp.topic, tp.partition, node(state.basePartitionState.leader), + state.basePartitionState.replicas.asScala.map(node).toArray, + state.basePartitionState.isr.asScala.map(node).toArray, + state.offlineReplicas.asScala.map(node).toArray) + } + val unauthorizedTopics = Collections.emptySet[String] + val internalTopics = getAllTopics().filter(Topic.isInternal).asJava + new Cluster(clusterId, nodes.values.filter(_ != null).toList.asJava, + partitions.toList.asJava, + unauthorizedTopics, internalTopics, + getControllerId.map(id => node(id)).orNull) + } + } + // This method returns the deleted TopicPartitions received from UpdateMetadataRequest def updateCache(correlationId: Int, updateMetadataRequest: UpdateMetadataRequest): Seq[TopicPartition] = { inWriteLock(partitionMetadataLock) { diff --git a/core/src/main/scala/kafka/server/QuotaFactory.scala b/core/src/main/scala/kafka/server/QuotaFactory.scala index 01441b573238c..1ee713b10bef3 100644 --- a/core/src/main/scala/kafka/server/QuotaFactory.scala +++ b/core/src/main/scala/kafka/server/QuotaFactory.scala @@ -20,6 +20,7 @@ import kafka.server.QuotaType._ import kafka.utils.Logging import org.apache.kafka.common.TopicPartition import org.apache.kafka.common.metrics.Metrics +import org.apache.kafka.server.quota.ClientQuotaCallback import org.apache.kafka.common.utils.Time object QuotaType { @@ -44,22 +45,28 @@ object QuotaFactory extends Logging { request: ClientRequestQuotaManager, leader: ReplicationQuotaManager, follower: ReplicationQuotaManager, - alterLogDirs: ReplicationQuotaManager) { + alterLogDirs: ReplicationQuotaManager, + clientQuotaCallback: Option[ClientQuotaCallback]) { def shutdown() { fetch.shutdown produce.shutdown request.shutdown + clientQuotaCallback.foreach(_.close()) } } def instantiate(cfg: KafkaConfig, metrics: Metrics, time: Time, threadNamePrefix: String): QuotaManagers = { + + val clientQuotaCallback = Option(cfg.getConfiguredInstance(KafkaConfig.ClientQuotaCallbackClassProp, + classOf[ClientQuotaCallback])) QuotaManagers( - new ClientQuotaManager(clientFetchConfig(cfg), metrics, Fetch, time, threadNamePrefix), - new ClientQuotaManager(clientProduceConfig(cfg), metrics, Produce, time, threadNamePrefix), - new ClientRequestQuotaManager(clientRequestConfig(cfg), metrics, time, threadNamePrefix), + new ClientQuotaManager(clientFetchConfig(cfg), metrics, Fetch, time, threadNamePrefix, clientQuotaCallback), + new ClientQuotaManager(clientProduceConfig(cfg), metrics, Produce, time, threadNamePrefix, clientQuotaCallback), + new ClientRequestQuotaManager(clientRequestConfig(cfg), metrics, time, threadNamePrefix, clientQuotaCallback), new ReplicationQuotaManager(replicationConfig(cfg), metrics, LeaderReplication, time), new ReplicationQuotaManager(replicationConfig(cfg), metrics, FollowerReplication, time), - new ReplicationQuotaManager(alterLogDirsReplicationConfig(cfg), metrics, AlterLogDirsReplication, time) + new ReplicationQuotaManager(alterLogDirsReplicationConfig(cfg), metrics, AlterLogDirsReplication, time), + clientQuotaCallback ) } diff --git a/core/src/main/scala/kafka/server/ReplicaManager.scala b/core/src/main/scala/kafka/server/ReplicaManager.scala index d4abc11be1b4e..0c2b0d805f8cc 100644 --- a/core/src/main/scala/kafka/server/ReplicaManager.scala +++ b/core/src/main/scala/kafka/server/ReplicaManager.scala @@ -1150,7 +1150,7 @@ class ReplicaManager(val config: KafkaConfig, for (partition <- partitionState.keys) responseMap.put(partition.topicPartition, Errors.NONE) - val partitionsToMakeLeaders: mutable.Set[Partition] = mutable.Set() + val partitionsToMakeLeaders = mutable.Set[Partition]() try { // First stop fetchers for all the partitions @@ -1218,15 +1218,16 @@ class ReplicaManager(val config: KafkaConfig, */ private def makeFollowers(controllerId: Int, epoch: Int, - partitionState: Map[Partition, LeaderAndIsrRequest.PartitionState], + partitionStates: Map[Partition, LeaderAndIsrRequest.PartitionState], correlationId: Int, responseMap: mutable.Map[TopicPartition, Errors]) : Set[Partition] = { - partitionState.keys.foreach { partition => + partitionStates.foreach { case (partition, partitionState) => stateChangeLogger.trace(s"Handling LeaderAndIsr request correlationId $correlationId from controller $controllerId " + - s"epoch $epoch starting the become-follower transition for partition ${partition.topicPartition}") + s"epoch $epoch starting the become-follower transition for partition ${partition.topicPartition} with leader " + + s"${partitionState.basePartitionState.leader}") } - for (partition <- partitionState.keys) + for (partition <- partitionStates.keys) responseMap.put(partition.topicPartition, Errors.NONE) val partitionsToMakeFollower: mutable.Set[Partition] = mutable.Set() @@ -1234,9 +1235,9 @@ class ReplicaManager(val config: KafkaConfig, try { // TODO: Delete leaders from LeaderAndIsrRequest - partitionState.foreach{ case (partition, partitionStateInfo) => + partitionStates.foreach { case (partition, partitionStateInfo) => + val newLeaderBrokerId = partitionStateInfo.basePartitionState.leader try { - val newLeaderBrokerId = partitionStateInfo.basePartitionState.leader metadataCache.getAliveBrokers.find(_.id == newLeaderBrokerId) match { // Only change partition state when the leader is available case Some(_) => @@ -1263,10 +1264,11 @@ class ReplicaManager(val config: KafkaConfig, case e: KafkaStorageException => stateChangeLogger.error(s"Skipped the become-follower state change with correlation id $correlationId from " + s"controller $controllerId epoch $epoch for partition ${partition.topicPartition} " + - s"(last update controller epoch ${partitionStateInfo.basePartitionState.controllerEpoch}) since the " + - s"replica for the partition is offline due to disk error $e") + s"(last update controller epoch ${partitionStateInfo.basePartitionState.controllerEpoch}) with leader " + + s"$newLeaderBrokerId since the replica for the partition is offline due to disk error $e") val dirOpt = getLogDir(partition.topicPartition) - error(s"Error while making broker the follower for partition $partition in dir $dirOpt", e) + error(s"Error while making broker the follower for partition $partition with leader " + + s"$newLeaderBrokerId in dir $dirOpt", e) responseMap.put(partition.topicPartition, Errors.KAFKA_STORAGE_ERROR) } } @@ -1274,7 +1276,8 @@ class ReplicaManager(val config: KafkaConfig, replicaFetcherManager.removeFetcherForPartitions(partitionsToMakeFollower.map(_.topicPartition)) partitionsToMakeFollower.foreach { partition => stateChangeLogger.trace(s"Stopped fetchers as part of become-follower request from controller $controllerId " + - s"epoch $epoch with correlation id $correlationId for partition ${partition.topicPartition}") + s"epoch $epoch with correlation id $correlationId for partition ${partition.topicPartition} with leader " + + s"${partitionStates(partition).basePartitionState.leader}") } partitionsToMakeFollower.foreach { partition => @@ -1286,14 +1289,15 @@ class ReplicaManager(val config: KafkaConfig, partitionsToMakeFollower.foreach { partition => stateChangeLogger.trace(s"Truncated logs and checkpointed recovery boundaries for partition " + s"${partition.topicPartition} as part of become-follower request with correlation id $correlationId from " + - s"controller $controllerId epoch $epoch") + s"controller $controllerId epoch $epoch with leader ${partitionStates(partition).basePartitionState.leader}") } if (isShuttingDown.get()) { partitionsToMakeFollower.foreach { partition => stateChangeLogger.trace(s"Skipped the adding-fetcher step of the become-follower state " + s"change with correlation id $correlationId from controller $controllerId epoch $epoch for " + - s"partition ${partition.topicPartition} since it is shutting down") + s"partition ${partition.topicPartition} with leader ${partitionStates(partition).basePartitionState.leader} " + + "since it is shutting down") } } else { @@ -1307,7 +1311,7 @@ class ReplicaManager(val config: KafkaConfig, partitionsToMakeFollower.foreach { partition => stateChangeLogger.trace(s"Started fetcher to new leader as part of become-follower " + s"request from controller $controllerId epoch $epoch with correlation id $correlationId for " + - s"partition ${partition.topicPartition}") + s"partition ${partition.topicPartition} with leader ${partitionStates(partition).basePartitionState.leader}") } } } catch { @@ -1318,9 +1322,10 @@ class ReplicaManager(val config: KafkaConfig, throw e } - partitionState.keys.foreach { partition => + partitionStates.keys.foreach { partition => stateChangeLogger.trace(s"Completed LeaderAndIsr request correlationId $correlationId from controller $controllerId " + - s"epoch $epoch for the become-follower transition for partition ${partition.topicPartition}") + s"epoch $epoch for the become-follower transition for partition ${partition.topicPartition} with leader " + + s"${partitionStates(partition).basePartitionState.leader}") } partitionsToMakeFollower diff --git a/core/src/main/scala/kafka/utils/VerifiableProperties.scala b/core/src/main/scala/kafka/utils/VerifiableProperties.scala index de4f654c1ee59..5d70db5e11d06 100755 --- a/core/src/main/scala/kafka/utils/VerifiableProperties.scala +++ b/core/src/main/scala/kafka/utils/VerifiableProperties.scala @@ -122,7 +122,7 @@ class VerifiableProperties(val props: Properties) extends Logging { require(v >= range._1 && v <= range._2, name + " has value " + v + " which is not in the range " + range + ".") v } - + /** * Get a required argument as a double * @param name The property name @@ -130,7 +130,7 @@ class VerifiableProperties(val props: Properties) extends Logging { * @throws IllegalArgumentException If the given property is not present */ def getDouble(name: String): Double = getString(name).toDouble - + /** * Get an optional argument as a double * @param name The property name @@ -141,7 +141,7 @@ class VerifiableProperties(val props: Properties) extends Logging { getDouble(name) else default - } + } /** * Read a boolean value from the properties instance @@ -158,7 +158,7 @@ class VerifiableProperties(val props: Properties) extends Logging { v.toBoolean } } - + def getBoolean(name: String) = getString(name).toBoolean /** @@ -178,7 +178,7 @@ class VerifiableProperties(val props: Properties) extends Logging { require(containsKey(name), "Missing required property '" + name + "'") getProperty(name) } - + /** * Get a Map[String, String] from a property list in the form k1:v2, k2:v2, ... */ diff --git a/core/src/test/scala/integration/kafka/api/BaseQuotaTest.scala b/core/src/test/scala/integration/kafka/api/BaseQuotaTest.scala index 9b1c2aad2f9ca..b265182af23ad 100644 --- a/core/src/test/scala/integration/kafka/api/BaseQuotaTest.scala +++ b/core/src/test/scala/integration/kafka/api/BaseQuotaTest.scala @@ -16,31 +16,29 @@ package kafka.api import java.util.{Collections, HashMap, Properties} -import kafka.server.{ClientQuotaManagerConfig, DynamicConfig, KafkaConfig, KafkaServer, QuotaId, QuotaType} +import kafka.api.QuotaTestClients._ +import kafka.server.{ClientQuotaManager, ClientQuotaManagerConfig, DynamicConfig, KafkaConfig, KafkaServer, QuotaType} import kafka.utils.TestUtils import org.apache.kafka.clients.consumer.{ConsumerConfig, KafkaConsumer} import org.apache.kafka.clients.producer._ import org.apache.kafka.clients.producer.internals.ErrorLoggingCallback -import org.apache.kafka.common.{MetricName, TopicPartition} +import org.apache.kafka.common.{Metric, MetricName, TopicPartition} import org.apache.kafka.common.metrics.{KafkaMetric, Quota} +import org.apache.kafka.common.security.auth.KafkaPrincipal import org.junit.Assert._ import org.junit.{Before, Test} -abstract class BaseQuotaTest extends IntegrationTestHarness { +import scala.collection.JavaConverters._ - def userPrincipal : String - def producerQuotaId : QuotaId - def consumerQuotaId : QuotaId - def overrideQuotas(producerQuota: Long, consumerQuota: Long, requestQuota: Double) - def removeQuotaOverrides() +abstract class BaseQuotaTest extends IntegrationTestHarness { override val serverCount = 2 val producerCount = 1 val consumerCount = 1 - private val producerBufferSize = 300000 protected def producerClientId = "QuotasTestProducer-1" protected def consumerClientId = "QuotasTestConsumer-1" + protected def createQuotaTestClients(topic: String, leaderNode: KafkaServer): QuotaTestClients this.serverConfig.setProperty(KafkaConfig.ControlledShutdownEnableProp, "false") this.serverConfig.setProperty(KafkaConfig.OffsetsTopicReplicationFactorProp, "2") @@ -49,7 +47,7 @@ abstract class BaseQuotaTest extends IntegrationTestHarness { this.serverConfig.setProperty(KafkaConfig.GroupMaxSessionTimeoutMsProp, "30000") this.serverConfig.setProperty(KafkaConfig.GroupInitialRebalanceDelayMsProp, "0") this.producerConfig.setProperty(ProducerConfig.ACKS_CONFIG, "0") - this.producerConfig.setProperty(ProducerConfig.BUFFER_MEMORY_CONFIG, producerBufferSize.toString) + this.producerConfig.setProperty(ProducerConfig.BUFFER_MEMORY_CONFIG, "300000") this.producerConfig.setProperty(ProducerConfig.CLIENT_ID_CONFIG, producerClientId) this.consumerConfig.setProperty(ConsumerConfig.GROUP_ID_CONFIG, "QuotasTest") this.consumerConfig.setProperty(ConsumerConfig.MAX_PARTITION_FETCH_BYTES_CONFIG, 4096.toString) @@ -63,9 +61,10 @@ abstract class BaseQuotaTest extends IntegrationTestHarness { val defaultConsumerQuota = 2500 val defaultRequestQuota = Int.MaxValue - var leaderNode: KafkaServer = null - var followerNode: KafkaServer = null - private val topic1 = "topic-1" + val topic1 = "topic-1" + var leaderNode: KafkaServer = _ + var followerNode: KafkaServer = _ + var quotaTestClients: QuotaTestClients = _ @Before override def setUp() { @@ -75,22 +74,19 @@ abstract class BaseQuotaTest extends IntegrationTestHarness { val leaders = createTopic(topic1, numPartitions, serverCount) leaderNode = if (leaders(0) == servers.head.config.brokerId) servers.head else servers(1) followerNode = if (leaders(0) != servers.head.config.brokerId) servers.head else servers(1) + quotaTestClients = createQuotaTestClients(topic1, leaderNode) } @Test def testThrottledProducerConsumer() { val numRecords = 1000 - val producer = producers.head - val produced = produceUntilThrottled(producer, numRecords) - assertTrue("Should have been throttled", producerThrottleMetric.value > 0) - verifyProducerThrottleTimeMetric(producer) + val produced = quotaTestClients.produceUntilThrottled(numRecords) + quotaTestClients.verifyProduceThrottle(expectThrottle = true) // Consumer should read in a bursty manner and get throttled immediately - val consumer = consumers.head - consumeUntilThrottled(consumer, produced) - assertTrue("Should have been throttled", consumerThrottleMetric.value > 0) - verifyConsumerThrottleTimeMetric(consumer) + quotaTestClients.consumeUntilThrottled(produced) + quotaTestClients.verifyConsumeThrottle(expectThrottle = true) } @Test @@ -100,154 +96,187 @@ abstract class BaseQuotaTest extends IntegrationTestHarness { props.put(DynamicConfig.Client.ProducerByteRateOverrideProp, Long.MaxValue.toString) props.put(DynamicConfig.Client.ConsumerByteRateOverrideProp, Long.MaxValue.toString) - overrideQuotas(Long.MaxValue, Long.MaxValue, Int.MaxValue) - waitForQuotaUpdate(Long.MaxValue, Long.MaxValue, Int.MaxValue) + quotaTestClients.overrideQuotas(Long.MaxValue, Long.MaxValue, Int.MaxValue) + quotaTestClients.waitForQuotaUpdate(Long.MaxValue, Long.MaxValue, Int.MaxValue) val numRecords = 1000 - assertEquals(numRecords, produceUntilThrottled(producers.head, numRecords)) - assertEquals("Should not have been throttled", 0.0, producerThrottleMetric.value, 0.0) + assertEquals(numRecords, quotaTestClients.produceUntilThrottled(numRecords)) + quotaTestClients.verifyProduceThrottle(expectThrottle = false) // The "client" consumer does not get throttled. - assertEquals(numRecords, consumeUntilThrottled(consumers.head, numRecords)) - assertEquals("Should not have been throttled", 0.0, consumerThrottleMetric.value, 0.0) + assertEquals(numRecords, quotaTestClients.consumeUntilThrottled(numRecords)) + quotaTestClients.verifyConsumeThrottle(expectThrottle = false) } @Test def testQuotaOverrideDelete() { // Override producer and consumer quotas to unlimited - overrideQuotas(Long.MaxValue, Long.MaxValue, Int.MaxValue) - waitForQuotaUpdate(Long.MaxValue, Long.MaxValue, Int.MaxValue) + quotaTestClients.overrideQuotas(Long.MaxValue, Long.MaxValue, Int.MaxValue) + quotaTestClients.waitForQuotaUpdate(Long.MaxValue, Long.MaxValue, Int.MaxValue) val numRecords = 1000 - assertEquals(numRecords, produceUntilThrottled(producers.head, numRecords)) - assertEquals("Should not have been throttled", 0.0, producerThrottleMetric.value, 0.0) - assertEquals(numRecords, consumeUntilThrottled(consumers.head, numRecords)) - assertEquals("Should not have been throttled", 0.0, consumerThrottleMetric.value, 0.0) + assertEquals(numRecords, quotaTestClients.produceUntilThrottled(numRecords)) + quotaTestClients.verifyProduceThrottle(expectThrottle = false) + assertEquals(numRecords, quotaTestClients.consumeUntilThrottled(numRecords)) + quotaTestClients.verifyConsumeThrottle(expectThrottle = false) // Delete producer and consumer quota overrides. Consumer and producer should now be // throttled since broker defaults are very small - removeQuotaOverrides() - val produced = produceUntilThrottled(producers.head, numRecords) - assertTrue("Should have been throttled", producerThrottleMetric.value > 0) + quotaTestClients.removeQuotaOverrides() + val produced = quotaTestClients.produceUntilThrottled(numRecords) + quotaTestClients.verifyProduceThrottle(expectThrottle = true) // Since producer may have been throttled after producing a couple of records, // consume from beginning till throttled consumers.head.seekToBeginning(Collections.singleton(new TopicPartition(topic1, 0))) - consumeUntilThrottled(consumers.head, numRecords + produced) - assertTrue("Should have been throttled", consumerThrottleMetric.value > 0) + quotaTestClients.consumeUntilThrottled(numRecords + produced) + quotaTestClients.verifyConsumeThrottle(expectThrottle = true) } @Test def testThrottledRequest() { - overrideQuotas(Long.MaxValue, Long.MaxValue, 0.1) - waitForQuotaUpdate(Long.MaxValue, Long.MaxValue, 0.1) + quotaTestClients.overrideQuotas(Long.MaxValue, Long.MaxValue, 0.1) + quotaTestClients.waitForQuotaUpdate(Long.MaxValue, Long.MaxValue, 0.1) val consumer = consumers.head consumer.subscribe(Collections.singleton(topic1)) val endTimeMs = System.currentTimeMillis + 10000 var throttled = false - while ((!throttled || exemptRequestMetric == null) && System.currentTimeMillis < endTimeMs) { + while ((!throttled || quotaTestClients.exemptRequestMetric == null) && System.currentTimeMillis < endTimeMs) { consumer.poll(100) - val throttleMetric = consumerRequestThrottleMetric - throttled = throttleMetric != null && throttleMetric.value > 0 + val throttleMetric = quotaTestClients.throttleMetric(QuotaType.Request, consumerClientId) + throttled = throttleMetric != null && metricValue(throttleMetric) > 0 } assertTrue("Should have been throttled", throttled) - verifyConsumerThrottleTimeMetric(consumer, Some(ClientQuotaManagerConfig.DefaultQuotaWindowSizeSeconds * 1000.0)) + quotaTestClients.verifyConsumerClientThrottleTimeMetric(expectThrottle = true, + Some(ClientQuotaManagerConfig.DefaultQuotaWindowSizeSeconds * 1000.0)) + + val exemptMetric = quotaTestClients.exemptRequestMetric + assertNotNull("Exempt requests not recorded", exemptMetric) + assertTrue("Exempt requests not recorded", metricValue(exemptMetric) > 0) + } +} + +object QuotaTestClients { + def metricValue(metric: Metric): Double = metric.metricValue().asInstanceOf[Double] +} + +abstract class QuotaTestClients(topic: String, + leaderNode: KafkaServer, + producerClientId: String, + consumerClientId: String, + producer: KafkaProducer[Array[Byte], Array[Byte]], + consumer: KafkaConsumer[Array[Byte], Array[Byte]]) { + + def userPrincipal : KafkaPrincipal + def overrideQuotas(producerQuota: Long, consumerQuota: Long, requestQuota: Double) + def removeQuotaOverrides() + + def quotaMetricTags(clientId: String): Map[String, String] - assertNotNull("Exempt requests not recorded", exemptRequestMetric) - assertTrue("Exempt requests not recorded", exemptRequestMetric.value > 0) + def quota(quotaManager: ClientQuotaManager, userPrincipal: KafkaPrincipal, clientId: String): Quota = { + quotaManager.quota(userPrincipal, clientId) } - def produceUntilThrottled(p: KafkaProducer[Array[Byte], Array[Byte]], maxRecords: Int): Int = { + def produceUntilThrottled(maxRecords: Int, waitForRequestCompletion: Boolean = true): Int = { var numProduced = 0 var throttled = false do { val payload = numProduced.toString.getBytes - p.send(new ProducerRecord[Array[Byte], Array[Byte]](topic1, null, null, payload), - new ErrorLoggingCallback(topic1, null, null, true)).get() + val future = producer.send(new ProducerRecord[Array[Byte], Array[Byte]](topic, null, null, payload), + new ErrorLoggingCallback(topic, null, null, true)) numProduced += 1 - val throttleMetric = producerThrottleMetric - throttled = throttleMetric != null && throttleMetric.value > 0 + do { + val metric = throttleMetric(QuotaType.Produce, producerClientId) + throttled = metric != null && metricValue(metric) > 0 + } while (!future.isDone && (!throttled || waitForRequestCompletion)) } while (numProduced < maxRecords && !throttled) numProduced } - def consumeUntilThrottled(consumer: KafkaConsumer[Array[Byte], Array[Byte]], maxRecords: Int): Int = { - consumer.subscribe(Collections.singleton(topic1)) + def consumeUntilThrottled(maxRecords: Int, waitForRequestCompletion: Boolean = true): Int = { + consumer.subscribe(Collections.singleton(topic)) var numConsumed = 0 var throttled = false do { numConsumed += consumer.poll(100).count - val throttleMetric = consumerThrottleMetric - throttled = throttleMetric != null && throttleMetric.value > 0 + val metric = throttleMetric(QuotaType.Fetch, consumerClientId) + throttled = metric != null && metricValue(metric) > 0 } while (numConsumed < maxRecords && !throttled) // If throttled, wait for the records from the last fetch to be received - if (throttled && numConsumed < maxRecords) { + if (throttled && numConsumed < maxRecords && waitForRequestCompletion) { val minRecords = numConsumed + 1 while (numConsumed < minRecords) - numConsumed += consumer.poll(100).count + numConsumed += consumer.poll(100).count } numConsumed } - def waitForQuotaUpdate(producerQuota: Long, consumerQuota: Long, requestQuota: Double) { - TestUtils.retry(10000) { - val quotaManagers = leaderNode.apis.quotas - val overrideProducerQuota = quotaManagers.produce.quota(userPrincipal, producerClientId) - val overrideConsumerQuota = quotaManagers.fetch.quota(userPrincipal, consumerClientId) - val overrideProducerRequestQuota = quotaManagers.request.quota(userPrincipal, producerClientId) - val overrideConsumerRequestQuota = quotaManagers.request.quota(userPrincipal, consumerClientId) + def verifyProduceThrottle(expectThrottle: Boolean, verifyClientMetric: Boolean = true): Unit = { + verifyThrottleTimeMetric(QuotaType.Produce, producerClientId, expectThrottle) + if (verifyClientMetric) + verifyProducerClientThrottleTimeMetric(expectThrottle) + } - assertEquals(s"ClientId $producerClientId of user $userPrincipal must have producer quota", Quota.upperBound(producerQuota), overrideProducerQuota) - assertEquals(s"ClientId $consumerClientId of user $userPrincipal must have consumer quota", Quota.upperBound(consumerQuota), overrideConsumerQuota) - assertEquals(s"ClientId $producerClientId of user $userPrincipal must have request quota", Quota.upperBound(requestQuota), overrideProducerRequestQuota) - assertEquals(s"ClientId $consumerClientId of user $userPrincipal must have request quota", Quota.upperBound(requestQuota), overrideConsumerRequestQuota) + def verifyConsumeThrottle(expectThrottle: Boolean, verifyClientMetric: Boolean = true): Unit = { + verifyThrottleTimeMetric(QuotaType.Fetch, consumerClientId, expectThrottle) + if (verifyClientMetric) + verifyConsumerClientThrottleTimeMetric(expectThrottle) + } + + def verifyThrottleTimeMetric(quotaType: QuotaType, clientId: String, expectThrottle: Boolean): Unit = { + val throttleMetricValue = metricValue(throttleMetric(quotaType, clientId)) + if (expectThrottle) { + assertTrue("Should have been throttled", throttleMetricValue > 0) + } else { + assertEquals("Should not have been throttled", 0.0, throttleMetricValue, 0.0) } } - private def verifyProducerThrottleTimeMetric(producer: KafkaProducer[_, _]) { + def throttleMetricName(quotaType: QuotaType, clientId: String): MetricName = { + leaderNode.metrics.metricName("throttle-time", + quotaType.toString, + quotaMetricTags(clientId).asJava) + } + + def throttleMetric(quotaType: QuotaType, clientId: String): KafkaMetric = { + leaderNode.metrics.metrics.get(throttleMetricName(quotaType, clientId)) + } + + def exemptRequestMetric: KafkaMetric = { + val metricName = leaderNode.metrics.metricName("exempt-request-time", QuotaType.Request.toString, "") + leaderNode.metrics.metrics.get(metricName) + } + + def verifyProducerClientThrottleTimeMetric(expectThrottle: Boolean) { val tags = new HashMap[String, String] tags.put("client-id", producerClientId) val avgMetric = producer.metrics.get(new MetricName("produce-throttle-time-avg", "producer-metrics", "", tags)) val maxMetric = producer.metrics.get(new MetricName("produce-throttle-time-max", "producer-metrics", "", tags)) - TestUtils.waitUntilTrue(() => avgMetric.value > 0.0 && maxMetric.value > 0.0, - s"Producer throttle metric not updated: avg=${avgMetric.value} max=${maxMetric.value}") + if (expectThrottle) { + TestUtils.waitUntilTrue(() => metricValue(avgMetric) > 0.0 && metricValue(maxMetric) > 0.0, + s"Producer throttle metric not updated: avg=${metricValue(avgMetric)} max=${metricValue(maxMetric)}") + } else + assertEquals("Should not have been throttled", 0.0, metricValue(maxMetric), 0.0) } - private def verifyConsumerThrottleTimeMetric(consumer: KafkaConsumer[_, _], maxThrottleTime: Option[Double] = None) { + def verifyConsumerClientThrottleTimeMetric(expectThrottle: Boolean, maxThrottleTime: Option[Double] = None) { val tags = new HashMap[String, String] tags.put("client-id", consumerClientId) val avgMetric = consumer.metrics.get(new MetricName("fetch-throttle-time-avg", "consumer-fetch-manager-metrics", "", tags)) val maxMetric = consumer.metrics.get(new MetricName("fetch-throttle-time-max", "consumer-fetch-manager-metrics", "", tags)) - TestUtils.waitUntilTrue(() => avgMetric.value > 0.0 && maxMetric.value > 0.0, - s"Consumer throttle metric not updated: avg=${avgMetric.value} max=${maxMetric.value}") - maxThrottleTime.foreach(max => assertTrue(s"Maximum consumer throttle too high: ${maxMetric.value}", maxMetric.value <= max)) - } - - private def throttleMetricName(quotaType: QuotaType, quotaId: QuotaId): MetricName = { - leaderNode.metrics.metricName("throttle-time", - quotaType.toString, - "Tracking throttle-time per user/client-id", - "user", quotaId.sanitizedUser.getOrElse(""), - "client-id", quotaId.clientId.getOrElse("")) - } - - def throttleMetric(quotaType: QuotaType, quotaId: QuotaId): KafkaMetric = { - leaderNode.metrics.metrics.get(throttleMetricName(quotaType, quotaId)) - } - - private def producerThrottleMetric = throttleMetric(QuotaType.Produce, producerQuotaId) - private def consumerThrottleMetric = throttleMetric(QuotaType.Fetch, consumerQuotaId) - private def consumerRequestThrottleMetric = throttleMetric(QuotaType.Request, consumerQuotaId) - - private def exemptRequestMetric: KafkaMetric = { - val metricName = leaderNode.metrics.metricName("exempt-request-time", QuotaType.Request.toString, "") - leaderNode.metrics.metrics.get(metricName) + if (expectThrottle) { + TestUtils.waitUntilTrue(() => metricValue(avgMetric) > 0.0 && metricValue(maxMetric) > 0.0, + s"Consumer throttle metric not updated: avg=${metricValue(avgMetric)} max=${metricValue(maxMetric)}") + maxThrottleTime.foreach(max => assertTrue(s"Maximum consumer throttle too high: ${metricValue(maxMetric)}", + metricValue(maxMetric) <= max)) + } else + assertEquals("Should not have been throttled", 0.0, metricValue(maxMetric), 0.0) } def quotaProperties(producerQuota: Long, consumerQuota: Long, requestQuota: Double): Properties = { @@ -257,4 +286,19 @@ abstract class BaseQuotaTest extends IntegrationTestHarness { props.put(DynamicConfig.Client.RequestPercentageOverrideProp, requestQuota.toString) props } + + def waitForQuotaUpdate(producerQuota: Long, consumerQuota: Long, requestQuota: Double, server: KafkaServer = leaderNode) { + TestUtils.retry(10000) { + val quotaManagers = server.apis.quotas + val overrideProducerQuota = quota(quotaManagers.produce, userPrincipal, producerClientId) + val overrideConsumerQuota = quota(quotaManagers.fetch, userPrincipal, consumerClientId) + val overrideProducerRequestQuota = quota(quotaManagers.request, userPrincipal, producerClientId) + val overrideConsumerRequestQuota = quota(quotaManagers.request, userPrincipal, consumerClientId) + + assertEquals(s"ClientId $producerClientId of user $userPrincipal must have producer quota", Quota.upperBound(producerQuota), overrideProducerQuota) + assertEquals(s"ClientId $consumerClientId of user $userPrincipal must have consumer quota", Quota.upperBound(consumerQuota), overrideConsumerQuota) + assertEquals(s"ClientId $producerClientId of user $userPrincipal must have request quota", Quota.upperBound(requestQuota), overrideProducerRequestQuota) + assertEquals(s"ClientId $consumerClientId of user $userPrincipal must have request quota", Quota.upperBound(requestQuota), overrideConsumerRequestQuota) + } + } } diff --git a/core/src/test/scala/integration/kafka/api/ClientIdQuotaTest.scala b/core/src/test/scala/integration/kafka/api/ClientIdQuotaTest.scala index 3e0832790b5de..b084b3ca5ea89 100644 --- a/core/src/test/scala/integration/kafka/api/ClientIdQuotaTest.scala +++ b/core/src/test/scala/integration/kafka/api/ClientIdQuotaTest.scala @@ -16,18 +16,15 @@ package kafka.api import java.util.Properties -import kafka.server.{DynamicConfig, KafkaConfig, QuotaId} +import kafka.server.{DynamicConfig, KafkaConfig, KafkaServer} import org.apache.kafka.common.security.auth.KafkaPrincipal import org.apache.kafka.common.utils.Sanitizer import org.junit.Before class ClientIdQuotaTest extends BaseQuotaTest { - override val userPrincipal = KafkaPrincipal.ANONYMOUS.getName override def producerClientId = "QuotasTestProducer-!@#$%^&*()" override def consumerClientId = "QuotasTestConsumer-!@#$%^&*()" - override val producerQuotaId = QuotaId(None, Some(producerClientId), Some(Sanitizer.sanitize(producerClientId))) - override val consumerQuotaId = QuotaId(None, Some(consumerClientId), Some(Sanitizer.sanitize(consumerClientId))) @Before override def setUp() { @@ -35,24 +32,35 @@ class ClientIdQuotaTest extends BaseQuotaTest { this.serverConfig.setProperty(KafkaConfig.ConsumerQuotaBytesPerSecondDefaultProp, defaultConsumerQuota.toString) super.setUp() } - override def overrideQuotas(producerQuota: Long, consumerQuota: Long, requestQuota: Double) { - val producerProps = new Properties() - producerProps.put(DynamicConfig.Client.ProducerByteRateOverrideProp, producerQuota.toString) - producerProps.put(DynamicConfig.Client.RequestPercentageOverrideProp, requestQuota.toString) - updateQuotaOverride(producerClientId, producerProps) - - val consumerProps = new Properties() - consumerProps.put(DynamicConfig.Client.ConsumerByteRateOverrideProp, consumerQuota.toString) - consumerProps.put(DynamicConfig.Client.RequestPercentageOverrideProp, requestQuota.toString) - updateQuotaOverride(consumerClientId, consumerProps) - } - override def removeQuotaOverrides() { - val emptyProps = new Properties - updateQuotaOverride(producerClientId, emptyProps) - updateQuotaOverride(consumerClientId, emptyProps) - } - private def updateQuotaOverride(clientId: String, properties: Properties) { - adminZkClient.changeClientIdConfig(Sanitizer.sanitize(clientId), properties) + override def createQuotaTestClients(topic: String, leaderNode: KafkaServer): QuotaTestClients = { + new QuotaTestClients(topic, leaderNode, producerClientId, consumerClientId, producers.head, consumers.head) { + override def userPrincipal: KafkaPrincipal = KafkaPrincipal.ANONYMOUS + override def quotaMetricTags(clientId: String): Map[String, String] = { + Map("user" -> "", "client-id" -> clientId) + } + + override def overrideQuotas(producerQuota: Long, consumerQuota: Long, requestQuota: Double) { + val producerProps = new Properties() + producerProps.put(DynamicConfig.Client.ProducerByteRateOverrideProp, producerQuota.toString) + producerProps.put(DynamicConfig.Client.RequestPercentageOverrideProp, requestQuota.toString) + updateQuotaOverride(producerClientId, producerProps) + + val consumerProps = new Properties() + consumerProps.put(DynamicConfig.Client.ConsumerByteRateOverrideProp, consumerQuota.toString) + consumerProps.put(DynamicConfig.Client.RequestPercentageOverrideProp, requestQuota.toString) + updateQuotaOverride(consumerClientId, consumerProps) + } + + override def removeQuotaOverrides() { + val emptyProps = new Properties + updateQuotaOverride(producerClientId, emptyProps) + updateQuotaOverride(consumerClientId, emptyProps) + } + + private def updateQuotaOverride(clientId: String, properties: Properties) { + adminZkClient.changeClientIdConfig(Sanitizer.sanitize(clientId), properties) + } + } } } diff --git a/core/src/test/scala/integration/kafka/api/CustomQuotaCallbackTest.scala b/core/src/test/scala/integration/kafka/api/CustomQuotaCallbackTest.scala new file mode 100644 index 0000000000000..018971b1f5ee8 --- /dev/null +++ b/core/src/test/scala/integration/kafka/api/CustomQuotaCallbackTest.scala @@ -0,0 +1,457 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + **/ + +package kafka.api + +import java.io.File +import java.{lang, util} +import java.util.concurrent.{ConcurrentHashMap, TimeUnit} +import java.util.concurrent.atomic.{AtomicBoolean, AtomicInteger} +import java.util.{Collections, Properties} + +import kafka.api.GroupedUserPrincipalBuilder._ +import kafka.api.GroupedUserQuotaCallback._ +import kafka.server._ +import kafka.utils.JaasTestUtils.ScramLoginModule +import kafka.utils.{JaasTestUtils, Logging, TestUtils} +import kafka.zk.ConfigEntityChangeNotificationZNode +import org.apache.kafka.clients.admin.{AdminClient, AdminClientConfig} +import org.apache.kafka.clients.consumer.{ConsumerConfig, KafkaConsumer} +import org.apache.kafka.clients.producer.{KafkaProducer, ProducerConfig, ProducerRecord} +import org.apache.kafka.common.{Cluster, Reconfigurable} +import org.apache.kafka.common.config.SaslConfigs +import org.apache.kafka.common.errors.SaslAuthenticationException +import org.apache.kafka.common.network.ListenerName +import org.apache.kafka.common.security.auth._ +import org.apache.kafka.common.security.scram.ScramCredential +import org.apache.kafka.server.quota._ +import org.junit.Assert._ +import org.junit.{After, Before, Test} + +import scala.collection.mutable.ArrayBuffer +import scala.collection.JavaConverters._ + +class CustomQuotaCallbackTest extends IntegrationTestHarness with SaslSetup { + + override protected def securityProtocol = SecurityProtocol.SASL_SSL + override protected def listenerName = new ListenerName("CLIENT") + override protected def interBrokerListenerName: ListenerName = new ListenerName("BROKER") + + override protected lazy val trustStoreFile = Some(File.createTempFile("truststore", ".jks")) + override val consumerCount: Int = 0 + override val producerCount: Int = 0 + override val serverCount: Int = 2 + + private val kafkaServerSaslMechanisms = Seq("SCRAM-SHA-256") + private val kafkaClientSaslMechanism = "SCRAM-SHA-256" + override protected val serverSaslProperties = Some(kafkaServerSaslProperties(kafkaServerSaslMechanisms, kafkaClientSaslMechanism)) + override protected val clientSaslProperties = Some(kafkaClientSaslProperties(kafkaClientSaslMechanism)) + private val adminClients = new ArrayBuffer[AdminClient]() + private var producerWithoutQuota: KafkaProducer[Array[Byte], Array[Byte]] = _ + + val defaultRequestQuota = 1000 + val defaultProduceQuota = 2000 * 1000 * 1000 + val defaultConsumeQuota = 1000 * 1000 * 1000 + + @Before + override def setUp() { + startSasl(jaasSections(kafkaServerSaslMechanisms, Some("SCRAM-SHA-256"), KafkaSasl, JaasTestUtils.KafkaServerContextName)) + this.serverConfig.setProperty(KafkaConfig.ProducerQuotaBytesPerSecondDefaultProp, Long.MaxValue.toString) + this.serverConfig.setProperty(KafkaConfig.ConsumerQuotaBytesPerSecondDefaultProp, Long.MaxValue.toString) + this.serverConfig.setProperty(KafkaConfig.ClientQuotaCallbackClassProp, classOf[GroupedUserQuotaCallback].getName) + this.serverConfig.setProperty(s"${listenerName.configPrefix}${KafkaConfig.PrincipalBuilderClassProp}", + classOf[GroupedUserPrincipalBuilder].getName) + this.serverConfig.setProperty(KafkaConfig.DeleteTopicEnableProp, "true") + super.setUp() + brokerList = TestUtils.bootstrapServers(servers, listenerName) + + producerConfig.put(SaslConfigs.SASL_JAAS_CONFIG, + ScramLoginModule(JaasTestUtils.KafkaScramAdmin, JaasTestUtils.KafkaScramAdminPassword).toString) + producerWithoutQuota = createNewProducer + producers += producerWithoutQuota + } + + @After + override def tearDown(): Unit = { + // Close producers and consumers without waiting for requests to complete + // to avoid waiting for throttled responses + producers.foreach(_.close(0, TimeUnit.MILLISECONDS)) + producers.clear() + consumers.foreach(_.close(0, TimeUnit.MILLISECONDS)) + consumers.clear() + super.tearDown() + } + + override def configureSecurityBeforeServersStart() { + super.configureSecurityBeforeServersStart() + zkClient.makeSurePersistentPathExists(ConfigEntityChangeNotificationZNode.path) + createScramCredentials(zkConnect, JaasTestUtils.KafkaScramAdmin, JaasTestUtils.KafkaScramAdminPassword) + } + + @Test + def testCustomQuotaCallback() { + // Large quota override, should not throttle + var brokerId = 0 + var user = createGroupWithOneUser("group0_user1", brokerId) + user.configureAndWaitForQuota(1000000, 2000000) + quotaLimitCalls.values.foreach(_.set(0)) + user.produceConsume(expectProduceThrottle = false, expectConsumeThrottle = false) + + // ClientQuotaCallback#quotaLimit is invoked by each quota manager once for each new client + assertEquals(1, quotaLimitCalls(ClientQuotaType.PRODUCE).get) + assertEquals(1, quotaLimitCalls(ClientQuotaType.FETCH).get) + assertTrue(s"Too many quotaLimit calls $quotaLimitCalls", quotaLimitCalls(ClientQuotaType.REQUEST).get <= serverCount) + // Large quota updated to small quota, should throttle + user.configureAndWaitForQuota(9000, 3000) + user.produceConsume(expectProduceThrottle = true, expectConsumeThrottle = true) + + // Quota override deletion - verify default quota applied (large quota, no throttling) + user = addUser("group0_user2", brokerId) + user.removeQuotaOverrides() + user.waitForQuotaUpdate(defaultProduceQuota, defaultConsumeQuota, defaultRequestQuota) + user.removeThrottleMetrics() // since group was throttled before + user.produceConsume(expectProduceThrottle = false, expectConsumeThrottle = false) + + // Make default quota smaller, should throttle + user.configureAndWaitForQuota(8000, 2500, divisor = 1, group = None) + user.produceConsume(expectProduceThrottle = true, expectConsumeThrottle = true) + + // Configure large quota override, should not throttle + user = addUser("group0_user3", brokerId) + user.configureAndWaitForQuota(2000000, 2000000) + user.removeThrottleMetrics() // since group was throttled before + user.produceConsume(expectProduceThrottle = false, expectConsumeThrottle = false) + + // Quota large enough for one partition, should not throttle + brokerId = 1 + user = createGroupWithOneUser("group1_user1", brokerId) + user.configureAndWaitForQuota(8000 * 100, 2500 * 100) + user.produceConsume(expectProduceThrottle = false, expectConsumeThrottle = false) + + // Create large number of partitions on another broker, should result in throttling on first partition + val largeTopic = "group1_largeTopic" + createTopic(largeTopic, numPartitions = 99, leader = 0) + user.waitForQuotaUpdate(8000, 2500, defaultRequestQuota) + user.produceConsume(expectProduceThrottle = true, expectConsumeThrottle = true) + + // Remove quota override and test default quota applied with scaling based on partitions + user = addUser("group1_user2", brokerId) + user.waitForQuotaUpdate(defaultProduceQuota / 100, defaultConsumeQuota / 100, defaultRequestQuota) + user.removeThrottleMetrics() // since group was throttled before + user.produceConsume(expectProduceThrottle = false, expectConsumeThrottle = false) + user.configureAndWaitForQuota(8000 * 100, 2500 * 100, divisor=100, group = None) + user.produceConsume(expectProduceThrottle = true, expectConsumeThrottle = true) + + // Remove the second topic with large number of partitions, verify no longer throttled + adminZkClient.deleteTopic(largeTopic) + user = addUser("group1_user3", brokerId) + user.waitForQuotaUpdate(8000 * 100, 2500 * 100, defaultRequestQuota) + user.removeThrottleMetrics() // since group was throttled before + user.produceConsume(expectProduceThrottle = false, expectConsumeThrottle = false) + + // Alter configs of custom callback dynamically + val adminClient = createAdminClient() + val newProps = new Properties + newProps.put(GroupedUserQuotaCallback.DefaultProduceQuotaProp, "8000") + newProps.put(GroupedUserQuotaCallback.DefaultFetchQuotaProp, "2500") + TestUtils.alterConfigs(servers, adminClient, newProps, perBrokerConfig = false) + user.waitForQuotaUpdate(8000, 2500, defaultRequestQuota) + user.produceConsume(expectProduceThrottle = true, expectConsumeThrottle = true) + + assertEquals(serverCount, callbackInstances.get) + } + + /** + * Creates a group with one user and one topic with one partition. + * @param firstUser First user to create in the group + * @param brokerId The broker id to use as leader of the partition + */ + private def createGroupWithOneUser(firstUser: String, brokerId: Int): GroupedUser = { + val user = addUser(firstUser, brokerId) + createTopic(user.topic, numPartitions = 1, brokerId) + user.configureAndWaitForQuota(defaultProduceQuota, defaultConsumeQuota, divisor = 1, group = None) + user + } + + private def createTopic(topic: String, numPartitions: Int, leader: Int): Unit = { + val assignment = (0 until numPartitions).map { i => i -> Seq(leader) }.toMap + TestUtils.createTopic(zkClient, topic, assignment, servers) + } + + private def createAdminClient(): AdminClient = { + val config = new util.HashMap[String, Object] + config.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, + TestUtils.bootstrapServers(servers, new ListenerName("BROKER"))) + clientSecurityProps("admin-client").asInstanceOf[util.Map[Object, Object]].asScala.foreach { case (key, value) => + config.put(key.toString, value) + } + config.put(SaslConfigs.SASL_JAAS_CONFIG, + ScramLoginModule(JaasTestUtils.KafkaScramAdmin, JaasTestUtils.KafkaScramAdminPassword).toString) + val adminClient = AdminClient.create(config) + adminClients += adminClient + adminClient + } + + private def produceWithoutThrottle(topic: String, numRecords: Int): Unit = { + (0 until numRecords).foreach { i => + val payload = i.toString.getBytes + producerWithoutQuota.send(new ProducerRecord[Array[Byte], Array[Byte]](topic, null, null, payload)) + } + } + + private def addUser(user: String, leader: Int): GroupedUser = { + + val password = s"$user:secret" + createScramCredentials(zkConnect, user, password) + servers.foreach { server => + val cache = server.credentialProvider.credentialCache.cache(kafkaClientSaslMechanism, classOf[ScramCredential]) + TestUtils.waitUntilTrue(() => cache.get(user) != null, "SCRAM credentials not created") + } + + val userGroup = group(user) + val topic = s"${userGroup}_topic" + val producerClientId = s"$user:producer-client-id" + val consumerClientId = s"$user:producer-client-id" + + producerConfig.put(ProducerConfig.CLIENT_ID_CONFIG, producerClientId) + producerConfig.put(SaslConfigs.SASL_JAAS_CONFIG, ScramLoginModule(user, password).toString) + val producer = createNewProducer + producers += producer + + consumerConfig.put(ConsumerConfig.CLIENT_ID_CONFIG, consumerClientId) + consumerConfig.put(ConsumerConfig.GROUP_ID_CONFIG, s"$user-group") + consumerConfig.put(SaslConfigs.SASL_JAAS_CONFIG, ScramLoginModule(user, password).toString) + val consumer = createNewConsumer + consumers += consumer + + GroupedUser(user, userGroup, topic, servers(leader), producerClientId, consumerClientId, producer, consumer) + } + + case class GroupedUser(user: String, userGroup: String, topic: String, leaderNode: KafkaServer, + producerClientId: String, consumerClientId: String, + producer: KafkaProducer[Array[Byte], Array[Byte]], + consumer: KafkaConsumer[Array[Byte], Array[Byte]]) extends + QuotaTestClients(topic, leaderNode, producerClientId, consumerClientId, producer, consumer) { + + override def userPrincipal: KafkaPrincipal = GroupedUserPrincipal(user, userGroup) + + override def quotaMetricTags(clientId: String): Map[String, String] = { + Map(GroupedUserQuotaCallback.QuotaGroupTag -> userGroup) + } + + override def overrideQuotas(producerQuota: Long, consumerQuota: Long, requestQuota: Double): Unit = { + configureQuota(userGroup, producerQuota, consumerQuota, requestQuota) + } + + override def removeQuotaOverrides(): Unit = { + adminZkClient.changeUserOrUserClientIdConfig(quotaEntityName(userGroup), new Properties) + } + + def configureQuota(userGroup: String, producerQuota: Long, consumerQuota: Long, requestQuota: Double): Unit = { + val quotaProps = quotaProperties(producerQuota, consumerQuota, requestQuota) + adminZkClient.changeUserOrUserClientIdConfig(quotaEntityName(userGroup), quotaProps) + } + + def configureAndWaitForQuota(produceQuota: Long, fetchQuota: Long, divisor: Int = 1, + group: Option[String] = Some(userGroup)): Unit = { + configureQuota(group.getOrElse(""), produceQuota, fetchQuota, defaultRequestQuota) + waitForQuotaUpdate(produceQuota / divisor, fetchQuota / divisor, defaultRequestQuota) + } + + def produceConsume(expectProduceThrottle: Boolean, expectConsumeThrottle: Boolean): Unit = { + val numRecords = 1000 + val produced = produceUntilThrottled(numRecords, waitForRequestCompletion = false) + verifyProduceThrottle(expectProduceThrottle, verifyClientMetric = false) + // make sure there are enough records on the topic to test consumer throttling + produceWithoutThrottle(topic, numRecords - produced) + consumeUntilThrottled(numRecords, waitForRequestCompletion = false) + verifyConsumeThrottle(expectConsumeThrottle, verifyClientMetric = false) + } + + def removeThrottleMetrics(): Unit = { + def removeSensors(quotaType: QuotaType, clientId: String): Unit = { + val sensorSuffix = quotaMetricTags(clientId).values.mkString(":") + leaderNode.metrics.removeSensor(s"${quotaType}ThrottleTime-$sensorSuffix") + leaderNode.metrics.removeSensor(s"$quotaType-$sensorSuffix") + } + removeSensors(QuotaType.Produce, producerClientId) + removeSensors(QuotaType.Fetch, consumerClientId) + removeSensors(QuotaType.Request, producerClientId) + removeSensors(QuotaType.Request, consumerClientId) + } + + private def quotaEntityName(userGroup: String): String = s"${userGroup}_" + } +} + +object GroupedUserPrincipalBuilder { + def group(str: String): String = { + if (str.indexOf("_") <= 0) + "" + else + str.substring(0, str.indexOf("_")) + } +} + +class GroupedUserPrincipalBuilder extends KafkaPrincipalBuilder { + override def build(context: AuthenticationContext): KafkaPrincipal = { + val securityProtocol = context.securityProtocol + if (securityProtocol == SecurityProtocol.SASL_PLAINTEXT || securityProtocol == SecurityProtocol.SASL_SSL) { + val user = context.asInstanceOf[SaslAuthenticationContext].server().getAuthorizationID + val userGroup = group(user) + if (userGroup.isEmpty) + new KafkaPrincipal(KafkaPrincipal.USER_TYPE, user) + else + GroupedUserPrincipal(user, userGroup) + } else + throw new IllegalStateException(s"Unexpected security protocol $securityProtocol") + } +} + +case class GroupedUserPrincipal(user: String, userGroup: String) extends KafkaPrincipal(KafkaPrincipal.USER_TYPE, user) + +object GroupedUserQuotaCallback { + val QuotaGroupTag = "group" + val DefaultProduceQuotaProp = "default.produce.quota" + val DefaultFetchQuotaProp = "default.fetch.quota" + val UnlimitedQuotaMetricTags = Collections.emptyMap[String, String] + val quotaLimitCalls = Map( + ClientQuotaType.PRODUCE -> new AtomicInteger, + ClientQuotaType.FETCH -> new AtomicInteger, + ClientQuotaType.REQUEST -> new AtomicInteger + ) + val callbackInstances = new AtomicInteger +} + +/** + * Quota callback for a grouped user. Both user principals and topics of each group + * are prefixed with the group name followed by '_'. This callback defines quotas of different + * types at the group level. Group quotas are configured in ZooKeeper as user quotas with + * the entity name "${group}_". Default group quotas are configured in ZooKeeper as user quotas + * with the entity name "_". + * + * Default group quotas may also be configured using the configuration options + * "default.produce.quota" and "default.fetch.quota" which can be reconfigured dynamically + * without restarting the broker. This tests custom reconfigurable options for quota callbacks, + */ +class GroupedUserQuotaCallback extends ClientQuotaCallback with Reconfigurable with Logging { + + var brokerId: Int = -1 + val customQuotasUpdated = ClientQuotaType.values.toList + .map(quotaType =>(quotaType -> new AtomicBoolean)).toMap + val quotas = ClientQuotaType.values.toList + .map(quotaType => (quotaType -> new ConcurrentHashMap[String, Double])).toMap + + val partitionRatio = new ConcurrentHashMap[String, Double]() + + override def configure(configs: util.Map[String, _]): Unit = { + brokerId = configs.get(KafkaConfig.BrokerIdProp).toString.toInt + callbackInstances.incrementAndGet + } + + override def reconfigurableConfigs: util.Set[String] = { + Set(DefaultProduceQuotaProp, DefaultFetchQuotaProp).asJava + } + + override def validateReconfiguration(configs: util.Map[String, _]): Unit = { + reconfigurableConfigs.asScala.foreach(configValue(configs, _)) + } + + override def reconfigure(configs: util.Map[String, _]): Unit = { + configValue(configs, DefaultProduceQuotaProp).foreach(value => quotas(ClientQuotaType.PRODUCE).put("", value)) + configValue(configs, DefaultFetchQuotaProp).foreach(value => quotas(ClientQuotaType.FETCH).put("", value)) + customQuotasUpdated.values.foreach(_.set(true)) + } + + private def configValue(configs: util.Map[String, _], key: String): Option[Long] = { + val value = configs.get(key) + if (value != null) Some(value.toString.toLong) else None + } + + override def quotaMetricTags(quotaType: ClientQuotaType, principal: KafkaPrincipal, clientId: String): util.Map[String, String] = { + principal match { + case groupPrincipal: GroupedUserPrincipal => + val userGroup = groupPrincipal.userGroup + val quotaLimit = quotaOrDefault(userGroup, quotaType) + if (quotaLimit != null) + Map(QuotaGroupTag -> userGroup).asJava + else + UnlimitedQuotaMetricTags + case _ => + UnlimitedQuotaMetricTags + } + } + + override def quotaLimit(quotaType: ClientQuotaType, metricTags: util.Map[String, String]): lang.Double = { + quotaLimitCalls(quotaType).incrementAndGet + val group = metricTags.get(QuotaGroupTag) + if (group != null) quotaOrDefault(group, quotaType) else null + } + + override def updateClusterMetadata(cluster: Cluster): Boolean = { + val topicsByGroup = cluster.topics.asScala.groupBy(group) + + !topicsByGroup.forall { case (group, groupTopics) => + val groupPartitions = groupTopics.flatMap(topic => cluster.partitionsForTopic(topic).asScala) + val totalPartitions = groupPartitions.size + val partitionsOnThisBroker = groupPartitions.count { p => p.leader != null && p.leader.id == brokerId } + val multiplier = if (totalPartitions == 0) + 1 + else if (partitionsOnThisBroker == 0) + 1.0 / totalPartitions + else + partitionsOnThisBroker.toDouble / totalPartitions + partitionRatio.put(group, multiplier) != multiplier + } + } + + override def updateQuota(quotaType: ClientQuotaType, quotaEntity: ClientQuotaEntity, newValue: Double): Unit = { + quotas(quotaType).put(userGroup(quotaEntity), newValue) + } + + override def removeQuota(quotaType: ClientQuotaType, quotaEntity: ClientQuotaEntity): Unit = { + quotas(quotaType).remove(userGroup(quotaEntity)) + } + + override def quotaResetRequired(quotaType: ClientQuotaType): Boolean = customQuotasUpdated(quotaType).getAndSet(false) + + def close(): Unit = {} + + private def userGroup(quotaEntity: ClientQuotaEntity): String = { + val configEntity = quotaEntity.configEntities.get(0) + if (configEntity.entityType == ClientQuotaEntity.ConfigEntityType.USER) + group(configEntity.name) + else + throw new IllegalArgumentException(s"Config entity type ${configEntity.entityType} is not supported") + } + + private def quotaOrDefault(group: String, quotaType: ClientQuotaType): lang.Double = { + val quotaMap = quotas(quotaType) + var quotaLimit: Any = quotaMap.get(group) + if (quotaLimit == null) + quotaLimit = quotaMap.get("") + if (quotaLimit != null) scaledQuota(quotaType, group, quotaLimit.asInstanceOf[Double]) else null + } + + private def scaledQuota(quotaType: ClientQuotaType, group: String, configuredQuota: Double): Double = { + if (quotaType == ClientQuotaType.REQUEST) + configuredQuota + else { + val multiplier = partitionRatio.get(group) + if (multiplier <= 0.0) configuredQuota else configuredQuota * multiplier + } + } +} + + diff --git a/core/src/test/scala/integration/kafka/api/DelegationTokenEndToEndAuthorizationTest.scala b/core/src/test/scala/integration/kafka/api/DelegationTokenEndToEndAuthorizationTest.scala index 24f435e9fb152..56a3b8a3f392a 100644 --- a/core/src/test/scala/integration/kafka/api/DelegationTokenEndToEndAuthorizationTest.scala +++ b/core/src/test/scala/integration/kafka/api/DelegationTokenEndToEndAuthorizationTest.scala @@ -18,13 +18,12 @@ package kafka.api import java.util -import kafka.admin.AdminClient import kafka.server.KafkaConfig import kafka.utils.{JaasTestUtils, TestUtils, ZkUtils} -import org.apache.kafka.clients.admin.AdminClientConfig +import org.apache.kafka.clients.admin.{AdminClient, AdminClientConfig} import org.apache.kafka.common.config.SaslConfigs import org.apache.kafka.common.security.auth.SecurityProtocol -import org.apache.kafka.common.security.scram.ScramMechanism +import org.apache.kafka.common.security.scram.internal.ScramMechanism import org.apache.kafka.common.security.token.delegation.DelegationToken import org.junit.Before @@ -83,9 +82,8 @@ class DelegationTokenEndToEndAuthorizationTest extends EndToEndAuthorizationTest val clientLoginContext = jaasClientLoginModule(kafkaClientSaslMechanism) config.put(SaslConfigs.SASL_JAAS_CONFIG, clientLoginContext) - val adminClient = AdminClient.create(config.asScala.toMap) - val (error, token) = adminClient.createToken(List()) - + val adminClient = AdminClient.create(config) + val token = adminClient.createDelegationToken().delegationToken().get() //wait for token to reach all the brokers TestUtils.waitUntilTrue(() => servers.forall(server => !server.tokenCache.tokens().isEmpty), "Timed out waiting for token to propagate to all servers") diff --git a/core/src/test/scala/integration/kafka/api/SaslEndToEndAuthorizationTest.scala b/core/src/test/scala/integration/kafka/api/SaslEndToEndAuthorizationTest.scala index f5409b1e47b8f..a5bf33171a4a0 100644 --- a/core/src/test/scala/integration/kafka/api/SaslEndToEndAuthorizationTest.scala +++ b/core/src/test/scala/integration/kafka/api/SaslEndToEndAuthorizationTest.scala @@ -63,6 +63,7 @@ abstract class SaslEndToEndAuthorizationTest extends EndToEndAuthorizationTest { consumer2Config ++= consumerConfig // consumer2 retrieves its credentials from the static JAAS configuration, so we test also this path consumer2Config.remove(SaslConfigs.SASL_JAAS_CONFIG) + consumer2Config.remove(SaslConfigs.SASL_CLIENT_CALLBACK_HANDLER_CLASS) val consumer2 = TestUtils.createNewConsumer(brokerList, securityProtocol = securityProtocol, diff --git a/core/src/test/scala/integration/kafka/api/SaslPlainSslEndToEndAuthorizationTest.scala b/core/src/test/scala/integration/kafka/api/SaslPlainSslEndToEndAuthorizationTest.scala index 08351aa0c7a3d..efb8c48c7c48a 100644 --- a/core/src/test/scala/integration/kafka/api/SaslPlainSslEndToEndAuthorizationTest.scala +++ b/core/src/test/scala/integration/kafka/api/SaslPlainSslEndToEndAuthorizationTest.scala @@ -16,22 +16,33 @@ */ package kafka.api -import kafka.utils.{CoreUtils, JaasTestUtils, TestUtils, ZkUtils} +import java.security.AccessController +import javax.security.auth.callback._ +import javax.security.auth.Subject +import javax.security.auth.login.AppConfigurationEntry + +import kafka.server.KafkaConfig +import kafka.utils.{CoreUtils, TestUtils, ZkUtils} +import kafka.utils.JaasTestUtils._ +import org.apache.kafka.common.config.SaslConfigs import org.apache.kafka.common.config.internals.BrokerSecurityConfigs +import org.apache.kafka.common.network.ListenerName import org.apache.kafka.common.security.JaasUtils -import org.apache.kafka.common.security.auth.{AuthenticationContext, KafkaPrincipal, KafkaPrincipalBuilder, SaslAuthenticationContext} +import org.apache.kafka.common.security.auth._ +import org.apache.kafka.common.security.plain.PlainAuthenticateCallback import org.junit.Test object SaslPlainSslEndToEndAuthorizationTest { + class TestPrincipalBuilder extends KafkaPrincipalBuilder { override def build(context: AuthenticationContext): KafkaPrincipal = { context match { case ctx: SaslAuthenticationContext => ctx.server.getAuthorizationID match { - case JaasTestUtils.KafkaPlainAdmin => + case KafkaPlainAdmin => new KafkaPrincipal(KafkaPrincipal.USER_TYPE, "admin") - case JaasTestUtils.KafkaPlainUser => + case KafkaPlainUser => new KafkaPrincipal(KafkaPrincipal.USER_TYPE, "user") case _ => KafkaPrincipal.ANONYMOUS @@ -39,18 +50,84 @@ object SaslPlainSslEndToEndAuthorizationTest { } } } + + object Credentials { + val allUsers = Map(KafkaPlainUser -> "user1-password", + KafkaPlainUser2 -> KafkaPlainPassword2, + KafkaPlainAdmin -> "broker-password") + } + + class TestServerCallbackHandler extends AuthenticateCallbackHandler { + def configure(configs: java.util.Map[String, _], saslMechanism: String, jaasConfigEntries: java.util.List[AppConfigurationEntry]) {} + def handle(callbacks: Array[Callback]) { + var username: String = null + for (callback <- callbacks) { + if (callback.isInstanceOf[NameCallback]) + username = callback.asInstanceOf[NameCallback].getDefaultName + else if (callback.isInstanceOf[PlainAuthenticateCallback]) { + val plainCallback = callback.asInstanceOf[PlainAuthenticateCallback] + plainCallback.authenticated(Credentials.allUsers(username) == new String(plainCallback.password)) + } else + throw new UnsupportedCallbackException(callback) + } + } + def close() {} + } + + class TestClientCallbackHandler extends AuthenticateCallbackHandler { + def configure(configs: java.util.Map[String, _], saslMechanism: String, jaasConfigEntries: java.util.List[AppConfigurationEntry]) {} + def handle(callbacks: Array[Callback]) { + val subject = Subject.getSubject(AccessController.getContext()) + val username = subject.getPublicCredentials(classOf[String]).iterator().next() + for (callback <- callbacks) { + if (callback.isInstanceOf[NameCallback]) + callback.asInstanceOf[NameCallback].setName(username) + else if (callback.isInstanceOf[PasswordCallback]) { + if (username == KafkaPlainUser || username == KafkaPlainAdmin) + callback.asInstanceOf[PasswordCallback].setPassword(Credentials.allUsers(username).toCharArray) + } else + throw new UnsupportedCallbackException(callback) + } + } + def close() {} + } } + +// This test uses SASL callback handler overrides for server connections of Kafka broker +// and client connections of Kafka producers and consumers. Client connections from Kafka brokers +// used for inter-broker communication also use custom callback handlers. The second client used in +// the multi-user test SaslEndToEndAuthorizationTest#testTwoConsumersWithDifferentSaslCredentials uses +// static JAAS configuration with default callback handlers to test those code paths as well. class SaslPlainSslEndToEndAuthorizationTest extends SaslEndToEndAuthorizationTest { - import SaslPlainSslEndToEndAuthorizationTest.TestPrincipalBuilder + import SaslPlainSslEndToEndAuthorizationTest._ this.serverConfig.setProperty(BrokerSecurityConfigs.PRINCIPAL_BUILDER_CLASS_CONFIG, classOf[TestPrincipalBuilder].getName) + this.serverConfig.put(KafkaConfig.SaslClientCallbackHandlerClassProp, classOf[TestClientCallbackHandler].getName) + val mechanismPrefix = ListenerName.forSecurityProtocol(SecurityProtocol.SASL_SSL).saslMechanismConfigPrefix("PLAIN") + this.serverConfig.put(s"$mechanismPrefix${KafkaConfig.SaslServerCallbackHandlerClassProp}", classOf[TestServerCallbackHandler].getName) + this.producerConfig.put(SaslConfigs.SASL_CLIENT_CALLBACK_HANDLER_CLASS, classOf[TestClientCallbackHandler].getName) + this.consumerConfig.put(SaslConfigs.SASL_CLIENT_CALLBACK_HANDLER_CLASS, classOf[TestClientCallbackHandler].getName) + private val plainLogin = s"org.apache.kafka.common.security.plain.PlainLoginModule username=$KafkaPlainUser required;" + this.producerConfig.put(SaslConfigs.SASL_JAAS_CONFIG, plainLogin) + this.consumerConfig.put(SaslConfigs.SASL_JAAS_CONFIG, plainLogin) override protected def kafkaClientSaslMechanism = "PLAIN" override protected def kafkaServerSaslMechanisms = List("PLAIN") + override val clientPrincipal = "user" override val kafkaPrincipal = "admin" + override def jaasSections(kafkaServerSaslMechanisms: Seq[String], + kafkaClientSaslMechanism: Option[String], + mode: SaslSetupMode, + kafkaServerEntryName: String): Seq[JaasSection] = { + val brokerLogin = new PlainLoginModule(KafkaPlainAdmin, "") // Password provided by callback handler + val clientLogin = new PlainLoginModule(KafkaPlainUser2, KafkaPlainPassword2) + Seq(JaasSection(kafkaServerEntryName, Seq(brokerLogin)), + JaasSection(KafkaClientContextName, Seq(clientLogin))) ++ zkSections + } + /** * Checks that secure paths created by broker and acl paths created by AclCommand * have expected ACLs. diff --git a/core/src/test/scala/integration/kafka/api/SaslScramSslEndToEndAuthorizationTest.scala b/core/src/test/scala/integration/kafka/api/SaslScramSslEndToEndAuthorizationTest.scala index 000fc219b4914..d304ffc1cae84 100644 --- a/core/src/test/scala/integration/kafka/api/SaslScramSslEndToEndAuthorizationTest.scala +++ b/core/src/test/scala/integration/kafka/api/SaslScramSslEndToEndAuthorizationTest.scala @@ -16,9 +16,9 @@ */ package kafka.api -import org.apache.kafka.common.security.scram.ScramMechanism import kafka.utils.JaasTestUtils import kafka.zk.ConfigEntityChangeNotificationZNode +import org.apache.kafka.common.security.scram.internal.ScramMechanism import scala.collection.JavaConverters._ import org.junit.Before diff --git a/core/src/test/scala/integration/kafka/api/SaslSetup.scala b/core/src/test/scala/integration/kafka/api/SaslSetup.scala index 273b24759e9c8..ab2819e0ed674 100644 --- a/core/src/test/scala/integration/kafka/api/SaslSetup.scala +++ b/core/src/test/scala/integration/kafka/api/SaslSetup.scala @@ -30,7 +30,7 @@ import org.apache.kafka.common.config.SaslConfigs import org.apache.kafka.common.config.internals.BrokerSecurityConfigs import org.apache.kafka.common.security.JaasUtils import org.apache.kafka.common.security.authenticator.LoginManager -import org.apache.kafka.common.security.scram.ScramMechanism +import org.apache.kafka.common.security.scram.internal.ScramMechanism /* * Implements an enumeration for the modes enabled here: diff --git a/core/src/test/scala/integration/kafka/api/UserClientIdQuotaTest.scala b/core/src/test/scala/integration/kafka/api/UserClientIdQuotaTest.scala index 453ac910269b1..47c8f5fa1da62 100644 --- a/core/src/test/scala/integration/kafka/api/UserClientIdQuotaTest.scala +++ b/core/src/test/scala/integration/kafka/api/UserClientIdQuotaTest.scala @@ -18,7 +18,7 @@ import java.io.File import java.util.Properties import kafka.server._ -import org.apache.kafka.common.security.auth.SecurityProtocol +import org.apache.kafka.common.security.auth.{KafkaPrincipal, SecurityProtocol} import org.apache.kafka.common.utils.Sanitizer import org.junit.Before @@ -27,11 +27,8 @@ class UserClientIdQuotaTest extends BaseQuotaTest { override protected def securityProtocol = SecurityProtocol.SSL override protected lazy val trustStoreFile = Some(File.createTempFile("truststore", ".jks")) - override val userPrincipal = "O=A client,CN=localhost" override def producerClientId = "QuotasTestProducer-!@#$%^&*()" override def consumerClientId = "QuotasTestConsumer-!@#$%^&*()" - override def producerQuotaId = QuotaId(Some(Sanitizer.sanitize(userPrincipal)), Some(producerClientId), Some(Sanitizer.sanitize(producerClientId))) - override def consumerQuotaId = QuotaId(Some(Sanitizer.sanitize(userPrincipal)), Some(consumerClientId), Some(Sanitizer.sanitize(consumerClientId))) @Before override def setUp() { @@ -39,30 +36,41 @@ class UserClientIdQuotaTest extends BaseQuotaTest { this.serverConfig.setProperty(KafkaConfig.ProducerQuotaBytesPerSecondDefaultProp, Long.MaxValue.toString) this.serverConfig.setProperty(KafkaConfig.ConsumerQuotaBytesPerSecondDefaultProp, Long.MaxValue.toString) super.setUp() - val defaultProps = quotaProperties(defaultProducerQuota, defaultConsumerQuota, defaultRequestQuota) + val defaultProps = quotaTestClients.quotaProperties(defaultProducerQuota, defaultConsumerQuota, defaultRequestQuota) adminZkClient.changeUserOrUserClientIdConfig(ConfigEntityName.Default + "/clients/" + ConfigEntityName.Default, defaultProps) - waitForQuotaUpdate(defaultProducerQuota, defaultConsumerQuota, defaultRequestQuota) + quotaTestClients.waitForQuotaUpdate(defaultProducerQuota, defaultConsumerQuota, defaultRequestQuota) } - override def overrideQuotas(producerQuota: Long, consumerQuota: Long, requestQuota: Double) { - val producerProps = new Properties() - producerProps.setProperty(DynamicConfig.Client.ProducerByteRateOverrideProp, producerQuota.toString) - producerProps.setProperty(DynamicConfig.Client.RequestPercentageOverrideProp, requestQuota.toString) - updateQuotaOverride(userPrincipal, producerClientId, producerProps) + override def createQuotaTestClients(topic: String, leaderNode: KafkaServer): QuotaTestClients = { + new QuotaTestClients(topic, leaderNode, producerClientId, consumerClientId, producers.head, consumers.head) { + override def userPrincipal: KafkaPrincipal = new KafkaPrincipal(KafkaPrincipal.USER_TYPE, "O=A client,CN=localhost") + override def quotaMetricTags(clientId: String): Map[String, String] = { + Map("user" -> Sanitizer.sanitize(userPrincipal.getName), "client-id" -> clientId) + } - val consumerProps = new Properties() - consumerProps.setProperty(DynamicConfig.Client.ConsumerByteRateOverrideProp, consumerQuota.toString) - consumerProps.setProperty(DynamicConfig.Client.RequestPercentageOverrideProp, requestQuota.toString) - updateQuotaOverride(userPrincipal, consumerClientId, consumerProps) - } + override def overrideQuotas(producerQuota: Long, consumerQuota: Long, requestQuota: Double) { + val producerProps = new Properties() + producerProps.setProperty(DynamicConfig.Client.ProducerByteRateOverrideProp, producerQuota.toString) + producerProps.setProperty(DynamicConfig.Client.RequestPercentageOverrideProp, requestQuota.toString) + updateQuotaOverride(userPrincipal.getName, producerClientId, producerProps) - override def removeQuotaOverrides() { - val emptyProps = new Properties - adminZkClient.changeUserOrUserClientIdConfig(Sanitizer.sanitize(userPrincipal) + "/clients/" + Sanitizer.sanitize(producerClientId), emptyProps) - adminZkClient.changeUserOrUserClientIdConfig(Sanitizer.sanitize(userPrincipal) + "/clients/" + Sanitizer.sanitize(consumerClientId), emptyProps) - } + val consumerProps = new Properties() + consumerProps.setProperty(DynamicConfig.Client.ConsumerByteRateOverrideProp, consumerQuota.toString) + consumerProps.setProperty(DynamicConfig.Client.RequestPercentageOverrideProp, requestQuota.toString) + updateQuotaOverride(userPrincipal.getName, consumerClientId, consumerProps) + } + + override def removeQuotaOverrides() { + val emptyProps = new Properties + adminZkClient.changeUserOrUserClientIdConfig(Sanitizer.sanitize(userPrincipal.getName) + + "/clients/" + Sanitizer.sanitize(producerClientId), emptyProps) + adminZkClient.changeUserOrUserClientIdConfig(Sanitizer.sanitize(userPrincipal.getName) + + "/clients/" + Sanitizer.sanitize(consumerClientId), emptyProps) + } - private def updateQuotaOverride(userPrincipal: String, clientId: String, properties: Properties) { - adminZkClient.changeUserOrUserClientIdConfig(Sanitizer.sanitize(userPrincipal) + "/clients/" + Sanitizer.sanitize(clientId), properties) + private def updateQuotaOverride(userPrincipal: String, clientId: String, properties: Properties) { + adminZkClient.changeUserOrUserClientIdConfig(Sanitizer.sanitize(userPrincipal) + "/clients/" + Sanitizer.sanitize(clientId), properties) + } + } } } diff --git a/core/src/test/scala/integration/kafka/api/UserQuotaTest.scala b/core/src/test/scala/integration/kafka/api/UserQuotaTest.scala index 91a92faf68f5c..3386c91ab81bb 100644 --- a/core/src/test/scala/integration/kafka/api/UserQuotaTest.scala +++ b/core/src/test/scala/integration/kafka/api/UserQuotaTest.scala @@ -17,9 +17,9 @@ package kafka.api import java.io.File import java.util.Properties -import kafka.server.{ConfigEntityName, KafkaConfig, QuotaId} +import kafka.server.{ConfigEntityName, KafkaConfig, KafkaServer} import kafka.utils.JaasTestUtils -import org.apache.kafka.common.security.auth.SecurityProtocol +import org.apache.kafka.common.security.auth.{KafkaPrincipal, SecurityProtocol} import org.apache.kafka.common.utils.Sanitizer import org.junit.{After, Before} @@ -32,20 +32,15 @@ class UserQuotaTest extends BaseQuotaTest with SaslSetup { override protected val serverSaslProperties = Some(kafkaServerSaslProperties(kafkaServerSaslMechanisms, kafkaClientSaslMechanism)) override protected val clientSaslProperties = Some(kafkaClientSaslProperties(kafkaClientSaslMechanism)) - override val userPrincipal = JaasTestUtils.KafkaClientPrincipalUnqualifiedName2 - override val producerQuotaId = QuotaId(Some(userPrincipal), None, None) - override val consumerQuotaId = QuotaId(Some(userPrincipal), None, None) - - @Before override def setUp() { startSasl(jaasSections(kafkaServerSaslMechanisms, Some("GSSAPI"), KafkaSasl, JaasTestUtils.KafkaServerContextName)) this.serverConfig.setProperty(KafkaConfig.ProducerQuotaBytesPerSecondDefaultProp, Long.MaxValue.toString) this.serverConfig.setProperty(KafkaConfig.ConsumerQuotaBytesPerSecondDefaultProp, Long.MaxValue.toString) super.setUp() - val defaultProps = quotaProperties(defaultProducerQuota, defaultConsumerQuota, defaultRequestQuota) + val defaultProps = quotaTestClients.quotaProperties(defaultProducerQuota, defaultConsumerQuota, defaultRequestQuota) adminZkClient.changeUserOrUserClientIdConfig(ConfigEntityName.Default, defaultProps) - waitForQuotaUpdate(defaultProducerQuota, defaultConsumerQuota, defaultRequestQuota) + quotaTestClients.waitForQuotaUpdate(defaultProducerQuota, defaultConsumerQuota, defaultRequestQuota) } @After @@ -54,18 +49,27 @@ class UserQuotaTest extends BaseQuotaTest with SaslSetup { closeSasl() } - override def overrideQuotas(producerQuota: Long, consumerQuota: Long, requestQuota: Double) { - val props = quotaProperties(producerQuota, consumerQuota, requestQuota) - updateQuotaOverride(props) - } + override def createQuotaTestClients(topic: String, leaderNode: KafkaServer): QuotaTestClients = { + new QuotaTestClients(topic, leaderNode, producerClientId, consumerClientId, producers.head, consumers.head) { + override val userPrincipal = new KafkaPrincipal(KafkaPrincipal.USER_TYPE, JaasTestUtils.KafkaClientPrincipalUnqualifiedName2) + override def quotaMetricTags(clientId: String): Map[String, String] = { + Map("user" -> userPrincipal.getName, "client-id" -> "") + } - override def removeQuotaOverrides() { - val emptyProps = new Properties - updateQuotaOverride(emptyProps) - updateQuotaOverride(emptyProps) - } + override def overrideQuotas(producerQuota: Long, consumerQuota: Long, requestQuota: Double) { + val props = quotaProperties(producerQuota, consumerQuota, requestQuota) + updateQuotaOverride(props) + } + + override def removeQuotaOverrides() { + val emptyProps = new Properties + updateQuotaOverride(emptyProps) + updateQuotaOverride(emptyProps) + } - private def updateQuotaOverride(properties: Properties) { - adminZkClient.changeUserOrUserClientIdConfig(Sanitizer.sanitize(userPrincipal), properties) + private def updateQuotaOverride(properties: Properties) { + adminZkClient.changeUserOrUserClientIdConfig(Sanitizer.sanitize(userPrincipal.getName), properties) + } + } } } diff --git a/core/src/test/scala/integration/kafka/server/DynamicBrokerReconfigurationTest.scala b/core/src/test/scala/integration/kafka/server/DynamicBrokerReconfigurationTest.scala index e0ab55c215841..bb62fb7fc6736 100644 --- a/core/src/test/scala/integration/kafka/server/DynamicBrokerReconfigurationTest.scala +++ b/core/src/test/scala/integration/kafka/server/DynamicBrokerReconfigurationTest.scala @@ -746,7 +746,7 @@ class DynamicBrokerReconfigurationTest extends ZooKeeperTestHarness with SaslSet val unknownConfig = "some.config" props.put(unknownConfig, "some.config.value") - alterConfigs(adminClients.head, props, perBrokerConfig = true).all.get + TestUtils.alterConfigs(servers, adminClients.head, props, perBrokerConfig = true).all.get TestUtils.waitUntilTrue(() => servers.forall(server => server.config.listeners.size == existingListenerCount + 1), "Listener config not updated") @@ -799,7 +799,7 @@ class DynamicBrokerReconfigurationTest extends ZooKeeperTestHarness with SaslSet listenerProps.foreach(props.remove) props.put(KafkaConfig.ListenersProp, listeners) props.put(KafkaConfig.ListenerSecurityProtocolMapProp, listenerMap) - alterConfigs(adminClients.head, props, perBrokerConfig = true).all.get + TestUtils.alterConfigs(servers, adminClients.head, props, perBrokerConfig = true).all.get TestUtils.waitUntilTrue(() => servers.forall(server => server.config.listeners.size == existingListenerCount - 1), "Listeners not updated") @@ -1054,20 +1054,6 @@ class DynamicBrokerReconfigurationTest extends ZooKeeperTestHarness with SaslSet assertTrue(s"Advertised listener update not propagated by controller: $endpoints", altered) } - private def alterConfigs(adminClient: AdminClient, props: Properties, perBrokerConfig: Boolean): AlterConfigsResult = { - val configEntries = props.asScala.map { case (k, v) => new ConfigEntry(k, v) }.toList.asJava - val newConfig = new Config(configEntries) - val configs = if (perBrokerConfig) { - servers.map { server => - val resource = new ConfigResource(ConfigResource.Type.BROKER, server.config.brokerId.toString) - (resource, newConfig) - }.toMap.asJava - } else { - Map(new ConfigResource(ConfigResource.Type.BROKER, "") -> newConfig).asJava - } - adminClient.alterConfigs(configs) - } - private def alterConfigsOnServer(server: KafkaServer, props: Properties): Unit = { val configEntries = props.asScala.map { case (k, v) => new ConfigEntry(k, v) }.toList.asJava val newConfig = new Config(configEntries) @@ -1077,7 +1063,7 @@ class DynamicBrokerReconfigurationTest extends ZooKeeperTestHarness with SaslSet } private def reconfigureServers(newProps: Properties, perBrokerConfig: Boolean, aPropToVerify: (String, String), expectFailure: Boolean = false): Unit = { - val alterResult = alterConfigs(adminClients.head, newProps, perBrokerConfig) + val alterResult = TestUtils.alterConfigs(servers, adminClients.head, newProps, perBrokerConfig) if (expectFailure) { val oldProps = servers.head.config.values.asScala.filterKeys(newProps.containsKey) val brokerResources = if (perBrokerConfig) diff --git a/core/src/test/scala/unit/kafka/admin/ConfigCommandTest.scala b/core/src/test/scala/unit/kafka/admin/ConfigCommandTest.scala index a17f060996b47..66e98f5b1cefa 100644 --- a/core/src/test/scala/unit/kafka/admin/ConfigCommandTest.scala +++ b/core/src/test/scala/unit/kafka/admin/ConfigCommandTest.scala @@ -28,7 +28,7 @@ import org.apache.kafka.clients.admin._ import org.apache.kafka.common.config.ConfigResource import org.apache.kafka.common.internals.KafkaFutureImpl import org.apache.kafka.common.Node -import org.apache.kafka.common.security.scram.ScramCredentialUtils +import org.apache.kafka.common.security.scram.internal.ScramCredentialUtils import org.apache.kafka.common.utils.Sanitizer import org.easymock.EasyMock import org.junit.Assert._ diff --git a/core/src/test/scala/unit/kafka/admin/DelegationTokenCommandTest.scala b/core/src/test/scala/unit/kafka/admin/DelegationTokenCommandTest.scala new file mode 100644 index 0000000000000..6ae8f5e83ac3c --- /dev/null +++ b/core/src/test/scala/unit/kafka/admin/DelegationTokenCommandTest.scala @@ -0,0 +1,147 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package kafka.admin + +import java.util + +import kafka.admin.DelegationTokenCommand.DelegationTokenCommandOptions +import kafka.api.{KafkaSasl, SaslSetup} +import kafka.server.{BaseRequestTest, KafkaConfig} +import kafka.utils.{JaasTestUtils, TestUtils} +import org.apache.kafka.clients.admin.AdminClientConfig +import org.apache.kafka.common.security.auth.SecurityProtocol +import org.junit.Assert._ +import org.junit.{After, Before, Test} + +import scala.collection.JavaConverters._ +import scala.collection.mutable.ListBuffer +import scala.concurrent.ExecutionException + +class DelegationTokenCommandTest extends BaseRequestTest with SaslSetup { + override protected def securityProtocol = SecurityProtocol.SASL_PLAINTEXT + private val kafkaClientSaslMechanism = "PLAIN" + private val kafkaServerSaslMechanisms = List("PLAIN") + protected override val serverSaslProperties = Some(kafkaServerSaslProperties(kafkaServerSaslMechanisms, kafkaClientSaslMechanism)) + protected override val clientSaslProperties = Some(kafkaClientSaslProperties(kafkaClientSaslMechanism)) + var adminClient: org.apache.kafka.clients.admin.AdminClient = null + + override def numBrokers = 1 + + @Before + override def setUp(): Unit = { + startSasl(jaasSections(kafkaServerSaslMechanisms, Some(kafkaClientSaslMechanism), KafkaSasl, JaasTestUtils.KafkaServerContextName)) + super.setUp() + } + + override def generateConfigs = { + val props = TestUtils.createBrokerConfigs(numBrokers, zkConnect, + enableControlledShutdown = false, enableDeleteTopic = true, + interBrokerSecurityProtocol = Some(securityProtocol), + trustStoreFile = trustStoreFile, saslProperties = serverSaslProperties, enableToken = true) + props.foreach(propertyOverrides) + props.map(KafkaConfig.fromProps) + } + + private def createAdminConfig():util.Map[String, Object] = { + val config = new util.HashMap[String, Object] + config.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, brokerList) + val securityProps: util.Map[Object, Object] = + TestUtils.adminClientSecurityConfigs(securityProtocol, trustStoreFile, clientSaslProperties) + securityProps.asScala.foreach { case (key, value) => config.put(key.asInstanceOf[String], value) } + config + } + + @Test + def testDelegationTokenRequests(): Unit = { + adminClient = org.apache.kafka.clients.admin.AdminClient.create(createAdminConfig) + val renewer1 = "User:renewer1" + val renewer2 = "User:renewer2" + + // create token1 with renewer1 + val tokenCreated = DelegationTokenCommand.createToken(adminClient, getCreateOpts(List(renewer1))) + + var tokens = DelegationTokenCommand.describeToken(adminClient, getDescribeOpts(List())) + assertTrue(tokens.size == 1) + val token1 = tokens.head + assertEquals(token1, tokenCreated) + + // create token2 with renewer2 + val token2 = DelegationTokenCommand.createToken(adminClient, getCreateOpts(List(renewer2))) + + tokens = DelegationTokenCommand.describeToken(adminClient, getDescribeOpts(List())) + assertTrue(tokens.size == 2) + assertEquals(Set(token1, token2), tokens.toSet) + + //get tokens for renewer2 + tokens = DelegationTokenCommand.describeToken(adminClient, getDescribeOpts(List(renewer2))) + assertTrue(tokens.size == 1) + assertEquals(Set(token2), tokens.toSet) + + //test renewing tokens + val expiryTimestamp = DelegationTokenCommand.renewToken(adminClient, getRenewOpts(token1.hmacAsBase64String())) + val renewedToken = DelegationTokenCommand.describeToken(adminClient, getDescribeOpts(List(renewer1))).head + assertEquals(expiryTimestamp, renewedToken.tokenInfo().expiryTimestamp()) + + //test expire tokens + DelegationTokenCommand.expireToken(adminClient, getExpireOpts(token1.hmacAsBase64String())) + DelegationTokenCommand.expireToken(adminClient, getExpireOpts(token2.hmacAsBase64String())) + + tokens = DelegationTokenCommand.describeToken(adminClient, getDescribeOpts(List())) + assertTrue(tokens.size == 0) + + //create token with invalid renewer principal type + intercept[ExecutionException](DelegationTokenCommand.createToken(adminClient, getCreateOpts(List("Group:Renewer3")))) + + // try describing tokens for unknown owner + assertTrue(DelegationTokenCommand.describeToken(adminClient, getDescribeOpts(List("User:Unknown"))).isEmpty) + } + + private def getCreateOpts(renewers: List[String]): DelegationTokenCommandOptions = { + val opts = ListBuffer("--bootstrap-server", brokerList, "--max-life-time-period", "-1", + "--command-config", "testfile", "--create") + renewers.foreach(renewer => opts ++= ListBuffer("--renewer-principal", renewer)) + new DelegationTokenCommandOptions(opts.toArray) + } + + private def getDescribeOpts(owners: List[String]): DelegationTokenCommandOptions = { + val opts = ListBuffer("--bootstrap-server", brokerList, "--command-config", "testfile", "--describe") + owners.foreach(owner => opts ++= ListBuffer("--owner-principal", owner)) + new DelegationTokenCommandOptions(opts.toArray) + } + + private def getRenewOpts(hmac: String): DelegationTokenCommandOptions = { + val opts = Array("--bootstrap-server", brokerList, "--command-config", "testfile", "--renew", + "--renew-time-period", "-1", + "--hmac", hmac) + new DelegationTokenCommandOptions(opts) + } + + private def getExpireOpts(hmac: String): DelegationTokenCommandOptions = { + val opts = Array("--bootstrap-server", brokerList, "--command-config", "testfile", "--expire", + "--expiry-time-period", "-1", + "--hmac", hmac) + new DelegationTokenCommandOptions(opts) + } + + @After + override def tearDown(): Unit = { + if (adminClient != null) + adminClient.close() + super.tearDown() + closeSasl() + } +} diff --git a/core/src/test/scala/unit/kafka/controller/PartitionLeaderElectionAlgorithmsTest.scala b/core/src/test/scala/unit/kafka/controller/PartitionLeaderElectionAlgorithmsTest.scala index f149fc93a4991..113a39d54300e 100644 --- a/core/src/test/scala/unit/kafka/controller/PartitionLeaderElectionAlgorithmsTest.scala +++ b/core/src/test/scala/unit/kafka/controller/PartitionLeaderElectionAlgorithmsTest.scala @@ -17,10 +17,17 @@ package kafka.controller import org.junit.Assert._ -import org.junit.Test +import org.junit.{Before, Test} import org.scalatest.junit.JUnitSuite class PartitionLeaderElectionAlgorithmsTest extends JUnitSuite { + private var controllerContext: ControllerContext = null + + @Before + def setUp(): Unit = { + controllerContext = new ControllerContext + controllerContext.stats.removeMetric("UncleanLeaderElectionsPerSec") + } @Test def testOfflinePartitionLeaderElection(): Unit = { @@ -30,7 +37,8 @@ class PartitionLeaderElectionAlgorithmsTest extends JUnitSuite { val leaderOpt = PartitionLeaderElectionAlgorithms.offlinePartitionLeaderElection(assignment, isr, liveReplicas, - uncleanLeaderElectionEnabled = false) + uncleanLeaderElectionEnabled = false, + controllerContext) assertEquals(Option(4), leaderOpt) } @@ -42,9 +50,12 @@ class PartitionLeaderElectionAlgorithmsTest extends JUnitSuite { val leaderOpt = PartitionLeaderElectionAlgorithms.offlinePartitionLeaderElection(assignment, isr, liveReplicas, - uncleanLeaderElectionEnabled = false) + uncleanLeaderElectionEnabled = false, + controllerContext) assertEquals(None, leaderOpt) + assertEquals(0, controllerContext.stats.uncleanLeaderElectionRate.count()) } + @Test def testOfflinePartitionLeaderElectionLastIsrOfflineUncleanLeaderElectionEnabled(): Unit = { val assignment = Seq(2, 4) @@ -53,8 +64,10 @@ class PartitionLeaderElectionAlgorithmsTest extends JUnitSuite { val leaderOpt = PartitionLeaderElectionAlgorithms.offlinePartitionLeaderElection(assignment, isr, liveReplicas, - uncleanLeaderElectionEnabled = true) + uncleanLeaderElectionEnabled = true, + controllerContext) assertEquals(Option(4), leaderOpt) + assertEquals(1, controllerContext.stats.uncleanLeaderElectionRate.count()) } @Test @@ -62,10 +75,9 @@ class PartitionLeaderElectionAlgorithmsTest extends JUnitSuite { val reassignment = Seq(2, 4) val isr = Seq(2, 4) val liveReplicas = Set(4) - val leaderOpt = PartitionLeaderElectionAlgorithms.offlinePartitionLeaderElection(reassignment, + val leaderOpt = PartitionLeaderElectionAlgorithms.reassignPartitionLeaderElection(reassignment, isr, - liveReplicas, - uncleanLeaderElectionEnabled = false) + liveReplicas) assertEquals(Option(4), leaderOpt) } diff --git a/core/src/test/scala/unit/kafka/integration/UncleanLeaderElectionTest.scala b/core/src/test/scala/unit/kafka/integration/UncleanLeaderElectionTest.scala index 5269f92d3584d..608f3a6f56115 100755 --- a/core/src/test/scala/unit/kafka/integration/UncleanLeaderElectionTest.scala +++ b/core/src/test/scala/unit/kafka/integration/UncleanLeaderElectionTest.scala @@ -191,12 +191,17 @@ class UncleanLeaderElectionTest extends ZooKeeperTestHarness { produceMessage(servers, topic, "second") assertEquals(List("first", "second"), consumeAllMessages(topic)) + //remove any previous unclean election metric + servers.map(server => server.kafkaController.controllerContext.stats.removeMetric("UncleanLeaderElectionsPerSec")) + // shutdown leader and then restart follower servers.filter(server => server.config.brokerId == leaderId).map(server => shutdownServer(server)) - servers.filter(server => server.config.brokerId == followerId).map(server => server.startup()) + val followerServer = servers.find(_.config.brokerId == followerId).get + followerServer.startup() // wait until new leader is (uncleanly) elected waitUntilLeaderIsElectedOrChanged(zkClient, topic, partitionId, newLeaderOpt = Some(followerId)) + assertEquals(1, followerServer.kafkaController.controllerContext.stats.uncleanLeaderElectionRate.count()) produceMessage(servers, topic, "third") @@ -224,12 +229,17 @@ class UncleanLeaderElectionTest extends ZooKeeperTestHarness { produceMessage(servers, topic, "second") assertEquals(List("first", "second"), consumeAllMessages(topic)) + //remove any previous unclean election metric + servers.map(server => server.kafkaController.controllerContext.stats.removeMetric("UncleanLeaderElectionsPerSec")) + // shutdown leader and then restart follower servers.filter(server => server.config.brokerId == leaderId).map(server => shutdownServer(server)) - servers.filter(server => server.config.brokerId == followerId).map(server => server.startup()) + val followerServer = servers.find(_.config.brokerId == followerId).get + followerServer.startup() // verify that unclean election to non-ISR follower does not occur waitUntilLeaderIsElectedOrChanged(zkClient, topic, partitionId, newLeaderOpt = Some(-1)) + assertEquals(0, followerServer.kafkaController.controllerContext.stats.uncleanLeaderElectionRate.count()) // message production and consumption should both fail while leader is down try { diff --git a/core/src/test/scala/unit/kafka/network/SocketServerTest.scala b/core/src/test/scala/unit/kafka/network/SocketServerTest.scala index 0dad3c71230fb..e6dadbb42536a 100644 --- a/core/src/test/scala/unit/kafka/network/SocketServerTest.scala +++ b/core/src/test/scala/unit/kafka/network/SocketServerTest.scala @@ -38,7 +38,7 @@ import org.apache.kafka.common.protocol.{ApiKeys, Errors} import org.apache.kafka.common.record.MemoryRecords import org.apache.kafka.common.requests.{AbstractRequest, ProduceRequest, RequestHeader} import org.apache.kafka.common.security.auth.{KafkaPrincipal, SecurityProtocol} -import org.apache.kafka.common.security.scram.ScramMechanism +import org.apache.kafka.common.security.scram.internal.ScramMechanism import org.apache.kafka.common.utils.{LogContext, MockTime, Time} import org.apache.log4j.Level import org.junit.Assert._ diff --git a/core/src/test/scala/unit/kafka/security/token/delegation/DelegationTokenManagerTest.scala b/core/src/test/scala/unit/kafka/security/token/delegation/DelegationTokenManagerTest.scala index 8c035482ee2ff..6093622a18822 100644 --- a/core/src/test/scala/unit/kafka/security/token/delegation/DelegationTokenManagerTest.scala +++ b/core/src/test/scala/unit/kafka/security/token/delegation/DelegationTokenManagerTest.scala @@ -29,8 +29,9 @@ import kafka.utils.TestUtils import kafka.zk.ZooKeeperTestHarness import org.apache.kafka.common.protocol.Errors import org.apache.kafka.common.security.auth.KafkaPrincipal -import org.apache.kafka.common.security.scram.ScramMechanism -import org.apache.kafka.common.security.token.delegation.{DelegationToken, DelegationTokenCache, TokenInformation} +import org.apache.kafka.common.security.scram.internal.ScramMechanism +import org.apache.kafka.common.security.token.delegation.internal.DelegationTokenCache +import org.apache.kafka.common.security.token.delegation.{DelegationToken, TokenInformation} import org.apache.kafka.common.utils.{MockTime, SecurityUtils} import org.junit.Assert._ import org.junit.{After, Before, Test} diff --git a/core/src/test/scala/unit/kafka/server/ClientQuotaManagerTest.scala b/core/src/test/scala/unit/kafka/server/ClientQuotaManagerTest.scala index 1aabbb3b1d476..c0bad91d0d74c 100644 --- a/core/src/test/scala/unit/kafka/server/ClientQuotaManagerTest.scala +++ b/core/src/test/scala/unit/kafka/server/ClientQuotaManagerTest.scala @@ -18,7 +18,10 @@ package kafka.server import java.util.Collections +import kafka.network.RequestChannel.Session +import kafka.server.QuotaType._ import org.apache.kafka.common.metrics.{MetricConfig, Metrics, Quota} +import org.apache.kafka.common.security.auth.KafkaPrincipal import org.apache.kafka.common.utils.{MockTime, Sanitizer} import org.junit.Assert.{assertEquals, assertTrue} import org.junit.{Before, Test} @@ -38,43 +41,49 @@ class ClientQuotaManagerTest { numCallbacks = 0 } + private def maybeRecordAndThrottle(quotaManager: ClientQuotaManager, user: String, clientId: String, value: Double): Int = { + val principal = new KafkaPrincipal(KafkaPrincipal.USER_TYPE, user) + quotaManager.maybeRecordAndThrottle(Session(principal, null),clientId, value, this.callback) + } + private def testQuotaParsing(config: ClientQuotaManagerConfig, client1: UserClient, client2: UserClient, randomClient: UserClient, defaultConfigClient: UserClient) { - val clientMetrics = new ClientQuotaManager(config, newMetrics, QuotaType.Produce, time, "") + val clientMetrics = new ClientQuotaManager(config, newMetrics, Produce, time, "") try { // Case 1: Update the quota. Assert that the new quota value is returned clientMetrics.updateQuota(client1.configUser, client1.configClientId, client1.sanitizedConfigClientId, Some(new Quota(2000, true))) clientMetrics.updateQuota(client2.configUser, client2.configClientId, client2.sanitizedConfigClientId, Some(new Quota(4000, true))) - assertEquals("Default producer quota should be " + config.quotaBytesPerSecondDefault, new Quota(config.quotaBytesPerSecondDefault, true), clientMetrics.quota(randomClient.user, randomClient.clientId)) - assertEquals("Should return the overridden value (2000)", new Quota(2000, true), clientMetrics.quota(client1.user, client1.clientId)) - assertEquals("Should return the overridden value (4000)", new Quota(4000, true), clientMetrics.quota(client2.user, client2.clientId)) + assertEquals("Default producer quota should be " + config.quotaBytesPerSecondDefault, + config.quotaBytesPerSecondDefault, clientMetrics.quota(randomClient.user, randomClient.clientId).bound, 0.0) + assertEquals("Should return the overridden value (2000)", 2000, clientMetrics.quota(client1.user, client1.clientId).bound, 0.0) + assertEquals("Should return the overridden value (4000)", 4000, clientMetrics.quota(client2.user, client2.clientId).bound, 0.0) // p1 should be throttled using the overridden quota - var throttleTimeMs = clientMetrics.maybeRecordAndThrottle(client1.user, client1.clientId, 2500 * config.numQuotaSamples, this.callback) + var throttleTimeMs = maybeRecordAndThrottle(clientMetrics, client1.user, client1.clientId, 2500 * config.numQuotaSamples) assertTrue(s"throttleTimeMs should be > 0. was $throttleTimeMs", throttleTimeMs > 0) // Case 2: Change quota again. The quota should be updated within KafkaMetrics as well since the sensor was created. // p1 should not longer be throttled after the quota change clientMetrics.updateQuota(client1.configUser, client1.configClientId, client1.sanitizedConfigClientId, Some(new Quota(3000, true))) - assertEquals("Should return the newly overridden value (3000)", new Quota(3000, true), clientMetrics.quota(client1.user, client1.clientId)) + assertEquals("Should return the newly overridden value (3000)", 3000, clientMetrics.quota(client1.user, client1.clientId).bound, 0.0) - throttleTimeMs = clientMetrics.maybeRecordAndThrottle(client1.user, client1.clientId, 0, this.callback) + throttleTimeMs = maybeRecordAndThrottle(clientMetrics, client1.user, client1.clientId, 0) assertEquals(s"throttleTimeMs should be 0. was $throttleTimeMs", 0, throttleTimeMs) // Case 3: Change quota back to default. Should be throttled again clientMetrics.updateQuota(client1.configUser, client1.configClientId, client1.sanitizedConfigClientId, Some(new Quota(500, true))) - assertEquals("Should return the default value (500)", new Quota(500, true), clientMetrics.quota(client1.user, client1.clientId)) + assertEquals("Should return the default value (500)", 500, clientMetrics.quota(client1.user, client1.clientId).bound, 0.0) - throttleTimeMs = clientMetrics.maybeRecordAndThrottle(client1.user, client1.clientId, 0, this.callback) + throttleTimeMs = maybeRecordAndThrottle(clientMetrics, client1.user, client1.clientId, 0) assertTrue(s"throttleTimeMs should be > 0. was $throttleTimeMs", throttleTimeMs > 0) // Case 4: Set high default quota, remove p1 quota. p1 should no longer be throttled clientMetrics.updateQuota(client1.configUser, client1.configClientId, client1.sanitizedConfigClientId, None) clientMetrics.updateQuota(defaultConfigClient.configUser, defaultConfigClient.configClientId, defaultConfigClient.sanitizedConfigClientId, Some(new Quota(4000, true))) - assertEquals("Should return the newly overridden value (4000)", new Quota(4000, true), clientMetrics.quota(client1.user, client1.clientId)) + assertEquals("Should return the newly overridden value (4000)", 4000, clientMetrics.quota(client1.user, client1.clientId).bound, 0.0) - throttleTimeMs = clientMetrics.maybeRecordAndThrottle(client1.user, client1.clientId, 1000 * config.numQuotaSamples, this.callback) + throttleTimeMs = maybeRecordAndThrottle(clientMetrics, client1.user, client1.clientId, 1000 * config.numQuotaSamples) assertEquals(s"throttleTimeMs should be 0. was $throttleTimeMs", 0, throttleTimeMs) } finally { @@ -150,11 +159,11 @@ class ClientQuotaManagerTest { @Test def testQuotaConfigPrecedence() { val quotaManager = new ClientQuotaManager(ClientQuotaManagerConfig(quotaBytesPerSecondDefault=Long.MaxValue), - newMetrics, QuotaType.Produce, time, "") + newMetrics, Produce, time, "") def checkQuota(user: String, clientId: String, expectedBound: Int, value: Int, expectThrottle: Boolean) { - assertEquals(new Quota(expectedBound, true), quotaManager.quota(user, clientId)) - val throttleTimeMs = quotaManager.maybeRecordAndThrottle(user, clientId, value * config.numQuotaSamples, this.callback) + assertEquals(expectedBound, quotaManager.quota(user, clientId).bound, 0.0) + val throttleTimeMs = maybeRecordAndThrottle(quotaManager, user, clientId, value * config.numQuotaSamples) if (expectThrottle) assertTrue(s"throttleTimeMs should be > 0. was $throttleTimeMs", throttleTimeMs > 0) else @@ -223,14 +232,14 @@ class ClientQuotaManagerTest { @Test def testQuotaViolation() { val metrics = newMetrics - val clientMetrics = new ClientQuotaManager(config, metrics, QuotaType.Produce, time, "") + val clientMetrics = new ClientQuotaManager(config, metrics, Produce, time, "") val queueSizeMetric = metrics.metrics().get(metrics.metricName("queue-size", "Produce", "")) try { /* We have 10 second windows. Make sure that there is no quota violation * if we produce under the quota */ for (_ <- 0 until 10) { - clientMetrics.maybeRecordAndThrottle("ANONYMOUS", "unknown", 400, callback) + maybeRecordAndThrottle(clientMetrics, "ANONYMOUS", "unknown", 400) time.sleep(1000) } assertEquals(10, numCallbacks) @@ -241,7 +250,7 @@ class ClientQuotaManagerTest { // (600 - quota)/quota*window-size = (600-500)/500*10.5 seconds = 2100 // 10.5 seconds because the last window is half complete time.sleep(500) - val sleepTime = clientMetrics.maybeRecordAndThrottle("ANONYMOUS", "unknown", 2300, callback) + val sleepTime = maybeRecordAndThrottle(clientMetrics, "ANONYMOUS", "unknown", 2300) assertEquals("Should be throttled", 2100, sleepTime) assertEquals(1, queueSizeMetric.value().toInt) @@ -257,12 +266,12 @@ class ClientQuotaManagerTest { // Could continue to see delays until the bursty sample disappears for (_ <- 0 until 10) { - clientMetrics.maybeRecordAndThrottle("ANONYMOUS", "unknown", 400, callback) + maybeRecordAndThrottle(clientMetrics, "ANONYMOUS", "unknown", 400) time.sleep(1000) } assertEquals("Should be unthrottled since bursty sample has rolled over", - 0, clientMetrics.maybeRecordAndThrottle("ANONYMOUS", "unknown", 0, callback)) + 0, maybeRecordAndThrottle(clientMetrics, "ANONYMOUS", "unknown", 0)) } finally { clientMetrics.shutdown() } @@ -271,7 +280,7 @@ class ClientQuotaManagerTest { @Test def testRequestPercentageQuotaViolation() { val metrics = newMetrics - val quotaManager = new ClientRequestQuotaManager(config, metrics, time, "") + val quotaManager = new ClientRequestQuotaManager(config, metrics, time, "", None) quotaManager.updateQuota(Some("ANONYMOUS"), Some("test-client"), Some("test-client"), Some(Quota.upperBound(1))) val queueSizeMetric = metrics.metrics().get(metrics.metricName("queue-size", "Request", "")) def millisToPercent(millis: Double) = millis * 1000 * 1000 * ClientQuotaManagerConfig.NanosToPercentagePerSecond @@ -280,7 +289,7 @@ class ClientQuotaManagerTest { * if we are under the quota */ for (_ <- 0 until 10) { - quotaManager.maybeRecordAndThrottle("ANONYMOUS", "test-client", millisToPercent(4), callback) + maybeRecordAndThrottle(quotaManager, "ANONYMOUS", "test-client", millisToPercent(4)) time.sleep(1000) } assertEquals(10, numCallbacks) @@ -292,7 +301,7 @@ class ClientQuotaManagerTest { // (10.2 - quota)/quota*window-size = (10.2-10)/10*10.5 seconds = 210ms // 10.5 seconds interval because the last window is half complete time.sleep(500) - val throttleTime = quotaManager.maybeRecordAndThrottle("ANONYMOUS", "test-client", millisToPercent(67.1), callback) + val throttleTime = maybeRecordAndThrottle(quotaManager, "ANONYMOUS", "test-client", millisToPercent(67.1)) assertEquals("Should be throttled", 210, throttleTime) assertEquals(1, queueSizeMetric.value().toInt) @@ -308,22 +317,22 @@ class ClientQuotaManagerTest { // Could continue to see delays until the bursty sample disappears for (_ <- 0 until 11) { - quotaManager.maybeRecordAndThrottle("ANONYMOUS", "test-client", millisToPercent(4), callback) + maybeRecordAndThrottle(quotaManager, "ANONYMOUS", "test-client", millisToPercent(4)) time.sleep(1000) } assertEquals("Should be unthrottled since bursty sample has rolled over", - 0, quotaManager.maybeRecordAndThrottle("ANONYMOUS", "test-client", 0, callback)) + 0, maybeRecordAndThrottle(quotaManager, "ANONYMOUS", "test-client", 0)) // Create a very large spike which requires > one quota window to bring within quota - assertEquals(1000, quotaManager.maybeRecordAndThrottle("ANONYMOUS", "test-client", millisToPercent(500), callback)) + assertEquals(1000, maybeRecordAndThrottle(quotaManager, "ANONYMOUS", "test-client", millisToPercent(500))) for (_ <- 0 until 10) { time.sleep(1000) - assertEquals(1000, quotaManager.maybeRecordAndThrottle("ANONYMOUS", "test-client", 0, callback)) + assertEquals(1000, maybeRecordAndThrottle(quotaManager, "ANONYMOUS", "test-client", 0)) } time.sleep(1000) assertEquals("Should be unthrottled since bursty sample has rolled over", - 0, quotaManager.maybeRecordAndThrottle("ANONYMOUS", "test-client", 0, callback)) + 0, maybeRecordAndThrottle(quotaManager, "ANONYMOUS", "test-client", 0)) } finally { quotaManager.shutdown() @@ -333,13 +342,13 @@ class ClientQuotaManagerTest { @Test def testExpireThrottleTimeSensor() { val metrics = newMetrics - val clientMetrics = new ClientQuotaManager(config, metrics, QuotaType.Produce, time, "") + val clientMetrics = new ClientQuotaManager(config, metrics, Produce, time, "") try { - clientMetrics.maybeRecordAndThrottle("ANONYMOUS", "client1", 100, callback) + maybeRecordAndThrottle(clientMetrics, "ANONYMOUS", "client1", 100) // remove the throttle time sensor metrics.removeSensor("ProduceThrottleTime-:client1") // should not throw an exception even if the throttle time sensor does not exist. - val throttleTime = clientMetrics.maybeRecordAndThrottle("ANONYMOUS", "client1", 10000, callback) + val throttleTime = maybeRecordAndThrottle(clientMetrics, "ANONYMOUS", "client1", 10000) assertTrue("Should be throttled", throttleTime > 0) // the sensor should get recreated val throttleTimeSensor = metrics.getSensor("ProduceThrottleTime-:client1") @@ -352,14 +361,14 @@ class ClientQuotaManagerTest { @Test def testExpireQuotaSensors() { val metrics = newMetrics - val clientMetrics = new ClientQuotaManager(config, metrics, QuotaType.Produce, time, "") + val clientMetrics = new ClientQuotaManager(config, metrics, Produce, time, "") try { - clientMetrics.maybeRecordAndThrottle("ANONYMOUS", "client1", 100, callback) + maybeRecordAndThrottle(clientMetrics, "ANONYMOUS", "client1", 100) // remove all the sensors metrics.removeSensor("ProduceThrottleTime-:client1") metrics.removeSensor("Produce-ANONYMOUS:client1") // should not throw an exception - val throttleTime = clientMetrics.maybeRecordAndThrottle("ANONYMOUS", "client1", 10000, callback) + val throttleTime = maybeRecordAndThrottle(clientMetrics, "ANONYMOUS", "client1", 10000) assertTrue("Should be throttled", throttleTime > 0) // all the sensors should get recreated @@ -376,10 +385,10 @@ class ClientQuotaManagerTest { @Test def testClientIdNotSanitized() { val metrics = newMetrics - val clientMetrics = new ClientQuotaManager(config, metrics, QuotaType.Produce, time, "") + val clientMetrics = new ClientQuotaManager(config, metrics, Produce, time, "") val clientId = "client@#$%" try { - clientMetrics.maybeRecordAndThrottle("ANONYMOUS", clientId, 100, callback) + maybeRecordAndThrottle(clientMetrics, "ANONYMOUS", clientId, 100) // The metrics should use the raw client ID, even if the reporters internally sanitize them val throttleTimeSensor = metrics.getSensor("ProduceThrottleTime-:" + clientId) diff --git a/core/src/test/scala/unit/kafka/server/DelegationTokenRequestsOnPlainTextTest.scala b/core/src/test/scala/unit/kafka/server/DelegationTokenRequestsOnPlainTextTest.scala index 4c42dd277242a..3d4be53198510 100644 --- a/core/src/test/scala/unit/kafka/server/DelegationTokenRequestsOnPlainTextTest.scala +++ b/core/src/test/scala/unit/kafka/server/DelegationTokenRequestsOnPlainTextTest.scala @@ -19,14 +19,13 @@ package kafka.server import java.nio.ByteBuffer import java.util -import kafka.admin.AdminClient import kafka.utils.TestUtils -import org.apache.kafka.clients.admin.AdminClientConfig -import org.apache.kafka.common.protocol.Errors -import org.junit.Assert._ +import org.apache.kafka.clients.admin.{AdminClient, AdminClientConfig} +import org.apache.kafka.common.errors.UnsupportedByAuthenticationException import org.junit.{After, Before, Test} import scala.collection.JavaConverters._ +import scala.concurrent.ExecutionException class DelegationTokenRequestsOnPlainTextTest extends BaseRequestTest { var adminClient: AdminClient = null @@ -49,21 +48,19 @@ class DelegationTokenRequestsOnPlainTextTest extends BaseRequestTest { @Test def testDelegationTokenRequests(): Unit = { - adminClient = AdminClient.create(createAdminConfig.asScala.toMap) + adminClient = AdminClient.create(createAdminConfig) - val createResponse = adminClient.createToken(List()) - assertEquals(Errors.DELEGATION_TOKEN_REQUEST_NOT_ALLOWED, createResponse._1) + val createResult = adminClient.createDelegationToken() + intercept[ExecutionException](createResult.delegationToken().get()).getCause.isInstanceOf[UnsupportedByAuthenticationException] - val describeResponse = adminClient.describeToken(List()) - assertEquals(Errors.DELEGATION_TOKEN_REQUEST_NOT_ALLOWED, describeResponse._1) + val describeResult = adminClient.describeDelegationToken() + intercept[ExecutionException](describeResult.delegationTokens().get()).getCause.isInstanceOf[UnsupportedByAuthenticationException] - //test renewing tokens - val renewResponse = adminClient.renewToken(ByteBuffer.wrap("".getBytes())) - assertEquals(Errors.DELEGATION_TOKEN_REQUEST_NOT_ALLOWED, renewResponse._1) + val renewResult = adminClient.renewDelegationToken("".getBytes()) + intercept[ExecutionException](renewResult.expiryTimestamp().get()).getCause.isInstanceOf[UnsupportedByAuthenticationException] - //test expire tokens tokens - val expireResponse = adminClient.expireToken(ByteBuffer.wrap("".getBytes())) - assertEquals(Errors.DELEGATION_TOKEN_REQUEST_NOT_ALLOWED, expireResponse._1) + val expireResult = adminClient.expireDelegationToken("".getBytes()) + intercept[ExecutionException](expireResult.expiryTimestamp().get()).getCause.isInstanceOf[UnsupportedByAuthenticationException] } diff --git a/core/src/test/scala/unit/kafka/server/DelegationTokenRequestsTest.scala b/core/src/test/scala/unit/kafka/server/DelegationTokenRequestsTest.scala index 55bf5fd022cc2..a00275084fbc3 100644 --- a/core/src/test/scala/unit/kafka/server/DelegationTokenRequestsTest.scala +++ b/core/src/test/scala/unit/kafka/server/DelegationTokenRequestsTest.scala @@ -18,17 +18,17 @@ package kafka.server import java.util -import kafka.admin.AdminClient import kafka.api.{KafkaSasl, SaslSetup} import kafka.utils.{JaasTestUtils, TestUtils} -import org.apache.kafka.clients.admin.AdminClientConfig -import org.apache.kafka.common.protocol.Errors +import org.apache.kafka.clients.admin.{AdminClient, AdminClientConfig, CreateDelegationTokenOptions, DescribeDelegationTokenOptions} +import org.apache.kafka.common.errors.InvalidPrincipalTypeException import org.apache.kafka.common.security.auth.SecurityProtocol import org.apache.kafka.common.utils.SecurityUtils import org.junit.Assert._ import org.junit.{After, Before, Test} import scala.collection.JavaConverters._ +import scala.concurrent.ExecutionException class DelegationTokenRequestsTest extends BaseRequestTest with SaslSetup { override protected def securityProtocol = SecurityProtocol.SASL_PLAINTEXT @@ -46,15 +46,6 @@ class DelegationTokenRequestsTest extends BaseRequestTest with SaslSetup { super.setUp() } - def createAdminConfig():util.Map[String, Object] = { - val config = new util.HashMap[String, Object] - config.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, brokerList) - val securityProps: util.Map[Object, Object] = - TestUtils.adminClientSecurityConfigs(securityProtocol, trustStoreFile, clientSaslProperties) - securityProps.asScala.foreach { case (key, value) => config.put(key.asInstanceOf[String], value) } - config - } - override def generateConfigs = { val props = TestUtils.createBrokerConfigs(numBrokers, zkConnect, enableControlledShutdown = false, enableDeleteTopic = true, @@ -64,46 +55,73 @@ class DelegationTokenRequestsTest extends BaseRequestTest with SaslSetup { props.map(KafkaConfig.fromProps) } + private def createAdminConfig():util.Map[String, Object] = { + val config = new util.HashMap[String, Object] + config.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, brokerList) + val securityProps: util.Map[Object, Object] = + TestUtils.adminClientSecurityConfigs(securityProtocol, trustStoreFile, clientSaslProperties) + securityProps.asScala.foreach { case (key, value) => config.put(key.asInstanceOf[String], value) } + config + } + @Test def testDelegationTokenRequests(): Unit = { - adminClient = AdminClient.create(createAdminConfig.asScala.toMap) - - // test creating token - val renewer1 = List(SecurityUtils.parseKafkaPrincipal("User:" + JaasTestUtils.KafkaPlainUser)) - val tokenResult1 = adminClient.createToken(renewer1) - assertEquals(Errors.NONE, tokenResult1._1) - var token1 = adminClient.describeToken(null)._2.head - assertEquals(token1, tokenResult1._2) + adminClient = AdminClient.create(createAdminConfig) + + // create token1 with renewer1 + val renewer1 = List(SecurityUtils.parseKafkaPrincipal("User:renewer1")).asJava + val createResult1 = adminClient.createDelegationToken(new CreateDelegationTokenOptions().renewers(renewer1)) + val tokenCreated = createResult1.delegationToken().get() + + //test describe token + var tokens = adminClient.describeDelegationToken().delegationTokens().get() + assertTrue(tokens.size() == 1) + var token1 = tokens.get(0) + assertEquals(token1, tokenCreated) + + // create token2 with renewer2 + val renewer2 = List(SecurityUtils.parseKafkaPrincipal("User:renewer2")).asJava + val createResult2 = adminClient.createDelegationToken(new CreateDelegationTokenOptions().renewers(renewer2)) + val token2 = createResult2.delegationToken().get() + + //get all tokens + tokens = adminClient.describeDelegationToken().delegationTokens().get() + assertTrue(tokens.size() == 2) + assertEquals(Set(token1, token2), tokens.asScala.toSet) + + //get tokens for renewer2 + tokens = adminClient.describeDelegationToken(new DescribeDelegationTokenOptions().owners(renewer2)).delegationTokens().get() + assertTrue(tokens.size() == 1) + assertEquals(Set(token2), tokens.asScala.toSet) //test renewing tokens - val renewResponse = adminClient.renewToken(token1.hmacBuffer()) - assertEquals(Errors.NONE, renewResponse._1) - - token1 = adminClient.describeToken(null)._2.head - assertEquals(renewResponse._2, token1.tokenInfo().expiryTimestamp()) + val renewResult = adminClient.renewDelegationToken(token1.hmac()) + var expiryTimestamp = renewResult.expiryTimestamp().get() - //test describe tokens - val renewer2 = List(SecurityUtils.parseKafkaPrincipal("User:Renewer1")) - val tokenResult2 = adminClient.createToken(renewer2) - assertEquals(Errors.NONE, tokenResult2._1) - val token2 = tokenResult2._2 + val describeResult = adminClient.describeDelegationToken() + val tokenId = token1.tokenInfo().tokenId() + token1 = describeResult.delegationTokens().get().asScala.filter(dt => dt.tokenInfo().tokenId() == tokenId).head + assertEquals(expiryTimestamp, token1.tokenInfo().expiryTimestamp()) - assertTrue(adminClient.describeToken(null)._2.size == 2) + //test expire tokens + val expireResult1 = adminClient.expireDelegationToken(token1.hmac()) + expiryTimestamp = expireResult1.expiryTimestamp().get() - //test expire tokens tokens - val expireResponse1 = adminClient.expireToken(token1.hmacBuffer()) - assertEquals(Errors.NONE, expireResponse1._1) + val expireResult2 = adminClient.expireDelegationToken(token2.hmac()) + expiryTimestamp = expireResult2.expiryTimestamp().get() - val expireResponse2 = adminClient.expireToken(token2.hmacBuffer()) - assertEquals(Errors.NONE, expireResponse2._1) - - assertTrue(adminClient.describeToken(null)._2.size == 0) + tokens = adminClient.describeDelegationToken().delegationTokens().get() + assertTrue(tokens.size == 0) //create token with invalid principal type - val renewer3 = List(SecurityUtils.parseKafkaPrincipal("Group:Renewer1")) - val tokenResult3 = adminClient.createToken(renewer3) - assertEquals(Errors.INVALID_PRINCIPAL_TYPE, tokenResult3._1) - + val renewer3 = List(SecurityUtils.parseKafkaPrincipal("Group:Renewer3")).asJava + val createResult3 = adminClient.createDelegationToken(new CreateDelegationTokenOptions().renewers(renewer3)) + intercept[ExecutionException](createResult3.delegationToken().get()).getCause.isInstanceOf[InvalidPrincipalTypeException] + + // try describing tokens for unknown owner + val unknownOwner = List(SecurityUtils.parseKafkaPrincipal("User:Unknown")).asJava + tokens = adminClient.describeDelegationToken(new DescribeDelegationTokenOptions().owners(unknownOwner)).delegationTokens().get() + assertTrue(tokens.isEmpty) } @After diff --git a/core/src/test/scala/unit/kafka/server/DelegationTokenRequestsWithDisableTokenFeatureTest.scala b/core/src/test/scala/unit/kafka/server/DelegationTokenRequestsWithDisableTokenFeatureTest.scala index 0561cacb8f89b..8f8842bbe97e7 100644 --- a/core/src/test/scala/unit/kafka/server/DelegationTokenRequestsWithDisableTokenFeatureTest.scala +++ b/core/src/test/scala/unit/kafka/server/DelegationTokenRequestsWithDisableTokenFeatureTest.scala @@ -19,17 +19,15 @@ package kafka.server import java.nio.ByteBuffer import java.util -import kafka.admin.AdminClient import kafka.api.{KafkaSasl, SaslSetup} import kafka.utils.{JaasTestUtils, TestUtils} -import org.apache.kafka.clients.admin.AdminClientConfig -import org.apache.kafka.common.protocol.Errors -import org.apache.kafka.common.utils.SecurityUtils -import org.junit.Assert._ -import org.junit.{After, Before, Test} +import org.apache.kafka.clients.admin.{AdminClient, AdminClientConfig} +import org.apache.kafka.common.errors.DelegationTokenDisabledException import org.apache.kafka.common.security.auth.SecurityProtocol +import org.junit.{After, Before, Test} import scala.collection.JavaConverters._ +import scala.concurrent.ExecutionException class DelegationTokenRequestsWithDisableTokenFeatureTest extends BaseRequestTest with SaslSetup { override protected def securityProtocol = SecurityProtocol.SASL_PLAINTEXT @@ -58,23 +56,19 @@ class DelegationTokenRequestsWithDisableTokenFeatureTest extends BaseRequestTest @Test def testDelegationTokenRequests(): Unit = { - adminClient = AdminClient.create(createAdminConfig.asScala.toMap) - - val renewer1 = List(SecurityUtils.parseKafkaPrincipal("User:" + JaasTestUtils.KafkaPlainUser)) - val createResponse = adminClient.createToken(renewer1) - assertEquals(Errors.DELEGATION_TOKEN_AUTH_DISABLED, createResponse._1) + adminClient = AdminClient.create(createAdminConfig) - val describeResponse = adminClient.describeToken(List()) - assertEquals(Errors.DELEGATION_TOKEN_AUTH_DISABLED, describeResponse._1) + val createResult = adminClient.createDelegationToken() + intercept[ExecutionException](createResult.delegationToken().get()).getCause.isInstanceOf[DelegationTokenDisabledException] - //test renewing tokens - val renewResponse = adminClient.renewToken(ByteBuffer.wrap("".getBytes())) - assertEquals(Errors.DELEGATION_TOKEN_AUTH_DISABLED, renewResponse._1) + val describeResult = adminClient.describeDelegationToken() + intercept[ExecutionException](describeResult.delegationTokens().get()).getCause.isInstanceOf[DelegationTokenDisabledException] - //test expire tokens tokens - val expireResponse = adminClient.expireToken(ByteBuffer.wrap("".getBytes())) - assertEquals(Errors.DELEGATION_TOKEN_AUTH_DISABLED, expireResponse._1) + val renewResult = adminClient.renewDelegationToken("".getBytes()) + intercept[ExecutionException](renewResult.expiryTimestamp().get()).getCause.isInstanceOf[DelegationTokenDisabledException] + val expireResult = adminClient.expireDelegationToken("".getBytes()) + intercept[ExecutionException](expireResult.expiryTimestamp().get()).getCause.isInstanceOf[DelegationTokenDisabledException] } @After diff --git a/core/src/test/scala/unit/kafka/server/DynamicBrokerConfigTest.scala b/core/src/test/scala/unit/kafka/server/DynamicBrokerConfigTest.scala index 5c88bf27f6d4a..9c8acb4802426 100755 --- a/core/src/test/scala/unit/kafka/server/DynamicBrokerConfigTest.scala +++ b/core/src/test/scala/unit/kafka/server/DynamicBrokerConfigTest.scala @@ -27,11 +27,12 @@ import org.apache.kafka.common.config.{ConfigException, SslConfigs} import org.easymock.EasyMock import org.junit.Assert._ import org.junit.Test +import org.scalatest.junit.JUnitSuite import scala.collection.JavaConverters._ import scala.collection.Set -class DynamicBrokerConfigTest { +class DynamicBrokerConfigTest extends JUnitSuite { @Test def testConfigUpdate(): Unit = { @@ -126,6 +127,35 @@ class DynamicBrokerConfigTest { verifyConfigUpdateWithInvalidConfig(config, origProps, validProps, invalidProps) } + @Test + def testReconfigurableValidation(): Unit = { + val origProps = TestUtils.createBrokerConfig(0, TestUtils.MockZkConnect, port = 8181) + val config = KafkaConfig(origProps) + val invalidReconfigurableProps = Set(KafkaConfig.LogCleanerThreadsProp, KafkaConfig.BrokerIdProp, "some.prop") + val validReconfigurableProps = Set(KafkaConfig.LogCleanerThreadsProp, KafkaConfig.LogCleanerDedupeBufferSizeProp, "some.prop") + + def createReconfigurable(configs: Set[String]) = new Reconfigurable { + override def configure(configs: util.Map[String, _]): Unit = {} + override def reconfigurableConfigs(): util.Set[String] = configs.asJava + override def validateReconfiguration(configs: util.Map[String, _]): Unit = {} + override def reconfigure(configs: util.Map[String, _]): Unit = {} + } + intercept[IllegalArgumentException] { + config.dynamicConfig.addReconfigurable(createReconfigurable(invalidReconfigurableProps)) + } + config.dynamicConfig.addReconfigurable(createReconfigurable(validReconfigurableProps)) + + def createBrokerReconfigurable(configs: Set[String]) = new BrokerReconfigurable { + override def reconfigurableConfigs: collection.Set[String] = configs + override def validateReconfiguration(newConfig: KafkaConfig): Unit = {} + override def reconfigure(oldConfig: KafkaConfig, newConfig: KafkaConfig): Unit = {} + } + intercept[IllegalArgumentException] { + config.dynamicConfig.addBrokerReconfigurable(createBrokerReconfigurable(invalidReconfigurableProps)) + } + config.dynamicConfig.addBrokerReconfigurable(createBrokerReconfigurable(validReconfigurableProps)) + } + @Test def testSecurityConfigs(): Unit = { def verifyUpdate(name: String, value: Object): Unit = { diff --git a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala index 5de978cd0beb8..382364f7a8341 100644 --- a/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala +++ b/core/src/test/scala/unit/kafka/server/KafkaApisTest.scala @@ -68,7 +68,8 @@ class KafkaApisTest { private val clientQuotaManager = EasyMock.createNiceMock(classOf[ClientQuotaManager]) private val clientRequestQuotaManager = EasyMock.createNiceMock(classOf[ClientRequestQuotaManager]) private val replicaQuotaManager = EasyMock.createNiceMock(classOf[ReplicationQuotaManager]) - private val quotas = QuotaManagers(clientQuotaManager, clientQuotaManager, clientRequestQuotaManager, replicaQuotaManager, replicaQuotaManager, replicaQuotaManager) + private val quotas = QuotaManagers(clientQuotaManager, clientQuotaManager, clientRequestQuotaManager, + replicaQuotaManager, replicaQuotaManager, replicaQuotaManager, None) private val fetchManager = EasyMock.createNiceMock(classOf[FetchManager]) private val brokerTopicStats = new BrokerTopicStats private val clusterId = "clusterId" diff --git a/core/src/test/scala/unit/kafka/server/KafkaConfigTest.scala b/core/src/test/scala/unit/kafka/server/KafkaConfigTest.scala index 0213c12814e09..39cbe4068773e 100755 --- a/core/src/test/scala/unit/kafka/server/KafkaConfigTest.scala +++ b/core/src/test/scala/unit/kafka/server/KafkaConfigTest.scala @@ -692,6 +692,10 @@ class KafkaConfigTest { //Sasl Configs case KafkaConfig.SaslMechanismInterBrokerProtocolProp => // ignore case KafkaConfig.SaslEnabledMechanismsProp => + case KafkaConfig.SaslClientCallbackHandlerClassProp => + case KafkaConfig.SaslServerCallbackHandlerClassProp => + case KafkaConfig.SaslLoginClassProp => + case KafkaConfig.SaslLoginCallbackHandlerClassProp => case KafkaConfig.SaslKerberosServiceNameProp => // ignore string case KafkaConfig.SaslKerberosKinitCmdProp => case KafkaConfig.SaslKerberosTicketRenewWindowFactorProp => @@ -776,6 +780,15 @@ class KafkaConfigTest { assertFalse(isValidKafkaConfig(props)) } + @Test + def testMaxConnectionsPerIpProp() { + val props = TestUtils.createBrokerConfig(0, TestUtils.MockZkConnect, port = 8181) + props.put(KafkaConfig.MaxConnectionsPerIpProp, "0") + assertFalse(isValidKafkaConfig(props)) + props.put(KafkaConfig.MaxConnectionsPerIpOverridesProp, "127.0.0.1:100") + assertTrue(isValidKafkaConfig(props)) + } + private def assertPropertyInvalid(validRequiredProps: => Properties, name: String, values: Any*) { values.foreach((value) => { val props = validRequiredProps diff --git a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala index bfbae2bde036d..ed85415eacccd 100644 --- a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala +++ b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala @@ -23,6 +23,7 @@ import kafka.log.LogConfig import kafka.network.RequestChannel.Session import kafka.security.auth._ import kafka.utils.TestUtils + import org.apache.kafka.clients.admin.NewPartitions import org.apache.kafka.common.acl.{AccessControlEntry, AccessControlEntryFilter, AclBinding, AclBindingFilter, AclOperation, AclPermissionType} import org.apache.kafka.common.resource.{ResourceFilter, Resource => AdminResource, ResourceType => AdminResourceType} @@ -132,13 +133,16 @@ class RequestQuotaTest extends BaseRequestTest { waitAndCheckResults() } + def session(user: String): Session = Session(new KafkaPrincipal(KafkaPrincipal.USER_TYPE, user), null) + private def throttleTimeMetricValue(clientId: String): Double = { val metricName = leaderNode.metrics.metricName("throttle-time", QuotaType.Request.toString, "", "user", "", "client-id", clientId) - val sensor = leaderNode.quotaManagers.request.getOrCreateQuotaSensors("ANONYMOUS", clientId).throttleTimeSensor + val sensor = leaderNode.quotaManagers.request.getOrCreateQuotaSensors(session("ANONYMOUS"), + clientId).throttleTimeSensor metricValue(leaderNode.metrics.metrics.get(metricName), sensor) } @@ -148,7 +152,8 @@ class RequestQuotaTest extends BaseRequestTest { "", "user", "", "client-id", clientId) - val sensor = leaderNode.quotaManagers.request.getOrCreateQuotaSensors("ANONYMOUS", clientId).quotaSensor + val sensor = leaderNode.quotaManagers.request.getOrCreateQuotaSensors(session("ANONYMOUS"), + clientId).quotaSensor metricValue(leaderNode.metrics.metrics.get(metricName), sensor) } @@ -310,13 +315,13 @@ class RequestQuotaTest extends BaseRequestTest { new CreateDelegationTokenRequest.Builder(Collections.singletonList(SecurityUtils.parseKafkaPrincipal("User:test")), 1000) case ApiKeys.EXPIRE_DELEGATION_TOKEN => - new ExpireDelegationTokenRequest.Builder(ByteBuffer.allocate(10), 1000) + new ExpireDelegationTokenRequest.Builder("".getBytes, 1000) case ApiKeys.DESCRIBE_DELEGATION_TOKEN => new DescribeDelegationTokenRequest.Builder(Collections.singletonList(SecurityUtils.parseKafkaPrincipal("User:test"))) case ApiKeys.RENEW_DELEGATION_TOKEN => - new RenewDelegationTokenRequest.Builder(ByteBuffer.allocate(10), 1000) + new RenewDelegationTokenRequest.Builder("".getBytes, 1000) case ApiKeys.DELETE_GROUPS => new DeleteGroupsRequest.Builder(Collections.singleton("test-group")) diff --git a/core/src/test/scala/unit/kafka/utils/JaasTestUtils.scala b/core/src/test/scala/unit/kafka/utils/JaasTestUtils.scala index 9ce3b01025a75..10c73452a3cb2 100644 --- a/core/src/test/scala/unit/kafka/utils/JaasTestUtils.scala +++ b/core/src/test/scala/unit/kafka/utils/JaasTestUtils.scala @@ -114,7 +114,7 @@ object JaasTestUtils { val KafkaServerContextName = "KafkaServer" val KafkaServerPrincipalUnqualifiedName = "kafka" private val KafkaServerPrincipal = KafkaServerPrincipalUnqualifiedName + "/localhost@EXAMPLE.COM" - private val KafkaClientContextName = "KafkaClient" + val KafkaClientContextName = "KafkaClient" val KafkaClientPrincipalUnqualifiedName = "client" private val KafkaClientPrincipal = KafkaClientPrincipalUnqualifiedName + "@EXAMPLE.COM" val KafkaClientPrincipalUnqualifiedName2 = "client2" diff --git a/core/src/test/scala/unit/kafka/utils/TestUtils.scala b/core/src/test/scala/unit/kafka/utils/TestUtils.scala index 4b8740600ff7c..16b7e87487e5d 100755 --- a/core/src/test/scala/unit/kafka/utils/TestUtils.scala +++ b/core/src/test/scala/unit/kafka/utils/TestUtils.scala @@ -41,9 +41,11 @@ import Implicits._ import kafka.controller.LeaderIsrAndControllerEpoch import kafka.zk.{AdminZkClient, BrokerIdsZNode, BrokerInfo, KafkaZkClient} import org.apache.kafka.clients.CommonClientConfigs +import org.apache.kafka.clients.admin.{AdminClient, AlterConfigsResult, Config, ConfigEntry} import org.apache.kafka.clients.consumer.{ConsumerRecord, KafkaConsumer, OffsetAndMetadata, RangeAssignor} import org.apache.kafka.clients.producer.{KafkaProducer, ProducerConfig, ProducerRecord} import org.apache.kafka.common.TopicPartition +import org.apache.kafka.common.config.ConfigResource import org.apache.kafka.common.header.Header import org.apache.kafka.common.internals.Topic import org.apache.kafka.common.network.{ListenerName, Mode} @@ -1492,6 +1494,21 @@ object TestUtils extends Logging { } } + def alterConfigs(servers: Seq[KafkaServer], adminClient: AdminClient, props: Properties, + perBrokerConfig: Boolean): AlterConfigsResult = { + val configEntries = props.asScala.map { case (k, v) => new ConfigEntry(k, v) }.toList.asJava + val newConfig = new Config(configEntries) + val configs = if (perBrokerConfig) { + servers.map { server => + val resource = new ConfigResource(ConfigResource.Type.BROKER, server.config.brokerId.toString) + (resource, newConfig) + }.toMap.asJava + } else { + Map(new ConfigResource(ConfigResource.Type.BROKER, "") -> newConfig).asJava + } + adminClient.alterConfigs(configs) + } + /** * Capture the console output during the execution of the provided function. */ diff --git a/docs/streams/upgrade-guide.html b/docs/streams/upgrade-guide.html index d21d505671aa2..fdc0af1de1fb1 100644 --- a/docs/streams/upgrade-guide.html +++ b/docs/streams/upgrade-guide.html @@ -40,37 +40,64 @@

Upgrade Guide and API Changes

- If you want to upgrade from 1.0.x to 1.1.0 and you have customized window store implementations on the ReadOnlyWindowStore interface + If you want to upgrade from 1.0.x to 1.2.0 and you have customized window store implementations on the ReadOnlyWindowStore interface you'd need to update your code to incorporate the newly added public APIs. Otherwise, if you are using Java 7 you don't need to make any code changes as the public API is fully backward compatible; but if you are using Java 8 method references in your Kafka Streams code you might need to update your code to resolve method ambiguities. Hot-swaping the jar-file only might not work for this case. - See below for a complete list of 1.1.0 API and semantic changes that allow you to advance your application and/or simplify your code base. + See below a complete list of 1.2.0 and 1.1.0 + API and semantic changes that allow you to advance your application and/or simplify your code base.

- If you want to upgrade from 0.11.0.x to 1.0.0 you don't need to make any code changes as the public API is fully backward compatible. + If you want to upgrade from 0.10.2.x or 0.11.0.x to 1.2.x and you have customized window store implementations on the ReadOnlyWindowStore interface + you'd need to update your code to incorporate the newly added public APIs. + Otherwise, if you are using Java 7 you don't need to do any code changes as the public API is fully backward compatible; + but if you are using Java 8 method references in your Kafka Streams code you might need to update your code to resolve method ambiguities. However, some public APIs were deprecated and thus it is recommended to update your code eventually to allow for future upgrades. - See below for a complete list of 1.0.0 API and semantic changes that allow you to advance your application and/or simplify your code base. -

- -

- If you want to upgrade from 0.10.2.x to 0.11.0 you don't need to make any code changes as the public API is fully backward compatible. - However, some configuration parameters were deprecated and thus it is recommended to update your code eventually to allow for future upgrades. - See below for a complete list of 0.11.0 API and semantic changes that allow you to advance your application and/or simplify your code base. + See below a complete list of 1.2, 1.1, + 1.0, and 0.11.0 API + and semantic changes that allow you to advance your application and/or simplify your code base, including the usage of new features. + Additionally, Streams API 1.1.x requires broker on-disk message format version 0.10 or higher; thus, you need to make sure that the message + format is configured correctly before you upgrade your Kafka Streams application.

- If you want to upgrade from 0.10.1.x to 0.10.2, see the Upgrade Section for 0.10.2. - It highlights incompatible changes you need to consider to upgrade your code and application. - See below for a complete list of 0.10.2 API and semantic changes that allow you to advance your application and/or simplify your code base. + If you want to upgrade from 0.10.1.x to 1.2.x see the Upgrade Sections for 0.10.2, + 0.11.0, + 1.0, + 1.0, and + 1.2. + Note, that a brokers on-disk message format must be on version 0.10 or higher to run a Kafka Streams application version 1.2 or higher. + See below a complete list of 0.10.2, 0.11.0, + 1.0, 1.1, and 1.2 + API and semantical changes that allow you to advance your application and/or simplify your code base, including the usage of new features.

- If you want to upgrade from 0.10.0.x to 0.10.1, see the Upgrade Section for 0.10.1. - It highlights incompatible changes you need to consider to upgrade your code and application. - See below a complete list of 0.10.1 API changes that allow you to advance your application and/or simplify your code base, including the usage of new features. + Upgrading from 0.10.0.x to 1.2.0 directly is also possible. + Note, that a brokers must be on version 0.10.1 or higher and on-disk message format must be on version 0.10 or higher + to run a Kafka Streams application version 1.2 or higher. + See Streams API changes in 0.10.1, Streams API changes in 0.10.2, + Streams API changes in 0.11.0, Streams API changes in 1.0, and + Streams API changes in 1.1, and Streams API changes in 1.2 + for a complete list of API changes. + Upgrading to 1.2.0 requires two rolling bounces with config upgrade.from="0.10.0" set for first upgrade phase + (cf. KIP-268). + As an alternative, an offline upgrade is also possible.

+
    +
  • prepare your application instances for a rolling bounce and make sure that config upgrade.from is set to "0.10.0" for new version 1.2.0
    • +
  • bounce each instance of your application once
    • +
  • prepare your newly deployed 1.2.0 application instances for a second round of rolling bounces; make sure to remove the value for config upgrade.mode
    • +
  • bounce each instance of your application once more to complete the upgrade
    • +
+

Upgrading from 0.10.0.x to 1.2.0 in offline mode:

+
    +
  • stop all old (0.10.0.x) application instances
    • +
  • update your code and swap old code and jar file with new code and new jar file
    • +
  • restart all new (1.2.0) application instances
    • +

Streams API changes in 1.2.0

diff --git a/docs/upgrade.html b/docs/upgrade.html index 0c5f5fd3247a2..95f2c418c3be2 100644 --- a/docs/upgrade.html +++ b/docs/upgrade.html @@ -68,6 +68,7 @@
Notable changes in 1
  • KIP-186 increases the default offset retention time from 1 day to 7 days. This makes it less likely to "lose" offsets in an application that commits infrequently. It also increases the active set of offsets and therefore can increase memory usage on the broker. Note that the console consumer currently enables offset commit by default and can be the source of a large number of offsets which this change will now preserve for 7 days instead of 1. You can preserve the existing behavior by setting the broker config offsets.retention.minutes to 1440.
  • KAFKA-5674 extends the lower interval of max.connections.per.ip minimum to zero and therefore allows IP-based filtering of inbound connections.
    • +
  • New Kafka Streams configuration parameter upgrade.from added that allows rolling bounce upgrade from older version.
New Protocol Versions
@@ -76,7 +77,7 @@
New Prot
Upgrading a 1.2.0 Kafka Streams Application
  • Upgrading your Streams application from 1.1.0 to 1.2.0 does not require a broker upgrade. - A Kafka Streams 1.2.0 application can connect to 1.2, 1.1, 1.0, 0.11.0, 0.10.2 and 0.10.1 brokers (it is not possible to connect to 0.10.0 brokers though).
    • + A Kafka Streams 1.2.0 application can connect to 1.2, 1.1, 1.0, 0.11.0, 0.10.2 and 0.10.1 brokers (it is not possible to connect to 0.10.0 brokers though).
  • See Streams API changes in 1.2.0 for more details.
@@ -125,6 +126,14 @@

Upgrading from 0.8.x, 0.9.x, 0.1 Hot-swaping the jar-file only might not work. + +
Notable changes in 1.1.0
  • The kafka artifact in Maven no longer depends on log4j or slf4j-log4j12. Similarly to the kafka-clients artifact, users @@ -196,6 +205,14 @@

    Upgrading from 0.8.x, 0.9.x, 0.1 Similarly for the message format version.

    • + +
    Notable changes in 1.0.1
    -
    Upgrading a 1.0.0 Kafka Streams Application
    +
    Upgrading a 0.11.0 Kafka Streams Application
    • Upgrading your Streams application from 0.11.0 to 1.0.0 does not require a broker upgrade. - A Kafka Streams 1.0.0 application can connect to 0.11.0, 0.10.2 and 0.10.1 brokers (it is not possible to connect to 0.10.0 brokers though). - However, Kafka Streams 1.0 requires 0.10 message format or newer and does not work with older message formats.
      • + A Kafka Streams 1.0.0 application can connect to 0.11.0, 0.10.2 and 0.10.1 brokers (it is not possible to connect to 0.10.0 brokers though). + However, Kafka Streams 1.0 requires 0.10 message format or newer and does not work with older message formats.
    • If you are monitoring on streams metrics, you will need make some changes to the metrics names in your reporting and monitoring code, because the metrics sensor hierarchy was changed.
    • There are a few public APIs including ProcessorContext#schedule(), Processor#punctuate() and KStreamBuilder, TopologyBuilder are being deprecated by new APIs. - We recommend making corresponding code changes, which should be very minor since the new APIs look quite similar, when you upgrade. + We recommend making corresponding code changes, which should be very minor since the new APIs look quite similar, when you upgrade.
    • See Streams API changes in 1.0.0 for more details.
    +
    Upgrading a 0.10.2 Kafka Streams Application
    +
      +
    • Upgrading your Streams application from 0.10.2 to 1.0 does not require a broker upgrade. + A Kafka Streams 1.0 application can connect to 1.0, 0.11.0, 0.10.2 and 0.10.1 brokers (it is not possible to connect to 0.10.0 brokers though).
      • +
    • If you are monitoring on streams metrics, you will need make some changes to the metrics names in your reporting and monitoring code, because the metrics sensor hierarchy was changed.
      • +
    • There are a few public APIs including ProcessorContext#schedule(), Processor#punctuate() and KStreamBuilder, TopologyBuilder are being deprecated by new APIs. + We recommend making corresponding code changes, which should be very minor since the new APIs look quite similar, when you upgrade. +
    • If you specify customized key.serde, value.serde and timestamp.extractor in configs, it is recommended to use their replaced configure parameter as these configs are deprecated.
      • +
    • See Streams API changes in 0.11.0 for more details.
      • +
    + +
    Upgrading a 0.10.1 Kafka Streams Application
    +
      +
    • Upgrading your Streams application from 0.10.1 to 1.0 does not require a broker upgrade. + A Kafka Streams 1.0 application can connect to 1.0, 0.11.0, 0.10.2 and 0.10.1 brokers (it is not possible to connect to 0.10.0 brokers though).
      • +
    • You need to recompile your code. Just swapping the Kafka Streams library jar file will not work and will break your application.
      • +
    • If you are monitoring on streams metrics, you will need make some changes to the metrics names in your reporting and monitoring code, because the metrics sensor hierarchy was changed.
      • +
    • There are a few public APIs including ProcessorContext#schedule(), Processor#punctuate() and KStreamBuilder, TopologyBuilder are being deprecated by new APIs. + We recommend making corresponding code changes, which should be very minor since the new APIs look quite similar, when you upgrade. +
    • If you specify customized key.serde, value.serde and timestamp.extractor in configs, it is recommended to use their replaced configure parameter as these configs are deprecated.
      • +
    • If you use a custom (i.e., user implemented) timestamp extractor, you will need to update this code, because the TimestampExtractor interface was changed.
      • +
    • If you register custom metrics, you will need to update this code, because the StreamsMetric interface was changed.
      • +
    • See Streams API changes in 1.0.0, + Streams API changes in 0.11.0 and + Streams API changes in 0.10.2 for more details.
      • +
    + +
    Upgrading a 0.10.0 Kafka Streams Application
    +
      +
    • Upgrading your Streams application from 0.10.0 to 1.0 does require a broker upgrade because a Kafka Streams 1.0 application can only connect to 0.1, 0.11.0, 0.10.2, or 0.10.1 brokers.
      • +
    • There are couple of API changes, that are not backward compatible (cf. Streams API changes in 1.0.0, + Streams API changes in 0.11.0, + Streams API changes in 0.10.2, and + Streams API changes in 0.10.1 for more details). + Thus, you need to update and recompile your code. Just swapping the Kafka Streams library jar file will not work and will break your application.
      • + +
    • Upgrading from 0.10.0.x to 1.0.0 or 1.0.1 requires an offline upgrade (rolling bounce upgrade is not supported) + +
        +
      • stop all old (0.10.0.x) application instances
        • +
      • update your code and swap old code and jar file with new code and new jar file
        • +
      • restart all new (1.0.0 or 1.0.1) application instances
        • +
      +
      • +
    +

    Upgrading from 0.8.x, 0.9.x, 0.10.0.x, 0.10.1.x or 0.10.2.x to 0.11.0.0

    Kafka 0.11.0.0 introduces a new message format version as well as wire protocol changes. By following the recommended rolling upgrade plan below, you guarantee no downtime during the upgrade. However, please review the notable changes in 0.11.0.0 before upgrading. @@ -291,7 +365,7 @@

    Upgrading from 0.8.x, 0.9.x, 0
  • Upgrade the brokers one at a time: shut down the broker, update the code, and restart it.
    • @@ -320,11 +394,59 @@

    Upgrading from 0.8.x, 0.9.x, 0
    Upgrading a 0.10.2 Kafka Streams Application
    • Upgrading your Streams application from 0.10.2 to 0.11.0 does not require a broker upgrade. - A Kafka Streams 0.11.0 application can connect to 0.11.0, 0.10.2 and 0.10.1 brokers (it is not possible to connect to 0.10.0 brokers though).
      • + A Kafka Streams 0.11.0 application can connect to 0.11.0, 0.10.2 and 0.10.1 brokers (it is not possible to connect to 0.10.0 brokers though).
    • If you specify customized key.serde, value.serde and timestamp.extractor in configs, it is recommended to use their replaced configure parameter as these configs are deprecated.
    • See Streams API changes in 0.11.0 for more details.
    +

    Upgrading a 0.10.1 Kafka Streams Application
    +
      +
    • Upgrading your Streams application from 0.10.1 to 0.11.0 does not require a broker upgrade. + A Kafka Streams 0.11.0 application can connect to 0.11.0, 0.10.2 and 0.10.1 brokers (it is not possible to connect to 0.10.0 brokers though).
      • +
    • You need to recompile your code. Just swapping the Kafka Streams library jar file will not work and will break your application.
      • +
    • If you specify customized key.serde, value.serde and timestamp.extractor in configs, it is recommended to use their replaced configure parameter as these configs are deprecated.
      • +
    • If you use a custom (i.e., user implemented) timestamp extractor, you will need to update this code, because the TimestampExtractor interface was changed.
      • +
    • If you register custom metrics, you will need to update this code, because the StreamsMetric interface was changed.
      • +
    • See Streams API changes in 0.11.0 and + Streams API changes in 0.10.2 for more details.
      • +
    + +
    Upgrading a 0.10.0 Kafka Streams Application
    +
      +
    • Upgrading your Streams application from 0.10.0 to 0.11.0 does require a broker upgrade because a Kafka Streams 0.11.0 application can only connect to 0.11.0, 0.10.2, or 0.10.1 brokers.
      • +
    • There are couple of API changes, that are not backward compatible (cf. Streams API changes in 0.11.0, + Streams API changes in 0.10.2, and + Streams API changes in 0.10.1 for more details). + Thus, you need to update and recompile your code. Just swapping the Kafka Streams library jar file will not work and will break your application.
      • + +
    • Upgrading from 0.10.0.x to 0.11.0.0, 0.11.0.1, or 0.11.0.2 requires an offline upgrade (rolling bounce upgrade is not supported) +
        +
      • stop all old (0.10.0.x) application instances
        • +
      • update your code and swap old code and jar file with new code and new jar file
        • +
      • restart all new (0.11.0.0 , 0.11.0.1, or 0.11.0.2) application instances
        • +
      +
      • +
    + + +
    Notable changes in 0.11.0.0
    +
    Upgrading a 0.10.0 Kafka Streams Application
    +
      +
    • Upgrading your Streams application from 0.10.0 to 0.10.2 does require a broker upgrade because a Kafka Streams 0.10.2 application can only connect to 0.10.2 or 0.10.1 brokers.
      • +
    • There are couple of API changes, that are not backward compatible (cf. Streams API changes in 0.10.2 for more details). + Thus, you need to update and recompile your code. Just swapping the Kafka Streams library jar file will not work and will break your application.
      • + +
    • Upgrading from 0.10.0.x to 0.10.2.0 or 0.10.2.1 requires an offline upgrade (rolling bounce upgrade is not supported) +
        +
      • stop all old (0.10.0.x) application instances
        • +
      • update your code and swap old code and jar file with new code and new jar file
        • +
      • restart all new (0.10.2.0 or 0.10.2.1) application instances
        • +
      +
      • +
    + + +
    Notable changes in 0.10.2.1
    • The default values for two configurations of the StreamsConfig class were changed to improve the resiliency of Kafka Streams applications. The internal Kafka Streams producer retries default value was changed from 0 to 10. The internal Kafka Streams consumer max.poll.interval.ms default value was changed from 300000 to Integer.MAX_VALUE. @@ -552,7 +707,26 @@
      Upgrading a 0.10.0
      • Upgrading your Streams application from 0.10.0 to 0.10.1 does require a broker upgrade because a Kafka Streams 0.10.1 application can only connect to 0.10.1 brokers.
      • There are couple of API changes, that are not backward compatible (cf. Streams API changes in 0.10.1 for more details). - Thus, you need to update and recompile your code. Just swapping the Kafka Streams library jar file will not work and will break your application.
        • + Thus, you need to update and recompile your code. Just swapping the Kafka Streams library jar file will not work and will break your application. + +
      • Upgrading from 0.10.0.x to 0.10.1.0 or 0.10.1.1 requires an offline upgrade (rolling bounce upgrade is not supported) +
          +
        • stop all old (0.10.0.x) application instances
          • +
        • update your code and swap old code and jar file with new code and new jar file
          • +
        • restart all new (0.10.1.0 or 0.10.1.1) application instances
          • +
        +
      Notable changes in 0.10.1.0
      @@ -596,14 +770,17 @@
      New Pr

    Upgrading from 0.8.x or 0.9.x to 0.10.0.0

    +

    0.10.0.0 has potential breaking changes (please review before upgrading) and possible performance impact following the upgrade. By following the recommended rolling upgrade plan below, you guarantee no downtime and no performance impact during and following the upgrade.
    Note: Because new protocols are introduced, it is important to upgrade your Kafka clusters before upgrading your clients. -

    +

    +

    Notes to clients with version 0.9.0.0: Due to a bug introduced in 0.9.0.0, clients that depend on ZooKeeper (old Scala high-level Consumer and MirrorMaker if used with the old consumer) will not work with 0.10.0.x brokers. Therefore, 0.9.0.0 clients should be upgraded to 0.9.0.1 before brokers are upgraded to 0.10.0.x. This step is not necessary for 0.8.X or 0.9.0.1 clients. +

    For a rolling upgrade:

    diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index 32c0040d0ff3b..d6beba958ddaa 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -52,7 +52,7 @@ versions += [ apacheds: "2.0.0-M24", argparse4j: "0.7.0", bcpkix: "1.59", - easymock: "3.5.1", + easymock: "3.6", jackson: "2.9.5", jetty: "9.2.24.v20180105", jersey: "2.25.1", @@ -62,6 +62,11 @@ versions += [ jaxb: "2.3.0", jopt: "5.0.4", junit: "4.12", + kafka_0100: "0.10.0.1", + kafka_0101: "0.10.1.1", + kafka_0102: "0.10.2.1", + kafka_0110: "0.11.0.2", + kafka_10: "1.0.1", lz4: "1.4.1", metrics: "2.2.0", // PowerMock 1.x doesn't support Java 9, so use PowerMock 2.0.0 beta @@ -101,12 +106,16 @@ libs += [ jettyServlets: "org.eclipse.jetty:jetty-servlets:$versions.jetty", jerseyContainerServlet: "org.glassfish.jersey.containers:jersey-container-servlet:$versions.jersey", jmhCore: "org.openjdk.jmh:jmh-core:$versions.jmh", - jmhGeneratorAnnProcess: "org.openjdk.jmh:jmh-generator-annprocess:$versions.jmh", jmhCoreBenchmarks: "org.openjdk.jmh:jmh-core-benchmarks:$versions.jmh", + jmhGeneratorAnnProcess: "org.openjdk.jmh:jmh-generator-annprocess:$versions.jmh", + joptSimple: "net.sf.jopt-simple:jopt-simple:$versions.jopt", junit: "junit:junit:$versions.junit", + kafkaStreams_0100: "org.apache.kafka:kafka-streams:$versions.kafka_0100", + kafkaStreams_0101: "org.apache.kafka:kafka-streams:$versions.kafka_0101", + kafkaStreams_0102: "org.apache.kafka:kafka-streams:$versions.kafka_0102", + kafkaStreams_0110: "org.apache.kafka:kafka-streams:$versions.kafka_0110", + kafkaStreams_10: "org.apache.kafka:kafka-streams:$versions.kafka_10", log4j: "log4j:log4j:$versions.log4j", - scalaLogging: "com.typesafe.scala-logging:scala-logging_$versions.baseScala:$versions.scalaLogging", - joptSimple: "net.sf.jopt-simple:jopt-simple:$versions.jopt", lz4: "org.lz4:lz4-java:$versions.lz4", metrics: "com.yammer.metrics:metrics-core:$versions.metrics", powermockJunit4: "org.powermock:powermock-module-junit4:$versions.powermock", @@ -114,6 +123,7 @@ libs += [ reflections: "org.reflections:reflections:$versions.reflections", rocksDBJni: "org.rocksdb:rocksdbjni:$versions.rocksDB", scalaLibrary: "org.scala-lang:scala-library:$versions.scala", + scalaLogging: "com.typesafe.scala-logging:scala-logging_$versions.baseScala:$versions.scalaLogging", scalaReflect: "org.scala-lang:scala-reflect:$versions.scala", scalatest: "org.scalatest:scalatest_$versions.baseScala:$versions.scalatest", scoveragePlugin: "org.scoverage:scalac-scoverage-plugin_$versions.baseScala:$versions.scoverage", diff --git a/settings.gradle b/settings.gradle index e599d01215cc9..03136849fd543 100644 --- a/settings.gradle +++ b/settings.gradle @@ -13,5 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -include 'core', 'examples', 'clients', 'tools', 'streams', 'streams:test-utils', 'streams:examples', 'log4j-appender', +include 'core', 'examples', 'clients', 'tools', 'streams', 'streams:test-utils', 'streams:examples', + 'streams:upgrade-system-tests-0100', 'streams:upgrade-system-tests-0101', 'streams:upgrade-system-tests-0102', + 'streams:upgrade-system-tests-0110', 'streams:upgrade-system-tests-10', 'log4j-appender', 'connect:api', 'connect:transforms', 'connect:runtime', 'connect:json', 'connect:file', 'jmh-benchmarks' diff --git a/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java b/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java index 0a525169fa66d..819bebd43b690 100644 --- a/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java +++ b/streams/src/main/java/org/apache/kafka/streams/StreamsConfig.java @@ -167,6 +167,11 @@ public class StreamsConfig extends AbstractConfig { */ public static final String ADMIN_CLIENT_PREFIX = "admin."; + /** + * Config value for parameter {@link #UPGRADE_FROM_CONFIG "upgrade.from"} for upgrading an application from version {@code 0.10.0.x}. + */ + public static final String UPGRADE_FROM_0100 = "0.10.0"; + /** * Config value for parameter {@link #PROCESSING_GUARANTEE_CONFIG "processing.guarantee"} for at-least-once processing guarantees. */ @@ -340,6 +345,11 @@ public class StreamsConfig extends AbstractConfig { public static final String TIMESTAMP_EXTRACTOR_CLASS_CONFIG = "timestamp.extractor"; private static final String TIMESTAMP_EXTRACTOR_CLASS_DOC = "Timestamp extractor class that implements the org.apache.kafka.streams.processor.TimestampExtractor interface. This config is deprecated, use " + DEFAULT_TIMESTAMP_EXTRACTOR_CLASS_CONFIG + " instead"; + /** {@code upgrade.from} */ + public static final String UPGRADE_FROM_CONFIG = "upgrade.from"; + public static final String UPGRADE_FROM_DOC = "Allows upgrading from version 0.10.0 to version 0.10.1 (or newer) in a backward compatible way. " + + "Default is null. Accepted values are \"" + UPGRADE_FROM_0100 + "\" (for upgrading from 0.10.0.x)."; + /** * {@code value.serde} * @deprecated Use {@link #DEFAULT_VALUE_SERDE_CLASS_CONFIG} instead. @@ -562,6 +572,12 @@ public class StreamsConfig extends AbstractConfig { 10 * 60 * 1000L, Importance.LOW, STATE_CLEANUP_DELAY_MS_DOC) + .define(UPGRADE_FROM_CONFIG, + ConfigDef.Type.STRING, + null, + in(null, UPGRADE_FROM_0100), + Importance.LOW, + UPGRADE_FROM_DOC) .define(WINDOW_STORE_CHANGE_LOG_ADDITIONAL_RETENTION_MS_CONFIG, Type.LONG, 24 * 60 * 60 * 1000L, @@ -793,6 +809,7 @@ public Map getConsumerConfigs(final String groupId, consumerProps.put(CommonClientConfigs.CLIENT_ID_CONFIG, clientId + "-consumer"); // add configs required for stream partition assignor + consumerProps.put(UPGRADE_FROM_CONFIG, getString(UPGRADE_FROM_CONFIG)); consumerProps.put(REPLICATION_FACTOR_CONFIG, getInt(REPLICATION_FACTOR_CONFIG)); consumerProps.put(APPLICATION_SERVER_CONFIG, getString(APPLICATION_SERVER_CONFIG)); consumerProps.put(NUM_STANDBY_REPLICAS_CONFIG, getInt(NUM_STANDBY_REPLICAS_CONFIG)); diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/PunctuationQueue.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/PunctuationQueue.java index 80eda6c9dd8ba..354c602cb3f23 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/PunctuationQueue.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/PunctuationQueue.java @@ -52,7 +52,10 @@ boolean mayPunctuate(final long timestamp, final PunctuationType type, final Pro if (!sched.isCancelled()) { processorNodePunctuator.punctuate(sched.node(), timestamp, type, sched.punctuator()); - pq.add(sched.next(timestamp)); + // sched can be cancelled from within the punctuator + if (!sched.isCancelled()) { + pq.add(sched.next(timestamp)); + } punctuated = true; } diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java index 8d6e56a17aab8..4b2e1b8cfb753 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamTask.java @@ -358,8 +358,8 @@ private void commitOffsets(final boolean startNewTransaction) { producer.commitTransaction(); transactionInFlight = false; if (startNewTransaction) { - transactionInFlight = true; producer.beginTransaction(); + transactionInFlight = true; } } else { consumer.commitSync(consumedOffsetsAndMetadata); @@ -482,7 +482,7 @@ public void closeSuspended(boolean clean, if (eosEnabled) { if (!clean) { try { - if (!isZombie) { + if (!isZombie && transactionInFlight) { producer.abortTransaction(); } transactionInFlight = false; diff --git a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java index 0edbe2f523d12..97771e568795d 100644 --- a/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java +++ b/streams/src/main/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignor.java @@ -179,6 +179,8 @@ public int compare(final TopicPartition p1, private TaskManager taskManager; private PartitionGrouper partitionGrouper; + private int userMetadataVersion = SubscriptionInfo.LATEST_SUPPORTED_VERSION; + private InternalTopicManager internalTopicManager; private CopartitionedTopicsValidator copartitionedTopicsValidator; @@ -197,6 +199,12 @@ public void configure(final Map configs) { final LogContext logContext = new LogContext(logPrefix); log = logContext.logger(getClass()); + final String upgradeMode = (String) configs.get(StreamsConfig.UPGRADE_FROM_CONFIG); + if (StreamsConfig.UPGRADE_FROM_0100.equals(upgradeMode)) { + log.info("Downgrading metadata version from 2 to 1 for upgrade from 0.10.0.x."); + userMetadataVersion = 1; + } + final Object o = configs.get(StreamsConfig.InternalConfig.TASK_MANAGER_FOR_PARTITION_ASSIGNOR); if (o == null) { final KafkaException fatalException = new KafkaException("TaskManager is not specified"); @@ -255,6 +263,7 @@ public Subscription subscription(final Set topics) { final Set standbyTasks = taskManager.cachedTasksIds(); standbyTasks.removeAll(previousActiveTasks); final SubscriptionInfo data = new SubscriptionInfo( + userMetadataVersion, taskManager.processId(), previousActiveTasks, standbyTasks, diff --git a/streams/src/test/java/org/apache/kafka/streams/StreamsConfigTest.java b/streams/src/test/java/org/apache/kafka/streams/StreamsConfigTest.java index 0309659042d0d..87f7075922072 100644 --- a/streams/src/test/java/org/apache/kafka/streams/StreamsConfigTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/StreamsConfigTest.java @@ -472,6 +472,7 @@ public void shouldNotOverrideUserConfigCommitIntervalMsIfExactlyOnceEnabled() { assertThat(streamsConfig.getLong(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG), equalTo(commitIntervalMs)); } + @SuppressWarnings("deprecation") @Test public void shouldBeBackwardsCompatibleWithDeprecatedConfigs() { final Properties props = minimalStreamsConfig(); @@ -506,6 +507,7 @@ public void shouldUseCorrectDefaultsWhenNoneSpecified() { assertTrue(config.defaultTimestampExtractor() instanceof FailOnInvalidTimestamp); } + @SuppressWarnings("deprecation") @Test public void shouldSpecifyCorrectKeySerdeClassOnErrorUsingDeprecatedConfigs() { final Properties props = minimalStreamsConfig(); @@ -519,6 +521,7 @@ public void shouldSpecifyCorrectKeySerdeClassOnErrorUsingDeprecatedConfigs() { } } + @SuppressWarnings("deprecation") @Test public void shouldSpecifyCorrectKeySerdeClassOnError() { final Properties props = minimalStreamsConfig(); @@ -532,6 +535,7 @@ public void shouldSpecifyCorrectKeySerdeClassOnError() { } } + @SuppressWarnings("deprecation") @Test public void shouldSpecifyCorrectValueSerdeClassOnErrorUsingDeprecatedConfigs() { final Properties props = minimalStreamsConfig(); @@ -545,6 +549,7 @@ public void shouldSpecifyCorrectValueSerdeClassOnErrorUsingDeprecatedConfigs() { } } + @SuppressWarnings("deprecation") @Test public void shouldSpecifyCorrectValueSerdeClassOnError() { final Properties props = minimalStreamsConfig(); @@ -567,9 +572,7 @@ public void configure(final Map configs, final boolean isKey) { } @Override - public void close() { - - } + public void close() {} @Override public Serializer serializer() { diff --git a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamAggregationDedupIntegrationTest.java b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamAggregationDedupIntegrationTest.java index 4c12bb935448f..44e139a28bdbf 100644 --- a/streams/src/test/java/org/apache/kafka/streams/integration/KStreamAggregationDedupIntegrationTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/integration/KStreamAggregationDedupIntegrationTest.java @@ -313,6 +313,4 @@ private List> receiveMessages(final Deserializer } - - } diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/PunctuationQueueTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/PunctuationQueueTest.java index 09c7a0a318344..e799688c428fc 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/PunctuationQueueTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/PunctuationQueueTest.java @@ -17,6 +17,7 @@ package org.apache.kafka.streams.processor.internals; import org.apache.kafka.streams.processor.AbstractProcessor; +import org.apache.kafka.streams.processor.Cancellable; import org.apache.kafka.streams.processor.ProcessorContext; import org.apache.kafka.streams.processor.PunctuationType; import org.apache.kafka.streams.processor.Punctuator; @@ -126,6 +127,42 @@ public void punctuate(ProcessorNode node, long time, PunctuationType type, Punct assertEquals(4, processor.punctuatedAt.size()); } + @Test + public void testPunctuationIntervalCancelFromPunctuator() { + final TestProcessor processor = new TestProcessor(); + final ProcessorNode node = new ProcessorNode<>("test", processor, null); + final PunctuationQueue queue = new PunctuationQueue(); + final Punctuator punctuator = new Punctuator() { + @Override + public void punctuate(long timestamp) { + node.processor().punctuate(timestamp); + } + }; + + final PunctuationSchedule sched = new PunctuationSchedule(node, 0L, 100L, punctuator); + final long now = sched.timestamp - 100L; + + final Cancellable cancellable = queue.schedule(sched); + + ProcessorNodePunctuator processorNodePunctuator = new ProcessorNodePunctuator() { + @Override + public void punctuate(ProcessorNode node, long time, PunctuationType type, Punctuator punctuator) { + punctuator.punctuate(time); + // simulate scheduler cancelled from within punctuator + cancellable.cancel(); + } + }; + + queue.mayPunctuate(now, PunctuationType.STREAM_TIME, processorNodePunctuator); + assertEquals(0, processor.punctuatedAt.size()); + + queue.mayPunctuate(now + 100L, PunctuationType.STREAM_TIME, processorNodePunctuator); + assertEquals(1, processor.punctuatedAt.size()); + + queue.mayPunctuate(now + 200L, PunctuationType.STREAM_TIME, processorNodePunctuator); + assertEquals(1, processor.punctuatedAt.size()); + } + private static class TestProcessor extends AbstractProcessor { public final ArrayList punctuatedAt = new ArrayList<>(); diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamTaskTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamTaskTest.java index a30582905de27..d6a5276a43d7c 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamTaskTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamTaskTest.java @@ -782,6 +782,14 @@ public void shouldInitAndBeginTransactionOnCreateIfEosEnabled() { assertTrue(producer.transactionInFlight()); } + @Test + public void shouldNotThrowOnCloseIfTaskWasNotInitializedWithEosEnabled() { + task = createStatelessTask(true); + + assertTrue(!producer.transactionInFlight()); + task.close(false, false); + } + @Test public void shouldNotInitOrBeginTransactionOnCreateIfEosDisabled() { task = createStatelessTask(false); diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignorTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignorTest.java index b0c0d68287b19..e9ed9682066eb 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignorTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/StreamsPartitionAssignorTest.java @@ -131,7 +131,7 @@ private void configurePartitionAssignor(final Map props) { private void mockTaskManager(final Set prevTasks, final Set cachedTasks, final UUID processId, - final InternalTopologyBuilder builder) throws NoSuchFieldException, IllegalAccessException { + final InternalTopologyBuilder builder) { EasyMock.expect(taskManager.builder()).andReturn(builder).anyTimes(); EasyMock.expect(taskManager.prevActiveTaskIds()).andReturn(prevTasks).anyTimes(); EasyMock.expect(taskManager.cachedTasksIds()).andReturn(cachedTasks).anyTimes(); @@ -167,7 +167,7 @@ public void shouldInterleaveTasksByGroupId() { } @Test - public void testSubscription() throws Exception { + public void testSubscription() { builder.addSource(null, "source1", null, null, null, "topic1"); builder.addSource(null, "source2", null, null, null, "topic2"); builder.addProcessor("processor", new MockProcessorSupplier(), "source1", "source2"); @@ -195,7 +195,7 @@ public void testSubscription() throws Exception { } @Test - public void testAssignBasic() throws Exception { + public void testAssignBasic() { builder.addSource(null, "source1", null, null, null, "topic1"); builder.addSource(null, "source2", null, null, null, "topic2"); builder.addProcessor("processor", new MockProcessorSupplier(), "source1", "source2"); @@ -235,11 +235,9 @@ public void testAssignBasic() throws Exception { // check assignment info - Set allActiveTasks = new HashSet<>(); - // the first consumer AssignmentInfo info10 = checkAssignment(allTopics, assignments.get("consumer10")); - allActiveTasks.addAll(info10.activeTasks()); + Set allActiveTasks = new HashSet<>(info10.activeTasks()); // the second consumer AssignmentInfo info11 = checkAssignment(allTopics, assignments.get("consumer11")); @@ -259,7 +257,7 @@ public void testAssignBasic() throws Exception { } @Test - public void shouldAssignEvenlyAcrossConsumersOneClientMultipleThreads() throws Exception { + public void shouldAssignEvenlyAcrossConsumersOneClientMultipleThreads() { builder.addSource(null, "source1", null, null, null, "topic1"); builder.addSource(null, "source2", null, null, null, "topic2"); builder.addProcessor("processor", new MockProcessorSupplier(), "source1"); @@ -327,7 +325,7 @@ public void shouldAssignEvenlyAcrossConsumersOneClientMultipleThreads() throws E } @Test - public void testAssignWithPartialTopology() throws Exception { + public void testAssignWithPartialTopology() { builder.addSource(null, "source1", null, null, null, "topic1"); builder.addProcessor("processor1", new MockProcessorSupplier(), "source1"); builder.addStateStore(new MockStateStoreSupplier("store1", false), "processor1"); @@ -352,9 +350,8 @@ public void testAssignWithPartialTopology() throws Exception { Map assignments = partitionAssignor.assign(metadata, subscriptions); // check assignment info - Set allActiveTasks = new HashSet<>(); AssignmentInfo info10 = checkAssignment(Utils.mkSet("topic1"), assignments.get("consumer10")); - allActiveTasks.addAll(info10.activeTasks()); + Set allActiveTasks = new HashSet<>(info10.activeTasks()); assertEquals(3, allActiveTasks.size()); assertEquals(allTasks, new HashSet<>(allActiveTasks)); @@ -362,7 +359,7 @@ public void testAssignWithPartialTopology() throws Exception { @Test - public void testAssignEmptyMetadata() throws Exception { + public void testAssignEmptyMetadata() { builder.addSource(null, "source1", null, null, null, "topic1"); builder.addSource(null, "source2", null, null, null, "topic2"); builder.addProcessor("processor", new MockProcessorSupplier(), "source1", "source2"); @@ -392,9 +389,8 @@ public void testAssignEmptyMetadata() throws Exception { new HashSet<>(assignments.get("consumer10").partitions())); // check assignment info - Set allActiveTasks = new HashSet<>(); AssignmentInfo info10 = checkAssignment(Collections.emptySet(), assignments.get("consumer10")); - allActiveTasks.addAll(info10.activeTasks()); + Set allActiveTasks = new HashSet<>(info10.activeTasks()); assertEquals(0, allActiveTasks.size()); assertEquals(Collections.emptySet(), new HashSet<>(allActiveTasks)); @@ -417,7 +413,7 @@ public void testAssignEmptyMetadata() throws Exception { } @Test - public void testAssignWithNewTasks() throws Exception { + public void testAssignWithNewTasks() { builder.addSource(null, "source1", null, null, null, "topic1"); builder.addSource(null, "source2", null, null, null, "topic2"); builder.addSource(null, "source3", null, null, null, "topic3"); @@ -450,13 +446,9 @@ public void testAssignWithNewTasks() throws Exception { // check assigned partitions: since there is no previous task for topic 3 it will be assigned randomly so we cannot check exact match // also note that previously assigned partitions / tasks may not stay on the previous host since we may assign the new task first and // then later ones will be re-assigned to other hosts due to load balancing - Set allActiveTasks = new HashSet<>(); - Set allPartitions = new HashSet<>(); - AssignmentInfo info; - - info = AssignmentInfo.decode(assignments.get("consumer10").userData()); - allActiveTasks.addAll(info.activeTasks()); - allPartitions.addAll(assignments.get("consumer10").partitions()); + AssignmentInfo info = AssignmentInfo.decode(assignments.get("consumer10").userData()); + Set allActiveTasks = new HashSet<>(info.activeTasks()); + Set allPartitions = new HashSet<>(assignments.get("consumer10").partitions()); info = AssignmentInfo.decode(assignments.get("consumer11").userData()); allActiveTasks.addAll(info.activeTasks()); @@ -471,7 +463,7 @@ public void testAssignWithNewTasks() throws Exception { } @Test - public void testAssignWithStates() throws Exception { + public void testAssignWithStates() { builder.setApplicationId(applicationId); builder.addSource(null, "source1", null, null, null, "topic1"); builder.addSource(null, "source2", null, null, null, "topic2"); @@ -542,7 +534,10 @@ public void testAssignWithStates() throws Exception { assertEquals(Utils.mkSet(task10, task11, task12), tasksForState(applicationId, "store3", tasks, topicGroups)); } - private Set tasksForState(String applicationId, String storeName, List tasks, Map topicGroups) { + private Set tasksForState(final String applicationId, + final String storeName, + final List tasks, + final Map topicGroups) { final String changelogTopic = ProcessorStateManager.storeChangelogTopic(applicationId, storeName); Set ids = new HashSet<>(); @@ -560,7 +555,7 @@ private Set tasksForState(String applicationId, String storeName, List props = configProps(); props.put(StreamsConfig.NUM_STANDBY_REPLICAS_CONFIG, "1"); StreamsConfig streamsConfig = new StreamsConfig(props); @@ -598,13 +593,10 @@ public void testAssignWithStandbyReplicas() throws Exception { Map assignments = partitionAssignor.assign(metadata, subscriptions); - Set allActiveTasks = new HashSet<>(); - Set allStandbyTasks = new HashSet<>(); - // the first consumer AssignmentInfo info10 = checkAssignment(allTopics, assignments.get("consumer10")); - allActiveTasks.addAll(info10.activeTasks()); - allStandbyTasks.addAll(info10.standbyTasks().keySet()); + Set allActiveTasks = new HashSet<>(info10.activeTasks()); + Set allStandbyTasks = new HashSet<>(info10.standbyTasks().keySet()); // the second consumer AssignmentInfo info11 = checkAssignment(allTopics, assignments.get("consumer11")); @@ -632,7 +624,7 @@ public void testAssignWithStandbyReplicas() throws Exception { } @Test - public void testOnAssignment() throws Exception { + public void testOnAssignment() { configurePartitionAssignor(Collections.emptyMap()); final List activeTaskList = Utils.mkList(task0, task3); @@ -667,7 +659,7 @@ public void testOnAssignment() throws Exception { } @Test - public void testAssignWithInternalTopics() throws Exception { + public void testAssignWithInternalTopics() { builder.setApplicationId(applicationId); builder.addInternalTopic("topicX"); builder.addSource(null, "source1", null, null, null, "topic1"); @@ -697,7 +689,7 @@ public void testAssignWithInternalTopics() throws Exception { } @Test - public void testAssignWithInternalTopicThatsSourceIsAnotherInternalTopic() throws Exception { + public void testAssignWithInternalTopicThatsSourceIsAnotherInternalTopic() { String applicationId = "test"; builder.setApplicationId(applicationId); builder.addInternalTopic("topicX"); @@ -732,7 +724,7 @@ public void testAssignWithInternalTopicThatsSourceIsAnotherInternalTopic() throw } @Test - public void shouldGenerateTasksForAllCreatedPartitions() throws Exception { + public void shouldGenerateTasksForAllCreatedPartitions() { final StreamsBuilder builder = new StreamsBuilder(); final InternalTopologyBuilder internalTopologyBuilder = StreamsBuilderTest.internalTopologyBuilder(builder); @@ -832,7 +824,7 @@ public Object apply(final Object value1, final Object value2) { } @Test - public void shouldAddUserDefinedEndPointToSubscription() throws Exception { + public void shouldAddUserDefinedEndPointToSubscription() { builder.setApplicationId(applicationId); builder.addSource(null, "source", null, null, null, "input"); builder.addProcessor("processor", new MockProcessorSupplier(), "source"); @@ -851,7 +843,7 @@ public void shouldAddUserDefinedEndPointToSubscription() throws Exception { } @Test - public void shouldMapUserEndPointToTopicPartitions() throws Exception { + public void shouldMapUserEndPointToTopicPartitions() { builder.setApplicationId(applicationId); builder.addSource(null, "source", null, null, null, "topic1"); builder.addProcessor("processor", new MockProcessorSupplier(), "source"); @@ -881,7 +873,7 @@ public void shouldMapUserEndPointToTopicPartitions() throws Exception { } @Test - public void shouldThrowExceptionIfApplicationServerConfigIsNotHostPortPair() throws Exception { + public void shouldThrowExceptionIfApplicationServerConfigIsNotHostPortPair() { builder.setApplicationId(applicationId); mockTaskManager(Collections.emptySet(), Collections.emptySet(), UUID.randomUUID(), builder); @@ -908,7 +900,7 @@ public void shouldThrowExceptionIfApplicationServerConfigPortIsNotAnInteger() { } @Test - public void shouldNotLoopInfinitelyOnMissingMetadataAndShouldNotCreateRelatedTasks() throws Exception { + public void shouldNotLoopInfinitelyOnMissingMetadataAndShouldNotCreateRelatedTasks() { final StreamsBuilder builder = new StreamsBuilder(); final InternalTopologyBuilder internalTopologyBuilder = StreamsBuilderTest.internalTopologyBuilder(builder); @@ -1010,7 +1002,7 @@ public Object apply(final Object value1, final Object value2) { } @Test - public void shouldUpdateClusterMetadataAndHostInfoOnAssignment() throws Exception { + public void shouldUpdateClusterMetadataAndHostInfoOnAssignment() { final TopicPartition partitionOne = new TopicPartition("topic", 1); final TopicPartition partitionTwo = new TopicPartition("topic", 2); final Map> hostState = Collections.singletonMap( @@ -1028,7 +1020,7 @@ public void shouldUpdateClusterMetadataAndHostInfoOnAssignment() throws Exceptio } @Test - public void shouldNotAddStandbyTaskPartitionsToPartitionsForHost() throws Exception { + public void shouldNotAddStandbyTaskPartitionsToPartitionsForHost() { final StreamsBuilder builder = new StreamsBuilder(); final InternalTopologyBuilder internalTopologyBuilder = StreamsBuilderTest.internalTopologyBuilder(builder); @@ -1096,7 +1088,7 @@ public void shouldThrowKafkaExceptionIfStreamThreadConfigIsNotThreadDataProvider } @Test - public void shouldReturnLowestAssignmentVersionForDifferentSubscriptionVersions() throws Exception { + public void shouldReturnLowestAssignmentVersionForDifferentSubscriptionVersions() { final Map subscriptions = new HashMap<>(); final Set emptyTasks = Collections.emptySet(); subscriptions.put( @@ -1114,10 +1106,11 @@ public void shouldReturnLowestAssignmentVersionForDifferentSubscriptionVersions( ) ); - mockTaskManager(Collections.emptySet(), - Collections.emptySet(), + mockTaskManager( + emptyTasks, + emptyTasks, UUID.randomUUID(), - new InternalTopologyBuilder()); + builder); partitionAssignor.configure(configProps()); final Map assignment = partitionAssignor.assign(metadata, subscriptions); @@ -1126,6 +1119,22 @@ public void shouldReturnLowestAssignmentVersionForDifferentSubscriptionVersions( assertThat(AssignmentInfo.decode(assignment.get("consumer2").userData()).version(), equalTo(1)); } + @Test + public void shouldDownGradeSubscription() { + final Set emptyTasks = Collections.emptySet(); + + mockTaskManager( + emptyTasks, + emptyTasks, + UUID.randomUUID(), + builder); + configurePartitionAssignor(Collections.singletonMap(StreamsConfig.UPGRADE_FROM_CONFIG, (Object) StreamsConfig.UPGRADE_FROM_0100)); + + PartitionAssignor.Subscription subscription = partitionAssignor.subscription(Utils.mkSet("topic1")); + + assertThat(SubscriptionInfo.decode(subscription.userData()).version(), equalTo(1)); + } + private PartitionAssignor.Assignment createAssignment(final Map> firstHostState) { final AssignmentInfo info = new AssignmentInfo(Collections.emptyList(), Collections.>emptyMap(), diff --git a/streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/AssignmentInfoTest.java b/streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/AssignmentInfoTest.java index 726a5623cd560..c1020a98ba9f3 100644 --- a/streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/AssignmentInfoTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/processor/internals/assignment/AssignmentInfoTest.java @@ -68,7 +68,6 @@ public void shouldDecodePreviousVersion() throws IOException { assertEquals(1, decoded.version()); } - /** * This is a clone of what the V1 encoding did. The encode method has changed for V2 * so it is impossible to test compatibility without having this diff --git a/streams/src/test/java/org/apache/kafka/streams/tests/SmokeTestClient.java b/streams/src/test/java/org/apache/kafka/streams/tests/SmokeTestClient.java index e2493b2c5a05a..2936c63d8f8ab 100644 --- a/streams/src/test/java/org/apache/kafka/streams/tests/SmokeTestClient.java +++ b/streams/src/test/java/org/apache/kafka/streams/tests/SmokeTestClient.java @@ -19,6 +19,7 @@ import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.common.utils.Bytes; import org.apache.kafka.streams.Consumed; import org.apache.kafka.streams.KafkaStreams; import org.apache.kafka.streams.StreamsBuilder; @@ -30,10 +31,13 @@ import org.apache.kafka.streams.kstream.KTable; import org.apache.kafka.streams.kstream.Materialized; import org.apache.kafka.streams.kstream.Predicate; +import org.apache.kafka.streams.kstream.Produced; import org.apache.kafka.streams.kstream.Serialized; import org.apache.kafka.streams.kstream.TimeWindows; import org.apache.kafka.streams.kstream.ValueJoiner; +import org.apache.kafka.streams.state.KeyValueStore; import org.apache.kafka.streams.state.Stores; +import org.apache.kafka.streams.state.WindowStore; import java.util.Properties; import java.util.concurrent.TimeUnit; @@ -56,7 +60,7 @@ public void start() { streams = createKafkaStreams(streamsProperties, kafka); streams.setUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() { @Override - public void uncaughtException(Thread t, Throwable e) { + public void uncaughtException(final Thread t, final Throwable e) { System.out.println("SMOKE-TEST-CLIENT-EXCEPTION"); uncaughtException = true; e.printStackTrace(); @@ -93,38 +97,45 @@ public void close() { } } - private static KafkaStreams createKafkaStreams(final Properties props, final String kafka) { - props.put(StreamsConfig.APPLICATION_ID_CONFIG, "SmokeTest"); - props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, kafka); - props.put(StreamsConfig.NUM_STREAM_THREADS_CONFIG, 3); - props.put(StreamsConfig.NUM_STANDBY_REPLICAS_CONFIG, 2); - props.put(StreamsConfig.BUFFERED_RECORDS_PER_PARTITION_CONFIG, 100); - props.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 1000); - props.put(StreamsConfig.REPLICATION_FACTOR_CONFIG, 3); - props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); - props.put(ProducerConfig.RETRIES_CONFIG, Integer.MAX_VALUE); - props.put(ProducerConfig.ACKS_CONFIG, "all"); + private static Properties getStreamsConfig(final Properties props, final String kafka) { + final Properties config = new Properties(props); + config.put(StreamsConfig.APPLICATION_ID_CONFIG, "SmokeTest"); + config.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, kafka); + config.put(StreamsConfig.NUM_STREAM_THREADS_CONFIG, 3); + config.put(StreamsConfig.NUM_STANDBY_REPLICAS_CONFIG, 2); + config.put(StreamsConfig.BUFFERED_RECORDS_PER_PARTITION_CONFIG, 100); + config.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 1000); + config.put(StreamsConfig.REPLICATION_FACTOR_CONFIG, 3); + config.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + config.put(ProducerConfig.RETRIES_CONFIG, Integer.MAX_VALUE); + config.put(ProducerConfig.ACKS_CONFIG, "all"); //TODO remove this config or set to smaller value when KIP-91 is merged - props.put(StreamsConfig.producerPrefix(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG), 80000); + config.put(StreamsConfig.producerPrefix(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG), 80000); + + config.putAll(props); + return config; + } - StreamsBuilder builder = new StreamsBuilder(); - Consumed stringIntConsumed = Consumed.with(stringSerde, intSerde); - KStream source = builder.stream("data", stringIntConsumed); - source.to(stringSerde, intSerde, "echo"); - KStream data = source.filter(new Predicate() { + private static KafkaStreams createKafkaStreams(final Properties props, final String kafka) { + final StreamsBuilder builder = new StreamsBuilder(); + final Consumed stringIntConsumed = Consumed.with(stringSerde, intSerde); + final KStream source = builder.stream("data", stringIntConsumed); + source.to("echo", Produced.with(stringSerde, intSerde)); + final KStream data = source.filter(new Predicate() { @Override - public boolean test(String key, Integer value) { + public boolean test(final String key, final Integer value) { return value == null || value != END; } }); data.process(SmokeTestUtil.printProcessorSupplier("data")); // min - KGroupedStream - groupedData = + final KGroupedStream groupedData = data.groupByKey(Serialized.with(stringSerde, intSerde)); - groupedData.aggregate( + groupedData + .windowedBy(TimeWindows.of(TimeUnit.DAYS.toMillis(1))) + .aggregate( new Initializer() { public Integer apply() { return Integer.MAX_VALUE; @@ -132,21 +143,24 @@ public Integer apply() { }, new Aggregator() { @Override - public Integer apply(String aggKey, Integer value, Integer aggregate) { + public Integer apply(final String aggKey, final Integer value, final Integer aggregate) { return (value < aggregate) ? value : aggregate; } }, - TimeWindows.of(TimeUnit.DAYS.toMillis(1)), - intSerde, "uwin-min" - ).toStream().map( - new Unwindow() - ).to(stringSerde, intSerde, "min"); + Materialized.>as("uwin-min").withValueSerde(intSerde)) + .toStream(new Unwindow()) + .to("min", Produced.with(stringSerde, intSerde)); - KTable minTable = builder.table("min", stringIntConsumed); + final KTable minTable = builder.table( + "min", + Consumed.with(stringSerde, intSerde), + Materialized.>as("minStoreName")); minTable.toStream().process(SmokeTestUtil.printProcessorSupplier("min")); // max - groupedData.aggregate( + groupedData + .windowedBy(TimeWindows.of(TimeUnit.DAYS.toMillis(2))) + .aggregate( new Initializer() { public Integer apply() { return Integer.MIN_VALUE; @@ -154,21 +168,24 @@ public Integer apply() { }, new Aggregator() { @Override - public Integer apply(String aggKey, Integer value, Integer aggregate) { + public Integer apply(final String aggKey, final Integer value, final Integer aggregate) { return (value > aggregate) ? value : aggregate; } }, - TimeWindows.of(TimeUnit.DAYS.toMillis(2)), - intSerde, "uwin-max" - ).toStream().map( - new Unwindow() - ).to(stringSerde, intSerde, "max"); + Materialized.>as("uwin-max").withValueSerde(intSerde)) + .toStream(new Unwindow()) + .to("max", Produced.with(stringSerde, intSerde)); - KTable maxTable = builder.table("max", stringIntConsumed); + final KTable maxTable = builder.table( + "max", + Consumed.with(stringSerde, intSerde), + Materialized.>as("maxStoreName")); maxTable.toStream().process(SmokeTestUtil.printProcessorSupplier("max")); // sum - groupedData.aggregate( + groupedData + .windowedBy(TimeWindows.of(TimeUnit.DAYS.toMillis(2))) + .aggregate( new Initializer() { public Long apply() { return 0L; @@ -176,70 +193,74 @@ public Long apply() { }, new Aggregator() { @Override - public Long apply(String aggKey, Integer value, Long aggregate) { + public Long apply(final String aggKey, final Integer value, final Long aggregate) { return (long) value + aggregate; } }, - TimeWindows.of(TimeUnit.DAYS.toMillis(2)), - longSerde, "win-sum" - ).toStream().map( - new Unwindow() - ).to(stringSerde, longSerde, "sum"); - - Consumed stringLongConsumed = Consumed.with(stringSerde, longSerde); - KTable sumTable = builder.table("sum", stringLongConsumed); + Materialized.>as("win-sum").withValueSerde(longSerde)) + .toStream(new Unwindow()) + .to("sum", Produced.with(stringSerde, longSerde)); + + final Consumed stringLongConsumed = Consumed.with(stringSerde, longSerde); + final KTable sumTable = builder.table("sum", stringLongConsumed); sumTable.toStream().process(SmokeTestUtil.printProcessorSupplier("sum")); + // cnt - groupedData.count(TimeWindows.of(TimeUnit.DAYS.toMillis(2)), "uwin-cnt") - .toStream().map( - new Unwindow() - ).to(stringSerde, longSerde, "cnt"); + groupedData + .windowedBy(TimeWindows.of(TimeUnit.DAYS.toMillis(2))) + .count(Materialized.>as("uwin-cnt")) + .toStream(new Unwindow()) + .to("cnt", Produced.with(stringSerde, longSerde)); - KTable cntTable = builder.table("cnt", stringLongConsumed); + final KTable cntTable = builder.table( + "cnt", + Consumed.with(stringSerde, longSerde), + Materialized.>as("cntStoreName")); cntTable.toStream().process(SmokeTestUtil.printProcessorSupplier("cnt")); // dif - maxTable.join(minTable, + maxTable + .join( + minTable, new ValueJoiner() { - public Integer apply(Integer value1, Integer value2) { + public Integer apply(final Integer value1, final Integer value2) { return value1 - value2; } - } - ).to(stringSerde, intSerde, "dif"); + }) + .toStream() + .to("dif", Produced.with(stringSerde, intSerde)); // avg - sumTable.join( + sumTable + .join( cntTable, new ValueJoiner() { - public Double apply(Long value1, Long value2) { + public Double apply(final Long value1, final Long value2) { return (double) value1 / (double) value2; } - } - ).to(stringSerde, doubleSerde, "avg"); + }) + .toStream() + .to("avg", Produced.with(stringSerde, doubleSerde)); // test repartition - Agg agg = new Agg(); - cntTable.groupBy(agg.selector(), - Serialized.with(stringSerde, longSerde) - ).aggregate(agg.init(), - agg.adder(), - agg.remover(), - Materialized.as(Stores.inMemoryKeyValueStore("cntByCnt")) - .withKeySerde(Serdes.String()) - .withValueSerde(Serdes.Long()) - ).to(stringSerde, longSerde, "tagg"); - - final KafkaStreams streamsClient = new KafkaStreams(builder.build(), props); + final Agg agg = new Agg(); + cntTable.groupBy(agg.selector(), Serialized.with(stringSerde, longSerde)) + .aggregate(agg.init(), agg.adder(), agg.remover(), + Materialized.as(Stores.inMemoryKeyValueStore("cntByCnt")) + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.Long())) + .toStream() + .to("tagg", Produced.with(stringSerde, longSerde)); + + final KafkaStreams streamsClient = new KafkaStreams(builder.build(), getStreamsConfig(props, kafka)); streamsClient.setUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() { @Override - public void uncaughtException(Thread t, Throwable e) { + public void uncaughtException(final Thread t, final Throwable e) { System.out.println("FATAL: An unexpected exception is encountered on thread " + t + ": " + e); - streamsClient.close(30, TimeUnit.SECONDS); } }); return streamsClient; } - } diff --git a/streams/src/test/java/org/apache/kafka/streams/tests/SmokeTestDriver.java b/streams/src/test/java/org/apache/kafka/streams/tests/SmokeTestDriver.java index a3f520a82dbf8..fc7a26ee7a204 100644 --- a/streams/src/test/java/org/apache/kafka/streams/tests/SmokeTestDriver.java +++ b/streams/src/test/java/org/apache/kafka/streams/tests/SmokeTestDriver.java @@ -136,53 +136,65 @@ public void run() { System.out.println("shutdown"); } - public static Map> generate(String kafka, final int numKeys, final int maxRecordsPerKey) { + public static Map> generate(final String kafka, + final int numKeys, + final int maxRecordsPerKey) { + return generate(kafka, numKeys, maxRecordsPerKey, true); + } + + public static Map> generate(final String kafka, + final int numKeys, + final int maxRecordsPerKey, + final boolean autoTerminate) { final Properties producerProps = new Properties(); producerProps.put(ProducerConfig.CLIENT_ID_CONFIG, "SmokeTest"); producerProps.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, kafka); producerProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class); producerProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class); - // the next 4 config values make sure that all records are produced with no loss and - // no duplicates + // the next 2 config values make sure that all records are produced with no loss and no duplicates producerProps.put(ProducerConfig.RETRIES_CONFIG, Integer.MAX_VALUE); producerProps.put(ProducerConfig.ACKS_CONFIG, "all"); producerProps.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, 80000); - KafkaProducer producer = new KafkaProducer<>(producerProps); + final KafkaProducer producer = new KafkaProducer<>(producerProps); int numRecordsProduced = 0; - Map> allData = new HashMap<>(); - ValueList[] data = new ValueList[numKeys]; + final Map> allData = new HashMap<>(); + final ValueList[] data = new ValueList[numKeys]; for (int i = 0; i < numKeys; i++) { data[i] = new ValueList(i, i + maxRecordsPerKey - 1); allData.put(data[i].key, new HashSet()); } - Random rand = new Random(); + final Random rand = new Random(); - int remaining = data.length; + int remaining = 1; // dummy value must be positive if is false + if (autoTerminate) { + remaining = data.length; + } List> needRetry = new ArrayList<>(); while (remaining > 0) { - int index = rand.nextInt(remaining); - String key = data[index].key; + final int index = autoTerminate ? rand.nextInt(remaining) : rand.nextInt(numKeys); + final String key = data[index].key; int value = data[index].next(); - if (value < 0) { + if (autoTerminate && value < 0) { remaining--; data[index] = data[remaining]; } else { - ProducerRecord record = - new ProducerRecord<>("data", stringSerde.serializer().serialize("", key), intSerde.serializer().serialize("", value)); + final ProducerRecord record = + new ProducerRecord<>("data", stringSerde.serializer().serialize("", key), intSerde.serializer().serialize("", value)); producer.send(record, new TestCallback(record, needRetry)); numRecordsProduced++; allData.get(key).add(value); - if (numRecordsProduced % 100 == 0) + if (numRecordsProduced % 100 == 0) { System.out.println(numRecordsProduced + " records produced"); + } Utils.sleep(2); } } diff --git a/streams/src/test/java/org/apache/kafka/streams/tests/SmokeTestUtil.java b/streams/src/test/java/org/apache/kafka/streams/tests/SmokeTestUtil.java index dc4c91b4097ed..87ca82918a951 100644 --- a/streams/src/test/java/org/apache/kafka/streams/tests/SmokeTestUtil.java +++ b/streams/src/test/java/org/apache/kafka/streams/tests/SmokeTestUtil.java @@ -44,20 +44,15 @@ private static ProcessorSupplier printProcessorSupplier(final St public Processor get() { return new AbstractProcessor() { private int numRecordsProcessed = 0; - private ProcessorContext context; @Override public void init(final ProcessorContext context) { System.out.println("initializing processor: topic=" + topic + " taskId=" + context.taskId()); numRecordsProcessed = 0; - this.context = context; } @Override public void process(final Object key, final Object value) { - if (printOffset) { - System.out.println(">>> " + context.offset()); - } numRecordsProcessed++; if (numRecordsProcessed % 100 == 0) { System.out.println(System.currentTimeMillis()); @@ -66,19 +61,19 @@ public void process(final Object key, final Object value) { } @Override - public void punctuate(final long timestamp) { } + public void punctuate(final long timestamp) {} @Override - public void close() { } + public void close() {} }; } }; } - public static final class Unwindow implements KeyValueMapper, V, KeyValue> { + public static final class Unwindow implements KeyValueMapper, V, K> { @Override - public KeyValue apply(final Windowed winKey, final V value) { - return new KeyValue<>(winKey.key(), value); + public K apply(final Windowed winKey, final V value) { + return winKey.key(); } } diff --git a/streams/src/test/java/org/apache/kafka/streams/tests/StreamsSmokeTest.java b/streams/src/test/java/org/apache/kafka/streams/tests/StreamsSmokeTest.java index 27aba29bba6d8..41c3f6c58f0f6 100644 --- a/streams/src/test/java/org/apache/kafka/streams/tests/StreamsSmokeTest.java +++ b/streams/src/test/java/org/apache/kafka/streams/tests/StreamsSmokeTest.java @@ -18,7 +18,6 @@ import org.apache.kafka.common.utils.Utils; -import java.io.IOException; import java.util.Map; import java.util.Properties; import java.util.Set; @@ -26,22 +25,24 @@ public class StreamsSmokeTest { /** - * args ::= kafka propFileName command + * args ::= kafka propFileName command disableAutoTerminate * command := "run" | "process" * * @param args */ - public static void main(final String[] args) throws InterruptedException, IOException { + public static void main(final String[] args) throws Exception { final String kafka = args[0]; final String propFileName = args.length > 1 ? args[1] : null; final String command = args.length > 2 ? args[2] : null; + final boolean disableAutoTerminate = args.length > 3; final Properties streamsProperties = Utils.loadProps(propFileName); - System.out.println("StreamsTest instance started"); + System.out.println("StreamsTest instance started (StreamsSmokeTest)"); System.out.println("command=" + command); System.out.println("kafka=" + kafka); System.out.println("props=" + streamsProperties); + System.out.println("disableAutoTerminate=" + disableAutoTerminate); switch (command) { case "standalone": @@ -51,8 +52,12 @@ public static void main(final String[] args) throws InterruptedException, IOExce // this starts the driver (data generation and result verification) final int numKeys = 10; final int maxRecordsPerKey = 500; - Map> allData = SmokeTestDriver.generate(kafka, numKeys, maxRecordsPerKey); - SmokeTestDriver.verify(kafka, allData, maxRecordsPerKey); + if (disableAutoTerminate) { + SmokeTestDriver.generate(kafka, numKeys, maxRecordsPerKey, false); + } else { + Map> allData = SmokeTestDriver.generate(kafka, numKeys, maxRecordsPerKey); + SmokeTestDriver.verify(kafka, allData, maxRecordsPerKey); + } break; case "process": // this starts a KafkaStreams client diff --git a/streams/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java b/streams/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java new file mode 100644 index 0000000000000..69eea0b37c0ef --- /dev/null +++ b/streams/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.streams.tests; + +import org.apache.kafka.common.utils.Utils; +import org.apache.kafka.streams.KafkaStreams; +import org.apache.kafka.streams.StreamsBuilder; +import org.apache.kafka.streams.StreamsConfig; +import org.apache.kafka.streams.kstream.KStream; + +import java.util.Properties; + +public class StreamsUpgradeTest { + + @SuppressWarnings("unchecked") + public static void main(final String[] args) throws Exception { + if (args.length < 2) { + System.err.println("StreamsUpgradeTest requires two argument (kafka-url, properties-file) but only " + args.length + " provided: " + + (args.length > 0 ? args[0] : "")); + } + final String kafka = args[0]; + final String propFileName = args.length > 1 ? args[1] : null; + + final Properties streamsProperties = Utils.loadProps(propFileName); + + System.out.println("StreamsTest instance started (StreamsUpgradeTest trunk)"); + System.out.println("kafka=" + kafka); + System.out.println("props=" + streamsProperties); + + final StreamsBuilder builder = new StreamsBuilder(); + final KStream dataStream = builder.stream("data"); + dataStream.process(SmokeTestUtil.printProcessorSupplier("data")); + dataStream.to("echo"); + + final Properties config = new Properties(); + config.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, "StreamsUpgradeTest"); + config.setProperty(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, kafka); + config.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 1000); + config.putAll(streamsProperties); + + final KafkaStreams streams = new KafkaStreams(builder.build(), config); + streams.start(); + + Runtime.getRuntime().addShutdownHook(new Thread() { + @Override + public void run() { + System.out.println("closing Kafka Streams instance"); + System.out.flush(); + streams.close(); + System.out.println("UPGRADE-TEST-CLIENT-CLOSED"); + System.out.flush(); + } + }); + } +} diff --git a/streams/upgrade-system-tests-0100/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java b/streams/upgrade-system-tests-0100/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java new file mode 100644 index 0000000000000..32f96a01d99d4 --- /dev/null +++ b/streams/upgrade-system-tests-0100/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.streams.tests; + +import org.apache.kafka.common.utils.Utils; +import org.apache.kafka.streams.KafkaStreams; +import org.apache.kafka.streams.StreamsConfig; +import org.apache.kafka.streams.kstream.KStream; +import org.apache.kafka.streams.kstream.KStreamBuilder; +import org.apache.kafka.streams.processor.AbstractProcessor; +import org.apache.kafka.streams.processor.Processor; +import org.apache.kafka.streams.processor.ProcessorContext; +import org.apache.kafka.streams.processor.ProcessorSupplier; + +import java.util.Properties; + +public class StreamsUpgradeTest { + + @SuppressWarnings("unchecked") + public static void main(final String[] args) throws Exception { + if (args.length < 3) { + System.err.println("StreamsUpgradeTest requires three argument (kafka-url, zookeeper-url, properties-file) but only " + args.length + " provided: " + + (args.length > 0 ? args[0] + " " : "") + + (args.length > 1 ? args[1] : "")); + } + final String kafka = args[0]; + final String zookeeper = args[1]; + final String propFileName = args.length > 2 ? args[2] : null; + + final Properties streamsProperties = Utils.loadProps(propFileName); + + System.out.println("StreamsTest instance started (StreamsUpgradeTest v0.10.0)"); + System.out.println("kafka=" + kafka); + System.out.println("zookeeper=" + zookeeper); + System.out.println("props=" + streamsProperties); + + final KStreamBuilder builder = new KStreamBuilder(); + final KStream dataStream = builder.stream("data"); + dataStream.process(printProcessorSupplier()); + dataStream.to("echo"); + + final Properties config = new Properties(); + config.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, "StreamsUpgradeTest"); + config.setProperty(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, kafka); + config.setProperty(StreamsConfig.ZOOKEEPER_CONNECT_CONFIG, zookeeper); + config.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 1000); + config.putAll(streamsProperties); + + final KafkaStreams streams = new KafkaStreams(builder, config); + streams.start(); + + Runtime.getRuntime().addShutdownHook(new Thread() { + @Override + public void run() { + System.out.println("closing Kafka Streams instance"); + System.out.flush(); + streams.close(); + System.out.println("UPGRADE-TEST-CLIENT-CLOSED"); + System.out.flush(); + } + }); + } + + private static ProcessorSupplier printProcessorSupplier() { + return new ProcessorSupplier() { + public Processor get() { + return new AbstractProcessor() { + private int numRecordsProcessed = 0; + + @Override + public void init(final ProcessorContext context) { + System.out.println("initializing processor: topic=data taskId=" + context.taskId()); + numRecordsProcessed = 0; + } + + @Override + public void process(final K key, final V value) { + numRecordsProcessed++; + if (numRecordsProcessed % 100 == 0) { + System.out.println("processed " + numRecordsProcessed + " records from topic=data"); + } + } + + @Override + public void punctuate(final long timestamp) {} + + @Override + public void close() {} + }; + } + }; + } +} diff --git a/streams/upgrade-system-tests-0101/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java b/streams/upgrade-system-tests-0101/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java new file mode 100644 index 0000000000000..dcb05ca5d72da --- /dev/null +++ b/streams/upgrade-system-tests-0101/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.streams.tests; + +import org.apache.kafka.common.utils.Utils; +import org.apache.kafka.streams.KafkaStreams; +import org.apache.kafka.streams.StreamsConfig; +import org.apache.kafka.streams.kstream.KStream; +import org.apache.kafka.streams.kstream.KStreamBuilder; +import org.apache.kafka.streams.processor.AbstractProcessor; +import org.apache.kafka.streams.processor.Processor; +import org.apache.kafka.streams.processor.ProcessorContext; +import org.apache.kafka.streams.processor.ProcessorSupplier; + +import java.util.Properties; + +public class StreamsUpgradeTest { + + /** + * This test cannot be executed, as long as Kafka 0.10.1.2 is not released + */ + @SuppressWarnings("unchecked") + public static void main(final String[] args) throws Exception { + if (args.length < 3) { + System.err.println("StreamsUpgradeTest requires three argument (kafka-url, zookeeper-url, properties-file) but only " + args.length + " provided: " + + (args.length > 0 ? args[0] + " " : "") + + (args.length > 1 ? args[1] : "")); + } + final String kafka = args[0]; + final String zookeeper = args[1]; + final String propFileName = args.length > 2 ? args[2] : null; + + final Properties streamsProperties = Utils.loadProps(propFileName); + + System.out.println("StreamsTest instance started (StreamsUpgradeTest v0.10.1)"); + System.out.println("kafka=" + kafka); + System.out.println("zookeeper=" + zookeeper); + System.out.println("props=" + streamsProperties); + + final KStreamBuilder builder = new KStreamBuilder(); + final KStream dataStream = builder.stream("data"); + dataStream.process(printProcessorSupplier()); + dataStream.to("echo"); + + final Properties config = new Properties(); + config.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, "StreamsUpgradeTest"); + config.setProperty(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, kafka); + config.setProperty(StreamsConfig.ZOOKEEPER_CONNECT_CONFIG, zookeeper); + config.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 1000); + config.putAll(streamsProperties); + + final KafkaStreams streams = new KafkaStreams(builder, config); + streams.start(); + + Runtime.getRuntime().addShutdownHook(new Thread() { + @Override + public void run() { + System.out.println("closing Kafka Streams instance"); + System.out.flush(); + streams.close(); + System.out.println("UPGRADE-TEST-CLIENT-CLOSED"); + System.out.flush(); + } + }); + } + + private static ProcessorSupplier printProcessorSupplier() { + return new ProcessorSupplier() { + public Processor get() { + return new AbstractProcessor() { + private int numRecordsProcessed = 0; + + @Override + public void init(final ProcessorContext context) { + System.out.println("initializing processor: topic=data taskId=" + context.taskId()); + numRecordsProcessed = 0; + } + + @Override + public void process(final K key, final V value) { + numRecordsProcessed++; + if (numRecordsProcessed % 100 == 0) { + System.out.println("processed " + numRecordsProcessed + " records from topic=data"); + } + } + + @Override + public void punctuate(final long timestamp) {} + + @Override + public void close() {} + }; + } + }; + } +} diff --git a/streams/upgrade-system-tests-0102/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java b/streams/upgrade-system-tests-0102/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java new file mode 100644 index 0000000000000..fb4a4091610f8 --- /dev/null +++ b/streams/upgrade-system-tests-0102/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.streams.tests; + +import org.apache.kafka.common.utils.Utils; +import org.apache.kafka.streams.KafkaStreams; +import org.apache.kafka.streams.StreamsConfig; +import org.apache.kafka.streams.kstream.KStream; +import org.apache.kafka.streams.kstream.KStreamBuilder; +import org.apache.kafka.streams.processor.AbstractProcessor; +import org.apache.kafka.streams.processor.Processor; +import org.apache.kafka.streams.processor.ProcessorContext; +import org.apache.kafka.streams.processor.ProcessorSupplier; + +import java.util.Properties; + +public class StreamsUpgradeTest { + + /** + * This test cannot be executed, as long as Kafka 0.10.2.2 is not released + */ + @SuppressWarnings("unchecked") + public static void main(final String[] args) throws Exception { + if (args.length < 2) { + System.err.println("StreamsUpgradeTest requires three argument (kafka-url, properties-file) but only " + args.length + " provided: " + + (args.length > 0 ? args[0] : "")); + } + final String kafka = args[0]; + final String propFileName = args.length > 1 ? args[1] : null; + + final Properties streamsProperties = Utils.loadProps(propFileName); + + System.out.println("StreamsTest instance started (StreamsUpgradeTest v0.10.2)"); + System.out.println("kafka=" + kafka); + System.out.println("props=" + streamsProperties); + + final KStreamBuilder builder = new KStreamBuilder(); + final KStream dataStream = builder.stream("data"); + dataStream.process(printProcessorSupplier()); + dataStream.to("echo"); + + final Properties config = new Properties(); + config.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, "StreamsUpgradeTest"); + config.setProperty(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, kafka); + config.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 1000); + config.putAll(streamsProperties); + + final KafkaStreams streams = new KafkaStreams(builder, config); + streams.start(); + + Runtime.getRuntime().addShutdownHook(new Thread() { + @Override + public void run() { + streams.close(); + System.out.println("UPGRADE-TEST-CLIENT-CLOSED"); + System.out.flush(); + } + }); + } + + private static ProcessorSupplier printProcessorSupplier() { + return new ProcessorSupplier() { + public Processor get() { + return new AbstractProcessor() { + private int numRecordsProcessed = 0; + + @Override + public void init(final ProcessorContext context) { + System.out.println("initializing processor: topic=data taskId=" + context.taskId()); + numRecordsProcessed = 0; + } + + @Override + public void process(final K key, final V value) { + numRecordsProcessed++; + if (numRecordsProcessed % 100 == 0) { + System.out.println("processed " + numRecordsProcessed + " records from topic=data"); + } + } + + @Override + public void punctuate(final long timestamp) {} + + @Override + public void close() {} + }; + } + }; + } +} diff --git a/streams/upgrade-system-tests-0110/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java b/streams/upgrade-system-tests-0110/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java new file mode 100644 index 0000000000000..b1aad5dea9d90 --- /dev/null +++ b/streams/upgrade-system-tests-0110/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.streams.tests; + +import org.apache.kafka.common.utils.Utils; +import org.apache.kafka.streams.KafkaStreams; +import org.apache.kafka.streams.StreamsConfig; +import org.apache.kafka.streams.kstream.KStream; +import org.apache.kafka.streams.kstream.KStreamBuilder; +import org.apache.kafka.streams.processor.AbstractProcessor; +import org.apache.kafka.streams.processor.Processor; +import org.apache.kafka.streams.processor.ProcessorContext; +import org.apache.kafka.streams.processor.ProcessorSupplier; + +import java.util.Properties; + +public class StreamsUpgradeTest { + + /** + * This test cannot be executed, as long as Kafka 0.11.0.3 is not released + */ + @SuppressWarnings("unchecked") + public static void main(final String[] args) throws Exception { + if (args.length < 2) { + System.err.println("StreamsUpgradeTest requires three argument (kafka-url, properties-file) but only " + args.length + " provided: " + + (args.length > 0 ? args[0] : "")); + } + final String kafka = args[0]; + final String propFileName = args.length > 1 ? args[1] : null; + + final Properties streamsProperties = Utils.loadProps(propFileName); + + System.out.println("StreamsTest instance started (StreamsUpgradeTest v0.11.0)"); + System.out.println("kafka=" + kafka); + System.out.println("props=" + streamsProperties); + + final KStreamBuilder builder = new KStreamBuilder(); + final KStream dataStream = builder.stream("data"); + dataStream.process(printProcessorSupplier()); + dataStream.to("echo"); + + final Properties config = new Properties(); + config.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, "StreamsUpgradeTest"); + config.setProperty(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, kafka); + config.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 1000); + config.putAll(streamsProperties); + + final KafkaStreams streams = new KafkaStreams(builder, config); + streams.start(); + + Runtime.getRuntime().addShutdownHook(new Thread() { + @Override + public void run() { + streams.close(); + System.out.println("UPGRADE-TEST-CLIENT-CLOSED"); + System.out.flush(); + } + }); + } + + private static ProcessorSupplier printProcessorSupplier() { + return new ProcessorSupplier() { + public Processor get() { + return new AbstractProcessor() { + private int numRecordsProcessed = 0; + + @Override + public void init(final ProcessorContext context) { + System.out.println("initializing processor: topic=data taskId=" + context.taskId()); + numRecordsProcessed = 0; + } + + @Override + public void process(final K key, final V value) { + numRecordsProcessed++; + if (numRecordsProcessed % 100 == 0) { + System.out.println("processed " + numRecordsProcessed + " records from topic=data"); + } + } + + @Override + public void punctuate(final long timestamp) {} + + @Override + public void close() {} + }; + } + }; + } +} diff --git a/streams/upgrade-system-tests-10/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java b/streams/upgrade-system-tests-10/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java new file mode 100644 index 0000000000000..dc72f2dcc9d0d --- /dev/null +++ b/streams/upgrade-system-tests-10/src/test/java/org/apache/kafka/streams/tests/StreamsUpgradeTest.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.streams.tests; + +import org.apache.kafka.common.utils.Utils; +import org.apache.kafka.streams.KafkaStreams; +import org.apache.kafka.streams.StreamsBuilder; +import org.apache.kafka.streams.StreamsConfig; +import org.apache.kafka.streams.kstream.KStream; +import org.apache.kafka.streams.processor.AbstractProcessor; +import org.apache.kafka.streams.processor.Processor; +import org.apache.kafka.streams.processor.ProcessorContext; +import org.apache.kafka.streams.processor.ProcessorSupplier; + +import java.util.Properties; + +public class StreamsUpgradeTest { + + /** + * This test cannot be executed, as long as Kafka 1.0.2 is not released + */ + @SuppressWarnings("unchecked") + public static void main(final String[] args) throws Exception { + if (args.length < 2) { + System.err.println("StreamsUpgradeTest requires three argument (kafka-url, properties-file) but only " + args.length + " provided: " + + (args.length > 0 ? args[0] : "")); + } + final String kafka = args[0]; + final String propFileName = args.length > 1 ? args[1] : null; + + final Properties streamsProperties = Utils.loadProps(propFileName); + + System.out.println("StreamsTest instance started (StreamsUpgradeTest v1.0)"); + System.out.println("kafka=" + kafka); + System.out.println("props=" + streamsProperties); + + final StreamsBuilder builder = new StreamsBuilder(); + final KStream dataStream = builder.stream("data"); + dataStream.process(printProcessorSupplier()); + dataStream.to("echo"); + + final Properties config = new Properties(); + config.setProperty(StreamsConfig.APPLICATION_ID_CONFIG, "StreamsUpgradeTest"); + config.setProperty(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, kafka); + config.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 1000); + config.putAll(streamsProperties); + + final KafkaStreams streams = new KafkaStreams(builder.build(), config); + streams.start(); + + Runtime.getRuntime().addShutdownHook(new Thread() { + @Override + public void run() { + streams.close(); + System.out.println("UPGRADE-TEST-CLIENT-CLOSED"); + System.out.flush(); + } + }); + } + + private static ProcessorSupplier printProcessorSupplier() { + return new ProcessorSupplier() { + public Processor get() { + return new AbstractProcessor() { + private int numRecordsProcessed = 0; + + @Override + public void init(final ProcessorContext context) { + System.out.println("initializing processor: topic=data taskId=" + context.taskId()); + numRecordsProcessed = 0; + } + + @Override + public void process(final K key, final V value) { + numRecordsProcessed++; + if (numRecordsProcessed % 100 == 0) { + System.out.println("processed " + numRecordsProcessed + " records from topic=data"); + } + } + + @Override + public void punctuate(final long timestamp) {} + + @Override + public void close() {} + }; + } + }; + } +} diff --git a/tests/docker/Dockerfile b/tests/docker/Dockerfile index 57ca2423e3f25..25da8db59eaf2 100644 --- a/tests/docker/Dockerfile +++ b/tests/docker/Dockerfile @@ -39,24 +39,36 @@ COPY ./ssh-config /root/.ssh/config RUN ssh-keygen -q -t rsa -N '' -f /root/.ssh/id_rsa && cp -f /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys # Install binary test dependencies. +# we use the same versions as in vagrant/base.sh ARG KAFKA_MIRROR="https://s3-us-west-2.amazonaws.com/kafka-packages" -RUN mkdir -p "/opt/kafka-0.8.2.2" && chmod a+rw /opt/kafka-0.8.2.2 && curl -s "$KAFKA_MIRROR/kafka_2.10-0.8.2.2.tgz" | tar xz --strip-components=1 -C "/opt/kafka-0.8.2.2" +RUN mkdir -p "/opt/kafka-0.8.2.2" && chmod a+rw /opt/kafka-0.8.2.2 && curl -s "$KAFKA_MIRROR/kafka_2.11-0.8.2.2.tgz" | tar xz --strip-components=1 -C "/opt/kafka-0.8.2.2" RUN mkdir -p "/opt/kafka-0.9.0.1" && chmod a+rw /opt/kafka-0.9.0.1 && curl -s "$KAFKA_MIRROR/kafka_2.11-0.9.0.1.tgz" | tar xz --strip-components=1 -C "/opt/kafka-0.9.0.1" +RUN mkdir -p "/opt/kafka-0.10.0.0" && chmod a+rw /opt/kafka-0.10.0.0 && curl -s "$KAFKA_MIRROR/kafka_2.11-0.10.0.0.tgz" | tar xz --strip-components=1 -C "/opt/kafka-0.10.0.0" RUN mkdir -p "/opt/kafka-0.10.0.1" && chmod a+rw /opt/kafka-0.10.0.1 && curl -s "$KAFKA_MIRROR/kafka_2.11-0.10.0.1.tgz" | tar xz --strip-components=1 -C "/opt/kafka-0.10.0.1" +RUN mkdir -p "/opt/kafka-0.10.1.0" && chmod a+rw /opt/kafka-0.10.1.0 && curl -s "$KAFKA_MIRROR/kafka_2.11-0.10.1.0.tgz" | tar xz --strip-components=1 -C "/opt/kafka-0.10.1.0" RUN mkdir -p "/opt/kafka-0.10.1.1" && chmod a+rw /opt/kafka-0.10.1.1 && curl -s "$KAFKA_MIRROR/kafka_2.11-0.10.1.1.tgz" | tar xz --strip-components=1 -C "/opt/kafka-0.10.1.1" +RUN mkdir -p "/opt/kafka-0.10.2.0" && chmod a+rw /opt/kafka-0.10.2.0 && curl -s "$KAFKA_MIRROR/kafka_2.11-0.10.2.0.tgz" | tar xz --strip-components=1 -C "/opt/kafka-0.10.2.0" RUN mkdir -p "/opt/kafka-0.10.2.1" && chmod a+rw /opt/kafka-0.10.2.1 && curl -s "$KAFKA_MIRROR/kafka_2.11-0.10.2.1.tgz" | tar xz --strip-components=1 -C "/opt/kafka-0.10.2.1" RUN mkdir -p "/opt/kafka-0.11.0.0" && chmod a+rw /opt/kafka-0.11.0.0 && curl -s "$KAFKA_MIRROR/kafka_2.11-0.11.0.0.tgz" | tar xz --strip-components=1 -C "/opt/kafka-0.11.0.0" +RUN mkdir -p "/opt/kafka-0.11.0.1" && chmod a+rw /opt/kafka-0.11.0.1 && curl -s "$KAFKA_MIRROR/kafka_2.11-0.11.0.1.tgz" | tar xz --strip-components=1 -C "/opt/kafka-0.11.0.1" RUN mkdir -p "/opt/kafka-0.11.0.2" && chmod a+rw /opt/kafka-0.11.0.2 && curl -s "$KAFKA_MIRROR/kafka_2.11-0.11.0.2.tgz" | tar xz --strip-components=1 -C "/opt/kafka-0.11.0.2" RUN mkdir -p "/opt/kafka-1.0.0" && chmod a+rw /opt/kafka-1.0.0 && curl -s "$KAFKA_MIRROR/kafka_2.11-1.0.0.tgz" | tar xz --strip-components=1 -C "/opt/kafka-1.0.0" RUN mkdir -p "/opt/kafka-1.0.1" && chmod a+rw /opt/kafka-1.0.1 && curl -s "$KAFKA_MIRROR/kafka_2.11-1.0.1.tgz" | tar xz --strip-components=1 -C "/opt/kafka-1.0.1" +RUN mkdir -p "/opt/kafka-1.1.0" && chmod a+rw /opt/kafka-1.1.0 && curl -s "$KAFKA_MIRROR/kafka_2.11-1.1.0.tgz" | tar xz --strip-components=1 -C "/opt/kafka-1.1.0" # Streams test dependencies -RUN curl -s "$KAFKA_MIRROR/kafka-streams-0.10.1.1-test.jar" -o /opt/kafka-0.10.1.1/libs/kafka-streams-0.10.1.1-test.jar && \ - curl -s "$KAFKA_MIRROR/kafka-streams-0.10.2.1-test.jar" -o /opt/kafka-0.10.2.1/libs/kafka-streams-0.10.2.1-test.jar && \ - curl -s "$KAFKA_MIRROR/kafka-streams-0.11.0.0-test.jar" -o /opt/kafka-0.11.0.0/libs/kafka-streams-0.11.0.0-test.jar && \ - curl -s "$KAFKA_MIRROR/kafka-streams-0.11.0.2-test.jar" -o /opt/kafka-0.11.0.2/libs/kafka-streams-0.11.0.2-test.jar && \ - curl -s "$KAFKA_MIRROR/kafka-streams-1.0.0-test.jar" -o /opt/kafka-1.0.0/libs/kafka-streams-1.0.0-test.jar && \ - curl -s "$KAFKA_MIRROR/kafka-streams-1.0.1-test.jar" -o /opt/kafka-1.0.1/libs/kafka-streams-1.0.1-test.jar +RUN curl -s "$KAFKA_MIRROR/kafka-streams-0.10.0.0-test.jar" -o /opt/kafka-0.10.0.0/libs/kafka-streams-0.10.0.0-test.jar +RUN curl -s "$KAFKA_MIRROR/kafka-streams-0.10.0.1-test.jar" -o /opt/kafka-0.10.0.1/libs/kafka-streams-0.10.0.1-test.jar +RUN curl -s "$KAFKA_MIRROR/kafka-streams-0.10.1.0-test.jar" -o /opt/kafka-0.10.1.0/libs/kafka-streams-0.10.1.0-test.jar +RUN curl -s "$KAFKA_MIRROR/kafka-streams-0.10.1.1-test.jar" -o /opt/kafka-0.10.1.1/libs/kafka-streams-0.10.1.1-test.jar +RUN curl -s "$KAFKA_MIRROR/kafka-streams-0.10.2.0-test.jar" -o /opt/kafka-0.10.2.0/libs/kafka-streams-0.10.2.0-test.jar +RUN curl -s "$KAFKA_MIRROR/kafka-streams-0.10.2.1-test.jar" -o /opt/kafka-0.10.2.1/libs/kafka-streams-0.10.2.1-test.jar +RUN curl -s "$KAFKA_MIRROR/kafka-streams-0.11.0.0-test.jar" -o /opt/kafka-0.11.0.0/libs/kafka-streams-0.11.0.0-test.jar +RUN curl -s "$KAFKA_MIRROR/kafka-streams-0.11.0.1-test.jar" -o /opt/kafka-0.11.0.1/libs/kafka-streams-0.11.0.1-test.jar +RUN curl -s "$KAFKA_MIRROR/kafka-streams-0.11.0.2-test.jar" -o /opt/kafka-0.11.0.2/libs/kafka-streams-0.11.0.2-test.jar +RUN curl -s "$KAFKA_MIRROR/kafka-streams-1.0.0-test.jar" -o /opt/kafka-1.0.0/libs/kafka-streams-1.0.0-test.jar +RUN curl -s "$KAFKA_MIRROR/kafka-streams-1.0.1-test.jar" -o /opt/kafka-1.0.1/libs/kafka-streams-1.0.1-test.jar +RUN curl -s "$KAFKA_MIRROR/kafka-streams-1.1.0-test.jar" -o /opt/kafka-1.1.0/libs/kafka-streams-1.1.0-test.jar # The version of Kibosh to use for testing. # If you update this, also update vagrant/base.sy diff --git a/tests/kafkatest/services/streams.py b/tests/kafkatest/services/streams.py index d9b475e191b22..a5be816c737e9 100644 --- a/tests/kafkatest/services/streams.py +++ b/tests/kafkatest/services/streams.py @@ -21,6 +21,7 @@ from kafkatest.directory_layout.kafka_path import KafkaPathResolverMixin from kafkatest.services.monitor.jmx import JmxMixin from kafkatest.services.kafka import KafkaConfig +from kafkatest.version import LATEST_0_10_0, LATEST_0_10_1 STATE_DIR = "state.dir" @@ -39,6 +40,8 @@ class StreamsTestBaseService(KafkaPathResolverMixin, JmxMixin, Service): LOG4J_CONFIG_FILE = os.path.join(PERSISTENT_ROOT, "tools-log4j.properties") PID_FILE = os.path.join(PERSISTENT_ROOT, "streams.pid") + CLEAN_NODE_ENABLED = True + logs = { "streams_log": { "path": LOG_FILE, @@ -49,6 +52,114 @@ class StreamsTestBaseService(KafkaPathResolverMixin, JmxMixin, Service): "streams_stderr": { "path": STDERR_FILE, "collect_default": True}, + "streams_log.0-1": { + "path": LOG_FILE + ".0-1", + "collect_default": True}, + "streams_stdout.0-1": { + "path": STDOUT_FILE + ".0-1", + "collect_default": True}, + "streams_stderr.0-1": { + "path": STDERR_FILE + ".0-1", + "collect_default": True}, + "streams_log.0-2": { + "path": LOG_FILE + ".0-2", + "collect_default": True}, + "streams_stdout.0-2": { + "path": STDOUT_FILE + ".0-2", + "collect_default": True}, + "streams_stderr.0-2": { + "path": STDERR_FILE + ".0-2", + "collect_default": True}, + "streams_log.0-3": { + "path": LOG_FILE + ".0-3", + "collect_default": True}, + "streams_stdout.0-3": { + "path": STDOUT_FILE + ".0-3", + "collect_default": True}, + "streams_stderr.0-3": { + "path": STDERR_FILE + ".0-3", + "collect_default": True}, + "streams_log.0-4": { + "path": LOG_FILE + ".0-4", + "collect_default": True}, + "streams_stdout.0-4": { + "path": STDOUT_FILE + ".0-4", + "collect_default": True}, + "streams_stderr.0-4": { + "path": STDERR_FILE + ".0-4", + "collect_default": True}, + "streams_log.0-5": { + "path": LOG_FILE + ".0-5", + "collect_default": True}, + "streams_stdout.0-5": { + "path": STDOUT_FILE + ".0-5", + "collect_default": True}, + "streams_stderr.0-5": { + "path": STDERR_FILE + ".0-5", + "collect_default": True}, + "streams_log.0-6": { + "path": LOG_FILE + ".0-6", + "collect_default": True}, + "streams_stdout.0-6": { + "path": STDOUT_FILE + ".0-6", + "collect_default": True}, + "streams_stderr.0-6": { + "path": STDERR_FILE + ".0-6", + "collect_default": True}, + "streams_log.1-1": { + "path": LOG_FILE + ".1-1", + "collect_default": True}, + "streams_stdout.1-1": { + "path": STDOUT_FILE + ".1-1", + "collect_default": True}, + "streams_stderr.1-1": { + "path": STDERR_FILE + ".1-1", + "collect_default": True}, + "streams_log.1-2": { + "path": LOG_FILE + ".1-2", + "collect_default": True}, + "streams_stdout.1-2": { + "path": STDOUT_FILE + ".1-2", + "collect_default": True}, + "streams_stderr.1-2": { + "path": STDERR_FILE + ".1-2", + "collect_default": True}, + "streams_log.1-3": { + "path": LOG_FILE + ".1-3", + "collect_default": True}, + "streams_stdout.1-3": { + "path": STDOUT_FILE + ".1-3", + "collect_default": True}, + "streams_stderr.1-3": { + "path": STDERR_FILE + ".1-3", + "collect_default": True}, + "streams_log.1-4": { + "path": LOG_FILE + ".1-4", + "collect_default": True}, + "streams_stdout.1-4": { + "path": STDOUT_FILE + ".1-4", + "collect_default": True}, + "streams_stderr.1-4": { + "path": STDERR_FILE + ".1-4", + "collect_default": True}, + "streams_log.1-5": { + "path": LOG_FILE + ".1-5", + "collect_default": True}, + "streams_stdout.1-5": { + "path": STDOUT_FILE + ".1-5", + "collect_default": True}, + "streams_stderr.1-5": { + "path": STDERR_FILE + ".1-5", + "collect_default": True}, + "streams_log.1-6": { + "path": LOG_FILE + ".1-6", + "collect_default": True}, + "streams_stdout.1-6": { + "path": STDOUT_FILE + ".1-6", + "collect_default": True}, + "streams_stderr.1-6": { + "path": STDERR_FILE + ".1-6", + "collect_default": True}, "jmx_log": { "path": JMX_LOG_FILE, "collect_default": True}, @@ -120,7 +231,8 @@ def wait_node(self, node, timeout_sec=None): def clean_node(self, node): node.account.kill_process("streams", clean_shutdown=False, allow_fail=True) - node.account.ssh("rm -rf " + self.PERSISTENT_ROOT, allow_fail=False) + if self.CLEAN_NODE_ENABLED: + node.account.ssh("rm -rf " + self.PERSISTENT_ROOT, allow_fail=False) def start_cmd(self, node): args = self.args.copy() @@ -141,13 +253,13 @@ def start_cmd(self, node): return cmd - def prop_file(self, node): + def prop_file(self): cfg = KafkaConfig(**{STATE_DIR: self.PERSISTENT_ROOT}) return cfg.render() def start_node(self, node): node.account.mkdirs(self.PERSISTENT_ROOT) - prop_file = self.prop_file(node) + prop_file = self.prop_file() node.account.create_file(self.CONFIG_FILE, prop_file) node.account.create_file(self.LOG4J_CONFIG_FILE, self.render('tools_log4j.properties', log_file=self.LOG_FILE)) @@ -189,7 +301,28 @@ def clean_node(self, node): class StreamsSmokeTestDriverService(StreamsSmokeTestBaseService): def __init__(self, test_context, kafka): super(StreamsSmokeTestDriverService, self).__init__(test_context, kafka, "run") + self.DISABLE_AUTO_TERMINATE = "" + def disable_auto_terminate(self): + self.DISABLE_AUTO_TERMINATE = "disableAutoTerminate" + + def start_cmd(self, node): + args = self.args.copy() + args['kafka'] = self.kafka.bootstrap_servers() + args['config_file'] = self.CONFIG_FILE + args['stdout'] = self.STDOUT_FILE + args['stderr'] = self.STDERR_FILE + args['pidfile'] = self.PID_FILE + args['log4j'] = self.LOG4J_CONFIG_FILE + args['disable_auto_terminate'] = self.DISABLE_AUTO_TERMINATE + args['kafka_run_class'] = self.path.script("kafka-run-class.sh", node) + + cmd = "( export KAFKA_LOG4J_OPTS=\"-Dlog4j.configuration=file:%(log4j)s\"; " \ + "INCLUDE_TEST_JARS=true %(kafka_run_class)s %(streams_class_name)s " \ + " %(kafka)s %(config_file)s %(user_test_args)s %(disable_auto_terminate)s" \ + " & echo $! >&3 ) 1>> %(stdout)s 2>> %(stderr)s 3> %(pidfile)s" % args + + return cmd class StreamsSmokeTestJobRunnerService(StreamsSmokeTestBaseService): def __init__(self, test_context, kafka): @@ -273,3 +406,48 @@ def __init__(self, test_context, kafka, configs): kafka, "org.apache.kafka.streams.tests.StreamsRepeatingIntegerKeyProducer", configs) +class StreamsUpgradeTestJobRunnerService(StreamsTestBaseService): + def __init__(self, test_context, kafka): + super(StreamsUpgradeTestJobRunnerService, self).__init__(test_context, + kafka, + "org.apache.kafka.streams.tests.StreamsUpgradeTest", + "") + self.UPGRADE_FROM = None + + def set_version(self, kafka_streams_version): + self.KAFKA_STREAMS_VERSION = kafka_streams_version + + def set_upgrade_from(self, upgrade_from): + self.UPGRADE_FROM = upgrade_from + + def prop_file(self): + properties = {STATE_DIR: self.PERSISTENT_ROOT} + if self.UPGRADE_FROM is not None: + properties['upgrade.from'] = self.UPGRADE_FROM + + cfg = KafkaConfig(**properties) + return cfg.render() + + def start_cmd(self, node): + args = self.args.copy() + args['kafka'] = self.kafka.bootstrap_servers() + if self.KAFKA_STREAMS_VERSION == str(LATEST_0_10_0) or self.KAFKA_STREAMS_VERSION == str(LATEST_0_10_1): + args['zk'] = self.kafka.zk.connect_setting() + else: + args['zk'] = "" + args['config_file'] = self.CONFIG_FILE + args['stdout'] = self.STDOUT_FILE + args['stderr'] = self.STDERR_FILE + args['pidfile'] = self.PID_FILE + args['log4j'] = self.LOG4J_CONFIG_FILE + args['version'] = self.KAFKA_STREAMS_VERSION + args['kafka_run_class'] = self.path.script("kafka-run-class.sh", node) + + cmd = "( export KAFKA_LOG4J_OPTS=\"-Dlog4j.configuration=file:%(log4j)s\"; " \ + "INCLUDE_TEST_JARS=true UPGRADE_KAFKA_STREAMS_TEST_VERSION=%(version)s " \ + " %(kafka_run_class)s %(streams_class_name)s %(kafka)s %(zk)s %(config_file)s " \ + " & echo $! >&3 ) 1>> %(stdout)s 2>> %(stderr)s 3> %(pidfile)s" % args + + self.logger.info("Executing: " + cmd) + + return cmd diff --git a/tests/kafkatest/tests/streams/streams_upgrade_test.py b/tests/kafkatest/tests/streams/streams_upgrade_test.py index 3b38ff6f0f60f..fa79d571f366d 100644 --- a/tests/kafkatest/tests/streams/streams_upgrade_test.py +++ b/tests/kafkatest/tests/streams/streams_upgrade_test.py @@ -13,25 +13,50 @@ # See the License for the specific language governing permissions and # limitations under the License. -import time -from ducktape.mark import ignore -from ducktape.mark import matrix +from ducktape.mark import ignore, matrix, parametrize from ducktape.mark.resource import cluster from ducktape.tests.test import Test from kafkatest.services.kafka import KafkaService -from kafkatest.services.streams import StreamsSmokeTestDriverService, StreamsSmokeTestJobRunnerService +from kafkatest.services.streams import StreamsSmokeTestDriverService, StreamsSmokeTestJobRunnerService, StreamsUpgradeTestJobRunnerService from kafkatest.services.zookeeper import ZookeeperService -from kafkatest.version import LATEST_0_10_2, LATEST_0_11, LATEST_1_0, DEV_BRANCH, KafkaVersion +from kafkatest.version import LATEST_0_10_0, LATEST_0_10_1, LATEST_0_10_2, LATEST_0_11_0, LATEST_1_0, LATEST_1_1, DEV_BRANCH, DEV_VERSION, KafkaVersion +import random +import time -upgrade_versions = [str(LATEST_0_10_2), str(LATEST_0_11), str(LATEST_1_0), str(DEV_BRANCH)] +broker_upgrade_versions = [str(LATEST_0_10_1), str(LATEST_0_10_2), str(LATEST_0_11_0), str(LATEST_1_0), str(LATEST_1_1), str(DEV_BRANCH)] +simple_upgrade_versions_metadata_version_2 = [str(LATEST_0_10_1), str(LATEST_0_10_2), str(LATEST_0_11_0), str(LATEST_1_0), str(DEV_VERSION)] class StreamsUpgradeTest(Test): """ - Tests rolling upgrades and downgrades of the Kafka Streams library. + Test upgrading Kafka Streams (all version combination) + If metadata was changes, upgrade is more difficult + Metadata version was bumped in 0.10.1.0 """ def __init__(self, test_context): super(StreamsUpgradeTest, self).__init__(test_context) + self.topics = { + 'echo' : { 'partitions': 5 }, + 'data' : { 'partitions': 5 }, + } + + def perform_broker_upgrade(self, to_version): + self.logger.info("First pass bounce - rolling broker upgrade") + for node in self.kafka.nodes: + self.kafka.stop_node(node) + node.version = KafkaVersion(to_version) + self.kafka.start_node(node) + + @cluster(num_nodes=6) + @matrix(from_version=broker_upgrade_versions, to_version=broker_upgrade_versions) + def test_upgrade_downgrade_brokers(self, from_version, to_version): + """ + Start a smoke test client then perform rolling upgrades on the broker. + """ + + if from_version == to_version: + return + self.replication = 3 self.partitions = 1 self.isr = 2 @@ -58,112 +83,272 @@ def __init__(self, test_context): 'configs': {"min.insync.replicas": self.isr} } } + # Setup phase + self.zk = ZookeeperService(self.test_context, num_nodes=1) + self.zk.start() - def perform_streams_upgrade(self, to_version): - self.logger.info("First pass bounce - rolling streams upgrade") + # number of nodes needs to be >= 3 for the smoke test + self.kafka = KafkaService(self.test_context, num_nodes=3, + zk=self.zk, version=KafkaVersion(from_version), topics=self.topics) + self.kafka.start() - # get the node running the streams app - node = self.processor1.node - self.processor1.stop() + # allow some time for topics to be created + time.sleep(10) - # change it's version. This will automatically make it pick up a different - # JAR when it starts again - node.version = KafkaVersion(to_version) + self.driver = StreamsSmokeTestDriverService(self.test_context, self.kafka) + self.processor1 = StreamsSmokeTestJobRunnerService(self.test_context, self.kafka) + + self.driver.start() self.processor1.start() + time.sleep(15) - def perform_broker_upgrade(self, to_version): - self.logger.info("First pass bounce - rolling broker upgrade") - for node in self.kafka.nodes: - self.kafka.stop_node(node) - node.version = KafkaVersion(to_version) - self.kafka.start_node(node) + self.perform_broker_upgrade(to_version) - @ignore - @cluster(num_nodes=6) - @matrix(from_version=upgrade_versions, to_version=upgrade_versions) - def test_upgrade_downgrade_streams(self, from_version, to_version): - """ - Start a smoke test client, then abort (kill -9) and restart it a few times. - Ensure that all records are delivered. + time.sleep(15) + self.driver.wait() + self.driver.stop() + + self.processor1.stop() - Note, that just like tests/core/upgrade_test.py, a prerequisite for this test to succeed - if the inclusion of all parametrized versions of kafka in kafka/vagrant/base.sh - (search for get_kafka()). For streams in particular, that means that someone has manually - copies the kafka-stream-$version-test.jar in the right S3 bucket as shown in base.sh. + node = self.driver.node + node.account.ssh("grep ALL-RECORDS-DELIVERED %s" % self.driver.STDOUT_FILE, allow_fail=False) + self.processor1.node.account.ssh_capture("grep SMOKE-TEST-CLIENT-CLOSED %s" % self.processor1.STDOUT_FILE, allow_fail=False) + + @matrix(from_version=simple_upgrade_versions_metadata_version_2, to_version=simple_upgrade_versions_metadata_version_2) + def test_simple_upgrade_downgrade(self, from_version, to_version): + """ + Starts 3 KafkaStreams instances with , and upgrades one-by-one to """ - if from_version != to_version: - # Setup phase - self.zk = ZookeeperService(self.test_context, num_nodes=1) - self.zk.start() - # number of nodes needs to be >= 3 for the smoke test - self.kafka = KafkaService(self.test_context, num_nodes=3, - zk=self.zk, version=KafkaVersion(from_version), topics=self.topics) - self.kafka.start() + if from_version == to_version: + return - # allow some time for topics to be created - time.sleep(10) + self.zk = ZookeeperService(self.test_context, num_nodes=1) + self.zk.start() - self.driver = StreamsSmokeTestDriverService(self.test_context, self.kafka) - self.driver.node.version = KafkaVersion(from_version) - self.driver.start() + self.kafka = KafkaService(self.test_context, num_nodes=1, zk=self.zk, topics=self.topics) + self.kafka.start() - self.processor1 = StreamsSmokeTestJobRunnerService(self.test_context, self.kafka) - self.processor1.node.version = KafkaVersion(from_version) - self.processor1.start() + self.driver = StreamsSmokeTestDriverService(self.test_context, self.kafka) + self.driver.disable_auto_terminate() + self.processor1 = StreamsUpgradeTestJobRunnerService(self.test_context, self.kafka) + self.processor2 = StreamsUpgradeTestJobRunnerService(self.test_context, self.kafka) + self.processor3 = StreamsUpgradeTestJobRunnerService(self.test_context, self.kafka) - time.sleep(15) + self.driver.start() + self.start_all_nodes_with(from_version) - self.perform_streams_upgrade(to_version) + self.processors = [self.processor1, self.processor2, self.processor3] - time.sleep(15) - self.driver.wait() - self.driver.stop() + counter = 1 + random.seed() - self.processor1.stop() + # upgrade one-by-one via rolling bounce + random.shuffle(self.processors) + for p in self.processors: + p.CLEAN_NODE_ENABLED = False + self.do_rolling_bounce(p, None, to_version, counter) + counter = counter + 1 - self.driver.node.account.ssh("grep ALL-RECORDS-DELIVERED %s" % self.driver.STDOUT_FILE, allow_fail=False) - self.processor1.node.account.ssh_capture("grep SMOKE-TEST-CLIENT-CLOSED %s" % self.processor1.STDOUT_FILE, allow_fail=False) + # shutdown + self.driver.stop() + self.driver.wait() + random.shuffle(self.processors) + for p in self.processors: + node = p.node + with node.account.monitor_log(p.STDOUT_FILE) as monitor: + p.stop() + monitor.wait_until("UPGRADE-TEST-CLIENT-CLOSED", + timeout_sec=60, + err_msg="Never saw output 'UPGRADE-TEST-CLIENT-CLOSED' on" + str(node.account)) - @ignore - @cluster(num_nodes=6) - @matrix(from_version=upgrade_versions, to_version=upgrade_versions) - def test_upgrade_brokers(self, from_version, to_version): + self.driver.stop() + + #@parametrize(new_version=str(LATEST_0_10_1)) we cannot run this test until Kafka 0.10.1.2 is released + #@parametrize(new_version=str(LATEST_0_10_2)) we cannot run this test until Kafka 0.10.2.2 is released + #@parametrize(new_version=str(LATEST_0_11_0)) we cannot run this test until Kafka 0.11.0.3 is released + #@parametrize(new_version=str(LATEST_1_0)) we cannot run this test until Kafka 1.0.2 is released + #@parametrize(new_version=str(LATEST_1_1)) we cannot run this test until Kafka 1.1.1 is released + @parametrize(new_version=str(DEV_VERSION)) + def test_metadata_upgrade(self, new_version): """ - Start a smoke test client then perform rolling upgrades on the broker. + Starts 3 KafkaStreams instances with version 0.10.0, and upgrades one-by-one to """ - if from_version != to_version: - # Setup phase - self.zk = ZookeeperService(self.test_context, num_nodes=1) - self.zk.start() - # number of nodes needs to be >= 3 for the smoke test - self.kafka = KafkaService(self.test_context, num_nodes=3, - zk=self.zk, version=KafkaVersion(from_version), topics=self.topics) - self.kafka.start() + self.zk = ZookeeperService(self.test_context, num_nodes=1) + self.zk.start() + + self.kafka = KafkaService(self.test_context, num_nodes=1, zk=self.zk, topics=self.topics) + self.kafka.start() + + self.driver = StreamsSmokeTestDriverService(self.test_context, self.kafka) + self.driver.disable_auto_terminate() + self.processor1 = StreamsUpgradeTestJobRunnerService(self.test_context, self.kafka) + self.processor2 = StreamsUpgradeTestJobRunnerService(self.test_context, self.kafka) + self.processor3 = StreamsUpgradeTestJobRunnerService(self.test_context, self.kafka) + + self.driver.start() + self.start_all_nodes_with(str(LATEST_0_10_0)) + + self.processors = [self.processor1, self.processor2, self.processor3] + + counter = 1 + random.seed() + + # first rolling bounce + random.shuffle(self.processors) + for p in self.processors: + p.CLEAN_NODE_ENABLED = False + self.do_rolling_bounce(p, "0.10.0", new_version, counter) + counter = counter + 1 + + # second rolling bounce + random.shuffle(self.processors) + for p in self.processors: + self.do_rolling_bounce(p, None, new_version, counter) + counter = counter + 1 + + # shutdown + self.driver.stop() + self.driver.wait() + + random.shuffle(self.processors) + for p in self.processors: + node = p.node + with node.account.monitor_log(p.STDOUT_FILE) as monitor: + p.stop() + monitor.wait_until("UPGRADE-TEST-CLIENT-CLOSED", + timeout_sec=60, + err_msg="Never saw output 'UPGRADE-TEST-CLIENT-CLOSED' on" + str(node.account)) + + self.driver.stop() + + def start_all_nodes_with(self, version): + # start first with + self.prepare_for(self.processor1, version) + node1 = self.processor1.node + with node1.account.monitor_log(self.processor1.STDOUT_FILE) as monitor: + with node1.account.monitor_log(self.processor1.LOG_FILE) as log_monitor: + self.processor1.start() + log_monitor.wait_until("Kafka version : " + version, + timeout_sec=60, + err_msg="Could not detect Kafka Streams version " + version + " " + str(node1.account)) + monitor.wait_until("processed 100 records from topic", + timeout_sec=60, + err_msg="Never saw output 'processed 100 records from topic' on" + str(node1.account)) + + # start second with + self.prepare_for(self.processor2, version) + node2 = self.processor2.node + with node1.account.monitor_log(self.processor1.STDOUT_FILE) as first_monitor: + with node2.account.monitor_log(self.processor2.STDOUT_FILE) as second_monitor: + with node2.account.monitor_log(self.processor2.LOG_FILE) as log_monitor: + self.processor2.start() + log_monitor.wait_until("Kafka version : " + version, + timeout_sec=60, + err_msg="Could not detect Kafka Streams version " + version + " " + str(node2.account)) + first_monitor.wait_until("processed 100 records from topic", + timeout_sec=60, + err_msg="Never saw output 'processed 100 records from topic' on" + str(node1.account)) + second_monitor.wait_until("processed 100 records from topic", + timeout_sec=60, + err_msg="Never saw output 'processed 100 records from topic' on" + str(node2.account)) + + # start third with + self.prepare_for(self.processor3, version) + node3 = self.processor3.node + with node1.account.monitor_log(self.processor1.STDOUT_FILE) as first_monitor: + with node2.account.monitor_log(self.processor2.STDOUT_FILE) as second_monitor: + with node3.account.monitor_log(self.processor3.STDOUT_FILE) as third_monitor: + with node3.account.monitor_log(self.processor3.LOG_FILE) as log_monitor: + self.processor3.start() + log_monitor.wait_until("Kafka version : " + version, + timeout_sec=60, + err_msg="Could not detect Kafka Streams version " + version + " " + str(node3.account)) + first_monitor.wait_until("processed 100 records from topic", + timeout_sec=60, + err_msg="Never saw output 'processed 100 records from topic' on" + str(node1.account)) + second_monitor.wait_until("processed 100 records from topic", + timeout_sec=60, + err_msg="Never saw output 'processed 100 records from topic' on" + str(node2.account)) + third_monitor.wait_until("processed 100 records from topic", + timeout_sec=60, + err_msg="Never saw output 'processed 100 records from topic' on" + str(node3.account)) + + @staticmethod + def prepare_for(processor, version): + processor.node.account.ssh("rm -rf " + processor.PERSISTENT_ROOT, allow_fail=False) + if version == str(DEV_VERSION): + processor.set_version("") # set to TRUNK + else: + processor.set_version(version) + + def do_rolling_bounce(self, processor, upgrade_from, new_version, counter): + first_other_processor = None + second_other_processor = None + for p in self.processors: + if p != processor: + if first_other_processor is None: + first_other_processor = p + else: + second_other_processor = p + + node = processor.node + first_other_node = first_other_processor.node + second_other_node = second_other_processor.node - # allow some time for topics to be created - time.sleep(10) + # stop processor and wait for rebalance of others + with first_other_node.account.monitor_log(first_other_processor.STDOUT_FILE) as first_other_monitor: + with second_other_node.account.monitor_log(second_other_processor.STDOUT_FILE) as second_other_monitor: + processor.stop() + first_other_monitor.wait_until("processed 100 records from topic", + timeout_sec=60, + err_msg="Never saw output 'processed 100 records from topic' on" + str(first_other_node.account)) + second_other_monitor.wait_until("processed 100 records from topic", + timeout_sec=60, + err_msg="Never saw output 'processed 100 records from topic' on" + str(second_other_node.account)) + node.account.ssh_capture("grep UPGRADE-TEST-CLIENT-CLOSED %s" % processor.STDOUT_FILE, allow_fail=False) - # use the current (dev) version driver - self.driver = StreamsSmokeTestDriverService(self.test_context, self.kafka) - self.driver.node.version = KafkaVersion(from_version) - self.driver.start() + if upgrade_from is None: # upgrade disabled -- second round of rolling bounces + roll_counter = ".1-" # second round of rolling bounces + else: + roll_counter = ".0-" # first round of rolling boundes - self.processor1 = StreamsSmokeTestJobRunnerService(self.test_context, self.kafka) - self.processor1.node.version = KafkaVersion(from_version) - self.processor1.start() + node.account.ssh("mv " + processor.STDOUT_FILE + " " + processor.STDOUT_FILE + roll_counter + str(counter), allow_fail=False) + node.account.ssh("mv " + processor.STDERR_FILE + " " + processor.STDERR_FILE + roll_counter + str(counter), allow_fail=False) + node.account.ssh("mv " + processor.LOG_FILE + " " + processor.LOG_FILE + roll_counter + str(counter), allow_fail=False) - time.sleep(15) + if new_version == str(DEV_VERSION): + processor.set_version("") # set to TRUNK + else: + processor.set_version(new_version) + processor.set_upgrade_from(upgrade_from) - self.perform_broker_upgrade(to_version) + grep_metadata_error = "grep \"org.apache.kafka.streams.errors.TaskAssignmentException: unable to decode subscription data: version=2\" " + with node.account.monitor_log(processor.STDOUT_FILE) as monitor: + with node.account.monitor_log(processor.LOG_FILE) as log_monitor: + with first_other_node.account.monitor_log(first_other_processor.STDOUT_FILE) as first_other_monitor: + with second_other_node.account.monitor_log(second_other_processor.STDOUT_FILE) as second_other_monitor: + processor.start() - time.sleep(15) - self.driver.wait() - self.driver.stop() + log_monitor.wait_until("Kafka version : " + new_version, + timeout_sec=60, + err_msg="Could not detect Kafka Streams version " + new_version + " " + str(node.account)) + first_other_monitor.wait_until("processed 100 records from topic", + timeout_sec=60, + err_msg="Never saw output 'processed 100 records from topic' on" + str(first_other_node.account)) + found = list(first_other_node.account.ssh_capture(grep_metadata_error + first_other_processor.STDERR_FILE, allow_fail=True)) + if len(found) > 0: + raise Exception("Kafka Streams failed with 'unable to decode subscription data: version=2'") - self.processor1.stop() + second_other_monitor.wait_until("processed 100 records from topic", + timeout_sec=60, + err_msg="Never saw output 'processed 100 records from topic' on" + str(second_other_node.account)) + found = list(second_other_node.account.ssh_capture(grep_metadata_error + second_other_processor.STDERR_FILE, allow_fail=True)) + if len(found) > 0: + raise Exception("Kafka Streams failed with 'unable to decode subscription data: version=2'") - self.driver.node.account.ssh("grep ALL-RECORDS-DELIVERED %s" % self.driver.STDOUT_FILE, allow_fail=False) - self.processor1.node.account.ssh_capture("grep SMOKE-TEST-CLIENT-CLOSED %s" % self.processor1.STDOUT_FILE, allow_fail=False) + monitor.wait_until("processed 100 records from topic", + timeout_sec=60, + err_msg="Never saw output 'processed 100 records from topic' on" + str(node.account)) diff --git a/tests/kafkatest/version.py b/tests/kafkatest/version.py index b7071e79dae9f..66e5fcf18aabf 100644 --- a/tests/kafkatest/version.py +++ b/tests/kafkatest/version.py @@ -61,6 +61,7 @@ def get_version(node=None): return DEV_BRANCH DEV_BRANCH = KafkaVersion("dev") +DEV_VERSION = KafkaVersion("1.2.0-SNAPSHOT") # 0.8.2.X versions V_0_8_2_1 = KafkaVersion("0.8.2.1") @@ -89,7 +90,7 @@ def get_version(node=None): LATEST_0_10 = LATEST_0_10_2 -# 0.11.0.0 versions +# 0.11.0.x versions V_0_11_0_0 = KafkaVersion("0.11.0.0") V_0_11_0_1 = KafkaVersion("0.11.0.1") V_0_11_0_2 = KafkaVersion("0.11.0.2") diff --git a/tools/src/main/java/org/apache/kafka/trogdor/agent/WorkerManager.java b/tools/src/main/java/org/apache/kafka/trogdor/agent/WorkerManager.java index cda77738d8c6e..7c8de6d3f2259 100644 --- a/tools/src/main/java/org/apache/kafka/trogdor/agent/WorkerManager.java +++ b/tools/src/main/java/org/apache/kafka/trogdor/agent/WorkerManager.java @@ -22,6 +22,7 @@ import org.apache.kafka.common.internals.KafkaFutureImpl; import org.apache.kafka.common.utils.Scheduler; import org.apache.kafka.common.utils.Time; +import org.apache.kafka.common.utils.Utils; import org.apache.kafka.trogdor.common.Platform; import org.apache.kafka.trogdor.common.ThreadUtils; import org.apache.kafka.trogdor.rest.WorkerDone; @@ -29,6 +30,7 @@ import org.apache.kafka.trogdor.rest.WorkerStarting; import org.apache.kafka.trogdor.rest.WorkerStopping; import org.apache.kafka.trogdor.rest.WorkerState; +import org.apache.kafka.trogdor.task.AgentWorkerStatusTracker; import org.apache.kafka.trogdor.task.TaskSpec; import org.apache.kafka.trogdor.task.TaskWorker; import org.slf4j.Logger; @@ -43,7 +45,6 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicReference; public final class WorkerManager { private static final Logger log = LoggerFactory.getLogger(WorkerManager.class); @@ -190,7 +191,7 @@ class Worker { /** * The worker status. */ - private final AtomicReference status = new AtomicReference<>(""); + private final AgentWorkerStatusTracker status = new AgentWorkerStatusTracker(); /** * The time when this task was started. @@ -293,6 +294,8 @@ public void createWorker(final String id, TaskSpec spec) throws Exception { haltFuture.thenApply(new KafkaFuture.BaseFunction() { @Override public Void apply(String errorString) { + if (errorString == null) + errorString = ""; if (errorString.isEmpty()) { log.info("{}: Worker {} is halting.", nodeName, id); } else { @@ -306,8 +309,9 @@ public Void apply(String errorString) { try { worker.taskWorker.start(platform, worker.status, haltFuture); } catch (Exception e) { + log.info("{}: Worker {} start() exception", nodeName, id, e); stateChangeExecutor.submit(new HandleWorkerHalting(worker, - "worker.start() exception: " + e.getMessage(), true)); + "worker.start() exception: " + Utils.stackTrace(e), true)); } stateChangeExecutor.submit(new FinishCreatingWorker(worker)); } diff --git a/tools/src/main/java/org/apache/kafka/trogdor/common/WorkerUtils.java b/tools/src/main/java/org/apache/kafka/trogdor/common/WorkerUtils.java index 99c13c09febaf..3d4871a17296b 100644 --- a/tools/src/main/java/org/apache/kafka/trogdor/common/WorkerUtils.java +++ b/tools/src/main/java/org/apache/kafka/trogdor/common/WorkerUtils.java @@ -21,9 +21,14 @@ import org.apache.kafka.clients.admin.AdminClientConfig; import org.apache.kafka.clients.admin.DescribeTopicsOptions; import org.apache.kafka.clients.admin.DescribeTopicsResult; +import org.apache.kafka.clients.admin.ListTopicsOptions; +import org.apache.kafka.clients.admin.ListTopicsResult; import org.apache.kafka.clients.admin.NewTopic; import org.apache.kafka.clients.admin.TopicDescription; +import org.apache.kafka.clients.admin.TopicListing; import org.apache.kafka.common.KafkaException; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.TopicPartitionInfo; import org.apache.kafka.common.errors.NotEnoughReplicasException; import org.apache.kafka.common.errors.TimeoutException; import org.apache.kafka.common.errors.TopicExistsException; @@ -39,6 +44,7 @@ import java.util.Map; import java.util.Properties; import java.util.concurrent.Future; +import java.util.regex.Pattern; /** * Utilities for Trogdor TaskWorkers. @@ -74,7 +80,24 @@ public static int perSecToPerPeriod(float perSec, long periodMs) { return (int) perPeriod; } - private static final int CREATE_TOPICS_REQUEST_TIMEOUT = 25000; + /** + * Adds all properties from commonConf and then from clientConf to given 'props' (in + * that order, over-writing properties with the same keys). + * @param props Properties object that may contain zero or more properties + * @param commonConf Map with common client properties + * @param clientConf Map with client properties + */ + public static void addConfigsToProperties( + Properties props, Map commonConf, Map clientConf) { + for (Map.Entry commonEntry : commonConf.entrySet()) { + props.setProperty(commonEntry.getKey(), commonEntry.getValue()); + } + for (Map.Entry entry : clientConf.entrySet()) { + props.setProperty(entry.getKey(), entry.getValue()); + } + } + + private static final int ADMIN_REQUEST_TIMEOUT = 25000; private static final int CREATE_TOPICS_CALL_TIMEOUT = 180000; private static final int MAX_CREATE_TOPICS_BATCH_SIZE = 10; @@ -85,6 +108,9 @@ public static int perSecToPerPeriod(float perSec, long periodMs) { * * @param log The logger to use. * @param bootstrapServers The bootstrap server list. + * @param commonClientConf Common client config + * @param adminClientConf AdminClient config. This config has precedence over fields in + * common client config. * @param topics Maps topic names to partition assignments. * @param failOnExisting If true, the method will throw TopicExistsException if one or * more topics already exist. Otherwise, the existing topics are @@ -93,12 +119,14 @@ public static int perSecToPerPeriod(float perSec, long periodMs) { * number of partitions, the method throws RuntimeException. */ public static void createTopics( - Logger log, String bootstrapServers, + Logger log, String bootstrapServers, Map commonClientConf, + Map adminClientConf, Map topics, boolean failOnExisting) throws Throwable { // this method wraps the call to createTopics() that takes admin client, so that we can // unit test the functionality with MockAdminClient. The exception is caught and // re-thrown so that admin client is closed when the method returns. - try (AdminClient adminClient = createAdminClient(bootstrapServers)) { + try (AdminClient adminClient + = createAdminClient(bootstrapServers, commonClientConf, adminClientConf)) { createTopics(log, adminClient, topics, failOnExisting); } catch (Exception e) { log.warn("Failed to create or verify topics {}", topics, e); @@ -106,6 +134,32 @@ public static void createTopics( } } + /** + * Returns a list of all existing topic partitions that match the following criteria: topic + * name matches give regular expression 'topicRegex', topic is not internal, partitions are + * in range [startPartition, endPartition] + * + * @param log The logger to use. + * @param bootstrapServers The bootstrap server list. + * @param topicRegex Topic name regular expression + * @param startPartition Starting partition of partition range + * @param endPartition Ending partition of partition range + * @return List of topic partitions + * @throws Throwable If getting list of topics or their descriptions fails. + */ + public static Collection getMatchingTopicPartitions( + Logger log, String bootstrapServers, + Map commonClientConf, Map adminClientConf, + String topicRegex, int startPartition, int endPartition) throws Throwable { + try (AdminClient adminClient + = createAdminClient(bootstrapServers, commonClientConf, adminClientConf)) { + return getMatchingTopicPartitions(adminClient, topicRegex, startPartition, endPartition); + } catch (Exception e) { + log.warn("Failed to get topic partitions matching {}", topicRegex, e); + throw e; + } + } + /** * The actual create topics functionality is separated into this method and called from the * above method to be able to unit test with mock adminClient. @@ -211,7 +265,7 @@ private static void verifyTopics( Logger log, AdminClient adminClient, Collection topicsToVerify, Map topicsInfo) throws Throwable { DescribeTopicsResult topicsResult = adminClient.describeTopics( - topicsToVerify, new DescribeTopicsOptions().timeoutMs(CREATE_TOPICS_REQUEST_TIMEOUT)); + topicsToVerify, new DescribeTopicsOptions().timeoutMs(ADMIN_REQUEST_TIMEOUT)); Map topicDescriptionMap = topicsResult.all().get(); for (TopicDescription desc: topicDescriptionMap.values()) { // map will always contain the topic since all topics in 'topicsExists' are in given @@ -227,10 +281,56 @@ private static void verifyTopics( } } - private static AdminClient createAdminClient(String bootstrapServers) { + /** + * Returns list of existing, not internal, topics/partitions that match given pattern and + * where partitions are in range [startPartition, endPartition] + * @param adminClient AdminClient + * @param topicRegex Topic regular expression to match + * @return List of topic names + * @throws Throwable If failed to get list of existing topics + */ + static Collection getMatchingTopicPartitions( + AdminClient adminClient, String topicRegex, int startPartition, int endPartition) + throws Throwable { + final Pattern topicNamePattern = Pattern.compile(topicRegex); + + // first get list of matching topics + List matchedTopics = new ArrayList<>(); + ListTopicsResult res = adminClient.listTopics( + new ListTopicsOptions().timeoutMs(ADMIN_REQUEST_TIMEOUT)); + Map topicListingMap = res.namesToListings().get(); + for (Map.Entry topicListingEntry: topicListingMap.entrySet()) { + if (!topicListingEntry.getValue().isInternal() + && topicNamePattern.matcher(topicListingEntry.getKey()).matches()) { + matchedTopics.add(topicListingEntry.getKey()); + } + } + + // create a list of topic/partitions + List out = new ArrayList<>(); + DescribeTopicsResult topicsResult = adminClient.describeTopics( + matchedTopics, new DescribeTopicsOptions().timeoutMs(ADMIN_REQUEST_TIMEOUT)); + Map topicDescriptionMap = topicsResult.all().get(); + for (TopicDescription desc: topicDescriptionMap.values()) { + List partitions = desc.partitions(); + for (TopicPartitionInfo info: partitions) { + if ((info.partition() >= startPartition) && (info.partition() <= endPartition)) { + out.add(new TopicPartition(desc.name(), info.partition())); + } + } + } + return out; + } + + private static AdminClient createAdminClient( + String bootstrapServers, + Map commonClientConf, Map adminClientConf) { Properties props = new Properties(); props.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers); - props.put(AdminClientConfig.REQUEST_TIMEOUT_MS_CONFIG, CREATE_TOPICS_REQUEST_TIMEOUT); + props.put(AdminClientConfig.REQUEST_TIMEOUT_MS_CONFIG, ADMIN_REQUEST_TIMEOUT); + // first add common client config, and then admin client config to properties, possibly + // over-writing default or common properties. + addConfigsToProperties(props, commonClientConf, adminClientConf); return AdminClient.create(props); } } diff --git a/tools/src/main/java/org/apache/kafka/trogdor/coordinator/Coordinator.java b/tools/src/main/java/org/apache/kafka/trogdor/coordinator/Coordinator.java index b3418dd6732b1..717d7c7047a34 100644 --- a/tools/src/main/java/org/apache/kafka/trogdor/coordinator/Coordinator.java +++ b/tools/src/main/java/org/apache/kafka/trogdor/coordinator/Coordinator.java @@ -31,6 +31,7 @@ import org.apache.kafka.trogdor.rest.JsonRestServer; import org.apache.kafka.trogdor.rest.StopTaskRequest; import org.apache.kafka.trogdor.rest.StopTaskResponse; +import org.apache.kafka.trogdor.rest.TasksRequest; import org.apache.kafka.trogdor.rest.TasksResponse; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -94,8 +95,8 @@ public StopTaskResponse stopTask(StopTaskRequest request) throws Exception { return new StopTaskResponse(taskManager.stopTask(request.id())); } - public TasksResponse tasks() throws Exception { - return taskManager.tasks(); + public TasksResponse tasks(TasksRequest request) throws Exception { + return taskManager.tasks(request); } public void beginShutdown(boolean stopAgents) throws Exception { diff --git a/tools/src/main/java/org/apache/kafka/trogdor/coordinator/CoordinatorClient.java b/tools/src/main/java/org/apache/kafka/trogdor/coordinator/CoordinatorClient.java index 870b64ea347e5..0677296ee3cfd 100644 --- a/tools/src/main/java/org/apache/kafka/trogdor/coordinator/CoordinatorClient.java +++ b/tools/src/main/java/org/apache/kafka/trogdor/coordinator/CoordinatorClient.java @@ -33,10 +33,13 @@ import org.apache.kafka.trogdor.rest.JsonRestServer.HttpResponse; import org.apache.kafka.trogdor.rest.StopTaskRequest; import org.apache.kafka.trogdor.rest.StopTaskResponse; +import org.apache.kafka.trogdor.rest.TasksRequest; import org.apache.kafka.trogdor.rest.TasksResponse; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.ws.rs.core.UriBuilder; + import static net.sourceforge.argparse4j.impl.Arguments.store; import static net.sourceforge.argparse4j.impl.Arguments.storeTrue; @@ -127,9 +130,15 @@ public StopTaskResponse stopTask(StopTaskRequest request) throws Exception { return resp.body(); } - public TasksResponse tasks() throws Exception { + public TasksResponse tasks(TasksRequest request) throws Exception { + UriBuilder uriBuilder = UriBuilder.fromPath(url("/coordinator/tasks")); + uriBuilder.queryParam("taskId", request.taskIds().toArray(new String[0])); + uriBuilder.queryParam("firstStartMs", request.firstStartMs()); + uriBuilder.queryParam("lastStartMs", request.lastStartMs()); + uriBuilder.queryParam("firstEndMs", request.firstEndMs()); + uriBuilder.queryParam("lastEndMs", request.lastEndMs()); HttpResponse resp = - JsonRestServer.httpRequest(log, url("/coordinator/tasks"), "GET", + JsonRestServer.httpRequest(log, uriBuilder.build().toString(), "GET", null, new TypeReference() { }, maxTries); return resp.body(); } @@ -204,7 +213,8 @@ public static void main(String[] args) throws Exception { JsonUtil.toPrettyJsonString(client.status())); } else if (res.getBoolean("show_tasks")) { System.out.println("Got coordinator tasks: " + - JsonUtil.toPrettyJsonString(client.tasks())); + JsonUtil.toPrettyJsonString(client.tasks( + new TasksRequest(null, 0, 0, 0, 0)))); } else if (res.getString("create_task") != null) { client.createTask(JsonUtil.JSON_SERDE.readValue(res.getString("create_task"), CreateTaskRequest.class)); diff --git a/tools/src/main/java/org/apache/kafka/trogdor/coordinator/CoordinatorRestResource.java b/tools/src/main/java/org/apache/kafka/trogdor/coordinator/CoordinatorRestResource.java index 7775dd097151f..b8663ec4cc337 100644 --- a/tools/src/main/java/org/apache/kafka/trogdor/coordinator/CoordinatorRestResource.java +++ b/tools/src/main/java/org/apache/kafka/trogdor/coordinator/CoordinatorRestResource.java @@ -23,16 +23,20 @@ import org.apache.kafka.trogdor.rest.Empty; import org.apache.kafka.trogdor.rest.StopTaskRequest; import org.apache.kafka.trogdor.rest.StopTaskResponse; +import org.apache.kafka.trogdor.rest.TasksRequest; import org.apache.kafka.trogdor.rest.TasksResponse; import javax.servlet.ServletContext; import javax.ws.rs.Consumes; +import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; import javax.ws.rs.POST; import javax.ws.rs.PUT; import javax.ws.rs.Path; import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; import javax.ws.rs.core.MediaType; +import java.util.List; import java.util.concurrent.atomic.AtomicReference; @@ -69,8 +73,12 @@ public StopTaskResponse stopTask(StopTaskRequest request) throws Throwable { @GET @Path("/tasks") - public TasksResponse tasks() throws Throwable { - return coordinator().tasks(); + public TasksResponse tasks(@QueryParam("taskId") List taskId, + @DefaultValue("0") @QueryParam("firstStartMs") int firstStartMs, + @DefaultValue("0") @QueryParam("lastStartMs") int lastStartMs, + @DefaultValue("0") @QueryParam("firstEndMs") int firstEndMs, + @DefaultValue("0") @QueryParam("lastEndMs") int lastEndMs) throws Throwable { + return coordinator().tasks(new TasksRequest(taskId, firstStartMs, lastStartMs, firstEndMs, lastEndMs)); } @PUT diff --git a/tools/src/main/java/org/apache/kafka/trogdor/coordinator/NodeManager.java b/tools/src/main/java/org/apache/kafka/trogdor/coordinator/NodeManager.java index 0129007aa0d19..91ef9c2928a59 100644 --- a/tools/src/main/java/org/apache/kafka/trogdor/coordinator/NodeManager.java +++ b/tools/src/main/java/org/apache/kafka/trogdor/coordinator/NodeManager.java @@ -49,7 +49,6 @@ import org.apache.kafka.trogdor.rest.AgentStatusResponse; import org.apache.kafka.trogdor.rest.CreateWorkerRequest; import org.apache.kafka.trogdor.rest.StopWorkerRequest; -import org.apache.kafka.trogdor.rest.WorkerDone; import org.apache.kafka.trogdor.rest.WorkerReceiving; import org.apache.kafka.trogdor.rest.WorkerRunning; import org.apache.kafka.trogdor.rest.WorkerStarting; @@ -192,6 +191,9 @@ public void run() { // agents going down? return; } + if (log.isTraceEnabled()) { + log.trace("{}: got heartbeat status {}", node.name(), agentStatus); + } // Identify workers which we think should be running, but which do not appear // in the agent's response. We need to send startWorker requests for these. for (Map.Entry entry : workers.entrySet()) { @@ -203,40 +205,31 @@ public void run() { } } } - // Identify tasks which are running, but which we don't know about. - // Add these to the NodeManager as tasks that should not be running. for (Map.Entry entry : agentStatus.workers().entrySet()) { String id = entry.getKey(); WorkerState state = entry.getValue(); - if (!workers.containsKey(id)) { + ManagedWorker worker = workers.get(id); + if (worker == null) { + // Identify tasks which are running, but which we don't know about. + // Add these to the NodeManager as tasks that should not be running. log.warn("{}: scheduling unknown worker {} for stopping.", node.name(), id); workers.put(id, new ManagedWorker(id, state.spec(), false, state)); - } - } - // Handle workers which need to be stopped. Handle workers which have newly completed. - for (Map.Entry entry : agentStatus.workers().entrySet()) { - String id = entry.getKey(); - WorkerState state = entry.getValue(); - ManagedWorker worker = workers.get(id); - if (state instanceof WorkerStarting || state instanceof WorkerRunning) { - if (!worker.shouldRun) { - worker.tryStop(); - } - } else if (state instanceof WorkerDone) { - if (!(worker.state instanceof WorkerDone)) { - WorkerDone workerDoneState = (WorkerDone) state; - String error = workerDoneState.error(); - if (error.isEmpty()) { - log.info("{}: Worker {} finished with status '{}'", - node.name(), id, workerDoneState.status()); - } else { - log.warn("{}: Worker {} finished with error '{}' and status '{}'", - node.name(), id, error, workerDoneState.status()); + } else { + // Handle workers which need to be stopped. + if (state instanceof WorkerStarting || state instanceof WorkerRunning) { + if (!worker.shouldRun) { + worker.tryStop(); } - taskManager.handleWorkerCompletion(node.name(), worker.id, error); + } + // Notify the TaskManager if the worker state has changed. + if (worker.state.equals(state)) { + log.debug("{}: worker state is still {}", node.name(), worker.state); + } else { + log.info("{}: worker state changed from {} to {}", node.name(), worker.state, state); + worker.state = state; + taskManager.updateWorkerState(node.name(), worker.id, state); } } - worker.state = state; } } catch (Throwable e) { log.error("{}: Unhandled exception in NodeHeartbeatRunnable", node.name(), e); diff --git a/tools/src/main/java/org/apache/kafka/trogdor/coordinator/TaskManager.java b/tools/src/main/java/org/apache/kafka/trogdor/coordinator/TaskManager.java index 286f9049fac14..7e19c8b34ae3b 100644 --- a/tools/src/main/java/org/apache/kafka/trogdor/coordinator/TaskManager.java +++ b/tools/src/main/java/org/apache/kafka/trogdor/coordinator/TaskManager.java @@ -17,10 +17,14 @@ package org.apache.kafka.trogdor.coordinator; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.kafka.common.KafkaException; import org.apache.kafka.common.utils.Scheduler; import org.apache.kafka.common.utils.Time; import org.apache.kafka.common.utils.Utils; +import org.apache.kafka.trogdor.common.JsonUtil; import org.apache.kafka.trogdor.common.Node; import org.apache.kafka.trogdor.common.Platform; import org.apache.kafka.trogdor.common.ThreadUtils; @@ -29,14 +33,17 @@ import org.apache.kafka.trogdor.rest.TaskRunning; import org.apache.kafka.trogdor.rest.TaskState; import org.apache.kafka.trogdor.rest.TaskStopping; +import org.apache.kafka.trogdor.rest.TasksRequest; import org.apache.kafka.trogdor.rest.TasksResponse; +import org.apache.kafka.trogdor.rest.WorkerDone; +import org.apache.kafka.trogdor.rest.WorkerReceiving; +import org.apache.kafka.trogdor.rest.WorkerState; import org.apache.kafka.trogdor.task.TaskController; import org.apache.kafka.trogdor.task.TaskSpec; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.HashMap; -import java.util.HashSet; import java.util.Map; import java.util.Set; import java.util.TreeMap; @@ -171,16 +178,9 @@ class ManagedTask { private Future startFuture = null; /** - * The name of the worker nodes involved with this task. - * Null if the task is not running. + * The states of the workers involved with this task. */ - private Set workers = null; - - /** - * The names of the worker nodes which are still running this task. - * Null if the task is not running. - */ - private Set activeWorkers = null; + public Map workerStates = new TreeMap<>(); /** * If this is non-empty, a message describing how this task failed. @@ -240,14 +240,39 @@ TaskState taskState() { case PENDING: return new TaskPending(spec); case RUNNING: - return new TaskRunning(spec, startedMs); + return new TaskRunning(spec, startedMs, getCombinedStatus(workerStates)); case STOPPING: - return new TaskStopping(spec, startedMs); + return new TaskStopping(spec, startedMs, getCombinedStatus(workerStates)); case DONE: - return new TaskDone(spec, startedMs, doneMs, error, cancelled); + return new TaskDone(spec, startedMs, doneMs, error, cancelled, getCombinedStatus(workerStates)); } throw new RuntimeException("unreachable"); } + + TreeSet activeWorkers() { + TreeSet workerNames = new TreeSet<>(); + for (Map.Entry entry : workerStates.entrySet()) { + if (!entry.getValue().done()) { + workerNames.add(entry.getKey()); + } + } + return workerNames; + } + } + + private static final JsonNode getCombinedStatus(Map states) { + if (states.size() == 1) { + return states.values().iterator().next().status(); + } else { + ObjectNode objectNode = new ObjectNode(JsonNodeFactory.instance); + for (Map.Entry entry : states.entrySet()) { + JsonNode node = entry.getValue().status(); + if (node != null) { + objectNode.set(entry.getKey(), node); + } + } + return objectNode; + } } /** @@ -348,10 +373,8 @@ public Void call() throws Exception { log.info("Running task {} on node(s): {}", task.id, Utils.join(nodeNames, ", ")); task.state = ManagedTaskState.RUNNING; task.startedMs = time.milliseconds(); - task.workers = nodeNames; - task.activeWorkers = new HashSet<>(); - for (String workerName : task.workers) { - task.activeWorkers.add(workerName); + for (String workerName : nodeNames) { + task.workerStates.put(workerName, new WorkerReceiving(task.spec)); nodeManagers.get(workerName).createWorker(task.id, task.spec); } return null; @@ -397,15 +420,16 @@ public TaskSpec call() throws Exception { break; case RUNNING: task.cancelled = true; - if (task.activeWorkers.size() == 0) { + TreeSet activeWorkers = task.activeWorkers(); + if (activeWorkers.isEmpty()) { log.info("Task {} is now complete with error: {}", id, task.error); task.doneMs = time.milliseconds(); task.state = ManagedTaskState.DONE; } else { - for (String workerName : task.activeWorkers) { + for (String workerName : activeWorkers) { nodeManagers.get(workerName).stopWorker(id); } - log.info("Cancelling task {} on worker(s): {}", id, Utils.join(task.activeWorkers, ", ")); + log.info("Cancelling task {} on worker(s): {}", id, Utils.join(activeWorkers, ", ")); task.state = ManagedTaskState.STOPPING; } break; @@ -421,78 +445,100 @@ public TaskSpec call() throws Exception { } /** - * A callback NodeManager makes to indicate that a worker has completed. - * The task will transition to DONE once all workers are done. + * Update the state of a particular agent's worker. * - * @param nodeName The node name. + * @param nodeName The node where the agent is running. * @param id The worker name. - * @param error An empty string if there is no error, or an error string. + * @param state The worker state. */ - public void handleWorkerCompletion(String nodeName, String id, String error) { - executor.submit(new HandleWorkerCompletion(nodeName, id, error)); + public void updateWorkerState(String nodeName, String id, WorkerState state) { + executor.submit(new UpdateWorkerState(nodeName, id, state)); } - class HandleWorkerCompletion implements Callable { + class UpdateWorkerState implements Callable { private final String nodeName; private final String id; - private final String error; + private final WorkerState state; - HandleWorkerCompletion(String nodeName, String id, String error) { + UpdateWorkerState(String nodeName, String id, WorkerState state) { this.nodeName = nodeName; this.id = id; - this.error = error; + this.state = state; } @Override public Void call() throws Exception { ManagedTask task = tasks.get(id); if (task == null) { - log.error("Can't handle completion of unknown worker {} on node {}", + log.error("Can't update worker state unknown worker {} on node {}", id, nodeName); return null; } - if ((task.state == ManagedTaskState.PENDING) || (task.state == ManagedTaskState.DONE)) { - log.error("Task {} got unexpected worker completion from {} while " + - "in {} state.", id, nodeName, task.state); - return null; - } - boolean broadcastStop = false; - if (task.state == ManagedTaskState.RUNNING) { - task.state = ManagedTaskState.STOPPING; - broadcastStop = true; - } - task.maybeSetError(error); - task.activeWorkers.remove(nodeName); - if (task.activeWorkers.size() == 0) { - task.doneMs = time.milliseconds(); - task.state = ManagedTaskState.DONE; - log.info("Task {} is now complete on {} with error: {}", id, - Utils.join(task.workers, ", "), - task.error.isEmpty() ? "(none)" : task.error); - } else if (broadcastStop) { - log.info("Node {} stopped. Stopping task {} on worker(s): {}", - id, Utils.join(task.activeWorkers, ", ")); - for (String workerName : task.activeWorkers) { - nodeManagers.get(workerName).stopWorker(id); - } + WorkerState prevState = task.workerStates.get(nodeName); + log.debug("Task {}: Updating worker state for {} from {} to {}.", + id, nodeName, prevState, state); + task.workerStates.put(nodeName, state); + if (state.done() && (!prevState.done())) { + handleWorkerCompletion(task, nodeName, (WorkerDone) state); } return null; } } + /** + * Handle a worker being completed. + * + * @param task The task that owns the worker. + * @param nodeName The name of the node on which the worker is running. + * @param state The worker state. + */ + private void handleWorkerCompletion(ManagedTask task, String nodeName, WorkerDone state) { + if (state.error().isEmpty()) { + log.info("{}: Worker {} finished with status '{}'", + nodeName, task.id, JsonUtil.toJsonString(state.status())); + } else { + log.warn("{}: Worker {} finished with error '{}' and status '{}'", + nodeName, task.id, state.error(), JsonUtil.toJsonString(state.status())); + task.maybeSetError(state.error()); + } + if (task.activeWorkers().isEmpty()) { + task.doneMs = time.milliseconds(); + task.state = ManagedTaskState.DONE; + log.info("{}: Task {} is now complete on {} with error: {}", + nodeName, task.id, Utils.join(task.workerStates.keySet(), ", "), + task.error.isEmpty() ? "(none)" : task.error); + } else if ((task.state == ManagedTaskState.RUNNING) && (!task.error.isEmpty())) { + TreeSet activeWorkers = task.activeWorkers(); + log.info("{}: task {} stopped with error {}. Stopping worker(s): {}", + nodeName, task.id, task.error, Utils.join(activeWorkers, ", ")); + task.state = ManagedTaskState.STOPPING; + for (String workerName : activeWorkers) { + nodeManagers.get(workerName).stopWorker(task.id); + } + } + } + /** * Get information about the tasks being managed. */ - public TasksResponse tasks() throws ExecutionException, InterruptedException { - return executor.submit(new GetTasksResponse()).get(); + public TasksResponse tasks(TasksRequest request) throws ExecutionException, InterruptedException { + return executor.submit(new GetTasksResponse(request)).get(); } class GetTasksResponse implements Callable { + private final TasksRequest request; + + GetTasksResponse(TasksRequest request) { + this.request = request; + } + @Override public TasksResponse call() throws Exception { TreeMap states = new TreeMap<>(); for (ManagedTask task : tasks.values()) { - states.put(task.id, task.taskState()); + if (request.matches(task.id, task.startedMs, task.doneMs)) { + states.put(task.id, task.taskState()); + } } return new TasksResponse(states); } diff --git a/tools/src/main/java/org/apache/kafka/trogdor/fault/KiboshFaultWorker.java b/tools/src/main/java/org/apache/kafka/trogdor/fault/KiboshFaultWorker.java index 629d15e8147fe..97934a88bbdea 100644 --- a/tools/src/main/java/org/apache/kafka/trogdor/fault/KiboshFaultWorker.java +++ b/tools/src/main/java/org/apache/kafka/trogdor/fault/KiboshFaultWorker.java @@ -17,15 +17,15 @@ package org.apache.kafka.trogdor.fault; +import com.fasterxml.jackson.databind.node.TextNode; import org.apache.kafka.common.internals.KafkaFutureImpl; import org.apache.kafka.trogdor.common.Platform; import org.apache.kafka.trogdor.fault.Kibosh.KiboshFaultSpec; import org.apache.kafka.trogdor.task.TaskWorker; +import org.apache.kafka.trogdor.task.WorkerStatusTracker; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.concurrent.atomic.AtomicReference; - public class KiboshFaultWorker implements TaskWorker { private static final Logger log = LoggerFactory.getLogger(KiboshFaultWorker.class); @@ -35,6 +35,8 @@ public class KiboshFaultWorker implements TaskWorker { private final String mountPath; + private WorkerStatusTracker status; + public KiboshFaultWorker(String id, KiboshFaultSpec spec, String mountPath) { this.id = id; this.spec = spec; @@ -42,15 +44,20 @@ public KiboshFaultWorker(String id, KiboshFaultSpec spec, String mountPath) { } @Override - public void start(Platform platform, AtomicReference status, + public void start(Platform platform, WorkerStatusTracker status, KafkaFutureImpl errorFuture) throws Exception { log.info("Activating {} {}: {}.", spec.getClass().getSimpleName(), id, spec); + this.status = status; + this.status.update(new TextNode("Adding fault " + id)); Kibosh.INSTANCE.addFault(mountPath, spec); + this.status.update(new TextNode("Added fault " + id)); } @Override public void stop(Platform platform) throws Exception { log.info("Deactivating {} {}: {}.", spec.getClass().getSimpleName(), id, spec); + this.status.update(new TextNode("Removing fault " + id)); Kibosh.INSTANCE.removeFault(mountPath, spec); + this.status.update(new TextNode("Removed fault " + id)); } } diff --git a/tools/src/main/java/org/apache/kafka/trogdor/fault/NetworkPartitionFaultWorker.java b/tools/src/main/java/org/apache/kafka/trogdor/fault/NetworkPartitionFaultWorker.java index 787c5e06f187a..1b99a93d8fbd0 100644 --- a/tools/src/main/java/org/apache/kafka/trogdor/fault/NetworkPartitionFaultWorker.java +++ b/tools/src/main/java/org/apache/kafka/trogdor/fault/NetworkPartitionFaultWorker.java @@ -17,11 +17,13 @@ package org.apache.kafka.trogdor.fault; +import com.fasterxml.jackson.databind.node.TextNode; import org.apache.kafka.common.internals.KafkaFutureImpl; import org.apache.kafka.trogdor.common.Node; import org.apache.kafka.trogdor.common.Platform; import org.apache.kafka.trogdor.common.Topology; import org.apache.kafka.trogdor.task.TaskWorker; +import org.apache.kafka.trogdor.task.WorkerStatusTracker; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -29,7 +31,6 @@ import java.util.List; import java.util.Set; import java.util.TreeSet; -import java.util.concurrent.atomic.AtomicReference; public class NetworkPartitionFaultWorker implements TaskWorker { private static final Logger log = LoggerFactory.getLogger(NetworkPartitionFaultWorker.class); @@ -38,22 +39,29 @@ public class NetworkPartitionFaultWorker implements TaskWorker { private final List> partitionSets; + private WorkerStatusTracker status; + public NetworkPartitionFaultWorker(String id, List> partitionSets) { this.id = id; this.partitionSets = partitionSets; } @Override - public void start(Platform platform, AtomicReference status, + public void start(Platform platform, WorkerStatusTracker status, KafkaFutureImpl errorFuture) throws Exception { log.info("Activating NetworkPartitionFault {}.", id); + this.status = status; + this.status.update(new TextNode("creating network partition " + id)); runIptablesCommands(platform, "-A"); + this.status.update(new TextNode("created network partition " + id)); } @Override public void stop(Platform platform) throws Exception { log.info("Deactivating NetworkPartitionFault {}.", id); + this.status.update(new TextNode("removing network partition " + id)); runIptablesCommands(platform, "-D"); + this.status.update(new TextNode("removed network partition " + id)); } private void runIptablesCommands(Platform platform, String iptablesAction) throws Exception { diff --git a/tools/src/main/java/org/apache/kafka/trogdor/fault/ProcessStopFaultWorker.java b/tools/src/main/java/org/apache/kafka/trogdor/fault/ProcessStopFaultWorker.java index 66a8c6edb2015..d30eaf766b946 100644 --- a/tools/src/main/java/org/apache/kafka/trogdor/fault/ProcessStopFaultWorker.java +++ b/tools/src/main/java/org/apache/kafka/trogdor/fault/ProcessStopFaultWorker.java @@ -17,16 +17,17 @@ package org.apache.kafka.trogdor.fault; +import com.fasterxml.jackson.databind.node.TextNode; import org.apache.kafka.common.internals.KafkaFutureImpl; import org.apache.kafka.common.utils.Utils; import org.apache.kafka.trogdor.common.Platform; import org.apache.kafka.trogdor.task.TaskWorker; +import org.apache.kafka.trogdor.task.WorkerStatusTracker; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.atomic.AtomicReference; public class ProcessStopFaultWorker implements TaskWorker { private static final Logger log = LoggerFactory.getLogger(ProcessStopFaultWorker.class); @@ -35,22 +36,29 @@ public class ProcessStopFaultWorker implements TaskWorker { private final String javaProcessName; + private WorkerStatusTracker status; + public ProcessStopFaultWorker(String id, String javaProcessName) { this.id = id; this.javaProcessName = javaProcessName; } @Override - public void start(Platform platform, AtomicReference status, + public void start(Platform platform, WorkerStatusTracker status, KafkaFutureImpl errorFuture) throws Exception { + this.status = status; log.info("Activating ProcessStopFault {}.", id); + this.status.update(new TextNode("stopping " + javaProcessName)); sendSignals(platform, "SIGSTOP"); + this.status.update(new TextNode("stopped " + javaProcessName)); } @Override public void stop(Platform platform) throws Exception { log.info("Deactivating ProcessStopFault {}.", id); + this.status.update(new TextNode("resuming " + javaProcessName)); sendSignals(platform, "SIGCONT"); + this.status.update(new TextNode("resumed " + javaProcessName)); } private void sendSignals(Platform platform, String signalName) throws Exception { diff --git a/tools/src/main/java/org/apache/kafka/trogdor/rest/TaskDone.java b/tools/src/main/java/org/apache/kafka/trogdor/rest/TaskDone.java index 536d3f20b33d8..e8d6003bede35 100644 --- a/tools/src/main/java/org/apache/kafka/trogdor/rest/TaskDone.java +++ b/tools/src/main/java/org/apache/kafka/trogdor/rest/TaskDone.java @@ -16,9 +16,9 @@ */ package org.apache.kafka.trogdor.rest; - import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.JsonNode; import org.apache.kafka.trogdor.task.TaskSpec; /** @@ -50,8 +50,9 @@ public TaskDone(@JsonProperty("spec") TaskSpec spec, @JsonProperty("startedMs") long startedMs, @JsonProperty("doneMs") long doneMs, @JsonProperty("error") String error, - @JsonProperty("cancelled") boolean cancelled) { - super(spec); + @JsonProperty("cancelled") boolean cancelled, + @JsonProperty("status") JsonNode status) { + super(spec, status); this.startedMs = startedMs; this.doneMs = doneMs; this.error = error; diff --git a/tools/src/main/java/org/apache/kafka/trogdor/rest/TaskPending.java b/tools/src/main/java/org/apache/kafka/trogdor/rest/TaskPending.java index b0162d35c71df..7831301425cde 100644 --- a/tools/src/main/java/org/apache/kafka/trogdor/rest/TaskPending.java +++ b/tools/src/main/java/org/apache/kafka/trogdor/rest/TaskPending.java @@ -19,6 +19,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.node.NullNode; import org.apache.kafka.trogdor.task.TaskSpec; /** @@ -27,6 +28,6 @@ public class TaskPending extends TaskState { @JsonCreator public TaskPending(@JsonProperty("spec") TaskSpec spec) { - super(spec); + super(spec, NullNode.instance); } } diff --git a/tools/src/main/java/org/apache/kafka/trogdor/rest/TaskRunning.java b/tools/src/main/java/org/apache/kafka/trogdor/rest/TaskRunning.java index bff36766c4d1d..7a81bdf7867c1 100644 --- a/tools/src/main/java/org/apache/kafka/trogdor/rest/TaskRunning.java +++ b/tools/src/main/java/org/apache/kafka/trogdor/rest/TaskRunning.java @@ -19,6 +19,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.JsonNode; import org.apache.kafka.trogdor.task.TaskSpec; /** @@ -32,8 +33,9 @@ public class TaskRunning extends TaskState { @JsonCreator public TaskRunning(@JsonProperty("spec") TaskSpec spec, - @JsonProperty("startedMs") long startedMs) { - super(spec); + @JsonProperty("startedMs") long startedMs, + @JsonProperty("status") JsonNode status) { + super(spec, status); this.startedMs = startedMs; } diff --git a/tools/src/main/java/org/apache/kafka/trogdor/rest/TaskState.java b/tools/src/main/java/org/apache/kafka/trogdor/rest/TaskState.java index 28b61087ad043..0764e1445c881 100644 --- a/tools/src/main/java/org/apache/kafka/trogdor/rest/TaskState.java +++ b/tools/src/main/java/org/apache/kafka/trogdor/rest/TaskState.java @@ -20,6 +20,8 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.NullNode; import org.apache.kafka.trogdor.task.TaskSpec; /** @@ -37,12 +39,20 @@ public abstract class TaskState extends Message { private final TaskSpec spec; - public TaskState(TaskSpec spec) { + private final JsonNode status; + + public TaskState(TaskSpec spec, JsonNode status) { this.spec = spec; + this.status = status == null ? NullNode.instance : status; } @JsonProperty public TaskSpec spec() { return spec; } + + @JsonProperty + public JsonNode status() { + return status; + } } diff --git a/tools/src/main/java/org/apache/kafka/trogdor/rest/TaskStopping.java b/tools/src/main/java/org/apache/kafka/trogdor/rest/TaskStopping.java index 4446b75f9fcdd..d40b43c485c5a 100644 --- a/tools/src/main/java/org/apache/kafka/trogdor/rest/TaskStopping.java +++ b/tools/src/main/java/org/apache/kafka/trogdor/rest/TaskStopping.java @@ -19,6 +19,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.JsonNode; import org.apache.kafka.trogdor.task.TaskSpec; /** @@ -32,8 +33,9 @@ public class TaskStopping extends TaskState { @JsonCreator public TaskStopping(@JsonProperty("spec") TaskSpec spec, - @JsonProperty("startedMs") long startedMs) { - super(spec); + @JsonProperty("startedMs") long startedMs, + @JsonProperty("status") JsonNode status) { + super(spec, status); this.startedMs = startedMs; } diff --git a/tools/src/main/java/org/apache/kafka/trogdor/rest/TasksRequest.java b/tools/src/main/java/org/apache/kafka/trogdor/rest/TasksRequest.java new file mode 100644 index 0000000000000..24b438af6383d --- /dev/null +++ b/tools/src/main/java/org/apache/kafka/trogdor/rest/TasksRequest.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.trogdor.rest; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +/** + * The request to /coordinator/tasks + */ +public class TasksRequest extends Message { + /** + * The task IDs to list. + * An empty set of task IDs indicates that we should list all task IDs. + */ + private final Set taskIds; + + /** + * If this is non-zero, only tasks with a startMs at or after this time will be listed. + */ + private final long firstStartMs; + + /** + * If this is non-zero, only tasks with a startMs at or before this time will be listed. + */ + private final long lastStartMs; + + /** + * If this is non-zero, only tasks with an endMs at or after this time will be listed. + */ + private final long firstEndMs; + + /** + * If this is non-zero, only tasks with an endMs at or before this time will be listed. + */ + private final long lastEndMs; + + @JsonCreator + public TasksRequest(@JsonProperty("taskIds") Collection taskIds, + @JsonProperty("firstStartMs") long firstStartMs, + @JsonProperty("lastStartMs") long lastStartMs, + @JsonProperty("firstEndMs") long firstEndMs, + @JsonProperty("lastEndMs") long lastEndMs) { + this.taskIds = Collections.unmodifiableSet((taskIds == null) ? + new HashSet() : new HashSet<>(taskIds)); + this.firstStartMs = Math.max(0, firstStartMs); + this.lastStartMs = Math.max(0, lastStartMs); + this.firstEndMs = Math.max(0, firstEndMs); + this.lastEndMs = Math.max(0, lastEndMs); + } + + @JsonProperty + public Collection taskIds() { + return taskIds; + } + + @JsonProperty + public long firstStartMs() { + return firstStartMs; + } + + @JsonProperty + public long lastStartMs() { + return lastStartMs; + } + + @JsonProperty + public long firstEndMs() { + return firstEndMs; + } + + @JsonProperty + public long lastEndMs() { + return lastEndMs; + } + + /** + * Determine if this TaskRequest should return a particular task. + * + * @param taskId The task ID. + * @param startMs The task start time, or -1 if the task hasn't started. + * @param endMs The task end time, or -1 if the task hasn't ended. + * @return True if information about the task should be returned. + */ + public boolean matches(String taskId, long startMs, long endMs) { + if ((!taskIds.isEmpty()) && (!taskIds.contains(taskId))) { + return false; + } + if ((firstStartMs > 0) && (startMs < firstStartMs)) { + return false; + } + if ((lastStartMs > 0) && ((startMs < 0) || (startMs > lastStartMs))) { + return false; + } + if ((firstEndMs > 0) && (endMs < firstEndMs)) { + return false; + } + if ((lastEndMs > 0) && ((endMs < 0) || (endMs > lastEndMs))) { + return false; + } + return true; + } +} diff --git a/tools/src/main/java/org/apache/kafka/trogdor/rest/WorkerDone.java b/tools/src/main/java/org/apache/kafka/trogdor/rest/WorkerDone.java index e463ffc345159..500d3c6a0c24a 100644 --- a/tools/src/main/java/org/apache/kafka/trogdor/rest/WorkerDone.java +++ b/tools/src/main/java/org/apache/kafka/trogdor/rest/WorkerDone.java @@ -19,6 +19,8 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.NullNode; import org.apache.kafka.trogdor.task.TaskSpec; /** @@ -39,7 +41,7 @@ public class WorkerDone extends WorkerState { * The task status. The format will depend on the type of task that is * being run. */ - private final String status; + private final JsonNode status; /** * Empty if the task completed without error; the error message otherwise. @@ -50,12 +52,12 @@ public class WorkerDone extends WorkerState { public WorkerDone(@JsonProperty("spec") TaskSpec spec, @JsonProperty("startedMs") long startedMs, @JsonProperty("doneMs") long doneMs, - @JsonProperty("status") String status, + @JsonProperty("status") JsonNode status, @JsonProperty("error") String error) { super(spec); this.startedMs = startedMs; this.doneMs = doneMs; - this.status = status == null ? "" : status; + this.status = status == null ? NullNode.instance : status; this.error = error == null ? "" : error; } @@ -72,7 +74,7 @@ public long doneMs() { @JsonProperty @Override - public String status() { + public JsonNode status() { return status; } diff --git a/tools/src/main/java/org/apache/kafka/trogdor/rest/WorkerReceiving.java b/tools/src/main/java/org/apache/kafka/trogdor/rest/WorkerReceiving.java index d3e356555ba66..70687743f743e 100644 --- a/tools/src/main/java/org/apache/kafka/trogdor/rest/WorkerReceiving.java +++ b/tools/src/main/java/org/apache/kafka/trogdor/rest/WorkerReceiving.java @@ -19,6 +19,8 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.TextNode; import org.apache.kafka.trogdor.task.TaskSpec; /** @@ -30,4 +32,9 @@ public final class WorkerReceiving extends WorkerState { public WorkerReceiving(@JsonProperty("spec") TaskSpec spec) { super(spec); } + + @Override + public JsonNode status() { + return new TextNode("receiving"); + } } diff --git a/tools/src/main/java/org/apache/kafka/trogdor/rest/WorkerRunning.java b/tools/src/main/java/org/apache/kafka/trogdor/rest/WorkerRunning.java index e3b8d1932b2dd..af8ee88a1abcd 100644 --- a/tools/src/main/java/org/apache/kafka/trogdor/rest/WorkerRunning.java +++ b/tools/src/main/java/org/apache/kafka/trogdor/rest/WorkerRunning.java @@ -19,6 +19,8 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.NullNode; import org.apache.kafka.trogdor.task.TaskSpec; /** @@ -34,15 +36,15 @@ public class WorkerRunning extends WorkerState { * The task status. The format will depend on the type of task that is * being run. */ - private final String status; + private final JsonNode status; @JsonCreator public WorkerRunning(@JsonProperty("spec") TaskSpec spec, @JsonProperty("startedMs") long startedMs, - @JsonProperty("status") String status) { + @JsonProperty("status") JsonNode status) { super(spec); this.startedMs = startedMs; - this.status = status == null ? "" : status; + this.status = status == null ? NullNode.instance : status; } @JsonProperty @@ -53,7 +55,7 @@ public long startedMs() { @JsonProperty @Override - public String status() { + public JsonNode status() { return status; } diff --git a/tools/src/main/java/org/apache/kafka/trogdor/rest/WorkerStarting.java b/tools/src/main/java/org/apache/kafka/trogdor/rest/WorkerStarting.java index 3a766ea380890..b568ec1f8873c 100644 --- a/tools/src/main/java/org/apache/kafka/trogdor/rest/WorkerStarting.java +++ b/tools/src/main/java/org/apache/kafka/trogdor/rest/WorkerStarting.java @@ -19,6 +19,8 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.TextNode; import org.apache.kafka.trogdor.task.TaskSpec; /** @@ -29,4 +31,9 @@ public final class WorkerStarting extends WorkerState { public WorkerStarting(@JsonProperty("spec") TaskSpec spec) { super(spec); } + + @Override + public JsonNode status() { + return new TextNode("starting"); + } } diff --git a/tools/src/main/java/org/apache/kafka/trogdor/rest/WorkerState.java b/tools/src/main/java/org/apache/kafka/trogdor/rest/WorkerState.java index 6d7c687c33890..044d719f8944e 100644 --- a/tools/src/main/java/org/apache/kafka/trogdor/rest/WorkerState.java +++ b/tools/src/main/java/org/apache/kafka/trogdor/rest/WorkerState.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.databind.JsonNode; import org.apache.kafka.common.KafkaException; import org.apache.kafka.trogdor.task.TaskSpec; @@ -60,9 +61,7 @@ public long startedMs() { throw new KafkaException("invalid state"); } - public String status() { - throw new KafkaException("invalid state"); - } + public abstract JsonNode status(); public boolean running() { return false; diff --git a/tools/src/main/java/org/apache/kafka/trogdor/rest/WorkerStopping.java b/tools/src/main/java/org/apache/kafka/trogdor/rest/WorkerStopping.java index 777e5114edc45..9fbb3ff730683 100644 --- a/tools/src/main/java/org/apache/kafka/trogdor/rest/WorkerStopping.java +++ b/tools/src/main/java/org/apache/kafka/trogdor/rest/WorkerStopping.java @@ -19,6 +19,8 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.NullNode; import org.apache.kafka.trogdor.task.TaskSpec; /** @@ -34,15 +36,15 @@ public class WorkerStopping extends WorkerState { * The task status. The format will depend on the type of task that is * being run. */ - private final String status; + private final JsonNode status; @JsonCreator public WorkerStopping(@JsonProperty("spec") TaskSpec spec, @JsonProperty("startedMs") long startedMs, - @JsonProperty("status") String status) { + @JsonProperty("status") JsonNode status) { super(spec); this.startedMs = startedMs; - this.status = status == null ? "" : status; + this.status = status == null ? NullNode.instance : status; } @JsonProperty @@ -53,7 +55,7 @@ public long startedMs() { @JsonProperty @Override - public String status() { + public JsonNode status() { return status; } diff --git a/tools/src/main/java/org/apache/kafka/trogdor/task/AgentWorkerStatusTracker.java b/tools/src/main/java/org/apache/kafka/trogdor/task/AgentWorkerStatusTracker.java new file mode 100644 index 0000000000000..2ad8e4ee4aec7 --- /dev/null +++ b/tools/src/main/java/org/apache/kafka/trogdor/task/AgentWorkerStatusTracker.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.trogdor.task; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.NullNode; + +/** + * Tracks the status of a Trogdor worker. + */ +public class AgentWorkerStatusTracker implements WorkerStatusTracker { + private JsonNode status = NullNode.instance; + + @Override + public void update(JsonNode newStatus) { + JsonNode status = newStatus.deepCopy(); + synchronized (this) { + this.status = status; + } + } + + /** + * Retrieves the status. + */ + public synchronized JsonNode get() { + return status; + } +} diff --git a/tools/src/main/java/org/apache/kafka/trogdor/task/NoOpTaskWorker.java b/tools/src/main/java/org/apache/kafka/trogdor/task/NoOpTaskWorker.java index dfa80842260b4..77336d87059f3 100644 --- a/tools/src/main/java/org/apache/kafka/trogdor/task/NoOpTaskWorker.java +++ b/tools/src/main/java/org/apache/kafka/trogdor/task/NoOpTaskWorker.java @@ -17,30 +17,34 @@ package org.apache.kafka.trogdor.task; +import com.fasterxml.jackson.databind.node.TextNode; import org.apache.kafka.common.internals.KafkaFutureImpl; import org.apache.kafka.trogdor.common.Platform; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.concurrent.atomic.AtomicReference; - public class NoOpTaskWorker implements TaskWorker { private static final Logger log = LoggerFactory.getLogger(NoOpTaskWorker.class); private final String id; + private WorkerStatusTracker status; + public NoOpTaskWorker(String id) { this.id = id; } @Override - public void start(Platform platform, AtomicReference status, + public void start(Platform platform, WorkerStatusTracker status, KafkaFutureImpl errorFuture) throws Exception { log.info("{}: Activating NoOpTask.", id); + this.status = status; + this.status.update(new TextNode("active")); } @Override public void stop(Platform platform) throws Exception { log.info("{}: Deactivating NoOpTask.", id); + this.status.update(new TextNode("done")); } } diff --git a/tools/src/main/java/org/apache/kafka/trogdor/task/TaskSpec.java b/tools/src/main/java/org/apache/kafka/trogdor/task/TaskSpec.java index 84ed75a8c8ac9..af7a76f854119 100644 --- a/tools/src/main/java/org/apache/kafka/trogdor/task/TaskSpec.java +++ b/tools/src/main/java/org/apache/kafka/trogdor/task/TaskSpec.java @@ -21,6 +21,8 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; import org.apache.kafka.trogdor.common.JsonUtil; +import java.util.Collections; +import java.util.Map; import java.util.Objects; @@ -102,4 +104,8 @@ public final int hashCode() { public String toString() { return JsonUtil.toJsonString(this); } + + protected Map configOrEmptyMap(Map config) { + return (config == null) ? Collections.emptyMap() : config; + } } diff --git a/tools/src/main/java/org/apache/kafka/trogdor/task/TaskWorker.java b/tools/src/main/java/org/apache/kafka/trogdor/task/TaskWorker.java index 288eb9ce14763..042568f1a83c0 100644 --- a/tools/src/main/java/org/apache/kafka/trogdor/task/TaskWorker.java +++ b/tools/src/main/java/org/apache/kafka/trogdor/task/TaskWorker.java @@ -20,8 +20,6 @@ import org.apache.kafka.common.internals.KafkaFutureImpl; import org.apache.kafka.trogdor.common.Platform; -import java.util.concurrent.atomic.AtomicReference; - /** * The agent-side interface for implementing tasks. */ @@ -42,7 +40,7 @@ public interface TaskWorker { * * * @param platform The platform to use. - * @param status The current status string. The TaskWorker can update + * @param status The current status. The TaskWorker can update * this at any time to provide an updated status. * @param haltFuture A future which the worker should complete if it halts. * If it is completed with an empty string, that means the task @@ -53,7 +51,7 @@ public interface TaskWorker { * * @throws Exception If the TaskWorker failed to start. stop() will not be invoked. */ - void start(Platform platform, AtomicReference status, KafkaFutureImpl haltFuture) + void start(Platform platform, WorkerStatusTracker status, KafkaFutureImpl haltFuture) throws Exception; /** diff --git a/tools/src/main/java/org/apache/kafka/trogdor/task/WorkerStatusTracker.java b/tools/src/main/java/org/apache/kafka/trogdor/task/WorkerStatusTracker.java new file mode 100644 index 0000000000000..dfbc7ea6e446e --- /dev/null +++ b/tools/src/main/java/org/apache/kafka/trogdor/task/WorkerStatusTracker.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.trogdor.task; + +import com.fasterxml.jackson.databind.JsonNode; + +/** + * Tracks the status of a Trogdor worker. + */ +public interface WorkerStatusTracker { + /** + * Updates the status. + * + * @param status The new status. + */ + void update(JsonNode status); +} diff --git a/tools/src/main/java/org/apache/kafka/trogdor/workload/ConsumeBenchSpec.java b/tools/src/main/java/org/apache/kafka/trogdor/workload/ConsumeBenchSpec.java new file mode 100644 index 0000000000000..cef913bc01a5f --- /dev/null +++ b/tools/src/main/java/org/apache/kafka/trogdor/workload/ConsumeBenchSpec.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.trogdor.workload; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import org.apache.kafka.trogdor.common.Topology; +import org.apache.kafka.trogdor.task.TaskController; +import org.apache.kafka.trogdor.task.TaskSpec; +import org.apache.kafka.trogdor.task.TaskWorker; + +import java.util.Collections; +import java.util.Map; +import java.util.Set; + +/** + * The specification for a benchmark that produces messages to a set of topics. + */ +public class ConsumeBenchSpec extends TaskSpec { + + private final String consumerNode; + private final String bootstrapServers; + private final int targetMessagesPerSec; + private final int maxMessages; + private final Map consumerConf; + private final Map adminClientConf; + private final Map commonClientConf; + private final String topicRegex; + private final int startPartition; + private final int endPartition; + + + @JsonCreator + public ConsumeBenchSpec(@JsonProperty("startMs") long startMs, + @JsonProperty("durationMs") long durationMs, + @JsonProperty("consumerNode") String consumerNode, + @JsonProperty("bootstrapServers") String bootstrapServers, + @JsonProperty("targetMessagesPerSec") int targetMessagesPerSec, + @JsonProperty("maxMessages") int maxMessages, + @JsonProperty("consumerConf") Map consumerConf, + @JsonProperty("commonClientConf") Map commonClientConf, + @JsonProperty("adminClientConf") Map adminClientConf, + @JsonProperty("topicRegex") String topicRegex, + @JsonProperty("startPartition") int startPartition, + @JsonProperty("endPartition") int endPartition) { + super(startMs, durationMs); + this.consumerNode = (consumerNode == null) ? "" : consumerNode; + this.bootstrapServers = (bootstrapServers == null) ? "" : bootstrapServers; + this.targetMessagesPerSec = targetMessagesPerSec; + this.maxMessages = maxMessages; + this.consumerConf = configOrEmptyMap(consumerConf); + this.commonClientConf = configOrEmptyMap(commonClientConf); + this.adminClientConf = configOrEmptyMap(adminClientConf); + this.topicRegex = topicRegex; + this.startPartition = startPartition; + this.endPartition = endPartition; + } + + @JsonProperty + public String consumerNode() { + return consumerNode; + } + + @JsonProperty + public String bootstrapServers() { + return bootstrapServers; + } + + @JsonProperty + public int targetMessagesPerSec() { + return targetMessagesPerSec; + } + + @JsonProperty + public int maxMessages() { + return maxMessages; + } + + @JsonProperty + public Map consumerConf() { + return consumerConf; + } + + @JsonProperty + public Map commonClientConf() { + return commonClientConf; + } + + @JsonProperty + public Map adminClientConf() { + return adminClientConf; + } + + @JsonProperty + public String topicRegex() { + return topicRegex; + } + + @JsonProperty + public int startPartition() { + return startPartition; + } + + @JsonProperty + public int endPartition() { + return endPartition; + } + + @Override + public TaskController newController(String id) { + return new TaskController() { + @Override + public Set targetNodes(Topology topology) { + return Collections.singleton(consumerNode); + } + }; + } + + @Override + public TaskWorker newTaskWorker(String id) { + return new ConsumeBenchWorker(id, this); + } +} diff --git a/tools/src/main/java/org/apache/kafka/trogdor/workload/ConsumeBenchWorker.java b/tools/src/main/java/org/apache/kafka/trogdor/workload/ConsumeBenchWorker.java new file mode 100644 index 0000000000000..5c74d906601a4 --- /dev/null +++ b/tools/src/main/java/org/apache/kafka/trogdor/workload/ConsumeBenchWorker.java @@ -0,0 +1,298 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.trogdor.workload; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.config.ConfigException; +import org.apache.kafka.common.internals.KafkaFutureImpl; +import org.apache.kafka.common.serialization.ByteArrayDeserializer; +import org.apache.kafka.common.utils.Time; +import org.apache.kafka.common.utils.Utils; +import org.apache.kafka.trogdor.common.JsonUtil; +import org.apache.kafka.trogdor.common.Platform; +import org.apache.kafka.trogdor.common.ThreadUtils; +import org.apache.kafka.trogdor.common.WorkerUtils; +import org.apache.kafka.trogdor.task.WorkerStatusTracker; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.kafka.trogdor.task.TaskWorker; + +import java.util.Collection; +import java.util.Properties; +import java.util.concurrent.Callable; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +public class ConsumeBenchWorker implements TaskWorker { + private static final Logger log = LoggerFactory.getLogger(ConsumeBenchWorker.class); + + private static final int THROTTLE_PERIOD_MS = 100; + + private final String id; + private final ConsumeBenchSpec spec; + private final AtomicBoolean running = new AtomicBoolean(false); + private ScheduledExecutorService executor; + private WorkerStatusTracker status; + private KafkaFutureImpl doneFuture; + private KafkaConsumer consumer; + + public ConsumeBenchWorker(String id, ConsumeBenchSpec spec) { + this.id = id; + this.spec = spec; + } + + @Override + public void start(Platform platform, WorkerStatusTracker status, + KafkaFutureImpl doneFuture) throws Exception { + if (!running.compareAndSet(false, true)) { + throw new IllegalStateException("ConsumeBenchWorker is already running."); + } + log.info("{}: Activating ConsumeBenchWorker with {}", id, spec); + this.executor = Executors.newScheduledThreadPool( + 2, ThreadUtils.createThreadFactory("ConsumeBenchWorkerThread%d", false)); + this.status = status; + this.doneFuture = doneFuture; + executor.submit(new Prepare()); + } + + public class Prepare implements Runnable { + @Override + public void run() { + try { + // find topics to consume from based on provided topic regular expression + if (spec.topicRegex() == null) { + throw new ConfigException( + "Must provide topic name or regular expression to match existing topics."); + } + Collection topicPartitions = + WorkerUtils.getMatchingTopicPartitions( + log, spec.bootstrapServers(), + spec.commonClientConf(), spec.adminClientConf(), + spec.topicRegex(), spec.startPartition(), spec.endPartition()); + log.info("Will consume from {}", topicPartitions); + + executor.submit(new ConsumeMessages(topicPartitions)); + } catch (Throwable e) { + WorkerUtils.abort(log, "Prepare", e, doneFuture); + } + } + } + + public class ConsumeMessages implements Callable { + private final Histogram latencyHistogram; + private final Histogram messageSizeHistogram; + private final Future statusUpdaterFuture; + private final Throttle throttle; + + ConsumeMessages(Collection topicPartitions) { + this.latencyHistogram = new Histogram(5000); + this.messageSizeHistogram = new Histogram(2 * 1024 * 1024); + this.statusUpdaterFuture = executor.scheduleAtFixedRate( + new StatusUpdater(latencyHistogram, messageSizeHistogram), 1, 1, TimeUnit.MINUTES); + Properties props = new Properties(); + props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, spec.bootstrapServers()); + props.put(ConsumerConfig.CLIENT_ID_CONFIG, "consumer." + id); + props.put(ConsumerConfig.GROUP_ID_CONFIG, "consumer-group-1"); + props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + props.put(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG, 100000); + // these defaults maybe over-written by the user-specified commonClientConf or + // consumerConf + WorkerUtils.addConfigsToProperties(props, spec.commonClientConf(), spec.consumerConf()); + consumer = new KafkaConsumer<>(props, new ByteArrayDeserializer(), + new ByteArrayDeserializer()); + consumer.assign(topicPartitions); + int perPeriod = WorkerUtils.perSecToPerPeriod( + spec.targetMessagesPerSec(), THROTTLE_PERIOD_MS); + this.throttle = new Throttle(perPeriod, THROTTLE_PERIOD_MS); + } + + @Override + public Void call() throws Exception { + long messagesConsumed = 0; + long bytesConsumed = 0; + long startTimeMs = Time.SYSTEM.milliseconds(); + long startBatchMs = startTimeMs; + try { + while (messagesConsumed < spec.maxMessages()) { + ConsumerRecords records = consumer.poll(50); + if (records.isEmpty()) { + continue; + } + long endBatchMs = Time.SYSTEM.milliseconds(); + long elapsedBatchMs = endBatchMs - startBatchMs; + for (ConsumerRecord record : records) { + messagesConsumed++; + long messageBytes = 0; + if (record.key() != null) { + messageBytes += record.serializedKeySize(); + } + if (record.value() != null) { + messageBytes += record.serializedValueSize(); + } + latencyHistogram.add(elapsedBatchMs); + messageSizeHistogram.add(messageBytes); + bytesConsumed += messageBytes; + throttle.increment(); + } + startBatchMs = Time.SYSTEM.milliseconds(); + } + } catch (Exception e) { + WorkerUtils.abort(log, "ConsumeRecords", e, doneFuture); + } finally { + statusUpdaterFuture.cancel(false); + StatusData statusData = + new StatusUpdater(latencyHistogram, messageSizeHistogram).update(); + long curTimeMs = Time.SYSTEM.milliseconds(); + log.info("Consumed total number of messages={}, bytes={} in {} ms. status: {}", + messagesConsumed, bytesConsumed, curTimeMs - startTimeMs, statusData); + } + doneFuture.complete(""); + return null; + } + } + + public class StatusUpdater implements Runnable { + private final Histogram latencyHistogram; + private final Histogram messageSizeHistogram; + + StatusUpdater(Histogram latencyHistogram, Histogram messageSizeHistogram) { + this.latencyHistogram = latencyHistogram; + this.messageSizeHistogram = messageSizeHistogram; + } + + @Override + public void run() { + try { + update(); + } catch (Exception e) { + WorkerUtils.abort(log, "StatusUpdater", e, doneFuture); + } + } + + StatusData update() { + Histogram.Summary latSummary = latencyHistogram.summarize(StatusData.PERCENTILES); + Histogram.Summary msgSummary = messageSizeHistogram.summarize(StatusData.PERCENTILES); + StatusData statusData = new StatusData( + latSummary.numSamples(), + (long) (msgSummary.numSamples() * msgSummary.average()), + (long) msgSummary.average(), + latSummary.average(), + latSummary.percentiles().get(0).value(), + latSummary.percentiles().get(1).value(), + latSummary.percentiles().get(2).value()); + status.update(JsonUtil.JSON_SERDE.valueToTree(statusData)); + log.info("Status={}", JsonUtil.toJsonString(statusData)); + return statusData; + } + } + + public static class StatusData { + private final long totalMessagesReceived; + private final long totalBytesReceived; + private final long averageMessageSizeBytes; + private final float averageLatencyMs; + private final int p50LatencyMs; + private final int p95LatencyMs; + private final int p99LatencyMs; + + /** + * The percentiles to use when calculating the histogram data. + * These should match up with the p50LatencyMs, p95LatencyMs, etc. fields. + */ + final static float[] PERCENTILES = {0.5f, 0.95f, 0.99f}; + + @JsonCreator + StatusData(@JsonProperty("totalMessagesReceived") long totalMessagesReceived, + @JsonProperty("totalBytesReceived") long totalBytesReceived, + @JsonProperty("averageMessageSizeBytes") long averageMessageSizeBytes, + @JsonProperty("averageLatencyMs") float averageLatencyMs, + @JsonProperty("p50LatencyMs") int p50latencyMs, + @JsonProperty("p95LatencyMs") int p95latencyMs, + @JsonProperty("p99LatencyMs") int p99latencyMs) { + this.totalMessagesReceived = totalMessagesReceived; + this.totalBytesReceived = totalBytesReceived; + this.averageMessageSizeBytes = averageMessageSizeBytes; + this.averageLatencyMs = averageLatencyMs; + this.p50LatencyMs = p50latencyMs; + this.p95LatencyMs = p95latencyMs; + this.p99LatencyMs = p99latencyMs; + } + + @JsonProperty + public long totalMessagesReceived() { + return totalMessagesReceived; + } + + @JsonProperty + public long totalBytesReceived() { + return totalBytesReceived; + } + + @JsonProperty + public long averageMessageSizeBytes() { + return averageMessageSizeBytes; + } + + @JsonProperty + public float averageLatencyMs() { + return averageLatencyMs; + } + + @JsonProperty + public int p50LatencyMs() { + return p50LatencyMs; + } + + @JsonProperty + public int p95LatencyMs() { + return p95LatencyMs; + } + + @JsonProperty + public int p99LatencyMs() { + return p99LatencyMs; + } + } + + @Override + public void stop(Platform platform) throws Exception { + if (!running.compareAndSet(true, false)) { + throw new IllegalStateException("ConsumeBenchWorker is not running."); + } + log.info("{}: Deactivating ConsumeBenchWorker.", id); + doneFuture.complete(""); + executor.shutdownNow(); + executor.awaitTermination(1, TimeUnit.DAYS); + Utils.closeQuietly(consumer, "consumer"); + this.consumer = null; + this.executor = null; + this.status = null; + this.doneFuture = null; + } + +} diff --git a/tools/src/main/java/org/apache/kafka/trogdor/workload/NullPayloadGenerator.java b/tools/src/main/java/org/apache/kafka/trogdor/workload/NullPayloadGenerator.java new file mode 100644 index 0000000000000..e9799c0481e02 --- /dev/null +++ b/tools/src/main/java/org/apache/kafka/trogdor/workload/NullPayloadGenerator.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.trogdor.workload; + +import com.fasterxml.jackson.annotation.JsonCreator; + +/** + * A PayloadGenerator which always generates a null payload. + */ +public class NullPayloadGenerator implements PayloadGenerator { + @JsonCreator + public NullPayloadGenerator() { + } + + @Override + public byte[] generate(long position) { + return null; + } +} diff --git a/tools/src/main/java/org/apache/kafka/trogdor/workload/PayloadGenerator.java b/tools/src/main/java/org/apache/kafka/trogdor/workload/PayloadGenerator.java index 4895f217672b7..3c574bac810df 100644 --- a/tools/src/main/java/org/apache/kafka/trogdor/workload/PayloadGenerator.java +++ b/tools/src/main/java/org/apache/kafka/trogdor/workload/PayloadGenerator.java @@ -33,7 +33,8 @@ @JsonSubTypes(value = { @JsonSubTypes.Type(value = ConstantPayloadGenerator.class, name = "constant"), @JsonSubTypes.Type(value = SequentialPayloadGenerator.class, name = "sequential"), - @JsonSubTypes.Type(value = UniformRandomPayloadGenerator.class, name = "uniformRandom") + @JsonSubTypes.Type(value = UniformRandomPayloadGenerator.class, name = "uniformRandom"), + @JsonSubTypes.Type(value = NullPayloadGenerator.class, name = "null") }) public interface PayloadGenerator { /** diff --git a/tools/src/main/java/org/apache/kafka/trogdor/workload/ProduceBenchSpec.java b/tools/src/main/java/org/apache/kafka/trogdor/workload/ProduceBenchSpec.java index 7b1bedd183036..ec6e3096469b6 100644 --- a/tools/src/main/java/org/apache/kafka/trogdor/workload/ProduceBenchSpec.java +++ b/tools/src/main/java/org/apache/kafka/trogdor/workload/ProduceBenchSpec.java @@ -26,7 +26,6 @@ import java.util.Collections; import java.util.Map; -import java.util.TreeMap; import java.util.Set; /** @@ -45,6 +44,8 @@ public class ProduceBenchSpec extends TaskSpec { private final PayloadGenerator keyGenerator; private final PayloadGenerator valueGenerator; private final Map producerConf; + private final Map adminClientConf; + private final Map commonClientConf; private final int totalTopics; private final int activeTopics; private final String topicPrefix; @@ -61,6 +62,8 @@ public ProduceBenchSpec(@JsonProperty("startMs") long startMs, @JsonProperty("keyGenerator") PayloadGenerator keyGenerator, @JsonProperty("valueGenerator") PayloadGenerator valueGenerator, @JsonProperty("producerConf") Map producerConf, + @JsonProperty("commonClientConf") Map commonClientConf, + @JsonProperty("adminClientConf") Map adminClientConf, @JsonProperty("totalTopics") int totalTopics, @JsonProperty("activeTopics") int activeTopics, @JsonProperty("topicPrefix") String topicPrefix, @@ -75,7 +78,9 @@ public ProduceBenchSpec(@JsonProperty("startMs") long startMs, new SequentialPayloadGenerator(4, 0) : keyGenerator; this.valueGenerator = valueGenerator == null ? new ConstantPayloadGenerator(512, new byte[0]) : valueGenerator; - this.producerConf = (producerConf == null) ? new TreeMap() : producerConf; + this.producerConf = configOrEmptyMap(producerConf); + this.commonClientConf = configOrEmptyMap(commonClientConf); + this.adminClientConf = configOrEmptyMap(adminClientConf); this.totalTopics = totalTopics; this.activeTopics = activeTopics; this.topicPrefix = (topicPrefix == null) ? DEFAULT_TOPIC_PREFIX : topicPrefix; @@ -120,6 +125,16 @@ public Map producerConf() { return producerConf; } + @JsonProperty + public Map commonClientConf() { + return commonClientConf; + } + + @JsonProperty + public Map adminClientConf() { + return adminClientConf; + } + @JsonProperty public int totalTopics() { return totalTopics; diff --git a/tools/src/main/java/org/apache/kafka/trogdor/workload/ProduceBenchWorker.java b/tools/src/main/java/org/apache/kafka/trogdor/workload/ProduceBenchWorker.java index e291bae408842..4c3095f0fbd50 100644 --- a/tools/src/main/java/org/apache/kafka/trogdor/workload/ProduceBenchWorker.java +++ b/tools/src/main/java/org/apache/kafka/trogdor/workload/ProduceBenchWorker.java @@ -19,6 +19,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.node.TextNode; import org.apache.kafka.clients.admin.NewTopic; import org.apache.kafka.clients.producer.Callback; import org.apache.kafka.clients.producer.KafkaProducer; @@ -34,6 +35,7 @@ import org.apache.kafka.trogdor.common.ThreadUtils; import org.apache.kafka.trogdor.common.WorkerUtils; import org.apache.kafka.trogdor.task.TaskWorker; +import org.apache.kafka.trogdor.task.WorkerStatusTracker; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -46,7 +48,6 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicReference; public class ProduceBenchWorker implements TaskWorker { private static final Logger log = LoggerFactory.getLogger(ProduceBenchWorker.class); @@ -61,7 +62,7 @@ public class ProduceBenchWorker implements TaskWorker { private ScheduledExecutorService executor; - private AtomicReference status; + private WorkerStatusTracker status; private KafkaFutureImpl doneFuture; @@ -81,7 +82,7 @@ public ProduceBenchWorker(String id, ProduceBenchSpec spec) { } @Override - public void start(Platform platform, AtomicReference status, + public void start(Platform platform, WorkerStatusTracker status, KafkaFutureImpl doneFuture) throws Exception { if (!running.compareAndSet(false, true)) { throw new IllegalStateException("ProducerBenchWorker is already running."); @@ -109,10 +110,13 @@ public void run() { Map newTopics = new HashMap<>(); for (int i = 0; i < spec.totalTopics(); i++) { String name = topicIndexToName(i); - newTopics.put(name, new NewTopic(name, spec.numPartitions(), spec.replicationFactor())); + newTopics.put(name, new NewTopic(name, spec.numPartitions(), + spec.replicationFactor())); } - WorkerUtils.createTopics(log, spec.bootstrapServers(), newTopics, false); - + status.update(new TextNode("Creating " + spec.totalTopics() + " topic(s)")); + WorkerUtils.createTopics(log, spec.bootstrapServers(), spec.commonClientConf(), + spec.adminClientConf(), newTopics, false); + status.update(new TextNode("Created " + spec.totalTopics() + " topic(s)")); executor.submit(new SendRecords()); } catch (Throwable e) { WorkerUtils.abort(log, "Prepare", e, doneFuture); @@ -179,12 +183,12 @@ public class SendRecords implements Callable { this.histogram = new Histogram(5000); int perPeriod = WorkerUtils.perSecToPerPeriod(spec.targetMessagesPerSec(), THROTTLE_PERIOD_MS); this.statusUpdaterFuture = executor.scheduleWithFixedDelay( - new StatusUpdater(histogram), 1, 1, TimeUnit.MINUTES); + new StatusUpdater(histogram), 30, 30, TimeUnit.SECONDS); Properties props = new Properties(); props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, spec.bootstrapServers()); - for (Map.Entry entry : spec.producerConf().entrySet()) { - props.setProperty(entry.getKey(), entry.getValue()); - } + // add common client configs to producer properties, and then user-specified producer + // configs + WorkerUtils.addConfigsToProperties(props, spec.commonClientConf(), spec.producerConf()); this.producer = new KafkaProducer<>(props, new ByteArraySerializer(), new ByteArraySerializer()); this.keys = new PayloadIterator(spec.keyGenerator()); this.values = new PayloadIterator(spec.valueGenerator()); @@ -216,10 +220,10 @@ public Void call() throws Exception { WorkerUtils.abort(log, "SendRecords", e, doneFuture); } finally { statusUpdaterFuture.cancel(false); - new StatusUpdater(histogram).run(); + StatusData statusData = new StatusUpdater(histogram).update(); long curTimeMs = Time.SYSTEM.milliseconds(); log.info("Sent {} total record(s) in {} ms. status: {}", - histogram.summarize().numSamples(), curTimeMs - startTimeMs, status.get()); + histogram.summarize().numSamples(), curTimeMs - startTimeMs, statusData); } doneFuture.complete(""); return null; @@ -232,46 +236,54 @@ void recordDuration(long durationMs) { public class StatusUpdater implements Runnable { private final Histogram histogram; - private final float[] percentiles; StatusUpdater(Histogram histogram) { this.histogram = histogram; - this.percentiles = new float[] {0.50f, 0.95f, 0.99f}; } @Override public void run() { try { - Histogram.Summary summary = histogram.summarize(percentiles); - StatusData statusData = new StatusData(summary.numSamples(), summary.average(), - summary.percentiles().get(0).value(), - summary.percentiles().get(1).value(), - summary.percentiles().get(2).value()); - String statusDataString = JsonUtil.toJsonString(statusData); - status.set(statusDataString); + update(); } catch (Exception e) { WorkerUtils.abort(log, "StatusUpdater", e, doneFuture); } } + + StatusData update() { + Histogram.Summary summary = histogram.summarize(StatusData.PERCENTILES); + StatusData statusData = new StatusData(summary.numSamples(), summary.average(), + summary.percentiles().get(0).value(), + summary.percentiles().get(1).value(), + summary.percentiles().get(2).value()); + status.update(JsonUtil.JSON_SERDE.valueToTree(statusData)); + return statusData; + } } public static class StatusData { private final long totalSent; private final float averageLatencyMs; private final int p50LatencyMs; - private final int p90LatencyMs; + private final int p95LatencyMs; private final int p99LatencyMs; + /** + * The percentiles to use when calculating the histogram data. + * These should match up with the p50LatencyMs, p95LatencyMs, etc. fields. + */ + final static float[] PERCENTILES = {0.5f, 0.95f, 0.99f}; + @JsonCreator StatusData(@JsonProperty("totalSent") long totalSent, @JsonProperty("averageLatencyMs") float averageLatencyMs, @JsonProperty("p50LatencyMs") int p50latencyMs, - @JsonProperty("p90LatencyMs") int p90latencyMs, + @JsonProperty("p95LatencyMs") int p95latencyMs, @JsonProperty("p99LatencyMs") int p99latencyMs) { this.totalSent = totalSent; this.averageLatencyMs = averageLatencyMs; this.p50LatencyMs = p50latencyMs; - this.p90LatencyMs = p90latencyMs; + this.p95LatencyMs = p95latencyMs; this.p99LatencyMs = p99latencyMs; } @@ -291,8 +303,8 @@ public int p50LatencyMs() { } @JsonProperty - public int p90LatencyMs() { - return p90LatencyMs; + public int p95LatencyMs() { + return p95LatencyMs; } @JsonProperty diff --git a/tools/src/main/java/org/apache/kafka/trogdor/workload/RoundTripWorker.java b/tools/src/main/java/org/apache/kafka/trogdor/workload/RoundTripWorker.java index a05785c04e0d2..12b0c08a70041 100644 --- a/tools/src/main/java/org/apache/kafka/trogdor/workload/RoundTripWorker.java +++ b/tools/src/main/java/org/apache/kafka/trogdor/workload/RoundTripWorker.java @@ -39,6 +39,7 @@ import org.apache.kafka.trogdor.common.ThreadUtils; import org.apache.kafka.trogdor.common.WorkerUtils; import org.apache.kafka.trogdor.task.TaskWorker; +import org.apache.kafka.trogdor.task.WorkerStatusTracker; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -55,7 +56,6 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicReference; public class RoundTripWorker implements TaskWorker { private static final int THROTTLE_PERIOD_MS = 100; @@ -98,7 +98,7 @@ public RoundTripWorker(String id, RoundTripWorkloadSpec spec) { } @Override - public void start(Platform platform, AtomicReference status, + public void start(Platform platform, WorkerStatusTracker status, KafkaFutureImpl doneFuture) throws Exception { if (!running.compareAndSet(false, true)) { throw new IllegalStateException("RoundTripWorker is already running."); @@ -124,7 +124,7 @@ public void run() { throw new ConfigException("Invalid null or empty partitionAssignments."); } WorkerUtils.createTopics( - log, spec.bootstrapServers(), + log, spec.bootstrapServers(), spec.commonClientConf(), spec.adminClientConf(), Collections.singletonMap(TOPIC_NAME, new NewTopic(TOPIC_NAME, spec.partitionAssignments())), true); @@ -184,6 +184,8 @@ class ProducerRunnable implements Runnable { props.put(ProducerConfig.CLIENT_ID_CONFIG, "producer." + id); props.put(ProducerConfig.ACKS_CONFIG, "all"); props.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, 105000); + // user may over-write the defaults with common client config and producer config + WorkerUtils.addConfigsToProperties(props, spec.commonClientConf(), spec.producerConf()); producer = new KafkaProducer<>(props, new ByteArraySerializer(), new ByteArraySerializer()); int perPeriod = WorkerUtils. @@ -275,6 +277,8 @@ class ConsumerRunnable implements Runnable { props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); props.put(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG, 105000); props.put(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG, 100000); + // user may over-write the defaults with common client config and consumer config + WorkerUtils.addConfigsToProperties(props, spec.commonClientConf(), spec.consumerConf()); consumer = new KafkaConsumer<>(props, new ByteArrayDeserializer(), new ByteArrayDeserializer()); consumer.subscribe(Collections.singleton(TOPIC_NAME)); diff --git a/tools/src/main/java/org/apache/kafka/trogdor/workload/RoundTripWorkloadSpec.java b/tools/src/main/java/org/apache/kafka/trogdor/workload/RoundTripWorkloadSpec.java index 00bd833c05265..3d0e3ef2e6b4a 100644 --- a/tools/src/main/java/org/apache/kafka/trogdor/workload/RoundTripWorkloadSpec.java +++ b/tools/src/main/java/org/apache/kafka/trogdor/workload/RoundTripWorkloadSpec.java @@ -26,6 +26,7 @@ import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.NavigableMap; import java.util.Set; import java.util.TreeMap; @@ -41,12 +42,20 @@ public class RoundTripWorkloadSpec extends TaskSpec { private final NavigableMap> partitionAssignments; private final PayloadGenerator valueGenerator; private final int maxMessages; + private final Map commonClientConf; + private final Map producerConf; + private final Map consumerConf; + private final Map adminClientConf; @JsonCreator public RoundTripWorkloadSpec(@JsonProperty("startMs") long startMs, @JsonProperty("durationMs") long durationMs, @JsonProperty("clientNode") String clientNode, @JsonProperty("bootstrapServers") String bootstrapServers, + @JsonProperty("commonClientConf") Map commonClientConf, + @JsonProperty("adminClientConf") Map adminClientConf, + @JsonProperty("consumerConf") Map consumerConf, + @JsonProperty("producerConf") Map producerConf, @JsonProperty("targetMessagesPerSec") int targetMessagesPerSec, @JsonProperty("partitionAssignments") NavigableMap> partitionAssignments, @JsonProperty("valueGenerator") PayloadGenerator valueGenerator, @@ -60,6 +69,10 @@ public RoundTripWorkloadSpec(@JsonProperty("startMs") long startMs, this.valueGenerator = valueGenerator == null ? new UniformRandomPayloadGenerator(32, 123, 10) : valueGenerator; this.maxMessages = maxMessages; + this.commonClientConf = configOrEmptyMap(commonClientConf); + this.adminClientConf = configOrEmptyMap(adminClientConf); + this.producerConf = configOrEmptyMap(producerConf); + this.consumerConf = configOrEmptyMap(consumerConf); } @JsonProperty @@ -92,6 +105,26 @@ public int maxMessages() { return maxMessages; } + @JsonProperty + public Map commonClientConf() { + return commonClientConf; + } + + @JsonProperty + public Map adminClientConf() { + return adminClientConf; + } + + @JsonProperty + public Map producerConf() { + return producerConf; + } + + @JsonProperty + public Map consumerConf() { + return consumerConf; + } + @Override public TaskController newController(String id) { return new TaskController() { diff --git a/tools/src/test/java/org/apache/kafka/trogdor/agent/AgentTest.java b/tools/src/test/java/org/apache/kafka/trogdor/agent/AgentTest.java index 30d13b55cb930..61de5c98797cf 100644 --- a/tools/src/test/java/org/apache/kafka/trogdor/agent/AgentTest.java +++ b/tools/src/test/java/org/apache/kafka/trogdor/agent/AgentTest.java @@ -17,6 +17,7 @@ package org.apache.kafka.trogdor.agent; +import com.fasterxml.jackson.databind.node.TextNode; import org.apache.kafka.common.utils.MockScheduler; import org.apache.kafka.common.utils.MockTime; import org.apache.kafka.common.utils.Scheduler; @@ -122,7 +123,7 @@ public void testAgentCreateWorkers() throws Exception { CreateWorkerResponse response = client.createWorker(new CreateWorkerRequest("foo", fooSpec)); assertEquals(fooSpec.toString(), response.spec().toString()); new ExpectedTasks().addTask(new ExpectedTaskBuilder("foo"). - workerState(new WorkerRunning(fooSpec, 0, "")). + workerState(new WorkerRunning(fooSpec, 0, new TextNode("active"))). build()). waitFor(client); @@ -131,10 +132,10 @@ public void testAgentCreateWorkers() throws Exception { client.createWorker(new CreateWorkerRequest("bar", barSpec)); new ExpectedTasks(). addTask(new ExpectedTaskBuilder("foo"). - workerState(new WorkerRunning(fooSpec, 0, "")). + workerState(new WorkerRunning(fooSpec, 0, new TextNode("active"))). build()). addTask(new ExpectedTaskBuilder("bar"). - workerState(new WorkerRunning(barSpec, 0, "")). + workerState(new WorkerRunning(barSpec, 0, new TextNode("active"))). build()). waitFor(client); @@ -142,13 +143,13 @@ public void testAgentCreateWorkers() throws Exception { client.createWorker(new CreateWorkerRequest("baz", bazSpec)); new ExpectedTasks(). addTask(new ExpectedTaskBuilder("foo"). - workerState(new WorkerRunning(fooSpec, 0, "")). + workerState(new WorkerRunning(fooSpec, 0, new TextNode("active"))). build()). addTask(new ExpectedTaskBuilder("bar"). - workerState(new WorkerRunning(barSpec, 0, "")). + workerState(new WorkerRunning(barSpec, 0, new TextNode("active"))). build()). addTask(new ExpectedTaskBuilder("baz"). - workerState(new WorkerRunning(bazSpec, 0, "")). + workerState(new WorkerRunning(bazSpec, 0, new TextNode("active"))). build()). waitFor(client); @@ -169,7 +170,7 @@ public void testAgentFinishesTasks() throws Exception { client.createWorker(new CreateWorkerRequest("foo", fooSpec)); new ExpectedTasks(). addTask(new ExpectedTaskBuilder("foo"). - workerState(new WorkerRunning(fooSpec, 0, "")). + workerState(new WorkerRunning(fooSpec, 0, new TextNode("active"))). build()). waitFor(client); @@ -179,10 +180,10 @@ public void testAgentFinishesTasks() throws Exception { client.createWorker(new CreateWorkerRequest("bar", barSpec)); new ExpectedTasks(). addTask(new ExpectedTaskBuilder("foo"). - workerState(new WorkerRunning(fooSpec, 0, "")). + workerState(new WorkerRunning(fooSpec, 0, new TextNode("active"))). build()). addTask(new ExpectedTaskBuilder("bar"). - workerState(new WorkerRunning(barSpec, 1, "")). + workerState(new WorkerRunning(barSpec, 1, new TextNode("active"))). build()). waitFor(client); @@ -190,10 +191,10 @@ public void testAgentFinishesTasks() throws Exception { new ExpectedTasks(). addTask(new ExpectedTaskBuilder("foo"). - workerState(new WorkerDone(fooSpec, 0, 2, "", "")). + workerState(new WorkerDone(fooSpec, 0, 2, new TextNode("done"), "")). build()). addTask(new ExpectedTaskBuilder("bar"). - workerState(new WorkerRunning(barSpec, 1, "")). + workerState(new WorkerRunning(barSpec, 1, new TextNode("active"))). build()). waitFor(client); @@ -201,10 +202,10 @@ public void testAgentFinishesTasks() throws Exception { client.stopWorker(new StopWorkerRequest("bar")); new ExpectedTasks(). addTask(new ExpectedTaskBuilder("foo"). - workerState(new WorkerDone(fooSpec, 0, 2, "", "")). + workerState(new WorkerDone(fooSpec, 0, 2, new TextNode("done"), "")). build()). addTask(new ExpectedTaskBuilder("bar"). - workerState(new WorkerDone(barSpec, 1, 7, "", "")). + workerState(new WorkerDone(barSpec, 1, 7, new TextNode("done"), "")). build()). waitFor(client); @@ -221,34 +222,40 @@ public void testWorkerCompletions() throws Exception { maxTries(10).target("localhost", agent.port()).build(); new ExpectedTasks().waitFor(client); - SampleTaskSpec fooSpec = new SampleTaskSpec(0, 900000, 1, ""); + SampleTaskSpec fooSpec = new SampleTaskSpec(0, 900000, + Collections.singletonMap("node01", 1L), ""); client.createWorker(new CreateWorkerRequest("foo", fooSpec)); new ExpectedTasks(). addTask(new ExpectedTaskBuilder("foo"). - workerState(new WorkerRunning(fooSpec, 0, "")). + workerState(new WorkerRunning(fooSpec, 0, new TextNode("active"))). build()). waitFor(client); - SampleTaskSpec barSpec = new SampleTaskSpec(0, 900000, 2, "baz"); + SampleTaskSpec barSpec = new SampleTaskSpec(0, 900000, + Collections.singletonMap("node01", 2L), "baz"); client.createWorker(new CreateWorkerRequest("bar", barSpec)); time.sleep(1); new ExpectedTasks(). addTask(new ExpectedTaskBuilder("foo"). - workerState(new WorkerDone(fooSpec, 0, 1, "", "")). + workerState(new WorkerDone(fooSpec, 0, 1, + new TextNode("halted"), "")). build()). addTask(new ExpectedTaskBuilder("bar"). - workerState(new WorkerRunning(barSpec, 0, "")). + workerState(new WorkerRunning(barSpec, 0, + new TextNode("active"))). build()). waitFor(client); time.sleep(1); new ExpectedTasks(). addTask(new ExpectedTaskBuilder("foo"). - workerState(new WorkerDone(fooSpec, 0, 1, "", "")). + workerState(new WorkerDone(fooSpec, 0, 1, + new TextNode("halted"), "")). build()). addTask(new ExpectedTaskBuilder("bar"). - workerState(new WorkerDone(barSpec, 0, 2, "", "baz")). + workerState(new WorkerDone(barSpec, 0, 2, + new TextNode("halted"), "baz")). build()). waitFor(client); } @@ -289,7 +296,7 @@ public void testKiboshFaults() throws Exception { client.createWorker(new CreateWorkerRequest("foo", fooSpec)); new ExpectedTasks(). addTask(new ExpectedTaskBuilder("foo"). - workerState(new WorkerRunning(fooSpec, 0, "")). + workerState(new WorkerRunning(fooSpec, 0, new TextNode("Added fault foo"))). build()). waitFor(client); Assert.assertEquals(new KiboshControlFile(Collections.singletonList( @@ -299,9 +306,9 @@ public void testKiboshFaults() throws Exception { client.createWorker(new CreateWorkerRequest("bar", barSpec)); new ExpectedTasks(). addTask(new ExpectedTaskBuilder("foo"). - workerState(new WorkerRunning(fooSpec, 0, "")).build()). + workerState(new WorkerRunning(fooSpec, 0, new TextNode("Added fault foo"))).build()). addTask(new ExpectedTaskBuilder("bar"). - workerState(new WorkerRunning(barSpec, 0, "")).build()). + workerState(new WorkerRunning(barSpec, 0, new TextNode("Added fault bar"))).build()). waitFor(client); Assert.assertEquals(new KiboshControlFile(new ArrayList() {{ add(new KiboshFilesUnreadableFaultSpec("/foo", 123)); @@ -311,9 +318,9 @@ public void testKiboshFaults() throws Exception { client.stopWorker(new StopWorkerRequest("foo")); new ExpectedTasks(). addTask(new ExpectedTaskBuilder("foo"). - workerState(new WorkerDone(fooSpec, 0, 1, "", "")).build()). + workerState(new WorkerDone(fooSpec, 0, 1, new TextNode("Removed fault foo"), "")).build()). addTask(new ExpectedTaskBuilder("bar"). - workerState(new WorkerRunning(barSpec, 0, "")).build()). + workerState(new WorkerRunning(barSpec, 0, new TextNode("Added fault bar"))).build()). waitFor(client); Assert.assertEquals(new KiboshControlFile(Collections.singletonList( new KiboshFilesUnreadableFaultSpec("/bar", 456))), mockKibosh.read()); diff --git a/tools/src/test/java/org/apache/kafka/trogdor/common/ExpectedTasks.java b/tools/src/test/java/org/apache/kafka/trogdor/common/ExpectedTasks.java index f72779f844bf4..617bf34bcd9a8 100644 --- a/tools/src/test/java/org/apache/kafka/trogdor/common/ExpectedTasks.java +++ b/tools/src/test/java/org/apache/kafka/trogdor/common/ExpectedTasks.java @@ -25,6 +25,7 @@ import org.apache.kafka.trogdor.coordinator.CoordinatorClient; import org.apache.kafka.trogdor.rest.AgentStatusResponse; import org.apache.kafka.trogdor.rest.TaskState; +import org.apache.kafka.trogdor.rest.TasksRequest; import org.apache.kafka.trogdor.rest.TasksResponse; import org.apache.kafka.trogdor.rest.WorkerState; import org.apache.kafka.trogdor.task.TaskSpec; @@ -144,7 +145,7 @@ public ExpectedTasks waitFor(final CoordinatorClient client) throws InterruptedE public boolean conditionMet() { TasksResponse tasks = null; try { - tasks = client.tasks(); + tasks = client.tasks(new TasksRequest(null, 0, 0, 0, 0)); } catch (Exception e) { log.info("Unable to get coordinator tasks", e); throw new RuntimeException(e); diff --git a/tools/src/test/java/org/apache/kafka/trogdor/common/JsonSerializationTest.java b/tools/src/test/java/org/apache/kafka/trogdor/common/JsonSerializationTest.java index dee7614292a65..8101d9c6e4e4f 100644 --- a/tools/src/test/java/org/apache/kafka/trogdor/common/JsonSerializationTest.java +++ b/tools/src/test/java/org/apache/kafka/trogdor/common/JsonSerializationTest.java @@ -49,10 +49,10 @@ public void testDeserializationDoesNotProduceNulls() throws Exception { verify(new WorkerRunning(null, 0, null)); verify(new WorkerStopping(null, 0, null)); verify(new ProduceBenchSpec(0, 0, null, null, - 0, 0, null, null, null, 0, 0, "test-topic", 1, (short) 3)); - verify(new RoundTripWorkloadSpec(0, 0, null, null, + 0, 0, null, null, null, null, null, 0, 0, "test-topic", 1, (short) 3)); + verify(new RoundTripWorkloadSpec(0, 0, null, null, null, null, null, null, 0, null, null, 0)); - verify(new SampleTaskSpec(0, 0, 0, null)); + verify(new SampleTaskSpec(0, 0, null, null)); } private void verify(T val1) throws Exception { diff --git a/tools/src/test/java/org/apache/kafka/trogdor/common/WorkerUtilsTest.java b/tools/src/test/java/org/apache/kafka/trogdor/common/WorkerUtilsTest.java index 22b784600f650..a35efe199aa8f 100644 --- a/tools/src/test/java/org/apache/kafka/trogdor/common/WorkerUtilsTest.java +++ b/tools/src/test/java/org/apache/kafka/trogdor/common/WorkerUtilsTest.java @@ -18,8 +18,9 @@ package org.apache.kafka.trogdor.common; - import org.apache.kafka.clients.admin.TopicDescription; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.TopicPartitionInfo; import org.apache.kafka.common.Node; @@ -37,10 +38,13 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Properties; public class WorkerUtilsTest { @@ -208,4 +212,110 @@ public void testCreateNonExistingTopicsWithZeroTopicsDoesNothing() throws Throwa assertEquals(0, adminClient.listTopics().names().get().size()); } + @Test + public void testAddConfigsToPropertiesAddsAllConfigs() { + Properties props = new Properties(); + props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092"); + props.put(ProducerConfig.ACKS_CONFIG, "all"); + + Properties resultProps = new Properties(); + resultProps.putAll(props); + resultProps.put(ProducerConfig.CLIENT_ID_CONFIG, "test-client"); + resultProps.put(ProducerConfig.LINGER_MS_CONFIG, "1000"); + + WorkerUtils.addConfigsToProperties( + props, + Collections.singletonMap(ProducerConfig.CLIENT_ID_CONFIG, "test-client"), + Collections.singletonMap(ProducerConfig.LINGER_MS_CONFIG, "1000")); + assertEquals(resultProps, props); + } + + @Test + public void testCommonConfigOverwritesDefaultProps() { + Properties props = new Properties(); + props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092"); + props.put(ProducerConfig.ACKS_CONFIG, "all"); + + Properties resultProps = new Properties(); + resultProps.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092"); + resultProps.put(ProducerConfig.ACKS_CONFIG, "1"); + resultProps.put(ProducerConfig.LINGER_MS_CONFIG, "1000"); + + WorkerUtils.addConfigsToProperties( + props, + Collections.singletonMap(ProducerConfig.ACKS_CONFIG, "1"), + Collections.singletonMap(ProducerConfig.LINGER_MS_CONFIG, "1000")); + assertEquals(resultProps, props); + } + + @Test + public void testClientConfigOverwritesBothDefaultAndCommonConfigs() { + Properties props = new Properties(); + props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092"); + props.put(ProducerConfig.ACKS_CONFIG, "all"); + + Properties resultProps = new Properties(); + resultProps.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092"); + resultProps.put(ProducerConfig.ACKS_CONFIG, "0"); + + WorkerUtils.addConfigsToProperties( + props, + Collections.singletonMap(ProducerConfig.ACKS_CONFIG, "1"), + Collections.singletonMap(ProducerConfig.ACKS_CONFIG, "0")); + assertEquals(resultProps, props); + } + + @Test + public void testGetMatchingTopicPartitionsCorrectlyMatchesExactTopicName() throws Throwable { + final String topic1 = "existing-topic"; + final String topic2 = "another-topic"; + makeExistingTopicWithOneReplica(topic1, 10); + makeExistingTopicWithOneReplica(topic2, 20); + + Collection topicPartitions = + WorkerUtils.getMatchingTopicPartitions(adminClient, topic2, 0, 2); + assertEquals( + Utils.mkSet( + new TopicPartition(topic2, 0), new TopicPartition(topic2, 1), + new TopicPartition(topic2, 2) + ), + new HashSet<>(topicPartitions) + ); + } + + @Test + public void testGetMatchingTopicPartitionsCorrectlyMatchesTopics() throws Throwable { + final String topic1 = "test-topic"; + final String topic2 = "another-test-topic"; + final String topic3 = "one-more"; + makeExistingTopicWithOneReplica(topic1, 10); + makeExistingTopicWithOneReplica(topic2, 20); + makeExistingTopicWithOneReplica(topic3, 30); + + Collection topicPartitions = + WorkerUtils.getMatchingTopicPartitions(adminClient, ".*-topic$", 0, 1); + assertEquals( + Utils.mkSet( + new TopicPartition(topic1, 0), new TopicPartition(topic1, 1), + new TopicPartition(topic2, 0), new TopicPartition(topic2, 1) + ), + new HashSet<>(topicPartitions) + ); + } + + private void makeExistingTopicWithOneReplica(String topicName, int numPartitions) { + List tpInfo = new ArrayList<>(); + int brokerIndex = 0; + for (int i = 0; i < numPartitions; ++i) { + Node broker = cluster.get(brokerIndex); + tpInfo.add(new TopicPartitionInfo( + i, broker, singleReplica, Collections.emptyList())); + brokerIndex = (brokerIndex + 1) % cluster.size(); + } + adminClient.addTopic( + false, + topicName, + tpInfo, + null); + } } diff --git a/tools/src/test/java/org/apache/kafka/trogdor/coordinator/CoordinatorTest.java b/tools/src/test/java/org/apache/kafka/trogdor/coordinator/CoordinatorTest.java index 49738230e5395..34d7ffe61062c 100644 --- a/tools/src/test/java/org/apache/kafka/trogdor/coordinator/CoordinatorTest.java +++ b/tools/src/test/java/org/apache/kafka/trogdor/coordinator/CoordinatorTest.java @@ -17,6 +17,9 @@ package org.apache.kafka.trogdor.coordinator; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.TextNode; import org.apache.kafka.common.utils.MockScheduler; import org.apache.kafka.common.utils.MockTime; import org.apache.kafka.common.utils.Scheduler; @@ -36,9 +39,12 @@ import org.apache.kafka.trogdor.rest.TaskDone; import org.apache.kafka.trogdor.rest.TaskPending; import org.apache.kafka.trogdor.rest.TaskRunning; +import org.apache.kafka.trogdor.rest.TasksRequest; +import org.apache.kafka.trogdor.rest.TasksResponse; import org.apache.kafka.trogdor.rest.WorkerDone; import org.apache.kafka.trogdor.rest.WorkerRunning; import org.apache.kafka.trogdor.task.NoOpTaskSpec; +import org.apache.kafka.trogdor.task.SampleTaskSpec; import org.junit.Rule; import org.junit.rules.Timeout; import org.slf4j.Logger; @@ -47,9 +53,12 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.List; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; public class CoordinatorTest { private static final Logger log = LoggerFactory.getLogger(CoordinatorTest.class); @@ -90,8 +99,8 @@ public void testCreateTask() throws Exception { time.sleep(2); new ExpectedTasks(). addTask(new ExpectedTaskBuilder("foo"). - taskState(new TaskRunning(fooSpec, 2)). - workerState(new WorkerRunning(fooSpec, 2, "")). + taskState(new TaskRunning(fooSpec, 2, new TextNode("active"))). + workerState(new WorkerRunning(fooSpec, 2, new TextNode("active"))). build()). waitFor(cluster.coordinatorClient()). waitFor(cluster.agentClient("node02")); @@ -99,7 +108,7 @@ public void testCreateTask() throws Exception { time.sleep(3); new ExpectedTasks(). addTask(new ExpectedTaskBuilder("foo"). - taskState(new TaskDone(fooSpec, 2, 5, "", false)). + taskState(new TaskDone(fooSpec, 2, 5, "", false, new TextNode("done"))). build()). waitFor(cluster.coordinatorClient()); } @@ -127,26 +136,34 @@ public void testTaskDistribution() throws Exception { NoOpTaskSpec fooSpec = new NoOpTaskSpec(5, 2); coordinatorClient.createTask(new CreateTaskRequest("foo", fooSpec)); new ExpectedTasks(). - addTask(new ExpectedTaskBuilder("foo").taskState(new TaskPending(fooSpec)).build()). + addTask(new ExpectedTaskBuilder("foo").taskState( + new TaskPending(fooSpec)).build()). waitFor(coordinatorClient). waitFor(agentClient1). waitFor(agentClient2); time.sleep(11); + ObjectNode status1 = new ObjectNode(JsonNodeFactory.instance); + status1.set("node01", new TextNode("active")); + status1.set("node02", new TextNode("active")); new ExpectedTasks(). addTask(new ExpectedTaskBuilder("foo"). - taskState(new TaskRunning(fooSpec, 11)). - workerState(new WorkerRunning(fooSpec, 11, "")). + taskState(new TaskRunning(fooSpec, 11, status1)). + workerState(new WorkerRunning(fooSpec, 11, new TextNode("active"))). build()). waitFor(coordinatorClient). waitFor(agentClient1). waitFor(agentClient2); time.sleep(2); + ObjectNode status2 = new ObjectNode(JsonNodeFactory.instance); + status2.set("node01", new TextNode("done")); + status2.set("node02", new TextNode("done")); new ExpectedTasks(). addTask(new ExpectedTaskBuilder("foo"). - taskState(new TaskDone(fooSpec, 11, 13, "", false)). - workerState(new WorkerDone(fooSpec, 11, 13, "", "")). + taskState(new TaskDone(fooSpec, 11, 13, + "", false, status2)). + workerState(new WorkerDone(fooSpec, 11, 13, new TextNode("done"), "")). build()). waitFor(coordinatorClient). waitFor(agentClient1). @@ -182,21 +199,29 @@ public void testTaskCancellation() throws Exception { waitFor(agentClient2); time.sleep(11); + + ObjectNode status1 = new ObjectNode(JsonNodeFactory.instance); + status1.set("node01", new TextNode("active")); + status1.set("node02", new TextNode("active")); new ExpectedTasks(). addTask(new ExpectedTaskBuilder("foo"). - taskState(new TaskRunning(fooSpec, 11)). - workerState(new WorkerRunning(fooSpec, 11, "")). + taskState(new TaskRunning(fooSpec, 11, status1)). + workerState(new WorkerRunning(fooSpec, 11, new TextNode("active"))). build()). waitFor(coordinatorClient). waitFor(agentClient1). waitFor(agentClient2); + ObjectNode status2 = new ObjectNode(JsonNodeFactory.instance); + status2.set("node01", new TextNode("done")); + status2.set("node02", new TextNode("done")); time.sleep(1); coordinatorClient.stopTask(new StopTaskRequest("foo")); new ExpectedTasks(). addTask(new ExpectedTaskBuilder("foo"). - taskState(new TaskDone(fooSpec, 11, 12, "", true)). - workerState(new WorkerDone(fooSpec, 11, 12, "", "")). + taskState(new TaskDone(fooSpec, 11, 12, "", + true, status2)). + workerState(new WorkerDone(fooSpec, 11, 12, new TextNode("done"), "")). build()). waitFor(coordinatorClient). waitFor(agentClient1). @@ -302,4 +327,161 @@ private void checkLines(String prefix, CapturingCommandRunner runner) throws Int "-m comment --comment node02"). waitFor("node03", runner); } + + @Test + public void testTasksRequestMatches() throws Exception { + TasksRequest req1 = new TasksRequest(null, 0, 0, 0, 0); + assertTrue(req1.matches("foo1", -1, -1)); + assertTrue(req1.matches("bar1", 100, 200)); + assertTrue(req1.matches("baz1", 100, -1)); + + TasksRequest req2 = new TasksRequest(null, 100, 0, 0, 0); + assertFalse(req2.matches("foo1", -1, -1)); + assertTrue(req2.matches("bar1", 100, 200)); + assertFalse(req2.matches("bar1", 99, 200)); + assertFalse(req2.matches("baz1", 99, -1)); + + TasksRequest req3 = new TasksRequest(null, 200, 900, 200, 900); + assertFalse(req3.matches("foo1", -1, -1)); + assertFalse(req3.matches("bar1", 100, 200)); + assertFalse(req3.matches("bar1", 200, 1000)); + assertTrue(req3.matches("bar1", 200, 700)); + assertFalse(req3.matches("baz1", 101, -1)); + + List taskIds = new ArrayList<>(); + taskIds.add("foo1"); + taskIds.add("bar1"); + taskIds.add("baz1"); + TasksRequest req4 = new TasksRequest(taskIds, 1000, -1, -1, -1); + assertFalse(req4.matches("foo1", -1, -1)); + assertTrue(req4.matches("foo1", 1000, -1)); + assertFalse(req4.matches("foo1", 900, -1)); + assertFalse(req4.matches("baz2", 2000, -1)); + assertFalse(req4.matches("baz2", -1, -1)); + } + + @Test + public void testTasksRequest() throws Exception { + MockTime time = new MockTime(0, 0, 0); + Scheduler scheduler = new MockScheduler(time); + try (MiniTrogdorCluster cluster = new MiniTrogdorCluster.Builder(). + addCoordinator("node01"). + addAgent("node02"). + scheduler(scheduler). + build()) { + CoordinatorClient coordinatorClient = cluster.coordinatorClient(); + new ExpectedTasks().waitFor(coordinatorClient); + + NoOpTaskSpec fooSpec = new NoOpTaskSpec(1, 10); + NoOpTaskSpec barSpec = new NoOpTaskSpec(3, 1); + coordinatorClient.createTask(new CreateTaskRequest("foo", fooSpec)); + coordinatorClient.createTask(new CreateTaskRequest("bar", barSpec)); + new ExpectedTasks(). + addTask(new ExpectedTaskBuilder("foo"). + taskState(new TaskPending(fooSpec)). + build()). + addTask(new ExpectedTaskBuilder("bar"). + taskState(new TaskPending(barSpec)). + build()). + waitFor(coordinatorClient); + + assertEquals(0, coordinatorClient.tasks( + new TasksRequest(null, 10, 0, 10, 0)).tasks().size()); + TasksResponse resp1 = coordinatorClient.tasks( + new TasksRequest(Arrays.asList(new String[] {"foo", "baz" }), 0, 0, 0, 0)); + assertTrue(resp1.tasks().containsKey("foo")); + assertFalse(resp1.tasks().containsKey("bar")); + assertEquals(1, resp1.tasks().size()); + + time.sleep(2); + new ExpectedTasks(). + addTask(new ExpectedTaskBuilder("foo"). + taskState(new TaskRunning(fooSpec, 2, new TextNode("active"))). + workerState(new WorkerRunning(fooSpec, 2, new TextNode("active"))). + build()). + addTask(new ExpectedTaskBuilder("bar"). + taskState(new TaskPending(barSpec)). + build()). + waitFor(coordinatorClient). + waitFor(cluster.agentClient("node02")); + + TasksResponse resp2 = coordinatorClient.tasks( + new TasksRequest(null, 1, 0, 0, 0)); + assertTrue(resp2.tasks().containsKey("foo")); + assertFalse(resp2.tasks().containsKey("bar")); + assertEquals(1, resp2.tasks().size()); + + assertEquals(0, coordinatorClient.tasks( + new TasksRequest(null, 3, 0, 0, 0)).tasks().size()); + } + } + + @Test + public void testWorkersExitingAtDifferentTimes() throws Exception { + MockTime time = new MockTime(0, 0, 0); + Scheduler scheduler = new MockScheduler(time); + try (MiniTrogdorCluster cluster = new MiniTrogdorCluster.Builder(). + addCoordinator("node01"). + addAgent("node02"). + addAgent("node03"). + scheduler(scheduler). + build()) { + CoordinatorClient coordinatorClient = cluster.coordinatorClient(); + new ExpectedTasks().waitFor(coordinatorClient); + + HashMap nodeToExitMs = new HashMap<>(); + nodeToExitMs.put("node02", 10L); + nodeToExitMs.put("node03", 20L); + SampleTaskSpec fooSpec = + new SampleTaskSpec(2, 100, nodeToExitMs, ""); + coordinatorClient.createTask(new CreateTaskRequest("foo", fooSpec)); + new ExpectedTasks(). + addTask(new ExpectedTaskBuilder("foo"). + taskState(new TaskPending(fooSpec)). + build()). + waitFor(coordinatorClient); + + time.sleep(2); + ObjectNode status1 = new ObjectNode(JsonNodeFactory.instance); + status1.set("node02", new TextNode("active")); + status1.set("node03", new TextNode("active")); + new ExpectedTasks(). + addTask(new ExpectedTaskBuilder("foo"). + taskState(new TaskRunning(fooSpec, 2, status1)). + workerState(new WorkerRunning(fooSpec, 2, new TextNode("active"))). + build()). + waitFor(coordinatorClient). + waitFor(cluster.agentClient("node02")). + waitFor(cluster.agentClient("node03")); + + time.sleep(10); + ObjectNode status2 = new ObjectNode(JsonNodeFactory.instance); + status2.set("node02", new TextNode("halted")); + status2.set("node03", new TextNode("active")); + new ExpectedTasks(). + addTask(new ExpectedTaskBuilder("foo"). + taskState(new TaskRunning(fooSpec, 2, status2)). + workerState(new WorkerRunning(fooSpec, 2, new TextNode("active"))). + build()). + waitFor(coordinatorClient). + waitFor(cluster.agentClient("node03")); + new ExpectedTasks(). + addTask(new ExpectedTaskBuilder("foo"). + taskState(new TaskRunning(fooSpec, 2, status2)). + workerState(new WorkerDone(fooSpec, 2, 12, new TextNode("halted"), "")). + build()). + waitFor(cluster.agentClient("node02")); + + time.sleep(10); + ObjectNode status3 = new ObjectNode(JsonNodeFactory.instance); + status3.set("node02", new TextNode("halted")); + status3.set("node03", new TextNode("halted")); + new ExpectedTasks(). + addTask(new ExpectedTaskBuilder("foo"). + taskState(new TaskDone(fooSpec, 2, 22, "", + false, status3)). + build()). + waitFor(coordinatorClient); + } + } }; diff --git a/tools/src/test/java/org/apache/kafka/trogdor/task/SampleTaskSpec.java b/tools/src/test/java/org/apache/kafka/trogdor/task/SampleTaskSpec.java index 26fdfb273ecca..38a160fba21ed 100644 --- a/tools/src/test/java/org/apache/kafka/trogdor/task/SampleTaskSpec.java +++ b/tools/src/test/java/org/apache/kafka/trogdor/task/SampleTaskSpec.java @@ -20,23 +20,28 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + public class SampleTaskSpec extends TaskSpec { - private final long exitMs; + private final Map nodeToExitMs; private final String error; @JsonCreator public SampleTaskSpec(@JsonProperty("startMs") long startMs, @JsonProperty("durationMs") long durationMs, - @JsonProperty("exitMs") long exitMs, + @JsonProperty("nodeToExitMs") Map nodeToExitMs, @JsonProperty("error") String error) { super(startMs, durationMs); - this.exitMs = exitMs; + this.nodeToExitMs = nodeToExitMs == null ? new HashMap() : + Collections.unmodifiableMap(nodeToExitMs); this.error = error == null ? "" : error; } @JsonProperty - public long exitMs() { - return exitMs; + public Map nodeToExitMs() { + return nodeToExitMs; } @JsonProperty diff --git a/tools/src/test/java/org/apache/kafka/trogdor/task/SampleTaskWorker.java b/tools/src/test/java/org/apache/kafka/trogdor/task/SampleTaskWorker.java index ebac27e466ed6..ade055d3d0683 100644 --- a/tools/src/test/java/org/apache/kafka/trogdor/task/SampleTaskWorker.java +++ b/tools/src/test/java/org/apache/kafka/trogdor/task/SampleTaskWorker.java @@ -17,6 +17,7 @@ package org.apache.kafka.trogdor.task; +import com.fasterxml.jackson.databind.node.TextNode; import org.apache.kafka.common.internals.KafkaFutureImpl; import org.apache.kafka.trogdor.common.Platform; import org.apache.kafka.trogdor.common.ThreadUtils; @@ -26,12 +27,12 @@ import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReference; public class SampleTaskWorker implements TaskWorker { private final SampleTaskSpec spec; private final ScheduledExecutorService executor; private Future future; + private WorkerStatusTracker status; SampleTaskWorker(SampleTaskSpec spec) { this.spec = spec; @@ -41,17 +42,24 @@ public class SampleTaskWorker implements TaskWorker { } @Override - public synchronized void start(Platform platform, AtomicReference status, + public synchronized void start(Platform platform, WorkerStatusTracker status, final KafkaFutureImpl haltFuture) throws Exception { if (this.future != null) return; + this.status = status; + this.status.update(new TextNode("active")); + + Long exitMs = spec.nodeToExitMs().get(platform.curNode().name()); + if (exitMs == null) { + exitMs = Long.MAX_VALUE; + } this.future = platform.scheduler().schedule(executor, new Callable() { @Override public Void call() throws Exception { haltFuture.complete(spec.error()); return null; } - }, spec.exitMs()); + }, exitMs); } @Override @@ -59,5 +67,6 @@ public void stop(Platform platform) throws Exception { this.future.cancel(false); this.executor.shutdown(); this.executor.awaitTermination(1, TimeUnit.DAYS); + this.status.update(new TextNode("halted")); } }; diff --git a/tools/src/test/java/org/apache/kafka/trogdor/task/TaskSpecTest.java b/tools/src/test/java/org/apache/kafka/trogdor/task/TaskSpecTest.java index abd7e6250ff57..d8d4ca9d5fb76 100644 --- a/tools/src/test/java/org/apache/kafka/trogdor/task/TaskSpecTest.java +++ b/tools/src/test/java/org/apache/kafka/trogdor/task/TaskSpecTest.java @@ -40,11 +40,11 @@ public void testTaskSpecSerialization() throws Exception { } catch (InvalidTypeIdException e) { } String inputJson = "{\"class\":\"org.apache.kafka.trogdor.task.SampleTaskSpec\"," + - "\"startMs\":123,\"durationMs\":456,\"exitMs\":1000,\"error\":\"foo\"}"; + "\"startMs\":123,\"durationMs\":456,\"nodeToExitMs\":{\"node01\":1000},\"error\":\"foo\"}"; SampleTaskSpec spec = JsonUtil.JSON_SERDE.readValue(inputJson, SampleTaskSpec.class); assertEquals(123, spec.startMs()); assertEquals(456, spec.durationMs()); - assertEquals(1000, spec.exitMs()); + assertEquals(Long.valueOf(1000), spec.nodeToExitMs().get("node01")); assertEquals("foo", spec.error()); String outputJson = JsonUtil.toJsonString(spec); assertEquals(inputJson, outputJson); diff --git a/tools/src/test/java/org/apache/kafka/trogdor/workload/PayloadGeneratorTest.java b/tools/src/test/java/org/apache/kafka/trogdor/workload/PayloadGeneratorTest.java index 25ef2e326b6b9..0909dc0d05c36 100644 --- a/tools/src/test/java/org/apache/kafka/trogdor/workload/PayloadGeneratorTest.java +++ b/tools/src/test/java/org/apache/kafka/trogdor/workload/PayloadGeneratorTest.java @@ -139,4 +139,12 @@ public void testPayloadIterator() { iter.seek(0); assertEquals(0, iter.position()); } + + @Test + public void testNullPayloadGenerator() { + NullPayloadGenerator generator = new NullPayloadGenerator(); + assertEquals(null, generator.generate(0)); + assertEquals(null, generator.generate(1)); + assertEquals(null, generator.generate(100)); + } } diff --git a/vagrant/base.sh b/vagrant/base.sh index bfc349673518e..f5c03cca4fdf2 100755 --- a/vagrant/base.sh +++ b/vagrant/base.sh @@ -99,8 +99,10 @@ popd popd popd -# Test multiple Scala versions -get_kafka 0.8.2.2 2.10 +# Test multiple Kafka versions +# we want to use the latest Scala version per Kafka version +# however, we cannot pull in Scala 2.12 builds atm, because Scala 2.12 requires Java 8, but we use Java 7 to run the system tests +get_kafka 0.8.2.2 2.11 chmod a+rw /opt/kafka-0.8.2.2 get_kafka 0.9.0.1 2.11 chmod a+rw /opt/kafka-0.9.0.1 @@ -112,10 +114,10 @@ get_kafka 0.10.2.1 2.11 chmod a+rw /opt/kafka-0.10.2.1 get_kafka 0.11.0.2 2.11 chmod a+rw /opt/kafka-0.11.0.2 -get_kafka 1.0.0 2.11 -chmod a+rw /opt/kafka-1.0.0 get_kafka 1.0.1 2.11 chmod a+rw /opt/kafka-1.0.1 +get_kafka 1.1.0 2.11 +chmod a+rw /opt/kafka-1.1.0 # For EC2 nodes, we want to use /mnt, which should have the local disk. On local