From f34bd4ba0e61a910e77ba0a6c9152974737412ca Mon Sep 17 00:00:00 2001 From: Tom Bentley Date: Wed, 6 Sep 2017 15:39:24 +0100 Subject: [PATCH 01/13] KAFKA-5692: Change PreferredReplicaLeaderElectionCommand to use AdminClient See also KIP-183. This implements the following algorithm: 1. AdminClient sends ElectPreferredLeadersRequest. 2. KafakApis receives ElectPreferredLeadersRequest and delegates to ReplicaManager.electPreferredLeaders() 3. ReplicaManager delegates to KafkaController.electPreferredLeaders() 4. KafkaController adds a PreferredReplicaLeaderElection to the EventManager, 5. ReplicaManager.electPreferredLeaders()'s callback uses the delayedElectPreferredReplicasPurgatory to wait for the results of the election to appear in the metadata cache. If there are no results because of errors, or because the preferred leaders are already leading the partitions then a response is returned immediately. In the EventManager work thread the preferred leader is elected as follows: 1. The EventManager runs PreferredReplicaLeaderElection.process() 2. process() calls KafkaController.onPreferredReplicaElectionWithResults() 3. KafkaController.onPreferredReplicaElectionWithResults() calls the PartitionStateMachine.handleStateChangesWithResults() to perform the election (asynchronously the PSM will send LeaderAndIsrRequest to the new and old leaders and UpdateMetadataRequest to all brokers) then invokes the callback. Note: the change in parameter type for CollectionUtils.groupDataByTopic(). This makes sense because the AdminClient APIs use Collection consistently, rather than List or Set. If binary compatiblity is a consideration the old version should be kept, delegating to the new version. I had to add PartitionStateMachine.handleStateChangesWithResults() in order to be able to process a set of state changes in the PartitionStateMachine *and get back individual results*. At the same time I noticed that all callers of existing handleStateChange() were destructuring a TopicAndPartition that they already had in order to call handleStateChange(), and that handleStateChange() immediately instantiated a new TopicAndPartition. Since TopicAndPartition is immutable this is pointless, so I refactored it. handleStateChange() also now returns any exception it caught, which is necessary for handleStateChangesWithResults() --- .../kafka/clients/admin/AdminClient.java | 27 ++ .../admin/ElectPreferredLeadersOptions.java | 31 ++ .../admin/ElectPreferredLeadersResult.java | 116 +++++++ .../kafka/clients/admin/KafkaAdminClient.java | 68 ++++ .../apache/kafka/common/protocol/ApiKeys.java | 6 +- .../common/requests/AbstractRequest.java | 2 + .../common/requests/AbstractResponse.java | 2 + .../ElectPreferredLeadersRequest.java | 159 +++++++++ .../ElectPreferredLeadersResponse.java | 120 +++++++ .../kafka/common/utils/CollectionUtils.java | 3 +- .../clients/admin/KafkaAdminClientTest.java | 35 ++ .../common/requests/RequestResponseTest.java | 22 ++ ...referredReplicaLeaderElectionCommand.scala | 200 +++++++++--- .../main/scala/kafka/cluster/Partition.scala | 1 + .../kafka/controller/KafkaController.scala | 64 +++- .../controller/PartitionStateMachine.scala | 49 ++- .../server/DelayedElectPreferredLeader.scala | 96 ++++++ .../main/scala/kafka/server/KafkaApis.scala | 26 ++ .../scala/kafka/server/ReplicaManager.scala | 43 +++ core/src/main/scala/kafka/utils/ZkUtils.scala | 22 ++ .../scala/unit/kafka/admin/AdminTest.scala | 4 +- ...rredReplicaLeaderElectionCommandTest.scala | 307 ++++++++++++++++++ .../kafka/server/ReplicaManagerTest.scala | 4 +- .../unit/kafka/server/RequestQuotaTest.scala | 4 + 24 files changed, 1336 insertions(+), 75 deletions(-) create mode 100644 clients/src/main/java/org/apache/kafka/clients/admin/ElectPreferredLeadersOptions.java create mode 100644 clients/src/main/java/org/apache/kafka/clients/admin/ElectPreferredLeadersResult.java create mode 100644 clients/src/main/java/org/apache/kafka/common/requests/ElectPreferredLeadersRequest.java create mode 100644 clients/src/main/java/org/apache/kafka/common/requests/ElectPreferredLeadersResponse.java create mode 100644 core/src/main/scala/kafka/server/DelayedElectPreferredLeader.scala create mode 100644 core/src/test/scala/unit/kafka/admin/PreferredReplicaLeaderElectionCommandTest.scala diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/AdminClient.java b/clients/src/main/java/org/apache/kafka/clients/admin/AdminClient.java index fd695f9dc33d9..e209a20659ec3 100644 --- a/clients/src/main/java/org/apache/kafka/clients/admin/AdminClient.java +++ b/clients/src/main/java/org/apache/kafka/clients/admin/AdminClient.java @@ -17,6 +17,7 @@ package org.apache.kafka.clients.admin; +import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.TopicPartitionReplica; import org.apache.kafka.common.acl.AclBinding; import org.apache.kafka.common.acl.AclBindingFilter; @@ -508,4 +509,30 @@ public CreatePartitionsResult createPartitions(Map newPar public abstract CreatePartitionsResult createPartitions(Map newPartitions, CreatePartitionsOptions options); + /* + * Elect the preferred replica of the given {@code partitions} as leader, or + * elect the preferred replica for all partitions as leader if the argument to {@code partitions} is null. + * + * This is a convenience method for {@link #electPreferredLeaders(Collection, ElectPreferredLeadersOptions)} with default options. + * See the overload for more details. + * + * @param partitions The partitions for which the the preferred leader should be elected. + * @return The ElectPreferredLeadersResult. + */ + public ElectPreferredLeadersResult electPreferredLeaders(Collection partitions) { + return electPreferredLeaders(partitions, new ElectPreferredLeadersOptions()); + } + + /** + * Elect the preferred replica of the given {@code partitions} as leader, or + * elect the preferred replica for all partitions as leader if the argument to {@code partitions} is null. + * + * This operation is supported by brokers with version 1.0.0 or higher. + * + * @param partitions The partitions for which the the preferred leader should be elected. + * @param options The options to use when electing the preferred leaders. + * @return The ElectPreferredLeadersResult. + */ + public abstract ElectPreferredLeadersResult electPreferredLeaders(Collection partitions, ElectPreferredLeadersOptions options); + } diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/ElectPreferredLeadersOptions.java b/clients/src/main/java/org/apache/kafka/clients/admin/ElectPreferredLeadersOptions.java new file mode 100644 index 0000000000000..80b00975f7da9 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/clients/admin/ElectPreferredLeadersOptions.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.clients.admin; + +import org.apache.kafka.common.annotation.InterfaceStability; + +import java.util.Collection; + +/** + * Options for {@link AdminClient#electPreferredLeaders(Collection, ElectPreferredLeadersOptions)}. + * + * The API of this class is evolving, see {@link AdminClient} for details. + */ +@InterfaceStability.Evolving +public class ElectPreferredLeadersOptions extends AbstractOptions { +} diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/ElectPreferredLeadersResult.java b/clients/src/main/java/org/apache/kafka/clients/admin/ElectPreferredLeadersResult.java new file mode 100644 index 0000000000000..e2075b2ef95cb --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/clients/admin/ElectPreferredLeadersResult.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.clients.admin; + +import org.apache.kafka.common.KafkaFuture; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.annotation.InterfaceStability; +import org.apache.kafka.common.internals.KafkaFutureImpl; + +import java.util.Collection; +import java.util.Map; +import java.util.concurrent.ExecutionException; + +/** + * The result of {@link AdminClient#electPreferredLeaders(Collection, ElectPreferredLeadersOptions)} + * + *

The {@code KafkaFuture}s available from instances of this class may be completed + * exceptionally due to:

+ *
    + *
  • {@link org.apache.kafka.common.protocol.Errors#CLUSTER_AUTHORIZATION_FAILED CLUSTER_AUTHORIZATION_FAILED} if the authenticated user didn't have {@code Alter} access to the cluster.
    • + *
  • {@link org.apache.kafka.common.protocol.Errors#UNKNOWN_TOPIC_OR_PARTITION UNKNOWN_TOPIC_OR_PARTITION} if the topic or partition did not exist within the cluster.
    • + *
  • {@link org.apache.kafka.common.protocol.Errors#INVALID_TOPIC_EXCEPTION INVALID_TOPIC_EXCEPTION} if the topic was already queued for deletion.
    • + *
  • {@link org.apache.kafka.common.protocol.Errors#NOT_CONTROLLER NOT_CONTROLLER} if the request was sent to a broker that was not the controller for the cluster.
    • + *
  • {@link org.apache.kafka.common.protocol.Errors#REQUEST_TIMED_OUT REQUEST_TIMED_OUT} if the request timed out before the election was complete.
    • + *
  • {@link org.apache.kafka.common.protocol.Errors#UNKNOWN_SERVER_ERROR UNKNOWN_SERVER_ERROR} if the preferred leader was not alive or not in the ISR.
    • + *
+ * + * The API of this class is evolving, see {@link AdminClient} for details. + */ +@InterfaceStability.Evolving +public class ElectPreferredLeadersResult { + + private final KafkaFuture>> futures; + + ElectPreferredLeadersResult(KafkaFuture>> futures) { + this.futures = futures; + } + + /** Return a new future that has completed exceptionally */ + private static KafkaFuture exceptionalFuture(Exception e) { + KafkaFutureImpl future = new KafkaFutureImpl<>(); + future.completeExceptionally(e); + return future; + } + + /** + * Get the result of the election for the given {@code partition}. + * If there was not an election triggered for the given {@code partition}, the + * returned future will complete with an error. + */ + public KafkaFuture partitionResult(TopicPartition partition) { + final Map> map; + try { + map = futures.get(); + } catch (InterruptedException | ExecutionException e) { + return exceptionalFuture(e); + } + KafkaFuture result = map.get(partition); + if (result == null) { + KafkaFutureImpl future = new KafkaFutureImpl<>(); + future.completeExceptionally(new IllegalArgumentException( + "Preferred leader election for partition \"" + partition + "\" was not attempted")); + result = future; + } + return result; + } + + /** + *

Get a future for the topic partitions for which a leader election + * was attempted. A partition will be present in this result if + * an election was attempted even if the election was not successful.

+ * + *

This method is provided to discover the partitions attempted when + * {@link AdminClient#electPreferredLeaders(Collection)} is called + * with a null {@code partitions} argument.

+ */ + public KafkaFuture> partitions() { + final Map> map; + try { + map = futures.get(); + } catch (InterruptedException | ExecutionException e) { + return exceptionalFuture(e); + } + KafkaFutureImpl> result = new KafkaFutureImpl<>(); + result.complete(map.keySet()); + return result; + } + + /** + * Return a future which succeeds if all the topic elections succeed. + */ + public KafkaFuture all() { + final Map> map; + try { + map = futures.get(); + } catch (InterruptedException | ExecutionException e) { + return exceptionalFuture(e); + } + return KafkaFuture.allOf(map.values().toArray(new KafkaFuture[0])); + } +} diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java index ece27caeb405d..0f67ac453584c 100644 --- a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java +++ b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java @@ -86,6 +86,8 @@ import org.apache.kafka.common.requests.DescribeConfigsResponse; import org.apache.kafka.common.requests.DescribeLogDirsRequest; import org.apache.kafka.common.requests.DescribeLogDirsResponse; +import org.apache.kafka.common.requests.ElectPreferredLeadersRequest; +import org.apache.kafka.common.requests.ElectPreferredLeadersResponse; import org.apache.kafka.common.requests.MetadataRequest; import org.apache.kafka.common.requests.MetadataResponse; import org.apache.kafka.common.requests.Resource; @@ -1863,6 +1865,7 @@ public void handleResponse(AbstractResponse abstractResponse) { future.completeExceptionally(result.getValue().exception()); } } + } @Override @@ -1873,4 +1876,69 @@ void handleFailure(Throwable throwable) { return new CreatePartitionsResult(new HashMap>(futures)); } + public ElectPreferredLeadersResult electPreferredLeaders(final Collection partitions, ElectPreferredLeadersOptions options) { + + final KafkaFutureImpl>> futures = new KafkaFutureImpl<>(); + final Map> mapOfFutures; + final List requestList; + final boolean knownPartitions = partitions != null; + if (knownPartitions) { + mapOfFutures = new HashMap<>(partitions.size()); + for (TopicPartition partition : partitions) { + KafkaFutureImpl future = new KafkaFutureImpl(); + mapOfFutures.put(partition, future); + } + futures.complete(mapOfFutures); + requestList = new ArrayList<>(partitions); + } else { + mapOfFutures = new HashMap<>(); + requestList = null; + } + + final long now = time.milliseconds(); + runnable.call(new Call("electPreferredLeaders", calcDeadlineMs(now, options.timeoutMs()), + new ControllerNodeProvider()) { + + @Override + public AbstractRequest.Builder createRequest(int timeoutMs) { + return new ElectPreferredLeadersRequest.Builder(requestList, timeoutMs); + } + + @Override + public void handleResponse(AbstractResponse abstractResponse) { + ElectPreferredLeadersResponse response = (ElectPreferredLeadersResponse) abstractResponse; + // Iterate over the response partitions->errors because the argument partitions can be null + for (Map.Entry entry : response.errors().entrySet()) { + TopicPartition partition = entry.getKey(); + KafkaFutureImpl future; + if (knownPartitions) { + future = mapOfFutures.get(partition); + } else { + future = new KafkaFutureImpl(); + mapOfFutures.put(partition, future); + } + if (entry.getValue().isSuccess()) { + future.complete(null); + } else { + ApiException exception = entry.getValue().exception(); + future.completeExceptionally(exception); + } + } + if (!knownPartitions) { + futures.complete(mapOfFutures); + } + + } + + @Override + void handleFailure(Throwable throwable) { + if (knownPartitions) { + completeAllExceptionally(mapOfFutures.values(), throwable); + } else { + futures.completeExceptionally(throwable); + } + } + }, now); + return new ElectPreferredLeadersResult(futures); + } } diff --git a/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java b/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java index cf1bff557331b..ccdda1021167d 100644 --- a/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java +++ b/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java @@ -53,6 +53,8 @@ import org.apache.kafka.common.requests.DescribeGroupsResponse; import org.apache.kafka.common.requests.DescribeLogDirsRequest; import org.apache.kafka.common.requests.DescribeLogDirsResponse; +import org.apache.kafka.common.requests.ElectPreferredLeadersRequest; +import org.apache.kafka.common.requests.ElectPreferredLeadersResponse; import org.apache.kafka.common.requests.EndTxnRequest; import org.apache.kafka.common.requests.EndTxnResponse; import org.apache.kafka.common.requests.FetchRequest; @@ -171,7 +173,9 @@ public Struct parseResponse(short version, ByteBuffer buffer) { SASL_AUTHENTICATE(36, "SaslAuthenticate", SaslAuthenticateRequest.schemaVersions(), SaslAuthenticateResponse.schemaVersions()), CREATE_PARTITIONS(37, "CreatePartitions", CreatePartitionsRequest.schemaVersions(), - CreatePartitionsResponse.schemaVersions()); + CreatePartitionsResponse.schemaVersions()), + ELECT_PREFERRED_LEADERS(38, "ElectPreferredLeaders", ElectPreferredLeadersRequest.schemaVersions(), + ElectPreferredLeadersResponse.schemaVersions()); private static final ApiKeys[] ID_TO_TYPE; private static final int MIN_API_KEY = 0; diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java index 5a1c4f49926d9..7c1e687e2a159 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractRequest.java @@ -214,6 +214,8 @@ public static AbstractRequest parseRequest(ApiKeys apiKey, short apiVersion, Str return new SaslAuthenticateRequest(struct, apiVersion); case CREATE_PARTITIONS: return new CreatePartitionsRequest(struct, apiVersion); + case ELECT_PREFERRED_LEADERS: + return new ElectPreferredLeadersRequest(struct, apiVersion); default: throw new AssertionError(String.format("ApiKey %s is not currently handled in `parseRequest`, the " + "code should be updated to do so.", apiKey)); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java index 6294af4cf7214..381d2d7d52bb7 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/AbstractResponse.java @@ -146,6 +146,8 @@ public static AbstractResponse parseResponse(ApiKeys apiKey, Struct struct) { return new SaslAuthenticateResponse(struct); case CREATE_PARTITIONS: return new CreatePartitionsResponse(struct); + case ELECT_PREFERRED_LEADERS: + return new ElectPreferredLeadersResponse(struct); default: throw new AssertionError(String.format("ApiKey %s is not currently handled in `parseResponse`, the " + "code should be updated to do so.", apiKey)); diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ElectPreferredLeadersRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ElectPreferredLeadersRequest.java new file mode 100644 index 0000000000000..c55f4eea4b6a0 --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/common/requests/ElectPreferredLeadersRequest.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.common.requests; + +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.types.ArrayOf; +import org.apache.kafka.common.protocol.types.Field; +import org.apache.kafka.common.protocol.types.Schema; +import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.utils.CollectionUtils; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.apache.kafka.common.protocol.CommonFields.TOPIC_NAME; +import static org.apache.kafka.common.protocol.types.Type.INT32; + +public class ElectPreferredLeadersRequest extends AbstractRequest { + + private static final String TIMEOUT_KEY_NAME = "timeout"; + private static final String TOPIC_PARTITIONS_KEY_NAME = "topic_partitions"; + private static final String PARTITIONS_KEY_NAME = "partitions"; + + public static final Schema ELECT_PREFERRED_LEADERS_REQUEST_V0 = new Schema( + new Field("topic_partitions", ArrayOf.nullable( + new Schema( + TOPIC_NAME, + new Field("partitions", + new ArrayOf(INT32), + "The partitions of this topic whose preferred leader should be elected")))), + new Field("timeout", INT32, "The time in ms to wait for the elections to be completed.") + ); + + public static Schema[] schemaVersions() { + return new Schema[]{ELECT_PREFERRED_LEADERS_REQUEST_V0}; + } + + public static class Builder extends AbstractRequest.Builder { + + private final Set topicPartitions; + private final int timeout; + + public Builder(Collection topicPartitions, int timeout) { + super(ApiKeys.ELECT_PREFERRED_LEADERS); + this.topicPartitions = topicPartitions != null ? new HashSet<>(topicPartitions) : (Set) null; + this.timeout = timeout; + } + + @Override + public ElectPreferredLeadersRequest build(short version) { + return new ElectPreferredLeadersRequest(version, topicPartitions, timeout); + } + } + + private final Set topicPartitions; + private final int timeout; + + public ElectPreferredLeadersRequest(short version, Set topicPartitions, int timeout) { + super(version); + this.topicPartitions = topicPartitions; + this.timeout = timeout; + } + + public ElectPreferredLeadersRequest(Struct struct, short version) { + super(version); + Object[] topicPartitionsArray = struct.getArray(TOPIC_PARTITIONS_KEY_NAME); + if (topicPartitionsArray != null) { + topicPartitions = new HashSet<>(topicPartitionsArray.length); + for (Object topicPartitionObj : topicPartitionsArray) { + Struct topicPartitionStruct = (Struct) topicPartitionObj; + String topicName = topicPartitionStruct.get(TOPIC_NAME); + Object[] partitionsArray = topicPartitionStruct.getArray(PARTITIONS_KEY_NAME); + if (partitionsArray != null) { + for (Object partitionObj : partitionsArray) { + Integer partition = (Integer) partitionObj; + TopicPartition topicPartition = new TopicPartition(topicName, partition); + topicPartitions.add(topicPartition); + } + } + } + } else { + topicPartitions = null; + } + timeout = struct.getInt(TIMEOUT_KEY_NAME); + } + + public Set topicPartitions() { + return topicPartitions; + } + + public int timeout() { + return timeout; + } + + @Override + protected Struct toStruct() { + Struct struct = new Struct(ApiKeys.ELECT_PREFERRED_LEADERS.requestSchema(version())); + Struct[] topicPartitionsArray; + if (topicPartitions != null) { + Map> map = CollectionUtils.groupDataByTopic(topicPartitions); + List topicPartitionsList = new ArrayList<>(topicPartitions.size()); + for (Map.Entry> entry : map.entrySet()) { + Struct partitionStruct = struct.instance(TOPIC_PARTITIONS_KEY_NAME); + partitionStruct.set(TOPIC_NAME, entry.getKey()); + partitionStruct.set(PARTITIONS_KEY_NAME, entry.getValue().toArray()); + topicPartitionsList.add(partitionStruct); + } + topicPartitionsArray = topicPartitionsList.toArray(new Struct[topicPartitionsList.size()]); + } else { + topicPartitionsArray = null; + } + struct.set(TOPIC_PARTITIONS_KEY_NAME, topicPartitionsArray); + struct.set(TIMEOUT_KEY_NAME, timeout); + return struct; + } + + @Override + public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { + short version = version(); + switch (version) { + case 0: + ApiError error = ApiError.fromThrowable(e); + Map errors = new HashMap<>(topicPartitions.size()); + for (TopicPartition partition : topicPartitions) + errors.put(partition, error); + return new ElectPreferredLeadersResponse(throttleTimeMs, errors); + default: + throw new IllegalArgumentException(String.format("Version %d is not valid. Valid versions for %s are 0 to %d", + version, this.getClass().getSimpleName(), ApiKeys.ELECT_PREFERRED_LEADERS.latestVersion())); + } + } + + public static ElectPreferredLeadersRequest parse(ByteBuffer buffer, short version) { + return new ElectPreferredLeadersRequest(ApiKeys.ELECT_PREFERRED_LEADERS.parseRequest(version, buffer), version); + } + +} diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ElectPreferredLeadersResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ElectPreferredLeadersResponse.java new file mode 100644 index 0000000000000..0c7459a8d3eaa --- /dev/null +++ b/clients/src/main/java/org/apache/kafka/common/requests/ElectPreferredLeadersResponse.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.common.requests; + +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.types.ArrayOf; +import org.apache.kafka.common.protocol.types.Field; +import org.apache.kafka.common.protocol.types.Schema; +import org.apache.kafka.common.protocol.types.Struct; +import org.apache.kafka.common.utils.CollectionUtils; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.apache.kafka.common.protocol.CommonFields.ERROR_CODE; +import static org.apache.kafka.common.protocol.CommonFields.ERROR_MESSAGE; +import static org.apache.kafka.common.protocol.CommonFields.PARTITION_ID; +import static org.apache.kafka.common.protocol.CommonFields.THROTTLE_TIME_MS; +import static org.apache.kafka.common.protocol.CommonFields.TOPIC_NAME; + +public class ElectPreferredLeadersResponse extends AbstractResponse { + + private static final String REPLICA_ELECTION_RESULT_KEY_NAME = "replica_election_result"; + private static final String PARTITION_RESULTS_KEY_NAME = "partition_results"; + + public static final Schema ELECT_PREFERRED_LEADERS_RESPONSE_V0 = new Schema( + THROTTLE_TIME_MS, + new Field("replica_election_result", new ArrayOf(new Schema( + TOPIC_NAME, + new Field("partition_results", new ArrayOf( + new Schema( + PARTITION_ID, + ERROR_CODE, + ERROR_MESSAGE)), + "The results for each partition"))))); + + public static Schema[] schemaVersions() { + return new Schema[]{ELECT_PREFERRED_LEADERS_RESPONSE_V0}; + } + + private final int throttleTimeMs; + private final Map errors; + + public ElectPreferredLeadersResponse(int throttleTimeMs, Map errors) { + this.throttleTimeMs = throttleTimeMs; + this.errors = errors; + } + + public ElectPreferredLeadersResponse(Struct struct) { + throttleTimeMs = struct.get(THROTTLE_TIME_MS); + Object[] resourcesArray = struct.getArray(REPLICA_ELECTION_RESULT_KEY_NAME); + errors = new HashMap<>(resourcesArray.length); + for (Object partitionObj : resourcesArray) { + Struct topicStruct = (Struct) partitionObj; + String topicName = topicStruct.get(TOPIC_NAME); + Object[] partitionResults = topicStruct.getArray(PARTITION_RESULTS_KEY_NAME); + for (Object partitionResult : partitionResults) { + Struct partitionStruct = (Struct) partitionResult; + int partitionId = partitionStruct.get(PARTITION_ID); + ApiError error = new ApiError(partitionStruct); + errors.put(new TopicPartition(topicName, partitionId), error); + } + } + } + + public Map errors() { + return errors; + } + + public int throttleTimeMs() { + return throttleTimeMs; + } + + @Override + protected Struct toStruct(short version) { + Struct struct = new Struct(ApiKeys.ELECT_PREFERRED_LEADERS.responseSchema(version)); + struct.set(THROTTLE_TIME_MS, throttleTimeMs); + Map> groupedErrors = CollectionUtils.groupDataByTopic(errors); + List replicaElectionResultList = new ArrayList<>(errors.size()); + for (Map.Entry> topicToErrors : groupedErrors.entrySet()) { + Struct topicStruct = struct.instance(REPLICA_ELECTION_RESULT_KEY_NAME); + topicStruct.set(TOPIC_NAME, topicToErrors.getKey()); + List partitionResultList = new ArrayList<>(topicToErrors.getValue().size()); + for (Map.Entry partitionToError : topicToErrors.getValue().entrySet()) { + Struct partitionResultStruct = topicStruct.instance(PARTITION_RESULTS_KEY_NAME); + partitionResultStruct.set(PARTITION_ID, partitionToError.getKey()); + partitionToError.getValue().write(partitionResultStruct); + partitionResultList.add(partitionResultStruct); + } + topicStruct.set(PARTITION_RESULTS_KEY_NAME, partitionResultList.toArray()); + replicaElectionResultList.add(topicStruct); + } + struct.set(REPLICA_ELECTION_RESULT_KEY_NAME, replicaElectionResultList.toArray(new Struct[0])); + return struct; + } + + public static ElectPreferredLeadersResponse parse(ByteBuffer buffer, short version) { + return new ElectPreferredLeadersResponse(ApiKeys.ELECT_PREFERRED_LEADERS.parseResponse(version, buffer)); + } + +} diff --git a/clients/src/main/java/org/apache/kafka/common/utils/CollectionUtils.java b/clients/src/main/java/org/apache/kafka/common/utils/CollectionUtils.java index d7ab4e0b39caf..4322341190ebf 100644 --- a/clients/src/main/java/org/apache/kafka/common/utils/CollectionUtils.java +++ b/clients/src/main/java/org/apache/kafka/common/utils/CollectionUtils.java @@ -19,6 +19,7 @@ import org.apache.kafka.common.TopicPartition; import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -50,7 +51,7 @@ public static Map> groupDataByTopic(Map> groupDataByTopic(List partitions) { + public static Map> groupDataByTopic(Collection partitions) { Map> partitionsByTopic = new HashMap<>(); for (TopicPartition tp: partitions) { String topic = tp.topic(); diff --git a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java index 2412d0310de85..7b9ac7a8f8d87 100644 --- a/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java +++ b/clients/src/test/java/org/apache/kafka/clients/admin/KafkaAdminClientTest.java @@ -22,6 +22,7 @@ import org.apache.kafka.common.KafkaFuture; import org.apache.kafka.common.Node; import org.apache.kafka.common.PartitionInfo; +import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.acl.AccessControlEntry; import org.apache.kafka.common.acl.AccessControlEntryFilter; import org.apache.kafka.common.acl.AclBinding; @@ -29,6 +30,7 @@ import org.apache.kafka.common.acl.AclOperation; import org.apache.kafka.common.acl.AclPermissionType; import org.apache.kafka.common.config.ConfigResource; +import org.apache.kafka.common.errors.ClusterAuthorizationException; import org.apache.kafka.common.errors.InvalidTopicException; import org.apache.kafka.common.errors.SecurityDisabledException; import org.apache.kafka.common.errors.TimeoutException; @@ -43,6 +45,7 @@ import org.apache.kafka.common.requests.DeleteAclsResponse.AclFilterResponse; import org.apache.kafka.common.requests.DescribeAclsResponse; import org.apache.kafka.common.requests.DescribeConfigsResponse; +import org.apache.kafka.common.requests.ElectPreferredLeadersResponse; import org.apache.kafka.common.resource.Resource; import org.apache.kafka.common.resource.ResourceFilter; import org.apache.kafka.common.resource.ResourceType; @@ -310,6 +313,38 @@ public void testDeleteAcls() throws Exception { } } + @Test + public void testElectPreferredLeaders() throws Exception { + TopicPartition topic1 = new TopicPartition("topic", 0); + TopicPartition topic2 = new TopicPartition("topic", 2); + try (MockKafkaAdminClientEnv env = mockClientEnv()) { + env.kafkaClient().setNodeApiVersions(NodeApiVersions.create()); + env.kafkaClient().prepareMetadataUpdate(env.cluster(), Collections.emptySet()); + env.kafkaClient().setNode(env.cluster().controller()); + + // Test a call where one partition has an error. + HashMap map = new HashMap<>(); + map.put(topic1, ApiError.NONE); + map.put(topic2, ApiError.fromThrowable(new ClusterAuthorizationException(null))); + env.kafkaClient().prepareResponse(new ElectPreferredLeadersResponse(0, + map)); + ElectPreferredLeadersResult results = env.adminClient().electPreferredLeaders(asList(topic1, topic2)); + results.partitionResult(topic1).get(); + assertFutureError(results.partitionResult(topic2), ClusterAuthorizationException.class); + assertFutureError(results.all(), ClusterAuthorizationException.class); + + // Test a call where there are no errors. + map.put(topic1, ApiError.NONE); + map.put(topic2, ApiError.NONE); + env.kafkaClient().prepareResponse(new ElectPreferredLeadersResponse(0, + map)); + + results = env.adminClient().electPreferredLeaders(asList(topic1, topic2)); + results.partitionResult(topic1).get(); + results.partitionResult(topic2).get(); + } + } + /** * Test handling timeouts. */ diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java index edd1314c48bde..57e767d1a0d6c 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java @@ -245,6 +245,11 @@ public void testSerialization() throws Exception { checkRequest(createCreatePartitionsRequestWithAssignments()); checkErrorResponse(createCreatePartitionsRequest(), new InvalidTopicException()); checkResponse(createCreatePartitionsResponse(), 0); + checkRequest(createElectPreferredLeadersRequest()); + checkRequest(createElectPreferredLeadersRequestNullPartitions()); + checkErrorResponse(createElectPreferredLeadersRequest(), new UnknownServerException()); + checkResponse(createElectPreferredLeadersResponse(), 0); + } @Test @@ -1082,4 +1087,21 @@ private CreatePartitionsResponse createCreatePartitionsResponse() { return new CreatePartitionsResponse(42, results); } + private ElectPreferredLeadersRequest createElectPreferredLeadersRequestNullPartitions() { + return new ElectPreferredLeadersRequest.Builder(null, 100).build((short) 0); + } + + private ElectPreferredLeadersRequest createElectPreferredLeadersRequest() { + return new ElectPreferredLeadersRequest.Builder(asList( + new TopicPartition("my_topic", 1), + new TopicPartition("my_topic", 2)), 100).build((short) 0); + } + + private ElectPreferredLeadersResponse createElectPreferredLeadersResponse() { + Map errors = new HashMap<>(); + errors.put(new TopicPartition("my_topic", 0), ApiError.NONE); + errors.put(new TopicPartition("my_topic", 1), ApiError.fromThrowable(Errors.UNKNOWN_TOPIC_OR_PARTITION.exception("blah"))); + return new ElectPreferredLeadersResponse(200, errors); + } + } diff --git a/core/src/main/scala/kafka/admin/PreferredReplicaLeaderElectionCommand.scala b/core/src/main/scala/kafka/admin/PreferredReplicaLeaderElectionCommand.scala index c292fe61bea37..d351ff1121ed4 100755 --- a/core/src/main/scala/kafka/admin/PreferredReplicaLeaderElectionCommand.scala +++ b/core/src/main/scala/kafka/admin/PreferredReplicaLeaderElectionCommand.scala @@ -16,11 +16,20 @@ */ package kafka.admin +import java.util.Properties +import java.util.concurrent.{ExecutionException, TimeUnit} + import joptsimple.OptionParser import kafka.utils._ -import org.I0Itec.zkclient.ZkClient + +import collection.JavaConverters._ import org.I0Itec.zkclient.exception.ZkNodeExistsException -import kafka.common.{TopicAndPartition, AdminCommandFailedException} +import kafka.common.{AdminCommandFailedException, TopicAndPartition} +import kafka.utils.CommandLineUtils.printUsageAndDie +import org.apache.kafka.clients.admin.AdminClientConfig +import org.apache.kafka.common.KafkaFuture +import org.apache.kafka.common.errors.TimeoutException + import collection._ import org.apache.kafka.common.utils.Utils import org.apache.kafka.common.security.JaasUtils @@ -28,6 +37,17 @@ import org.apache.kafka.common.security.JaasUtils object PreferredReplicaLeaderElectionCommand extends Logging { def main(args: Array[String]): Unit = { + try { + run(args) + } catch { + case e: Throwable => + println("Failed to start preferred replica election") + println(Utils.stackTrace(e)) + System.exit(1); + } + } + /** Basically the same as main, but throws rather than calling System.exit */ + def run(args: Array[String]): Unit = { val parser = new OptionParser(false) val jsonFileOpt = parser.accepts("path-to-json-file", "The JSON file with the list of partitions " + "for which preferred replica leader election should be done, in the following format - \n" + @@ -36,66 +56,64 @@ object PreferredReplicaLeaderElectionCommand extends Logging { .withRequiredArg .describedAs("list of partitions for which preferred replica leader election needs to be triggered") .ofType(classOf[String]) - val zkConnectOpt = parser.accepts("zookeeper", "REQUIRED: The connection string for the zookeeper connection in the " + - "form host:port. Multiple URLS can be given to allow fail-over.") + val zkConnectOpt = parser.accepts("zookeeper", "DEPRECATED. The connection string for the zookeeper connection in the " + + "form host:port. Multiple URLS can be given to allow fail-over. REQUIRED unless --bootstrap-server is given.") .withRequiredArg .describedAs("urls") .ofType(classOf[String]) + val bootstrapServerOpt = parser.accepts("bootstrap-server", "A hostname and port for the broker to connect to, " + + "in the form host:port. Multiple URLs can be given. REQUIRED unless --zookeeper is given.") + .withRequiredArg + .describedAs("host:port") + .ofType(classOf[String]) if(args.length == 0) CommandLineUtils.printUsageAndDie(parser, "This tool causes leadership for each partition to be transferred back to the 'preferred replica'," + - " it can be used to balance leadership among the servers.") + " it can be used to balance leadership among the servers." + + " The preferred replica is the first one in the replica assignment (see kafka-reassign-partitions.sh)." + + " Using this command is not necessary when the broker is configured with \"auto.leader.rebalance.enable=true\".") val options = parser.parse(args : _*) - CommandLineUtils.checkRequiredArgs(parser, options, zkConnectOpt) + if (options.has(bootstrapServerOpt) && options.has(zkConnectOpt) + || !options.has(bootstrapServerOpt) && !options.has(zkConnectOpt)) { + printUsageAndDie(parser, "Exactly one of \"" + bootstrapServerOpt+ "\" or \"" + zkConnectOpt + "\" must be provided") + } + + val partitionsForPreferredReplicaElection = + if (!options.has(jsonFileOpt)) + None + else + Some(parsePreferredReplicaElectionData(Utils.readFileAsString(options.valueOf(jsonFileOpt)))) + + val preferredReplicaElectionCommand = if (options.has(zkConnectOpt)) { + Console.err.println(zkConnectOpt + " is deprecated and will be removed in a future version of Kafka.") + Console.err.println("Use " + bootstrapServerOpt + " instead to specify a broker to connect to.") + new ZkPreferredReplicaLeaderElectionCommand(ZkUtils(options.valueOf(zkConnectOpt), + 30000, + 30000, + JaasUtils.isZkSecurityEnabled())) + } else { + new AdminClientPreferredReplicaLeaderElectionCommand(options.valueOf(bootstrapServerOpt)) + } - val zkConnect = options.valueOf(zkConnectOpt) - var zkClient: ZkClient = null - var zkUtils: ZkUtils = null try { - zkClient = ZkUtils.createZkClient(zkConnect, 30000, 30000) - zkUtils = ZkUtils(zkConnect, - 30000, - 30000, - JaasUtils.isZkSecurityEnabled()) - val partitionsForPreferredReplicaElection = - if (!options.has(jsonFileOpt)) - zkUtils.getAllPartitions() - else - parsePreferredReplicaElectionData(Utils.readFileAsString(options.valueOf(jsonFileOpt))) - val preferredReplicaElectionCommand = new PreferredReplicaLeaderElectionCommand(zkUtils, partitionsForPreferredReplicaElection) - - preferredReplicaElectionCommand.moveLeaderToPreferredReplica() - } catch { - case e: Throwable => - println("Failed to start preferred replica election") - println(Utils.stackTrace(e)) + preferredReplicaElectionCommand.moveLeaderToPreferredReplica(partitionsForPreferredReplicaElection) } finally { - if (zkClient != null) - zkClient.close() + preferredReplicaElectionCommand.close() } } def parsePreferredReplicaElectionData(jsonString: String): immutable.Set[TopicAndPartition] = { - Json.parseFull(jsonString) match { - case Some(js) => - js.asJsonObject.get("partitions") match { - case Some(partitionsList) => - val partitionsRaw = partitionsList.asJsonArray.iterator.map(_.asJsonObject) - val partitions = partitionsRaw.map { p => - val topic = p("topic").to[String] - val partition = p("partition").to[Int] - TopicAndPartition(topic, partition) - }.toBuffer - val duplicatePartitions = CoreUtils.duplicates(partitions) - if (duplicatePartitions.nonEmpty) - throw new AdminOperationException("Preferred replica election data contains duplicate partitions: %s".format(duplicatePartitions.mkString(","))) - partitions.toSet - case None => throw new AdminOperationException("Preferred replica election data is empty") - } - case None => throw new AdminOperationException("Preferred replica election data is empty") + val partitionsUndergoingPreferredReplicaElection = + ZkUtils.parsePreferredReplicaElectionDataWithoutDedup(jsonString) + if (partitionsUndergoingPreferredReplicaElection.isEmpty) { + throw new AdminOperationException("Preferred replica election data is empty") } + val duplicatePartitions = CoreUtils.duplicates(partitionsUndergoingPreferredReplicaElection) + if (duplicatePartitions.nonEmpty) + throw new AdminOperationException("Preferred replica election data contains duplicate partitions: %s".format(duplicatePartitions.mkString(","))) + return partitionsUndergoingPreferredReplicaElection.toSet } def writePreferredReplicaElectionData(zkUtils: ZkUtils, @@ -116,15 +134,95 @@ object PreferredReplicaLeaderElectionCommand extends Logging { } } -class PreferredReplicaLeaderElectionCommand(zkUtils: ZkUtils, partitionsFromUser: scala.collection.Set[TopicAndPartition]) { - def moveLeaderToPreferredReplica() = { +trait PreferredReplicaLeaderElectionCommand { + /** + * Move the given partitions to their preferred leader. If the given partitions are none then move all partitions. + */ + def moveLeaderToPreferredReplica(partitionsFromUser: Option[scala.collection.Set[TopicAndPartition]]) : Unit + def close() : Unit +} + +class AdminClientPreferredReplicaLeaderElectionCommand(bootstrapServers: String) extends PreferredReplicaLeaderElectionCommand with Logging { + val props = new Properties() + props.setProperty(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers) + val adminClient: org.apache.kafka.clients.admin.AdminClient = org.apache.kafka.clients.admin.AdminClient.create(props) + + /** + * Wait until the given future has completed, then return whether it completed exceptionally. + * Because KafkaFuture.isCompletedExceptionally doesn't wait for a result + */ + private def completedExceptionally[T](future: KafkaFuture[T]): Boolean = { + try { + future.get() + false + } catch { + case (_: Throwable) => + true + } + } + + override def moveLeaderToPreferredReplica(partitionsFromUser: Option[Set[TopicAndPartition]]): Unit = { + val partitions = partitionsFromUser match { + case Some(partitionsFromUser) => partitionsFromUser.map(tp => tp.asTopicPartition).toSet.asJava + case None => null + } + debug(s"Calling AdminClient.electPreferredLeaders($partitions)") + val result = adminClient.electPreferredLeaders(partitions) + // wait for all results + val attemptedPartitions = try { + result.partitions().get.asScala + } catch { + case e: TimeoutException => + // We timed out, or don't even know the attempted partitions + println("Timeout waiting for election results") + return + case e: Throwable => + // We don't even know the attempted partitions + println("Error while making request") + e.printStackTrace() + return + } + val (exceptional, ok) = attemptedPartitions.map(tp => tp -> result.partitionResult(tp)). + partition{case (_, partitionResult) => completedExceptionally(partitionResult)} + if (!ok.isEmpty) { + println("Successfully completed preferred replica election for partitions %s".format(ok.map(_._1).mkString(", "))) + } + if (!exceptional.isEmpty) { + val adminException = new AdminCommandFailedException(s"${exceptional.size} preferred replica(s) could not be elected") + for ((partition, void) <- exceptional) { + val exception = try { + void.get() + new AdminCommandFailedException("Exceptional future with no exception") + } catch { + case e: ExecutionException => e.getCause + } + println(s"Error completing preferred replica election for partition $partition: $exception") + adminException.addSuppressed(exception) + } + throw adminException + } + } + + override def close(): Unit = { + debug("Closing AdminClient") + adminClient.close() + } +} + +class ZkPreferredReplicaLeaderElectionCommand(zkUtils: ZkUtils) extends PreferredReplicaLeaderElectionCommand { + + override def moveLeaderToPreferredReplica(partitionsFromUser: Option[scala.collection.Set[TopicAndPartition]]) = { try { - val topics = partitionsFromUser.map(_.topic).toSet + val partitions = partitionsFromUser match { + case Some(partitionsFromUser) => partitionsFromUser + case None => zkUtils.getAllPartitions() + } + val topics = partitions.map(_.topic).toSet val partitionsFromZk = zkUtils.getPartitionsForTopics(topics.toSeq).flatMap{ case (topic, partitions) => partitions.map(TopicAndPartition(topic, _)) }.toSet - val (validPartitions, invalidPartitions) = partitionsFromUser.partition(partitionsFromZk.contains) + val (validPartitions, invalidPartitions) = partitions.partition(partitionsFromZk.contains) PreferredReplicaLeaderElectionCommand.writePreferredReplicaElectionData(zkUtils, validPartitions) println("Successfully started preferred replica election for partitions %s".format(validPartitions)) @@ -133,4 +231,8 @@ class PreferredReplicaLeaderElectionCommand(zkUtils: ZkUtils, partitionsFromUser case e: Throwable => throw new AdminCommandFailedException("Admin command failed", e) } } + + override def close(): Unit = { + zkUtils.close() + } } diff --git a/core/src/main/scala/kafka/cluster/Partition.scala b/core/src/main/scala/kafka/cluster/Partition.scala index 1e3ab75477bc0..0089b0fb1becb 100755 --- a/core/src/main/scala/kafka/cluster/Partition.scala +++ b/core/src/main/scala/kafka/cluster/Partition.scala @@ -430,6 +430,7 @@ class Partition(val topic: String, replicaManager.tryCompleteDelayedFetch(requestKey) replicaManager.tryCompleteDelayedProduce(requestKey) replicaManager.tryCompleteDelayedDeleteRecords(requestKey) + replicaManager.tryCompleteElection(requestKey) } def maybeShrinkIsr(replicaMaxLagTimeMs: Long) { diff --git a/core/src/main/scala/kafka/controller/KafkaController.scala b/core/src/main/scala/kafka/controller/KafkaController.scala index 811ff6770c3c5..3cc8cb0450536 100644 --- a/core/src/main/scala/kafka/controller/KafkaController.scala +++ b/core/src/main/scala/kafka/controller/KafkaController.scala @@ -33,7 +33,7 @@ import org.I0Itec.zkclient.{IZkChildListener, IZkDataListener, IZkStateListener} import org.apache.kafka.common.errors.{BrokerNotAvailableException, ControllerMovedException} import org.apache.kafka.common.metrics.Metrics import org.apache.kafka.common.protocol.{ApiKeys, Errors} -import org.apache.kafka.common.requests.{AbstractRequest, AbstractResponse, StopReplicaResponse, LeaderAndIsrResponse} +import org.apache.kafka.common.requests.{AbstractRequest, AbstractResponse, ApiError, LeaderAndIsrResponse, StopReplicaResponse} import org.apache.kafka.common.utils.Time import org.apache.zookeeper.Watcher.Event.KeeperState @@ -627,13 +627,19 @@ class KafkaController(val config: KafkaConfig, zkUtils: ZkUtils, time: Time, met } def onPreferredReplicaElection(partitions: Set[TopicAndPartition], isTriggeredByAutoRebalance: Boolean = false) { + onPreferredReplicaElectionWithResults(partitions, isTriggeredByAutoRebalance).foreach(entry => + error("Error completing preferred replica leader election for partition %s".format(entry._1), entry._2) + ) + } + + private def onPreferredReplicaElectionWithResults(partitions: Set[TopicAndPartition], isTriggeredByAutoRebalance: Boolean = false, viaZk: Boolean = true): Map[TopicAndPartition, Throwable] = { info("Starting preferred replica leader election for partitions %s".format(partitions.mkString(","))) try { - partitionStateMachine.handleStateChanges(partitions, OnlinePartition, preferredReplicaPartitionLeaderSelector) - } catch { - case e: Throwable => error("Error completing preferred replica leader election for partitions %s".format(partitions.mkString(",")), e) + partitionStateMachine.handleStateChangesWithResults(partitions, OnlinePartition, preferredReplicaPartitionLeaderSelector) } finally { - removePartitionsFromPreferredReplicaElection(partitions, isTriggeredByAutoRebalance) + if (viaZk) { + removePartitionsFromPreferredReplicaElection(partitions, isTriggeredByAutoRebalance) + } } } @@ -1427,18 +1433,45 @@ class KafkaController(val config: KafkaConfig, zkUtils: ZkUtils, time: Time, met } } - case class PreferredReplicaLeaderElection(partitions: Set[TopicAndPartition]) extends ControllerEvent { + /** + * ControllerEvent to elect preferred leaders. + * + * @param partitions The partitions where the preferred leader should be elected + * @param viaZk Whether the election is triggered as a result of a change in the /admin/preferred_replica_election znode + * @param callback A callback invoked with the results of the election. + * The first argument is the set of partitions where an election was preformed successfully + * The second argument is a map of the partition to ApiError where either the election was not performed + * or the election was performed but there was an error. This map may have entries with + * ApiError.NONE values if the partition was already led by the preferred leader. + */ + case class PreferredReplicaLeaderElection(partitions: Set[TopicAndPartition], viaZk: Boolean = true, callback: (Set[TopicAndPartition], Map[TopicAndPartition, ApiError])=>Unit = { (_,_) => }) extends ControllerEvent { def state = ControllerState.ManualLeaderBalance override def process(): Unit = { - if (!isActive) return - val partitionsForTopicsToBeDeleted = partitions.filter(p => topicDeletionManager.isTopicQueuedUpForDeletion(p.topic)) - if (partitionsForTopicsToBeDeleted.nonEmpty) { - error("Skipping preferred replica election for partitions %s since the respective topics are being deleted" - .format(partitionsForTopicsToBeDeleted)) + if (!isActive) { + callback(Set(), partitions.map(partition => partition -> new ApiError(Errors.NOT_CONTROLLER, null)).toMap) + } else { + val (partitionsForTopicsToBeDeleted, livePartitions) = partitions.partition(partition => topicDeletionManager.isTopicQueuedUpForDeletion(partition.topic)) + if (partitionsForTopicsToBeDeleted.nonEmpty) { + error("Skipping preferred replica election for partitions %s since the respective topics are being deleted" + .format(partitionsForTopicsToBeDeleted)) + } + // partition those where preferred is already leader + val (electablePartitions, alreadyPreferred) = livePartitions.partition(partition => { + val assignedReplicas = controllerContext.partitionReplicaAssignment(partition) + val preferredReplica = assignedReplicas.head + val currentLeader = controllerContext.partitionLeadershipInfo(partition).leaderAndIsr.leader + currentLeader != preferredReplica + }) + + val electionErrors = onPreferredReplicaElectionWithResults(electablePartitions, viaZk=viaZk) + val successfulPartitions = electablePartitions -- electionErrors.keySet + val results = electionErrors.map(e => e._1 -> ApiError.fromThrowable(e._2)) ++ alreadyPreferred.map(_ -> ApiError.NONE) ++ partitionsForTopicsToBeDeleted.map(_-> new ApiError(Errors.INVALID_TOPIC_EXCEPTION, "The topic is being deleted")) + debug("PreferredReplicaLeaderElection waiting: %s, results: %s".format(successfulPartitions, results)) + callback(successfulPartitions, results) } - onPreferredReplicaElection(partitions -- partitionsForTopicsToBeDeleted) + } } @@ -1698,6 +1731,13 @@ class KafkaController(val config: KafkaConfig, zkUtils: ZkUtils, time: Time, met triggerControllerMove() } } + + def electPreferredLeaders(partitions: Set[TopicAndPartition], callback: (Set[TopicAndPartition], Map[TopicAndPartition, ApiError])=>Unit = { (_,_) => }): Unit = { + eventManager.put( + PreferredReplicaLeaderElection(partitions, + callback = callback + )) + } } /** diff --git a/core/src/main/scala/kafka/controller/PartitionStateMachine.scala b/core/src/main/scala/kafka/controller/PartitionStateMachine.scala index 9e75bc0006819..5a32d6d0aa9e7 100755 --- a/core/src/main/scala/kafka/controller/PartitionStateMachine.scala +++ b/core/src/main/scala/kafka/controller/PartitionStateMachine.scala @@ -81,7 +81,7 @@ class PartitionStateMachine(controller: KafkaController, stateChangeLogger: Stat for ((topicAndPartition, partitionState) <- partitionState if !controller.topicDeletionManager.isTopicQueuedUpForDeletion(topicAndPartition.topic)) { if (partitionState.equals(OfflinePartition) || partitionState.equals(NewPartition)) - handleStateChange(topicAndPartition.topic, topicAndPartition.partition, OnlinePartition, controller.offlinePartitionSelector, + handleStateChange(topicAndPartition, OnlinePartition, controller.offlinePartitionSelector, (new CallbackBuilder).build) } brokerRequestBatch.sendRequestsToBrokers(controller.epoch) @@ -107,7 +107,7 @@ class PartitionStateMachine(controller: KafkaController, stateChangeLogger: Stat try { brokerRequestBatch.newBatch() partitions.foreach { topicAndPartition => - handleStateChange(topicAndPartition.topic, topicAndPartition.partition, targetState, leaderSelector, callbacks) + handleStateChange(topicAndPartition, targetState, leaderSelector, callbacks) } brokerRequestBatch.sendRequestsToBrokers(controller.epoch) } catch { @@ -116,6 +116,35 @@ class PartitionStateMachine(controller: KafkaController, stateChangeLogger: Stat } } + /** + * Attempt to put each of the given partitions into the given targetState, returning + * a map of partition to exception for those partitions where the attempt failed with an exception. + * @param partitions The list of partitions that need to be transitioned to the target state + * @param targetState The state that the partitions should be moved to + */ + def handleStateChangesWithResults(partitions: Set[TopicAndPartition], targetState: PartitionState, + leaderSelector: PartitionLeaderSelector = noOpPartitionLeaderSelector, + callbacks: Callbacks = (new CallbackBuilder).build): Map[TopicAndPartition, Throwable] = { + info("Invoking state change to %s for partitions %s".format(targetState, partitions.mkString(","))) + val result = new mutable.HashMap[TopicAndPartition, Throwable]() + try { + brokerRequestBatch.newBatch() + partitions.foreach { topicAndPartition => { + handleStateChange(topicAndPartition, targetState, leaderSelector, callbacks) match { + case Some(exception) => result.put(topicAndPartition, exception) + case None => + } + }} + brokerRequestBatch.sendRequestsToBrokers(controller.epoch) + } catch { + case e: Throwable => + error("Error changing state of some partitions to %s state".format(targetState), e) + (partitions -- result.keySet).foreach(result.put(_, e)) + } + result + } + + /** * This API exercises the partition's state machine. It ensures that every state transition happens from a legal * previous state to the target state. Valid state transitions are: @@ -135,14 +164,14 @@ class PartitionStateMachine(controller: KafkaController, stateChangeLogger: Stat * * OfflinePartition -> NonExistentPartition * --nothing other than marking the partition state as NonExistentPartition - * @param topic The topic of the partition for which the state transition is invoked - * @param partition The partition for which the state transition is invoked + * @param topicAndPartition The topic and partition for which the state transition is invoked * @param targetState The end state that the partition should be moved to + * @param callbacks + * @return The exception, if the state transition failed, otherwise None */ - private def handleStateChange(topic: String, partition: Int, targetState: PartitionState, + private def handleStateChange(topicAndPartition: TopicAndPartition, targetState: PartitionState, leaderSelector: PartitionLeaderSelector, - callbacks: Callbacks) { - val topicAndPartition = TopicAndPartition(topic, partition) + callbacks: Callbacks): Option[Throwable] = { val currState = partitionState.getOrElseUpdate(topicAndPartition, NonExistentPartition) val stateChangeLog = stateChangeLogger.withControllerEpoch(controller.epoch) try { @@ -160,9 +189,9 @@ class PartitionStateMachine(controller: KafkaController, stateChangeLogger: Stat // initialize leader and isr path for new partition initializeLeaderAndIsrForPartition(topicAndPartition) case OfflinePartition => - electLeaderForPartition(topic, partition, leaderSelector) + electLeaderForPartition(topicAndPartition.topic, topicAndPartition.partition, leaderSelector) case OnlinePartition => // invoked when the leader needs to be re-elected - electLeaderForPartition(topic, partition, leaderSelector) + electLeaderForPartition(topicAndPartition.topic, topicAndPartition.partition, leaderSelector) case _ => // should never come here since illegal previous states are checked above } partitionState.put(topicAndPartition, OnlinePartition) @@ -179,10 +208,12 @@ class PartitionStateMachine(controller: KafkaController, stateChangeLogger: Stat partitionState.put(topicAndPartition, NonExistentPartition) // post: partition state is deleted from all brokers and zookeeper } + None } catch { case t: Throwable => stateChangeLog.error(s"Initiated state change for partition $topicAndPartition from $currState to $targetState failed", t) + Some(t) } } diff --git a/core/src/main/scala/kafka/server/DelayedElectPreferredLeader.scala b/core/src/main/scala/kafka/server/DelayedElectPreferredLeader.scala new file mode 100644 index 0000000000000..299136a870678 --- /dev/null +++ b/core/src/main/scala/kafka/server/DelayedElectPreferredLeader.scala @@ -0,0 +1,96 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kafka.server + +import java.util.concurrent.atomic.AtomicReference + +import kafka.common.TopicAndPartition +import kafka.controller.ControllerContext +import org.apache.kafka.common.TopicPartition +import org.apache.kafka.common.protocol.Errors +import org.apache.kafka.common.requests.ApiError + +import scala.collection.{Map, Set, mutable} + +/** + * A delayed elect preferred leader operation that can be created by the replica manager and watched + * in the elect preferred leader purgatory + */ +class DelayedElectPreferredLeader(delayMs: Long, + waiting: Set[TopicAndPartition], + results: Map[TopicAndPartition, ApiError], + replicaManager: ReplicaManager, + controllerContext: ControllerContext, + responseCallback: Map[TopicAndPartition, ApiError] => Unit) + extends DelayedOperation(delayMs) { + + val expectedLeaders: Map[TopicAndPartition, Int]= waiting.map(tp => tp -> controllerContext.partitionReplicaAssignment(tp).head).toMap + var waitingPartitions: Set[TopicAndPartition] = new mutable.HashSet[TopicAndPartition]() ++= waiting + val fullResults = new mutable.HashSet() ++= results + + /** + * Call-back to execute when a delayed operation gets expired and hence forced to complete. + */ + override def onExpiration(): Unit = {} + + /** + * Process for completing an operation; This function needs to be defined + * in subclasses and will be called exactly once in forceComplete() + */ + override def onComplete(): Unit = { + // This could be called to force complete, so I need the full list of partitions, so I can time them all out. + updateWaiting() + val timedout = waitingPartitions.map(partition => partition -> new ApiError(Errors.REQUEST_TIMED_OUT, null)).toMap + responseCallback(timedout ++ fullResults) + } + + private def timeoutWaiting = { + waitingPartitions.map(partition => partition -> new ApiError(Errors.REQUEST_TIMED_OUT, null)).toMap + } + + /** + * Try to complete the delayed operation by first checking if the operation + * can be completed by now. If yes execute the completion logic by calling + * forceComplete() and return true iff forceComplete returns true; otherwise return false + * + * This function needs to be defined in subclasses + */ + override def tryComplete(): Boolean = { + updateWaiting() + debug("tryComplete() waitingPartitions: %s".format(waitingPartitions)) + if (waitingPartitions.isEmpty) + forceComplete() + else + false + } + + private def updateWaiting() = { + waitingPartitions.foreach(tp => { + val ps = replicaManager.metadataCache.getPartitionInfo(tp.topic, tp.partition) + ps match { + case Some(ps) => + if (expectedLeaders(tp) == ps.basePartitionState.leader) { + waitingPartitions -= tp + fullResults += tp -> ApiError.NONE + } + case None => + } + } + ) + } +} diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala index 006670242fd6f..59f4d38b84ba3 100644 --- a/core/src/main/scala/kafka/server/KafkaApis.scala +++ b/core/src/main/scala/kafka/server/KafkaApis.scala @@ -134,6 +134,7 @@ class KafkaApis(val requestChannel: RequestChannel, case ApiKeys.DESCRIBE_LOG_DIRS => handleDescribeLogDirsRequest(request) case ApiKeys.SASL_AUTHENTICATE => handleSaslAuthenticateRequest(request) case ApiKeys.CREATE_PARTITIONS => handleCreatePartitionsRequest(request) + case ApiKeys.ELECT_PREFERRED_LEADERS => handleElectPreferredReplicaLeader(request) } } catch { case e: FatalExitError => throw e @@ -223,6 +224,11 @@ class KafkaApis(val requestChannel: RequestChannel, adminManager.tryCompleteDelayedTopicOperations(topic) } } + if (replicaManager.hasDelayedElectionOperations) { + updateMetadataRequest.partitionStates.asScala.foreach{ case (tp, ps) => { + replicaManager.tryCompleteElection(new TopicPartitionOperationKey(tp.topic(), tp.partition())) + }} + } sendResponseExemptThrottle(request, new UpdateMetadataResponse(Errors.NONE)) } else { sendResponseMaybeThrottle(request, _ => new UpdateMetadataResponse(Errors.CLUSTER_AUTHORIZATION_FAILED)) @@ -1926,6 +1932,26 @@ class KafkaApis(val requestChannel: RequestChannel, new AlterConfigsResponse(requestThrottleMs, (authorizedResult ++ unauthorizedResult).asJava)) } + def handleElectPreferredReplicaLeader(request: RequestChannel.Request): Unit = { + + val electionRequest = request.body[ElectPreferredLeadersRequest] + val partitions = + if (electionRequest.topicPartitions() == null) { + zkUtils.getAllPartitions() + } else { + electionRequest.topicPartitions().asScala.map(t => new TopicAndPartition(t)) + } + def sendResponseCallback(result: Map[TopicAndPartition, ApiError]): Unit = { + sendResponseMaybeThrottle(request, requestThrottleMs => + new ElectPreferredLeadersResponse(requestThrottleMs, result.map(entry => entry._1.asTopicPartition -> entry._2).asJava)) + } + if (!authorize(request.session, Alter, Resource.ClusterResource)) { + sendResponseCallback(partitions.map(partition => partition -> new ApiError(Errors.CLUSTER_AUTHORIZATION_FAILED, null)).toMap) + } else { + replicaManager.electPreferredLeaders(controller, partitions, sendResponseCallback, electionRequest.timeout) + } + } + private def configsAuthorizationApiError(session: RequestChannel.Session, resource: RResource): ApiError = { val error = resource.`type` match { case RResourceType.BROKER => Errors.CLUSTER_AUTHORIZATION_FAILED diff --git a/core/src/main/scala/kafka/server/ReplicaManager.scala b/core/src/main/scala/kafka/server/ReplicaManager.scala index 755a043b8a910..daa750095c9b2 100644 --- a/core/src/main/scala/kafka/server/ReplicaManager.scala +++ b/core/src/main/scala/kafka/server/ReplicaManager.scala @@ -23,6 +23,7 @@ import java.util.concurrent.atomic.{AtomicBoolean, AtomicLong} import com.yammer.metrics.core.Gauge import kafka.api._ import kafka.cluster.{Partition, Replica} +import kafka.common.TopicAndPartition import kafka.controller.{KafkaController, StateChangeLogger} import kafka.log.{Log, LogAppendInfo, LogManager} import kafka.metrics.KafkaMetricsGroup @@ -149,6 +150,7 @@ class ReplicaManager(val config: KafkaConfig, val delayedProducePurgatory: DelayedOperationPurgatory[DelayedProduce], val delayedFetchPurgatory: DelayedOperationPurgatory[DelayedFetch], val delayedDeleteRecordsPurgatory: DelayedOperationPurgatory[DelayedDeleteRecords], + val delayedElectPreferredLeaderPurgatory: DelayedOperationPurgatory[DelayedElectPreferredLeader], threadNamePrefix: Option[String]) extends Logging with KafkaMetricsGroup { def this(config: KafkaConfig, @@ -174,6 +176,8 @@ class ReplicaManager(val config: KafkaConfig, DelayedOperationPurgatory[DelayedDeleteRecords]( purgatoryName = "DeleteRecords", brokerId = config.brokerId, purgeInterval = config.deleteRecordsPurgatoryPurgeIntervalRequests), + DelayedOperationPurgatory[DelayedElectPreferredLeader]( + purgatoryName = "ElectPreferredLeader", brokerId = config.brokerId), threadNamePrefix) } @@ -313,6 +317,13 @@ class ReplicaManager(val config: KafkaConfig, debug("Request key %s unblocked %d DeleteRecordsRequest.".format(key.keyLabel, completed)) } + def hasDelayedElectionOperations = delayedElectPreferredLeaderPurgatory.delayed != 0 + + def tryCompleteElection(key: DelayedOperationKey): Unit = { + val completed = delayedElectPreferredLeaderPurgatory.checkAndComplete(key) + debug("Request key %s unblocked %d ElectPreferredLeader.".format(key.keyLabel, completed)) + } + def startup() { // start ISR expiration thread // A follower can lag behind leader for up to config.replicaLagTimeMaxMs x 1.5 before it is removed from ISR @@ -1411,6 +1422,7 @@ class ReplicaManager(val config: KafkaConfig, delayedFetchPurgatory.shutdown() delayedProducePurgatory.shutdown() delayedDeleteRecordsPurgatory.shutdown() + delayedElectPreferredLeaderPurgatory.shutdown() if (checkpointHW) checkpointHighWatermarks() info("Shut down completely") @@ -1434,5 +1446,36 @@ class ReplicaManager(val config: KafkaConfig, tp -> epochEndOffset } } + + def electPreferredLeaders(controller: KafkaController, + partitions: Set[TopicAndPartition], + responseCallback: Map[TopicAndPartition, ApiError] => Unit, + requestTimeout: Long): Unit = { + val partitionsFromZk = zkUtils.getPartitionsForTopics(partitions.map(_.topic).toSeq).flatMap { case (topic, partitions) => + partitions.map(TopicAndPartition(topic, _)) + }.toSet + val (validPartitions, invalidPartitions) = partitions.partition(partitionsFromZk.contains) + val invalidPartitionsResults = invalidPartitions.map(p => { + val msg = s"Skipping preferred replica leader election for partition ${p} since it doesn't exist." + logger.info(msg) + p -> new ApiError(Errors.UNKNOWN_TOPIC_OR_PARTITION, msg) + }).toMap + + def electionCallback(waiting: Set[TopicAndPartition], results: Map[TopicAndPartition, ApiError]) = { + if (waiting.nonEmpty) { + // timeout + val watchKeys = waiting.map(p => new TopicPartitionOperationKey(p.topic, p.partition)).toSeq + delayedElectPreferredLeaderPurgatory.tryCompleteElseWatch( + new DelayedElectPreferredLeader(requestTimeout, waiting, results ++ invalidPartitionsResults, + this, controller.controllerContext, responseCallback), + watchKeys) + } else { + // There are no partitions actually being elected, so return immediately + responseCallback(results ++ invalidPartitionsResults) + } + } + + controller.electPreferredLeaders(validPartitions, electionCallback) + } } diff --git a/core/src/main/scala/kafka/utils/ZkUtils.scala b/core/src/main/scala/kafka/utils/ZkUtils.scala index 1df0916b4be42..0c3a14c332f0d 100644 --- a/core/src/main/scala/kafka/utils/ZkUtils.scala +++ b/core/src/main/scala/kafka/utils/ZkUtils.scala @@ -206,6 +206,28 @@ object ZkUtils { Json.encode(Map("version" -> 1, "partitions" -> partitions.map(tp => Map("topic" -> tp.topic, "partition" -> tp.partition)))) } + def parsePreferredReplicaElectionDataWithoutDedup(jsonString: String): Seq[TopicAndPartition] = { + Json.parseFull(jsonString) match { + case Some(js) => + js.asJsonObject.get("partitions") match { + case Some(partitionsList) => + val partitionsRaw = partitionsList.asJsonArray.iterator.map(_.asJsonObject) + val partitions = partitionsRaw.map { p => + val topic = p("topic").to[String] + val partition = p("partition").to[Int] + TopicAndPartition(topic, partition) + }.toBuffer + partitions + case None => Seq.empty[TopicAndPartition] + } + case None => Seq.empty[TopicAndPartition] + } + } + + def parsePreferredReplicaElectionData(jsonString: String): Set[TopicAndPartition] = { + parsePreferredReplicaElectionDataWithoutDedup(jsonString).toSet + } + def formatAsReassignmentJson(partitionsToBeReassigned: Map[TopicAndPartition, Seq[Int]]): String = { Json.encode(Map( "version" -> 1, diff --git a/core/src/test/scala/unit/kafka/admin/AdminTest.scala b/core/src/test/scala/unit/kafka/admin/AdminTest.scala index 7f4eed77b312a..3fecc89f0ad04 100755 --- a/core/src/test/scala/unit/kafka/admin/AdminTest.scala +++ b/core/src/test/scala/unit/kafka/admin/AdminTest.scala @@ -350,8 +350,8 @@ class AdminTest extends ZooKeeperTestHarness with Logging with RackAwareTest { // broker 2 should be the leader since it was started first val currentLeader = TestUtils.waitUntilLeaderIsElectedOrChanged(zkUtils, topic, partition, oldLeaderOpt = None) // trigger preferred replica election - val preferredReplicaElection = new PreferredReplicaLeaderElectionCommand(zkUtils, Set(TopicAndPartition(topic, partition))) - preferredReplicaElection.moveLeaderToPreferredReplica() + val preferredReplicaElection = new ZkPreferredReplicaLeaderElectionCommand(zkUtils) + preferredReplicaElection.moveLeaderToPreferredReplica(Some(Set(TopicAndPartition(topic, partition)))) val newLeader = TestUtils.waitUntilLeaderIsElectedOrChanged(zkUtils, topic, partition, oldLeaderOpt = Some(currentLeader)) assertEquals("Preferred replica election failed", preferredReplica, newLeader) } diff --git a/core/src/test/scala/unit/kafka/admin/PreferredReplicaLeaderElectionCommandTest.scala b/core/src/test/scala/unit/kafka/admin/PreferredReplicaLeaderElectionCommandTest.scala new file mode 100644 index 0000000000000..7aab1071ba9a3 --- /dev/null +++ b/core/src/test/scala/unit/kafka/admin/PreferredReplicaLeaderElectionCommandTest.scala @@ -0,0 +1,307 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package unit.kafka.admin + +import java.io.File +import java.nio.charset.StandardCharsets +import java.nio.file.{Files, Paths} +import java.util +import java.util.Properties + +import kafka.admin.{AdminUtils, PreferredReplicaLeaderElectionCommand} +import kafka.common.{AdminCommandFailedException, TopicAndPartition} +import kafka.network.RequestChannel +import kafka.security.auth._ +import kafka.server.{KafkaConfig, KafkaServer} +import kafka.utils.{Logging, TestUtils, ZkUtils} +import kafka.zk.ZooKeeperTestHarness +import org.apache.kafka.common.TopicPartition +import org.apache.kafka.common.errors.{UnknownServerException, UnknownTopicOrPartitionException} +import org.apache.kafka.common.network.ListenerName +import org.apache.kafka.common.security.auth.KafkaPrincipal +import org.junit.Assert._ +import org.junit.{After, Ignore, Test} + +import scala.concurrent.ExecutionException + +class PreferredReplicaLeaderElectionCommandTest extends ZooKeeperTestHarness with Logging /*with RackAwareTest*/ { + + var servers: Seq[KafkaServer] = Seq() + + @After + override def tearDown() { + TestUtils.shutdownServers(servers) + super.tearDown() + } + + private def createTestTopicAndCluster(topicPartition: Map[TopicPartition, List[Int]], authorizer: Option[String] = None) { + + val brokerConfigs = TestUtils.createBrokerConfigs(3, zkConnect, false) + brokerConfigs.foreach(p => p.setProperty("auto.leader.rebalance.enable", "false") + ) + authorizer match { + case Some(className) => + brokerConfigs.foreach(p => p.setProperty("authorizer.class.name", className)) + case None => + } + createTestTopicAndCluster(topicPartition,brokerConfigs) + } + + private def createTestTopicAndCluster(partitionsAndAssignments: Map[TopicPartition, List[Int]], brokerConfigs: Seq[Properties]) { + // create brokers + servers = brokerConfigs.map(b => TestUtils.createServer(KafkaConfig.fromProps(b))) + // create the topic + partitionsAndAssignments.foreach(partitionAndAssignment => + AdminUtils.createOrUpdateTopicPartitionAssignmentPathInZK(zkUtils, partitionAndAssignment._1.topic(), + Map(partitionAndAssignment._1.partition -> partitionAndAssignment._2))) + // wait until replica log is created on every broker + TestUtils.waitUntilTrue(() => servers.forall(server => partitionsAndAssignments.forall(partitionAndAssignment => server.getLogManager().getLog(partitionAndAssignment._1).isDefined)), + "Replicas for topic test not created") + } + + /** Bounce the given server and wait for all servers to get metadata for the given partition */ + private def bounceServer(server: Int, partition: TopicPartition) { + info(s"Shutting down server $server so a non-preferred replica becomes leader") + servers(server).shutdown() + info(s"Starting server $server now that a non-preferred replica is leader") + servers(server).startup() + TestUtils.waitUntilTrue(() => servers.forall(server => server.metadataCache.getPartitionInfo(partition.topic(), partition.partition()).isDefined), + s"Replicas for partition $partition not created") + + servers + } + + private def getController() = { + servers.find(p => p.kafkaController.isActive) + } + + private def getLeader(topicPartition: TopicPartition) = { + servers(0).metadataCache.getPartitionInfo(topicPartition.topic(), topicPartition.partition()).get.basePartitionState.leader + } + + private def bootstrapServer: String = { + val port = servers(0).socketServer.boundPort(ListenerName.normalised("PLAINTEXT")) + info("Server bound to port "+port) + "localhost:" + port + } + + val testPartition = new TopicPartition("test", 0) + val testPartitionAssignment = List(1, 2, 0) + val testPartitionPreferredLeader = testPartitionAssignment.head + val testPartitionAndAssignment = Map(testPartition -> testPartitionAssignment) + + /** Test the case where no partitions are given (=> elect all partitions) */ + @Test + def testNoPartitionsGiven() { + createTestTopicAndCluster(testPartitionAndAssignment) + bounceServer(testPartitionPreferredLeader, testPartition) + // Check the leader for the partition is not the preferred one + assertNotEquals(testPartitionPreferredLeader, getLeader(testPartition)) + PreferredReplicaLeaderElectionCommand.run(Array( + "--bootstrap-server", bootstrapServer)) + // Check the leader for the partition IS the preferred one + assertEquals(testPartitionPreferredLeader, getLeader(testPartition)) + } + + private def toJsonFile(partitions: scala.collection.Set[TopicPartition]): File = { + val jsonFile = File.createTempFile("preferredreplicaelection", ".js") + jsonFile.deleteOnExit() + val jsonString = ZkUtils.preferredReplicaLeaderElectionZkData(partitions.map(new TopicAndPartition(_))) + info("Using json: "+jsonString) + Files.write(Paths.get(jsonFile.getAbsolutePath), jsonString.getBytes(StandardCharsets.UTF_8)) + jsonFile + } + + /** Test the case where a list of partitions is given */ + @Test + def testSingletonPartitionGiven() { + createTestTopicAndCluster(testPartitionAndAssignment) + bounceServer(testPartitionPreferredLeader, testPartition) + // Check the leader for the partition is not the preferred one + assertNotEquals(testPartitionPreferredLeader, getLeader(testPartition)) + val jsonFile = toJsonFile(testPartitionAndAssignment.keySet) + try { + PreferredReplicaLeaderElectionCommand.run(Array( + "--bootstrap-server", bootstrapServer, + "--path-to-json-file", jsonFile.getAbsolutePath)) + } finally { + jsonFile.delete() + } + // Check the leader for the partition IS the preferred one + assertEquals(testPartitionPreferredLeader, getLeader(testPartition)) + } + + /** Test the case where a topic does not exist */ + @Test + def testTopicDoesNotExist() { + val nonExistentPartition = new TopicPartition("does.not.exist", 0) + val nonExistentPartitionAssignment = List(1, 2, 0) + val nonExistentPartitionAndAssignment = Map(nonExistentPartition -> nonExistentPartitionAssignment) + + createTestTopicAndCluster(testPartitionAndAssignment) + val jsonFile = toJsonFile(nonExistentPartitionAndAssignment.keySet) + try { + PreferredReplicaLeaderElectionCommand.run(Array( + "--bootstrap-server", bootstrapServer, + "--path-to-json-file", jsonFile.getAbsolutePath)) + } catch { + case e: AdminCommandFailedException => + val suppressed = e.getSuppressed()(0) + assertTrue(suppressed.isInstanceOf[UnknownTopicOrPartitionException]) + } finally { + jsonFile.delete() + } + } + + + /** Test the case where several partitions are given */ + @Test + def testMultiplePartitionsSameAssignment() { + val testPartitionA = new TopicPartition("testA", 0) + val testPartitionB = new TopicPartition("testB", 0) + val testPartitionAssignment = List(1, 2, 0) + val testPartitionPreferredLeader = testPartitionAssignment.head + val testPartitionAndAssignment = Map(testPartitionA -> testPartitionAssignment, testPartitionB -> testPartitionAssignment) + + createTestTopicAndCluster(testPartitionAndAssignment) + bounceServer(testPartitionPreferredLeader, testPartitionA) + // Check the leader for the partition is not the preferred one + assertNotEquals(testPartitionPreferredLeader, getLeader(testPartitionA)) + assertNotEquals(testPartitionPreferredLeader, getLeader(testPartitionB)) + val jsonFile = toJsonFile(testPartitionAndAssignment.keySet) + try { + PreferredReplicaLeaderElectionCommand.run(Array( + "--bootstrap-server", bootstrapServer, + "--path-to-json-file", jsonFile.getAbsolutePath)) + } finally { + jsonFile.delete() + } + // Check the leader for the partition IS the preferred one + assertEquals(testPartitionPreferredLeader, getLeader(testPartitionA)) + assertEquals(testPartitionPreferredLeader, getLeader(testPartitionB)) + } + + /** What happens when the preferred replica is already the leader? */ + @Test + def testNoopElection() { + createTestTopicAndCluster(testPartitionAndAssignment) + // Don't bounce the server. Doublec heck the leader for the partition is the preferred one + assertEquals(testPartitionPreferredLeader, getLeader(testPartition)) + val jsonFile = toJsonFile(testPartitionAndAssignment.keySet) + try { + // Now do the election, even though the preferred replica is *already* the leader + PreferredReplicaLeaderElectionCommand.run(Array( + "--bootstrap-server", bootstrapServer, + "--path-to-json-file", jsonFile.getAbsolutePath)) + // Check the leader for the partition still is the preferred one + assertEquals(testPartitionPreferredLeader, getLeader(testPartition)) + } finally { + jsonFile.delete() + } + + } + + /** What happens if the preferred replica is offline? */ + @Test + def testWithOfflinePreferredReplica() { + createTestTopicAndCluster(testPartitionAndAssignment) + bounceServer(testPartitionPreferredLeader, testPartition) + // Check the leader for the partition is not the preferred one + val leader = getLeader(testPartition) + assertNotEquals(testPartitionPreferredLeader, leader) + // Now kill the preferred one + servers(testPartitionPreferredLeader).shutdown() + // Now try to elect the preferred one + val jsonFile = toJsonFile(testPartitionAndAssignment.keySet) + try { + PreferredReplicaLeaderElectionCommand.run(Array( + "--bootstrap-server", bootstrapServer, + "--path-to-json-file", jsonFile.getAbsolutePath)) + fail(); + } catch { + case e: AdminCommandFailedException => + assertEquals("1 preferred replica(s) could not be elected", e.getMessage) + val suppressed = e.getSuppressed()(0) + assertTrue(suppressed.isInstanceOf[UnknownServerException]) + assertTrue(suppressed.getMessage, suppressed.getMessage.contains("is either not alive or not in the isr")) + // Check we still have the same leader + assertEquals(leader, getLeader(testPartition)) + } finally { + jsonFile.delete() + } + } + + /** What happens if the controller gets killed just before an election? */ + @Test + def testTimeout() { + createTestTopicAndCluster(testPartitionAndAssignment) + bounceServer(testPartitionPreferredLeader, testPartition) + // Check the leader for the partition is not the preferred one + val leader = getLeader(testPartition) + assertNotEquals(testPartitionPreferredLeader, leader) + // Now kill the controller just before we trigger the election + servers(getController().get.kafkaController.getControllerID()).shutdown() + val jsonFile = toJsonFile(testPartitionAndAssignment.keySet) + try { + PreferredReplicaLeaderElectionCommand.run(Array( + "--bootstrap-server", bootstrapServer, + "--path-to-json-file", jsonFile.getAbsolutePath)) + fail(); + } catch { + case e: AdminCommandFailedException => + assertEquals("1 preferred replica(s) could not be elected", e.getMessage) + assertTrue(e.getSuppressed()(0).getMessage.contains("Timed out waiting for a node assignment")) + // Check we still have the same leader + assertEquals(leader, getLeader(testPartition)) + } finally { + jsonFile.delete() + } + } + + /** Test the case where a list of partitions is given */ + @Test + def testAuthzFailure() { + createTestTopicAndCluster(testPartitionAndAssignment, Some(classOf[PreferredReplicaLeaderElectionCommandTestAuthorizer].getName)) + bounceServer(testPartitionPreferredLeader, testPartition) + // Check the leader for the partition is not the preferred one + val leader = getLeader(testPartition) + assertNotEquals(testPartitionPreferredLeader, leader) + // Check the leader for the partition is not the preferred one + assertNotEquals(testPartitionPreferredLeader, getLeader(testPartition)) + val jsonFile = toJsonFile(testPartitionAndAssignment.keySet) + try { + PreferredReplicaLeaderElectionCommand.run(Array( + "--bootstrap-server", bootstrapServer, + "--path-to-json-file", jsonFile.getAbsolutePath)) + fail(); + } catch { + case e: AdminCommandFailedException => + assertEquals("1 preferred replica(s) could not be elected", e.getMessage) + assertTrue(e.getSuppressed()(0).getMessage.contains("Cluster authorization failed")) + // Check we still have the same leader + assertEquals(leader, getLeader(testPartition)) + } finally { + jsonFile.delete() + } + } + +} + +class PreferredReplicaLeaderElectionCommandTestAuthorizer extends SimpleAclAuthorizer { + override def authorize(session: RequestChannel.Session, operation: Operation, resource: Resource): Boolean = + operation != Alter || resource.resourceType != Cluster +} \ No newline at end of file diff --git a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala index 57fe5b5204ffc..76145e0646967 100644 --- a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala +++ b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala @@ -609,11 +609,13 @@ class ReplicaManagerTest { purgatoryName = "Fetch", timer, reaperEnabled = false) val mockDeleteRecordsPurgatory = new DelayedOperationPurgatory[DelayedDeleteRecords]( purgatoryName = "DeleteRecords", timer, reaperEnabled = false) + val mockDelayedElectPreferredLeaderPurgatory = new DelayedOperationPurgatory[DelayedElectPreferredLeader]( + purgatoryName = "DelayedElectPreferredLeader", timer, reaperEnabled = false) new ReplicaManager(config, metrics, time, zkUtils, new MockScheduler(time), mockLogMgr, new AtomicBoolean(false), QuotaFactory.instantiate(config, metrics, time).follower, new BrokerTopicStats, metadataCache, new LogDirFailureChannel(config.logDirs.size), mockProducePurgatory, mockFetchPurgatory, - mockDeleteRecordsPurgatory, Option(this.getClass.getName)) + mockDeleteRecordsPurgatory, mockDelayedElectPreferredLeaderPurgatory, Option(this.getClass.getName)) } } diff --git a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala index 4774e1d2b8774..e0d57fa874868 100644 --- a/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala +++ b/core/src/test/scala/unit/kafka/server/RequestQuotaTest.scala @@ -306,6 +306,9 @@ class RequestQuotaTest extends BaseRequestTest { Collections.singletonMap("topic-2", NewPartitions.increaseTo(1)), 0, false ) + case ApiKeys.ELECT_PREFERRED_LEADERS => + new ElectPreferredLeadersRequest.Builder(Collections.singleton(new TopicPartition("my_topic", 0)), 0) + case _ => throw new IllegalArgumentException("Unsupported API key " + apiKey) } @@ -400,6 +403,7 @@ class RequestQuotaTest extends BaseRequestTest { case ApiKeys.ALTER_REPLICA_LOG_DIRS => new AlterReplicaLogDirsResponse(response).throttleTimeMs case ApiKeys.DESCRIBE_LOG_DIRS => new DescribeLogDirsResponse(response).throttleTimeMs case ApiKeys.CREATE_PARTITIONS => new CreatePartitionsResponse(response).throttleTimeMs + case ApiKeys.ELECT_PREFERRED_LEADERS => new ElectPreferredLeadersResponse(response).throttleTimeMs case requestId => throw new IllegalArgumentException(s"No throttle time for $requestId") } } From 74677b59bf925a834b4f84c5c5718aa4eeb38902 Mon Sep 17 00:00:00 2001 From: Tom Bentley Date: Wed, 20 Sep 2017 12:57:19 +0100 Subject: [PATCH 02/13] Style --- ...referredReplicaLeaderElectionCommand.scala | 3 +- .../kafka/controller/KafkaController.scala | 38 ++++++++++++------- .../server/DelayedElectPreferredLeader.scala | 28 +++++++------- .../main/scala/kafka/server/KafkaApis.scala | 4 +- .../scala/kafka/server/ReplicaManager.scala | 8 ++-- ...rredReplicaLeaderElectionCommandTest.scala | 14 ++++--- 6 files changed, 53 insertions(+), 42 deletions(-) diff --git a/core/src/main/scala/kafka/admin/PreferredReplicaLeaderElectionCommand.scala b/core/src/main/scala/kafka/admin/PreferredReplicaLeaderElectionCommand.scala index d351ff1121ed4..2ddc9fbe2c8ba 100755 --- a/core/src/main/scala/kafka/admin/PreferredReplicaLeaderElectionCommand.scala +++ b/core/src/main/scala/kafka/admin/PreferredReplicaLeaderElectionCommand.scala @@ -136,7 +136,8 @@ object PreferredReplicaLeaderElectionCommand extends Logging { trait PreferredReplicaLeaderElectionCommand { /** - * Move the given partitions to their preferred leader. If the given partitions are none then move all partitions. + * Elect the preferred leader for the given {@code partitionsFromUser}. + * If the given {@code partitionsFromUser} are None then elect the preferred leader for all partitions. */ def moveLeaderToPreferredReplica(partitionsFromUser: Option[scala.collection.Set[TopicAndPartition]]) : Unit def close() : Unit diff --git a/core/src/main/scala/kafka/controller/KafkaController.scala b/core/src/main/scala/kafka/controller/KafkaController.scala index 3cc8cb0450536..83be90eeffa39 100644 --- a/core/src/main/scala/kafka/controller/KafkaController.scala +++ b/core/src/main/scala/kafka/controller/KafkaController.scala @@ -632,7 +632,10 @@ class KafkaController(val config: KafkaConfig, zkUtils: ZkUtils, time: Time, met ) } - private def onPreferredReplicaElectionWithResults(partitions: Set[TopicAndPartition], isTriggeredByAutoRebalance: Boolean = false, viaZk: Boolean = true): Map[TopicAndPartition, Throwable] = { + private def onPreferredReplicaElectionWithResults( + partitions: Set[TopicAndPartition], + isTriggeredByAutoRebalance: Boolean = false, + viaZk: Boolean = true): Map[TopicAndPartition, Throwable] = { info("Starting preferred replica leader election for partitions %s".format(partitions.mkString(","))) try { partitionStateMachine.handleStateChangesWithResults(partitions, OnlinePartition, preferredReplicaPartitionLeaderSelector) @@ -1433,18 +1436,22 @@ class KafkaController(val config: KafkaConfig, zkUtils: ZkUtils, time: Time, met } } - /** - * ControllerEvent to elect preferred leaders. + /** ControllerEvent to elect preferred leaders. * * @param partitions The partitions where the preferred leader should be elected - * @param viaZk Whether the election is triggered as a result of a change in the /admin/preferred_replica_election znode - * @param callback A callback invoked with the results of the election. - * The first argument is the set of partitions where an election was preformed successfully - * The second argument is a map of the partition to ApiError where either the election was not performed - * or the election was performed but there was an error. This map may have entries with - * ApiError.NONE values if the partition was already led by the preferred leader. + * @param viaZk Whether the election is triggered as a result of a change in + * the /admin/preferred_replica_election znode. + * @param callback A callback invoked with the results of the election. + * The first argument is the set of partitions where an election was preformed successfully + * The second argument is a map of the partition to ApiError where either the + * election was not performed or the election was performed but there was an error. + * This map may have entries with ApiError.NONE values if the partition was + * already led by the preferred leader. */ - case class PreferredReplicaLeaderElection(partitions: Set[TopicAndPartition], viaZk: Boolean = true, callback: (Set[TopicAndPartition], Map[TopicAndPartition, ApiError])=>Unit = { (_,_) => }) extends ControllerEvent { + case class PreferredReplicaLeaderElection(partitions: Set[TopicAndPartition], + viaZk: Boolean = true, + callback: (Set[TopicAndPartition], Map[TopicAndPartition, ApiError]) => Unit = { (_, _) => }) + extends ControllerEvent { def state = ControllerState.ManualLeaderBalance @@ -1452,22 +1459,25 @@ class KafkaController(val config: KafkaConfig, zkUtils: ZkUtils, time: Time, met if (!isActive) { callback(Set(), partitions.map(partition => partition -> new ApiError(Errors.NOT_CONTROLLER, null)).toMap) } else { - val (partitionsForTopicsToBeDeleted, livePartitions) = partitions.partition(partition => topicDeletionManager.isTopicQueuedUpForDeletion(partition.topic)) + val (partitionsForTopicsToBeDeleted, livePartitions) = partitions.partition(partition => + topicDeletionManager.isTopicQueuedUpForDeletion(partition.topic)) if (partitionsForTopicsToBeDeleted.nonEmpty) { error("Skipping preferred replica election for partitions %s since the respective topics are being deleted" .format(partitionsForTopicsToBeDeleted)) } // partition those where preferred is already leader - val (electablePartitions, alreadyPreferred) = livePartitions.partition(partition => { + val (electablePartitions, alreadyPreferred) = livePartitions.partition { partition => val assignedReplicas = controllerContext.partitionReplicaAssignment(partition) val preferredReplica = assignedReplicas.head val currentLeader = controllerContext.partitionLeadershipInfo(partition).leaderAndIsr.leader currentLeader != preferredReplica - }) + } val electionErrors = onPreferredReplicaElectionWithResults(electablePartitions, viaZk=viaZk) val successfulPartitions = electablePartitions -- electionErrors.keySet - val results = electionErrors.map(e => e._1 -> ApiError.fromThrowable(e._2)) ++ alreadyPreferred.map(_ -> ApiError.NONE) ++ partitionsForTopicsToBeDeleted.map(_-> new ApiError(Errors.INVALID_TOPIC_EXCEPTION, "The topic is being deleted")) + val results = electionErrors.map(e => e._1 -> ApiError.fromThrowable(e._2)) ++ + alreadyPreferred.map(_ -> ApiError.NONE) ++ + partitionsForTopicsToBeDeleted.map(_-> new ApiError(Errors.INVALID_TOPIC_EXCEPTION, "The topic is being deleted")) debug("PreferredReplicaLeaderElection waiting: %s, results: %s".format(successfulPartitions, results)) callback(successfulPartitions, results) } diff --git a/core/src/main/scala/kafka/server/DelayedElectPreferredLeader.scala b/core/src/main/scala/kafka/server/DelayedElectPreferredLeader.scala index 299136a870678..2984a8b2aba19 100644 --- a/core/src/main/scala/kafka/server/DelayedElectPreferredLeader.scala +++ b/core/src/main/scala/kafka/server/DelayedElectPreferredLeader.scala @@ -27,10 +27,9 @@ import org.apache.kafka.common.requests.ApiError import scala.collection.{Map, Set, mutable} -/** - * A delayed elect preferred leader operation that can be created by the replica manager and watched - * in the elect preferred leader purgatory - */ +/** A delayed elect preferred leader operation that can be created by the replica manager and watched + * in the elect preferred leader purgatory + */ class DelayedElectPreferredLeader(delayMs: Long, waiting: Set[TopicAndPartition], results: Map[TopicAndPartition, ApiError], @@ -80,17 +79,16 @@ class DelayedElectPreferredLeader(delayMs: Long, } private def updateWaiting() = { - waitingPartitions.foreach(tp => { - val ps = replicaManager.metadataCache.getPartitionInfo(tp.topic, tp.partition) - ps match { - case Some(ps) => - if (expectedLeaders(tp) == ps.basePartitionState.leader) { - waitingPartitions -= tp - fullResults += tp -> ApiError.NONE - } - case None => - } + waitingPartitions.foreach{tp => + val ps = replicaManager.metadataCache.getPartitionInfo(tp.topic, tp.partition) + ps match { + case Some(ps) => + if (expectedLeaders(tp) == ps.basePartitionState.leader) { + waitingPartitions -= tp + fullResults += tp -> ApiError.NONE + } + case None => } - ) + } } } diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala index 59f4d38b84ba3..f5f3a72a89006 100644 --- a/core/src/main/scala/kafka/server/KafkaApis.scala +++ b/core/src/main/scala/kafka/server/KafkaApis.scala @@ -225,9 +225,9 @@ class KafkaApis(val requestChannel: RequestChannel, } } if (replicaManager.hasDelayedElectionOperations) { - updateMetadataRequest.partitionStates.asScala.foreach{ case (tp, ps) => { + updateMetadataRequest.partitionStates.asScala.foreach { case (tp, ps) => replicaManager.tryCompleteElection(new TopicPartitionOperationKey(tp.topic(), tp.partition())) - }} + } } sendResponseExemptThrottle(request, new UpdateMetadataResponse(Errors.NONE)) } else { diff --git a/core/src/main/scala/kafka/server/ReplicaManager.scala b/core/src/main/scala/kafka/server/ReplicaManager.scala index daa750095c9b2..0b441dc8bb6be 100644 --- a/core/src/main/scala/kafka/server/ReplicaManager.scala +++ b/core/src/main/scala/kafka/server/ReplicaManager.scala @@ -1451,15 +1451,15 @@ class ReplicaManager(val config: KafkaConfig, partitions: Set[TopicAndPartition], responseCallback: Map[TopicAndPartition, ApiError] => Unit, requestTimeout: Long): Unit = { - val partitionsFromZk = zkUtils.getPartitionsForTopics(partitions.map(_.topic).toSeq).flatMap { case (topic, partitions) => - partitions.map(TopicAndPartition(topic, _)) + val partitionsFromZk = zkUtils.getPartitionsForTopics(partitions.map(_.topic).toSeq).flatMap { + case (topic, partitions) => partitions.map(TopicAndPartition(topic, _)) }.toSet val (validPartitions, invalidPartitions) = partitions.partition(partitionsFromZk.contains) - val invalidPartitionsResults = invalidPartitions.map(p => { + val invalidPartitionsResults = invalidPartitions.map { p => val msg = s"Skipping preferred replica leader election for partition ${p} since it doesn't exist." logger.info(msg) p -> new ApiError(Errors.UNKNOWN_TOPIC_OR_PARTITION, msg) - }).toMap + }.toMap def electionCallback(waiting: Set[TopicAndPartition], results: Map[TopicAndPartition, ApiError]) = { if (waiting.nonEmpty) { diff --git a/core/src/test/scala/unit/kafka/admin/PreferredReplicaLeaderElectionCommandTest.scala b/core/src/test/scala/unit/kafka/admin/PreferredReplicaLeaderElectionCommandTest.scala index 7aab1071ba9a3..ca6a39d634aca 100644 --- a/core/src/test/scala/unit/kafka/admin/PreferredReplicaLeaderElectionCommandTest.scala +++ b/core/src/test/scala/unit/kafka/admin/PreferredReplicaLeaderElectionCommandTest.scala @@ -48,11 +48,11 @@ class PreferredReplicaLeaderElectionCommandTest extends ZooKeeperTestHarness wit super.tearDown() } - private def createTestTopicAndCluster(topicPartition: Map[TopicPartition, List[Int]], authorizer: Option[String] = None) { + private def createTestTopicAndCluster(topicPartition: Map[TopicPartition, List[Int]], + authorizer: Option[String] = None) { val brokerConfigs = TestUtils.createBrokerConfigs(3, zkConnect, false) - brokerConfigs.foreach(p => p.setProperty("auto.leader.rebalance.enable", "false") - ) + brokerConfigs.foreach(p => p.setProperty("auto.leader.rebalance.enable", "false")) authorizer match { case Some(className) => brokerConfigs.foreach(p => p.setProperty("authorizer.class.name", className)) @@ -61,13 +61,15 @@ class PreferredReplicaLeaderElectionCommandTest extends ZooKeeperTestHarness wit createTestTopicAndCluster(topicPartition,brokerConfigs) } - private def createTestTopicAndCluster(partitionsAndAssignments: Map[TopicPartition, List[Int]], brokerConfigs: Seq[Properties]) { + private def createTestTopicAndCluster(partitionsAndAssignments: Map[TopicPartition, List[Int]], + brokerConfigs: Seq[Properties]) { // create brokers servers = brokerConfigs.map(b => TestUtils.createServer(KafkaConfig.fromProps(b))) // create the topic - partitionsAndAssignments.foreach(partitionAndAssignment => + partitionsAndAssignments.foreach { partitionAndAssignment => AdminUtils.createOrUpdateTopicPartitionAssignmentPathInZK(zkUtils, partitionAndAssignment._1.topic(), - Map(partitionAndAssignment._1.partition -> partitionAndAssignment._2))) + Map(partitionAndAssignment._1.partition -> partitionAndAssignment._2)) + } // wait until replica log is created on every broker TestUtils.waitUntilTrue(() => servers.forall(server => partitionsAndAssignments.forall(partitionAndAssignment => server.getLogManager().getLog(partitionAndAssignment._1).isDefined)), "Replicas for topic test not created") From 89a3bf441cbf55fbc41be188b18a9404cbb8e6ee Mon Sep 17 00:00:00 2001 From: Tom Bentley Date: Fri, 22 Sep 2017 15:30:58 +0100 Subject: [PATCH 03/13] Properly detect and throw correct exception when leader unavailable --- .../PreferredLeaderUnavailableException.scala | 23 +++++++++++++++++++ .../kafka/controller/KafkaController.scala | 10 +++++++- .../controller/PartitionLeaderSelector.scala | 4 ++-- ...rredReplicaLeaderElectionCommandTest.scala | 4 ++-- 4 files changed, 36 insertions(+), 5 deletions(-) create mode 100644 core/src/main/scala/kafka/common/PreferredLeaderUnavailableException.scala diff --git a/core/src/main/scala/kafka/common/PreferredLeaderUnavailableException.scala b/core/src/main/scala/kafka/common/PreferredLeaderUnavailableException.scala new file mode 100644 index 0000000000000..4101a199d6d28 --- /dev/null +++ b/core/src/main/scala/kafka/common/PreferredLeaderUnavailableException.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kafka.common + +class PreferredLeaderUnavailableException(message: String, cause: Throwable) extends StateChangeFailedException(message, cause) { + def this(message: String) = this(message, null) + def this() = this(null, null) +} \ No newline at end of file diff --git a/core/src/main/scala/kafka/controller/KafkaController.scala b/core/src/main/scala/kafka/controller/KafkaController.scala index 83be90eeffa39..973ca301e259f 100644 --- a/core/src/main/scala/kafka/controller/KafkaController.scala +++ b/core/src/main/scala/kafka/controller/KafkaController.scala @@ -1475,7 +1475,15 @@ class KafkaController(val config: KafkaConfig, zkUtils: ZkUtils, time: Time, met val electionErrors = onPreferredReplicaElectionWithResults(electablePartitions, viaZk=viaZk) val successfulPartitions = electablePartitions -- electionErrors.keySet - val results = electionErrors.map(e => e._1 -> ApiError.fromThrowable(e._2)) ++ + val results = electionErrors.map { case (partition, ex) => + val apiError = if (ex.isInstanceOf[StateChangeFailedException] && + ex.getCause != null && + ex.getCause.isInstanceOf[PreferredLeaderUnavailableException]) + new ApiError(Errors.LEADER_NOT_AVAILABLE, ex.getCause.getMessage) + else + ApiError.fromThrowable(ex) + partition -> apiError + } ++ alreadyPreferred.map(_ -> ApiError.NONE) ++ partitionsForTopicsToBeDeleted.map(_-> new ApiError(Errors.INVALID_TOPIC_EXCEPTION, "The topic is being deleted")) debug("PreferredReplicaLeaderElection waiting: %s, results: %s".format(successfulPartitions, results)) diff --git a/core/src/main/scala/kafka/controller/PartitionLeaderSelector.scala b/core/src/main/scala/kafka/controller/PartitionLeaderSelector.scala index e534ff3e85bd6..d19d7e0f691cc 100644 --- a/core/src/main/scala/kafka/controller/PartitionLeaderSelector.scala +++ b/core/src/main/scala/kafka/controller/PartitionLeaderSelector.scala @@ -18,7 +18,7 @@ package kafka.controller import kafka.admin.AdminUtils import kafka.api.LeaderAndIsr -import kafka.common.{LeaderElectionNotNeededException, NoReplicaOnlineException, StateChangeFailedException, TopicAndPartition} +import kafka.common.{LeaderElectionNotNeededException, NoReplicaOnlineException, PreferredLeaderUnavailableException, StateChangeFailedException, TopicAndPartition} import kafka.log.LogConfig import kafka.server.{ConfigType, KafkaConfig} import kafka.utils.Logging @@ -154,7 +154,7 @@ class PreferredReplicaPartitionLeaderSelector(controllerContext: ControllerConte val newLeaderAndIsr = currentLeaderAndIsr.newLeader(preferredReplica) (newLeaderAndIsr, assignedReplicas) } else { - throw new StateChangeFailedException(s"Preferred replica $preferredReplica for partition $topicAndPartition " + + throw new PreferredLeaderUnavailableException(s"Preferred replica $preferredReplica for partition $topicAndPartition " + s"is either not alive or not in the isr. Current leader and ISR: [$currentLeaderAndIsr]") } } diff --git a/core/src/test/scala/unit/kafka/admin/PreferredReplicaLeaderElectionCommandTest.scala b/core/src/test/scala/unit/kafka/admin/PreferredReplicaLeaderElectionCommandTest.scala index ca6a39d634aca..e09defa5e8db5 100644 --- a/core/src/test/scala/unit/kafka/admin/PreferredReplicaLeaderElectionCommandTest.scala +++ b/core/src/test/scala/unit/kafka/admin/PreferredReplicaLeaderElectionCommandTest.scala @@ -30,7 +30,7 @@ import kafka.server.{KafkaConfig, KafkaServer} import kafka.utils.{Logging, TestUtils, ZkUtils} import kafka.zk.ZooKeeperTestHarness import org.apache.kafka.common.TopicPartition -import org.apache.kafka.common.errors.{UnknownServerException, UnknownTopicOrPartitionException} +import org.apache.kafka.common.errors.{LeaderNotAvailableException, UnknownServerException, UnknownTopicOrPartitionException} import org.apache.kafka.common.network.ListenerName import org.apache.kafka.common.security.auth.KafkaPrincipal import org.junit.Assert._ @@ -238,7 +238,7 @@ class PreferredReplicaLeaderElectionCommandTest extends ZooKeeperTestHarness wit case e: AdminCommandFailedException => assertEquals("1 preferred replica(s) could not be elected", e.getMessage) val suppressed = e.getSuppressed()(0) - assertTrue(suppressed.isInstanceOf[UnknownServerException]) + assertTrue(suppressed.isInstanceOf[LeaderNotAvailableException]) assertTrue(suppressed.getMessage, suppressed.getMessage.contains("is either not alive or not in the isr")) // Check we still have the same leader assertEquals(leader, getLeader(testPartition)) From d3cc0f5c272573d688e614e08b66bfec9cf49fa7 Mon Sep 17 00:00:00 2001 From: Tom Bentley Date: Wed, 20 Sep 2017 14:19:26 +0100 Subject: [PATCH 04/13] WIP --- .../admin/ElectPreferredLeadersResult.java | 6 ++- ...referredReplicaLeaderElectionCommand.scala | 34 +++++++----- ...rredReplicaLeaderElectionCommandTest.scala | 52 ++++++++++++++----- 3 files changed, 66 insertions(+), 26 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/ElectPreferredLeadersResult.java b/clients/src/main/java/org/apache/kafka/clients/admin/ElectPreferredLeadersResult.java index e2075b2ef95cb..133c335021dfc 100644 --- a/clients/src/main/java/org/apache/kafka/clients/admin/ElectPreferredLeadersResult.java +++ b/clients/src/main/java/org/apache/kafka/clients/admin/ElectPreferredLeadersResult.java @@ -52,7 +52,7 @@ public class ElectPreferredLeadersResult { } /** Return a new future that has completed exceptionally */ - private static KafkaFuture exceptionalFuture(Exception e) { + private static KafkaFuture exceptionalFuture(Throwable e) { KafkaFutureImpl future = new KafkaFutureImpl<>(); future.completeExceptionally(e); return future; @@ -93,8 +93,10 @@ public KafkaFuture> partitions() { final Map> map; try { map = futures.get(); - } catch (InterruptedException | ExecutionException e) { + } catch (InterruptedException e) { return exceptionalFuture(e); + } catch (ExecutionException e) { + return exceptionalFuture(e.getCause()); } KafkaFutureImpl> result = new KafkaFutureImpl<>(); result.complete(map.keySet()); diff --git a/core/src/main/scala/kafka/admin/PreferredReplicaLeaderElectionCommand.scala b/core/src/main/scala/kafka/admin/PreferredReplicaLeaderElectionCommand.scala index 2ddc9fbe2c8ba..65395d9da96f6 100755 --- a/core/src/main/scala/kafka/admin/PreferredReplicaLeaderElectionCommand.scala +++ b/core/src/main/scala/kafka/admin/PreferredReplicaLeaderElectionCommand.scala @@ -17,7 +17,7 @@ package kafka.admin import java.util.Properties -import java.util.concurrent.{ExecutionException, TimeUnit} +import java.util.concurrent.{ExecutionException} import joptsimple.OptionParser import kafka.utils._ @@ -47,7 +47,7 @@ object PreferredReplicaLeaderElectionCommand extends Logging { } } /** Basically the same as main, but throws rather than calling System.exit */ - def run(args: Array[String]): Unit = { + def run(args: Array[String], timeout: Option[Int] = None): Unit = { val parser = new OptionParser(false) val jsonFileOpt = parser.accepts("path-to-json-file", "The JSON file with the list of partitions " + "for which preferred replica leader election should be done, in the following format - \n" + @@ -57,12 +57,12 @@ object PreferredReplicaLeaderElectionCommand extends Logging { .describedAs("list of partitions for which preferred replica leader election needs to be triggered") .ofType(classOf[String]) val zkConnectOpt = parser.accepts("zookeeper", "DEPRECATED. The connection string for the zookeeper connection in the " + - "form host:port. Multiple URLS can be given to allow fail-over. REQUIRED unless --bootstrap-server is given.") + "form host:port. Multiple comma-separated URLS can be given to allow fail-over. REQUIRED unless --bootstrap-server is given.") .withRequiredArg .describedAs("urls") .ofType(classOf[String]) val bootstrapServerOpt = parser.accepts("bootstrap-server", "A hostname and port for the broker to connect to, " + - "in the form host:port. Multiple URLs can be given. REQUIRED unless --zookeeper is given.") + "in the form host:port. Multiple comma-separated URLs can be given. REQUIRED unless --zookeeper is given.") .withRequiredArg .describedAs("host:port") .ofType(classOf[String]) @@ -90,11 +90,11 @@ object PreferredReplicaLeaderElectionCommand extends Logging { Console.err.println(zkConnectOpt + " is deprecated and will be removed in a future version of Kafka.") Console.err.println("Use " + bootstrapServerOpt + " instead to specify a broker to connect to.") new ZkPreferredReplicaLeaderElectionCommand(ZkUtils(options.valueOf(zkConnectOpt), - 30000, - 30000, + timeout.getOrElse(30000), + timeout.getOrElse(30000), JaasUtils.isZkSecurityEnabled())) } else { - new AdminClientPreferredReplicaLeaderElectionCommand(options.valueOf(bootstrapServerOpt)) + new AdminClientPreferredReplicaLeaderElectionCommand(options.valueOf(bootstrapServerOpt), timeout) } try { @@ -143,9 +143,15 @@ trait PreferredReplicaLeaderElectionCommand { def close() : Unit } -class AdminClientPreferredReplicaLeaderElectionCommand(bootstrapServers: String) extends PreferredReplicaLeaderElectionCommand with Logging { +class AdminClientPreferredReplicaLeaderElectionCommand(bootstrapServers: String, timeout: Option[Int]) extends PreferredReplicaLeaderElectionCommand with Logging { val props = new Properties() props.setProperty(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers) + timeout match { + case Some(timeout) => + props.setProperty(AdminClientConfig.REQUEST_TIMEOUT_MS_CONFIG, timeout.toString) + case None => + } + val adminClient: org.apache.kafka.clients.admin.AdminClient = org.apache.kafka.clients.admin.AdminClient.create(props) /** @@ -173,10 +179,14 @@ class AdminClientPreferredReplicaLeaderElectionCommand(bootstrapServers: String) val attemptedPartitions = try { result.partitions().get.asScala } catch { - case e: TimeoutException => - // We timed out, or don't even know the attempted partitions - println("Timeout waiting for election results") - return + case e: ExecutionException => + val cause = e.getCause + if (cause.isInstanceOf[TimeoutException]) { + // We timed out, or don't even know the attempted partitions + println("Timeout waiting for election results") + return + } + throw cause case e: Throwable => // We don't even know the attempted partitions println("Error while making request") diff --git a/core/src/test/scala/unit/kafka/admin/PreferredReplicaLeaderElectionCommandTest.scala b/core/src/test/scala/unit/kafka/admin/PreferredReplicaLeaderElectionCommandTest.scala index e09defa5e8db5..b39c36a4c8960 100644 --- a/core/src/test/scala/unit/kafka/admin/PreferredReplicaLeaderElectionCommandTest.scala +++ b/core/src/test/scala/unit/kafka/admin/PreferredReplicaLeaderElectionCommandTest.scala @@ -95,10 +95,10 @@ class PreferredReplicaLeaderElectionCommandTest extends ZooKeeperTestHarness wit servers(0).metadataCache.getPartitionInfo(topicPartition.topic(), topicPartition.partition()).get.basePartitionState.leader } - private def bootstrapServer: String = { - val port = servers(0).socketServer.boundPort(ListenerName.normalised("PLAINTEXT")) + private def bootstrapServer(broker: Int = 0): String = { + val port = servers(broker).socketServer.boundPort(ListenerName.normalised("PLAINTEXT")) info("Server bound to port "+port) - "localhost:" + port + s"localhost:$port" } val testPartition = new TopicPartition("test", 0) @@ -106,6 +106,33 @@ class PreferredReplicaLeaderElectionCommandTest extends ZooKeeperTestHarness wit val testPartitionPreferredLeader = testPartitionAssignment.head val testPartitionAndAssignment = Map(testPartition -> testPartitionAssignment) + /** Test the case multiple values are given for --bootstrap-broker */ + @Test + def testMultipleBrokersGiven() { + createTestTopicAndCluster(testPartitionAndAssignment) + bounceServer(testPartitionPreferredLeader, testPartition) + // Check the leader for the partition is not the preferred one + assertNotEquals(testPartitionPreferredLeader, getLeader(testPartition)) + PreferredReplicaLeaderElectionCommand.run(Array( + "--bootstrap-server", s"${bootstrapServer(1)},${bootstrapServer(0)}")) + // Check the leader for the partition IS the preferred one + assertEquals(testPartitionPreferredLeader, getLeader(testPartition)) + } + + /** Test the case when an invalid broker is given for --bootstrap-broker */ + @Test + def testInvalidBrokerGiven() { + createTestTopicAndCluster(testPartitionAndAssignment) + bounceServer(testPartitionPreferredLeader, testPartition) + // Check the leader for the partition is not the preferred one + assertNotEquals(testPartitionPreferredLeader, getLeader(testPartition)) + PreferredReplicaLeaderElectionCommand.run(Array( + "--bootstrap-server", "example.com:1234"), + timeout = Some(10000)) + // Check the leader for the partition is STILL not the preferred one + assertNotEquals(testPartitionPreferredLeader, getLeader(testPartition)) + } + /** Test the case where no partitions are given (=> elect all partitions) */ @Test def testNoPartitionsGiven() { @@ -114,7 +141,7 @@ class PreferredReplicaLeaderElectionCommandTest extends ZooKeeperTestHarness wit // Check the leader for the partition is not the preferred one assertNotEquals(testPartitionPreferredLeader, getLeader(testPartition)) PreferredReplicaLeaderElectionCommand.run(Array( - "--bootstrap-server", bootstrapServer)) + "--bootstrap-server", bootstrapServer())) // Check the leader for the partition IS the preferred one assertEquals(testPartitionPreferredLeader, getLeader(testPartition)) } @@ -138,7 +165,7 @@ class PreferredReplicaLeaderElectionCommandTest extends ZooKeeperTestHarness wit val jsonFile = toJsonFile(testPartitionAndAssignment.keySet) try { PreferredReplicaLeaderElectionCommand.run(Array( - "--bootstrap-server", bootstrapServer, + "--bootstrap-server", bootstrapServer(), "--path-to-json-file", jsonFile.getAbsolutePath)) } finally { jsonFile.delete() @@ -158,7 +185,7 @@ class PreferredReplicaLeaderElectionCommandTest extends ZooKeeperTestHarness wit val jsonFile = toJsonFile(nonExistentPartitionAndAssignment.keySet) try { PreferredReplicaLeaderElectionCommand.run(Array( - "--bootstrap-server", bootstrapServer, + "--bootstrap-server", bootstrapServer(), "--path-to-json-file", jsonFile.getAbsolutePath)) } catch { case e: AdminCommandFailedException => @@ -187,7 +214,7 @@ class PreferredReplicaLeaderElectionCommandTest extends ZooKeeperTestHarness wit val jsonFile = toJsonFile(testPartitionAndAssignment.keySet) try { PreferredReplicaLeaderElectionCommand.run(Array( - "--bootstrap-server", bootstrapServer, + "--bootstrap-server", bootstrapServer(), "--path-to-json-file", jsonFile.getAbsolutePath)) } finally { jsonFile.delete() @@ -207,7 +234,7 @@ class PreferredReplicaLeaderElectionCommandTest extends ZooKeeperTestHarness wit try { // Now do the election, even though the preferred replica is *already* the leader PreferredReplicaLeaderElectionCommand.run(Array( - "--bootstrap-server", bootstrapServer, + "--bootstrap-server", bootstrapServer(), "--path-to-json-file", jsonFile.getAbsolutePath)) // Check the leader for the partition still is the preferred one assertEquals(testPartitionPreferredLeader, getLeader(testPartition)) @@ -231,7 +258,7 @@ class PreferredReplicaLeaderElectionCommandTest extends ZooKeeperTestHarness wit val jsonFile = toJsonFile(testPartitionAndAssignment.keySet) try { PreferredReplicaLeaderElectionCommand.run(Array( - "--bootstrap-server", bootstrapServer, + "--bootstrap-server", bootstrapServer(), "--path-to-json-file", jsonFile.getAbsolutePath)) fail(); } catch { @@ -260,8 +287,9 @@ class PreferredReplicaLeaderElectionCommandTest extends ZooKeeperTestHarness wit val jsonFile = toJsonFile(testPartitionAndAssignment.keySet) try { PreferredReplicaLeaderElectionCommand.run(Array( - "--bootstrap-server", bootstrapServer, - "--path-to-json-file", jsonFile.getAbsolutePath)) + "--bootstrap-server", bootstrapServer(), + "--path-to-json-file", jsonFile.getAbsolutePath), + timeout = Some(10000)) fail(); } catch { case e: AdminCommandFailedException => @@ -287,7 +315,7 @@ class PreferredReplicaLeaderElectionCommandTest extends ZooKeeperTestHarness wit val jsonFile = toJsonFile(testPartitionAndAssignment.keySet) try { PreferredReplicaLeaderElectionCommand.run(Array( - "--bootstrap-server", bootstrapServer, + "--bootstrap-server", bootstrapServer(), "--path-to-json-file", jsonFile.getAbsolutePath)) fail(); } catch { From 98fe16acccf9a42732ae159bc84abb308be7a4da Mon Sep 17 00:00:00 2001 From: Tom Bentley Date: Fri, 22 Sep 2017 15:34:49 +0100 Subject: [PATCH 05/13] Javadoc + formatting --- .../kafka/clients/admin/AdminClient.java | 29 +++++++- .../admin/ElectPreferredLeadersResult.java | 16 +---- .../kafka/clients/admin/KafkaAdminClient.java | 6 +- .../ElectPreferredLeadersRequest.java | 9 +-- .../ElectPreferredLeadersResponse.java | 4 +- .../common/requests/RequestResponseTest.java | 1 - ...referredReplicaLeaderElectionCommand.scala | 72 ++++++++++--------- .../scala/unit/kafka/admin/AdminTest.scala | 2 +- 8 files changed, 81 insertions(+), 58 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/AdminClient.java b/clients/src/main/java/org/apache/kafka/clients/admin/AdminClient.java index e209a20659ec3..5b3b7bca268cd 100644 --- a/clients/src/main/java/org/apache/kafka/clients/admin/AdminClient.java +++ b/clients/src/main/java/org/apache/kafka/clients/admin/AdminClient.java @@ -527,12 +527,37 @@ public ElectPreferredLeadersResult electPreferredLeaders(CollectionThe {@code KafkaFuture}s available from instances of this class may be completed + * exceptionally due to:

+ *
    + *
  • {@link org.apache.kafka.common.errors.ClusterAuthorizationException} + * if the authenticated user didn't have alter access to the cluster.
    • + *
  • {@link org.apache.kafka.common.errors.UnknownTopicOrPartitionException} + * if the topic or partition did not exist within the cluster.
    • + *
  • {@link org.apache.kafka.common.errors.InvalidTopicException} + * if the topic was already queued for deletion.
    • + *
  • {@link org.apache.kafka.common.errors.NotControllerException} + * if the request was sent to a broker that was not the controller for the cluster.
    • + *
  • {@link org.apache.kafka.common.errors.TimeoutException} + * if the request timed out before the election was complete.
    • + *
  • {@link org.apache.kafka.common.errors.LeaderNotAvailableException} + * if the preferred leader was not alive or not in the ISR.
    • + *
* * @param partitions The partitions for which the the preferred leader should be elected. * @param options The options to use when electing the preferred leaders. * @return The ElectPreferredLeadersResult. */ - public abstract ElectPreferredLeadersResult electPreferredLeaders(Collection partitions, ElectPreferredLeadersOptions options); + public abstract ElectPreferredLeadersResult electPreferredLeaders(Collection partitions, + ElectPreferredLeadersOptions options); } diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/ElectPreferredLeadersResult.java b/clients/src/main/java/org/apache/kafka/clients/admin/ElectPreferredLeadersResult.java index 133c335021dfc..350414be5ca1b 100644 --- a/clients/src/main/java/org/apache/kafka/clients/admin/ElectPreferredLeadersResult.java +++ b/clients/src/main/java/org/apache/kafka/clients/admin/ElectPreferredLeadersResult.java @@ -24,22 +24,12 @@ import java.util.Collection; import java.util.Map; +import java.util.Set; import java.util.concurrent.ExecutionException; /** * The result of {@link AdminClient#electPreferredLeaders(Collection, ElectPreferredLeadersOptions)} * - *

The {@code KafkaFuture}s available from instances of this class may be completed - * exceptionally due to:

- *
    - *
  • {@link org.apache.kafka.common.protocol.Errors#CLUSTER_AUTHORIZATION_FAILED CLUSTER_AUTHORIZATION_FAILED} if the authenticated user didn't have {@code Alter} access to the cluster.
    • - *
  • {@link org.apache.kafka.common.protocol.Errors#UNKNOWN_TOPIC_OR_PARTITION UNKNOWN_TOPIC_OR_PARTITION} if the topic or partition did not exist within the cluster.
    • - *
  • {@link org.apache.kafka.common.protocol.Errors#INVALID_TOPIC_EXCEPTION INVALID_TOPIC_EXCEPTION} if the topic was already queued for deletion.
    • - *
  • {@link org.apache.kafka.common.protocol.Errors#NOT_CONTROLLER NOT_CONTROLLER} if the request was sent to a broker that was not the controller for the cluster.
    • - *
  • {@link org.apache.kafka.common.protocol.Errors#REQUEST_TIMED_OUT REQUEST_TIMED_OUT} if the request timed out before the election was complete.
    • - *
  • {@link org.apache.kafka.common.protocol.Errors#UNKNOWN_SERVER_ERROR UNKNOWN_SERVER_ERROR} if the preferred leader was not alive or not in the ISR.
    • - *
- * * The API of this class is evolving, see {@link AdminClient} for details. */ @InterfaceStability.Evolving @@ -89,7 +79,7 @@ public KafkaFuture partitionResult(TopicPartition partition) { * {@link AdminClient#electPreferredLeaders(Collection)} is called * with a null {@code partitions} argument.

*/ - public KafkaFuture> partitions() { + public KafkaFuture> partitions() { final Map> map; try { map = futures.get(); @@ -98,7 +88,7 @@ public KafkaFuture> partitions() { } catch (ExecutionException e) { return exceptionalFuture(e.getCause()); } - KafkaFutureImpl> result = new KafkaFutureImpl<>(); + KafkaFutureImpl> result = new KafkaFutureImpl<>(); result.complete(map.keySet()); return result; } diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java index 0f67ac453584c..21b9761877814 100644 --- a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java +++ b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java @@ -1865,7 +1865,6 @@ public void handleResponse(AbstractResponse abstractResponse) { future.completeExceptionally(result.getValue().exception()); } } - } @Override @@ -1876,7 +1875,9 @@ void handleFailure(Throwable throwable) { return new CreatePartitionsResult(new HashMap>(futures)); } - public ElectPreferredLeadersResult electPreferredLeaders(final Collection partitions, ElectPreferredLeadersOptions options) { + @Override + public ElectPreferredLeadersResult electPreferredLeaders(final Collection partitions, + ElectPreferredLeadersOptions options) { final KafkaFutureImpl>> futures = new KafkaFutureImpl<>(); final Map> mapOfFutures; @@ -1927,7 +1928,6 @@ public void handleResponse(AbstractResponse abstractResponse) { if (!knownPartitions) { futures.complete(mapOfFutures); } - } @Override diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ElectPreferredLeadersRequest.java b/clients/src/main/java/org/apache/kafka/common/requests/ElectPreferredLeadersRequest.java index c55f4eea4b6a0..7a219c35dcf82 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ElectPreferredLeadersRequest.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ElectPreferredLeadersRequest.java @@ -44,13 +44,13 @@ public class ElectPreferredLeadersRequest extends AbstractRequest { private static final String PARTITIONS_KEY_NAME = "partitions"; public static final Schema ELECT_PREFERRED_LEADERS_REQUEST_V0 = new Schema( - new Field("topic_partitions", ArrayOf.nullable( + new Field(TOPIC_PARTITIONS_KEY_NAME, ArrayOf.nullable( new Schema( TOPIC_NAME, - new Field("partitions", + new Field(PARTITIONS_KEY_NAME, new ArrayOf(INT32), "The partitions of this topic whose preferred leader should be elected")))), - new Field("timeout", INT32, "The time in ms to wait for the elections to be completed.") + new Field(TIMEOUT_KEY_NAME, INT32, "The time in ms to wait for the elections to be completed.") ); public static Schema[] schemaVersions() { @@ -147,7 +147,8 @@ public AbstractResponse getErrorResponse(int throttleTimeMs, Throwable e) { errors.put(partition, error); return new ElectPreferredLeadersResponse(throttleTimeMs, errors); default: - throw new IllegalArgumentException(String.format("Version %d is not valid. Valid versions for %s are 0 to %d", + throw new IllegalArgumentException(String.format( + "Version %d is not valid. Valid versions for %s are 0 to %d", version, this.getClass().getSimpleName(), ApiKeys.ELECT_PREFERRED_LEADERS.latestVersion())); } } diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ElectPreferredLeadersResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ElectPreferredLeadersResponse.java index 0c7459a8d3eaa..e61282f94331b 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ElectPreferredLeadersResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ElectPreferredLeadersResponse.java @@ -44,9 +44,9 @@ public class ElectPreferredLeadersResponse extends AbstractResponse { public static final Schema ELECT_PREFERRED_LEADERS_RESPONSE_V0 = new Schema( THROTTLE_TIME_MS, - new Field("replica_election_result", new ArrayOf(new Schema( + new Field(REPLICA_ELECTION_RESULT_KEY_NAME, new ArrayOf(new Schema( TOPIC_NAME, - new Field("partition_results", new ArrayOf( + new Field(PARTITION_RESULTS_KEY_NAME, new ArrayOf( new Schema( PARTITION_ID, ERROR_CODE, diff --git a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java index 57e767d1a0d6c..2aa9394d4a390 100644 --- a/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java +++ b/clients/src/test/java/org/apache/kafka/common/requests/RequestResponseTest.java @@ -249,7 +249,6 @@ public void testSerialization() throws Exception { checkRequest(createElectPreferredLeadersRequestNullPartitions()); checkErrorResponse(createElectPreferredLeadersRequest(), new UnknownServerException()); checkResponse(createElectPreferredLeadersResponse(), 0); - } @Test diff --git a/core/src/main/scala/kafka/admin/PreferredReplicaLeaderElectionCommand.scala b/core/src/main/scala/kafka/admin/PreferredReplicaLeaderElectionCommand.scala index 65395d9da96f6..f8eada7a48280 100755 --- a/core/src/main/scala/kafka/admin/PreferredReplicaLeaderElectionCommand.scala +++ b/core/src/main/scala/kafka/admin/PreferredReplicaLeaderElectionCommand.scala @@ -67,7 +67,7 @@ object PreferredReplicaLeaderElectionCommand extends Logging { .describedAs("host:port") .ofType(classOf[String]) - if(args.length == 0) + if (args.length == 0) CommandLineUtils.printUsageAndDie(parser, "This tool causes leadership for each partition to be transferred back to the 'preferred replica'," + " it can be used to balance leadership among the servers." + " The preferred replica is the first one in the replica assignment (see kafka-reassign-partitions.sh)." + @@ -75,20 +75,19 @@ object PreferredReplicaLeaderElectionCommand extends Logging { val options = parser.parse(args : _*) - if (options.has(bootstrapServerOpt) && options.has(zkConnectOpt) - || !options.has(bootstrapServerOpt) && !options.has(zkConnectOpt)) { - printUsageAndDie(parser, "Exactly one of \"" + bootstrapServerOpt+ "\" or \"" + zkConnectOpt + "\" must be provided") + if (options.has(bootstrapServerOpt) == options.has(zkConnectOpt)) { + printUsageAndDie(parser, s"Exactly one of '${bootstrapServerOpt}' or '${zkConnectOpt}' must be provided") } val partitionsForPreferredReplicaElection = - if (!options.has(jsonFileOpt)) - None - else + if (options.has(jsonFileOpt)) Some(parsePreferredReplicaElectionData(Utils.readFileAsString(options.valueOf(jsonFileOpt)))) + else + None val preferredReplicaElectionCommand = if (options.has(zkConnectOpt)) { - Console.err.println(zkConnectOpt + " is deprecated and will be removed in a future version of Kafka.") - Console.err.println("Use " + bootstrapServerOpt + " instead to specify a broker to connect to.") + Console.err.println(s"$zkConnectOpt is deprecated and will be removed in a future version of Kafka.") + Console.err.println(s"Use $bootstrapServerOpt instead to specify a broker to connect to.") new ZkPreferredReplicaLeaderElectionCommand(ZkUtils(options.valueOf(zkConnectOpt), timeout.getOrElse(30000), timeout.getOrElse(30000), @@ -98,52 +97,55 @@ object PreferredReplicaLeaderElectionCommand extends Logging { } try { - preferredReplicaElectionCommand.moveLeaderToPreferredReplica(partitionsForPreferredReplicaElection) + preferredReplicaElectionCommand.electPreferredLeaders(partitionsForPreferredReplicaElection) } finally { preferredReplicaElectionCommand.close() } } - def parsePreferredReplicaElectionData(jsonString: String): immutable.Set[TopicAndPartition] = { - val partitionsUndergoingPreferredReplicaElection = + def parsePreferredReplicaElectionData(jsonString: String): Set[TopicAndPartition] = { + val partitionsForElection = ZkUtils.parsePreferredReplicaElectionDataWithoutDedup(jsonString) - if (partitionsUndergoingPreferredReplicaElection.isEmpty) { + if (partitionsForElection.isEmpty) { throw new AdminOperationException("Preferred replica election data is empty") } - val duplicatePartitions = CoreUtils.duplicates(partitionsUndergoingPreferredReplicaElection) + val duplicatePartitions = CoreUtils.duplicates(partitionsForElection) if (duplicatePartitions.nonEmpty) - throw new AdminOperationException("Preferred replica election data contains duplicate partitions: %s".format(duplicatePartitions.mkString(","))) - return partitionsUndergoingPreferredReplicaElection.toSet + throw new AdminOperationException( + s"Preferred replica election data contains duplicate partitions: ${duplicatePartitions.mkString(",")}") + return partitionsForElection.toSet } def writePreferredReplicaElectionData(zkUtils: ZkUtils, - partitionsUndergoingPreferredReplicaElection: scala.collection.Set[TopicAndPartition]) { + partitionsForElection: Set[TopicAndPartition]) { val zkPath = ZkUtils.PreferredReplicaLeaderElectionPath - val jsonData = ZkUtils.preferredReplicaLeaderElectionZkData(partitionsUndergoingPreferredReplicaElection) + val jsonData = ZkUtils.preferredReplicaLeaderElectionZkData(partitionsForElection) try { zkUtils.createPersistentPath(zkPath, jsonData) - println("Created preferred replica election path with %s".format(jsonData)) + println(s"Created preferred replica election path with $jsonData") } catch { case _: ZkNodeExistsException => val partitionsUndergoingPreferredReplicaElection = PreferredReplicaLeaderElectionCommand.parsePreferredReplicaElectionData(zkUtils.readData(zkPath)._1) throw new AdminOperationException("Preferred replica leader election currently in progress for " + - "%s. Aborting operation".format(partitionsUndergoingPreferredReplicaElection)) + s"$partitionsUndergoingPreferredReplicaElection. Aborting operation") case e2: Throwable => throw new AdminOperationException(e2.toString) } } } +/** Abstraction over different ways to perform a leader election */ trait PreferredReplicaLeaderElectionCommand { - /** - * Elect the preferred leader for the given {@code partitionsFromUser}. - * If the given {@code partitionsFromUser} are None then elect the preferred leader for all partitions. + /** Elect the preferred leader for the given {@code partitionsForElection}. + * If the given {@code partitionsForElection} are None then elect the preferred leader for all partitions. */ - def moveLeaderToPreferredReplica(partitionsFromUser: Option[scala.collection.Set[TopicAndPartition]]) : Unit + def electPreferredLeaders(partitionsForElection: Option[Set[TopicAndPartition]]) : Unit def close() : Unit } -class AdminClientPreferredReplicaLeaderElectionCommand(bootstrapServers: String, timeout: Option[Int]) extends PreferredReplicaLeaderElectionCommand with Logging { +/** Election via AdminClient.electPreferredLeaders() */ +class AdminClientPreferredReplicaLeaderElectionCommand(bootstrapServers: String, timeout: Option[Int]) + extends PreferredReplicaLeaderElectionCommand with Logging { val props = new Properties() props.setProperty(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers) timeout match { @@ -152,7 +154,7 @@ class AdminClientPreferredReplicaLeaderElectionCommand(bootstrapServers: String, case None => } - val adminClient: org.apache.kafka.clients.admin.AdminClient = org.apache.kafka.clients.admin.AdminClient.create(props) + val adminClient = org.apache.kafka.clients.admin.AdminClient.create(props) /** * Wait until the given future has completed, then return whether it completed exceptionally. @@ -168,7 +170,7 @@ class AdminClientPreferredReplicaLeaderElectionCommand(bootstrapServers: String, } } - override def moveLeaderToPreferredReplica(partitionsFromUser: Option[Set[TopicAndPartition]]): Unit = { + override def electPreferredLeaders(partitionsFromUser: Option[Set[TopicAndPartition]]): Unit = { val partitions = partitionsFromUser match { case Some(partitionsFromUser) => partitionsFromUser.map(tp => tp.asTopicPartition).toSet.asJava case None => null @@ -196,10 +198,11 @@ class AdminClientPreferredReplicaLeaderElectionCommand(bootstrapServers: String, val (exceptional, ok) = attemptedPartitions.map(tp => tp -> result.partitionResult(tp)). partition{case (_, partitionResult) => completedExceptionally(partitionResult)} if (!ok.isEmpty) { - println("Successfully completed preferred replica election for partitions %s".format(ok.map(_._1).mkString(", "))) + println(s"Successfully completed preferred replica election for partitions ${ok.map(_._1).mkString(", ")}") } if (!exceptional.isEmpty) { - val adminException = new AdminCommandFailedException(s"${exceptional.size} preferred replica(s) could not be elected") + val adminException = new AdminCommandFailedException( + s"${exceptional.size} preferred replica(s) could not be elected") for ((partition, void) <- exceptional) { val exception = try { void.get() @@ -220,9 +223,13 @@ class AdminClientPreferredReplicaLeaderElectionCommand(bootstrapServers: String, } } +/** Election via ZooKeeper + * @deprecated since Kafka 1.1.0. Use [[kafka.admin.AdminClientPreferredReplicaLeaderElectionCommand]] instead. + */ +@Deprecated class ZkPreferredReplicaLeaderElectionCommand(zkUtils: ZkUtils) extends PreferredReplicaLeaderElectionCommand { - override def moveLeaderToPreferredReplica(partitionsFromUser: Option[scala.collection.Set[TopicAndPartition]]) = { + override def electPreferredLeaders(partitionsFromUser: Option[scala.collection.Set[TopicAndPartition]]) = { try { val partitions = partitionsFromUser match { case Some(partitionsFromUser) => partitionsFromUser @@ -236,8 +243,9 @@ class ZkPreferredReplicaLeaderElectionCommand(zkUtils: ZkUtils) extends Preferre val (validPartitions, invalidPartitions) = partitions.partition(partitionsFromZk.contains) PreferredReplicaLeaderElectionCommand.writePreferredReplicaElectionData(zkUtils, validPartitions) - println("Successfully started preferred replica election for partitions %s".format(validPartitions)) - invalidPartitions.foreach(p => println("Skipping preferred replica leader election for partition %s since it doesn't exist.".format(p))) + println(s"Successfully started preferred replica election for partitions $validPartitions") + invalidPartitions.foreach(p => println( + s"Skipping preferred replica leader election for partition $p since it doesn't exist.")) } catch { case e: Throwable => throw new AdminCommandFailedException("Admin command failed", e) } diff --git a/core/src/test/scala/unit/kafka/admin/AdminTest.scala b/core/src/test/scala/unit/kafka/admin/AdminTest.scala index 3fecc89f0ad04..9beaf9a4ae78e 100755 --- a/core/src/test/scala/unit/kafka/admin/AdminTest.scala +++ b/core/src/test/scala/unit/kafka/admin/AdminTest.scala @@ -351,7 +351,7 @@ class AdminTest extends ZooKeeperTestHarness with Logging with RackAwareTest { val currentLeader = TestUtils.waitUntilLeaderIsElectedOrChanged(zkUtils, topic, partition, oldLeaderOpt = None) // trigger preferred replica election val preferredReplicaElection = new ZkPreferredReplicaLeaderElectionCommand(zkUtils) - preferredReplicaElection.moveLeaderToPreferredReplica(Some(Set(TopicAndPartition(topic, partition)))) + preferredReplicaElection.electPreferredLeaders(Some(Set(TopicAndPartition(topic, partition)))) val newLeader = TestUtils.waitUntilLeaderIsElectedOrChanged(zkUtils, topic, partition, oldLeaderOpt = Some(currentLeader)) assertEquals("Preferred replica election failed", preferredReplica, newLeader) } From 756a5d558f8a03f2e8709bef8eddef702fcb26bf Mon Sep 17 00:00:00 2001 From: Tom Bentley Date: Fri, 22 Sep 2017 15:36:10 +0100 Subject: [PATCH 06/13] Formatting and DelayedElectionOperation --- .../kafka/controller/KafkaController.scala | 24 +++++++-------- .../controller/PartitionStateMachine.scala | 2 +- .../server/DelayedElectPreferredLeader.scala | 30 +++++++++---------- .../scala/kafka/server/ReplicaManager.scala | 14 ++++----- 4 files changed, 34 insertions(+), 36 deletions(-) diff --git a/core/src/main/scala/kafka/controller/KafkaController.scala b/core/src/main/scala/kafka/controller/KafkaController.scala index 973ca301e259f..6d901af0ed178 100644 --- a/core/src/main/scala/kafka/controller/KafkaController.scala +++ b/core/src/main/scala/kafka/controller/KafkaController.scala @@ -627,16 +627,17 @@ class KafkaController(val config: KafkaConfig, zkUtils: ZkUtils, time: Time, met } def onPreferredReplicaElection(partitions: Set[TopicAndPartition], isTriggeredByAutoRebalance: Boolean = false) { - onPreferredReplicaElectionWithResults(partitions, isTriggeredByAutoRebalance).foreach(entry => - error("Error completing preferred replica leader election for partition %s".format(entry._1), entry._2) - ) + onPreferredReplicaElectionWithResults(partitions, isTriggeredByAutoRebalance).foreach { + case (topicPartition, ex) => + error(s"Error completing preferred replica leader election for partition $topicPartition", ex) + } } private def onPreferredReplicaElectionWithResults( partitions: Set[TopicAndPartition], isTriggeredByAutoRebalance: Boolean = false, viaZk: Boolean = true): Map[TopicAndPartition, Throwable] = { - info("Starting preferred replica leader election for partitions %s".format(partitions.mkString(","))) + info(s"Starting preferred replica leader election for partitions ${partitions.mkString(",")}") try { partitionStateMachine.handleStateChangesWithResults(partitions, OnlinePartition, preferredReplicaPartitionLeaderSelector) } finally { @@ -1457,13 +1458,13 @@ class KafkaController(val config: KafkaConfig, zkUtils: ZkUtils, time: Time, met override def process(): Unit = { if (!isActive) { - callback(Set(), partitions.map(partition => partition -> new ApiError(Errors.NOT_CONTROLLER, null)).toMap) + callback(Set(), partitions.map(_ -> new ApiError(Errors.NOT_CONTROLLER, null)).toMap) } else { - val (partitionsForTopicsToBeDeleted, livePartitions) = partitions.partition(partition => + val (partitionsBeingDeleted, livePartitions) = partitions.partition(partition => topicDeletionManager.isTopicQueuedUpForDeletion(partition.topic)) - if (partitionsForTopicsToBeDeleted.nonEmpty) { - error("Skipping preferred replica election for partitions %s since the respective topics are being deleted" - .format(partitionsForTopicsToBeDeleted)) + if (partitionsBeingDeleted.nonEmpty) { + error(s"Skipping preferred replica election for partitions $partitionsBeingDeleted " + + s"since the respective topics are being deleted") } // partition those where preferred is already leader val (electablePartitions, alreadyPreferred) = livePartitions.partition { partition => @@ -1485,11 +1486,10 @@ class KafkaController(val config: KafkaConfig, zkUtils: ZkUtils, time: Time, met partition -> apiError } ++ alreadyPreferred.map(_ -> ApiError.NONE) ++ - partitionsForTopicsToBeDeleted.map(_-> new ApiError(Errors.INVALID_TOPIC_EXCEPTION, "The topic is being deleted")) - debug("PreferredReplicaLeaderElection waiting: %s, results: %s".format(successfulPartitions, results)) + partitionsBeingDeleted.map(_-> new ApiError(Errors.INVALID_TOPIC_EXCEPTION, "The topic is being deleted")) + debug(s"PreferredReplicaLeaderElection waiting: $successfulPartitions, results: $results") callback(successfulPartitions, results) } - } } diff --git a/core/src/main/scala/kafka/controller/PartitionStateMachine.scala b/core/src/main/scala/kafka/controller/PartitionStateMachine.scala index 5a32d6d0aa9e7..e0efd643dfaef 100755 --- a/core/src/main/scala/kafka/controller/PartitionStateMachine.scala +++ b/core/src/main/scala/kafka/controller/PartitionStateMachine.scala @@ -125,7 +125,7 @@ class PartitionStateMachine(controller: KafkaController, stateChangeLogger: Stat def handleStateChangesWithResults(partitions: Set[TopicAndPartition], targetState: PartitionState, leaderSelector: PartitionLeaderSelector = noOpPartitionLeaderSelector, callbacks: Callbacks = (new CallbackBuilder).build): Map[TopicAndPartition, Throwable] = { - info("Invoking state change to %s for partitions %s".format(targetState, partitions.mkString(","))) + info(s"Invoking state change to $targetState for partitions ${partitions.mkString(",")}") val result = new mutable.HashMap[TopicAndPartition, Throwable]() try { brokerRequestBatch.newBatch() diff --git a/core/src/main/scala/kafka/server/DelayedElectPreferredLeader.scala b/core/src/main/scala/kafka/server/DelayedElectPreferredLeader.scala index 2984a8b2aba19..14c3a79fbd37b 100644 --- a/core/src/main/scala/kafka/server/DelayedElectPreferredLeader.scala +++ b/core/src/main/scala/kafka/server/DelayedElectPreferredLeader.scala @@ -27,20 +27,20 @@ import org.apache.kafka.common.requests.ApiError import scala.collection.{Map, Set, mutable} +case class ElectPreferredLeaderMetadata(partition: TopicAndPartition, leader: Int) + /** A delayed elect preferred leader operation that can be created by the replica manager and watched * in the elect preferred leader purgatory */ class DelayedElectPreferredLeader(delayMs: Long, - waiting: Set[TopicAndPartition], + expectedLeaders: Set[ElectPreferredLeaderMetadata], results: Map[TopicAndPartition, ApiError], replicaManager: ReplicaManager, - controllerContext: ControllerContext, responseCallback: Map[TopicAndPartition, ApiError] => Unit) extends DelayedOperation(delayMs) { - val expectedLeaders: Map[TopicAndPartition, Int]= waiting.map(tp => tp -> controllerContext.partitionReplicaAssignment(tp).head).toMap - var waitingPartitions: Set[TopicAndPartition] = new mutable.HashSet[TopicAndPartition]() ++= waiting - val fullResults = new mutable.HashSet() ++= results + var waitingPartitions = expectedLeaders.to[mutable.Set] + val fullResults = results.to[mutable.Set] /** * Call-back to execute when a delayed operation gets expired and hence forced to complete. @@ -54,7 +54,7 @@ class DelayedElectPreferredLeader(delayMs: Long, override def onComplete(): Unit = { // This could be called to force complete, so I need the full list of partitions, so I can time them all out. updateWaiting() - val timedout = waitingPartitions.map(partition => partition -> new ApiError(Errors.REQUEST_TIMED_OUT, null)).toMap + val timedout = waitingPartitions.map(meta => meta.partition -> new ApiError(Errors.REQUEST_TIMED_OUT, null)).toMap responseCallback(timedout ++ fullResults) } @@ -71,24 +71,22 @@ class DelayedElectPreferredLeader(delayMs: Long, */ override def tryComplete(): Boolean = { updateWaiting() - debug("tryComplete() waitingPartitions: %s".format(waitingPartitions)) - if (waitingPartitions.isEmpty) - forceComplete() - else - false + debug(s"tryComplete() waitingPartitions: $waitingPartitions") + waitingPartitions.isEmpty && forceComplete() } private def updateWaiting() = { - waitingPartitions.foreach{tp => - val ps = replicaManager.metadataCache.getPartitionInfo(tp.topic, tp.partition) + waitingPartitions.foreach{m => + val ps = replicaManager.metadataCache.getPartitionInfo(m.partition.topic, m.partition.partition) ps match { case Some(ps) => - if (expectedLeaders(tp) == ps.basePartitionState.leader) { - waitingPartitions -= tp - fullResults += tp -> ApiError.NONE + if (m.leader == ps.basePartitionState.leader) { + waitingPartitions -= m + fullResults += m.partition -> ApiError.NONE } case None => } } } + } diff --git a/core/src/main/scala/kafka/server/ReplicaManager.scala b/core/src/main/scala/kafka/server/ReplicaManager.scala index 0b441dc8bb6be..50643bded28f0 100644 --- a/core/src/main/scala/kafka/server/ReplicaManager.scala +++ b/core/src/main/scala/kafka/server/ReplicaManager.scala @@ -1451,23 +1451,23 @@ class ReplicaManager(val config: KafkaConfig, partitions: Set[TopicAndPartition], responseCallback: Map[TopicAndPartition, ApiError] => Unit, requestTimeout: Long): Unit = { - val partitionsFromZk = zkUtils.getPartitionsForTopics(partitions.map(_.topic).toSeq).flatMap { - case (topic, partitions) => partitions.map(TopicAndPartition(topic, _)) - }.toSet - val (validPartitions, invalidPartitions) = partitions.partition(partitionsFromZk.contains) + val (validPartitions, invalidPartitions) = partitions.partition(tp => metadataCache.contains(tp.asTopicPartition)) val invalidPartitionsResults = invalidPartitions.map { p => val msg = s"Skipping preferred replica leader election for partition ${p} since it doesn't exist." logger.info(msg) - p -> new ApiError(Errors.UNKNOWN_TOPIC_OR_PARTITION, msg) + p -> new ApiError(Errors.UNKNOWN_TOPIC_OR_PARTITION, s"The partition '$p' does not exist.") }.toMap def electionCallback(waiting: Set[TopicAndPartition], results: Map[TopicAndPartition, ApiError]) = { if (waiting.nonEmpty) { // timeout + val expectedLeaders = waiting.map( + tp => ElectPreferredLeaderMetadata(tp, controller.controllerContext.partitionReplicaAssignment(tp).head)) + val watchKeys = waiting.map(p => new TopicPartitionOperationKey(p.topic, p.partition)).toSeq delayedElectPreferredLeaderPurgatory.tryCompleteElseWatch( - new DelayedElectPreferredLeader(requestTimeout, waiting, results ++ invalidPartitionsResults, - this, controller.controllerContext, responseCallback), + new DelayedElectPreferredLeader(requestTimeout, expectedLeaders, results ++ invalidPartitionsResults, + this, responseCallback), watchKeys) } else { // There are no partitions actually being elected, so return immediately From c734c0eef9e77f9efe620d969158e9f8eb73bf9a Mon Sep 17 00:00:00 2001 From: Tom Bentley Date: Fri, 22 Sep 2017 15:36:37 +0100 Subject: [PATCH 07/13] Add a test TODO: Add Authorizer test --- .../api/AdminClientIntegrationTest.scala | 176 ++++++++++++++++++ 1 file changed, 176 insertions(+) diff --git a/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala index a18b21795de61..1b6ef4d9d10fc 100644 --- a/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala +++ b/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala @@ -22,6 +22,7 @@ import java.util.Arrays.asList import java.util.concurrent.{ExecutionException, TimeUnit} import java.io.File +import kafka.common.TopicAndPartition import org.apache.kafka.clients.admin.KafkaAdminClientTest import org.apache.kafka.common.utils.{Time, Utils} import kafka.integration.KafkaServerTestHarness @@ -736,6 +737,181 @@ class AdminClientIntegrationTest extends KafkaServerTestHarness with Logging { client.close() assertEquals(1, factory.failuresInjected) } + + @Test + def testElectPreferredLeaders(): Unit = { + client = AdminClient.create(createConfig) + + var prefer0 = Seq(0, 1, 2) + var prefer1 = Seq(1, 2, 0) + var prefer2 = Seq(2, 0, 1) + + val partition1 = new TopicPartition("elect-preferred-leaders-topic-1", 0) + TestUtils.createTopic(zkUtils, partition1.topic, Map[Int, Seq[Int]](partition1.partition -> prefer0), servers) + + val partition2 = new TopicPartition("elect-preferred-leaders-topic-2", 0) + TestUtils.createTopic(zkUtils, partition2.topic, Map[Int, Seq[Int]](partition2.partition -> prefer0), servers) + + def getLeader(topicPartition: TopicPartition) = + client.describeTopics(asList(topicPartition.topic)).values.get(topicPartition.topic). + get.partitions.get(topicPartition.partition).leader.id + + def changePreferredLeader(newAssignment: Seq[Int]) = { + val prior1 = getLeader(partition1) + val prior2 = getLeader(partition2) + + zkUtils.updatePartitionReassignmentData(Map( + new TopicAndPartition(partition1) -> newAssignment, + new TopicAndPartition(partition2) -> newAssignment)) + // Check the leader hasn't moved + assertEquals(prior1, getLeader(partition1)) + assertEquals(prior1, getLeader(partition2)) + } + + // Check current leaders are 0 + assertEquals(0, getLeader(partition1)) + assertEquals(0, getLeader(partition2)) + + // Noop election + var electResult = client.electPreferredLeaders(asList(partition1)) + electResult.partitionResult(partition1).get() + assertEquals(0, getLeader(partition1)) + + // Noop election with null partitions + electResult = client.electPreferredLeaders(null) + electResult.partitionResult(partition1).get() + assertEquals(0, getLeader(partition1)) + electResult.partitionResult(partition2).get() + assertEquals(0, getLeader(partition2)) + + // Now change the preferred leader to 1 + changePreferredLeader(prefer1) + + // meaningful election + electResult = client.electPreferredLeaders(asList(partition1)) + assertEquals(Set(partition1).asJava, electResult.partitions.get) + electResult.partitionResult(partition1).get() + assertEquals(1, getLeader(partition1)) + // topic 2 unchanged + try { + electResult.partitionResult(partition2).get() + fail("topic 2 wasn't requested") + } catch { + case e: ExecutionException => + val cause = e.getCause + assertTrue(cause.isInstanceOf[IllegalArgumentException]) + assertEquals("Preferred leader election for partition \"elect-preferred-leaders-topic-2-0\" was not attempted", + cause.getMessage) + assertEquals(0, getLeader(partition2)) + } + + // meaningful election with null partitions + electResult = client.electPreferredLeaders(null) + assertEquals(Set(partition1, partition2).asJava, electResult.partitions.get) + electResult.partitionResult(partition1).get() + assertEquals(1, getLeader(partition1)) + electResult.partitionResult(partition2).get() + assertEquals(1, getLeader(partition2)) + + // unknown topic + val unknownPartition = new TopicPartition("topic-does-not-exist", 0) + electResult = client.electPreferredLeaders(asList(unknownPartition)) + assertEquals(Set(unknownPartition).asJava, electResult.partitions.get) + try { + electResult.partitionResult(unknownPartition).get() + } catch { + case e: Exception => + val cause = e.getCause + assertTrue(cause.isInstanceOf[UnknownTopicOrPartitionException]) + assertEquals("The partition 'topic-does-not-exist-0' does not exist.", + cause.getMessage) + assertEquals(1, getLeader(partition1)) + assertEquals(1, getLeader(partition2)) + } + + // Now change the preferred leader to 2 + changePreferredLeader(prefer2) + + // mixed results + electResult = client.electPreferredLeaders(asList(unknownPartition, partition1)) + assertEquals(Set(unknownPartition, partition1).asJava, electResult.partitions.get) + assertEquals(2, getLeader(partition1)) + assertEquals(1, getLeader(partition2)) + try { + electResult.partitionResult(unknownPartition).get() + } catch { + case e: Exception => + val cause = e.getCause + assertTrue(cause.isInstanceOf[UnknownTopicOrPartitionException]) + assertEquals("The partition 'topic-does-not-exist-0' does not exist.", + cause.getMessage) + } + + // dupe partitions + electResult = client.electPreferredLeaders(asList(partition2, partition2)) + assertEquals(Set(partition2).asJava, electResult.partitions.get) + electResult.partitionResult(partition2).get() + assertEquals(2, getLeader(partition2)) + + // Now change the preferred leader to 1 + changePreferredLeader(prefer1) + // but shut it down... + servers(1).shutdown() + //assertEquals(2, getLeader(partition1)) + //assertEquals(2, getLeader(partition2)) + + // ... now what happens if we try to elect the preferred leader and it's down? + val shortTimeout = new ElectPreferredLeadersOptions().timeoutMs(10000) + electResult = client.electPreferredLeaders(asList(partition1), shortTimeout) + assertEquals(Set(partition1).asJava, electResult.partitions.get) + try { + electResult.partitionResult(partition1).get() + fail() + } catch { + case e: Exception => + val cause = e.getCause + cause.printStackTrace() + assertTrue(cause.isInstanceOf[LeaderNotAvailableException]) + assertTrue(s"Wrong message ${cause.getMessage}", cause.getMessage.contains( + "Preferred replica 1 for partition elect-preferred-leaders-topic-1-0 is either not alive or not in the isr.")) + } + assertEquals(2, getLeader(partition1)) + + // preferred leader unavailable with null argument + electResult = client.electPreferredLeaders(null, shortTimeout) + assertEquals(Set(partition1, partition2).asJava, electResult.partitions.get) + try { + electResult.partitionResult(partition1).get() + fail() + } catch { + case e: Exception => + val cause = e.getCause + cause.printStackTrace() + assertTrue(cause.isInstanceOf[LeaderNotAvailableException]) + assertTrue(s"Wrong message ${cause.getMessage}", cause.getMessage.contains( + "Preferred replica 1 for partition elect-preferred-leaders-topic-1-0 is either not alive or not in the isr.")) + } + try { + electResult.partitionResult(partition2).get() + fail() + } catch { + case e: Exception => + val cause = e.getCause + cause.printStackTrace() + assertTrue(cause.isInstanceOf[LeaderNotAvailableException]) + assertTrue(s"Wrong message ${cause.getMessage}", cause.getMessage.contains( + "Preferred replica 1 for partition elect-preferred-leaders-topic-2-0 is either not alive or not in the isr.")) + } + + assertEquals(2, getLeader(partition1)) + assertEquals(2, getLeader(partition2)) + + + // TODO authz failure + + // TODO timeout + + } } object AdminClientIntegrationTest { From 736383b7e6801270e33c6cb1aab0aa27f8f115a5 Mon Sep 17 00:00:00 2001 From: Tom Bentley Date: Wed, 27 Sep 2017 16:18:44 +0100 Subject: [PATCH 08/13] Add ELECT_PREFERRED_LEADERS to AuthorizerIntegrationTests --- .../main/scala/kafka/server/KafkaApis.scala | 40 +++++++++---------- .../kafka/api/AuthorizerIntegrationTest.scala | 17 +++++--- 2 files changed, 31 insertions(+), 26 deletions(-) diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala index f5f3a72a89006..d7c7328c8d366 100644 --- a/core/src/main/scala/kafka/server/KafkaApis.scala +++ b/core/src/main/scala/kafka/server/KafkaApis.scala @@ -1932,26 +1932,6 @@ class KafkaApis(val requestChannel: RequestChannel, new AlterConfigsResponse(requestThrottleMs, (authorizedResult ++ unauthorizedResult).asJava)) } - def handleElectPreferredReplicaLeader(request: RequestChannel.Request): Unit = { - - val electionRequest = request.body[ElectPreferredLeadersRequest] - val partitions = - if (electionRequest.topicPartitions() == null) { - zkUtils.getAllPartitions() - } else { - electionRequest.topicPartitions().asScala.map(t => new TopicAndPartition(t)) - } - def sendResponseCallback(result: Map[TopicAndPartition, ApiError]): Unit = { - sendResponseMaybeThrottle(request, requestThrottleMs => - new ElectPreferredLeadersResponse(requestThrottleMs, result.map(entry => entry._1.asTopicPartition -> entry._2).asJava)) - } - if (!authorize(request.session, Alter, Resource.ClusterResource)) { - sendResponseCallback(partitions.map(partition => partition -> new ApiError(Errors.CLUSTER_AUTHORIZATION_FAILED, null)).toMap) - } else { - replicaManager.electPreferredLeaders(controller, partitions, sendResponseCallback, electionRequest.timeout) - } - } - private def configsAuthorizationApiError(session: RequestChannel.Session, resource: RResource): ApiError = { val error = resource.`type` match { case RResourceType.BROKER => Errors.CLUSTER_AUTHORIZATION_FAILED @@ -2012,6 +1992,26 @@ class KafkaApis(val requestChannel: RequestChannel, sendResponseMaybeThrottle(request, throttleTimeMs => new DescribeLogDirsResponse(throttleTimeMs, logDirInfos.asJava)) } + def handleElectPreferredReplicaLeader(request: RequestChannel.Request): Unit = { + + val electionRequest = request.body[ElectPreferredLeadersRequest] + val partitions = + if (electionRequest.topicPartitions() == null) { + zkUtils.getAllPartitions() + } else { + electionRequest.topicPartitions().asScala.map(t => new TopicAndPartition(t)) + } + def sendResponseCallback(result: Map[TopicAndPartition, ApiError]): Unit = { + sendResponseMaybeThrottle(request, requestThrottleMs => + new ElectPreferredLeadersResponse(requestThrottleMs, result.map(entry => entry._1.asTopicPartition -> entry._2).asJava)) + } + if (!authorize(request.session, Alter, Resource.ClusterResource)) { + sendResponseCallback(partitions.map(partition => partition -> new ApiError(Errors.CLUSTER_AUTHORIZATION_FAILED, null)).toMap) + } else { + replicaManager.electPreferredLeaders(controller, partitions, sendResponseCallback, electionRequest.timeout) + } + } + def authorizeClusterAction(request: RequestChannel.Request): Unit = { if (!authorize(request.session, ClusterAction, Resource.ClusterResource)) throw new ClusterAuthorizationException(s"Request $request is not authorized.") diff --git a/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala index 522fcd31db90f..463f11ea01e68 100644 --- a/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala +++ b/core/src/test/scala/integration/kafka/api/AuthorizerIntegrationTest.scala @@ -146,7 +146,8 @@ class AuthorizerIntegrationTest extends BaseRequestTest { ApiKeys.DESCRIBE_ACLS -> classOf[DescribeAclsResponse], ApiKeys.ALTER_REPLICA_LOG_DIRS -> classOf[AlterReplicaLogDirsResponse], ApiKeys.DESCRIBE_LOG_DIRS -> classOf[DescribeLogDirsResponse], - ApiKeys.CREATE_PARTITIONS -> classOf[CreatePartitionsResponse] + ApiKeys.CREATE_PARTITIONS -> classOf[CreatePartitionsResponse], + ApiKeys.ELECT_PREFERRED_LEADERS -> classOf[ElectPreferredLeadersResponse] ) val requestKeyToError = Map[ApiKeys, Nothing => Errors]( @@ -186,7 +187,8 @@ class AuthorizerIntegrationTest extends BaseRequestTest { ApiKeys.ALTER_REPLICA_LOG_DIRS -> ((resp: AlterReplicaLogDirsResponse) => resp.responses.get(tp)), ApiKeys.DESCRIBE_LOG_DIRS -> ((resp: DescribeLogDirsResponse) => if (resp.logDirInfos.size() > 0) resp.logDirInfos.asScala.head._2.error else Errors.CLUSTER_AUTHORIZATION_FAILED), - ApiKeys.CREATE_PARTITIONS -> ((resp: CreatePartitionsResponse) => resp.errors.asScala.find(_._1 == topic).get._2.error) + ApiKeys.CREATE_PARTITIONS -> ((resp: CreatePartitionsResponse) => resp.errors.asScala.find(_._1 == topic).get._2.error), + ApiKeys.ELECT_PREFERRED_LEADERS -> ((resp: ElectPreferredLeadersResponse) => resp.errors.get(tp).error) ) val requestKeysToAcls = Map[ApiKeys, Map[Resource, Set[Acl]]]( @@ -223,8 +225,8 @@ class AuthorizerIntegrationTest extends BaseRequestTest { ApiKeys.DELETE_ACLS -> clusterAlterAcl, ApiKeys.ALTER_REPLICA_LOG_DIRS -> clusterAlterAcl, ApiKeys.DESCRIBE_LOG_DIRS -> clusterDescribeAcl, - ApiKeys.CREATE_PARTITIONS -> topicAlterAcl - + ApiKeys.CREATE_PARTITIONS -> topicAlterAcl, + ApiKeys.ELECT_PREFERRED_LEADERS -> clusterAlterAcl ) @Before @@ -386,6 +388,7 @@ class AuthorizerIntegrationTest extends BaseRequestTest { private def addOffsetsToTxnRequest = new AddOffsetsToTxnRequest.Builder(transactionalId, 1, 1, group).build() + private def electPreferredLeadersRequest = new ElectPreferredLeadersRequest.Builder(Collections.singleton(tp), 10000).build() @Test def testAuthorizationWithTopicExisting() { @@ -419,7 +422,8 @@ class AuthorizerIntegrationTest extends BaseRequestTest { ApiKeys.DESCRIBE_LOG_DIRS -> describeLogDirsRequest, ApiKeys.CREATE_PARTITIONS -> createPartitionsRequest, ApiKeys.ADD_PARTITIONS_TO_TXN -> addPartitionsToTxnRequest, - ApiKeys.ADD_OFFSETS_TO_TXN -> addOffsetsToTxnRequest + ApiKeys.ADD_OFFSETS_TO_TXN -> addOffsetsToTxnRequest, + ApiKeys.ELECT_PREFERRED_LEADERS -> electPreferredLeadersRequest ) for ((key, request) <- requestKeyToRequest) { @@ -463,7 +467,8 @@ class AuthorizerIntegrationTest extends BaseRequestTest { ApiKeys.DELETE_RECORDS -> deleteRecordsRequest, ApiKeys.ADD_PARTITIONS_TO_TXN -> addPartitionsToTxnRequest, ApiKeys.ADD_OFFSETS_TO_TXN -> addOffsetsToTxnRequest, - ApiKeys.CREATE_PARTITIONS -> createPartitionsRequest + ApiKeys.CREATE_PARTITIONS -> createPartitionsRequest, + ApiKeys.ELECT_PREFERRED_LEADERS -> electPreferredLeadersRequest ) for ((key, request) <- requestKeyToRequest) { From f8c1d1ba0fe33ea9258a4b99a071a53d3c77028a Mon Sep 17 00:00:00 2001 From: Tom Bentley Date: Wed, 27 Sep 2017 21:15:45 +0100 Subject: [PATCH 09/13] Fix flaky test --- .../api/AdminClientIntegrationTest.scala | 69 ++++++++++++------- 1 file changed, 44 insertions(+), 25 deletions(-) diff --git a/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala b/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala index 1b6ef4d9d10fc..c62ca743c1fd2 100644 --- a/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala +++ b/core/src/test/scala/integration/kafka/api/AdminClientIntegrationTest.scala @@ -44,6 +44,7 @@ import org.junit.Assert._ import scala.util.Random import scala.collection.JavaConverters._ +import scala.collection.mutable /** * An integration test of the KafkaAdminClient. @@ -752,37 +753,54 @@ class AdminClientIntegrationTest extends KafkaServerTestHarness with Logging { val partition2 = new TopicPartition("elect-preferred-leaders-topic-2", 0) TestUtils.createTopic(zkUtils, partition2.topic, Map[Int, Seq[Int]](partition2.partition -> prefer0), servers) - def getLeader(topicPartition: TopicPartition) = + def currentLeader(topicPartition: TopicPartition) = client.describeTopics(asList(topicPartition.topic)).values.get(topicPartition.topic). get.partitions.get(topicPartition.partition).leader.id + def preferredLeader(topicPartition: TopicPartition) = + client.describeTopics(asList(topicPartition.topic)).values.get(topicPartition.topic). + get.partitions.get(topicPartition.partition).replicas.get(0).id + + def waitForLeaderToBecome(topicPartition: TopicPartition, leader: Int) = + TestUtils.waitUntilTrue(() => currentLeader(topicPartition) == leader, s"Expected leader to become $leader", 10000) + + /** Changes the preferred leader without changing the current leader. */ def changePreferredLeader(newAssignment: Seq[Int]) = { - val prior1 = getLeader(partition1) - val prior2 = getLeader(partition2) + var preferred = newAssignment.head + val prior1 = currentLeader(partition1) + val prior2 = currentLeader(partition2) - zkUtils.updatePartitionReassignmentData(Map( - new TopicAndPartition(partition1) -> newAssignment, - new TopicAndPartition(partition2) -> newAssignment)) + val m = mutable.Map.empty[TopicAndPartition, Seq[Int]]; + + if (prior1 != preferred) + m += new TopicAndPartition(partition1) -> newAssignment + if (prior2 != preferred) + m += new TopicAndPartition(partition2) -> newAssignment + + zkUtils.updatePartitionReassignmentData(m) + TestUtils.waitUntilTrue( + () => preferredLeader(partition1) == preferred && preferredLeader(partition2) == preferred, + s"Expected preferred leader to become $preferred, but is ${preferredLeader(partition1)} and ${preferredLeader(partition2)}", 10000) // Check the leader hasn't moved - assertEquals(prior1, getLeader(partition1)) - assertEquals(prior1, getLeader(partition2)) + assertEquals(prior1, currentLeader(partition1)) + assertEquals(prior2, currentLeader(partition2)) } // Check current leaders are 0 - assertEquals(0, getLeader(partition1)) - assertEquals(0, getLeader(partition2)) + assertEquals(0, currentLeader(partition1)) + assertEquals(0, currentLeader(partition2)) // Noop election var electResult = client.electPreferredLeaders(asList(partition1)) electResult.partitionResult(partition1).get() - assertEquals(0, getLeader(partition1)) + assertEquals(0, currentLeader(partition1)) // Noop election with null partitions electResult = client.electPreferredLeaders(null) electResult.partitionResult(partition1).get() - assertEquals(0, getLeader(partition1)) + assertEquals(0, currentLeader(partition1)) electResult.partitionResult(partition2).get() - assertEquals(0, getLeader(partition2)) + assertEquals(0, currentLeader(partition2)) // Now change the preferred leader to 1 changePreferredLeader(prefer1) @@ -791,7 +809,8 @@ class AdminClientIntegrationTest extends KafkaServerTestHarness with Logging { electResult = client.electPreferredLeaders(asList(partition1)) assertEquals(Set(partition1).asJava, electResult.partitions.get) electResult.partitionResult(partition1).get() - assertEquals(1, getLeader(partition1)) + waitForLeaderToBecome(partition1, 1) + // topic 2 unchanged try { electResult.partitionResult(partition2).get() @@ -802,16 +821,16 @@ class AdminClientIntegrationTest extends KafkaServerTestHarness with Logging { assertTrue(cause.isInstanceOf[IllegalArgumentException]) assertEquals("Preferred leader election for partition \"elect-preferred-leaders-topic-2-0\" was not attempted", cause.getMessage) - assertEquals(0, getLeader(partition2)) + assertEquals(0, currentLeader(partition2)) } // meaningful election with null partitions electResult = client.electPreferredLeaders(null) assertEquals(Set(partition1, partition2).asJava, electResult.partitions.get) electResult.partitionResult(partition1).get() - assertEquals(1, getLeader(partition1)) + waitForLeaderToBecome(partition1, 1) electResult.partitionResult(partition2).get() - assertEquals(1, getLeader(partition2)) + waitForLeaderToBecome(partition2, 1) // unknown topic val unknownPartition = new TopicPartition("topic-does-not-exist", 0) @@ -825,8 +844,8 @@ class AdminClientIntegrationTest extends KafkaServerTestHarness with Logging { assertTrue(cause.isInstanceOf[UnknownTopicOrPartitionException]) assertEquals("The partition 'topic-does-not-exist-0' does not exist.", cause.getMessage) - assertEquals(1, getLeader(partition1)) - assertEquals(1, getLeader(partition2)) + assertEquals(1, currentLeader(partition1)) + assertEquals(1, currentLeader(partition2)) } // Now change the preferred leader to 2 @@ -835,8 +854,8 @@ class AdminClientIntegrationTest extends KafkaServerTestHarness with Logging { // mixed results electResult = client.electPreferredLeaders(asList(unknownPartition, partition1)) assertEquals(Set(unknownPartition, partition1).asJava, electResult.partitions.get) - assertEquals(2, getLeader(partition1)) - assertEquals(1, getLeader(partition2)) + waitForLeaderToBecome(partition1, 2) + assertEquals(1, currentLeader(partition2)) try { electResult.partitionResult(unknownPartition).get() } catch { @@ -851,7 +870,7 @@ class AdminClientIntegrationTest extends KafkaServerTestHarness with Logging { electResult = client.electPreferredLeaders(asList(partition2, partition2)) assertEquals(Set(partition2).asJava, electResult.partitions.get) electResult.partitionResult(partition2).get() - assertEquals(2, getLeader(partition2)) + waitForLeaderToBecome(partition2, 2) // Now change the preferred leader to 1 changePreferredLeader(prefer1) @@ -875,7 +894,7 @@ class AdminClientIntegrationTest extends KafkaServerTestHarness with Logging { assertTrue(s"Wrong message ${cause.getMessage}", cause.getMessage.contains( "Preferred replica 1 for partition elect-preferred-leaders-topic-1-0 is either not alive or not in the isr.")) } - assertEquals(2, getLeader(partition1)) + assertEquals(2, currentLeader(partition1)) // preferred leader unavailable with null argument electResult = client.electPreferredLeaders(null, shortTimeout) @@ -903,8 +922,8 @@ class AdminClientIntegrationTest extends KafkaServerTestHarness with Logging { "Preferred replica 1 for partition elect-preferred-leaders-topic-2-0 is either not alive or not in the isr.")) } - assertEquals(2, getLeader(partition1)) - assertEquals(2, getLeader(partition2)) + assertEquals(2, currentLeader(partition1)) + assertEquals(2, currentLeader(partition2)) // TODO authz failure From 64fcbfc1e3cbf5949431f8c7ca4011f19fa3ac3d Mon Sep 17 00:00:00 2001 From: Tom Bentley Date: Thu, 12 Oct 2017 12:30:58 +0100 Subject: [PATCH 10/13] Add errorCounts() method --- .../common/requests/ElectPreferredLeadersResponse.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/clients/src/main/java/org/apache/kafka/common/requests/ElectPreferredLeadersResponse.java b/clients/src/main/java/org/apache/kafka/common/requests/ElectPreferredLeadersResponse.java index e61282f94331b..ee1059e318cf6 100644 --- a/clients/src/main/java/org/apache/kafka/common/requests/ElectPreferredLeadersResponse.java +++ b/clients/src/main/java/org/apache/kafka/common/requests/ElectPreferredLeadersResponse.java @@ -19,6 +19,7 @@ import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.protocol.types.ArrayOf; import org.apache.kafka.common.protocol.types.Field; import org.apache.kafka.common.protocol.types.Schema; @@ -90,6 +91,11 @@ public int throttleTimeMs() { return throttleTimeMs; } + @Override + public Map errorCounts() { + return apiErrorCounts(errors); + } + @Override protected Struct toStruct(short version) { Struct struct = new Struct(ApiKeys.ELECT_PREFERRED_LEADERS.responseSchema(version)); From b4787c9a6d1dc8b0fbc7d78833cd19389cdc06cf Mon Sep 17 00:00:00 2001 From: Tom Bentley Date: Mon, 30 Oct 2017 11:16:59 +0000 Subject: [PATCH 11/13] Move exceptionalFuture() to KafkaFuture. --- .../admin/ElectPreferredLeadersResult.java | 15 ++++----------- .../java/org/apache/kafka/common/KafkaFuture.java | 9 +++++++++ 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/ElectPreferredLeadersResult.java b/clients/src/main/java/org/apache/kafka/clients/admin/ElectPreferredLeadersResult.java index 350414be5ca1b..5d732e7a7510b 100644 --- a/clients/src/main/java/org/apache/kafka/clients/admin/ElectPreferredLeadersResult.java +++ b/clients/src/main/java/org/apache/kafka/clients/admin/ElectPreferredLeadersResult.java @@ -41,13 +41,6 @@ public class ElectPreferredLeadersResult { this.futures = futures; } - /** Return a new future that has completed exceptionally */ - private static KafkaFuture exceptionalFuture(Throwable e) { - KafkaFutureImpl future = new KafkaFutureImpl<>(); - future.completeExceptionally(e); - return future; - } - /** * Get the result of the election for the given {@code partition}. * If there was not an election triggered for the given {@code partition}, the @@ -58,7 +51,7 @@ public KafkaFuture partitionResult(TopicPartition partition) { try { map = futures.get(); } catch (InterruptedException | ExecutionException e) { - return exceptionalFuture(e); + return KafkaFuture.exceptionalFuture(e); } KafkaFuture result = map.get(partition); if (result == null) { @@ -84,9 +77,9 @@ public KafkaFuture> partitions() { try { map = futures.get(); } catch (InterruptedException e) { - return exceptionalFuture(e); + return KafkaFuture.exceptionalFuture(e); } catch (ExecutionException e) { - return exceptionalFuture(e.getCause()); + return KafkaFuture.exceptionalFuture(e.getCause()); } KafkaFutureImpl> result = new KafkaFutureImpl<>(); result.complete(map.keySet()); @@ -101,7 +94,7 @@ public KafkaFuture all() { try { map = futures.get(); } catch (InterruptedException | ExecutionException e) { - return exceptionalFuture(e); + return KafkaFuture.exceptionalFuture(e); } return KafkaFuture.allOf(map.values().toArray(new KafkaFuture[0])); } diff --git a/clients/src/main/java/org/apache/kafka/common/KafkaFuture.java b/clients/src/main/java/org/apache/kafka/common/KafkaFuture.java index 23e218137a93a..c1367e4cb6104 100644 --- a/clients/src/main/java/org/apache/kafka/common/KafkaFuture.java +++ b/clients/src/main/java/org/apache/kafka/common/KafkaFuture.java @@ -84,6 +84,15 @@ public static KafkaFuture completedFuture(U value) { return future; } + /** + * Return a new future that has completed exceptionally with the given exception. + */ + public static KafkaFuture exceptionalFuture(Throwable exception) { + KafkaFutureImpl future = new KafkaFutureImpl<>(); + future.completeExceptionally(exception); + return future; + } + /** * Returns a new KafkaFuture that is completed when all the given futures have completed. If * any future throws an exception, the returned future returns it. If multiple futures throw From 3a92833ed321e7e9d885497fcf286cdfaede27e7 Mon Sep 17 00:00:00 2001 From: Tom Bentley Date: Mon, 30 Oct 2017 18:30:19 +0000 Subject: [PATCH 12/13] WIP solution using thenCompose() --- .../admin/ElectPreferredLeadersResult.java | 59 +++++++++---------- .../kafka/clients/admin/KafkaAdminClient.java | 4 ++ .../org/apache/kafka/common/KafkaFuture.java | 2 + .../common/internals/KafkaFutureImpl.java | 42 +++++++++++++ 4 files changed, 76 insertions(+), 31 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/ElectPreferredLeadersResult.java b/clients/src/main/java/org/apache/kafka/clients/admin/ElectPreferredLeadersResult.java index 5d732e7a7510b..db02399ce2991 100644 --- a/clients/src/main/java/org/apache/kafka/clients/admin/ElectPreferredLeadersResult.java +++ b/clients/src/main/java/org/apache/kafka/clients/admin/ElectPreferredLeadersResult.java @@ -25,7 +25,6 @@ import java.util.Collection; import java.util.Map; import java.util.Set; -import java.util.concurrent.ExecutionException; /** * The result of {@link AdminClient#electPreferredLeaders(Collection, ElectPreferredLeadersOptions)} @@ -46,21 +45,22 @@ public class ElectPreferredLeadersResult { * If there was not an election triggered for the given {@code partition}, the * returned future will complete with an error. */ - public KafkaFuture partitionResult(TopicPartition partition) { - final Map> map; - try { - map = futures.get(); - } catch (InterruptedException | ExecutionException e) { - return KafkaFuture.exceptionalFuture(e); + public KafkaFuture partitionResult(final TopicPartition partition) { + class Function>> + extends KafkaFuture.Function> { + @Override + public KafkaFuture apply(T map) { + KafkaFuture result = map.get(partition); + if (result == null) { + KafkaFutureImpl future = new KafkaFutureImpl<>(); + future.completeExceptionally(new IllegalArgumentException( + "Preferred leader election for partition \"" + partition + "\" was not attempted")); + result = future; + } + return result; + } } - KafkaFuture result = map.get(partition); - if (result == null) { - KafkaFutureImpl future = new KafkaFutureImpl<>(); - future.completeExceptionally(new IllegalArgumentException( - "Preferred leader election for partition \"" + partition + "\" was not attempted")); - result = future; - } - return result; + return futures.thenCompose(new Function()); } /** @@ -73,29 +73,26 @@ public KafkaFuture partitionResult(TopicPartition partition) { * with a null {@code partitions} argument.

*/ public KafkaFuture> partitions() { - final Map> map; - try { - map = futures.get(); - } catch (InterruptedException e) { - return KafkaFuture.exceptionalFuture(e); - } catch (ExecutionException e) { - return KafkaFuture.exceptionalFuture(e.getCause()); + class Function>> extends KafkaFuture.Function> { + @Override + public Set apply(T map) { + return map.keySet(); + } } - KafkaFutureImpl> result = new KafkaFutureImpl<>(); - result.complete(map.keySet()); - return result; + return futures.thenApply(new Function()); } /** * Return a future which succeeds if all the topic elections succeed. */ public KafkaFuture all() { - final Map> map; - try { - map = futures.get(); - } catch (InterruptedException | ExecutionException e) { - return KafkaFuture.exceptionalFuture(e); + class Function>> extends KafkaFuture.Function> { + + @Override + public KafkaFuture apply(T map) { + return KafkaFuture.allOf(map.values().toArray(new KafkaFuture[0])); + } } - return KafkaFuture.allOf(map.values().toArray(new KafkaFuture[0])); + return futures.thenCompose(new Function()); } } diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java index 21b9761877814..7e470b44c9821 100644 --- a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java +++ b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java @@ -1914,6 +1914,10 @@ public void handleResponse(AbstractResponse abstractResponse) { KafkaFutureImpl future; if (knownPartitions) { future = mapOfFutures.get(partition); + if (future == null) { + future = new KafkaFutureImpl(); + future.completeExceptionally(new IllegalStateException("Unexpected partition in response")); + } } else { future = new KafkaFutureImpl(); mapOfFutures.put(partition, future); diff --git a/clients/src/main/java/org/apache/kafka/common/KafkaFuture.java b/clients/src/main/java/org/apache/kafka/common/KafkaFuture.java index c1367e4cb6104..0d6ac534d8f03 100644 --- a/clients/src/main/java/org/apache/kafka/common/KafkaFuture.java +++ b/clients/src/main/java/org/apache/kafka/common/KafkaFuture.java @@ -113,6 +113,8 @@ public static KafkaFuture allOf(KafkaFuture... futures) { */ public abstract KafkaFuture thenApply(Function function); + public abstract KafkaFuture thenCompose(Function> function); + protected abstract void addWaiter(BiConsumer action); /** diff --git a/clients/src/main/java/org/apache/kafka/common/internals/KafkaFutureImpl.java b/clients/src/main/java/org/apache/kafka/common/internals/KafkaFutureImpl.java index 9ca019b2661ae..d25ea150a1882 100644 --- a/clients/src/main/java/org/apache/kafka/common/internals/KafkaFutureImpl.java +++ b/clients/src/main/java/org/apache/kafka/common/internals/KafkaFutureImpl.java @@ -70,6 +70,41 @@ public void accept(A a, Throwable exception) { } } + private static class Composer extends BiConsumer { + private final Function> function; + private final KafkaFutureImpl future; + + Composer(Function> function, KafkaFutureImpl future) { + this.function = function; + this.future = future; + } + + // Called when "this" completes + @Override + public void accept(A a, Throwable exception) { + if (exception != null) { + future.completeExceptionally(exception); + } else { + try { + // TODO typecast eek! + KafkaFutureImpl mapped = (KafkaFutureImpl) function.apply(a); + mapped.addWaiter(new BiConsumer() { + @Override + public void accept(B b, Throwable throwable) { + if (throwable != null) { + future.completeExceptionally(throwable); + } else { + future.complete(b); + } + } + }); + } catch (Throwable t) { + future.completeExceptionally(t); + } + } + } + } + private static class SingleWaiter extends BiConsumer { private R value = null; private Throwable exception = null; @@ -146,6 +181,13 @@ public KafkaFuture thenApply(Function function) { return future; } + @Override + public KafkaFuture thenCompose(Function> function) { + KafkaFutureImpl future = new KafkaFutureImpl(); + addWaiter(new Composer<>(function, future)); + return future; + } + @Override protected synchronized void addWaiter(BiConsumer action) { if (exception != null) { From 0a57af9172985c4c382087aa167bd4421b375d5d Mon Sep 17 00:00:00 2001 From: "Colin P. Mccabe" Date: Mon, 6 Nov 2017 14:06:32 -0800 Subject: [PATCH 13/13] Rework futures --- .../admin/ElectPreferredLeadersResult.java | 75 ++++++++++++------- .../kafka/clients/admin/KafkaAdminClient.java | 54 ++----------- 2 files changed, 54 insertions(+), 75 deletions(-) diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/ElectPreferredLeadersResult.java b/clients/src/main/java/org/apache/kafka/clients/admin/ElectPreferredLeadersResult.java index db02399ce2991..c2521a5893b49 100644 --- a/clients/src/main/java/org/apache/kafka/clients/admin/ElectPreferredLeadersResult.java +++ b/clients/src/main/java/org/apache/kafka/clients/admin/ElectPreferredLeadersResult.java @@ -20,7 +20,10 @@ import org.apache.kafka.common.KafkaFuture; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.annotation.InterfaceStability; +import org.apache.kafka.common.errors.ApiException; +import org.apache.kafka.common.errors.UnknownTopicOrPartitionException; import org.apache.kafka.common.internals.KafkaFutureImpl; +import org.apache.kafka.common.requests.ApiError; import java.util.Collection; import java.util.Map; @@ -34,10 +37,10 @@ @InterfaceStability.Evolving public class ElectPreferredLeadersResult { - private final KafkaFuture>> futures; + private final KafkaFutureImpl> electionFuture; - ElectPreferredLeadersResult(KafkaFuture>> futures) { - this.futures = futures; + ElectPreferredLeadersResult(KafkaFutureImpl> electionFuture) { + this.electionFuture = electionFuture; } /** @@ -46,21 +49,26 @@ public class ElectPreferredLeadersResult { * returned future will complete with an error. */ public KafkaFuture partitionResult(final TopicPartition partition) { - class Function>> - extends KafkaFuture.Function> { + final KafkaFutureImpl result = new KafkaFutureImpl<>(); + electionFuture.thenApply(new KafkaFuture.Function, Void>() { @Override - public KafkaFuture apply(T map) { - KafkaFuture result = map.get(partition); - if (result == null) { - KafkaFutureImpl future = new KafkaFutureImpl<>(); - future.completeExceptionally(new IllegalArgumentException( - "Preferred leader election for partition \"" + partition + "\" was not attempted")); - result = future; + public Void apply(Map topicPartitions) { + if (!topicPartitions.containsKey(partition)) { + result.completeExceptionally(new UnknownTopicOrPartitionException( + "Preferred leader election for partition \"" + partition + + "\" was not attempted")); + } else { + ApiException exception = topicPartitions.get(partition).exception(); + if (exception == null) { + result.complete(null); + } else { + result.completeExceptionally(exception); + } } - return result; + return null; } - } - return futures.thenCompose(new Function()); + }); + return result; } /** @@ -73,26 +81,41 @@ public KafkaFuture apply(T map) { * with a null {@code partitions} argument.

*/ public KafkaFuture> partitions() { - class Function>> extends KafkaFuture.Function> { + final KafkaFutureImpl> result = new KafkaFutureImpl<>(); + electionFuture.thenApply(new KafkaFuture.Function, Void>() { @Override - public Set apply(T map) { - return map.keySet(); + public Void apply(Map topicPartitions) { + for (ApiError apiError : topicPartitions.values()) { + if (apiError.isFailure()) { + result.completeExceptionally(apiError.exception()); + return null; + } + } + result.complete(topicPartitions.keySet()); + return null; } - } - return futures.thenApply(new Function()); + }); + return result; } /** * Return a future which succeeds if all the topic elections succeed. */ public KafkaFuture all() { - class Function>> extends KafkaFuture.Function> { - + final KafkaFutureImpl result = new KafkaFutureImpl<>(); + electionFuture.thenApply(new KafkaFuture.Function, Void>() { @Override - public KafkaFuture apply(T map) { - return KafkaFuture.allOf(map.values().toArray(new KafkaFuture[0])); + public Void apply(Map topicPartitions) { + for (ApiError apiError : topicPartitions.values()) { + if (apiError.isFailure()) { + result.completeExceptionally(apiError.exception()); + return null; + } + } + result.complete(null); + return null; } - } - return futures.thenCompose(new Function()); + }); + return result; } } diff --git a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java index 7e470b44c9821..4e41c59b5e2c4 100644 --- a/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java +++ b/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java @@ -1878,71 +1878,27 @@ void handleFailure(Throwable throwable) { @Override public ElectPreferredLeadersResult electPreferredLeaders(final Collection partitions, ElectPreferredLeadersOptions options) { - - final KafkaFutureImpl>> futures = new KafkaFutureImpl<>(); - final Map> mapOfFutures; - final List requestList; - final boolean knownPartitions = partitions != null; - if (knownPartitions) { - mapOfFutures = new HashMap<>(partitions.size()); - for (TopicPartition partition : partitions) { - KafkaFutureImpl future = new KafkaFutureImpl(); - mapOfFutures.put(partition, future); - } - futures.complete(mapOfFutures); - requestList = new ArrayList<>(partitions); - } else { - mapOfFutures = new HashMap<>(); - requestList = null; - } - + final KafkaFutureImpl> electionFuture = new KafkaFutureImpl<>(); final long now = time.milliseconds(); runnable.call(new Call("electPreferredLeaders", calcDeadlineMs(now, options.timeoutMs()), new ControllerNodeProvider()) { @Override public AbstractRequest.Builder createRequest(int timeoutMs) { - return new ElectPreferredLeadersRequest.Builder(requestList, timeoutMs); + return new ElectPreferredLeadersRequest.Builder(partitions, timeoutMs); } @Override public void handleResponse(AbstractResponse abstractResponse) { ElectPreferredLeadersResponse response = (ElectPreferredLeadersResponse) abstractResponse; - // Iterate over the response partitions->errors because the argument partitions can be null - for (Map.Entry entry : response.errors().entrySet()) { - TopicPartition partition = entry.getKey(); - KafkaFutureImpl future; - if (knownPartitions) { - future = mapOfFutures.get(partition); - if (future == null) { - future = new KafkaFutureImpl(); - future.completeExceptionally(new IllegalStateException("Unexpected partition in response")); - } - } else { - future = new KafkaFutureImpl(); - mapOfFutures.put(partition, future); - } - if (entry.getValue().isSuccess()) { - future.complete(null); - } else { - ApiException exception = entry.getValue().exception(); - future.completeExceptionally(exception); - } - } - if (!knownPartitions) { - futures.complete(mapOfFutures); - } + electionFuture.complete(response.errors()); } @Override void handleFailure(Throwable throwable) { - if (knownPartitions) { - completeAllExceptionally(mapOfFutures.values(), throwable); - } else { - futures.completeExceptionally(throwable); - } + electionFuture.completeExceptionally(throwable); } }, now); - return new ElectPreferredLeadersResult(futures); + return new ElectPreferredLeadersResult(electionFuture); } }