From ffe57e52aa9bdfac32623e4b15daf85a64f0b497 Mon Sep 17 00:00:00 2001 From: nizhikov Date: Tue, 14 Feb 2023 13:26:06 +0300 Subject: [PATCH 01/78] KAFKA-14595 Move case classes from PartitionReassignmentState to java --- .../kafka/tools/reassign/ActiveMoveState.java | 81 ++++++++ .../tools/reassign/CancelledMoveState.java | 73 ++++++++ .../tools/reassign/CompletedMoveState.java | 63 +++++++ .../kafka/tools/reassign/LogDirMoveState.java | 28 +++ .../reassign/MissingLogDirMoveState.java | 63 +++++++ .../reassign/MissingReplicaMoveState.java | 63 +++++++ .../kafka/tools/reassign/PartitionMove.java | 68 +++++++ .../reassign/PartitionReassignmentState.java | 78 ++++++++ .../TerseReassignmentFailureException.java | 34 ++++ .../admin/ReassignPartitionsCommand.scala | 176 ++++-------------- .../ReassignPartitionsIntegrationTest.scala | 47 ++--- .../admin/ReassignPartitionsUnitTest.scala | 41 ++-- 12 files changed, 636 insertions(+), 179 deletions(-) create mode 100644 core/src/main/java/org/apache/kafka/tools/reassign/ActiveMoveState.java create mode 100644 core/src/main/java/org/apache/kafka/tools/reassign/CancelledMoveState.java create mode 100644 core/src/main/java/org/apache/kafka/tools/reassign/CompletedMoveState.java create mode 100644 core/src/main/java/org/apache/kafka/tools/reassign/LogDirMoveState.java create mode 100644 core/src/main/java/org/apache/kafka/tools/reassign/MissingLogDirMoveState.java create mode 100644 core/src/main/java/org/apache/kafka/tools/reassign/MissingReplicaMoveState.java create mode 100644 core/src/main/java/org/apache/kafka/tools/reassign/PartitionMove.java create mode 100644 core/src/main/java/org/apache/kafka/tools/reassign/PartitionReassignmentState.java create mode 100644 core/src/main/java/org/apache/kafka/tools/reassign/TerseReassignmentFailureException.java diff --git a/core/src/main/java/org/apache/kafka/tools/reassign/ActiveMoveState.java b/core/src/main/java/org/apache/kafka/tools/reassign/ActiveMoveState.java new file mode 100644 index 0000000000000..271b3f6926b5f --- /dev/null +++ b/core/src/main/java/org/apache/kafka/tools/reassign/ActiveMoveState.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.tools.reassign; + +import java.util.Objects; + +/** + * A replica log directory move state where the move is in progress. + */ +public final class ActiveMoveState implements LogDirMoveState { + private final String currentLogDir; + + private final String targetLogDir; + + private final String futureLogDir; + + /** + * @param currentLogDir The current log directory. + * @param futureLogDir The log directory that the replica is moving to. + * @param targetLogDir The log directory that we wanted the replica to move to. + */ + public ActiveMoveState(String currentLogDir, String targetLogDir, String futureLogDir) { + this.currentLogDir = currentLogDir; + this.targetLogDir = targetLogDir; + this.futureLogDir = futureLogDir; + } + + public String currentLogDir() { + return currentLogDir; + } + + public String targetLogDir() { + return targetLogDir; + } + + public String futureLogDir() { + return futureLogDir; + } + + @Override + public boolean done() { + return false; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ActiveMoveState that = (ActiveMoveState) o; + return Objects.equals(currentLogDir, that.currentLogDir) && Objects.equals(targetLogDir, that.targetLogDir) && Objects.equals(futureLogDir, that.futureLogDir); + } + + @Override + public int hashCode() { + return Objects.hash(currentLogDir, targetLogDir, futureLogDir); + } + + @Override + public String toString() { + return "ActiveMoveState{" + + "currentLogDir='" + currentLogDir + '\'' + + ", targetLogDir='" + targetLogDir + '\'' + + ", futureLogDir='" + futureLogDir + '\'' + + '}'; + } +} diff --git a/core/src/main/java/org/apache/kafka/tools/reassign/CancelledMoveState.java b/core/src/main/java/org/apache/kafka/tools/reassign/CancelledMoveState.java new file mode 100644 index 0000000000000..fec8036d0910c --- /dev/null +++ b/core/src/main/java/org/apache/kafka/tools/reassign/CancelledMoveState.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.tools.reassign; + +import java.util.Objects; + +/** + * A replica log directory move state where there is no move in progress, but we did not + * reach the target log directory. + */ +public final class CancelledMoveState implements LogDirMoveState { + private final String currentLogDir; + + private final String targetLogDir; + + /** + * @param currentLogDir The current log directory. + * @param targetLogDir The log directory that we wanted the replica to move to. + */ + public CancelledMoveState(String currentLogDir, String targetLogDir) { + this.currentLogDir = currentLogDir; + this.targetLogDir = targetLogDir; + } + + public String currentLogDir() { + return currentLogDir; + } + + public String targetLogDir() { + return targetLogDir; + } + + @Override + public boolean done() { + return true; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + CancelledMoveState that = (CancelledMoveState) o; + return Objects.equals(currentLogDir, that.currentLogDir) && Objects.equals(targetLogDir, that.targetLogDir); + } + + @Override + public int hashCode() { + return Objects.hash(currentLogDir, targetLogDir); + } + + @Override + public String toString() { + return "CancelledMoveState{" + + "currentLogDir='" + currentLogDir + '\'' + + ", targetLogDir='" + targetLogDir + '\'' + + '}'; + } +} \ No newline at end of file diff --git a/core/src/main/java/org/apache/kafka/tools/reassign/CompletedMoveState.java b/core/src/main/java/org/apache/kafka/tools/reassign/CompletedMoveState.java new file mode 100644 index 0000000000000..ce936e8c6d7af --- /dev/null +++ b/core/src/main/java/org/apache/kafka/tools/reassign/CompletedMoveState.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.tools.reassign; + +import java.util.Objects; + +/** + * The completed replica log directory move state. + */ +public final class CompletedMoveState implements LogDirMoveState { + private final String targetLogDir; + + /** + * @param targetLogDir The log directory that we wanted the replica to move to. + */ + public CompletedMoveState(String targetLogDir) { + this.targetLogDir = targetLogDir; + } + + public String targetLogDir() { + return targetLogDir; + } + + @Override + public boolean done() { + return true; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + CompletedMoveState that = (CompletedMoveState) o; + return Objects.equals(targetLogDir, that.targetLogDir); + } + + @Override + public int hashCode() { + return Objects.hash(targetLogDir); + } + + @Override + public String toString() { + return "CompletedMoveState{" + + "targetLogDir='" + targetLogDir + '\'' + + '}'; + } +} diff --git a/core/src/main/java/org/apache/kafka/tools/reassign/LogDirMoveState.java b/core/src/main/java/org/apache/kafka/tools/reassign/LogDirMoveState.java new file mode 100644 index 0000000000000..deb2f64e32161 --- /dev/null +++ b/core/src/main/java/org/apache/kafka/tools/reassign/LogDirMoveState.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.tools.reassign; + +/** + * The state of a replica log directory movement. + */ +public interface LogDirMoveState { + /** + * True if the move is done without errors. + */ + boolean done(); +} diff --git a/core/src/main/java/org/apache/kafka/tools/reassign/MissingLogDirMoveState.java b/core/src/main/java/org/apache/kafka/tools/reassign/MissingLogDirMoveState.java new file mode 100644 index 0000000000000..0143a5b035f82 --- /dev/null +++ b/core/src/main/java/org/apache/kafka/tools/reassign/MissingLogDirMoveState.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.tools.reassign; + +import java.util.Objects; + +/** + * A replica log directory move state where the source replica is missing. + */ +public final class MissingLogDirMoveState implements LogDirMoveState { + private final String targetLogDir; + + /** + * @param targetLogDir The log directory that we wanted the replica to move to. + */ + public MissingLogDirMoveState(String targetLogDir) { + this.targetLogDir = targetLogDir; + } + + public String targetLogDir() { + return targetLogDir; + } + + @Override + public boolean done() { + return false; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + MissingLogDirMoveState that = (MissingLogDirMoveState) o; + return Objects.equals(targetLogDir, that.targetLogDir); + } + + @Override + public int hashCode() { + return Objects.hash(targetLogDir); + } + + @Override + public String toString() { + return "MissingLogDirMoveState{" + + "targetLogDir='" + targetLogDir + '\'' + + '}'; + } +} diff --git a/core/src/main/java/org/apache/kafka/tools/reassign/MissingReplicaMoveState.java b/core/src/main/java/org/apache/kafka/tools/reassign/MissingReplicaMoveState.java new file mode 100644 index 0000000000000..41d0356ec4756 --- /dev/null +++ b/core/src/main/java/org/apache/kafka/tools/reassign/MissingReplicaMoveState.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.tools.reassign; + +import java.util.Objects; + +/** + * A replica log directory move state where the source log directory is missing. + */ +public final class MissingReplicaMoveState implements LogDirMoveState { + private final String targetLogDir; + + /** + * @param targetLogDir The log directory that we wanted the replica to move to. + */ + public MissingReplicaMoveState(String targetLogDir) { + this.targetLogDir = targetLogDir; + } + + public String targetLogDir() { + return targetLogDir; + } + + @Override + public boolean done() { + return false; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + MissingReplicaMoveState that = (MissingReplicaMoveState) o; + return Objects.equals(targetLogDir, that.targetLogDir); + } + + @Override + public int hashCode() { + return Objects.hash(targetLogDir); + } + + @Override + public String toString() { + return "MissingReplicaMoveState{" + + "targetLogDir='" + targetLogDir + '\'' + + '}'; + } +} diff --git a/core/src/main/java/org/apache/kafka/tools/reassign/PartitionMove.java b/core/src/main/java/org/apache/kafka/tools/reassign/PartitionMove.java new file mode 100644 index 0000000000000..ffd00056fdbdd --- /dev/null +++ b/core/src/main/java/org/apache/kafka/tools/reassign/PartitionMove.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.tools.reassign; + +import java.util.Objects; +import java.util.Set; + +/** + * A partition movement. The source and destination brokers may overlap. + */ +public final class PartitionMove { + private final Set sources; + + private final Set destinations; + + /** + * @param sources The source brokers. + * @param destinations The destination brokers. + */ + public PartitionMove(Set sources, Set destinations) { + this.sources = sources; + this.destinations = destinations; + } + + public Set sources() { + return sources; + } + + public Set destinations() { + return destinations; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + PartitionMove that = (PartitionMove) o; + return Objects.equals(sources, that.sources) && Objects.equals(destinations, that.destinations); + } + + @Override + public int hashCode() { + return Objects.hash(sources, destinations); + } + + @Override + public String toString() { + return "PartitionMove{" + + "sources=" + sources + + ", destinations=" + destinations + + '}'; + } +} diff --git a/core/src/main/java/org/apache/kafka/tools/reassign/PartitionReassignmentState.java b/core/src/main/java/org/apache/kafka/tools/reassign/PartitionReassignmentState.java new file mode 100644 index 0000000000000..db6a49263cb05 --- /dev/null +++ b/core/src/main/java/org/apache/kafka/tools/reassign/PartitionReassignmentState.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.tools.reassign; + +import java.util.List; +import java.util.Objects; + +/** + * The state of a partition reassignment. The current replicas and target replicas + * may overlap. + */ +public final class PartitionReassignmentState { + private final List currentReplicas; + + private final List targetReplicas; + + private final boolean done; + + /** + * @param currentReplicas The current replicas. + * @param targetReplicas The target replicas. + * @param done True if the reassignment is done. + */ + public PartitionReassignmentState(List currentReplicas, List targetReplicas, boolean done) { + this.currentReplicas = currentReplicas; + this.targetReplicas = targetReplicas; + this.done = done; + } + + public List currentReplicas() { + return currentReplicas; + } + + public List targetReplicas() { + return targetReplicas; + } + + public boolean done() { + return done; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + PartitionReassignmentState that = (PartitionReassignmentState) o; + return done == that.done && Objects.equals(currentReplicas, that.currentReplicas) && Objects.equals(targetReplicas, that.targetReplicas); + } + + @Override + public int hashCode() { + return Objects.hash(currentReplicas, targetReplicas, done); + } + + @Override + public String toString() { + return "PartitionReassignmentState{" + + "currentReplicas=" + currentReplicas + + ", targetReplicas=" + targetReplicas + + ", done=" + done + + '}'; + } +} diff --git a/core/src/main/java/org/apache/kafka/tools/reassign/TerseReassignmentFailureException.java b/core/src/main/java/org/apache/kafka/tools/reassign/TerseReassignmentFailureException.java new file mode 100644 index 0000000000000..524e25ff53cf8 --- /dev/null +++ b/core/src/main/java/org/apache/kafka/tools/reassign/TerseReassignmentFailureException.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.tools.reassign; + +import org.apache.kafka.common.KafkaException; + +/** + * An exception thrown to indicate that the command has failed, but we don't want to + * print a stack trace. + */ +public class TerseReassignmentFailureException extends KafkaException { + /** + * @param message The message to print out before exiting. A stack trace will not + * be printed. + */ + public TerseReassignmentFailureException(String message) { + super(message); + } +} diff --git a/core/src/main/scala/kafka/admin/ReassignPartitionsCommand.scala b/core/src/main/scala/kafka/admin/ReassignPartitionsCommand.scala index d1b75c483eba8..e8096f0e30a52 100755 --- a/core/src/main/scala/kafka/admin/ReassignPartitionsCommand.scala +++ b/core/src/main/scala/kafka/admin/ReassignPartitionsCommand.scala @@ -29,9 +29,10 @@ import org.apache.kafka.clients.admin.{Admin, AdminClientConfig, AlterConfigOp, import org.apache.kafka.common.config.ConfigResource import org.apache.kafka.common.errors.{ReplicaNotAvailableException, UnknownTopicOrPartitionException} import org.apache.kafka.common.utils.{Time, Utils} -import org.apache.kafka.common.{KafkaException, KafkaFuture, TopicPartition, TopicPartitionReplica} +import org.apache.kafka.common.{KafkaFuture, TopicPartition, TopicPartitionReplica} import org.apache.kafka.server.util.{CommandDefaultOptions, CommandLineUtils} import org.apache.kafka.storage.internals.log.LogConfig +import org.apache.kafka.tools.reassign.{ActiveMoveState, CancelledMoveState, CompletedMoveState, LogDirMoveState, MissingLogDirMoveState, MissingReplicaMoveState, PartitionMove, PartitionReassignmentState, TerseReassignmentFailureException} import scala.jdk.CollectionConverters._ import scala.collection.{Map, Seq, mutable} @@ -92,104 +93,6 @@ object ReassignPartitionsCommand extends Logging { */ type MoveMap = mutable.Map[String, mutable.Map[Int, PartitionMove]] - /** - * A partition movement. The source and destination brokers may overlap. - * - * @param sources The source brokers. - * @param destinations The destination brokers. - */ - sealed case class PartitionMove(sources: mutable.Set[Int], - destinations: mutable.Set[Int]) { } - - /** - * The state of a partition reassignment. The current replicas and target replicas - * may overlap. - * - * @param currentReplicas The current replicas. - * @param targetReplicas The target replicas. - * @param done True if the reassignment is done. - */ - sealed case class PartitionReassignmentState(currentReplicas: Seq[Int], - targetReplicas: Seq[Int], - done: Boolean) {} - - /** - * The state of a replica log directory movement. - */ - sealed trait LogDirMoveState { - /** - * True if the move is done without errors. - */ - def done: Boolean - } - - /** - * A replica log directory move state where the source log directory is missing. - * - * @param targetLogDir The log directory that we wanted the replica to move to. - */ - sealed case class MissingReplicaMoveState(targetLogDir: String) - extends LogDirMoveState { - override def done = false - } - - /** - * A replica log directory move state where the source replica is missing. - * - * @param targetLogDir The log directory that we wanted the replica to move to. - */ - sealed case class MissingLogDirMoveState(targetLogDir: String) - extends LogDirMoveState { - override def done = false - } - - /** - * A replica log directory move state where the move is in progress. - * - * @param currentLogDir The current log directory. - * @param futureLogDir The log directory that the replica is moving to. - * @param targetLogDir The log directory that we wanted the replica to move to. - */ - sealed case class ActiveMoveState(currentLogDir: String, - targetLogDir: String, - futureLogDir: String) - extends LogDirMoveState { - override def done = false - } - - /** - * A replica log directory move state where there is no move in progress, but we did not - * reach the target log directory. - * - * @param currentLogDir The current log directory. - * @param targetLogDir The log directory that we wanted the replica to move to. - */ - sealed case class CancelledMoveState(currentLogDir: String, - targetLogDir: String) - extends LogDirMoveState { - override def done = true - } - - /** - * The completed replica log directory move state. - * - * @param targetLogDir The log directory that we wanted the replica to move to. - */ - sealed case class CompletedMoveState(targetLogDir: String) - extends LogDirMoveState { - override def done = true - } - - /** - * An exception thrown to indicate that the command has failed, but we don't want to - * print a stack trace. - * - * @param message The message to print out before exiting. A stack trace will not - * be printed. - */ - class TerseReassignmentFailureException(message: String) extends KafkaException(message) { - } - def main(args: Array[String]): Unit = { val opts = validateAndParseArgs(args) var failed = true @@ -329,14 +232,14 @@ object ReassignPartitionsCommand extends Logging { bld.append("Status of partition reassignment:") states.keySet.toBuffer.sortWith(compareTopicPartitions).foreach { topicPartition => val state = states(topicPartition) - if (state.done) { - if (state.currentReplicas.equals(state.targetReplicas)) { + if (state.done()) { + if (state.currentReplicas().equals(state.targetReplicas)) { bld.append("Reassignment of partition %s is completed.". format(topicPartition.toString)) } else { bld.append(s"There is no active reassignment of partition ${topicPartition}, " + - s"but replica set is ${state.currentReplicas.mkString(",")} rather than " + - s"${state.targetReplicas.mkString(",")}.") + s"but replica set is ${state.currentReplicas()} rather than " + + s"${state.targetReplicas}.") } } else { bld.append("Reassignment of partition %s is still in progress.".format(topicPartition)) @@ -364,10 +267,9 @@ object ReassignPartitionsCommand extends Logging { } val foundResults = foundReassignments.map { case (part, targetReplicas) => (part, - PartitionReassignmentState( - currentReassignments(part).replicas. - asScala.map(i => i.asInstanceOf[Int]), - targetReplicas, + new PartitionReassignmentState( + currentReassignments(part).replicas, + targetReplicas.map(i => i.asInstanceOf[Integer]).asJava, false)) } val topicNamesToLookUp = new mutable.HashSet[String]() @@ -381,9 +283,9 @@ object ReassignPartitionsCommand extends Logging { case (part, targetReplicas) => currentReassignments.get(part) match { case Some(reassignment) => (part, - PartitionReassignmentState( - reassignment.replicas.asScala.map(_.asInstanceOf[Int]), - targetReplicas, + new PartitionReassignmentState( + reassignment.replicas, + targetReplicas.map(i => i.asInstanceOf[Integer]).asJava, false)) case None => (part, topicDescriptionFutureToState(part.partition, @@ -402,13 +304,13 @@ object ReassignPartitionsCommand extends Logging { if (topicDescription.partitions().size() < partition) { throw new ExecutionException("Too few partitions found", new UnknownTopicOrPartitionException()) } - PartitionReassignmentState( - topicDescription.partitions.get(partition).replicas.asScala.map(_.id), - targetReplicas, + new PartitionReassignmentState( + topicDescription.partitions.get(partition).replicas.asScala.map(_.id.asInstanceOf[Integer]).asJava, + targetReplicas.map(i => i.asInstanceOf[Integer]).asJava, true) } catch { case t: ExecutionException if t.getCause.isInstanceOf[UnknownTopicOrPartitionException] => - PartitionReassignmentState(Seq(), targetReplicas, true) + new PartitionReassignmentState(java.util.Collections.emptyList(), targetReplicas.map(i => i.asInstanceOf[Integer]).asJava, true) } } @@ -451,17 +353,17 @@ object ReassignPartitionsCommand extends Logging { targetMoves.keySet.asJava).all().get().asScala targetMoves.map { case (replica, targetLogDir) => val moveState = replicaLogDirInfos.get(replica) match { - case None => MissingReplicaMoveState(targetLogDir) + case None => new MissingReplicaMoveState(targetLogDir) case Some(info) => if (info.getCurrentReplicaLogDir == null) { - MissingLogDirMoveState(targetLogDir) + new MissingLogDirMoveState(targetLogDir) } else if (info.getFutureReplicaLogDir == null) { if (info.getCurrentReplicaLogDir.equals(targetLogDir)) { - CompletedMoveState(targetLogDir) + new CompletedMoveState(targetLogDir) } else { - CancelledMoveState(info.getCurrentReplicaLogDir, targetLogDir) + new CancelledMoveState(info.getCurrentReplicaLogDir, targetLogDir) } } else { - ActiveMoveState(info.getCurrentReplicaLogDir(), + new ActiveMoveState(info.getCurrentReplicaLogDir(), targetLogDir, info.getFutureReplicaLogDir) } @@ -483,27 +385,29 @@ object ReassignPartitionsCommand extends Logging { states.keySet.toBuffer.sortWith(compareTopicPartitionReplicas).foreach { replica => val state = states(replica) state match { - case MissingLogDirMoveState(_) => + case _: MissingLogDirMoveState => bld.append(s"Partition ${replica.topic}-${replica.partition} is not found " + s"in any live log dir on broker ${replica.brokerId}. There is likely an " + s"offline log directory on the broker.") - case MissingReplicaMoveState(_) => + case _: MissingReplicaMoveState => bld.append(s"Partition ${replica.topic}-${replica.partition} cannot be found " + s"in any live log directory on broker ${replica.brokerId}.") - case ActiveMoveState(_, targetLogDir, futureLogDir) => - if (targetLogDir.equals(futureLogDir)) { + case s: ActiveMoveState => + if (s.targetLogDir.equals(s.futureLogDir)) { bld.append(s"Reassignment of replica $replica is still in progress.") } else { bld.append(s"Partition ${replica.topic}-${replica.partition} on broker " + - s"${replica.brokerId} is being moved to log dir $futureLogDir " + - s"instead of $targetLogDir.") + s"${replica.brokerId} is being moved to log dir ${s.futureLogDir} " + + s"instead of ${s.targetLogDir}.") } - case CancelledMoveState(currentLogDir, targetLogDir) => + case s: CancelledMoveState => bld.append(s"Partition ${replica.topic}-${replica.partition} on broker " + - s"${replica.brokerId} is not being moved from log dir $currentLogDir to " + - s"$targetLogDir.") - case CompletedMoveState(_) => + s"${replica.brokerId} is not being moved from log dir ${s.currentLogDir} to " + + s"${s.targetLogDir}.") + case _: CompletedMoveState => bld.append(s"Reassignment of replica $replica completed successfully.") + case _ => + throw new IllegalStateException(s"Unknown state $state") } } bld.mkString(System.lineSeparator()) @@ -962,7 +866,7 @@ object ReassignPartitionsCommand extends Logging { val destinations = mutable.Set[Int]() ++ addingReplicas val partMoves = moveMap.getOrElseUpdate(part.topic, new mutable.HashMap[Int, PartitionMove]) - partMoves.put(part.partition, PartitionMove(sources, destinations)) + partMoves.put(part.partition, new PartitionMove(sources.map(i => i.asInstanceOf[Integer]).asJava, destinations.map(i => i.asInstanceOf[Integer]).asJava)) } moveMap } @@ -989,14 +893,14 @@ object ReassignPartitionsCommand extends Logging { // If there is a reassignment in progress, use the sources from moveMap, otherwise // use the sources from currentParts val sources = mutable.Set[Int]() ++ (partMoves.get(part.partition) match { - case Some(move) => move.sources.toSeq + case Some(move) => move.sources.asScala.toSeq case None => currentParts.getOrElse(part, throw new RuntimeException(s"Trying to reassign a topic partition $part with 0 replicas")) }) val destinations = mutable.Set[Int]() ++ replicas.diff(sources.toSeq) partMoves.put(part.partition, - PartitionMove(sources, destinations)) + new PartitionMove(sources.map(i => i.asInstanceOf[Integer]).asJava, destinations.map(i => i.asInstanceOf[Integer]).asJava)) } moveMap } @@ -1012,7 +916,7 @@ object ReassignPartitionsCommand extends Logging { case (topicName, partMoveMap) => { val components = new mutable.TreeSet[String] partMoveMap.forKeyValue { (partId, move) => - move.sources.foreach(source => components.add("%d:%d".format(partId, source))) + move.sources.forEach(source => components.add("%d:%d".format(partId, source))) } (topicName, components.mkString(",")) } @@ -1030,7 +934,7 @@ object ReassignPartitionsCommand extends Logging { case (topicName, partMoveMap) => { val components = new mutable.TreeSet[String] partMoveMap.forKeyValue { (partId, move) => - move.destinations.foreach(destination => + move.destinations.forEach(destination => if (!move.sources.contains(destination)) { components.add("%d:%d".format(partId, destination)) }) @@ -1051,8 +955,8 @@ object ReassignPartitionsCommand extends Logging { moveMap.values.foreach { _.values.foreach { partMove => - partMove.sources.foreach(reassigningBrokers.add) - partMove.destinations.foreach(reassigningBrokers.add) + partMove.sources.forEach(i => reassigningBrokers.add(i)) + partMove.destinations.forEach(i => reassigningBrokers.add(i)) } } reassigningBrokers.toSet diff --git a/core/src/test/scala/integration/kafka/admin/ReassignPartitionsIntegrationTest.scala b/core/src/test/scala/integration/kafka/admin/ReassignPartitionsIntegrationTest.scala index d3a04da4d1344..40341908589d1 100644 --- a/core/src/test/scala/integration/kafka/admin/ReassignPartitionsIntegrationTest.scala +++ b/core/src/test/scala/integration/kafka/admin/ReassignPartitionsIntegrationTest.scala @@ -27,13 +27,14 @@ import org.apache.kafka.common.config.ConfigResource import org.apache.kafka.common.utils.Utils import org.apache.kafka.common.{TopicPartition, TopicPartitionReplica} import org.apache.kafka.server.common.MetadataVersion.IBP_2_7_IV1 +import org.apache.kafka.tools.reassign.{ActiveMoveState, CompletedMoveState, PartitionReassignmentState} import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertTrue} import org.junit.jupiter.api.{AfterEach, Timeout} import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.ValueSource import java.io.Closeable -import java.util.{Collections, HashMap, List} +import java.util.{Arrays, Collections, HashMap, List} import scala.collection.{Map, Seq, mutable} import scala.jdk.CollectionConverters._ @@ -113,8 +114,8 @@ class ReassignPartitionsIntegrationTest extends QuorumTestHarness { // Check that the assignment has not yet been started yet. val initialAssignment = Map( - foo0 -> PartitionReassignmentState(Seq(0, 1, 2), Seq(0, 1, 3), true), - bar0 -> PartitionReassignmentState(Seq(3, 2, 1), Seq(3, 2, 0), true) + foo0 -> new PartitionReassignmentState(Arrays.asList(0, 1, 2), Arrays.asList(0, 1, 3), true), + bar0 -> new PartitionReassignmentState(Arrays.asList(3, 2, 1), Arrays.asList(3, 2, 0), true) ) waitForVerifyAssignment(cluster.adminClient, assignment, false, VerifyAssignmentResult(initialAssignment)) @@ -123,8 +124,8 @@ class ReassignPartitionsIntegrationTest extends QuorumTestHarness { runExecuteAssignment(cluster.adminClient, false, assignment, -1L, -1L) assertEquals(unthrottledBrokerConfigs, describeBrokerLevelThrottles(unthrottledBrokerConfigs.keySet.toSeq)) val finalAssignment = Map( - foo0 -> PartitionReassignmentState(Seq(0, 1, 3), Seq(0, 1, 3), true), - bar0 -> PartitionReassignmentState(Seq(3, 2, 0), Seq(3, 2, 0), true) + foo0 -> new PartitionReassignmentState(Arrays.asList(0, 1, 3), Arrays.asList(0, 1, 3), true), + bar0 -> new PartitionReassignmentState(Arrays.asList(3, 2, 0), Arrays.asList(3, 2, 0), true) ) val verifyAssignmentResult = runVerifyAssignment(cluster.adminClient, assignment, false) @@ -158,7 +159,7 @@ class ReassignPartitionsIntegrationTest extends QuorumTestHarness { // Execute the assignment runExecuteAssignment(cluster.adminClient, false, assignment, -1L, -1L) val finalAssignment = Map(part -> - PartitionReassignmentState(Seq(3, 1, 2), Seq(3, 1, 2), true)) + new PartitionReassignmentState(Arrays.asList(3, 1, 2), Arrays.asList(3, 1, 2), true)) // Wait for the assignment to complete waitForVerifyAssignment(cluster.adminClient, assignment, false, @@ -196,9 +197,9 @@ class ReassignPartitionsIntegrationTest extends QuorumTestHarness { val finalAssignment = Map( new TopicPartition("foo", 0) -> - PartitionReassignmentState(Seq(0, 3, 2), Seq(0, 3, 2), true), + new PartitionReassignmentState(Arrays.asList(0, 3, 2), Arrays.asList(0, 3, 2), true), new TopicPartition("baz", 2) -> - PartitionReassignmentState(Seq(3, 2, 1), Seq(3, 2, 1), true)) + new PartitionReassignmentState(Arrays.asList(3, 2, 1), Arrays.asList(3, 2, 1), true)) // Now remove the throttles. waitForVerifyAssignment(cluster.adminClient, assignment, false, @@ -224,9 +225,9 @@ class ReassignPartitionsIntegrationTest extends QuorumTestHarness { // Check that the assignment has not yet been started yet. val initialAssignment = Map( new TopicPartition("foo", 0) -> - PartitionReassignmentState(Seq(0, 1, 2), Seq(0, 3, 2), true), + new PartitionReassignmentState(Arrays.asList(0, 1, 2), Arrays.asList(0, 3, 2), true), new TopicPartition("baz", 2) -> - PartitionReassignmentState(Seq(0, 2, 1), Seq(3, 2, 1), true)) + new PartitionReassignmentState(Arrays.asList(0, 2, 1), Arrays.asList(3, 2, 1), true)) assertEquals(VerifyAssignmentResult(initialAssignment), runVerifyAssignment(cluster.adminClient, assignment, false)) assertEquals(unthrottledBrokerConfigs, describeBrokerLevelThrottles(unthrottledBrokerConfigs.keySet.toSeq)) @@ -237,9 +238,9 @@ class ReassignPartitionsIntegrationTest extends QuorumTestHarness { val finalAssignment = Map( new TopicPartition("foo", 0) -> - PartitionReassignmentState(Seq(0, 3, 2), Seq(0, 3, 2), true), + new PartitionReassignmentState(Arrays.asList(0, 3, 2), Arrays.asList(0, 3, 2), true), new TopicPartition("baz", 2) -> - PartitionReassignmentState(Seq(3, 2, 1), Seq(3, 2, 1), true)) + new PartitionReassignmentState(Arrays.asList(3, 2, 1), Arrays.asList(3, 2, 1), true)) // Wait for the assignment to complete TestUtils.waitUntilTrue( @@ -250,8 +251,8 @@ class ReassignPartitionsIntegrationTest extends QuorumTestHarness { true } else { assertFalse(result.partStates.forall(_._2.done), s"Expected at least one partition reassignment to be ongoing when result = $result") - assertEquals(Seq(0, 3, 2), result.partStates(new TopicPartition("foo", 0)).targetReplicas) - assertEquals(Seq(3, 2, 1), result.partStates(new TopicPartition("baz", 2)).targetReplicas) + assertEquals(Arrays.asList(0, 3, 2), result.partStates(new TopicPartition("foo", 0)).targetReplicas) + assertEquals(Arrays.asList(3, 2, 1), result.partStates(new TopicPartition("baz", 2)).targetReplicas) logger.info(s"Current result: ${result}") waitForInterBrokerThrottle(Set(0, 1, 2, 3), interBrokerThrottle) false @@ -288,7 +289,7 @@ class ReassignPartitionsIntegrationTest extends QuorumTestHarness { } TestUtils.removeReplicationThrottleForPartitions(cluster.adminClient, Seq(0,1,2,3), Set(part)) val finalAssignment = Map(part -> - PartitionReassignmentState(Seq(3, 2, 1), Seq(3, 2, 1), true)) + new PartitionReassignmentState(Arrays.asList(3, 2, 1), Arrays.asList(3, 2, 1), true)) waitForVerifyAssignment(cluster.adminClient, assignment, false, VerifyAssignmentResult(finalAssignment)) } @@ -320,8 +321,8 @@ class ReassignPartitionsIntegrationTest extends QuorumTestHarness { // from completing before this runs. waitForVerifyAssignment(cluster.adminClient, assignment, true, VerifyAssignmentResult(Map( - foo0 -> PartitionReassignmentState(Seq(0, 1, 3, 2), Seq(0, 1, 3), false), - baz1 -> PartitionReassignmentState(Seq(0, 2, 3, 1), Seq(0, 2, 3), false)), + foo0 -> new PartitionReassignmentState(Arrays.asList(0, 1, 3, 2), Arrays.asList(0, 1, 3), false), + baz1 -> new PartitionReassignmentState(Arrays.asList(0, 2, 3, 1), Arrays.asList(0, 2, 3), false)), true, Map(), false)) // Cancel the reassignment. assertEquals((Set(foo0, baz1), Set()), runCancelAssignment(cluster.adminClient, assignment, true)) @@ -476,10 +477,10 @@ class ReassignPartitionsIntegrationTest extends QuorumTestHarness { // Check the output of --verify waitForVerifyAssignment(cluster.adminClient, reassignment.json, true, VerifyAssignmentResult(Map( - topicPartition -> PartitionReassignmentState(Seq(0, 1, 2), Seq(0, 1, 2), true) + topicPartition -> new PartitionReassignmentState(Arrays.asList(0, 1, 2), Arrays.asList(0, 1, 2), true) ), false, Map( new TopicPartitionReplica(topicPartition.topic, topicPartition.partition, 0) -> - ActiveMoveState(reassignment.currentDir, reassignment.targetDir, reassignment.targetDir) + new ActiveMoveState(reassignment.currentDir, reassignment.targetDir, reassignment.targetDir) ), true)) waitForLogDirThrottle(Set(0), logDirThrottle) @@ -494,10 +495,10 @@ class ReassignPartitionsIntegrationTest extends QuorumTestHarness { // Wait for the directory movement to complete. waitForVerifyAssignment(cluster.adminClient, reassignment.json, true, VerifyAssignmentResult(Map( - topicPartition -> PartitionReassignmentState(Seq(0, 1, 2), Seq(0, 1, 2), true) + topicPartition -> new PartitionReassignmentState(Arrays.asList(0, 1, 2), Arrays.asList(0, 1, 2), true) ), false, Map( new TopicPartitionReplica(topicPartition.topic, topicPartition.partition, 0) -> - CompletedMoveState(reassignment.targetDir) + new CompletedMoveState(reassignment.targetDir) ), false)) val info1 = new BrokerDirs(cluster.adminClient.describeLogDirs(0.to(4). @@ -532,10 +533,10 @@ class ReassignPartitionsIntegrationTest extends QuorumTestHarness { waitForVerifyAssignment(cluster.adminClient, reassignment.json, true, VerifyAssignmentResult(Map( - topicPartition -> PartitionReassignmentState(Seq(0, 1, 2), Seq(0, 1, 2), true) + topicPartition -> new PartitionReassignmentState(Arrays.asList(0, 1, 2), Arrays.asList(0, 1, 2), true) ), false, Map( new TopicPartitionReplica(topicPartition.topic, topicPartition.partition, targetBrokerId) -> - CompletedMoveState(reassignment.targetDir) + new CompletedMoveState(reassignment.targetDir) ), false)) } diff --git a/core/src/test/scala/unit/kafka/admin/ReassignPartitionsUnitTest.scala b/core/src/test/scala/unit/kafka/admin/ReassignPartitionsUnitTest.scala index 3b7c1ab719168..0d74d7790eed6 100644 --- a/core/src/test/scala/unit/kafka/admin/ReassignPartitionsUnitTest.scala +++ b/core/src/test/scala/unit/kafka/admin/ReassignPartitionsUnitTest.scala @@ -26,6 +26,7 @@ import org.apache.kafka.clients.admin.{Config, MockAdminClient, PartitionReassig import org.apache.kafka.common.config.ConfigResource import org.apache.kafka.common.errors.{InvalidReplicationFactorException, UnknownTopicOrPartitionException} import org.apache.kafka.common.{Node, TopicPartition, TopicPartitionInfo, TopicPartitionReplica} +import org.apache.kafka.tools.reassign.{ActiveMoveState, CancelledMoveState, CompletedMoveState, MissingLogDirMoveState, MissingReplicaMoveState, PartitionMove, PartitionReassignmentState} import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertThrows, assertTrue} import org.junit.jupiter.api.{AfterEach, BeforeEach, Test, Timeout} @@ -71,11 +72,11 @@ class ReassignPartitionsUnitTest { mkString(System.lineSeparator()), partitionReassignmentStatesToString(Map( new TopicPartition("foo", 0) -> - PartitionReassignmentState(Seq(1, 2, 3), Seq(1, 2, 3), true), + new PartitionReassignmentState(Arrays.asList(1, 2, 3), Arrays.asList(1, 2, 3), true), new TopicPartition("foo", 1) -> - PartitionReassignmentState(Seq(1, 2, 3), Seq(1, 2, 4), false), + new PartitionReassignmentState(Arrays.asList(1, 2, 3), Arrays.asList(1, 2, 4), false), new TopicPartition("bar", 0) -> - PartitionReassignmentState(Seq(1, 2, 3), Seq(1, 2, 4), false), + new PartitionReassignmentState(Arrays.asList(1, 2, 3), Arrays.asList(1, 2, 4), false), ))) } @@ -108,8 +109,8 @@ class ReassignPartitionsUnitTest { assertEquals(Map(new TopicPartition("quux", 0) -> classOf[UnknownTopicOrPartitionException]), reassignmentResult) assertEquals((Map( - new TopicPartition("foo", 0) -> PartitionReassignmentState(Seq(0,1,2), Seq(0,1,3), false), - new TopicPartition("foo", 1) -> PartitionReassignmentState(Seq(1,2,3), Seq(1,2,3), true) + new TopicPartition("foo", 0) -> new PartitionReassignmentState(Arrays.asList(0,1,2), Arrays.asList(0,1,3), false), + new TopicPartition("foo", 1) -> new PartitionReassignmentState(Arrays.asList(1,2,3), Arrays.asList(1,2,3), true) ), true), findPartitionReassignmentStates(adminClient, Seq( (new TopicPartition("foo", 0), Seq(0,1,3)), @@ -123,8 +124,8 @@ class ReassignPartitionsUnitTest { assertEquals(Map(new TopicPartition("quux", 2) -> classOf[UnknownTopicOrPartitionException]), cancelResult) assertEquals((Map( - new TopicPartition("foo", 0) -> PartitionReassignmentState(Seq(0,1,2), Seq(0,1,3), true), - new TopicPartition("foo", 1) -> PartitionReassignmentState(Seq(1,2,3), Seq(1,2,3), true) + new TopicPartition("foo", 0) -> new PartitionReassignmentState(Arrays.asList(0,1,2), Arrays.asList(0,1,3), true), + new TopicPartition("foo", 1) -> new PartitionReassignmentState(Arrays.asList(1,2,3), Arrays.asList(1,2,3), true) ), false), findPartitionReassignmentStates(adminClient, Seq( (new TopicPartition("foo", 0), Seq(0,1,3)), @@ -188,15 +189,15 @@ class ReassignPartitionsUnitTest { "Partition quux-2 is not found in any live log dir on broker 1. " + "There is likely an offline log directory on the broker.").mkString(System.lineSeparator()), replicaMoveStatesToString(Map( - new TopicPartitionReplica("bar", 0, 0) -> CompletedMoveState("/tmp/kafka-logs0"), - new TopicPartitionReplica("foo", 0, 0) -> ActiveMoveState("/tmp/kafka-logs0", + new TopicPartitionReplica("bar", 0, 0) -> new CompletedMoveState("/tmp/kafka-logs0"), + new TopicPartitionReplica("foo", 0, 0) -> new ActiveMoveState("/tmp/kafka-logs0", "/tmp/kafka-logs1", "/tmp/kafka-logs1"), - new TopicPartitionReplica("foo", 1, 0) -> CancelledMoveState("/tmp/kafka-logs0", + new TopicPartitionReplica("foo", 1, 0) -> new CancelledMoveState("/tmp/kafka-logs0", "/tmp/kafka-logs1"), - new TopicPartitionReplica("quux", 0, 0) -> MissingReplicaMoveState("/tmp/kafka-logs1"), - new TopicPartitionReplica("quux", 1, 1) -> ActiveMoveState("/tmp/kafka-logs0", + new TopicPartitionReplica("quux", 0, 0) -> new MissingReplicaMoveState("/tmp/kafka-logs1"), + new TopicPartitionReplica("quux", 1, 1) -> new ActiveMoveState("/tmp/kafka-logs0", "/tmp/kafka-logs1", "/tmp/kafka-logs2"), - new TopicPartitionReplica("quux", 2, 1) -> MissingLogDirMoveState("/tmp/kafka-logs1") + new TopicPartitionReplica("quux", 2, 1) -> new MissingLogDirMoveState("/tmp/kafka-logs1") ))) } @@ -419,14 +420,14 @@ class ReassignPartitionsUnitTest { assertEquals( mutable.Map("foo" -> mutable.Map( - 0 -> PartitionMove(mutable.Set(1,2,3), mutable.Set(5)), - 1 -> PartitionMove(mutable.Set(4,5,6), mutable.Set(7,8)), - 2 -> PartitionMove(mutable.Set(1,2), mutable.Set(3,4)), - 3 -> PartitionMove(mutable.Set(1,2), mutable.Set(5,6)), - 4 -> PartitionMove(mutable.Set(1,2), mutable.Set(3)), - 5 -> PartitionMove(mutable.Set(1,2), mutable.Set(3,4,5,6)) + 0 -> new PartitionMove(Set[Integer](1,2,3).asJava, Set[Integer](5).asJava), + 1 -> new PartitionMove(Set[Integer](4,5,6).asJava, Set[Integer](7,8).asJava), + 2 -> new PartitionMove(Set[Integer](1,2).asJava, Set[Integer](3,4).asJava), + 3 -> new PartitionMove(Set[Integer](1,2).asJava, Set[Integer](5,6).asJava), + 4 -> new PartitionMove(Set[Integer](1,2).asJava, Set[Integer](3).asJava), + 5 -> new PartitionMove(Set[Integer](1,2).asJava, Set[Integer](3,4,5,6).asJava) ), "bar" -> mutable.Map( - 0 -> PartitionMove(mutable.Set(2,3,4), mutable.Set(1)), + 0 -> new PartitionMove(Set[Integer](2,3,4).asJava, Set[Integer](1).asJava), )), moveMap) assertEquals(Map( From e87507518fd0eeb9d267e9665743927ffb75b04e Mon Sep 17 00:00:00 2001 From: nizhikov Date: Wed, 15 Feb 2023 11:05:06 +0300 Subject: [PATCH 02/78] KAFKA-14595 Move ReassignPartitionsCommandOptions to java --- .../ReassignPartitionsCommandOptions.java | 160 ++++++++++++++++++ .../admin/ReassignPartitionsCommand.scala | 71 +------- 2 files changed, 162 insertions(+), 69 deletions(-) create mode 100644 core/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommandOptions.java diff --git a/core/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommandOptions.java b/core/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommandOptions.java new file mode 100644 index 0000000000000..acf768d8488cd --- /dev/null +++ b/core/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommandOptions.java @@ -0,0 +1,160 @@ +package org.apache.kafka.tools.reassign; + +import joptsimple.ArgumentAcceptingOptionSpec; +import joptsimple.OptionSpecBuilder; +import org.apache.kafka.server.util.CommandDefaultOptions; + +public class ReassignPartitionsCommandOptions extends CommandDefaultOptions { + // Actions + private final OptionSpecBuilder verifyOpt; + private final OptionSpecBuilder generateOpt; + private final OptionSpecBuilder executeOpt; + private final OptionSpecBuilder cancelOpt; + private final OptionSpecBuilder listOpt; + + // Arguments + private final ArgumentAcceptingOptionSpec bootstrapServerOpt; + private final ArgumentAcceptingOptionSpec commandConfigOpt; + private final ArgumentAcceptingOptionSpec reassignmentJsonFileOpt; + private final ArgumentAcceptingOptionSpec topicsToMoveJsonFileOpt; + private final ArgumentAcceptingOptionSpec brokerListOpt; + private final OptionSpecBuilder disableRackAware; + private final ArgumentAcceptingOptionSpec interBrokerThrottleOpt; + private final ArgumentAcceptingOptionSpec replicaAlterLogDirsThrottleOpt; + private final ArgumentAcceptingOptionSpec timeoutOpt; + private final OptionSpecBuilder additionalOpt; + private final OptionSpecBuilder preserveThrottlesOpt; + + public ReassignPartitionsCommandOptions(String[] args) { + super(args); + + verifyOpt = parser.accepts("verify", "Verify if the reassignment completed as specified by the " + + "--reassignment-json-file option. If there is a throttle engaged for the replicas specified, and the rebalance has completed, the throttle will be removed"); + generateOpt = parser.accepts("generate", "Generate a candidate partition reassignment configuration." + + " Note that this only generates a candidate assignment, it does not execute it."); + executeOpt = parser.accepts("execute", "Kick off the reassignment as specified by the --reassignment-json-file option."); + cancelOpt = parser.accepts("cancel", "Cancel an active reassignment."); + listOpt = parser.accepts("list", "List all active partition reassignments."); + + // Arguments + bootstrapServerOpt = parser.accepts("bootstrap-server", "REQUIRED: the server(s) to use for bootstrapping.") + .withRequiredArg() + .describedAs("Server(s) to use for bootstrapping") + .ofType(String.class); + + commandConfigOpt = parser.accepts("command-config", "Property file containing configs to be passed to Admin Client.") + .withRequiredArg() + .describedAs("Admin client property file") + .ofType(String.class); + + reassignmentJsonFileOpt = parser.accepts("reassignment-json-file", "The JSON file with the partition reassignment configuration" + + "The format to use is - \n" + + "{\"partitions\":\n\t[{\"topic\": \"foo\",\n\t \"partition\": 1,\n\t \"replicas\": [1,2,3],\n\t \"log_dirs\": [\"dir1\",\"dir2\",\"dir3\"] }],\n\"version\":1\n}\n" + + "Note that \"log_dirs\" is optional. When it is specified, its length must equal the length of the replicas list. The value in this list " + + "can be either \"any\" or the absolution path of the log directory on the broker. If absolute log directory path is specified, the replica will be moved to the specified log directory on the broker.") + .withRequiredArg() + .describedAs("manual assignment json file path") + .ofType(String.class); + topicsToMoveJsonFileOpt = parser.accepts("topics-to-move-json-file", "Generate a reassignment configuration to move the partitions" + + " of the specified topics to the list of brokers specified by the --broker-list option. The format to use is - \n" + + "{\"topics\":\n\t[{\"topic\": \"foo\"},{\"topic\": \"foo1\"}],\n\"version\":1\n}") + .withRequiredArg() + .describedAs("topics to reassign json file path") + .ofType(String.class); + brokerListOpt = parser.accepts("broker-list", "The list of brokers to which the partitions need to be reassigned" + + " in the form \"0,1,2\". This is required if --topics-to-move-json-file is used to generate reassignment configuration") + .withRequiredArg() + .describedAs("brokerlist") + .ofType(String.class); + disableRackAware = parser.accepts("disable-rack-aware", "Disable rack aware replica assignment"); + interBrokerThrottleOpt = parser.accepts("throttle", "The movement of partitions between brokers will be throttled to this value (bytes/sec). " + + "This option can be included with --execute when a reassignment is started, and it can be altered by resubmitting the current reassignment " + + "along with the --additional flag. The throttle rate should be at least 1 KB/s.") + .withRequiredArg() + .describedAs("throttle") + .ofType(Long.class) + .defaultsTo(-1L); + replicaAlterLogDirsThrottleOpt = parser.accepts("replica-alter-log-dirs-throttle", + "The movement of replicas between log directories on the same broker will be throttled to this value (bytes/sec). " + + "This option can be included with --execute when a reassignment is started, and it can be altered by resubmitting the current reassignment " + + "along with the --additional flag. The throttle rate should be at least 1 KB/s.") + .withRequiredArg() + .describedAs("replicaAlterLogDirsThrottle") + .ofType(Long.class) + .defaultsTo(-1L); + timeoutOpt = parser.accepts("timeout", "The maximum time in ms to wait for log directory replica assignment to begin.") + .withRequiredArg() + .describedAs("timeout") + .ofType(Long.class) + .defaultsTo(10000L); + additionalOpt = parser.accepts("additional", "Execute this reassignment in addition to any " + + "other ongoing ones. This option can also be used to change the throttle of an ongoing reassignment."); + preserveThrottlesOpt = parser.accepts("preserve-throttles", "Do not modify broker or topic throttles."); + + options = parser.parse(args); + } + + public OptionSpecBuilder verifyOpt() { + return verifyOpt; + } + + public OptionSpecBuilder generateOpt() { + return generateOpt; + } + + public OptionSpecBuilder executeOpt() { + return executeOpt; + } + + public OptionSpecBuilder cancelOpt() { + return cancelOpt; + } + + public OptionSpecBuilder listOpt() { + return listOpt; + } + + public ArgumentAcceptingOptionSpec bootstrapServerOpt() { + return bootstrapServerOpt; + } + + public ArgumentAcceptingOptionSpec commandConfigOpt() { + return commandConfigOpt; + } + + public ArgumentAcceptingOptionSpec reassignmentJsonFileOpt() { + return reassignmentJsonFileOpt; + } + + public ArgumentAcceptingOptionSpec topicsToMoveJsonFileOpt() { + return topicsToMoveJsonFileOpt; + } + + public ArgumentAcceptingOptionSpec brokerListOpt() { + return brokerListOpt; + } + + public OptionSpecBuilder disableRackAware() { + return disableRackAware; + } + + public ArgumentAcceptingOptionSpec interBrokerThrottleOpt() { + return interBrokerThrottleOpt; + } + + public ArgumentAcceptingOptionSpec replicaAlterLogDirsThrottleOpt() { + return replicaAlterLogDirsThrottleOpt; + } + + public ArgumentAcceptingOptionSpec timeoutOpt() { + return timeoutOpt; + } + + public OptionSpecBuilder additionalOpt() { + return additionalOpt; + } + + public OptionSpecBuilder preserveThrottlesOpt() { + return preserveThrottlesOpt; + } +} diff --git a/core/src/main/scala/kafka/admin/ReassignPartitionsCommand.scala b/core/src/main/scala/kafka/admin/ReassignPartitionsCommand.scala index e8096f0e30a52..e48bf89cedd86 100755 --- a/core/src/main/scala/kafka/admin/ReassignPartitionsCommand.scala +++ b/core/src/main/scala/kafka/admin/ReassignPartitionsCommand.scala @@ -30,9 +30,9 @@ import org.apache.kafka.common.config.ConfigResource import org.apache.kafka.common.errors.{ReplicaNotAvailableException, UnknownTopicOrPartitionException} import org.apache.kafka.common.utils.{Time, Utils} import org.apache.kafka.common.{KafkaFuture, TopicPartition, TopicPartitionReplica} -import org.apache.kafka.server.util.{CommandDefaultOptions, CommandLineUtils} +import org.apache.kafka.server.util.CommandLineUtils import org.apache.kafka.storage.internals.log.LogConfig -import org.apache.kafka.tools.reassign.{ActiveMoveState, CancelledMoveState, CompletedMoveState, LogDirMoveState, MissingLogDirMoveState, MissingReplicaMoveState, PartitionMove, PartitionReassignmentState, TerseReassignmentFailureException} +import org.apache.kafka.tools.reassign.{ActiveMoveState, CancelledMoveState, CompletedMoveState, LogDirMoveState, MissingLogDirMoveState, MissingReplicaMoveState, PartitionMove, PartitionReassignmentState, ReassignPartitionsCommandOptions, TerseReassignmentFailureException} import scala.jdk.CollectionConverters._ import scala.collection.{Map, Seq, mutable} @@ -1334,71 +1334,4 @@ object ReassignPartitionsCommand extends Logging { } }.toSet } - - sealed class ReassignPartitionsCommandOptions(args: Array[String]) extends CommandDefaultOptions(args) { - // Actions - val verifyOpt = parser.accepts("verify", "Verify if the reassignment completed as specified by the " + - "--reassignment-json-file option. If there is a throttle engaged for the replicas specified, and the rebalance has completed, the throttle will be removed") - val generateOpt = parser.accepts("generate", "Generate a candidate partition reassignment configuration." + - " Note that this only generates a candidate assignment, it does not execute it.") - val executeOpt = parser.accepts("execute", "Kick off the reassignment as specified by the --reassignment-json-file option.") - val cancelOpt = parser.accepts("cancel", "Cancel an active reassignment.") - val listOpt = parser.accepts("list", "List all active partition reassignments.") - - // Arguments - val bootstrapServerOpt = parser.accepts("bootstrap-server", "REQUIRED: the server(s) to use for bootstrapping.") - .withRequiredArg - .describedAs("Server(s) to use for bootstrapping") - .ofType(classOf[String]) - - val commandConfigOpt = parser.accepts("command-config", "Property file containing configs to be passed to Admin Client.") - .withRequiredArg - .describedAs("Admin client property file") - .ofType(classOf[String]) - - val reassignmentJsonFileOpt = parser.accepts("reassignment-json-file", "The JSON file with the partition reassignment configuration" + - "The format to use is - \n" + - "{\"partitions\":\n\t[{\"topic\": \"foo\",\n\t \"partition\": 1,\n\t \"replicas\": [1,2,3],\n\t \"log_dirs\": [\"dir1\",\"dir2\",\"dir3\"] }],\n\"version\":1\n}\n" + - "Note that \"log_dirs\" is optional. When it is specified, its length must equal the length of the replicas list. The value in this list " + - "can be either \"any\" or the absolution path of the log directory on the broker. If absolute log directory path is specified, the replica will be moved to the specified log directory on the broker.") - .withRequiredArg - .describedAs("manual assignment json file path") - .ofType(classOf[String]) - val topicsToMoveJsonFileOpt = parser.accepts("topics-to-move-json-file", "Generate a reassignment configuration to move the partitions" + - " of the specified topics to the list of brokers specified by the --broker-list option. The format to use is - \n" + - "{\"topics\":\n\t[{\"topic\": \"foo\"},{\"topic\": \"foo1\"}],\n\"version\":1\n}") - .withRequiredArg - .describedAs("topics to reassign json file path") - .ofType(classOf[String]) - val brokerListOpt = parser.accepts("broker-list", "The list of brokers to which the partitions need to be reassigned" + - " in the form \"0,1,2\". This is required if --topics-to-move-json-file is used to generate reassignment configuration") - .withRequiredArg - .describedAs("brokerlist") - .ofType(classOf[String]) - val disableRackAware = parser.accepts("disable-rack-aware", "Disable rack aware replica assignment") - val interBrokerThrottleOpt = parser.accepts("throttle", "The movement of partitions between brokers will be throttled to this value (bytes/sec). " + - "This option can be included with --execute when a reassignment is started, and it can be altered by resubmitting the current reassignment " + - "along with the --additional flag. The throttle rate should be at least 1 KB/s.") - .withRequiredArg() - .describedAs("throttle") - .ofType(classOf[Long]) - .defaultsTo(-1) - val replicaAlterLogDirsThrottleOpt = parser.accepts("replica-alter-log-dirs-throttle", - "The movement of replicas between log directories on the same broker will be throttled to this value (bytes/sec). " + - "This option can be included with --execute when a reassignment is started, and it can be altered by resubmitting the current reassignment " + - "along with the --additional flag. The throttle rate should be at least 1 KB/s.") - .withRequiredArg() - .describedAs("replicaAlterLogDirsThrottle") - .ofType(classOf[Long]) - .defaultsTo(-1) - val timeoutOpt = parser.accepts("timeout", "The maximum time in ms to wait for log directory replica assignment to begin.") - .withRequiredArg() - .describedAs("timeout") - .ofType(classOf[Long]) - .defaultsTo(10000) - val additionalOpt = parser.accepts("additional", "Execute this reassignment in addition to any " + - "other ongoing ones. This option can also be used to change the throttle of an ongoing reassignment.") - val preserveThrottlesOpt = parser.accepts("preserve-throttles", "Do not modify broker or topic throttles.") - options = parser.parse(args : _*) - } } From ca8dc2a12e779bf66e2bd1fac403a883e0c01b2b Mon Sep 17 00:00:00 2001 From: nizhikov Date: Wed, 15 Feb 2023 11:26:16 +0300 Subject: [PATCH 03/78] KAFKA-14595 Move ReassignPartitionsCommandOptions to java --- .../ReassignPartitionsCommandOptions.java | 17 ++++ .../reassign/VerifyAssignmentResult.java | 91 +++++++++++++++++++ .../admin/ReassignPartitionsCommand.scala | 17 +--- .../ReassignPartitionsIntegrationTest.scala | 48 +++++----- 4 files changed, 134 insertions(+), 39 deletions(-) create mode 100644 core/src/main/java/org/apache/kafka/tools/reassign/VerifyAssignmentResult.java diff --git a/core/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommandOptions.java b/core/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommandOptions.java index acf768d8488cd..3d1b05c8cc655 100644 --- a/core/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommandOptions.java +++ b/core/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommandOptions.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.kafka.tools.reassign; import joptsimple.ArgumentAcceptingOptionSpec; diff --git a/core/src/main/java/org/apache/kafka/tools/reassign/VerifyAssignmentResult.java b/core/src/main/java/org/apache/kafka/tools/reassign/VerifyAssignmentResult.java new file mode 100644 index 0000000000000..2b6ba5cb72af5 --- /dev/null +++ b/core/src/main/java/org/apache/kafka/tools/reassign/VerifyAssignmentResult.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.tools.reassign; + +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.TopicPartitionReplica; + +import java.util.Collections; +import java.util.Map; +import java.util.Objects; + +/** + * A result returned from verifyAssignment. + */ +public final class VerifyAssignmentResult { + private final Map partStates; + private final boolean partsOngoing; + private final Map moveStates; + private final boolean movesOngoing; + + public VerifyAssignmentResult(Map partStates) { + this(partStates, false, Collections.emptyMap(), false); + } + + /** + * @param partStates A map from partitions to reassignment states. + * @param partsOngoing True if there are any ongoing partition reassignments. + * @param moveStates A map from log directories to movement states. + * @param movesOngoing True if there are any ongoing moves that we know about. + */ + public VerifyAssignmentResult(Map partStates, boolean partsOngoing, Map moveStates, boolean movesOngoing) { + this.partStates = partStates; + this.partsOngoing = partsOngoing; + this.moveStates = moveStates; + this.movesOngoing = movesOngoing; + } + + public Map partStates() { + return partStates; + } + + public boolean partsOngoing() { + return partsOngoing; + } + + public Map moveStates() { + return moveStates; + } + + public boolean movesOngoing() { + return movesOngoing; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + VerifyAssignmentResult that = (VerifyAssignmentResult) o; + return partsOngoing == that.partsOngoing && movesOngoing == that.movesOngoing && Objects.equals(partStates, that.partStates) && Objects.equals(moveStates, that.moveStates); + } + + @Override + public int hashCode() { + return Objects.hash(partStates, partsOngoing, moveStates, movesOngoing); + } + + @Override + public String toString() { + return "VerifyAssignmentResult{" + + "partStates=" + partStates + + ", partsOngoing=" + partsOngoing + + ", moveStates=" + moveStates + + ", movesOngoing=" + movesOngoing + + '}'; + } +} diff --git a/core/src/main/scala/kafka/admin/ReassignPartitionsCommand.scala b/core/src/main/scala/kafka/admin/ReassignPartitionsCommand.scala index e48bf89cedd86..f0769afe7c551 100755 --- a/core/src/main/scala/kafka/admin/ReassignPartitionsCommand.scala +++ b/core/src/main/scala/kafka/admin/ReassignPartitionsCommand.scala @@ -32,7 +32,7 @@ import org.apache.kafka.common.utils.{Time, Utils} import org.apache.kafka.common.{KafkaFuture, TopicPartition, TopicPartitionReplica} import org.apache.kafka.server.util.CommandLineUtils import org.apache.kafka.storage.internals.log.LogConfig -import org.apache.kafka.tools.reassign.{ActiveMoveState, CancelledMoveState, CompletedMoveState, LogDirMoveState, MissingLogDirMoveState, MissingReplicaMoveState, PartitionMove, PartitionReassignmentState, ReassignPartitionsCommandOptions, TerseReassignmentFailureException} +import org.apache.kafka.tools.reassign.{ActiveMoveState, CancelledMoveState, CompletedMoveState, LogDirMoveState, MissingLogDirMoveState, MissingReplicaMoveState, PartitionMove, PartitionReassignmentState, ReassignPartitionsCommandOptions, TerseReassignmentFailureException, VerifyAssignmentResult} import scala.jdk.CollectionConverters._ import scala.collection.{Map, Seq, mutable} @@ -156,19 +156,6 @@ object ReassignPartitionsCommand extends Logging { } } - /** - * A result returned from verifyAssignment. - * - * @param partStates A map from partitions to reassignment states. - * @param partsOngoing True if there are any ongoing partition reassignments. - * @param moveStates A map from log directories to movement states. - * @param movesOngoing True if there are any ongoing moves that we know about. - */ - case class VerifyAssignmentResult(partStates: Map[TopicPartition, PartitionReassignmentState], - partsOngoing: Boolean = false, - moveStates: Map[TopicPartitionReplica, LogDirMoveState] = Map.empty, - movesOngoing: Boolean = false) - /** * The entry point for the --verify command. * @@ -190,7 +177,7 @@ object ReassignPartitionsCommand extends Logging { // previous reassignments. clearAllThrottles(adminClient, targetParts) } - VerifyAssignmentResult(partStates, partsOngoing, moveStates, movesOngoing) + new VerifyAssignmentResult(partStates.asJava, partsOngoing, moveStates.asJava, movesOngoing) } /** diff --git a/core/src/test/scala/integration/kafka/admin/ReassignPartitionsIntegrationTest.scala b/core/src/test/scala/integration/kafka/admin/ReassignPartitionsIntegrationTest.scala index 40341908589d1..84d3c3c0f4f5d 100644 --- a/core/src/test/scala/integration/kafka/admin/ReassignPartitionsIntegrationTest.scala +++ b/core/src/test/scala/integration/kafka/admin/ReassignPartitionsIntegrationTest.scala @@ -27,7 +27,7 @@ import org.apache.kafka.common.config.ConfigResource import org.apache.kafka.common.utils.Utils import org.apache.kafka.common.{TopicPartition, TopicPartitionReplica} import org.apache.kafka.server.common.MetadataVersion.IBP_2_7_IV1 -import org.apache.kafka.tools.reassign.{ActiveMoveState, CompletedMoveState, PartitionReassignmentState} +import org.apache.kafka.tools.reassign.{ActiveMoveState, CompletedMoveState, LogDirMoveState, PartitionReassignmentState, VerifyAssignmentResult} import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertTrue} import org.junit.jupiter.api.{AfterEach, Timeout} import org.junit.jupiter.params.ParameterizedTest @@ -118,7 +118,7 @@ class ReassignPartitionsIntegrationTest extends QuorumTestHarness { bar0 -> new PartitionReassignmentState(Arrays.asList(3, 2, 1), Arrays.asList(3, 2, 0), true) ) waitForVerifyAssignment(cluster.adminClient, assignment, false, - VerifyAssignmentResult(initialAssignment)) + new VerifyAssignmentResult(initialAssignment.asJava)) // Execute the assignment runExecuteAssignment(cluster.adminClient, false, assignment, -1L, -1L) @@ -133,7 +133,7 @@ class ReassignPartitionsIntegrationTest extends QuorumTestHarness { // Wait for the assignment to complete waitForVerifyAssignment(cluster.adminClient, assignment, false, - VerifyAssignmentResult(finalAssignment)) + new VerifyAssignmentResult(finalAssignment.asJava)) assertEquals(unthrottledBrokerConfigs, describeBrokerLevelThrottles(unthrottledBrokerConfigs.keySet.toSeq)) @@ -163,7 +163,7 @@ class ReassignPartitionsIntegrationTest extends QuorumTestHarness { // Wait for the assignment to complete waitForVerifyAssignment(cluster.adminClient, assignment, false, - VerifyAssignmentResult(finalAssignment)) + new VerifyAssignmentResult(finalAssignment.asJava)) TestUtils.waitUntilTrue(() => { cluster.servers(3).replicaManager.onlinePartition(part). @@ -203,7 +203,7 @@ class ReassignPartitionsIntegrationTest extends QuorumTestHarness { // Now remove the throttles. waitForVerifyAssignment(cluster.adminClient, assignment, false, - VerifyAssignmentResult(finalAssignment)) + new VerifyAssignmentResult(finalAssignment.asJava)) waitForBrokerLevelThrottles(unthrottledBrokerConfigs) } @@ -228,7 +228,7 @@ class ReassignPartitionsIntegrationTest extends QuorumTestHarness { new PartitionReassignmentState(Arrays.asList(0, 1, 2), Arrays.asList(0, 3, 2), true), new TopicPartition("baz", 2) -> new PartitionReassignmentState(Arrays.asList(0, 2, 1), Arrays.asList(3, 2, 1), true)) - assertEquals(VerifyAssignmentResult(initialAssignment), runVerifyAssignment(cluster.adminClient, assignment, false)) + assertEquals(new VerifyAssignmentResult(initialAssignment.asJava), runVerifyAssignment(cluster.adminClient, assignment, false)) assertEquals(unthrottledBrokerConfigs, describeBrokerLevelThrottles(unthrottledBrokerConfigs.keySet.toSeq)) // Execute the assignment @@ -250,21 +250,21 @@ class ReassignPartitionsIntegrationTest extends QuorumTestHarness { if (!result.partsOngoing) { true } else { - assertFalse(result.partStates.forall(_._2.done), s"Expected at least one partition reassignment to be ongoing when result = $result") - assertEquals(Arrays.asList(0, 3, 2), result.partStates(new TopicPartition("foo", 0)).targetReplicas) - assertEquals(Arrays.asList(3, 2, 1), result.partStates(new TopicPartition("baz", 2)).targetReplicas) + //assertFalse(result.partStates.forall(_._2.done), s"Expected at least one partition reassignment to be ongoing when result = $result") + assertEquals(Arrays.asList(0, 3, 2), result.partStates().get(new TopicPartition("foo", 0)).targetReplicas) + assertEquals(Arrays.asList(3, 2, 1), result.partStates().get(new TopicPartition("baz", 2)).targetReplicas) logger.info(s"Current result: ${result}") waitForInterBrokerThrottle(Set(0, 1, 2, 3), interBrokerThrottle) false } }, "Expected reassignment to complete.") waitForVerifyAssignment(cluster.adminClient, assignment, true, - VerifyAssignmentResult(finalAssignment)) + new VerifyAssignmentResult(finalAssignment.asJava)) // The throttles should still have been preserved, since we ran with --preserve-throttles waitForInterBrokerThrottle(Set(0, 1, 2, 3), interBrokerThrottle) // Now remove the throttles. waitForVerifyAssignment(cluster.adminClient, assignment, false, - VerifyAssignmentResult(finalAssignment)) + new VerifyAssignmentResult(finalAssignment.asJava)) waitForBrokerLevelThrottles(unthrottledBrokerConfigs) } @@ -291,7 +291,7 @@ class ReassignPartitionsIntegrationTest extends QuorumTestHarness { val finalAssignment = Map(part -> new PartitionReassignmentState(Arrays.asList(3, 2, 1), Arrays.asList(3, 2, 1), true)) waitForVerifyAssignment(cluster.adminClient, assignment, false, - VerifyAssignmentResult(finalAssignment)) + new VerifyAssignmentResult(finalAssignment.asJava)) } /** @@ -320,10 +320,10 @@ class ReassignPartitionsIntegrationTest extends QuorumTestHarness { // Verify that the reassignment is running. The very low throttle should keep it // from completing before this runs. waitForVerifyAssignment(cluster.adminClient, assignment, true, - VerifyAssignmentResult(Map( + new VerifyAssignmentResult(Map( foo0 -> new PartitionReassignmentState(Arrays.asList(0, 1, 3, 2), Arrays.asList(0, 1, 3), false), - baz1 -> new PartitionReassignmentState(Arrays.asList(0, 2, 3, 1), Arrays.asList(0, 2, 3), false)), - true, Map(), false)) + baz1 -> new PartitionReassignmentState(Arrays.asList(0, 2, 3, 1), Arrays.asList(0, 2, 3), false)).asJava, + true, Collections.emptyMap(), false)) // Cancel the reassignment. assertEquals((Set(foo0, baz1), Set()), runCancelAssignment(cluster.adminClient, assignment, true)) // Broker throttles are still active because we passed --preserve-throttles @@ -476,12 +476,12 @@ class ReassignPartitionsIntegrationTest extends QuorumTestHarness { // Check the output of --verify waitForVerifyAssignment(cluster.adminClient, reassignment.json, true, - VerifyAssignmentResult(Map( + new VerifyAssignmentResult(Map( topicPartition -> new PartitionReassignmentState(Arrays.asList(0, 1, 2), Arrays.asList(0, 1, 2), true) - ), false, Map( + ).asJava, false, Map[TopicPartitionReplica, LogDirMoveState]( new TopicPartitionReplica(topicPartition.topic, topicPartition.partition, 0) -> new ActiveMoveState(reassignment.currentDir, reassignment.targetDir, reassignment.targetDir) - ), true)) + ).asJava, true)) waitForLogDirThrottle(Set(0), logDirThrottle) // Remove the throttle @@ -494,12 +494,12 @@ class ReassignPartitionsIntegrationTest extends QuorumTestHarness { // Wait for the directory movement to complete. waitForVerifyAssignment(cluster.adminClient, reassignment.json, true, - VerifyAssignmentResult(Map( + new VerifyAssignmentResult(Map( topicPartition -> new PartitionReassignmentState(Arrays.asList(0, 1, 2), Arrays.asList(0, 1, 2), true) - ), false, Map( + ).asJava, false, Map[TopicPartitionReplica, LogDirMoveState]( new TopicPartitionReplica(topicPartition.topic, topicPartition.partition, 0) -> new CompletedMoveState(reassignment.targetDir) - ), false)) + ).asJava, false)) val info1 = new BrokerDirs(cluster.adminClient.describeLogDirs(0.to(4). map(_.asInstanceOf[Integer]).asJavaCollection), 0) @@ -532,12 +532,12 @@ class ReassignPartitionsIntegrationTest extends QuorumTestHarness { waitForLogDirThrottle(Set(0), updatedLogDirThrottle) waitForVerifyAssignment(cluster.adminClient, reassignment.json, true, - VerifyAssignmentResult(Map( + new VerifyAssignmentResult(Map( topicPartition -> new PartitionReassignmentState(Arrays.asList(0, 1, 2), Arrays.asList(0, 1, 2), true) - ), false, Map( + ).asJava, false, Map[TopicPartitionReplica, LogDirMoveState]( new TopicPartitionReplica(topicPartition.topic, topicPartition.partition, targetBrokerId) -> new CompletedMoveState(reassignment.targetDir) - ), false)) + ).asJava, false)) } case class LogDirReassignment(json: String, currentDir: String, targetDir: String) From 634d1853a8dce0bb1c3216674699d99fa2dc6b59 Mon Sep 17 00:00:00 2001 From: nizhikov Date: Wed, 15 Feb 2023 13:55:04 +0300 Subject: [PATCH 04/78] KAFKA-14595 Move ReassignPartitionsCommandOptions to java --- .../kafka/tools/reassign => kafka/tools}/ActiveMoveState.java | 2 +- .../tools/reassign => kafka/tools}/CancelledMoveState.java | 2 +- .../tools/reassign => kafka/tools}/CompletedMoveState.java | 2 +- .../kafka/tools/reassign => kafka/tools}/LogDirMoveState.java | 2 +- .../tools/reassign => kafka/tools}/MissingLogDirMoveState.java | 2 +- .../tools/reassign => kafka/tools}/MissingReplicaMoveState.java | 2 +- .../kafka/tools/reassign => kafka/tools}/PartitionMove.java | 2 +- .../reassign => kafka/tools}/PartitionReassignmentState.java | 2 +- .../tools}/ReassignPartitionsCommandOptions.java | 2 +- .../tools}/TerseReassignmentFailureException.java | 2 +- .../tools/reassign => kafka/tools}/VerifyAssignmentResult.java | 2 +- core/src/main/scala/kafka/admin/ReassignPartitionsCommand.scala | 2 +- .../kafka/admin/ReassignPartitionsIntegrationTest.scala | 2 +- .../scala/unit/kafka/admin/ReassignPartitionsUnitTest.scala | 2 +- 14 files changed, 14 insertions(+), 14 deletions(-) rename core/src/main/java/{org/apache/kafka/tools/reassign => kafka/tools}/ActiveMoveState.java (98%) rename core/src/main/java/{org/apache/kafka/tools/reassign => kafka/tools}/CancelledMoveState.java (98%) rename core/src/main/java/{org/apache/kafka/tools/reassign => kafka/tools}/CompletedMoveState.java (97%) rename core/src/main/java/{org/apache/kafka/tools/reassign => kafka/tools}/LogDirMoveState.java (95%) rename core/src/main/java/{org/apache/kafka/tools/reassign => kafka/tools}/MissingLogDirMoveState.java (97%) rename core/src/main/java/{org/apache/kafka/tools/reassign => kafka/tools}/MissingReplicaMoveState.java (97%) rename core/src/main/java/{org/apache/kafka/tools/reassign => kafka/tools}/PartitionMove.java (98%) rename core/src/main/java/{org/apache/kafka/tools/reassign => kafka/tools}/PartitionReassignmentState.java (98%) rename core/src/main/java/{org/apache/kafka/tools/reassign => kafka/tools}/ReassignPartitionsCommandOptions.java (99%) rename core/src/main/java/{org/apache/kafka/tools/reassign => kafka/tools}/TerseReassignmentFailureException.java (96%) rename core/src/main/java/{org/apache/kafka/tools/reassign => kafka/tools}/VerifyAssignmentResult.java (98%) diff --git a/core/src/main/java/org/apache/kafka/tools/reassign/ActiveMoveState.java b/core/src/main/java/kafka/tools/ActiveMoveState.java similarity index 98% rename from core/src/main/java/org/apache/kafka/tools/reassign/ActiveMoveState.java rename to core/src/main/java/kafka/tools/ActiveMoveState.java index 271b3f6926b5f..df97da6c782df 100644 --- a/core/src/main/java/org/apache/kafka/tools/reassign/ActiveMoveState.java +++ b/core/src/main/java/kafka/tools/ActiveMoveState.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.kafka.tools.reassign; +package kafka.tools; import java.util.Objects; diff --git a/core/src/main/java/org/apache/kafka/tools/reassign/CancelledMoveState.java b/core/src/main/java/kafka/tools/CancelledMoveState.java similarity index 98% rename from core/src/main/java/org/apache/kafka/tools/reassign/CancelledMoveState.java rename to core/src/main/java/kafka/tools/CancelledMoveState.java index fec8036d0910c..12b8997db3452 100644 --- a/core/src/main/java/org/apache/kafka/tools/reassign/CancelledMoveState.java +++ b/core/src/main/java/kafka/tools/CancelledMoveState.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.kafka.tools.reassign; +package kafka.tools; import java.util.Objects; diff --git a/core/src/main/java/org/apache/kafka/tools/reassign/CompletedMoveState.java b/core/src/main/java/kafka/tools/CompletedMoveState.java similarity index 97% rename from core/src/main/java/org/apache/kafka/tools/reassign/CompletedMoveState.java rename to core/src/main/java/kafka/tools/CompletedMoveState.java index ce936e8c6d7af..b9d655d4dc1a4 100644 --- a/core/src/main/java/org/apache/kafka/tools/reassign/CompletedMoveState.java +++ b/core/src/main/java/kafka/tools/CompletedMoveState.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.kafka.tools.reassign; +package kafka.tools; import java.util.Objects; diff --git a/core/src/main/java/org/apache/kafka/tools/reassign/LogDirMoveState.java b/core/src/main/java/kafka/tools/LogDirMoveState.java similarity index 95% rename from core/src/main/java/org/apache/kafka/tools/reassign/LogDirMoveState.java rename to core/src/main/java/kafka/tools/LogDirMoveState.java index deb2f64e32161..091557da79cab 100644 --- a/core/src/main/java/org/apache/kafka/tools/reassign/LogDirMoveState.java +++ b/core/src/main/java/kafka/tools/LogDirMoveState.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.kafka.tools.reassign; +package kafka.tools; /** * The state of a replica log directory movement. diff --git a/core/src/main/java/org/apache/kafka/tools/reassign/MissingLogDirMoveState.java b/core/src/main/java/kafka/tools/MissingLogDirMoveState.java similarity index 97% rename from core/src/main/java/org/apache/kafka/tools/reassign/MissingLogDirMoveState.java rename to core/src/main/java/kafka/tools/MissingLogDirMoveState.java index 0143a5b035f82..d4e8276976c65 100644 --- a/core/src/main/java/org/apache/kafka/tools/reassign/MissingLogDirMoveState.java +++ b/core/src/main/java/kafka/tools/MissingLogDirMoveState.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.kafka.tools.reassign; +package kafka.tools; import java.util.Objects; diff --git a/core/src/main/java/org/apache/kafka/tools/reassign/MissingReplicaMoveState.java b/core/src/main/java/kafka/tools/MissingReplicaMoveState.java similarity index 97% rename from core/src/main/java/org/apache/kafka/tools/reassign/MissingReplicaMoveState.java rename to core/src/main/java/kafka/tools/MissingReplicaMoveState.java index 41d0356ec4756..33ab43f87e642 100644 --- a/core/src/main/java/org/apache/kafka/tools/reassign/MissingReplicaMoveState.java +++ b/core/src/main/java/kafka/tools/MissingReplicaMoveState.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.kafka.tools.reassign; +package kafka.tools; import java.util.Objects; diff --git a/core/src/main/java/org/apache/kafka/tools/reassign/PartitionMove.java b/core/src/main/java/kafka/tools/PartitionMove.java similarity index 98% rename from core/src/main/java/org/apache/kafka/tools/reassign/PartitionMove.java rename to core/src/main/java/kafka/tools/PartitionMove.java index ffd00056fdbdd..8652176889249 100644 --- a/core/src/main/java/org/apache/kafka/tools/reassign/PartitionMove.java +++ b/core/src/main/java/kafka/tools/PartitionMove.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.kafka.tools.reassign; +package kafka.tools; import java.util.Objects; import java.util.Set; diff --git a/core/src/main/java/org/apache/kafka/tools/reassign/PartitionReassignmentState.java b/core/src/main/java/kafka/tools/PartitionReassignmentState.java similarity index 98% rename from core/src/main/java/org/apache/kafka/tools/reassign/PartitionReassignmentState.java rename to core/src/main/java/kafka/tools/PartitionReassignmentState.java index db6a49263cb05..39dcdb6694587 100644 --- a/core/src/main/java/org/apache/kafka/tools/reassign/PartitionReassignmentState.java +++ b/core/src/main/java/kafka/tools/PartitionReassignmentState.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.kafka.tools.reassign; +package kafka.tools; import java.util.List; import java.util.Objects; diff --git a/core/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommandOptions.java b/core/src/main/java/kafka/tools/ReassignPartitionsCommandOptions.java similarity index 99% rename from core/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommandOptions.java rename to core/src/main/java/kafka/tools/ReassignPartitionsCommandOptions.java index 3d1b05c8cc655..7c40d568d2172 100644 --- a/core/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommandOptions.java +++ b/core/src/main/java/kafka/tools/ReassignPartitionsCommandOptions.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.kafka.tools.reassign; +package kafka.tools; import joptsimple.ArgumentAcceptingOptionSpec; import joptsimple.OptionSpecBuilder; diff --git a/core/src/main/java/org/apache/kafka/tools/reassign/TerseReassignmentFailureException.java b/core/src/main/java/kafka/tools/TerseReassignmentFailureException.java similarity index 96% rename from core/src/main/java/org/apache/kafka/tools/reassign/TerseReassignmentFailureException.java rename to core/src/main/java/kafka/tools/TerseReassignmentFailureException.java index 524e25ff53cf8..bed649ecc0c7d 100644 --- a/core/src/main/java/org/apache/kafka/tools/reassign/TerseReassignmentFailureException.java +++ b/core/src/main/java/kafka/tools/TerseReassignmentFailureException.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.kafka.tools.reassign; +package kafka.tools; import org.apache.kafka.common.KafkaException; diff --git a/core/src/main/java/org/apache/kafka/tools/reassign/VerifyAssignmentResult.java b/core/src/main/java/kafka/tools/VerifyAssignmentResult.java similarity index 98% rename from core/src/main/java/org/apache/kafka/tools/reassign/VerifyAssignmentResult.java rename to core/src/main/java/kafka/tools/VerifyAssignmentResult.java index 2b6ba5cb72af5..5f5ca07cd57e8 100644 --- a/core/src/main/java/org/apache/kafka/tools/reassign/VerifyAssignmentResult.java +++ b/core/src/main/java/kafka/tools/VerifyAssignmentResult.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.kafka.tools.reassign; +package kafka.tools; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.TopicPartitionReplica; diff --git a/core/src/main/scala/kafka/admin/ReassignPartitionsCommand.scala b/core/src/main/scala/kafka/admin/ReassignPartitionsCommand.scala index f0769afe7c551..78da75e9e4c99 100755 --- a/core/src/main/scala/kafka/admin/ReassignPartitionsCommand.scala +++ b/core/src/main/scala/kafka/admin/ReassignPartitionsCommand.scala @@ -21,6 +21,7 @@ import java.util.Optional import java.util.concurrent.ExecutionException import kafka.common.AdminCommandFailedException import kafka.server.DynamicConfig +import kafka.tools.{ActiveMoveState, CancelledMoveState, CompletedMoveState, LogDirMoveState, MissingLogDirMoveState, MissingReplicaMoveState, PartitionMove, PartitionReassignmentState, ReassignPartitionsCommandOptions, TerseReassignmentFailureException, VerifyAssignmentResult} import kafka.utils.{CoreUtils, Exit, Json, Logging} import kafka.utils.Implicits._ import kafka.utils.json.JsonValue @@ -32,7 +33,6 @@ import org.apache.kafka.common.utils.{Time, Utils} import org.apache.kafka.common.{KafkaFuture, TopicPartition, TopicPartitionReplica} import org.apache.kafka.server.util.CommandLineUtils import org.apache.kafka.storage.internals.log.LogConfig -import org.apache.kafka.tools.reassign.{ActiveMoveState, CancelledMoveState, CompletedMoveState, LogDirMoveState, MissingLogDirMoveState, MissingReplicaMoveState, PartitionMove, PartitionReassignmentState, ReassignPartitionsCommandOptions, TerseReassignmentFailureException, VerifyAssignmentResult} import scala.jdk.CollectionConverters._ import scala.collection.{Map, Seq, mutable} diff --git a/core/src/test/scala/integration/kafka/admin/ReassignPartitionsIntegrationTest.scala b/core/src/test/scala/integration/kafka/admin/ReassignPartitionsIntegrationTest.scala index 84d3c3c0f4f5d..21f96b4fe241a 100644 --- a/core/src/test/scala/integration/kafka/admin/ReassignPartitionsIntegrationTest.scala +++ b/core/src/test/scala/integration/kafka/admin/ReassignPartitionsIntegrationTest.scala @@ -19,6 +19,7 @@ package kafka.admin import kafka.admin.ReassignPartitionsCommand._ import kafka.server._ +import kafka.tools.{ActiveMoveState, CompletedMoveState, LogDirMoveState, PartitionReassignmentState, VerifyAssignmentResult} import kafka.utils.Implicits._ import kafka.utils.{TestInfoUtils, TestUtils} import org.apache.kafka.clients.admin._ @@ -27,7 +28,6 @@ import org.apache.kafka.common.config.ConfigResource import org.apache.kafka.common.utils.Utils import org.apache.kafka.common.{TopicPartition, TopicPartitionReplica} import org.apache.kafka.server.common.MetadataVersion.IBP_2_7_IV1 -import org.apache.kafka.tools.reassign.{ActiveMoveState, CompletedMoveState, LogDirMoveState, PartitionReassignmentState, VerifyAssignmentResult} import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertTrue} import org.junit.jupiter.api.{AfterEach, Timeout} import org.junit.jupiter.params.ParameterizedTest diff --git a/core/src/test/scala/unit/kafka/admin/ReassignPartitionsUnitTest.scala b/core/src/test/scala/unit/kafka/admin/ReassignPartitionsUnitTest.scala index 0d74d7790eed6..35a8688830054 100644 --- a/core/src/test/scala/unit/kafka/admin/ReassignPartitionsUnitTest.scala +++ b/core/src/test/scala/unit/kafka/admin/ReassignPartitionsUnitTest.scala @@ -21,12 +21,12 @@ import java.util.concurrent.ExecutionException import java.util.{Arrays, Collections} import kafka.admin.ReassignPartitionsCommand._ import kafka.common.AdminCommandFailedException +import kafka.tools.{ActiveMoveState, CancelledMoveState, CompletedMoveState, MissingLogDirMoveState, MissingReplicaMoveState, PartitionMove, PartitionReassignmentState} import kafka.utils.Exit import org.apache.kafka.clients.admin.{Config, MockAdminClient, PartitionReassignment} import org.apache.kafka.common.config.ConfigResource import org.apache.kafka.common.errors.{InvalidReplicationFactorException, UnknownTopicOrPartitionException} import org.apache.kafka.common.{Node, TopicPartition, TopicPartitionInfo, TopicPartitionReplica} -import org.apache.kafka.tools.reassign.{ActiveMoveState, CancelledMoveState, CompletedMoveState, MissingLogDirMoveState, MissingReplicaMoveState, PartitionMove, PartitionReassignmentState} import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertThrows, assertTrue} import org.junit.jupiter.api.{AfterEach, BeforeEach, Test, Timeout} From 28ae6bbe2da89f643c571c2ba46cb6077077ed01 Mon Sep 17 00:00:00 2001 From: nizhikov Date: Wed, 15 Feb 2023 14:48:25 +0300 Subject: [PATCH 05/78] KAFKA-14595 Compilation fix --- core/src/main/java/kafka/tools/VerifyAssignmentResult.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/kafka/tools/VerifyAssignmentResult.java b/core/src/main/java/kafka/tools/VerifyAssignmentResult.java index 5f5ca07cd57e8..944d8d50e1f00 100644 --- a/core/src/main/java/kafka/tools/VerifyAssignmentResult.java +++ b/core/src/main/java/kafka/tools/VerifyAssignmentResult.java @@ -43,7 +43,12 @@ public VerifyAssignmentResult(Map pa * @param moveStates A map from log directories to movement states. * @param movesOngoing True if there are any ongoing moves that we know about. */ - public VerifyAssignmentResult(Map partStates, boolean partsOngoing, Map moveStates, boolean movesOngoing) { + public VerifyAssignmentResult( + Map partStates, + boolean partsOngoing, + Map moveStates, + boolean movesOngoing + ) { this.partStates = partStates; this.partsOngoing = partsOngoing; this.moveStates = moveStates; From cd4095de7d06e9bd1dbd4a8f7f30bde0d9494d70 Mon Sep 17 00:00:00 2001 From: nizhikov Date: Tue, 28 Feb 2023 16:24:44 +0300 Subject: [PATCH 06/78] KAFKA-14595 Move new classes to tools --- .../admin/ReassignPartitionsCommand.scala | 260 +++++++++++++++--- .../ReassignPartitionsIntegrationTest.scala | 87 +++--- .../admin/ReassignPartitionsUnitTest.scala | 41 ++- .../tools/reassign}/ActiveMoveState.java | 2 +- .../tools/reassign}/CancelledMoveState.java | 2 +- .../tools/reassign}/CompletedMoveState.java | 2 +- .../tools/reassign}/LogDirMoveState.java | 2 +- .../reassign}/MissingLogDirMoveState.java | 2 +- .../reassign}/MissingReplicaMoveState.java | 2 +- .../kafka/tools/reassign}/PartitionMove.java | 2 +- .../reassign}/PartitionReassignmentState.java | 2 +- .../ReassignPartitionsCommandOptions.java | 2 +- .../TerseReassignmentFailureException.java | 2 +- .../reassign}/VerifyAssignmentResult.java | 2 +- 14 files changed, 292 insertions(+), 118 deletions(-) rename {core/src/main/java/kafka/tools => tools/src/main/java/org/apache/kafka/tools/reassign}/ActiveMoveState.java (98%) rename {core/src/main/java/kafka/tools => tools/src/main/java/org/apache/kafka/tools/reassign}/CancelledMoveState.java (98%) rename {core/src/main/java/kafka/tools => tools/src/main/java/org/apache/kafka/tools/reassign}/CompletedMoveState.java (97%) rename {core/src/main/java/kafka/tools => tools/src/main/java/org/apache/kafka/tools/reassign}/LogDirMoveState.java (95%) rename {core/src/main/java/kafka/tools => tools/src/main/java/org/apache/kafka/tools/reassign}/MissingLogDirMoveState.java (97%) rename {core/src/main/java/kafka/tools => tools/src/main/java/org/apache/kafka/tools/reassign}/MissingReplicaMoveState.java (97%) rename {core/src/main/java/kafka/tools => tools/src/main/java/org/apache/kafka/tools/reassign}/PartitionMove.java (98%) rename {core/src/main/java/kafka/tools => tools/src/main/java/org/apache/kafka/tools/reassign}/PartitionReassignmentState.java (98%) rename {core/src/main/java/kafka/tools => tools/src/main/java/org/apache/kafka/tools/reassign}/ReassignPartitionsCommandOptions.java (99%) rename {core/src/main/java/kafka/tools => tools/src/main/java/org/apache/kafka/tools/reassign}/TerseReassignmentFailureException.java (96%) rename {core/src/main/java/kafka/tools => tools/src/main/java/org/apache/kafka/tools/reassign}/VerifyAssignmentResult.java (98%) diff --git a/core/src/main/scala/kafka/admin/ReassignPartitionsCommand.scala b/core/src/main/scala/kafka/admin/ReassignPartitionsCommand.scala index 78da75e9e4c99..d1b75c483eba8 100755 --- a/core/src/main/scala/kafka/admin/ReassignPartitionsCommand.scala +++ b/core/src/main/scala/kafka/admin/ReassignPartitionsCommand.scala @@ -21,7 +21,6 @@ import java.util.Optional import java.util.concurrent.ExecutionException import kafka.common.AdminCommandFailedException import kafka.server.DynamicConfig -import kafka.tools.{ActiveMoveState, CancelledMoveState, CompletedMoveState, LogDirMoveState, MissingLogDirMoveState, MissingReplicaMoveState, PartitionMove, PartitionReassignmentState, ReassignPartitionsCommandOptions, TerseReassignmentFailureException, VerifyAssignmentResult} import kafka.utils.{CoreUtils, Exit, Json, Logging} import kafka.utils.Implicits._ import kafka.utils.json.JsonValue @@ -30,8 +29,8 @@ import org.apache.kafka.clients.admin.{Admin, AdminClientConfig, AlterConfigOp, import org.apache.kafka.common.config.ConfigResource import org.apache.kafka.common.errors.{ReplicaNotAvailableException, UnknownTopicOrPartitionException} import org.apache.kafka.common.utils.{Time, Utils} -import org.apache.kafka.common.{KafkaFuture, TopicPartition, TopicPartitionReplica} -import org.apache.kafka.server.util.CommandLineUtils +import org.apache.kafka.common.{KafkaException, KafkaFuture, TopicPartition, TopicPartitionReplica} +import org.apache.kafka.server.util.{CommandDefaultOptions, CommandLineUtils} import org.apache.kafka.storage.internals.log.LogConfig import scala.jdk.CollectionConverters._ @@ -93,6 +92,104 @@ object ReassignPartitionsCommand extends Logging { */ type MoveMap = mutable.Map[String, mutable.Map[Int, PartitionMove]] + /** + * A partition movement. The source and destination brokers may overlap. + * + * @param sources The source brokers. + * @param destinations The destination brokers. + */ + sealed case class PartitionMove(sources: mutable.Set[Int], + destinations: mutable.Set[Int]) { } + + /** + * The state of a partition reassignment. The current replicas and target replicas + * may overlap. + * + * @param currentReplicas The current replicas. + * @param targetReplicas The target replicas. + * @param done True if the reassignment is done. + */ + sealed case class PartitionReassignmentState(currentReplicas: Seq[Int], + targetReplicas: Seq[Int], + done: Boolean) {} + + /** + * The state of a replica log directory movement. + */ + sealed trait LogDirMoveState { + /** + * True if the move is done without errors. + */ + def done: Boolean + } + + /** + * A replica log directory move state where the source log directory is missing. + * + * @param targetLogDir The log directory that we wanted the replica to move to. + */ + sealed case class MissingReplicaMoveState(targetLogDir: String) + extends LogDirMoveState { + override def done = false + } + + /** + * A replica log directory move state where the source replica is missing. + * + * @param targetLogDir The log directory that we wanted the replica to move to. + */ + sealed case class MissingLogDirMoveState(targetLogDir: String) + extends LogDirMoveState { + override def done = false + } + + /** + * A replica log directory move state where the move is in progress. + * + * @param currentLogDir The current log directory. + * @param futureLogDir The log directory that the replica is moving to. + * @param targetLogDir The log directory that we wanted the replica to move to. + */ + sealed case class ActiveMoveState(currentLogDir: String, + targetLogDir: String, + futureLogDir: String) + extends LogDirMoveState { + override def done = false + } + + /** + * A replica log directory move state where there is no move in progress, but we did not + * reach the target log directory. + * + * @param currentLogDir The current log directory. + * @param targetLogDir The log directory that we wanted the replica to move to. + */ + sealed case class CancelledMoveState(currentLogDir: String, + targetLogDir: String) + extends LogDirMoveState { + override def done = true + } + + /** + * The completed replica log directory move state. + * + * @param targetLogDir The log directory that we wanted the replica to move to. + */ + sealed case class CompletedMoveState(targetLogDir: String) + extends LogDirMoveState { + override def done = true + } + + /** + * An exception thrown to indicate that the command has failed, but we don't want to + * print a stack trace. + * + * @param message The message to print out before exiting. A stack trace will not + * be printed. + */ + class TerseReassignmentFailureException(message: String) extends KafkaException(message) { + } + def main(args: Array[String]): Unit = { val opts = validateAndParseArgs(args) var failed = true @@ -156,6 +253,19 @@ object ReassignPartitionsCommand extends Logging { } } + /** + * A result returned from verifyAssignment. + * + * @param partStates A map from partitions to reassignment states. + * @param partsOngoing True if there are any ongoing partition reassignments. + * @param moveStates A map from log directories to movement states. + * @param movesOngoing True if there are any ongoing moves that we know about. + */ + case class VerifyAssignmentResult(partStates: Map[TopicPartition, PartitionReassignmentState], + partsOngoing: Boolean = false, + moveStates: Map[TopicPartitionReplica, LogDirMoveState] = Map.empty, + movesOngoing: Boolean = false) + /** * The entry point for the --verify command. * @@ -177,7 +287,7 @@ object ReassignPartitionsCommand extends Logging { // previous reassignments. clearAllThrottles(adminClient, targetParts) } - new VerifyAssignmentResult(partStates.asJava, partsOngoing, moveStates.asJava, movesOngoing) + VerifyAssignmentResult(partStates, partsOngoing, moveStates, movesOngoing) } /** @@ -219,14 +329,14 @@ object ReassignPartitionsCommand extends Logging { bld.append("Status of partition reassignment:") states.keySet.toBuffer.sortWith(compareTopicPartitions).foreach { topicPartition => val state = states(topicPartition) - if (state.done()) { - if (state.currentReplicas().equals(state.targetReplicas)) { + if (state.done) { + if (state.currentReplicas.equals(state.targetReplicas)) { bld.append("Reassignment of partition %s is completed.". format(topicPartition.toString)) } else { bld.append(s"There is no active reassignment of partition ${topicPartition}, " + - s"but replica set is ${state.currentReplicas()} rather than " + - s"${state.targetReplicas}.") + s"but replica set is ${state.currentReplicas.mkString(",")} rather than " + + s"${state.targetReplicas.mkString(",")}.") } } else { bld.append("Reassignment of partition %s is still in progress.".format(topicPartition)) @@ -254,9 +364,10 @@ object ReassignPartitionsCommand extends Logging { } val foundResults = foundReassignments.map { case (part, targetReplicas) => (part, - new PartitionReassignmentState( - currentReassignments(part).replicas, - targetReplicas.map(i => i.asInstanceOf[Integer]).asJava, + PartitionReassignmentState( + currentReassignments(part).replicas. + asScala.map(i => i.asInstanceOf[Int]), + targetReplicas, false)) } val topicNamesToLookUp = new mutable.HashSet[String]() @@ -270,9 +381,9 @@ object ReassignPartitionsCommand extends Logging { case (part, targetReplicas) => currentReassignments.get(part) match { case Some(reassignment) => (part, - new PartitionReassignmentState( - reassignment.replicas, - targetReplicas.map(i => i.asInstanceOf[Integer]).asJava, + PartitionReassignmentState( + reassignment.replicas.asScala.map(_.asInstanceOf[Int]), + targetReplicas, false)) case None => (part, topicDescriptionFutureToState(part.partition, @@ -291,13 +402,13 @@ object ReassignPartitionsCommand extends Logging { if (topicDescription.partitions().size() < partition) { throw new ExecutionException("Too few partitions found", new UnknownTopicOrPartitionException()) } - new PartitionReassignmentState( - topicDescription.partitions.get(partition).replicas.asScala.map(_.id.asInstanceOf[Integer]).asJava, - targetReplicas.map(i => i.asInstanceOf[Integer]).asJava, + PartitionReassignmentState( + topicDescription.partitions.get(partition).replicas.asScala.map(_.id), + targetReplicas, true) } catch { case t: ExecutionException if t.getCause.isInstanceOf[UnknownTopicOrPartitionException] => - new PartitionReassignmentState(java.util.Collections.emptyList(), targetReplicas.map(i => i.asInstanceOf[Integer]).asJava, true) + PartitionReassignmentState(Seq(), targetReplicas, true) } } @@ -340,17 +451,17 @@ object ReassignPartitionsCommand extends Logging { targetMoves.keySet.asJava).all().get().asScala targetMoves.map { case (replica, targetLogDir) => val moveState = replicaLogDirInfos.get(replica) match { - case None => new MissingReplicaMoveState(targetLogDir) + case None => MissingReplicaMoveState(targetLogDir) case Some(info) => if (info.getCurrentReplicaLogDir == null) { - new MissingLogDirMoveState(targetLogDir) + MissingLogDirMoveState(targetLogDir) } else if (info.getFutureReplicaLogDir == null) { if (info.getCurrentReplicaLogDir.equals(targetLogDir)) { - new CompletedMoveState(targetLogDir) + CompletedMoveState(targetLogDir) } else { - new CancelledMoveState(info.getCurrentReplicaLogDir, targetLogDir) + CancelledMoveState(info.getCurrentReplicaLogDir, targetLogDir) } } else { - new ActiveMoveState(info.getCurrentReplicaLogDir(), + ActiveMoveState(info.getCurrentReplicaLogDir(), targetLogDir, info.getFutureReplicaLogDir) } @@ -372,29 +483,27 @@ object ReassignPartitionsCommand extends Logging { states.keySet.toBuffer.sortWith(compareTopicPartitionReplicas).foreach { replica => val state = states(replica) state match { - case _: MissingLogDirMoveState => + case MissingLogDirMoveState(_) => bld.append(s"Partition ${replica.topic}-${replica.partition} is not found " + s"in any live log dir on broker ${replica.brokerId}. There is likely an " + s"offline log directory on the broker.") - case _: MissingReplicaMoveState => + case MissingReplicaMoveState(_) => bld.append(s"Partition ${replica.topic}-${replica.partition} cannot be found " + s"in any live log directory on broker ${replica.brokerId}.") - case s: ActiveMoveState => - if (s.targetLogDir.equals(s.futureLogDir)) { + case ActiveMoveState(_, targetLogDir, futureLogDir) => + if (targetLogDir.equals(futureLogDir)) { bld.append(s"Reassignment of replica $replica is still in progress.") } else { bld.append(s"Partition ${replica.topic}-${replica.partition} on broker " + - s"${replica.brokerId} is being moved to log dir ${s.futureLogDir} " + - s"instead of ${s.targetLogDir}.") + s"${replica.brokerId} is being moved to log dir $futureLogDir " + + s"instead of $targetLogDir.") } - case s: CancelledMoveState => + case CancelledMoveState(currentLogDir, targetLogDir) => bld.append(s"Partition ${replica.topic}-${replica.partition} on broker " + - s"${replica.brokerId} is not being moved from log dir ${s.currentLogDir} to " + - s"${s.targetLogDir}.") - case _: CompletedMoveState => + s"${replica.brokerId} is not being moved from log dir $currentLogDir to " + + s"$targetLogDir.") + case CompletedMoveState(_) => bld.append(s"Reassignment of replica $replica completed successfully.") - case _ => - throw new IllegalStateException(s"Unknown state $state") } } bld.mkString(System.lineSeparator()) @@ -853,7 +962,7 @@ object ReassignPartitionsCommand extends Logging { val destinations = mutable.Set[Int]() ++ addingReplicas val partMoves = moveMap.getOrElseUpdate(part.topic, new mutable.HashMap[Int, PartitionMove]) - partMoves.put(part.partition, new PartitionMove(sources.map(i => i.asInstanceOf[Integer]).asJava, destinations.map(i => i.asInstanceOf[Integer]).asJava)) + partMoves.put(part.partition, PartitionMove(sources, destinations)) } moveMap } @@ -880,14 +989,14 @@ object ReassignPartitionsCommand extends Logging { // If there is a reassignment in progress, use the sources from moveMap, otherwise // use the sources from currentParts val sources = mutable.Set[Int]() ++ (partMoves.get(part.partition) match { - case Some(move) => move.sources.asScala.toSeq + case Some(move) => move.sources.toSeq case None => currentParts.getOrElse(part, throw new RuntimeException(s"Trying to reassign a topic partition $part with 0 replicas")) }) val destinations = mutable.Set[Int]() ++ replicas.diff(sources.toSeq) partMoves.put(part.partition, - new PartitionMove(sources.map(i => i.asInstanceOf[Integer]).asJava, destinations.map(i => i.asInstanceOf[Integer]).asJava)) + PartitionMove(sources, destinations)) } moveMap } @@ -903,7 +1012,7 @@ object ReassignPartitionsCommand extends Logging { case (topicName, partMoveMap) => { val components = new mutable.TreeSet[String] partMoveMap.forKeyValue { (partId, move) => - move.sources.forEach(source => components.add("%d:%d".format(partId, source))) + move.sources.foreach(source => components.add("%d:%d".format(partId, source))) } (topicName, components.mkString(",")) } @@ -921,7 +1030,7 @@ object ReassignPartitionsCommand extends Logging { case (topicName, partMoveMap) => { val components = new mutable.TreeSet[String] partMoveMap.forKeyValue { (partId, move) => - move.destinations.forEach(destination => + move.destinations.foreach(destination => if (!move.sources.contains(destination)) { components.add("%d:%d".format(partId, destination)) }) @@ -942,8 +1051,8 @@ object ReassignPartitionsCommand extends Logging { moveMap.values.foreach { _.values.foreach { partMove => - partMove.sources.forEach(i => reassigningBrokers.add(i)) - partMove.destinations.forEach(i => reassigningBrokers.add(i)) + partMove.sources.foreach(reassigningBrokers.add) + partMove.destinations.foreach(reassigningBrokers.add) } } reassigningBrokers.toSet @@ -1321,4 +1430,71 @@ object ReassignPartitionsCommand extends Logging { } }.toSet } + + sealed class ReassignPartitionsCommandOptions(args: Array[String]) extends CommandDefaultOptions(args) { + // Actions + val verifyOpt = parser.accepts("verify", "Verify if the reassignment completed as specified by the " + + "--reassignment-json-file option. If there is a throttle engaged for the replicas specified, and the rebalance has completed, the throttle will be removed") + val generateOpt = parser.accepts("generate", "Generate a candidate partition reassignment configuration." + + " Note that this only generates a candidate assignment, it does not execute it.") + val executeOpt = parser.accepts("execute", "Kick off the reassignment as specified by the --reassignment-json-file option.") + val cancelOpt = parser.accepts("cancel", "Cancel an active reassignment.") + val listOpt = parser.accepts("list", "List all active partition reassignments.") + + // Arguments + val bootstrapServerOpt = parser.accepts("bootstrap-server", "REQUIRED: the server(s) to use for bootstrapping.") + .withRequiredArg + .describedAs("Server(s) to use for bootstrapping") + .ofType(classOf[String]) + + val commandConfigOpt = parser.accepts("command-config", "Property file containing configs to be passed to Admin Client.") + .withRequiredArg + .describedAs("Admin client property file") + .ofType(classOf[String]) + + val reassignmentJsonFileOpt = parser.accepts("reassignment-json-file", "The JSON file with the partition reassignment configuration" + + "The format to use is - \n" + + "{\"partitions\":\n\t[{\"topic\": \"foo\",\n\t \"partition\": 1,\n\t \"replicas\": [1,2,3],\n\t \"log_dirs\": [\"dir1\",\"dir2\",\"dir3\"] }],\n\"version\":1\n}\n" + + "Note that \"log_dirs\" is optional. When it is specified, its length must equal the length of the replicas list. The value in this list " + + "can be either \"any\" or the absolution path of the log directory on the broker. If absolute log directory path is specified, the replica will be moved to the specified log directory on the broker.") + .withRequiredArg + .describedAs("manual assignment json file path") + .ofType(classOf[String]) + val topicsToMoveJsonFileOpt = parser.accepts("topics-to-move-json-file", "Generate a reassignment configuration to move the partitions" + + " of the specified topics to the list of brokers specified by the --broker-list option. The format to use is - \n" + + "{\"topics\":\n\t[{\"topic\": \"foo\"},{\"topic\": \"foo1\"}],\n\"version\":1\n}") + .withRequiredArg + .describedAs("topics to reassign json file path") + .ofType(classOf[String]) + val brokerListOpt = parser.accepts("broker-list", "The list of brokers to which the partitions need to be reassigned" + + " in the form \"0,1,2\". This is required if --topics-to-move-json-file is used to generate reassignment configuration") + .withRequiredArg + .describedAs("brokerlist") + .ofType(classOf[String]) + val disableRackAware = parser.accepts("disable-rack-aware", "Disable rack aware replica assignment") + val interBrokerThrottleOpt = parser.accepts("throttle", "The movement of partitions between brokers will be throttled to this value (bytes/sec). " + + "This option can be included with --execute when a reassignment is started, and it can be altered by resubmitting the current reassignment " + + "along with the --additional flag. The throttle rate should be at least 1 KB/s.") + .withRequiredArg() + .describedAs("throttle") + .ofType(classOf[Long]) + .defaultsTo(-1) + val replicaAlterLogDirsThrottleOpt = parser.accepts("replica-alter-log-dirs-throttle", + "The movement of replicas between log directories on the same broker will be throttled to this value (bytes/sec). " + + "This option can be included with --execute when a reassignment is started, and it can be altered by resubmitting the current reassignment " + + "along with the --additional flag. The throttle rate should be at least 1 KB/s.") + .withRequiredArg() + .describedAs("replicaAlterLogDirsThrottle") + .ofType(classOf[Long]) + .defaultsTo(-1) + val timeoutOpt = parser.accepts("timeout", "The maximum time in ms to wait for log directory replica assignment to begin.") + .withRequiredArg() + .describedAs("timeout") + .ofType(classOf[Long]) + .defaultsTo(10000) + val additionalOpt = parser.accepts("additional", "Execute this reassignment in addition to any " + + "other ongoing ones. This option can also be used to change the throttle of an ongoing reassignment.") + val preserveThrottlesOpt = parser.accepts("preserve-throttles", "Do not modify broker or topic throttles.") + options = parser.parse(args : _*) + } } diff --git a/core/src/test/scala/integration/kafka/admin/ReassignPartitionsIntegrationTest.scala b/core/src/test/scala/integration/kafka/admin/ReassignPartitionsIntegrationTest.scala index 21f96b4fe241a..d3a04da4d1344 100644 --- a/core/src/test/scala/integration/kafka/admin/ReassignPartitionsIntegrationTest.scala +++ b/core/src/test/scala/integration/kafka/admin/ReassignPartitionsIntegrationTest.scala @@ -19,7 +19,6 @@ package kafka.admin import kafka.admin.ReassignPartitionsCommand._ import kafka.server._ -import kafka.tools.{ActiveMoveState, CompletedMoveState, LogDirMoveState, PartitionReassignmentState, VerifyAssignmentResult} import kafka.utils.Implicits._ import kafka.utils.{TestInfoUtils, TestUtils} import org.apache.kafka.clients.admin._ @@ -34,7 +33,7 @@ import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.ValueSource import java.io.Closeable -import java.util.{Arrays, Collections, HashMap, List} +import java.util.{Collections, HashMap, List} import scala.collection.{Map, Seq, mutable} import scala.jdk.CollectionConverters._ @@ -114,18 +113,18 @@ class ReassignPartitionsIntegrationTest extends QuorumTestHarness { // Check that the assignment has not yet been started yet. val initialAssignment = Map( - foo0 -> new PartitionReassignmentState(Arrays.asList(0, 1, 2), Arrays.asList(0, 1, 3), true), - bar0 -> new PartitionReassignmentState(Arrays.asList(3, 2, 1), Arrays.asList(3, 2, 0), true) + foo0 -> PartitionReassignmentState(Seq(0, 1, 2), Seq(0, 1, 3), true), + bar0 -> PartitionReassignmentState(Seq(3, 2, 1), Seq(3, 2, 0), true) ) waitForVerifyAssignment(cluster.adminClient, assignment, false, - new VerifyAssignmentResult(initialAssignment.asJava)) + VerifyAssignmentResult(initialAssignment)) // Execute the assignment runExecuteAssignment(cluster.adminClient, false, assignment, -1L, -1L) assertEquals(unthrottledBrokerConfigs, describeBrokerLevelThrottles(unthrottledBrokerConfigs.keySet.toSeq)) val finalAssignment = Map( - foo0 -> new PartitionReassignmentState(Arrays.asList(0, 1, 3), Arrays.asList(0, 1, 3), true), - bar0 -> new PartitionReassignmentState(Arrays.asList(3, 2, 0), Arrays.asList(3, 2, 0), true) + foo0 -> PartitionReassignmentState(Seq(0, 1, 3), Seq(0, 1, 3), true), + bar0 -> PartitionReassignmentState(Seq(3, 2, 0), Seq(3, 2, 0), true) ) val verifyAssignmentResult = runVerifyAssignment(cluster.adminClient, assignment, false) @@ -133,7 +132,7 @@ class ReassignPartitionsIntegrationTest extends QuorumTestHarness { // Wait for the assignment to complete waitForVerifyAssignment(cluster.adminClient, assignment, false, - new VerifyAssignmentResult(finalAssignment.asJava)) + VerifyAssignmentResult(finalAssignment)) assertEquals(unthrottledBrokerConfigs, describeBrokerLevelThrottles(unthrottledBrokerConfigs.keySet.toSeq)) @@ -159,11 +158,11 @@ class ReassignPartitionsIntegrationTest extends QuorumTestHarness { // Execute the assignment runExecuteAssignment(cluster.adminClient, false, assignment, -1L, -1L) val finalAssignment = Map(part -> - new PartitionReassignmentState(Arrays.asList(3, 1, 2), Arrays.asList(3, 1, 2), true)) + PartitionReassignmentState(Seq(3, 1, 2), Seq(3, 1, 2), true)) // Wait for the assignment to complete waitForVerifyAssignment(cluster.adminClient, assignment, false, - new VerifyAssignmentResult(finalAssignment.asJava)) + VerifyAssignmentResult(finalAssignment)) TestUtils.waitUntilTrue(() => { cluster.servers(3).replicaManager.onlinePartition(part). @@ -197,13 +196,13 @@ class ReassignPartitionsIntegrationTest extends QuorumTestHarness { val finalAssignment = Map( new TopicPartition("foo", 0) -> - new PartitionReassignmentState(Arrays.asList(0, 3, 2), Arrays.asList(0, 3, 2), true), + PartitionReassignmentState(Seq(0, 3, 2), Seq(0, 3, 2), true), new TopicPartition("baz", 2) -> - new PartitionReassignmentState(Arrays.asList(3, 2, 1), Arrays.asList(3, 2, 1), true)) + PartitionReassignmentState(Seq(3, 2, 1), Seq(3, 2, 1), true)) // Now remove the throttles. waitForVerifyAssignment(cluster.adminClient, assignment, false, - new VerifyAssignmentResult(finalAssignment.asJava)) + VerifyAssignmentResult(finalAssignment)) waitForBrokerLevelThrottles(unthrottledBrokerConfigs) } @@ -225,10 +224,10 @@ class ReassignPartitionsIntegrationTest extends QuorumTestHarness { // Check that the assignment has not yet been started yet. val initialAssignment = Map( new TopicPartition("foo", 0) -> - new PartitionReassignmentState(Arrays.asList(0, 1, 2), Arrays.asList(0, 3, 2), true), + PartitionReassignmentState(Seq(0, 1, 2), Seq(0, 3, 2), true), new TopicPartition("baz", 2) -> - new PartitionReassignmentState(Arrays.asList(0, 2, 1), Arrays.asList(3, 2, 1), true)) - assertEquals(new VerifyAssignmentResult(initialAssignment.asJava), runVerifyAssignment(cluster.adminClient, assignment, false)) + PartitionReassignmentState(Seq(0, 2, 1), Seq(3, 2, 1), true)) + assertEquals(VerifyAssignmentResult(initialAssignment), runVerifyAssignment(cluster.adminClient, assignment, false)) assertEquals(unthrottledBrokerConfigs, describeBrokerLevelThrottles(unthrottledBrokerConfigs.keySet.toSeq)) // Execute the assignment @@ -238,9 +237,9 @@ class ReassignPartitionsIntegrationTest extends QuorumTestHarness { val finalAssignment = Map( new TopicPartition("foo", 0) -> - new PartitionReassignmentState(Arrays.asList(0, 3, 2), Arrays.asList(0, 3, 2), true), + PartitionReassignmentState(Seq(0, 3, 2), Seq(0, 3, 2), true), new TopicPartition("baz", 2) -> - new PartitionReassignmentState(Arrays.asList(3, 2, 1), Arrays.asList(3, 2, 1), true)) + PartitionReassignmentState(Seq(3, 2, 1), Seq(3, 2, 1), true)) // Wait for the assignment to complete TestUtils.waitUntilTrue( @@ -250,21 +249,21 @@ class ReassignPartitionsIntegrationTest extends QuorumTestHarness { if (!result.partsOngoing) { true } else { - //assertFalse(result.partStates.forall(_._2.done), s"Expected at least one partition reassignment to be ongoing when result = $result") - assertEquals(Arrays.asList(0, 3, 2), result.partStates().get(new TopicPartition("foo", 0)).targetReplicas) - assertEquals(Arrays.asList(3, 2, 1), result.partStates().get(new TopicPartition("baz", 2)).targetReplicas) + assertFalse(result.partStates.forall(_._2.done), s"Expected at least one partition reassignment to be ongoing when result = $result") + assertEquals(Seq(0, 3, 2), result.partStates(new TopicPartition("foo", 0)).targetReplicas) + assertEquals(Seq(3, 2, 1), result.partStates(new TopicPartition("baz", 2)).targetReplicas) logger.info(s"Current result: ${result}") waitForInterBrokerThrottle(Set(0, 1, 2, 3), interBrokerThrottle) false } }, "Expected reassignment to complete.") waitForVerifyAssignment(cluster.adminClient, assignment, true, - new VerifyAssignmentResult(finalAssignment.asJava)) + VerifyAssignmentResult(finalAssignment)) // The throttles should still have been preserved, since we ran with --preserve-throttles waitForInterBrokerThrottle(Set(0, 1, 2, 3), interBrokerThrottle) // Now remove the throttles. waitForVerifyAssignment(cluster.adminClient, assignment, false, - new VerifyAssignmentResult(finalAssignment.asJava)) + VerifyAssignmentResult(finalAssignment)) waitForBrokerLevelThrottles(unthrottledBrokerConfigs) } @@ -289,9 +288,9 @@ class ReassignPartitionsIntegrationTest extends QuorumTestHarness { } TestUtils.removeReplicationThrottleForPartitions(cluster.adminClient, Seq(0,1,2,3), Set(part)) val finalAssignment = Map(part -> - new PartitionReassignmentState(Arrays.asList(3, 2, 1), Arrays.asList(3, 2, 1), true)) + PartitionReassignmentState(Seq(3, 2, 1), Seq(3, 2, 1), true)) waitForVerifyAssignment(cluster.adminClient, assignment, false, - new VerifyAssignmentResult(finalAssignment.asJava)) + VerifyAssignmentResult(finalAssignment)) } /** @@ -320,10 +319,10 @@ class ReassignPartitionsIntegrationTest extends QuorumTestHarness { // Verify that the reassignment is running. The very low throttle should keep it // from completing before this runs. waitForVerifyAssignment(cluster.adminClient, assignment, true, - new VerifyAssignmentResult(Map( - foo0 -> new PartitionReassignmentState(Arrays.asList(0, 1, 3, 2), Arrays.asList(0, 1, 3), false), - baz1 -> new PartitionReassignmentState(Arrays.asList(0, 2, 3, 1), Arrays.asList(0, 2, 3), false)).asJava, - true, Collections.emptyMap(), false)) + VerifyAssignmentResult(Map( + foo0 -> PartitionReassignmentState(Seq(0, 1, 3, 2), Seq(0, 1, 3), false), + baz1 -> PartitionReassignmentState(Seq(0, 2, 3, 1), Seq(0, 2, 3), false)), + true, Map(), false)) // Cancel the reassignment. assertEquals((Set(foo0, baz1), Set()), runCancelAssignment(cluster.adminClient, assignment, true)) // Broker throttles are still active because we passed --preserve-throttles @@ -476,12 +475,12 @@ class ReassignPartitionsIntegrationTest extends QuorumTestHarness { // Check the output of --verify waitForVerifyAssignment(cluster.adminClient, reassignment.json, true, - new VerifyAssignmentResult(Map( - topicPartition -> new PartitionReassignmentState(Arrays.asList(0, 1, 2), Arrays.asList(0, 1, 2), true) - ).asJava, false, Map[TopicPartitionReplica, LogDirMoveState]( + VerifyAssignmentResult(Map( + topicPartition -> PartitionReassignmentState(Seq(0, 1, 2), Seq(0, 1, 2), true) + ), false, Map( new TopicPartitionReplica(topicPartition.topic, topicPartition.partition, 0) -> - new ActiveMoveState(reassignment.currentDir, reassignment.targetDir, reassignment.targetDir) - ).asJava, true)) + ActiveMoveState(reassignment.currentDir, reassignment.targetDir, reassignment.targetDir) + ), true)) waitForLogDirThrottle(Set(0), logDirThrottle) // Remove the throttle @@ -494,12 +493,12 @@ class ReassignPartitionsIntegrationTest extends QuorumTestHarness { // Wait for the directory movement to complete. waitForVerifyAssignment(cluster.adminClient, reassignment.json, true, - new VerifyAssignmentResult(Map( - topicPartition -> new PartitionReassignmentState(Arrays.asList(0, 1, 2), Arrays.asList(0, 1, 2), true) - ).asJava, false, Map[TopicPartitionReplica, LogDirMoveState]( + VerifyAssignmentResult(Map( + topicPartition -> PartitionReassignmentState(Seq(0, 1, 2), Seq(0, 1, 2), true) + ), false, Map( new TopicPartitionReplica(topicPartition.topic, topicPartition.partition, 0) -> - new CompletedMoveState(reassignment.targetDir) - ).asJava, false)) + CompletedMoveState(reassignment.targetDir) + ), false)) val info1 = new BrokerDirs(cluster.adminClient.describeLogDirs(0.to(4). map(_.asInstanceOf[Integer]).asJavaCollection), 0) @@ -532,12 +531,12 @@ class ReassignPartitionsIntegrationTest extends QuorumTestHarness { waitForLogDirThrottle(Set(0), updatedLogDirThrottle) waitForVerifyAssignment(cluster.adminClient, reassignment.json, true, - new VerifyAssignmentResult(Map( - topicPartition -> new PartitionReassignmentState(Arrays.asList(0, 1, 2), Arrays.asList(0, 1, 2), true) - ).asJava, false, Map[TopicPartitionReplica, LogDirMoveState]( + VerifyAssignmentResult(Map( + topicPartition -> PartitionReassignmentState(Seq(0, 1, 2), Seq(0, 1, 2), true) + ), false, Map( new TopicPartitionReplica(topicPartition.topic, topicPartition.partition, targetBrokerId) -> - new CompletedMoveState(reassignment.targetDir) - ).asJava, false)) + CompletedMoveState(reassignment.targetDir) + ), false)) } case class LogDirReassignment(json: String, currentDir: String, targetDir: String) diff --git a/core/src/test/scala/unit/kafka/admin/ReassignPartitionsUnitTest.scala b/core/src/test/scala/unit/kafka/admin/ReassignPartitionsUnitTest.scala index 35a8688830054..3b7c1ab719168 100644 --- a/core/src/test/scala/unit/kafka/admin/ReassignPartitionsUnitTest.scala +++ b/core/src/test/scala/unit/kafka/admin/ReassignPartitionsUnitTest.scala @@ -21,7 +21,6 @@ import java.util.concurrent.ExecutionException import java.util.{Arrays, Collections} import kafka.admin.ReassignPartitionsCommand._ import kafka.common.AdminCommandFailedException -import kafka.tools.{ActiveMoveState, CancelledMoveState, CompletedMoveState, MissingLogDirMoveState, MissingReplicaMoveState, PartitionMove, PartitionReassignmentState} import kafka.utils.Exit import org.apache.kafka.clients.admin.{Config, MockAdminClient, PartitionReassignment} import org.apache.kafka.common.config.ConfigResource @@ -72,11 +71,11 @@ class ReassignPartitionsUnitTest { mkString(System.lineSeparator()), partitionReassignmentStatesToString(Map( new TopicPartition("foo", 0) -> - new PartitionReassignmentState(Arrays.asList(1, 2, 3), Arrays.asList(1, 2, 3), true), + PartitionReassignmentState(Seq(1, 2, 3), Seq(1, 2, 3), true), new TopicPartition("foo", 1) -> - new PartitionReassignmentState(Arrays.asList(1, 2, 3), Arrays.asList(1, 2, 4), false), + PartitionReassignmentState(Seq(1, 2, 3), Seq(1, 2, 4), false), new TopicPartition("bar", 0) -> - new PartitionReassignmentState(Arrays.asList(1, 2, 3), Arrays.asList(1, 2, 4), false), + PartitionReassignmentState(Seq(1, 2, 3), Seq(1, 2, 4), false), ))) } @@ -109,8 +108,8 @@ class ReassignPartitionsUnitTest { assertEquals(Map(new TopicPartition("quux", 0) -> classOf[UnknownTopicOrPartitionException]), reassignmentResult) assertEquals((Map( - new TopicPartition("foo", 0) -> new PartitionReassignmentState(Arrays.asList(0,1,2), Arrays.asList(0,1,3), false), - new TopicPartition("foo", 1) -> new PartitionReassignmentState(Arrays.asList(1,2,3), Arrays.asList(1,2,3), true) + new TopicPartition("foo", 0) -> PartitionReassignmentState(Seq(0,1,2), Seq(0,1,3), false), + new TopicPartition("foo", 1) -> PartitionReassignmentState(Seq(1,2,3), Seq(1,2,3), true) ), true), findPartitionReassignmentStates(adminClient, Seq( (new TopicPartition("foo", 0), Seq(0,1,3)), @@ -124,8 +123,8 @@ class ReassignPartitionsUnitTest { assertEquals(Map(new TopicPartition("quux", 2) -> classOf[UnknownTopicOrPartitionException]), cancelResult) assertEquals((Map( - new TopicPartition("foo", 0) -> new PartitionReassignmentState(Arrays.asList(0,1,2), Arrays.asList(0,1,3), true), - new TopicPartition("foo", 1) -> new PartitionReassignmentState(Arrays.asList(1,2,3), Arrays.asList(1,2,3), true) + new TopicPartition("foo", 0) -> PartitionReassignmentState(Seq(0,1,2), Seq(0,1,3), true), + new TopicPartition("foo", 1) -> PartitionReassignmentState(Seq(1,2,3), Seq(1,2,3), true) ), false), findPartitionReassignmentStates(adminClient, Seq( (new TopicPartition("foo", 0), Seq(0,1,3)), @@ -189,15 +188,15 @@ class ReassignPartitionsUnitTest { "Partition quux-2 is not found in any live log dir on broker 1. " + "There is likely an offline log directory on the broker.").mkString(System.lineSeparator()), replicaMoveStatesToString(Map( - new TopicPartitionReplica("bar", 0, 0) -> new CompletedMoveState("/tmp/kafka-logs0"), - new TopicPartitionReplica("foo", 0, 0) -> new ActiveMoveState("/tmp/kafka-logs0", + new TopicPartitionReplica("bar", 0, 0) -> CompletedMoveState("/tmp/kafka-logs0"), + new TopicPartitionReplica("foo", 0, 0) -> ActiveMoveState("/tmp/kafka-logs0", "/tmp/kafka-logs1", "/tmp/kafka-logs1"), - new TopicPartitionReplica("foo", 1, 0) -> new CancelledMoveState("/tmp/kafka-logs0", + new TopicPartitionReplica("foo", 1, 0) -> CancelledMoveState("/tmp/kafka-logs0", "/tmp/kafka-logs1"), - new TopicPartitionReplica("quux", 0, 0) -> new MissingReplicaMoveState("/tmp/kafka-logs1"), - new TopicPartitionReplica("quux", 1, 1) -> new ActiveMoveState("/tmp/kafka-logs0", + new TopicPartitionReplica("quux", 0, 0) -> MissingReplicaMoveState("/tmp/kafka-logs1"), + new TopicPartitionReplica("quux", 1, 1) -> ActiveMoveState("/tmp/kafka-logs0", "/tmp/kafka-logs1", "/tmp/kafka-logs2"), - new TopicPartitionReplica("quux", 2, 1) -> new MissingLogDirMoveState("/tmp/kafka-logs1") + new TopicPartitionReplica("quux", 2, 1) -> MissingLogDirMoveState("/tmp/kafka-logs1") ))) } @@ -420,14 +419,14 @@ class ReassignPartitionsUnitTest { assertEquals( mutable.Map("foo" -> mutable.Map( - 0 -> new PartitionMove(Set[Integer](1,2,3).asJava, Set[Integer](5).asJava), - 1 -> new PartitionMove(Set[Integer](4,5,6).asJava, Set[Integer](7,8).asJava), - 2 -> new PartitionMove(Set[Integer](1,2).asJava, Set[Integer](3,4).asJava), - 3 -> new PartitionMove(Set[Integer](1,2).asJava, Set[Integer](5,6).asJava), - 4 -> new PartitionMove(Set[Integer](1,2).asJava, Set[Integer](3).asJava), - 5 -> new PartitionMove(Set[Integer](1,2).asJava, Set[Integer](3,4,5,6).asJava) + 0 -> PartitionMove(mutable.Set(1,2,3), mutable.Set(5)), + 1 -> PartitionMove(mutable.Set(4,5,6), mutable.Set(7,8)), + 2 -> PartitionMove(mutable.Set(1,2), mutable.Set(3,4)), + 3 -> PartitionMove(mutable.Set(1,2), mutable.Set(5,6)), + 4 -> PartitionMove(mutable.Set(1,2), mutable.Set(3)), + 5 -> PartitionMove(mutable.Set(1,2), mutable.Set(3,4,5,6)) ), "bar" -> mutable.Map( - 0 -> new PartitionMove(Set[Integer](2,3,4).asJava, Set[Integer](1).asJava), + 0 -> PartitionMove(mutable.Set(2,3,4), mutable.Set(1)), )), moveMap) assertEquals(Map( diff --git a/core/src/main/java/kafka/tools/ActiveMoveState.java b/tools/src/main/java/org/apache/kafka/tools/reassign/ActiveMoveState.java similarity index 98% rename from core/src/main/java/kafka/tools/ActiveMoveState.java rename to tools/src/main/java/org/apache/kafka/tools/reassign/ActiveMoveState.java index df97da6c782df..271b3f6926b5f 100644 --- a/core/src/main/java/kafka/tools/ActiveMoveState.java +++ b/tools/src/main/java/org/apache/kafka/tools/reassign/ActiveMoveState.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package kafka.tools; +package org.apache.kafka.tools.reassign; import java.util.Objects; diff --git a/core/src/main/java/kafka/tools/CancelledMoveState.java b/tools/src/main/java/org/apache/kafka/tools/reassign/CancelledMoveState.java similarity index 98% rename from core/src/main/java/kafka/tools/CancelledMoveState.java rename to tools/src/main/java/org/apache/kafka/tools/reassign/CancelledMoveState.java index 12b8997db3452..fec8036d0910c 100644 --- a/core/src/main/java/kafka/tools/CancelledMoveState.java +++ b/tools/src/main/java/org/apache/kafka/tools/reassign/CancelledMoveState.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package kafka.tools; +package org.apache.kafka.tools.reassign; import java.util.Objects; diff --git a/core/src/main/java/kafka/tools/CompletedMoveState.java b/tools/src/main/java/org/apache/kafka/tools/reassign/CompletedMoveState.java similarity index 97% rename from core/src/main/java/kafka/tools/CompletedMoveState.java rename to tools/src/main/java/org/apache/kafka/tools/reassign/CompletedMoveState.java index b9d655d4dc1a4..ce936e8c6d7af 100644 --- a/core/src/main/java/kafka/tools/CompletedMoveState.java +++ b/tools/src/main/java/org/apache/kafka/tools/reassign/CompletedMoveState.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package kafka.tools; +package org.apache.kafka.tools.reassign; import java.util.Objects; diff --git a/core/src/main/java/kafka/tools/LogDirMoveState.java b/tools/src/main/java/org/apache/kafka/tools/reassign/LogDirMoveState.java similarity index 95% rename from core/src/main/java/kafka/tools/LogDirMoveState.java rename to tools/src/main/java/org/apache/kafka/tools/reassign/LogDirMoveState.java index 091557da79cab..deb2f64e32161 100644 --- a/core/src/main/java/kafka/tools/LogDirMoveState.java +++ b/tools/src/main/java/org/apache/kafka/tools/reassign/LogDirMoveState.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package kafka.tools; +package org.apache.kafka.tools.reassign; /** * The state of a replica log directory movement. diff --git a/core/src/main/java/kafka/tools/MissingLogDirMoveState.java b/tools/src/main/java/org/apache/kafka/tools/reassign/MissingLogDirMoveState.java similarity index 97% rename from core/src/main/java/kafka/tools/MissingLogDirMoveState.java rename to tools/src/main/java/org/apache/kafka/tools/reassign/MissingLogDirMoveState.java index d4e8276976c65..0143a5b035f82 100644 --- a/core/src/main/java/kafka/tools/MissingLogDirMoveState.java +++ b/tools/src/main/java/org/apache/kafka/tools/reassign/MissingLogDirMoveState.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package kafka.tools; +package org.apache.kafka.tools.reassign; import java.util.Objects; diff --git a/core/src/main/java/kafka/tools/MissingReplicaMoveState.java b/tools/src/main/java/org/apache/kafka/tools/reassign/MissingReplicaMoveState.java similarity index 97% rename from core/src/main/java/kafka/tools/MissingReplicaMoveState.java rename to tools/src/main/java/org/apache/kafka/tools/reassign/MissingReplicaMoveState.java index 33ab43f87e642..41d0356ec4756 100644 --- a/core/src/main/java/kafka/tools/MissingReplicaMoveState.java +++ b/tools/src/main/java/org/apache/kafka/tools/reassign/MissingReplicaMoveState.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package kafka.tools; +package org.apache.kafka.tools.reassign; import java.util.Objects; diff --git a/core/src/main/java/kafka/tools/PartitionMove.java b/tools/src/main/java/org/apache/kafka/tools/reassign/PartitionMove.java similarity index 98% rename from core/src/main/java/kafka/tools/PartitionMove.java rename to tools/src/main/java/org/apache/kafka/tools/reassign/PartitionMove.java index 8652176889249..ffd00056fdbdd 100644 --- a/core/src/main/java/kafka/tools/PartitionMove.java +++ b/tools/src/main/java/org/apache/kafka/tools/reassign/PartitionMove.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package kafka.tools; +package org.apache.kafka.tools.reassign; import java.util.Objects; import java.util.Set; diff --git a/core/src/main/java/kafka/tools/PartitionReassignmentState.java b/tools/src/main/java/org/apache/kafka/tools/reassign/PartitionReassignmentState.java similarity index 98% rename from core/src/main/java/kafka/tools/PartitionReassignmentState.java rename to tools/src/main/java/org/apache/kafka/tools/reassign/PartitionReassignmentState.java index 39dcdb6694587..db6a49263cb05 100644 --- a/core/src/main/java/kafka/tools/PartitionReassignmentState.java +++ b/tools/src/main/java/org/apache/kafka/tools/reassign/PartitionReassignmentState.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package kafka.tools; +package org.apache.kafka.tools.reassign; import java.util.List; import java.util.Objects; diff --git a/core/src/main/java/kafka/tools/ReassignPartitionsCommandOptions.java b/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommandOptions.java similarity index 99% rename from core/src/main/java/kafka/tools/ReassignPartitionsCommandOptions.java rename to tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommandOptions.java index 7c40d568d2172..3d1b05c8cc655 100644 --- a/core/src/main/java/kafka/tools/ReassignPartitionsCommandOptions.java +++ b/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommandOptions.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package kafka.tools; +package org.apache.kafka.tools.reassign; import joptsimple.ArgumentAcceptingOptionSpec; import joptsimple.OptionSpecBuilder; diff --git a/core/src/main/java/kafka/tools/TerseReassignmentFailureException.java b/tools/src/main/java/org/apache/kafka/tools/reassign/TerseReassignmentFailureException.java similarity index 96% rename from core/src/main/java/kafka/tools/TerseReassignmentFailureException.java rename to tools/src/main/java/org/apache/kafka/tools/reassign/TerseReassignmentFailureException.java index bed649ecc0c7d..524e25ff53cf8 100644 --- a/core/src/main/java/kafka/tools/TerseReassignmentFailureException.java +++ b/tools/src/main/java/org/apache/kafka/tools/reassign/TerseReassignmentFailureException.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package kafka.tools; +package org.apache.kafka.tools.reassign; import org.apache.kafka.common.KafkaException; diff --git a/core/src/main/java/kafka/tools/VerifyAssignmentResult.java b/tools/src/main/java/org/apache/kafka/tools/reassign/VerifyAssignmentResult.java similarity index 98% rename from core/src/main/java/kafka/tools/VerifyAssignmentResult.java rename to tools/src/main/java/org/apache/kafka/tools/reassign/VerifyAssignmentResult.java index 944d8d50e1f00..4061ab81c86dc 100644 --- a/core/src/main/java/kafka/tools/VerifyAssignmentResult.java +++ b/tools/src/main/java/org/apache/kafka/tools/reassign/VerifyAssignmentResult.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package kafka.tools; +package org.apache.kafka.tools.reassign; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.TopicPartitionReplica; From 30be4ee7428c3fd028767b994d8397e197ca6411 Mon Sep 17 00:00:00 2001 From: nizhikov Date: Sun, 9 Jul 2023 17:30:43 +0300 Subject: [PATCH 07/78] KAFKA-14595 Move new classes to tools --- .../ReassignPartitionsCommandOptions.java | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommandOptions.java b/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommandOptions.java index 3d1b05c8cc655..b1382a8490411 100644 --- a/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommandOptions.java +++ b/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommandOptions.java @@ -17,7 +17,7 @@ package org.apache.kafka.tools.reassign; -import joptsimple.ArgumentAcceptingOptionSpec; +import joptsimple.OptionSpec; import joptsimple.OptionSpecBuilder; import org.apache.kafka.server.util.CommandDefaultOptions; @@ -30,15 +30,15 @@ public class ReassignPartitionsCommandOptions extends CommandDefaultOptions { private final OptionSpecBuilder listOpt; // Arguments - private final ArgumentAcceptingOptionSpec bootstrapServerOpt; - private final ArgumentAcceptingOptionSpec commandConfigOpt; - private final ArgumentAcceptingOptionSpec reassignmentJsonFileOpt; - private final ArgumentAcceptingOptionSpec topicsToMoveJsonFileOpt; - private final ArgumentAcceptingOptionSpec brokerListOpt; + private final OptionSpec bootstrapServerOpt; + private final OptionSpec commandConfigOpt; + private final OptionSpec reassignmentJsonFileOpt; + private final OptionSpec topicsToMoveJsonFileOpt; + private final OptionSpec brokerListOpt; private final OptionSpecBuilder disableRackAware; - private final ArgumentAcceptingOptionSpec interBrokerThrottleOpt; - private final ArgumentAcceptingOptionSpec replicaAlterLogDirsThrottleOpt; - private final ArgumentAcceptingOptionSpec timeoutOpt; + private final OptionSpec interBrokerThrottleOpt; + private final OptionSpec replicaAlterLogDirsThrottleOpt; + private final OptionSpec timeoutOpt; private final OptionSpecBuilder additionalOpt; private final OptionSpecBuilder preserveThrottlesOpt; @@ -131,23 +131,23 @@ public OptionSpecBuilder listOpt() { return listOpt; } - public ArgumentAcceptingOptionSpec bootstrapServerOpt() { + public OptionSpec bootstrapServerOpt() { return bootstrapServerOpt; } - public ArgumentAcceptingOptionSpec commandConfigOpt() { + public OptionSpec commandConfigOpt() { return commandConfigOpt; } - public ArgumentAcceptingOptionSpec reassignmentJsonFileOpt() { + public OptionSpec reassignmentJsonFileOpt() { return reassignmentJsonFileOpt; } - public ArgumentAcceptingOptionSpec topicsToMoveJsonFileOpt() { + public OptionSpec topicsToMoveJsonFileOpt() { return topicsToMoveJsonFileOpt; } - public ArgumentAcceptingOptionSpec brokerListOpt() { + public OptionSpec brokerListOpt() { return brokerListOpt; } @@ -155,15 +155,15 @@ public OptionSpecBuilder disableRackAware() { return disableRackAware; } - public ArgumentAcceptingOptionSpec interBrokerThrottleOpt() { + public OptionSpec interBrokerThrottleOpt() { return interBrokerThrottleOpt; } - public ArgumentAcceptingOptionSpec replicaAlterLogDirsThrottleOpt() { + public OptionSpec replicaAlterLogDirsThrottleOpt() { return replicaAlterLogDirsThrottleOpt; } - public ArgumentAcceptingOptionSpec timeoutOpt() { + public OptionSpec timeoutOpt() { return timeoutOpt; } From d2bd8ac2074b02ef2cb2a77dc14dada12748dcf8 Mon Sep 17 00:00:00 2001 From: nizhikov Date: Sun, 9 Jul 2023 19:04:22 +0300 Subject: [PATCH 08/78] KAFKA-14595 Move new classes to tools --- .../ReassignPartitionsCommandOptions.java | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommandOptions.java b/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommandOptions.java index b1382a8490411..fd2b8d848e630 100644 --- a/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommandOptions.java +++ b/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommandOptions.java @@ -23,11 +23,11 @@ public class ReassignPartitionsCommandOptions extends CommandDefaultOptions { // Actions - private final OptionSpecBuilder verifyOpt; - private final OptionSpecBuilder generateOpt; - private final OptionSpecBuilder executeOpt; - private final OptionSpecBuilder cancelOpt; - private final OptionSpecBuilder listOpt; + private final OptionSpec verifyOpt; + private final OptionSpec generateOpt; + private final OptionSpec executeOpt; + private final OptionSpec cancelOpt; + private final OptionSpec listOpt; // Arguments private final OptionSpec bootstrapServerOpt; @@ -39,8 +39,8 @@ public class ReassignPartitionsCommandOptions extends CommandDefaultOptions { private final OptionSpec interBrokerThrottleOpt; private final OptionSpec replicaAlterLogDirsThrottleOpt; private final OptionSpec timeoutOpt; - private final OptionSpecBuilder additionalOpt; - private final OptionSpecBuilder preserveThrottlesOpt; + private final OptionSpec additionalOpt; + private final OptionSpec preserveThrottlesOpt; public ReassignPartitionsCommandOptions(String[] args) { super(args); @@ -111,23 +111,23 @@ public ReassignPartitionsCommandOptions(String[] args) { options = parser.parse(args); } - public OptionSpecBuilder verifyOpt() { + public OptionSpec verifyOpt() { return verifyOpt; } - public OptionSpecBuilder generateOpt() { + public OptionSpec generateOpt() { return generateOpt; } - public OptionSpecBuilder executeOpt() { + public OptionSpec executeOpt() { return executeOpt; } - public OptionSpecBuilder cancelOpt() { + public OptionSpec cancelOpt() { return cancelOpt; } - public OptionSpecBuilder listOpt() { + public OptionSpec listOpt() { return listOpt; } @@ -167,11 +167,11 @@ public OptionSpec timeoutOpt() { return timeoutOpt; } - public OptionSpecBuilder additionalOpt() { + public OptionSpec additionalOpt() { return additionalOpt; } - public OptionSpecBuilder preserveThrottlesOpt() { + public OptionSpec preserveThrottlesOpt() { return preserveThrottlesOpt; } } From de2e6603d99ced648b61c8c21e5a2246166f7382 Mon Sep 17 00:00:00 2001 From: nizhikov Date: Sun, 9 Jul 2023 19:05:52 +0300 Subject: [PATCH 09/78] KAFKA-14595 Move new classes to tools --- .../tools/reassign/ReassignPartitionsCommandOptions.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommandOptions.java b/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommandOptions.java index fd2b8d848e630..b70f167f77695 100644 --- a/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommandOptions.java +++ b/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommandOptions.java @@ -18,7 +18,6 @@ package org.apache.kafka.tools.reassign; import joptsimple.OptionSpec; -import joptsimple.OptionSpecBuilder; import org.apache.kafka.server.util.CommandDefaultOptions; public class ReassignPartitionsCommandOptions extends CommandDefaultOptions { @@ -35,7 +34,7 @@ public class ReassignPartitionsCommandOptions extends CommandDefaultOptions { private final OptionSpec reassignmentJsonFileOpt; private final OptionSpec topicsToMoveJsonFileOpt; private final OptionSpec brokerListOpt; - private final OptionSpecBuilder disableRackAware; + private final OptionSpec disableRackAware; private final OptionSpec interBrokerThrottleOpt; private final OptionSpec replicaAlterLogDirsThrottleOpt; private final OptionSpec timeoutOpt; @@ -151,7 +150,7 @@ public OptionSpec brokerListOpt() { return brokerListOpt; } - public OptionSpecBuilder disableRackAware() { + public OptionSpec disableRackAware() { return disableRackAware; } From fa4b0b0d9c4b151d504231229cda06c29ca4f68f Mon Sep 17 00:00:00 2001 From: nizhikov Date: Sat, 22 Jul 2023 12:30:15 +0300 Subject: [PATCH 10/78] KAFKA-14595 Code review fixes --- .../kafka/tools/reassign/ActiveMoveState.java | 44 ++--------------- .../tools/reassign/CancelledMoveState.java | 39 ++------------- .../tools/reassign/CompletedMoveState.java | 30 +----------- .../kafka/tools/reassign/LogDirMoveState.java | 2 +- .../reassign/MissingLogDirMoveState.java | 30 +----------- .../reassign/MissingReplicaMoveState.java | 30 +----------- .../kafka/tools/reassign/PartitionMove.java | 36 ++------------ .../reassign/PartitionReassignmentState.java | 43 ++--------------- .../reassign/VerifyAssignmentResult.java | 48 ++----------------- 9 files changed, 26 insertions(+), 276 deletions(-) diff --git a/tools/src/main/java/org/apache/kafka/tools/reassign/ActiveMoveState.java b/tools/src/main/java/org/apache/kafka/tools/reassign/ActiveMoveState.java index 271b3f6926b5f..fa9af20e59378 100644 --- a/tools/src/main/java/org/apache/kafka/tools/reassign/ActiveMoveState.java +++ b/tools/src/main/java/org/apache/kafka/tools/reassign/ActiveMoveState.java @@ -17,17 +17,15 @@ package org.apache.kafka.tools.reassign; -import java.util.Objects; - /** * A replica log directory move state where the move is in progress. */ -public final class ActiveMoveState implements LogDirMoveState { - private final String currentLogDir; +final class ActiveMoveState implements LogDirMoveState { + public final String currentLogDir; - private final String targetLogDir; + public final String targetLogDir; - private final String futureLogDir; + public final String futureLogDir; /** * @param currentLogDir The current log directory. @@ -40,42 +38,8 @@ public ActiveMoveState(String currentLogDir, String targetLogDir, String futureL this.futureLogDir = futureLogDir; } - public String currentLogDir() { - return currentLogDir; - } - - public String targetLogDir() { - return targetLogDir; - } - - public String futureLogDir() { - return futureLogDir; - } - @Override public boolean done() { return false; } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - ActiveMoveState that = (ActiveMoveState) o; - return Objects.equals(currentLogDir, that.currentLogDir) && Objects.equals(targetLogDir, that.targetLogDir) && Objects.equals(futureLogDir, that.futureLogDir); - } - - @Override - public int hashCode() { - return Objects.hash(currentLogDir, targetLogDir, futureLogDir); - } - - @Override - public String toString() { - return "ActiveMoveState{" + - "currentLogDir='" + currentLogDir + '\'' + - ", targetLogDir='" + targetLogDir + '\'' + - ", futureLogDir='" + futureLogDir + '\'' + - '}'; - } } diff --git a/tools/src/main/java/org/apache/kafka/tools/reassign/CancelledMoveState.java b/tools/src/main/java/org/apache/kafka/tools/reassign/CancelledMoveState.java index fec8036d0910c..5c4fbf98f5312 100644 --- a/tools/src/main/java/org/apache/kafka/tools/reassign/CancelledMoveState.java +++ b/tools/src/main/java/org/apache/kafka/tools/reassign/CancelledMoveState.java @@ -17,16 +17,14 @@ package org.apache.kafka.tools.reassign; -import java.util.Objects; - /** * A replica log directory move state where there is no move in progress, but we did not * reach the target log directory. */ -public final class CancelledMoveState implements LogDirMoveState { - private final String currentLogDir; +final class CancelledMoveState implements LogDirMoveState { + public final String currentLogDir; - private final String targetLogDir; + public final String targetLogDir; /** * @param currentLogDir The current log directory. @@ -37,37 +35,8 @@ public CancelledMoveState(String currentLogDir, String targetLogDir) { this.targetLogDir = targetLogDir; } - public String currentLogDir() { - return currentLogDir; - } - - public String targetLogDir() { - return targetLogDir; - } - @Override public boolean done() { return true; } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - CancelledMoveState that = (CancelledMoveState) o; - return Objects.equals(currentLogDir, that.currentLogDir) && Objects.equals(targetLogDir, that.targetLogDir); - } - - @Override - public int hashCode() { - return Objects.hash(currentLogDir, targetLogDir); - } - - @Override - public String toString() { - return "CancelledMoveState{" + - "currentLogDir='" + currentLogDir + '\'' + - ", targetLogDir='" + targetLogDir + '\'' + - '}'; - } -} \ No newline at end of file +} diff --git a/tools/src/main/java/org/apache/kafka/tools/reassign/CompletedMoveState.java b/tools/src/main/java/org/apache/kafka/tools/reassign/CompletedMoveState.java index ce936e8c6d7af..af18195db4542 100644 --- a/tools/src/main/java/org/apache/kafka/tools/reassign/CompletedMoveState.java +++ b/tools/src/main/java/org/apache/kafka/tools/reassign/CompletedMoveState.java @@ -17,13 +17,11 @@ package org.apache.kafka.tools.reassign; -import java.util.Objects; - /** * The completed replica log directory move state. */ -public final class CompletedMoveState implements LogDirMoveState { - private final String targetLogDir; +final class CompletedMoveState implements LogDirMoveState { + public final String targetLogDir; /** * @param targetLogDir The log directory that we wanted the replica to move to. @@ -32,32 +30,8 @@ public CompletedMoveState(String targetLogDir) { this.targetLogDir = targetLogDir; } - public String targetLogDir() { - return targetLogDir; - } - @Override public boolean done() { return true; } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - CompletedMoveState that = (CompletedMoveState) o; - return Objects.equals(targetLogDir, that.targetLogDir); - } - - @Override - public int hashCode() { - return Objects.hash(targetLogDir); - } - - @Override - public String toString() { - return "CompletedMoveState{" + - "targetLogDir='" + targetLogDir + '\'' + - '}'; - } } diff --git a/tools/src/main/java/org/apache/kafka/tools/reassign/LogDirMoveState.java b/tools/src/main/java/org/apache/kafka/tools/reassign/LogDirMoveState.java index deb2f64e32161..983d6af016761 100644 --- a/tools/src/main/java/org/apache/kafka/tools/reassign/LogDirMoveState.java +++ b/tools/src/main/java/org/apache/kafka/tools/reassign/LogDirMoveState.java @@ -20,7 +20,7 @@ /** * The state of a replica log directory movement. */ -public interface LogDirMoveState { +interface LogDirMoveState { /** * True if the move is done without errors. */ diff --git a/tools/src/main/java/org/apache/kafka/tools/reassign/MissingLogDirMoveState.java b/tools/src/main/java/org/apache/kafka/tools/reassign/MissingLogDirMoveState.java index 0143a5b035f82..d647e045ef3a8 100644 --- a/tools/src/main/java/org/apache/kafka/tools/reassign/MissingLogDirMoveState.java +++ b/tools/src/main/java/org/apache/kafka/tools/reassign/MissingLogDirMoveState.java @@ -17,13 +17,11 @@ package org.apache.kafka.tools.reassign; -import java.util.Objects; - /** * A replica log directory move state where the source replica is missing. */ -public final class MissingLogDirMoveState implements LogDirMoveState { - private final String targetLogDir; +final class MissingLogDirMoveState implements LogDirMoveState { + public final String targetLogDir; /** * @param targetLogDir The log directory that we wanted the replica to move to. @@ -32,32 +30,8 @@ public MissingLogDirMoveState(String targetLogDir) { this.targetLogDir = targetLogDir; } - public String targetLogDir() { - return targetLogDir; - } - @Override public boolean done() { return false; } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - MissingLogDirMoveState that = (MissingLogDirMoveState) o; - return Objects.equals(targetLogDir, that.targetLogDir); - } - - @Override - public int hashCode() { - return Objects.hash(targetLogDir); - } - - @Override - public String toString() { - return "MissingLogDirMoveState{" + - "targetLogDir='" + targetLogDir + '\'' + - '}'; - } } diff --git a/tools/src/main/java/org/apache/kafka/tools/reassign/MissingReplicaMoveState.java b/tools/src/main/java/org/apache/kafka/tools/reassign/MissingReplicaMoveState.java index 41d0356ec4756..04c978f6b79df 100644 --- a/tools/src/main/java/org/apache/kafka/tools/reassign/MissingReplicaMoveState.java +++ b/tools/src/main/java/org/apache/kafka/tools/reassign/MissingReplicaMoveState.java @@ -17,13 +17,11 @@ package org.apache.kafka.tools.reassign; -import java.util.Objects; - /** * A replica log directory move state where the source log directory is missing. */ -public final class MissingReplicaMoveState implements LogDirMoveState { - private final String targetLogDir; +final class MissingReplicaMoveState implements LogDirMoveState { + public final String targetLogDir; /** * @param targetLogDir The log directory that we wanted the replica to move to. @@ -32,32 +30,8 @@ public MissingReplicaMoveState(String targetLogDir) { this.targetLogDir = targetLogDir; } - public String targetLogDir() { - return targetLogDir; - } - @Override public boolean done() { return false; } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - MissingReplicaMoveState that = (MissingReplicaMoveState) o; - return Objects.equals(targetLogDir, that.targetLogDir); - } - - @Override - public int hashCode() { - return Objects.hash(targetLogDir); - } - - @Override - public String toString() { - return "MissingReplicaMoveState{" + - "targetLogDir='" + targetLogDir + '\'' + - '}'; - } } diff --git a/tools/src/main/java/org/apache/kafka/tools/reassign/PartitionMove.java b/tools/src/main/java/org/apache/kafka/tools/reassign/PartitionMove.java index ffd00056fdbdd..76c467ccf1859 100644 --- a/tools/src/main/java/org/apache/kafka/tools/reassign/PartitionMove.java +++ b/tools/src/main/java/org/apache/kafka/tools/reassign/PartitionMove.java @@ -17,16 +17,15 @@ package org.apache.kafka.tools.reassign; -import java.util.Objects; import java.util.Set; /** * A partition movement. The source and destination brokers may overlap. */ -public final class PartitionMove { - private final Set sources; +final class PartitionMove { + public final Set sources; - private final Set destinations; + public final Set destinations; /** * @param sources The source brokers. @@ -36,33 +35,4 @@ public PartitionMove(Set sources, Set destinations) { this.sources = sources; this.destinations = destinations; } - - public Set sources() { - return sources; - } - - public Set destinations() { - return destinations; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - PartitionMove that = (PartitionMove) o; - return Objects.equals(sources, that.sources) && Objects.equals(destinations, that.destinations); - } - - @Override - public int hashCode() { - return Objects.hash(sources, destinations); - } - - @Override - public String toString() { - return "PartitionMove{" + - "sources=" + sources + - ", destinations=" + destinations + - '}'; - } } diff --git a/tools/src/main/java/org/apache/kafka/tools/reassign/PartitionReassignmentState.java b/tools/src/main/java/org/apache/kafka/tools/reassign/PartitionReassignmentState.java index db6a49263cb05..a7e0f81086f8f 100644 --- a/tools/src/main/java/org/apache/kafka/tools/reassign/PartitionReassignmentState.java +++ b/tools/src/main/java/org/apache/kafka/tools/reassign/PartitionReassignmentState.java @@ -18,18 +18,17 @@ package org.apache.kafka.tools.reassign; import java.util.List; -import java.util.Objects; /** * The state of a partition reassignment. The current replicas and target replicas * may overlap. */ -public final class PartitionReassignmentState { - private final List currentReplicas; +final class PartitionReassignmentState { + public final List currentReplicas; - private final List targetReplicas; + public final List targetReplicas; - private final boolean done; + public final boolean done; /** * @param currentReplicas The current replicas. @@ -41,38 +40,4 @@ public PartitionReassignmentState(List currentReplicas, List t this.targetReplicas = targetReplicas; this.done = done; } - - public List currentReplicas() { - return currentReplicas; - } - - public List targetReplicas() { - return targetReplicas; - } - - public boolean done() { - return done; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - PartitionReassignmentState that = (PartitionReassignmentState) o; - return done == that.done && Objects.equals(currentReplicas, that.currentReplicas) && Objects.equals(targetReplicas, that.targetReplicas); - } - - @Override - public int hashCode() { - return Objects.hash(currentReplicas, targetReplicas, done); - } - - @Override - public String toString() { - return "PartitionReassignmentState{" + - "currentReplicas=" + currentReplicas + - ", targetReplicas=" + targetReplicas + - ", done=" + done + - '}'; - } } diff --git a/tools/src/main/java/org/apache/kafka/tools/reassign/VerifyAssignmentResult.java b/tools/src/main/java/org/apache/kafka/tools/reassign/VerifyAssignmentResult.java index 4061ab81c86dc..7721066ab0024 100644 --- a/tools/src/main/java/org/apache/kafka/tools/reassign/VerifyAssignmentResult.java +++ b/tools/src/main/java/org/apache/kafka/tools/reassign/VerifyAssignmentResult.java @@ -22,16 +22,15 @@ import java.util.Collections; import java.util.Map; -import java.util.Objects; /** * A result returned from verifyAssignment. */ public final class VerifyAssignmentResult { - private final Map partStates; - private final boolean partsOngoing; - private final Map moveStates; - private final boolean movesOngoing; + public final Map partStates; + public final boolean partsOngoing; + public final Map moveStates; + public final boolean movesOngoing; public VerifyAssignmentResult(Map partStates) { this(partStates, false, Collections.emptyMap(), false); @@ -54,43 +53,4 @@ public VerifyAssignmentResult( this.moveStates = moveStates; this.movesOngoing = movesOngoing; } - - public Map partStates() { - return partStates; - } - - public boolean partsOngoing() { - return partsOngoing; - } - - public Map moveStates() { - return moveStates; - } - - public boolean movesOngoing() { - return movesOngoing; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - VerifyAssignmentResult that = (VerifyAssignmentResult) o; - return partsOngoing == that.partsOngoing && movesOngoing == that.movesOngoing && Objects.equals(partStates, that.partStates) && Objects.equals(moveStates, that.moveStates); - } - - @Override - public int hashCode() { - return Objects.hash(partStates, partsOngoing, moveStates, movesOngoing); - } - - @Override - public String toString() { - return "VerifyAssignmentResult{" + - "partStates=" + partStates + - ", partsOngoing=" + partsOngoing + - ", moveStates=" + moveStates + - ", movesOngoing=" + movesOngoing + - '}'; - } } From 0ea355209358773d0b16f6559d2493153ae21bff Mon Sep 17 00:00:00 2001 From: nizhikov Date: Mon, 24 Jul 2023 10:53:23 +0300 Subject: [PATCH 11/78] KAFKA-14595 Stubs for all methods in scala version of command --- .../kafka/tools/reassign/BrokerMetadata.java | 31 + .../reassign/ReassignPartitionCommand.java | 753 ++++++++++++++++++ .../ReassignPartitionsCommandOptions.java | 32 +- .../apache/kafka/tools/reassign/Tuple.java | 49 ++ 4 files changed, 849 insertions(+), 16 deletions(-) create mode 100644 tools/src/main/java/org/apache/kafka/tools/reassign/BrokerMetadata.java create mode 100644 tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionCommand.java create mode 100644 tools/src/main/java/org/apache/kafka/tools/reassign/Tuple.java diff --git a/tools/src/main/java/org/apache/kafka/tools/reassign/BrokerMetadata.java b/tools/src/main/java/org/apache/kafka/tools/reassign/BrokerMetadata.java new file mode 100644 index 0000000000000..b6d416aff97bb --- /dev/null +++ b/tools/src/main/java/org/apache/kafka/tools/reassign/BrokerMetadata.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.tools.reassign; + +import java.util.Optional; + +public class BrokerMetadata { + public final int id; + + public final Optional rack; + + public BrokerMetadata(int id, Optional rack) { + this.id = id; + this.rack = rack; + } +} diff --git a/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionCommand.java b/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionCommand.java new file mode 100644 index 0000000000000..0ea01eee785c2 --- /dev/null +++ b/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionCommand.java @@ -0,0 +1,753 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.tools.reassign; + +import joptsimple.OptionSpec; +import org.apache.kafka.clients.admin.Admin; +import org.apache.kafka.clients.admin.AdminClientConfig; +import org.apache.kafka.clients.admin.PartitionReassignment; +import org.apache.kafka.clients.admin.TopicDescription; +import org.apache.kafka.common.KafkaFuture; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.TopicPartitionReplica; +import org.apache.kafka.common.utils.Exit; +import org.apache.kafka.common.utils.Time; +import org.apache.kafka.common.utils.Utils; +import org.apache.kafka.server.util.CommandLineUtils; +import org.apache.kafka.server.util.json.JsonValue; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * + */ +public class ReassignPartitionCommand { + private static final String AnyLogDir = "any"; + + private static final String helpText = "This tool helps to move topic partitions between replicas."; + + /** + * The earliest version of the partition reassignment JSON. We will default to this + * version if no other version number is given. + */ + public static final int EarliestVersion = 1; + + /** + * The earliest version of the JSON for each partition reassignment topic. We will + * default to this version if no other version number is given. + */ + public static final int EarliestTopicsJsonVersion = 1; + + // Throttles that are set at the level of an individual broker. + //DynamicConfig.Broker.LeaderReplicationThrottledRateProp + private static final String brokerLevelLeaderThrottle = "leader.replication.throttled.rate"; + //DynamicConfig.Broker.FollowerReplicationThrottledRateProp + private static final String brokerLevelFollowerThrottle = "follower.replication.throttled.rate"; + //DynamicConfig.Broker.ReplicaAlterLogDirsIoMaxBytesPerSecondProp + private static final String brokerLevelLogDirThrottle = "replica.alter.log.dirs.io.max.bytes.per.second"; + private static final List brokerLevelThrottles = Arrays.asList( + brokerLevelLeaderThrottle, + brokerLevelFollowerThrottle, + brokerLevelLogDirThrottle + ); + + // Throttles that are set at the level of an individual topic. + //LogConfig.LEADER_REPLICATION_THROTTLED_REPLICAS_CONFIG + private static final String topicLevelLeaderThrottle = "leader.replication.throttled.replicas"; + //LogConfig.FOLLOWER_REPLICATION_THROTTLED_REPLICAS_CONFIG + private static final String topicLevelFollowerThrottle = "follower.replication.throttled.replicas"; + private static final List topicLevelThrottles = Arrays.asList( + topicLevelLeaderThrottle, + topicLevelFollowerThrottle + ); + + private static final String cannotExecuteBecauseOfExistingMessage = "Cannot execute because " + + "there is an existing partition assignment. Use --additional to override this and " + + "create a new partition assignment in addition to the existing one. The --additional " + + "flag can also be used to change the throttle by resubmitting the current reassignment."; + + private static final String youMustRunVerifyPeriodicallyMessage = "Warning: You must run " + + "--verify periodically, until the reassignment completes, to ensure the throttle " + + "is removed."; + + public static void main(String[] args) { + ReassignPartitionsCommandOptions opts = validateAndParseArgs(args); + boolean failed = true; + Admin adminClient = null; + + try { + Properties props = opts.options.has(opts.commandConfigOpt) + ? Utils.loadProps(opts.options.valueOf(opts.commandConfigOpt)) + : new Properties(); + props.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, opts.options.valueOf(opts.bootstrapServerOpt)); + props.putIfAbsent(AdminClientConfig.CLIENT_ID_CONFIG, "reassign-partitions-tool"); + adminClient = Admin.create(props); + handleAction(adminClient, opts); + failed = false; + } catch (TerseReassignmentFailureException e) { + System.out.println(e.getMessage()); + } catch (Throwable e) { + System.out.println("Error: " + e.getMessage()); + System.out.println(Utils.stackTrace(e)); + } finally { + // It's good to do this after printing any error stack trace. + if (adminClient != null) { + adminClient.close(); + } + } + // If the command failed, exit with a non-zero exit code. + if (failed) { + Exit.exit(1); + } + } + + private static void handleAction(Admin adminClient, ReassignPartitionsCommandOptions opts) throws IOException { + if (opts.options.has(opts.verifyOpt)) { + verifyAssignment(adminClient, + Utils.readFileAsString(opts.options.valueOf(opts.reassignmentJsonFileOpt)), + opts.options.has(opts.preserveThrottlesOpt)); + } else if (opts.options.has(opts.generateOpt)) { + generateAssignment(adminClient, + Utils.readFileAsString(opts.options.valueOf(opts.topicsToMoveJsonFileOpt)), + opts.options.valueOf(opts.brokerListOpt), + !opts.options.has(opts.disableRackAware)); + } else if (opts.options.has(opts.executeOpt)) { + executeAssignment(adminClient, + opts.options.has(opts.additionalOpt), + Utils.readFileAsString(opts.options.valueOf(opts.reassignmentJsonFileOpt)), + opts.options.valueOf(opts.interBrokerThrottleOpt), + opts.options.valueOf(opts.replicaAlterLogDirsThrottleOpt), + opts.options.valueOf(opts.timeoutOpt), + Time.SYSTEM); + } else if (opts.options.has(opts.cancelOpt)) { + cancelAssignment(adminClient, + Utils.readFileAsString(opts.options.valueOf(opts.reassignmentJsonFileOpt)), + opts.options.has(opts.preserveThrottlesOpt), + opts.options.valueOf(opts.timeoutOpt), + Time.SYSTEM); + } else if (opts.options.has(opts.listOpt)) { + listReassignments(adminClient); + } else { + throw new RuntimeException("Unsupported action."); + } + } + + /** + * The entry point for the --verify command. + * + * @param adminClient The AdminClient to use. + * @param jsonString The JSON string to use for the topics and partitions to verify. + * @param preserveThrottles True if we should avoid changing topic or broker throttles. + * + * @return A result that is useful for testing. + */ + private static VerifyAssignmentResult verifyAssignment(Admin adminClient, String jsonString, Boolean preserveThrottles) { + return null; + } + + /** + * Verify the partition reassignments specified by the user. + * + * @param adminClient The AdminClient to use. + * @param targets The partition reassignments specified by the user. + * + * @return A tuple of the partition reassignment states, and a + * boolean which is true if there are no ongoing + * reassignments (including reassignments not described + * in the JSON file.) + */ + private static Tuple, Boolean> verifyPartitionAssignments(Admin adminClient, + List>> targets) { + return null; + } + + + private static boolean compareTopicPartition(TopicPartition a, TopicPartition b) { + return false; + } + + private static boolean compareTopicPartitionReplicas(TopicPartition a, TopicPartition b) { + return false; + } + + /** + * Convert partition reassignment states to a human-readable string. + * + * @param states A map from topic partitions to states. + * @return A string summarizing the partition reassignment states. + */ + private static String partitionReassignmentStatesToString(Map states) { + return null; + } + + /** + * Find the state of the specified partition reassignments. + * + * @param adminClient The Admin client to use. + * @param targetReassignments The reassignments we want to learn about. + * + * @return A tuple containing the reassignment states for each topic + * partition, plus whether there are any ongoing reassignments. + */ + private static Tuple, Boolean> findPartitionReassignmentStates(Admin adminClient, + List>> targetReassignments) { + return null; + } + + private static PartitionReassignmentState topicDescriptionFutureToState(int partition, + KafkaFuture future, + List targetReplicas) { + return null; + } + + /** + * Verify the replica reassignments specified by the user. + * + * @param adminClient The AdminClient to use. + * @param targetReassignments The replica reassignments specified by the user. + * + * @return A tuple of the replica states, and a boolean which is true + * if there are any ongoing replica moves. + * + * Note: Unlike in verifyPartitionAssignments, we will + * return false here even if there are unrelated ongoing + * reassignments. (We don't have an efficient API that + * returns all ongoing replica reassignments.) + */ + private static Tuple, Boolean> verifyReplicaMoves(Admin adminClient, + Map targetReassignments) { + return null; + } + + /** + * Find the state of the specified partition reassignments. + * + * @param adminClient The AdminClient to use. + * @param targetMoves The movements we want to learn about. The map is keyed + * by TopicPartitionReplica, and its values are target log + * directories. + * + * @return The states for each replica movement. + */ + private static MapfindLogDirMoveStates(Admin adminClient, + Map targetMoves) { + return null; + } + + /** + * Convert replica move states to a human-readable string. + * + * @param states A map from topic partition replicas to states. + * @return A tuple of a summary string, and a boolean describing + * whether there are any active replica moves. + */ + private static String replicaMoveStatesToString(Map states) { + return null; + } + + /** + * Clear all topic-level and broker-level throttles. + * + * @param adminClient The AdminClient to use. + * @param targetParts The target partitions loaded from the JSON file. + */ + private static void clearAllThrottles(Admin adminClient, List>> targetParts) { + + } + + /** + * Clear all throttles which have been set at the broker level. + * + * @param adminClient The AdminClient to use. + * @param brokers The brokers to clear the throttles for. + */ + private static void clearBrokerLevelThrottles(Admin adminClient, Set brokers) { + + } + + /** + * Clear the reassignment throttles for the specified topics. + * + * @param adminClient The AdminClient to use. + * @param topics The topics to clear the throttles for. + */ + private static void clearTopicLevelThrottles(Admin adminClient, Set topics) { + + } + + /** + * The entry point for the --generate command. + * + * @param adminClient The AdminClient to use. + * @param reassignmentJson The JSON string to use for the topics to reassign. + * @param brokerListString The comma-separated string of broker IDs to use. + * @param enableRackAwareness True if rack-awareness should be enabled. + * + * @return A tuple containing the proposed assignment and the + * current assignment. + */ + private static Tuple>, Map>> generateAssignment(Admin adminClient, + String reassignmentJson, + String brokerListString, + Boolean enableRackAwareness) { + return null; + } + + /** + * Calculate the new partition assignments to suggest in --generate. + * + * @param currentAssignment The current partition assignments. + * @param brokerMetadatas The rack information for each broker. + * + * @return A map from partitions to the proposed assignments for each. + */ + private static Map> calculateAssignment(Map> currentAssignment, + List brokerMetadatas) { + return null; + } + + private static Map describeTopics(Admin adminClient, + Set topics) { + return null; + } + + /** + * Get the current replica assignments for some topics. + * + * @param adminClient The AdminClient to use. + * @param topics The topics to get information about. + * @return A map from partitions to broker assignments. + * If any topic can't be found, an exception will be thrown. + */ + private static Map> getReplicaAssignmentForTopics(Admin adminClient, + List topics) { + return null; + } + + /** + * Get the current replica assignments for some partitions. + * + * @param adminClient The AdminClient to use. + * @param partitions The partitions to get information about. + * @return A map from partitions to broker assignments. + * If any topic can't be found, an exception will be thrown. + */ + private static Map> getReplicaAssignmentForPartitions(Admin adminClient, + Set partitions) { + return null; + } + + /** + * Find the rack information for some brokers. + * + * @param adminClient The AdminClient object. + * @param brokers The brokers to gather metadata about. + * @param enableRackAwareness True if we should return rack information, and throw an + * exception if it is inconsistent. + * + * @return The metadata for each broker that was found. + * Brokers that were not found will be omitted. + */ + private static List getBrokerMetadata(Admin adminClient, List brokers, boolean enableRackAwareness) { + return null; + } + + /** + * Parse and validate data gathered from the command-line for --generate + * In particular, we parse the JSON and validate that duplicate brokers and + * topics don't appear. + * + * @param reassignmentJson The JSON passed to --generate . + * @param brokerList A list of brokers passed to --generate. + * + * @return A tuple of brokers to reassign, topics to reassign + */ + private static Tuple, List> parseGenerateAssignmentArgs(String reassignmentJson, + String brokerList) { + return null; + } + + /** + * The entry point for the --execute and --execute-additional commands. + * + * @param adminClient The AdminClient to use. + * @param additional Whether --additional was passed. + * @param reassignmentJson The JSON string to use for the topics to reassign. + * @param interBrokerThrottle The inter-broker throttle to use, or a negative + * number to skip using a throttle. + * @param logDirThrottle The replica log directory throttle to use, or a + * negative number to skip using a throttle. + * @param timeoutMs The maximum time in ms to wait for log directory + * replica assignment to begin. + * @param time The Time object to use. + */ + private static void executeAssignment(Admin adminClient, + Boolean additional, + String reassignmentJson, + Long interBrokerThrottle, + Long logDirThrottle, + Long timeoutMs, + Time time) { + + } + + /** + * Execute some partition log directory movements. + * + * @param adminClient The AdminClient to use. + * @param proposedReplicas A map from TopicPartitionReplicas to the + * directories to move them to. + * @param timeoutMs The maximum time in ms to wait for log directory + * replica assignment to begin. + * @param time The Time object to use. + */ + private static void executeMoves(Admin adminClient, + Map proposedReplicas, + Long timeoutMs, + Time time) { + + } + + /** + * Entry point for the --list command. + * + * @param adminClient The AdminClient to use. + */ + private static void listReassignments(Admin adminClient) { + + } + + /** + * Convert the current partition reassignments to text. + * + * @param adminClient The AdminClient to use. + * @return A string describing the current partition reassignments. + */ + private static String curReassignmentsToString(Admin adminClient) { + return null; + } + + /** + * Verify that all the brokers in an assignment exist. + * + * @param adminClient The AdminClient to use. + * @param brokers The broker IDs to verify. + */ + private static void verifyBrokerIds(Admin adminClient, Set brokers) { + + } + + /** + * Return the string which we want to print to describe the current partition assignment. + * + * @param proposedParts The proposed partition assignment. + * @param currentParts The current partition assignment. + * + * @return The string to print. We will only print information about + * partitions that appear in the proposed partition assignment. + */ + private static String currentPartitionReplicaAssignmentToString(Map> proposedParts, + Map> currentParts) { + return null; + } + + /** + * Execute the given partition reassignments. + * + * @param adminClient The admin client object to use. + * @param reassignments A map from topic names to target replica assignments. + * @return A map from partition objects to error strings. + */ + private static Map alterPartitionReassignments(Admin adminClient, + Map> reassignments) { + return null; + } + + /** + * Cancel the given partition reassignments. + * + * @param adminClient The admin client object to use. + * @param reassignments The partition reassignments to cancel. + * @return A map from partition objects to error strings. + */ + private static Map cancelPartitionReassignments(Admin adminClient, + Set reassignments) { + return null; + } + + /** + * Compute the in progress partition move from the current reassignments. + * @param currentReassignments All replicas, adding replicas and removing replicas of target partitions + */ + private static Map> calculateCurrentMoveMap(Map currentReassignments) { + return null; + } + + /** + * Calculate the global map of all partitions that are moving. + * + * @param currentReassignments The currently active reassignments. + * @param proposedParts The proposed location of the partitions (destinations replicas only). + * @param currentParts The current location of the partitions that we are + * proposing to move. + * @return A map from topic name to partition map. + * The partition map is keyed on partition index and contains + * the movements for that partition. + */ + private static Map> calculateProposedMoveMap(Map currentReassignments, + Map> proposedParts, + Map> currentParts) { + return null; + } + + /** + * Calculate the leader throttle configurations to use. + * + * @param moveMap The movements. + * @return A map from topic names to leader throttle configurations. + */ + private static Map calculateLeaderThrottles(Map> moveMap) { + return null; + } + + /** + * Calculate the follower throttle configurations to use. + * + * @param moveMap The movements. + * @return A map from topic names to follower throttle configurations. + */ + private static Map calculateFollowerThrottles(Map> moveMap) { + return null; + } + + /** + * Calculate all the brokers which are involved in the given partition reassignments. + * + * @param moveMap The partition movements. + * @return A set of all the brokers involved. + */ + private static List calculateReassigningBrokers(Map> moveMap) { + return null; + } + + /** + * Calculate all the brokers which are involved in the given directory movements. + * + * @param replicaMoves The replica movements. + * @return A set of all the brokers involved. + */ + private static Set calculateMovingBrokers(Set replicaMoves) { + return null; + } + + /** + * Modify the topic configurations that control inter-broker throttling. + * + * @param adminClient The adminClient object to use. + * @param leaderThrottles A map from topic names to leader throttle configurations. + * @param followerThrottles A map from topic names to follower throttle configurations. + */ + private static void modifyTopicThrottles(Admin adminClient, + Map leaderThrottles, + Map followerThrottles) { + } + + private static void modifyReassignmentThrottle(Admin admin, Map> moveMap, Long interBrokerThrottle) { + + } + + /** + * Modify the leader/follower replication throttles for a set of brokers. + * + * @param adminClient The Admin instance to use + * @param reassigningBrokers The set of brokers involved in the reassignment + * @param interBrokerThrottle The new throttle (ignored if less than 0) + */ + private static void modifyInterBrokerThrottle(Admin adminClient, + Set reassigningBrokers, + Long interBrokerThrottle) { + + } + + /** + * Modify the log dir reassignment throttle for a set of brokers. + * + * @param admin The Admin instance to use + * @param movingBrokers The set of broker to alter the throttle of + * @param logDirThrottle The new throttle (ignored if less than 0) + */ + private static void modifyLogDirThrottle(Admin admin, + Set movingBrokers, + Long logDirThrottle) { + + } + + /** + * Parse the reassignment JSON string passed to the --execute command. + * + * @param reassignmentJson The JSON string. + * @return A tuple of the partitions to be reassigned and the replicas + * to be reassigned. + */ + private static Tuple>, Map> parseExecuteAssignmentArgs(String reassignmentJson) { + return null; + } + + /** + * The entry point for the --cancel command. + * + * @param adminClient The AdminClient to use. + * @param jsonString The JSON string to use for the topics and partitions to cancel. + * @param preserveThrottles True if we should avoid changing topic or broker throttles. + * @param timeoutMs The maximum time in ms to wait for log directory + * replica assignment to begin. + * @param time The Time object to use. + * + * @return A tuple of the partition reassignments that were cancelled, + * and the replica movements that were cancelled. + */ + private static Tuple, Set> cancelAssignment(Admin adminClient, + String jsonString, + Boolean preserveThrottles, + Long timeoutMs, + Time time) { + return null; + } + + private static String formatAsReassignmentJson(Map> partitionsToBeReassigned, + Map replicaLogDirAssignment) { + return null; + } + + private List parseTopicsData(String jsonData) { + return null; + } + + private static List parseTopicsData(int version, JsonValue js) { + return null; + } + + private static Tuple>>, Map> parsePartitionReassignmentData(String jsonData) { + return null; + } + + // Parses without deduplicating keys so the data can be checked before allowing reassignment to proceed + private static Tuple>>, Map> parsePartitionReassignmentData(int version, JsonValue jsonData) { + return null; + } + + + private static ReassignPartitionsCommandOptions validateAndParseArgs(String[] args) { + ReassignPartitionsCommandOptions opts = new ReassignPartitionsCommandOptions(args); + + CommandLineUtils.maybePrintHelpOrVersion(opts, helpText); + + // Determine which action we should perform. + List> validActions = Arrays.asList(opts.generateOpt, opts.executeOpt, opts.verifyOpt, + opts.cancelOpt, opts.listOpt); + + List> allActions = validActions.stream() + .filter(a -> opts.options.has(a)) + .collect(Collectors.toList()); + + if (allActions.size() != 1) { + CommandLineUtils.printUsageAndExit(opts.parser, String.format("Command must include exactly one action: %s", + validActions.stream().map(a -> "--" + a.options().get(0)).collect(Collectors.joining(", ")))); + } + + OptionSpec action = allActions.get(0); + + if (!opts.options.has(opts.bootstrapServerOpt)) + CommandLineUtils.printUsageAndExit(opts.parser, "Please specify --bootstrap-server"); + + // Make sure that we have all the required arguments for our action. + Map, List>> requiredArgs = new HashMap<>(); + + requiredArgs.put(opts.verifyOpt, Collections.singletonList( + opts.reassignmentJsonFileOpt + )); + requiredArgs.put(opts.generateOpt, Arrays.asList( + opts.topicsToMoveJsonFileOpt, + opts.brokerListOpt + )); + requiredArgs.put(opts.executeOpt, Collections.singletonList( + opts.reassignmentJsonFileOpt + )); + requiredArgs.put(opts.cancelOpt, Collections.singletonList( + opts.reassignmentJsonFileOpt + )); + requiredArgs.put(opts.listOpt, Collections.emptyList()); + + CommandLineUtils.checkRequiredArgs(opts.parser, opts.options, requiredArgs.get(action).toArray(new OptionSpec[0])); + + Map, List>> permittedArgs = new HashMap<>(); + + permittedArgs.put(opts.verifyOpt, Arrays.asList( + opts.bootstrapServerOpt, + opts.commandConfigOpt, + opts.preserveThrottlesOpt + )); + permittedArgs.put(opts.generateOpt, Arrays.asList( + opts.bootstrapServerOpt, + opts.brokerListOpt, + opts.commandConfigOpt, + opts.disableRackAware + )); + permittedArgs.put(opts.executeOpt, Arrays.asList( + opts.additionalOpt, + opts.bootstrapServerOpt, + opts.commandConfigOpt, + opts.interBrokerThrottleOpt, + opts.replicaAlterLogDirsThrottleOpt, + opts.timeoutOpt + )); + permittedArgs.put(opts.cancelOpt, Arrays.asList( + opts.bootstrapServerOpt, + opts.commandConfigOpt, + opts.preserveThrottlesOpt, + opts.timeoutOpt + )); + permittedArgs.put(opts.listOpt, Arrays.asList( + opts.bootstrapServerOpt, + opts.commandConfigOpt + )); + + opts.options.specs().forEach(opt -> { + if (!opt.equals(action) && + !requiredArgs.getOrDefault(action, Collections.emptyList()).contains(opt) && + !permittedArgs.getOrDefault(action, Collections.emptyList()).contains(opt)) { + CommandLineUtils.printUsageAndExit(opts.parser, + String.format("Option \"%s\" can't be used with action \"%s\"", opt, action)); + } + }); + + return opts; + } + + private static Set alterReplicaLogDirs(Admin adminClient, Map assignment) { + return null; + } +} diff --git a/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommandOptions.java b/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommandOptions.java index b70f167f77695..d0e97c036dda2 100644 --- a/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommandOptions.java +++ b/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommandOptions.java @@ -22,24 +22,24 @@ public class ReassignPartitionsCommandOptions extends CommandDefaultOptions { // Actions - private final OptionSpec verifyOpt; - private final OptionSpec generateOpt; - private final OptionSpec executeOpt; - private final OptionSpec cancelOpt; - private final OptionSpec listOpt; + final OptionSpec verifyOpt; + final OptionSpec generateOpt; + final OptionSpec executeOpt; + final OptionSpec cancelOpt; + final OptionSpec listOpt; // Arguments - private final OptionSpec bootstrapServerOpt; - private final OptionSpec commandConfigOpt; - private final OptionSpec reassignmentJsonFileOpt; - private final OptionSpec topicsToMoveJsonFileOpt; - private final OptionSpec brokerListOpt; - private final OptionSpec disableRackAware; - private final OptionSpec interBrokerThrottleOpt; - private final OptionSpec replicaAlterLogDirsThrottleOpt; - private final OptionSpec timeoutOpt; - private final OptionSpec additionalOpt; - private final OptionSpec preserveThrottlesOpt; + final OptionSpec bootstrapServerOpt; + final OptionSpec commandConfigOpt; + final OptionSpec reassignmentJsonFileOpt; + final OptionSpec topicsToMoveJsonFileOpt; + final OptionSpec brokerListOpt; + final OptionSpec disableRackAware; + final OptionSpec interBrokerThrottleOpt; + final OptionSpec replicaAlterLogDirsThrottleOpt; + final OptionSpec timeoutOpt; + final OptionSpec additionalOpt; + final OptionSpec preserveThrottlesOpt; public ReassignPartitionsCommandOptions(String[] args) { super(args); diff --git a/tools/src/main/java/org/apache/kafka/tools/reassign/Tuple.java b/tools/src/main/java/org/apache/kafka/tools/reassign/Tuple.java new file mode 100644 index 0000000000000..6606bfe7d3866 --- /dev/null +++ b/tools/src/main/java/org/apache/kafka/tools/reassign/Tuple.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.tools.reassign; + +import java.util.Objects; + +public final class Tuple { + public final V1 v1; + + public final V2 v2; + + public Tuple(V1 v1, V2 v2) { + this.v1 = v1; + this.v2 = v2; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Tuple tuple = (Tuple) o; + return Objects.equals(v1, tuple.v1) && Objects.equals(v2, tuple.v2); + } + + @Override + public int hashCode() { + return Objects.hash(v1, v2); + } + + @Override + public String toString() { + return "Tuple{v1=" + v1 + ", v2=" + v2 + '}'; + } +} From 4924947f2e0655e883d41fcb96556d73fe1ceb3c Mon Sep 17 00:00:00 2001 From: nizhikov Date: Mon, 24 Jul 2023 21:38:25 +0300 Subject: [PATCH 12/78] KAFKA-14595 WIP --- .../org/apache/kafka/tools/ToolsUtils.java | 13 + .../reassign/ReassignPartitionCommand.java | 379 ++++++++++++++++-- 2 files changed, 353 insertions(+), 39 deletions(-) diff --git a/tools/src/main/java/org/apache/kafka/tools/ToolsUtils.java b/tools/src/main/java/org/apache/kafka/tools/ToolsUtils.java index 1a391f33d673a..c5f36c22629f1 100644 --- a/tools/src/main/java/org/apache/kafka/tools/ToolsUtils.java +++ b/tools/src/main/java/org/apache/kafka/tools/ToolsUtils.java @@ -20,8 +20,11 @@ import org.apache.kafka.common.MetricName; import java.io.PrintStream; +import java.util.Collection; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.TreeMap; import java.util.stream.Collectors; @@ -99,4 +102,14 @@ public static void prettyPrintTable( printRow(columnLengths, headers, out); rows.forEach(row -> printRow(columnLengths, row, out)); } + + public static Set duplicates(Collection s) { + Set duplicates = new HashSet<>(); + Set elements = new HashSet<>(); + for (T t : s) { + if (!elements.add(t)) + duplicates.add(t); + } + return duplicates; + } } diff --git a/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionCommand.java b/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionCommand.java index 0ea01eee785c2..b0ccc1e6dafd0 100644 --- a/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionCommand.java +++ b/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionCommand.java @@ -20,25 +20,40 @@ import joptsimple.OptionSpec; import org.apache.kafka.clients.admin.Admin; import org.apache.kafka.clients.admin.AdminClientConfig; +import org.apache.kafka.clients.admin.AlterConfigOp; +import org.apache.kafka.clients.admin.ConfigEntry; +import org.apache.kafka.clients.admin.DescribeReplicaLogDirsResult; import org.apache.kafka.clients.admin.PartitionReassignment; import org.apache.kafka.clients.admin.TopicDescription; import org.apache.kafka.common.KafkaFuture; +import org.apache.kafka.common.Node; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.TopicPartitionReplica; +import org.apache.kafka.common.config.ConfigResource; +import org.apache.kafka.common.errors.UnknownTopicOrPartitionException; import org.apache.kafka.common.utils.Exit; import org.apache.kafka.common.utils.Time; import org.apache.kafka.common.utils.Utils; +import org.apache.kafka.server.common.AdminCommandFailedException; +import org.apache.kafka.server.common.AdminOperationException; import org.apache.kafka.server.util.CommandLineUtils; import org.apache.kafka.server.util.json.JsonValue; +import org.apache.kafka.tools.ToolsUtils; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.Optional; import java.util.Properties; import java.util.Set; +import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; /** @@ -124,7 +139,7 @@ public static void main(String[] args) { } } - private static void handleAction(Admin adminClient, ReassignPartitionsCommandOptions opts) throws IOException { + private static void handleAction(Admin adminClient, ReassignPartitionsCommandOptions opts) throws IOException, ExecutionException, InterruptedException { if (opts.options.has(opts.verifyOpt)) { verifyAssignment(adminClient, Utils.readFileAsString(opts.options.valueOf(opts.reassignmentJsonFileOpt)), @@ -164,8 +179,31 @@ private static void handleAction(Admin adminClient, ReassignPartitionsCommandOpt * * @return A result that is useful for testing. */ - private static VerifyAssignmentResult verifyAssignment(Admin adminClient, String jsonString, Boolean preserveThrottles) { - return null; + private static VerifyAssignmentResult verifyAssignment(Admin adminClient, String jsonString, Boolean preserveThrottles) throws ExecutionException, InterruptedException { + Tuple>>, Map> t0 = parsePartitionReassignmentData(jsonString); + + List>> targetParts = t0.v1; + Map targetLogDirs = t0.v2; + + Tuple, Boolean> t1 = verifyPartitionAssignments(adminClient, targetParts); + + Map partStates = t1.v1; + Boolean partsOngoing = t1.v2; + + Tuple, Boolean> t2 = verifyReplicaMoves(adminClient, targetLogDirs); + + Map moveStates = t2.v1; + Boolean movesOngoing = t2.v2; + + if (!partsOngoing && !movesOngoing && !preserveThrottles) { + // If the partition assignments and replica assignments are done, clear any throttles + // that were set. We have to clear all throttles, because we don't have enough + // information to know all of the source brokers that might have been involved in the + // previous reassignments. + clearAllThrottles(adminClient, targetParts); + } + + return new VerifyAssignmentResult(partStates, partsOngoing, moveStates, movesOngoing); } /** @@ -180,17 +218,29 @@ private static VerifyAssignmentResult verifyAssignment(Admin adminClient, String * in the JSON file.) */ private static Tuple, Boolean> verifyPartitionAssignments(Admin adminClient, - List>> targets) { - return null; - } + List>> targets) throws ExecutionException, InterruptedException { + Tuple, Boolean> t0 = findPartitionReassignmentStates(adminClient, targets); + Map partStates = t0.v1; + Boolean partsOngoing = t0.v2; - private static boolean compareTopicPartition(TopicPartition a, TopicPartition b) { - return false; + System.out.print(partitionReassignmentStatesToString(partStates)); + return new Tuple<>(partStates, partsOngoing); } - private static boolean compareTopicPartitionReplicas(TopicPartition a, TopicPartition b) { - return false; + private static int compareTopicPartitions(TopicPartition a, TopicPartition b) { + int topicOrder = Objects.compare(a.topic(), b.topic(), String::compareTo); + return topicOrder == 0 ? Integer.compare(a.partition(), b.partition()) : topicOrder; + } + + private static int compareTopicPartitionReplicas(TopicPartitionReplica a, TopicPartitionReplica b) { + int brokerOrder = Integer.compare(a.brokerId(), b.brokerId()); + + if (brokerOrder != 0) + return brokerOrder; + + int topicOrder = Objects.compare(a.topic(), b.topic(), String::compareTo); + return topicOrder == 0 ? Integer.compare(a.partition(), b.partition()) : topicOrder; } /** @@ -200,7 +250,26 @@ private static boolean compareTopicPartitionReplicas(TopicPartition a, TopicPart * @return A string summarizing the partition reassignment states. */ private static String partitionReassignmentStatesToString(Map states) { - return null; + List bld = new ArrayList<>(); + bld.add("Status of partition reassignment:"); + states.keySet().stream().sorted(ReassignPartitionCommand::compareTopicPartitions).forEach(topicPartition -> { + PartitionReassignmentState state = states.get(topicPartition); + if (state.done) { + if (state.currentReplicas.equals(state.targetReplicas)) { + bld.add(String.format("Reassignment of partition %s is completed.", topicPartition)); + } else { + String currentReplicaStr = state.currentReplicas.stream().map(String::valueOf).collect(Collectors.joining(",")); + String targetReplicaStr = state.targetReplicas.stream().map(String::valueOf).collect(Collectors.joining(",")); + + bld.add("There is no active reassignment of partition " + topicPartition + ", " + + "but replica set is " + currentReplicaStr + " rather than " + + targetReplicaStr + "."); + } + } else { + bld.add(String.format("Reassignment of partition %s is still in progress.", topicPartition)); + } + }); + return bld.stream().collect(Collectors.joining(System.lineSeparator())); } /** @@ -213,14 +282,82 @@ private static String partitionReassignmentStatesToString(Map, Boolean> findPartitionReassignmentStates(Admin adminClient, - List>> targetReassignments) { - return null; + List>> targetReassignments) throws ExecutionException, InterruptedException { + Map currentReassignments = adminClient. + listPartitionReassignments().reassignments().get(); + + List>> foundReassignments = new ArrayList<>(); + List>> notFoundReassignments = new ArrayList<>(); + + targetReassignments.forEach(reassignment -> { + if (targetReassignments.contains(reassignment)) + foundReassignments.add(reassignment); + else + notFoundReassignments.add(reassignment); + }); + + List> foundResults = foundReassignments.stream().map(e -> { + TopicPartition part = e.v1; + List targetReplicas = e.v2; + return new Tuple<>(part, + new PartitionReassignmentState( + currentReassignments.get(part).replicas(), + targetReplicas, + false)); + }).collect(Collectors.toList()); + + Set topicNamesToLookUp = new HashSet<>(); + notFoundReassignments.forEach(e -> { + TopicPartition part = e.v1; + if (!currentReassignments.containsKey(part)) + topicNamesToLookUp.add(part.topic()); + }); + + Map> topicDescriptions = adminClient. + describeTopics(topicNamesToLookUp).topicNameValues(); + + List> notFoundResults = new ArrayList<>(); + for (Tuple> e : notFoundReassignments) { + TopicPartition part = e.v1; + List targetReplicas = e.v2; + + if (currentReassignments.containsKey(part)) { + PartitionReassignment reassignment = currentReassignments.get(part); + notFoundResults.add(new Tuple<>(part, new PartitionReassignmentState( + reassignment.replicas(), + targetReplicas, + false))); + } else { + notFoundResults.add(new Tuple<>(part, topicDescriptionFutureToState(part.partition(), + topicDescriptions.get(part.topic()), targetReplicas))); + } + } + + Map allResults = new HashMap<>(); + foundResults.forEach(e -> allResults.put(e.v1, e.v2)); + notFoundResults.forEach(e -> allResults.put(e.v1, e.v2)); + + return new Tuple<>(allResults, currentReassignments.size() > 0); } private static PartitionReassignmentState topicDescriptionFutureToState(int partition, KafkaFuture future, - List targetReplicas) { - return null; + List targetReplicas) throws InterruptedException, ExecutionException { + try { + TopicDescription topicDescription = future.get(); + if (topicDescription.partitions().size() < partition) { + throw new ExecutionException("Too few partitions found", new UnknownTopicOrPartitionException()); + } + return new PartitionReassignmentState( + topicDescription.partitions().get(partition).replicas().stream().map(Node::id).collect(Collectors.toList()), + targetReplicas, + true); + } catch (ExecutionException t) { + if (t.getCause() instanceof UnknownTopicOrPartitionException) + return new PartitionReassignmentState(Collections.emptyList(), targetReplicas, true); + + throw t; + } } /** @@ -238,8 +375,10 @@ private static PartitionReassignmentState topicDescriptionFutureToState(int part * returns all ongoing replica reassignments.) */ private static Tuple, Boolean> verifyReplicaMoves(Admin adminClient, - Map targetReassignments) { - return null; + Map targetReassignments) throws ExecutionException, InterruptedException { + Map moveStates = findLogDirMoveStates(adminClient, targetReassignments); + System.out.println(replicaMoveStatesToString(moveStates)); + return new Tuple<>(moveStates, moveStates.values().stream().noneMatch(LogDirMoveState::done)); } /** @@ -252,9 +391,41 @@ private static Tuple, Boolean> verif * * @return The states for each replica movement. */ - private static MapfindLogDirMoveStates(Admin adminClient, - Map targetMoves) { - return null; + private static Map findLogDirMoveStates(Admin adminClient, + Map targetMoves) throws ExecutionException, InterruptedException { + Map replicaLogDirInfos = adminClient. + describeReplicaLogDirs(targetMoves.keySet()).all().get(); + + Map res = new HashMap<>(); + + targetMoves.entrySet().forEach(e -> { + TopicPartitionReplica replica = e.getKey(); + String targetLogDir = e.getValue(); + + LogDirMoveState moveState; + + if (!replicaLogDirInfos.containsKey(replica)) { + moveState = new MissingReplicaMoveState(targetLogDir); + } else { + DescribeReplicaLogDirsResult.ReplicaLogDirInfo info = replicaLogDirInfos.get(replica); + if (info.getCurrentReplicaLogDir() == null){ + moveState = new MissingLogDirMoveState(targetLogDir); + } else if (info.getFutureReplicaLogDir() == null) { + if (info.getCurrentReplicaLogDir().equals(targetLogDir)) { + moveState = new CompletedMoveState(targetLogDir); + } else { + moveState = new CancelledMoveState(info.getCurrentReplicaLogDir(), targetLogDir); + } + } else { + moveState = new ActiveMoveState(info.getCurrentReplicaLogDir(), + targetLogDir, + info.getFutureReplicaLogDir()); + } + } + res.put(replica, moveState); + }); + + return res; } /** @@ -265,7 +436,38 @@ private static Tuple, Boolean> verif * whether there are any active replica moves. */ private static String replicaMoveStatesToString(Map states) { - return null; + List bld = new ArrayList<>(); + states.keySet().stream().sorted(ReassignPartitionCommand::compareTopicPartitionReplicas).forEach(replica -> { + LogDirMoveState state = states.get(replica); + if (state instanceof MissingLogDirMoveState) { + bld.add("Partition " + replica.topic() + "-" + replica.partition() + " is not found " + + "in any live log dir on broker " + replica.brokerId() + ". There is likely an " + + "offline log directory on the broker."); + } else if (state instanceof MissingReplicaMoveState) { + bld.add("Partition " + replica.topic() + "-" + replica.partition() + " cannot be found " + + "in any live log directory on broker " + replica.brokerId() + "."); + } else if (state instanceof ActiveMoveState) { + String targetLogDir = ((ActiveMoveState) state).targetLogDir; + String futureLogDir = ((ActiveMoveState) state).futureLogDir; + if (targetLogDir.equals(futureLogDir)) { + bld.add("Reassignment of replica " + replica + " is still in progress."); + } else { + bld.add("Partition " + replica.topic() + "-" + replica.partition() + " on broker " + + replica.brokerId() + " is being moved to log dir " + futureLogDir + " " + + "instead of " + targetLogDir + "."); + } + } else if (state instanceof CancelledMoveState) { + String targetLogDir = ((CancelledMoveState) state).targetLogDir; + String currentLogDir = ((CancelledMoveState) state).currentLogDir; + bld.add("Partition " + replica.topic() + "-" + replica.partition() + " on broker " + + replica.brokerId() + " is not being moved from log dir " + currentLogDir + " to " + + targetLogDir + "."); + } else if (state instanceof CompletedMoveState) { + bld.add("Reassignment of replica " + replica + " completed successfully."); + } + }); + + return bld.stream().collect(Collectors.joining(System.lineSeparator())); } /** @@ -274,8 +476,19 @@ private static String replicaMoveStatesToString(Map>> targetParts) { - + private static void clearAllThrottles(Admin adminClient, List>> targetParts) throws ExecutionException, InterruptedException { + Set activeBrokers = adminClient.describeCluster().nodes().get().stream().map(Node::id).collect(Collectors.toSet()); + Set brokers = new HashSet<>(activeBrokers); + targetParts.forEach(t -> brokers.addAll(t.v2)); + + System.out.printf("Clearing broker-level throttles on broker%s %s%n", + brokers.size() == 1 ? "" : "s", brokers.stream().map(Object::toString).collect(Collectors.joining(","))); + clearBrokerLevelThrottles(adminClient, brokers); + + Set topics = targetParts.stream().map(t -> t.v1.topic()).collect(Collectors.toSet()); + System.out.printf("Clearing topic-level throttles on topic%s %s%n", + topics.size() == 1 ? "" : "s", topics.stream().map(Object::toString).collect(Collectors.joining(","))); + clearTopicLevelThrottles(adminClient, topics); } /** @@ -284,8 +497,14 @@ private static void clearAllThrottles(Admin adminClient, List brokers) { - + private static void clearBrokerLevelThrottles(Admin adminClient, Set brokers) throws ExecutionException, InterruptedException { + Map> configOps = new HashMap<>(); + brokers.forEach(brokerId -> configOps.put( + new ConfigResource(ConfigResource.Type.BROKER, brokerId.toString()), + brokerLevelThrottles.stream().map(throttle -> new AlterConfigOp( + new ConfigEntry(throttle, null), AlterConfigOp.OpType.DELETE)).collect(Collectors.toList()) + )); + adminClient.incrementalAlterConfigs(configOps).all().get(); } /** @@ -294,8 +513,15 @@ private static void clearBrokerLevelThrottles(Admin adminClient, Set br * @param adminClient The AdminClient to use. * @param topics The topics to clear the throttles for. */ - private static void clearTopicLevelThrottles(Admin adminClient, Set topics) { + private static void clearTopicLevelThrottles(Admin adminClient, Set topics) throws ExecutionException, InterruptedException { + Map> configOps = new HashMap<>(); + topics.forEach(topicName -> configOps.put( + new ConfigResource(ConfigResource.Type.TOPIC, topicName), + topicLevelThrottles.stream().map(throttle -> new AlterConfigOp(new ConfigEntry(throttle, null), + AlterConfigOp.OpType.DELETE)).collect(Collectors.toList()) + )); + adminClient.incrementalAlterConfigs(configOps).all().get(); } /** @@ -312,8 +538,20 @@ private static void clearTopicLevelThrottles(Admin adminClient, Set topi private static Tuple>, Map>> generateAssignment(Admin adminClient, String reassignmentJson, String brokerListString, - Boolean enableRackAwareness) { - return null; + Boolean enableRackAwareness) throws ExecutionException, InterruptedException { + Tuple, List> t0 = parseGenerateAssignmentArgs(reassignmentJson, brokerListString); + + List brokersToReassign = t0.v1; + List topicsToReassign = t0.v2; + + Map> currentAssignments = getReplicaAssignmentForTopics(adminClient, topicsToReassign); + List brokerMetadatas = getBrokerMetadata(adminClient, brokersToReassign, enableRackAwareness); + Map> proposedAssignments = calculateAssignment(currentAssignments, brokerMetadatas); + System.out.printf("Current partition replica assignment\n%s\n%n", + formatAsReassignmentJson(currentAssignments, Collections.emptyMap())); + System.out.printf("Proposed partition reassignment configuration\n%s", + formatAsReassignmentJson(proposedAssignments, Collections.emptyMap())); + return new Tuple<>(proposedAssignments, currentAssignments); } /** @@ -326,12 +564,37 @@ private static Tuple>, Map> calculateAssignment(Map> currentAssignment, List brokerMetadatas) { - return null; + Map>>> groupedByTopic = new HashMap<>(); + for (Map.Entry> e : currentAssignment.entrySet()) + groupedByTopic.computeIfAbsent(e.getKey().topic(), k -> new ArrayList<>()).add(e); + HashMap> proposedAssignments = new HashMap<>(); + groupedByTopic.forEach((topic, assignment) -> { + List replicas = assignment.get(0).getValue(); + Map> assignedReplicas = null/*AdminUtils. + assignReplicasToBrokers(brokerMetadatas, assignment.size(), replicas.size())*/; + assignedReplicas.forEach((partition, replicas0) -> + proposedAssignments.put(new TopicPartition(topic, partition), replicas0)); + }); + return proposedAssignments; } private static Map describeTopics(Admin adminClient, - Set topics) { - return null; + Set topics) throws ExecutionException, InterruptedException { + Map> futures = adminClient.describeTopics(topics).topicNameValues(); + Map res = new HashMap<>(); + for (Map.Entry> e : futures.entrySet()) { + String topicName = e.getKey(); + KafkaFuture topicDescriptionFuture = e.getValue(); + try { + res.put(topicName, topicDescriptionFuture.get()); + } catch (ExecutionException t) { + if (t.getCause() instanceof UnknownTopicOrPartitionException) + throw new ExecutionException( + new UnknownTopicOrPartitionException("Topic " + topicName + " not found.")); + throw t; + } + } + return res; } /** @@ -343,8 +606,16 @@ private static Map describeTopics(Admin adminClient, * If any topic can't be found, an exception will be thrown. */ private static Map> getReplicaAssignmentForTopics(Admin adminClient, - List topics) { - return null; + List topics) throws ExecutionException, InterruptedException { + + Map> res = new HashMap<>(); + describeTopics(adminClient, new HashSet<>(topics)).forEach((topicName, topicDescription) -> + topicDescription.partitions().forEach(info -> res.put( + new TopicPartition(topicName, info.partition()), + info.replicas().stream().map(Node::id).collect(Collectors.toList()) + ) + )); + return res; } /** @@ -356,8 +627,16 @@ private static Map> getReplicaAssignmentForTopics( * If any topic can't be found, an exception will be thrown. */ private static Map> getReplicaAssignmentForPartitions(Admin adminClient, - Set partitions) { - return null; + Set partitions) throws ExecutionException, InterruptedException { + Map> res = new HashMap<>(); + describeTopics(adminClient, partitions.stream().map(TopicPartition::topic).collect(Collectors.toSet())).forEach((topicName, topicDescription) -> { + topicDescription.partitions().forEach(info -> { + TopicPartition tp = new TopicPartition(topicName, info.partition()); + if (partitions.contains(tp)) + res.put(tp, info.replicas().stream().map(Node::id).collect(Collectors.toList())); + }); + }); + return res; } /** @@ -371,8 +650,21 @@ private static Map> getReplicaAssignmentForPartiti * @return The metadata for each broker that was found. * Brokers that were not found will be omitted. */ - private static List getBrokerMetadata(Admin adminClient, List brokers, boolean enableRackAwareness) { - return null; + private static List getBrokerMetadata(Admin adminClient, List brokers, boolean enableRackAwareness) throws ExecutionException, InterruptedException { + Set brokerSet = new HashSet<>(brokers); + List results = adminClient.describeCluster().nodes().get().stream(). + filter(node -> brokerSet.contains(node.id())). + map(node -> (enableRackAwareness && node.rack() != null) + ? new BrokerMetadata(node.id(), Optional.of(node.rack())) + : new BrokerMetadata(node.id(), Optional.empty())).collect(Collectors.toList()); + + long numRackless = results.stream().filter(m -> !m.rack.isPresent()).count(); + if (enableRackAwareness && numRackless != 0 && numRackless != results.size()) { + throw new AdminOperationException("Not all brokers have rack information. Add " + + "--disable-rack-aware in command line to make replica assignment without rack " + + "information."); + } + return results; } /** @@ -387,7 +679,16 @@ private static List getBrokerMetadata(Admin adminClient, List, List> parseGenerateAssignmentArgs(String reassignmentJson, String brokerList) { - return null; + List brokerListToReassign = Arrays.stream(brokerList.split(",")).map(Integer::parseInt).collect(Collectors.toList()); + Set duplicateReassignments = ToolsUtils.duplicates(brokerListToReassign); + if (!duplicateReassignments.isEmpty()) + throw new AdminCommandFailedException(String.format("Broker list contains duplicate entries: %s", duplicateReassignments)); + List topicsToReassign = parseTopicsData(reassignmentJson); + Set duplicateTopicsToReassign = ToolsUtils.duplicates(topicsToReassign); + if (!duplicateTopicsToReassign.isEmpty()) + throw new AdminCommandFailedException(String.format("List of topics to reassign contains duplicate entries: %s", + duplicateTopicsToReassign)); + return new Tuple<>(brokerListToReassign, topicsToReassign); } /** @@ -642,7 +943,7 @@ private static String formatAsReassignmentJson(Map return null; } - private List parseTopicsData(String jsonData) { + private static List parseTopicsData(String jsonData) { return null; } From 86ec36137e03f65fd62afb3b1bd0fe83ef493ce0 Mon Sep 17 00:00:00 2001 From: nizhikov Date: Tue, 25 Jul 2023 09:34:36 +0300 Subject: [PATCH 13/78] KAFKA-14595 WIP * ReassignPartitionCommand rewritten * AdminUtils stub --- .../org/apache/kafka/admin/AdminUtils.java | 163 ++++++ .../apache/kafka/admin}/BrokerMetadata.java | 2 +- .../reassign/ReassignPartitionCommand.java | 541 ++++++++++++++++-- 3 files changed, 649 insertions(+), 57 deletions(-) create mode 100644 server-common/src/main/java/org/apache/kafka/admin/AdminUtils.java rename {tools/src/main/java/org/apache/kafka/tools/reassign => server-common/src/main/java/org/apache/kafka/admin}/BrokerMetadata.java (96%) diff --git a/server-common/src/main/java/org/apache/kafka/admin/AdminUtils.java b/server-common/src/main/java/org/apache/kafka/admin/AdminUtils.java new file mode 100644 index 0000000000000..58437b40c310c --- /dev/null +++ b/server-common/src/main/java/org/apache/kafka/admin/AdminUtils.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.admin; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.Map; +import java.util.Random; + +public class AdminUtils { + private static final Logger log = LoggerFactory.getLogger(AdminUtils.class); + + static final Random rand = new Random(); + + static final String AdminClientId = "__admin_client"; + + public static Map> assignReplicasToBrokers(Iterable brokerMetadatas, + int nPartitions, + int replicationFactor) { + return assignReplicasToBrokers(brokerMetadatas, nPartitions, replicationFactor, -1, -1); + } + + /** + * There are 3 goals of replica assignment: + * + *
    + *
  1. Spread the replicas evenly among brokers.
    2. + *
  2. For partitions assigned to a particular broker, their other replicas are spread over the other brokers.
    3. + *
  3. If all brokers have rack information, assign the replicas for each partition to different racks if possible
    4. + *
+ * + * To achieve this goal for replica assignment without considering racks, we: + *
    + *
  1. Assign the first replica of each partition by round-robin, starting from a random position in the broker list.
    2. + *
  2. Assign the remaining replicas of each partition with an increasing shift.
    3. + *
+ * + * Here is an example of assigning + * + * + * + * + * + * + * + * + *
broker-0broker-1broker-2broker-3broker-4 
p0 p1 p2 p3 p4 (1st replica)
p5 p6 p7 p8 p9 (1st replica)
p4 p0 p1 p2 p3 (2nd replica)
p8 p9 p5 p6 p7 (2nd replica)
p3 p4 p0 p1 p2 (3nd replica)
p7 p8 p9 p5 p6 (3nd replica)
+ * + *

+ * To create rack aware assignment, this API will first create a rack alternated broker list. For example, + * from this brokerID -> rack mapping:

+ * 0 -> "rack1", 1 -> "rack3", 2 -> "rack3", 3 -> "rack2", 4 -> "rack2", 5 -> "rack1" + *

+ *

+ * The rack alternated list will be: + *

+ * 0, 3, 1, 5, 4, 2 + *

+ *

+ * Then an easy round-robin assignment can be applied. Assume 6 partitions with replication factor of 3, the assignment + * will be: + *

+ * 0 -> 0,3,1
+ * 1 -> 3,1,5
+ * 2 -> 1,5,4
+ * 3 -> 5,4,2
+ * 4 -> 4,2,0
+ * 5 -> 2,0,3
+ *
+ *

+ * Once it has completed the first round-robin, if there are more partitions to assign, the algorithm will start + * shifting the followers. This is to ensure we will not always get the same set of sequences. + * In this case, if there is another partition to assign (partition #6), the assignment will be: + *

+ * 6 -> 0,4,2 (instead of repeating 0,3,1 as partition 0) + *

+ *

+ * The rack aware assignment always chooses the 1st replica of the partition using round robin on the rack alternated + * broker list. For rest of the replicas, it will be biased towards brokers on racks that do not have + * any replica assignment, until every rack has a replica. Then the assignment will go back to round-robin on + * the broker list. + *

+ *
+ *

+ * As the result, if the number of replicas is equal to or greater than the number of racks, it will ensure that + * each rack will get at least one replica. Otherwise, each rack will get at most one replica. In a perfect + * situation where the number of replicas is the same as the number of racks and each rack has the same number of + * brokers, it guarantees that the replica distribution is even across brokers and racks. + *

+ * @return a Map from partition id to replica ids + * @throws AdminOperationException If rack information is supplied but it is incomplete, or if it is not possible to + * assign each replica to a unique rack. + * + */ + public static Map> assignReplicasToBrokers(Iterable brokerMetadatas, + int nPartitions, + int replicationFactor, + int fixedStartIndex, + int startPartitionId) { + return null; + } + + private static Map> assignReplicasToBrokersRackUnaware(int nPartitions, + int replicationFactor, + Iterable brokerList, + int fixedStartIndex, + int startPartitionId) { + return null; + } + + private static Map> assignReplicasToBrokersRackAware(int nPartitions, + int replicationFactor, + Iterable brokerMetadatas, + int fixedStartIndex, + int startPartitionId) { + return null; + } + + /** + * Given broker and rack information, returns a list of brokers alternated by the rack. Assume + * this is the rack and its brokers: + * + * rack1: 0, 1, 2 + * rack2: 3, 4, 5 + * rack3: 6, 7, 8 + * + * This API would return the list of 0, 3, 6, 1, 4, 7, 2, 5, 8 + * + * This is essential to make sure that the assignReplicasToBrokers API can use such list and + * assign replicas to brokers in a simple round-robin fashion, while ensuring an even + * distribution of leader and replica counts on each broker and that replicas are + * distributed to all racks. + */ + static List getRackAlternatedBrokerList(Map brokerRackMap) { + return null; + } + + static Map> getInverseMap(Map brokerRackMap) { + return null; + } + + static int replicaIndex(int firstReplicaIndex, int secondReplicaShift, int replicaIndex, int nBrokers) { + long shift = 1 + (secondReplicaShift + replicaIndex) % (nBrokers - 1); + return (int) ((firstReplicaIndex + shift) % nBrokers); + } +} diff --git a/tools/src/main/java/org/apache/kafka/tools/reassign/BrokerMetadata.java b/server-common/src/main/java/org/apache/kafka/admin/BrokerMetadata.java similarity index 96% rename from tools/src/main/java/org/apache/kafka/tools/reassign/BrokerMetadata.java rename to server-common/src/main/java/org/apache/kafka/admin/BrokerMetadata.java index b6d416aff97bb..722a28c9fba57 100644 --- a/tools/src/main/java/org/apache/kafka/tools/reassign/BrokerMetadata.java +++ b/server-common/src/main/java/org/apache/kafka/admin/BrokerMetadata.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.kafka.tools.reassign; +package org.apache.kafka.admin; import java.util.Optional; diff --git a/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionCommand.java b/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionCommand.java index b0ccc1e6dafd0..79ea8569ad82b 100644 --- a/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionCommand.java +++ b/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionCommand.java @@ -17,12 +17,18 @@ package org.apache.kafka.tools.reassign; +import com.fasterxml.jackson.core.JsonParseException; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonMappingException; import joptsimple.OptionSpec; +import org.apache.kafka.admin.AdminUtils; +import org.apache.kafka.admin.BrokerMetadata; import org.apache.kafka.clients.admin.Admin; import org.apache.kafka.clients.admin.AdminClientConfig; import org.apache.kafka.clients.admin.AlterConfigOp; import org.apache.kafka.clients.admin.ConfigEntry; import org.apache.kafka.clients.admin.DescribeReplicaLogDirsResult; +import org.apache.kafka.clients.admin.NewPartitionReassignment; import org.apache.kafka.clients.admin.PartitionReassignment; import org.apache.kafka.clients.admin.TopicDescription; import org.apache.kafka.common.KafkaFuture; @@ -30,6 +36,7 @@ import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.TopicPartitionReplica; import org.apache.kafka.common.config.ConfigResource; +import org.apache.kafka.common.errors.ReplicaNotAvailableException; import org.apache.kafka.common.errors.UnknownTopicOrPartitionException; import org.apache.kafka.common.utils.Exit; import org.apache.kafka.common.utils.Time; @@ -37,6 +44,9 @@ import org.apache.kafka.server.common.AdminCommandFailedException; import org.apache.kafka.server.common.AdminOperationException; import org.apache.kafka.server.util.CommandLineUtils; +import org.apache.kafka.server.util.Json; +import org.apache.kafka.server.util.json.DecodeJson; +import org.apache.kafka.server.util.json.JsonObject; import org.apache.kafka.server.util.json.JsonValue; import org.apache.kafka.tools.ToolsUtils; @@ -47,34 +57,38 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Properties; import java.util.Set; +import java.util.TreeSet; import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; -/** - * - */ public class ReassignPartitionCommand { - private static final String AnyLogDir = "any"; + private static final String ANY_LOG_DIR = "any"; + + private static final String HELP_TEXT = "This tool helps to move topic partitions between replicas."; - private static final String helpText = "This tool helps to move topic partitions between replicas."; + private static final DecodeJson.DecodeInteger INT = new DecodeJson.DecodeInteger(); + + private static final DecodeJson.DecodeString STRING = new DecodeJson.DecodeString(); /** * The earliest version of the partition reassignment JSON. We will default to this * version if no other version number is given. */ - public static final int EarliestVersion = 1; + static final int EARLIEST_VERSION = 1; /** * The earliest version of the JSON for each partition reassignment topic. We will * default to this version if no other version number is given. */ - public static final int EarliestTopicsJsonVersion = 1; + static final int EARLIEST_TOPICS_JSON_VERSION = 1; // Throttles that are set at the level of an individual broker. //DynamicConfig.Broker.LeaderReplicationThrottledRateProp @@ -179,7 +193,7 @@ private static void handleAction(Admin adminClient, ReassignPartitionsCommandOpt * * @return A result that is useful for testing. */ - private static VerifyAssignmentResult verifyAssignment(Admin adminClient, String jsonString, Boolean preserveThrottles) throws ExecutionException, InterruptedException { + private static VerifyAssignmentResult verifyAssignment(Admin adminClient, String jsonString, Boolean preserveThrottles) throws ExecutionException, InterruptedException, JsonProcessingException { Tuple>>, Map> t0 = parsePartitionReassignmentData(jsonString); List>> targetParts = t0.v1; @@ -538,7 +552,7 @@ private static void clearTopicLevelThrottles(Admin adminClient, Set topi private static Tuple>, Map>> generateAssignment(Admin adminClient, String reassignmentJson, String brokerListString, - Boolean enableRackAwareness) throws ExecutionException, InterruptedException { + Boolean enableRackAwareness) throws ExecutionException, InterruptedException, JsonProcessingException { Tuple, List> t0 = parseGenerateAssignmentArgs(reassignmentJson, brokerListString); List brokersToReassign = t0.v1; @@ -570,8 +584,8 @@ private static Map> calculateAssignment(Map> proposedAssignments = new HashMap<>(); groupedByTopic.forEach((topic, assignment) -> { List replicas = assignment.get(0).getValue(); - Map> assignedReplicas = null/*AdminUtils. - assignReplicasToBrokers(brokerMetadatas, assignment.size(), replicas.size())*/; + Map> assignedReplicas = AdminUtils. + assignReplicasToBrokers(brokerMetadatas, assignment.size(), replicas.size()); assignedReplicas.forEach((partition, replicas0) -> proposedAssignments.put(new TopicPartition(topic, partition), replicas0)); }); @@ -678,7 +692,7 @@ private static List getBrokerMetadata(Admin adminClient, List, List> parseGenerateAssignmentArgs(String reassignmentJson, - String brokerList) { + String brokerList) throws JsonMappingException { List brokerListToReassign = Arrays.stream(brokerList.split(",")).map(Integer::parseInt).collect(Collectors.toList()); Set duplicateReassignments = ToolsUtils.duplicates(brokerListToReassign); if (!duplicateReassignments.isEmpty()) @@ -711,8 +725,58 @@ private static void executeAssignment(Admin adminClient, Long interBrokerThrottle, Long logDirThrottle, Long timeoutMs, - Time time) { + Time time) throws ExecutionException, InterruptedException, JsonProcessingException { + Tuple>, Map> t0 = parseExecuteAssignmentArgs(reassignmentJson); + + Map> proposedParts = t0.v1; + Map proposedReplicas = t0.v2; + Map currentReassignments = adminClient. + listPartitionReassignments().reassignments().get(); + // If there is an existing assignment, check for --additional before proceeding. + // This helps avoid surprising users. + if (!additional && !currentReassignments.isEmpty()) { + throw new TerseReassignmentFailureException(cannotExecuteBecauseOfExistingMessage); + } + Set brokers = new HashSet<>(); + proposedParts.values().forEach(brokers::addAll); + + verifyBrokerIds(adminClient, brokers); + Map> currentParts = getReplicaAssignmentForPartitions(adminClient, proposedParts.keySet()); + System.out.println(currentPartitionReplicaAssignmentToString(proposedParts, currentParts)); + + if (interBrokerThrottle >= 0 || logDirThrottle >= 0) { + System.out.println(youMustRunVerifyPeriodicallyMessage); + + if (interBrokerThrottle >= 0) { + Map> moveMap = calculateProposedMoveMap(currentReassignments, proposedParts, currentParts); + modifyReassignmentThrottle(adminClient, moveMap, interBrokerThrottle); + } + if (logDirThrottle >= 0) { + Set movingBrokers = calculateMovingBrokers(proposedReplicas.keySet()); + modifyLogDirThrottle(adminClient, movingBrokers, logDirThrottle); + } + } + + // Execute the partition reassignments. + Map errors = alterPartitionReassignments(adminClient, proposedParts); + if (!errors.isEmpty()) { + throw new TerseReassignmentFailureException( + String.format("Error reassigning partition(s):%n%s", + errors.keySet().stream(). + sorted(ReassignPartitionCommand::compareTopicPartitions). + map(part -> part + ": " + errors.get(part).getMessage()). + collect(Collectors.joining(System.lineSeparator())))); + } + System.out.printf("Successfully started partition reassignment%s for %s%n", + proposedParts.size() == 1 ? "" : "s", + proposedParts.keySet().stream(). + sorted(ReassignPartitionCommand::compareTopicPartitions). + map(Objects::toString). + collect(Collectors.joining(","))); + if (!proposedReplicas.isEmpty()) { + executeMoves(adminClient, proposedReplicas, timeoutMs, time); + } } /** @@ -728,8 +792,41 @@ private static void executeAssignment(Admin adminClient, private static void executeMoves(Admin adminClient, Map proposedReplicas, Long timeoutMs, - Time time) { + Time time) throws InterruptedException { + long startTimeMs = time.milliseconds(); + Map pendingReplicas = new HashMap<>(); + pendingReplicas.putAll(proposedReplicas); + boolean done = false; + do { + Set completed = alterReplicaLogDirs(adminClient, pendingReplicas); + if (!completed.isEmpty()) { + System.out.printf("Successfully started log directory move%s for: %s", + completed.size() == 1 ? "" : "s", + completed.stream(). + sorted(ReassignPartitionCommand::compareTopicPartitionReplicas). + map(Object::toString). + collect(Collectors.joining(","))); + } + completed.forEach(pendingReplicas::remove); + if (pendingReplicas.isEmpty()) { + done = true; + } else if (time.milliseconds() >= startTimeMs + timeoutMs) { + throw new TerseReassignmentFailureException(String.format( + "Timed out before log directory move%s could be started for: %s", + pendingReplicas.size() == 1 ? "" : "s", + pendingReplicas.keySet().stream(). + sorted(ReassignPartitionCommand::compareTopicPartitionReplicas). + map(Object::toString). + collect(Collectors.joining(",")))); + } else { + // If a replica has been moved to a new host and we also specified a particular + // log directory, we will have to keep retrying the alterReplicaLogDirs + // call. It can't take effect until the replica is moved to that host. + time.sleep(100); + } + + } while (!done); } /** @@ -737,8 +834,8 @@ private static void executeMoves(Admin adminClient, * * @param adminClient The AdminClient to use. */ - private static void listReassignments(Admin adminClient) { - + private static void listReassignments(Admin adminClient) throws ExecutionException, InterruptedException { + System.out.println(curReassignmentsToString(adminClient)); } /** @@ -747,8 +844,30 @@ private static void listReassignments(Admin adminClient) { * @param adminClient The AdminClient to use. * @return A string describing the current partition reassignments. */ - private static String curReassignmentsToString(Admin adminClient) { - return null; + private static String curReassignmentsToString(Admin adminClient) throws ExecutionException, InterruptedException { + Map currentReassignments = adminClient. + listPartitionReassignments().reassignments().get(); + String text = currentReassignments.keySet().stream().sorted(ReassignPartitionCommand::compareTopicPartitions).map(part -> { + PartitionReassignment reassignment = currentReassignments.get(part); + List replicas = reassignment.replicas(); + List addingReplicas = reassignment.addingReplicas(); + List removingReplicas = reassignment.removingReplicas(); + + return String.format("%s: replicas: %s.%s%s", + part, + replicas.stream().map(Object::toString).collect(Collectors.joining(",")), + addingReplicas.isEmpty() ? "" : String.format(" adding: %s.", addingReplicas.stream(). + map(Object::toString). + collect(Collectors.joining(","))), + removingReplicas.isEmpty() ? "" : String.format(" removing: %s.", removingReplicas.stream(). + map(Object::toString). + collect(Collectors.joining(","))) + ); + }).collect(Collectors.joining(System.lineSeparator())); + + return text.length() == 0 + ? "No partition reassignments found." + : String.format("Current partition reassignments:%n%s", text); } /** @@ -757,8 +876,13 @@ private static String curReassignmentsToString(Admin adminClient) { * @param adminClient The AdminClient to use. * @param brokers The broker IDs to verify. */ - private static void verifyBrokerIds(Admin adminClient, Set brokers) { - + private static void verifyBrokerIds(Admin adminClient, Set brokers) throws ExecutionException, InterruptedException { + Set allNodeIds = adminClient.describeCluster().nodes().get().stream().map(Node::id).collect(Collectors.toSet()); + Optional unknown = brokers.stream(). + filter(i -> !allNodeIds.contains(i)). + findFirst(); + if (unknown.isPresent()) + throw new AdminCommandFailedException("Unknown broker id " + unknown.get()); } /** @@ -771,8 +895,14 @@ private static void verifyBrokerIds(Admin adminClient, Set brokers) { * partitions that appear in the proposed partition assignment. */ private static String currentPartitionReplicaAssignmentToString(Map> proposedParts, - Map> currentParts) { - return null; + Map> currentParts) throws JsonProcessingException { + Map> partitionsToBeReassigned = currentParts.entrySet().stream(). + filter(e -> proposedParts.containsKey(e.getKey())). + collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + + return String.format("Current partition replica assignment%n%n%s%n%nSave this to use as the %s", + formatAsReassignmentJson(partitionsToBeReassigned, Collections.emptyMap()), + "--reassignment-json-file option during rollback"); } /** @@ -783,8 +913,24 @@ private static String currentPartitionReplicaAssignmentToString(Map alterPartitionReassignments(Admin adminClient, - Map> reassignments) { - return null; + Map> reassignments) throws InterruptedException { + Map> args = new HashMap<>(); + reassignments.entrySet().forEach(e -> { + TopicPartition part = e.getKey(); + List replicas = e.getValue(); + + args.put(part, Optional.of(new NewPartitionReassignment(replicas))); + }); + Map> results = adminClient.alterPartitionReassignments(args).values(); + Map errors = new HashMap<>(); + for (Map.Entry> e : results.entrySet()) { + try { + e.getValue().get(); + } catch(ExecutionException t) { + errors.put(e.getKey(), t.getCause()); + } + } + return errors; } /** @@ -795,8 +941,20 @@ private static Map alterPartitionReassignments(Admin * @return A map from partition objects to error strings. */ private static Map cancelPartitionReassignments(Admin adminClient, - Set reassignments) { - return null; + Set reassignments) throws InterruptedException { + Map> args = new HashMap<>(); + reassignments.forEach(part -> args.put(part, Optional.empty())); + + Map> results = adminClient.alterPartitionReassignments(args).values(); + Map errors = new HashMap<>(); + for (Map.Entry> e : results.entrySet()) { + try { + e.getValue().get(); + } catch(ExecutionException t) { + errors.put(e.getKey(), t.getCause()); + } + } + return errors; } /** @@ -804,7 +962,13 @@ private static Map cancelPartitionReassignments(Admin * @param currentReassignments All replicas, adding replicas and removing replicas of target partitions */ private static Map> calculateCurrentMoveMap(Map currentReassignments) { - return null; + Map> moveMap = new HashMap<>(); + // Add the current reassignments to the move map. + currentReassignments.entrySet().forEach(e -> { + TopicPartition part = e.getKey(); + PartitionReassignment reassignment = e.getValue(); + }); + return moveMap; } /** @@ -821,7 +985,28 @@ private static Map> calculateCurrentMoveMap( private static Map> calculateProposedMoveMap(Map currentReassignments, Map> proposedParts, Map> currentParts) { - return null; + Map> moveMap = calculateCurrentMoveMap(currentReassignments); + for (Map.Entry> e : proposedParts.entrySet()) { + TopicPartition part = e.getKey(); + List replicas = e.getValue(); + Map partMoves = moveMap.computeIfAbsent(part.topic(), k -> new HashMap<>()); + + // If there is a reassignment in progress, use the sources from moveMap, otherwise + // use the sources from currentParts + Set sources = new HashSet<>(); + + if (partMoves.containsKey(part.partition())) { + PartitionMove move = partMoves.get(part.partition()); + sources.addAll(move.sources); + } else if (!currentParts.containsKey(part)) + throw new RuntimeException("Trying to reassign a topic partition " + part + " with 0 replicas"); + + Set destinations = new HashSet<>(replicas); + destinations.removeAll(sources); + + partMoves.put(part.partition(), new PartitionMove(sources, destinations)); + } + return moveMap; } /** @@ -831,7 +1016,14 @@ private static Map> calculateProposedMoveMap * @return A map from topic names to leader throttle configurations. */ private static Map calculateLeaderThrottles(Map> moveMap) { - return null; + Map results = new HashMap<>(); + moveMap.forEach((topicName, partMoveMap) -> { + Set components = new TreeSet<>(); + partMoveMap.forEach((partId, move) -> + move.sources.forEach(source -> components.add(String.format("%d:%d", partId, source)))); + results.put(topicName, String.join(",", components)); + }); + return results; } /** @@ -841,7 +1033,20 @@ private static Map calculateLeaderThrottles(Map calculateFollowerThrottles(Map> moveMap) { - return null; + Map results = new HashMap<>(); + moveMap.forEach((topicName, partMoveMap) -> { + Set components = new TreeSet<>(); + partMoveMap.forEach((partId, move) -> + move.destinations.forEach(destination -> { + if (!move.sources.contains(destination)) { + components.add(String.format("%d:%d", partId, destination)); + } + }) + ); + results.put(topicName, String.join(",", components)); + }); + + return results; } /** @@ -850,8 +1055,15 @@ private static Map calculateFollowerThrottles(Map calculateReassigningBrokers(Map> moveMap) { - return null; + private static Set calculateReassigningBrokers(Map> moveMap) { + Set reassigningBrokers = new TreeSet<>(); + moveMap.values().forEach(partMoveMap -> { + partMoveMap.values().forEach(partMove -> { + reassigningBrokers.addAll(partMove.sources); + reassigningBrokers.addAll(partMove.destinations); + }); + }); + return reassigningBrokers; } /** @@ -861,7 +1073,7 @@ private static List calculateReassigningBrokers(Map calculateMovingBrokers(Set replicaMoves) { - return null; + return replicaMoves.stream().map(TopicPartitionReplica::brokerId).collect(Collectors.toSet()); } /** @@ -873,11 +1085,32 @@ private static Set calculateMovingBrokers(Set re */ private static void modifyTopicThrottles(Admin adminClient, Map leaderThrottles, - Map followerThrottles) { + Map followerThrottles) throws ExecutionException, InterruptedException { + Map> configs = new HashMap<>(); + Set topicNames = new HashSet<>(leaderThrottles.keySet()); + topicNames.addAll(followerThrottles.keySet()); + topicNames.forEach(topicName -> { + List ops = new ArrayList<>(); + if (leaderThrottles.containsKey(topicName)) { + ops.add(new AlterConfigOp(new ConfigEntry(topicLevelLeaderThrottle, leaderThrottles.get(topicName)), AlterConfigOp.OpType.SET)); + } + if (followerThrottles.containsKey(topicName)) { + ops.add(new AlterConfigOp(new ConfigEntry(topicLevelFollowerThrottle, leaderThrottles.get(topicName)), AlterConfigOp.OpType.SET)); + } + if (!ops.isEmpty()) { + configs.put(new ConfigResource(ConfigResource.Type.TOPIC, topicName), ops); + } + }); + adminClient.incrementalAlterConfigs(configs).all().get(); } - private static void modifyReassignmentThrottle(Admin admin, Map> moveMap, Long interBrokerThrottle) { + private static void modifyReassignmentThrottle(Admin admin, Map> moveMap, Long interBrokerThrottle) throws ExecutionException, InterruptedException { + Map leaderThrottles = calculateLeaderThrottles(moveMap); + Map followerThrottles = calculateFollowerThrottles(moveMap); + modifyTopicThrottles(admin, leaderThrottles, followerThrottles); + Set reassigningBrokers = calculateReassigningBrokers(moveMap); + modifyInterBrokerThrottle(admin, reassigningBrokers, interBrokerThrottle); } /** @@ -889,8 +1122,20 @@ private static void modifyReassignmentThrottle(Admin admin, Map reassigningBrokers, - Long interBrokerThrottle) { - + long interBrokerThrottle) throws ExecutionException, InterruptedException { + if (interBrokerThrottle >= 0) { + Map> configs = new HashMap<>(); + reassigningBrokers.forEach(brokerId -> { + List ops = new ArrayList<>(); + ops.add(new AlterConfigOp(new ConfigEntry(brokerLevelLeaderThrottle, + Long.toString(interBrokerThrottle)), AlterConfigOp.OpType.SET)); + ops.add(new AlterConfigOp(new ConfigEntry(brokerLevelFollowerThrottle, + Long.toString(interBrokerThrottle)), AlterConfigOp.OpType.SET)); + configs.put(new ConfigResource(ConfigResource.Type.BROKER, Long.toString(brokerId)), ops); + }); + adminClient.incrementalAlterConfigs(configs).all().get(); + System.out.println("The inter-broker throttle limit was set to " + interBrokerThrottle + " B/s"); + } } /** @@ -902,8 +1147,17 @@ private static void modifyInterBrokerThrottle(Admin adminClient, */ private static void modifyLogDirThrottle(Admin admin, Set movingBrokers, - Long logDirThrottle) { - + long logDirThrottle) throws ExecutionException, InterruptedException { + if (logDirThrottle >= 0) { + Map> configs = new HashMap<>(); + movingBrokers.forEach(brokerId -> { + List ops = new ArrayList<>(); + ops.add(new AlterConfigOp(new ConfigEntry(brokerLevelLogDirThrottle, Long.toString(logDirThrottle)), AlterConfigOp.OpType.SET)); + configs.put(new ConfigResource(ConfigResource.Type.BROKER, Long.toString(brokerId)), ops); + }); + admin.incrementalAlterConfigs(configs).all().get(); + System.out.println("The replica-alter-dir throttle limit was set to " + logDirThrottle + " B/s"); + } } /** @@ -913,8 +1167,37 @@ private static void modifyLogDirThrottle(Admin admin, * @return A tuple of the partitions to be reassigned and the replicas * to be reassigned. */ - private static Tuple>, Map> parseExecuteAssignmentArgs(String reassignmentJson) { - return null; + private static Tuple>, Map> parseExecuteAssignmentArgs(String reassignmentJson) throws JsonProcessingException { + Tuple>>, Map> t0 = parsePartitionReassignmentData(reassignmentJson); + + List>> partitionsToBeReassigned = t0.v1; + Map replicaAssignment = t0.v2; + + if (partitionsToBeReassigned.isEmpty()) + throw new AdminCommandFailedException("Partition reassignment list cannot be empty"); + if (partitionsToBeReassigned.stream().anyMatch(t -> t.v2.isEmpty())) { + throw new AdminCommandFailedException("Partition replica list cannot be empty"); + } + Set duplicateReassignedPartitions = ToolsUtils.duplicates(partitionsToBeReassigned.stream().map(t -> t.v1).collect(Collectors.toSet())); + if (!duplicateReassignedPartitions.isEmpty()) { + throw new AdminCommandFailedException(String.format( + "Partition reassignment contains duplicate topic partitions: %s", + duplicateReassignedPartitions.stream().map(Object::toString).collect(Collectors.joining(","))) + ); + } + List>> duplicateEntries = partitionsToBeReassigned.stream() + .map(t -> new Tuple<>(t.v1, ToolsUtils.duplicates(t.v2))) + .filter(t -> !t.v2.isEmpty()) + .collect(Collectors.toList()); + if (!duplicateEntries.isEmpty()) { + String duplicatesMsg = duplicateEntries.stream().map(t -> + String.format("%s contains multiple entries for %s", + t.v1, + t.v2.stream().map(Object::toString).collect(Collectors.joining(","))) + ).collect(Collectors.joining(". ")); + throw new AdminCommandFailedException(String.format("Partition replica lists may not contain duplicate entries: %s", duplicatesMsg)); + } + return new Tuple<>(partitionsToBeReassigned.stream().collect(Collectors.toMap(t -> t.v1, t -> t.v2)), replicaAssignment); } /** @@ -934,37 +1217,165 @@ private static Tuple, Set> cancelAssi String jsonString, Boolean preserveThrottles, Long timeoutMs, - Time time) { - return null; + Time time) throws ExecutionException, InterruptedException, JsonProcessingException { + Tuple>>, Map> t0 = parsePartitionReassignmentData(jsonString); + + List>> targetParts = t0.v1; + Map targetReplicas = t0.v2; + Set targetPartsSet = targetParts.stream().map(t -> t.v1).collect(Collectors.toSet()); + Set curReassigningParts = new HashSet<>(); + adminClient.listPartitionReassignments(targetPartsSet).reassignments().get().forEach((part, reassignment) -> { + if (reassignment.addingReplicas().isEmpty() || !reassignment.removingReplicas().isEmpty()) + curReassigningParts.add(part); + }); + if (!curReassigningParts.isEmpty()) { + Map errors = cancelPartitionReassignments(adminClient, curReassigningParts); + if (!errors.isEmpty()) { + throw new TerseReassignmentFailureException(String.format( + "Error cancelling partition reassignment%s for:%n%s", + errors.size() == 1 ? "" : "s", + errors.keySet().stream(). + sorted(ReassignPartitionCommand::compareTopicPartitions). + map(part -> part + ": " + errors.get(part).getMessage()).collect(Collectors.joining(System.lineSeparator()))) + ); + } + System.out.printf("Successfully cancelled partition reassignment%s for: %s", + curReassigningParts.size() == 1 ? "" : "s", + curReassigningParts.stream().sorted(ReassignPartitionCommand::compareTopicPartitions).map(Object::toString).collect(Collectors.joining(",")) + ); + } else { + System.out.println("None of the specified partition reassignments are active."); + } + Map curMovingParts = new HashMap<>(); + findLogDirMoveStates(adminClient, targetReplicas).forEach((part, moveState) -> { + if (moveState instanceof ActiveMoveState) + curMovingParts.put(part, ((ActiveMoveState) moveState).currentLogDir); + }); + if (curMovingParts.isEmpty()) { + System.out.print("None of the specified partition moves are active."); + } else { + executeMoves(adminClient, curMovingParts, timeoutMs, time); + } + if (!preserveThrottles) { + clearAllThrottles(adminClient, targetParts); + } + return new Tuple<>(curReassigningParts, curMovingParts.keySet()); } private static String formatAsReassignmentJson(Map> partitionsToBeReassigned, - Map replicaLogDirAssignment) { - return null; + Map replicaLogDirAssignment) throws JsonProcessingException { + Map> partitions = new LinkedHashMap<>(); + partitionsToBeReassigned.keySet().stream().sorted(ReassignPartitionCommand::compareTopicPartitions).forEach(tp -> { + List replicas = partitionsToBeReassigned.get(tp); + Map data = new LinkedHashMap<>(); + + data.put("topic", tp.topic()); + data.put("partition", tp.partition()); + data.put("replicas", replicas); + data.put("log_dirs", replicas.stream(). + map(r -> replicaLogDirAssignment.getOrDefault(new TopicPartitionReplica(tp.topic(), tp.partition(), r), ANY_LOG_DIR)). + collect(Collectors.toList())); + + partitions.put(tp, data); + }); + + Map results = new HashMap<>(); + + results.put("version", 1); + results.put("partitions", partitions); + + return Json.encodeAsString(results); } - private static List parseTopicsData(String jsonData) { - return null; + private static List parseTopicsData(String jsonData) throws JsonMappingException { + Optional parsed = Json.parseFull(jsonData); + if (parsed.isPresent()) { + JsonValue js = parsed.get(); + Optional version = js.asJsonObject().get("version"); + return parseTopicsData(version.isPresent() ? version.get().to(INT) : EARLIEST_TOPICS_JSON_VERSION, js); + } else { + throw new AdminOperationException("The input string is not a valid JSON"); + } } - private static List parseTopicsData(int version, JsonValue js) { - return null; + private static List parseTopicsData(int version, JsonValue js) throws JsonMappingException { + switch (version) { + case 1: + List results = new ArrayList<>(); + Optional partitionsSeq = js.asJsonObject().get("topics"); + if (partitionsSeq.isPresent()) { + Iterator iter = partitionsSeq.get().asJsonArray().iterator(); + while (iter.hasNext()) { + results.add(iter.next().asJsonObject().apply("topic").to(STRING)); + } + } + return results; + + default: + throw new AdminOperationException("Not supported version field value " + version); + } } - private static Tuple>>, Map> parsePartitionReassignmentData(String jsonData) { - return null; + private static Tuple>>, Map> parsePartitionReassignmentData(String jsonData) throws JsonProcessingException { + JsonValue js; + try { + js = Json.tryParseFull(jsonData); + } catch(JsonParseException f) { + throw new AdminOperationException(f); + } + Optional version = js.asJsonObject().get("version"); + return parsePartitionReassignmentData(version.isPresent() ? version.get().to(INT) : EARLIEST_VERSION, js); } // Parses without deduplicating keys so the data can be checked before allowing reassignment to proceed - private static Tuple>>, Map> parsePartitionReassignmentData(int version, JsonValue jsonData) { - return null; + private static Tuple>>, Map> parsePartitionReassignmentData(int version, JsonValue jsonData) throws JsonMappingException { + switch (version) { + case 1: + List>> partitionAssignment = new ArrayList<>(); + Map replicaAssignment = new HashMap<>(); + + Optional partitionsSeq = jsonData.asJsonObject().get("topics"); + if (partitionsSeq.isPresent()) { + Iterator iter = partitionsSeq.get().asJsonArray().iterator(); + while (iter.hasNext()) { + JsonObject partitionFields = iter.next().asJsonObject(); + String topic = partitionFields.apply("topic").to(STRING); + int partition = partitionFields.apply("partition").to(INT); + List newReplicas = partitionFields.apply("replicas").to(null); + Optional logDirsOpts = partitionFields.get("log_dirs"); + List newLogDirs; + if (logDirsOpts.isPresent()) + newLogDirs = logDirsOpts.get().to(null); + else + newLogDirs = newReplicas.stream().map(r -> ANY_LOG_DIR).collect(Collectors.toList()); + if (newReplicas.size() != newLogDirs.size()) + throw new AdminCommandFailedException("Size of replicas list " + newReplicas + " is different from " + + "size of log dirs list " + newLogDirs + " for partition " + new TopicPartition(topic, partition)); + partitionAssignment.add(new Tuple<>(new TopicPartition(topic, partition), newReplicas)); + for (int i = 0; i < newLogDirs.size(); i++) { + Integer replica = newReplicas.get(i); + String logDir = newLogDirs.get(i); + + if (logDir.equals(ANY_LOG_DIR)) + continue; + + replicaAssignment.put(new TopicPartitionReplica(topic, partition, replica), logDir); + } + } + } + + return new Tuple<>(partitionAssignment, replicaAssignment); + + default: + throw new AdminOperationException("Not supported version field value " + version); + } } private static ReassignPartitionsCommandOptions validateAndParseArgs(String[] args) { ReassignPartitionsCommandOptions opts = new ReassignPartitionsCommandOptions(args); - CommandLineUtils.maybePrintHelpOrVersion(opts, helpText); + CommandLineUtils.maybePrintHelpOrVersion(opts, HELP_TEXT); // Determine which action we should perform. List> validActions = Arrays.asList(opts.generateOpt, opts.executeOpt, opts.verifyOpt, @@ -1048,7 +1459,25 @@ private static ReassignPartitionsCommandOptions validateAndParseArgs(String[] ar return opts; } - private static Set alterReplicaLogDirs(Admin adminClient, Map assignment) { - return null; + private static Set alterReplicaLogDirs(Admin adminClient, + Map assignment) throws InterruptedException { + Set results = new HashSet<>(); + Map> values = adminClient.alterReplicaLogDirs(assignment).values(); + + for (Map.Entry> e : values.entrySet()) { + TopicPartitionReplica replica = e.getKey(); + KafkaFuture future = e.getValue(); + try { + future.get(); + results.add(replica); + } catch (ExecutionException t) { + // Ignore ReplicaNotAvailableException. It is OK if the replica is not + // available at this moment. + if (t.getCause() instanceof ReplicaNotAvailableException) + continue; + throw new AdminCommandFailedException("Failed to alter dir for " + replica, t); + } + } + return results; } } From 780d9061c44caa31ddad6d2d540b157a8b08fd08 Mon Sep 17 00:00:00 2001 From: nizhikov Date: Tue, 25 Jul 2023 12:45:14 +0300 Subject: [PATCH 14/78] KAFKA-14595 checkstyle fixed --- checkstyle/checkstyle.xml | 2 +- checkstyle/import-control-server-common.xml | 3 + checkstyle/import-control.xml | 3 + .../org/apache/kafka/admin/AdminUtils.java | 132 +++++++++++++++--- .../apache/kafka/admin/BrokerMetadata.java | 7 +- .../reassign/ReassignPartitionCommand.java | 65 +++++---- .../apache/kafka/tools/reassign/Tuple.java | 7 +- 7 files changed, 157 insertions(+), 62 deletions(-) diff --git a/checkstyle/checkstyle.xml b/checkstyle/checkstyle.xml index bf2d339da8c76..f79622009a79f 100644 --- a/checkstyle/checkstyle.xml +++ b/checkstyle/checkstyle.xml @@ -119,7 +119,7 @@ - + diff --git a/checkstyle/import-control-server-common.xml b/checkstyle/import-control-server-common.xml index ebf9984c7a066..a8d032c622113 100644 --- a/checkstyle/import-control-server-common.xml +++ b/checkstyle/import-control-server-common.xml @@ -101,4 +101,7 @@ + + + diff --git a/checkstyle/import-control.xml b/checkstyle/import-control.xml index 7ecc4d5a1d140..a334d58ce56ef 100644 --- a/checkstyle/import-control.xml +++ b/checkstyle/import-control.xml @@ -296,9 +296,11 @@ + + @@ -313,6 +315,7 @@ + diff --git a/server-common/src/main/java/org/apache/kafka/admin/AdminUtils.java b/server-common/src/main/java/org/apache/kafka/admin/AdminUtils.java index 58437b40c310c..13ffa807fc203 100644 --- a/server-common/src/main/java/org/apache/kafka/admin/AdminUtils.java +++ b/server-common/src/main/java/org/apache/kafka/admin/AdminUtils.java @@ -1,12 +1,12 @@ /* * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with + * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at + * the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -14,24 +14,28 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.kafka.admin; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.apache.kafka.common.errors.InvalidPartitionsException; +import org.apache.kafka.common.errors.InvalidReplicationFactorException; +import org.apache.kafka.server.common.AdminOperationException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Random; +import java.util.Set; +import java.util.stream.Collectors; public class AdminUtils { - private static final Logger log = LoggerFactory.getLogger(AdminUtils.class); - - static final Random rand = new Random(); - - static final String AdminClientId = "__admin_client"; + static final Random RAND = new Random(); - public static Map> assignReplicasToBrokers(Iterable brokerMetadatas, + public static Map> assignReplicasToBrokers(Collection brokerMetadatas, int nPartitions, int replicationFactor) { return assignReplicasToBrokers(brokerMetadatas, nPartitions, replicationFactor, -1, -1); @@ -109,28 +113,101 @@ public static Map> assignReplicasToBrokers(Iterable> assignReplicasToBrokers(Iterable brokerMetadatas, + public static Map> assignReplicasToBrokers(Collection brokerMetadatas, int nPartitions, int replicationFactor, int fixedStartIndex, int startPartitionId) { - return null; + if (nPartitions <= 0) + throw new InvalidPartitionsException("Number of partitions must be larger than 0."); + if (replicationFactor <= 0) + throw new InvalidReplicationFactorException("Replication factor must be larger than 0."); + if (replicationFactor > brokerMetadatas.size()) + throw new InvalidReplicationFactorException("Replication factor: " + replicationFactor + " larger than available brokers: " + brokerMetadatas.size() + "."); + if (brokerMetadatas.stream().noneMatch(b -> b.rack.isPresent())) + return assignReplicasToBrokersRackUnaware(nPartitions, replicationFactor, brokerMetadatas.stream().map(b -> b.id).collect(Collectors.toList()), fixedStartIndex, + startPartitionId); + else { + if (brokerMetadatas.stream().anyMatch(b -> !b.rack.isPresent())) + throw new AdminOperationException("Not all brokers have rack information for replica rack aware assignment."); + return assignReplicasToBrokersRackAware(nPartitions, replicationFactor, brokerMetadatas, fixedStartIndex, + startPartitionId); + } } private static Map> assignReplicasToBrokersRackUnaware(int nPartitions, int replicationFactor, - Iterable brokerList, + Collection brokerList, int fixedStartIndex, int startPartitionId) { - return null; + Map> ret = new HashMap<>(); + Integer[] brokerArray = brokerList.toArray(new Integer[0]); + int startIndex = fixedStartIndex >= 0 ? fixedStartIndex : RAND.nextInt(brokerArray.length); + int currentPartitionId = Math.max(0, startPartitionId); + int nextReplicaShift = fixedStartIndex >= 0 ? fixedStartIndex : RAND.nextInt(brokerArray.length); + for (int i = 0; i < nPartitions; i++) { + if (currentPartitionId > 0 && (currentPartitionId % brokerArray.length == 0)) + nextReplicaShift += 1; + int firstReplicaIndex = (currentPartitionId + startIndex) % brokerArray.length; + List replicaBuffer = new ArrayList<>(); + replicaBuffer.add(brokerArray[firstReplicaIndex]); + for (int j = 0; j < replicationFactor - 1; j++) + replicaBuffer.add(brokerArray[replicaIndex(firstReplicaIndex, nextReplicaShift, j, brokerArray.length)]); + ret.put(currentPartitionId, replicaBuffer); + currentPartitionId += 1; + } + return ret; } private static Map> assignReplicasToBrokersRackAware(int nPartitions, int replicationFactor, - Iterable brokerMetadatas, + Collection brokerMetadatas, int fixedStartIndex, int startPartitionId) { - return null; + Map brokerRackMap = new HashMap<>(); + brokerMetadatas.forEach(m -> brokerRackMap.put(m.id, m.rack.get())); + int numRacks = new HashSet<>(brokerRackMap.values()).size(); + List arrangedBrokerList = getRackAlternatedBrokerList(brokerRackMap); + int numBrokers = arrangedBrokerList.size(); + Map> ret = new HashMap<>(); + int startIndex = fixedStartIndex >= 0 ? fixedStartIndex : RAND.nextInt(arrangedBrokerList.size()); + int currentPartitionId = Math.max(0, startPartitionId); + int nextReplicaShift = fixedStartIndex >= 0 ? fixedStartIndex : RAND.nextInt(arrangedBrokerList.size()); + for (int i = 0; i < nPartitions; i++) { + if (currentPartitionId > 0 && (currentPartitionId % arrangedBrokerList.size() == 0)) + nextReplicaShift += 1; + int firstReplicaIndex = (currentPartitionId + startIndex) % arrangedBrokerList.size(); + int leader = arrangedBrokerList.get(firstReplicaIndex); + List replicaBuffer = new ArrayList<>(); + replicaBuffer.add(leader); + Set racksWithReplicas = new HashSet<>(); + racksWithReplicas.add(brokerRackMap.get(leader)); + Set brokersWithReplicas = new HashSet<>(); + brokersWithReplicas.add(leader); + int k = 0; + for (int j = 0; j < replicationFactor - 1; j++) { + boolean done = false; + while (!done) { + Integer broker = arrangedBrokerList.get(replicaIndex(firstReplicaIndex, nextReplicaShift * numRacks, k, arrangedBrokerList.size())); + String rack = brokerRackMap.get(broker); + // Skip this broker if + // 1. there is already a broker in the same rack that has assigned a replica AND there is one or more racks + // that do not have any replica, or + // 2. the broker has already assigned a replica AND there is one or more brokers that do not have replica assigned + if ((!racksWithReplicas.contains(rack) || racksWithReplicas.size() == numRacks) + && (!brokersWithReplicas.contains(broker) || brokersWithReplicas.size() == numBrokers)) { + replicaBuffer.add(broker); + racksWithReplicas.add(rack); + brokersWithReplicas.add(broker); + done = true; + } + k += 1; + } + } + ret.put(currentPartitionId, replicaBuffer); + currentPartitionId += 1; + } + return ret; } /** @@ -149,11 +226,26 @@ private static Map> assignReplicasToBrokersRackAware(int * distributed to all racks. */ static List getRackAlternatedBrokerList(Map brokerRackMap) { - return null; + Map> brokersIteratorByRack = new HashMap<>(); + getInverseMap(brokerRackMap).forEach((rack, brokers) -> brokersIteratorByRack.put(rack, brokers.iterator())); + String[] racks = brokersIteratorByRack.keySet().toArray(new String[0]); + Arrays.sort(racks); + List results = new ArrayList<>(); + int rackIndex = 0; + while (results.size() < brokerRackMap.size()) { + Iterator rackIterator = brokersIteratorByRack.get(racks[rackIndex]); + if (rackIterator.hasNext()) + results.add(rackIterator.next()); + rackIndex = (rackIndex + 1) % racks.length; + } + return results; } static Map> getInverseMap(Map brokerRackMap) { - return null; + Map> results = new HashMap<>(); + brokerRackMap.forEach((id, rack) -> results.computeIfAbsent(rack, key -> new ArrayList<>()).add(id)); + results.forEach((rack, rackAndIdList) -> rackAndIdList.sort(Integer::compareTo)); + return results; } static int replicaIndex(int firstReplicaIndex, int secondReplicaShift, int replicaIndex, int nBrokers) { diff --git a/server-common/src/main/java/org/apache/kafka/admin/BrokerMetadata.java b/server-common/src/main/java/org/apache/kafka/admin/BrokerMetadata.java index 722a28c9fba57..de7bca7e5749e 100644 --- a/server-common/src/main/java/org/apache/kafka/admin/BrokerMetadata.java +++ b/server-common/src/main/java/org/apache/kafka/admin/BrokerMetadata.java @@ -1,12 +1,12 @@ /* * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with + * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at + * the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.kafka.admin; import java.util.Optional; diff --git a/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionCommand.java b/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionCommand.java index 79ea8569ad82b..2393fefe8aa9a 100644 --- a/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionCommand.java +++ b/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionCommand.java @@ -1,12 +1,12 @@ /* * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with + * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at + * the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.kafka.tools.reassign; import com.fasterxml.jackson.core.JsonParseException; @@ -92,33 +91,33 @@ public class ReassignPartitionCommand { // Throttles that are set at the level of an individual broker. //DynamicConfig.Broker.LeaderReplicationThrottledRateProp - private static final String brokerLevelLeaderThrottle = "leader.replication.throttled.rate"; + private static final String BROKER_LEVEL_LEADER_THROTTLE = "leader.replication.throttled.rate"; //DynamicConfig.Broker.FollowerReplicationThrottledRateProp - private static final String brokerLevelFollowerThrottle = "follower.replication.throttled.rate"; + private static final String BROKER_LEVEL_FOLLOWER_THROTTLE = "follower.replication.throttled.rate"; //DynamicConfig.Broker.ReplicaAlterLogDirsIoMaxBytesPerSecondProp - private static final String brokerLevelLogDirThrottle = "replica.alter.log.dirs.io.max.bytes.per.second"; - private static final List brokerLevelThrottles = Arrays.asList( - brokerLevelLeaderThrottle, - brokerLevelFollowerThrottle, - brokerLevelLogDirThrottle + private static final String BROKER_LEVEL_LOG_DIR_THROTTLE = "replica.alter.log.dirs.io.max.bytes.per.second"; + private static final List BROKER_LEVEL_THROTTLES = Arrays.asList( + BROKER_LEVEL_LEADER_THROTTLE, + BROKER_LEVEL_FOLLOWER_THROTTLE, + BROKER_LEVEL_LOG_DIR_THROTTLE ); // Throttles that are set at the level of an individual topic. //LogConfig.LEADER_REPLICATION_THROTTLED_REPLICAS_CONFIG - private static final String topicLevelLeaderThrottle = "leader.replication.throttled.replicas"; + private static final String TOPIC_LEVEL_LEADER_THROTTLE = "leader.replication.throttled.replicas"; //LogConfig.FOLLOWER_REPLICATION_THROTTLED_REPLICAS_CONFIG - private static final String topicLevelFollowerThrottle = "follower.replication.throttled.replicas"; - private static final List topicLevelThrottles = Arrays.asList( - topicLevelLeaderThrottle, - topicLevelFollowerThrottle + private static final String TOPIC_LEVEL_FOLLOWER_THROTTLE = "follower.replication.throttled.replicas"; + private static final List TOPIC_LEVEL_THROTTLES = Arrays.asList( + TOPIC_LEVEL_LEADER_THROTTLE, + TOPIC_LEVEL_FOLLOWER_THROTTLE ); - private static final String cannotExecuteBecauseOfExistingMessage = "Cannot execute because " + + private static final String CANNOT_EXECUTE_BECAUSE_OF_EXISTING_MESSAGE = "Cannot execute because " + "there is an existing partition assignment. Use --additional to override this and " + "create a new partition assignment in addition to the existing one. The --additional " + "flag can also be used to change the throttle by resubmitting the current reassignment."; - private static final String youMustRunVerifyPeriodicallyMessage = "Warning: You must run " + + private static final String YOU_MUST_RUN_VERIFY_PERIODICALLY_MESSAGE = "Warning: You must run " + "--verify periodically, until the reassignment completes, to ensure the throttle " + "is removed."; @@ -422,7 +421,7 @@ private static Map findLogDirMoveStates( moveState = new MissingReplicaMoveState(targetLogDir); } else { DescribeReplicaLogDirsResult.ReplicaLogDirInfo info = replicaLogDirInfos.get(replica); - if (info.getCurrentReplicaLogDir() == null){ + if (info.getCurrentReplicaLogDir() == null) { moveState = new MissingLogDirMoveState(targetLogDir); } else if (info.getFutureReplicaLogDir() == null) { if (info.getCurrentReplicaLogDir().equals(targetLogDir)) { @@ -515,7 +514,7 @@ private static void clearBrokerLevelThrottles(Admin adminClient, Set br Map> configOps = new HashMap<>(); brokers.forEach(brokerId -> configOps.put( new ConfigResource(ConfigResource.Type.BROKER, brokerId.toString()), - brokerLevelThrottles.stream().map(throttle -> new AlterConfigOp( + BROKER_LEVEL_THROTTLES.stream().map(throttle -> new AlterConfigOp( new ConfigEntry(throttle, null), AlterConfigOp.OpType.DELETE)).collect(Collectors.toList()) )); adminClient.incrementalAlterConfigs(configOps).all().get(); @@ -531,7 +530,7 @@ private static void clearTopicLevelThrottles(Admin adminClient, Set topi Map> configOps = new HashMap<>(); topics.forEach(topicName -> configOps.put( new ConfigResource(ConfigResource.Type.TOPIC, topicName), - topicLevelThrottles.stream().map(throttle -> new AlterConfigOp(new ConfigEntry(throttle, null), + TOPIC_LEVEL_THROTTLES.stream().map(throttle -> new AlterConfigOp(new ConfigEntry(throttle, null), AlterConfigOp.OpType.DELETE)).collect(Collectors.toList()) )); @@ -561,9 +560,9 @@ private static Tuple>, Map> currentAssignments = getReplicaAssignmentForTopics(adminClient, topicsToReassign); List brokerMetadatas = getBrokerMetadata(adminClient, brokersToReassign, enableRackAwareness); Map> proposedAssignments = calculateAssignment(currentAssignments, brokerMetadatas); - System.out.printf("Current partition replica assignment\n%s\n%n", + System.out.printf("Current partition replica assignment%n%s%n%n", formatAsReassignmentJson(currentAssignments, Collections.emptyMap())); - System.out.printf("Proposed partition reassignment configuration\n%s", + System.out.printf("Proposed partition reassignment configuration%n%s", formatAsReassignmentJson(proposedAssignments, Collections.emptyMap())); return new Tuple<>(proposedAssignments, currentAssignments); } @@ -735,7 +734,7 @@ private static void executeAssignment(Admin adminClient, // If there is an existing assignment, check for --additional before proceeding. // This helps avoid surprising users. if (!additional && !currentReassignments.isEmpty()) { - throw new TerseReassignmentFailureException(cannotExecuteBecauseOfExistingMessage); + throw new TerseReassignmentFailureException(CANNOT_EXECUTE_BECAUSE_OF_EXISTING_MESSAGE); } Set brokers = new HashSet<>(); proposedParts.values().forEach(brokers::addAll); @@ -745,7 +744,7 @@ private static void executeAssignment(Admin adminClient, System.out.println(currentPartitionReplicaAssignmentToString(proposedParts, currentParts)); if (interBrokerThrottle >= 0 || logDirThrottle >= 0) { - System.out.println(youMustRunVerifyPeriodicallyMessage); + System.out.println(YOU_MUST_RUN_VERIFY_PERIODICALLY_MESSAGE); if (interBrokerThrottle >= 0) { Map> moveMap = calculateProposedMoveMap(currentReassignments, proposedParts, currentParts); @@ -926,7 +925,7 @@ private static Map alterPartitionReassignments(Admin for (Map.Entry> e : results.entrySet()) { try { e.getValue().get(); - } catch(ExecutionException t) { + } catch (ExecutionException t) { errors.put(e.getKey(), t.getCause()); } } @@ -950,7 +949,7 @@ private static Map cancelPartitionReassignments(Admin for (Map.Entry> e : results.entrySet()) { try { e.getValue().get(); - } catch(ExecutionException t) { + } catch (ExecutionException t) { errors.put(e.getKey(), t.getCause()); } } @@ -1092,10 +1091,10 @@ private static void modifyTopicThrottles(Admin adminClient, topicNames.forEach(topicName -> { List ops = new ArrayList<>(); if (leaderThrottles.containsKey(topicName)) { - ops.add(new AlterConfigOp(new ConfigEntry(topicLevelLeaderThrottle, leaderThrottles.get(topicName)), AlterConfigOp.OpType.SET)); + ops.add(new AlterConfigOp(new ConfigEntry(TOPIC_LEVEL_LEADER_THROTTLE, leaderThrottles.get(topicName)), AlterConfigOp.OpType.SET)); } if (followerThrottles.containsKey(topicName)) { - ops.add(new AlterConfigOp(new ConfigEntry(topicLevelFollowerThrottle, leaderThrottles.get(topicName)), AlterConfigOp.OpType.SET)); + ops.add(new AlterConfigOp(new ConfigEntry(TOPIC_LEVEL_FOLLOWER_THROTTLE, leaderThrottles.get(topicName)), AlterConfigOp.OpType.SET)); } if (!ops.isEmpty()) { configs.put(new ConfigResource(ConfigResource.Type.TOPIC, topicName), ops); @@ -1127,9 +1126,9 @@ private static void modifyInterBrokerThrottle(Admin adminClient, Map> configs = new HashMap<>(); reassigningBrokers.forEach(brokerId -> { List ops = new ArrayList<>(); - ops.add(new AlterConfigOp(new ConfigEntry(brokerLevelLeaderThrottle, + ops.add(new AlterConfigOp(new ConfigEntry(BROKER_LEVEL_LEADER_THROTTLE, Long.toString(interBrokerThrottle)), AlterConfigOp.OpType.SET)); - ops.add(new AlterConfigOp(new ConfigEntry(brokerLevelFollowerThrottle, + ops.add(new AlterConfigOp(new ConfigEntry(BROKER_LEVEL_FOLLOWER_THROTTLE, Long.toString(interBrokerThrottle)), AlterConfigOp.OpType.SET)); configs.put(new ConfigResource(ConfigResource.Type.BROKER, Long.toString(brokerId)), ops); }); @@ -1152,7 +1151,7 @@ private static void modifyLogDirThrottle(Admin admin, Map> configs = new HashMap<>(); movingBrokers.forEach(brokerId -> { List ops = new ArrayList<>(); - ops.add(new AlterConfigOp(new ConfigEntry(brokerLevelLogDirThrottle, Long.toString(logDirThrottle)), AlterConfigOp.OpType.SET)); + ops.add(new AlterConfigOp(new ConfigEntry(BROKER_LEVEL_LOG_DIR_THROTTLE, Long.toString(logDirThrottle)), AlterConfigOp.OpType.SET)); configs.put(new ConfigResource(ConfigResource.Type.BROKER, Long.toString(brokerId)), ops); }); admin.incrementalAlterConfigs(configs).all().get(); @@ -1320,7 +1319,7 @@ private static Tuple>>, Map version = js.asJsonObject().get("version"); diff --git a/tools/src/main/java/org/apache/kafka/tools/reassign/Tuple.java b/tools/src/main/java/org/apache/kafka/tools/reassign/Tuple.java index 6606bfe7d3866..6943e62f22fcf 100644 --- a/tools/src/main/java/org/apache/kafka/tools/reassign/Tuple.java +++ b/tools/src/main/java/org/apache/kafka/tools/reassign/Tuple.java @@ -1,12 +1,12 @@ /* * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with + * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at + * the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -14,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.kafka.tools.reassign; import java.util.Objects; From 22a1382a7ec4782367d17fd095f74a92999711c4 Mon Sep 17 00:00:00 2001 From: nizhikov Date: Tue, 25 Jul 2023 16:16:50 +0300 Subject: [PATCH 15/78] KAFKA-14595 scala BrokerMetadata removed --- .../main/scala/kafka/admin/AdminUtils.scala | 9 ++++---- .../scala/kafka/admin/BrokerMetadata.scala | 23 ------------------- .../admin/ReassignPartitionsCommand.scala | 7 +++--- .../main/scala/kafka/server/KafkaApis.scala | 4 ++-- .../scala/kafka/server/MetadataCache.scala | 2 +- .../server/metadata/KRaftMetadataCache.scala | 4 ++-- .../server/metadata/ZkMetadataCache.scala | 7 +++--- .../main/scala/kafka/zk/AdminZkClient.scala | 11 +++++---- .../unit/kafka/admin/AdminRackAwareTest.scala | 4 +++- .../unit/kafka/admin/RackAwareTest.scala | 8 +++++-- .../admin/ReassignPartitionsUnitTest.scala | 15 ++++++------ .../scala/unit/kafka/utils/TestUtils.scala | 7 +++--- .../unit/kafka/zk/AdminZkClientTest.scala | 8 +++---- .../org/apache/kafka/admin/AdminUtils.java | 2 ++ .../apache/kafka/admin/BrokerMetadata.java | 22 ++++++++++++++++++ 15 files changed, 72 insertions(+), 61 deletions(-) delete mode 100644 core/src/main/scala/kafka/admin/BrokerMetadata.scala diff --git a/core/src/main/scala/kafka/admin/AdminUtils.scala b/core/src/main/scala/kafka/admin/AdminUtils.scala index 5ac09ab534895..1ab4643397d71 100644 --- a/core/src/main/scala/kafka/admin/AdminUtils.scala +++ b/core/src/main/scala/kafka/admin/AdminUtils.scala @@ -19,6 +19,7 @@ package kafka.admin import java.util.Random import kafka.utils.Logging +import org.apache.kafka.admin.BrokerMetadata import org.apache.kafka.common.errors.{InvalidPartitionsException, InvalidReplicationFactorException} import org.apache.kafka.server.common.AdminOperationException @@ -111,11 +112,11 @@ object AdminUtils extends Logging { throw new InvalidReplicationFactorException("Replication factor must be larger than 0.") if (replicationFactor > brokerMetadatas.size) throw new InvalidReplicationFactorException(s"Replication factor: $replicationFactor larger than available brokers: ${brokerMetadatas.size}.") - if (brokerMetadatas.forall(_.rack.isEmpty)) + if (brokerMetadatas.forall(!_.rack.isPresent)) assignReplicasToBrokersRackUnaware(nPartitions, replicationFactor, brokerMetadatas.map(_.id), fixedStartIndex, startPartitionId) else { - if (brokerMetadatas.exists(_.rack.isEmpty)) + if (brokerMetadatas.exists(!_.rack.isPresent)) throw new AdminOperationException("Not all brokers have rack information for replica rack aware assignment.") assignReplicasToBrokersRackAware(nPartitions, replicationFactor, brokerMetadatas, fixedStartIndex, startPartitionId) @@ -150,8 +151,8 @@ object AdminUtils extends Logging { brokerMetadatas: Iterable[BrokerMetadata], fixedStartIndex: Int, startPartitionId: Int): Map[Int, Seq[Int]] = { - val brokerRackMap = brokerMetadatas.collect { case BrokerMetadata(id, Some(rack)) => - id -> rack + val brokerRackMap = brokerMetadatas.collect { bm => + bm.id -> bm.rack.get() }.toMap val numRacks = brokerRackMap.values.toSet.size val arrangedBrokerList = getRackAlternatedBrokerList(brokerRackMap) diff --git a/core/src/main/scala/kafka/admin/BrokerMetadata.scala b/core/src/main/scala/kafka/admin/BrokerMetadata.scala deleted file mode 100644 index 86831e376e5bd..0000000000000 --- a/core/src/main/scala/kafka/admin/BrokerMetadata.scala +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE - * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file - * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -package kafka.admin - -/** - * Broker metadata used by admin tools. - * - * @param id an integer that uniquely identifies this broker - * @param rack the rack of the broker, which is used to in rack aware partition assignment for fault tolerance. - * Examples: "RACK1", "us-east-1d" - */ -case class BrokerMetadata(id: Int, rack: Option[String]) diff --git a/core/src/main/scala/kafka/admin/ReassignPartitionsCommand.scala b/core/src/main/scala/kafka/admin/ReassignPartitionsCommand.scala index 40f688c7085e2..22ae9c281b0ed 100755 --- a/core/src/main/scala/kafka/admin/ReassignPartitionsCommand.scala +++ b/core/src/main/scala/kafka/admin/ReassignPartitionsCommand.scala @@ -23,6 +23,7 @@ import kafka.server.DynamicConfig import kafka.utils.{CoreUtils, Exit, Json, Logging} import kafka.utils.Implicits._ import kafka.utils.json.JsonValue +import org.apache.kafka.admin.BrokerMetadata import org.apache.kafka.clients.admin.AlterConfigOp.OpType import org.apache.kafka.clients.admin.{Admin, AdminClientConfig, AlterConfigOp, ConfigEntry, NewPartitionReassignment, PartitionReassignment, TopicDescription} import org.apache.kafka.common.config.ConfigResource @@ -688,12 +689,12 @@ object ReassignPartitionsCommand extends Logging { filter(node => brokerSet.contains(node.id)). map { node => if (enableRackAwareness && node.rack != null) { - BrokerMetadata(node.id, Some(node.rack)) + new BrokerMetadata(node.id, Optional.of(node.rack)) } else { - BrokerMetadata(node.id, None) + new BrokerMetadata(node.id, Optional.empty()) } }.toSeq - val numRackless = results.count(_.rack.isEmpty) + val numRackless = results.count(!_.rack.isPresent) if (enableRackAwareness && numRackless != 0 && numRackless != results.size) { throw new AdminOperationException("Not all brokers have rack information. Add " + "--disable-rack-aware in command line to make replica assignment without rack " + diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala index 5d6233d110005..15c4e356bdce2 100644 --- a/core/src/main/scala/kafka/server/KafkaApis.scala +++ b/core/src/main/scala/kafka/server/KafkaApis.scala @@ -17,7 +17,6 @@ package kafka.server -import kafka.admin.AdminUtils import kafka.api.ElectLeadersRequestOps import kafka.controller.ReplicaAssignment import kafka.coordinator.transaction.{InitProducerIdResult, TransactionCoordinator} @@ -26,6 +25,7 @@ import kafka.server.QuotaFactory.{QuotaManagers, UnboundedQuota} import kafka.server.metadata.ConfigRepository import kafka.utils.Implicits._ import kafka.utils.{CoreUtils, Logging} +import org.apache.kafka.admin.AdminUtils import org.apache.kafka.clients.admin.AlterConfigOp.OpType import org.apache.kafka.clients.admin.{AlterConfigOp, ConfigEntry} import org.apache.kafka.common.acl.AclOperation._ @@ -668,7 +668,7 @@ class KafkaApis(val requestChannel: RequestChannel, if (authorizedRequestInfo.isEmpty) sendResponseCallback(Map.empty) else { - val internalTopicsAllowed = request.header.clientId == AdminUtils.AdminClientId + val internalTopicsAllowed = request.header.clientId == AdminUtils.ADMIN_CLIENT_ID val transactionStatePartition = if (produceRequest.transactionalId() == null) diff --git a/core/src/main/scala/kafka/server/MetadataCache.scala b/core/src/main/scala/kafka/server/MetadataCache.scala index ecd64c17b91f3..2e60ce2ba7bd9 100755 --- a/core/src/main/scala/kafka/server/MetadataCache.scala +++ b/core/src/main/scala/kafka/server/MetadataCache.scala @@ -17,8 +17,8 @@ package kafka.server -import kafka.admin.BrokerMetadata import kafka.server.metadata.{KRaftMetadataCache, ZkMetadataCache} +import org.apache.kafka.admin.BrokerMetadata import org.apache.kafka.common.message.{MetadataResponseData, UpdateMetadataRequestData} import org.apache.kafka.common.network.ListenerName import org.apache.kafka.common.{Cluster, Node, TopicPartition, Uuid} diff --git a/core/src/main/scala/kafka/server/metadata/KRaftMetadataCache.scala b/core/src/main/scala/kafka/server/metadata/KRaftMetadataCache.scala index 5e3ed11022b4f..559ccb2e9179a 100644 --- a/core/src/main/scala/kafka/server/metadata/KRaftMetadataCache.scala +++ b/core/src/main/scala/kafka/server/metadata/KRaftMetadataCache.scala @@ -20,6 +20,7 @@ package kafka.server.metadata import kafka.controller.StateChangeLogger import kafka.server.{CachedControllerId, KRaftCachedControllerId, MetadataCache} import kafka.utils.Logging +import org.apache.kafka.admin.BrokerMetadata import org.apache.kafka.common.internals.Topic import org.apache.kafka.common.message.MetadataResponseData.{MetadataResponsePartition, MetadataResponseTopic} import org.apache.kafka.common.{Cluster, Node, PartitionInfo, TopicPartition, Uuid} @@ -32,7 +33,6 @@ import org.apache.kafka.image.MetadataImage import java.util import java.util.{Collections, Properties} import java.util.concurrent.ThreadLocalRandom -import kafka.admin.BrokerMetadata import org.apache.kafka.common.config.ConfigResource import org.apache.kafka.common.message.{DescribeClientQuotasRequestData, DescribeClientQuotasResponseData} import org.apache.kafka.common.message.{DescribeUserScramCredentialsRequestData, DescribeUserScramCredentialsResponseData} @@ -215,7 +215,7 @@ class KRaftMetadataCache(val brokerId: Int) extends MetadataCache with Logging w private def getAliveBrokers(image: MetadataImage): Iterable[BrokerMetadata] = { image.cluster().brokers().values().asScala.filterNot(_.fenced()). - map(b => BrokerMetadata(b.id, b.rack.asScala)) + map(b => new BrokerMetadata(b.id, b.rack)) } override def getAliveBrokerNode(brokerId: Int, listenerName: ListenerName): Option[Node] = { diff --git a/core/src/main/scala/kafka/server/metadata/ZkMetadataCache.scala b/core/src/main/scala/kafka/server/metadata/ZkMetadataCache.scala index 9159b7918809b..302d3fbf8de06 100755 --- a/core/src/main/scala/kafka/server/metadata/ZkMetadataCache.scala +++ b/core/src/main/scala/kafka/server/metadata/ZkMetadataCache.scala @@ -18,10 +18,8 @@ package kafka.server.metadata import java.util -import java.util.Collections +import java.util.{Collections, Optional} import java.util.concurrent.locks.{ReentrantLock, ReentrantReadWriteLock} -import kafka.admin.BrokerMetadata - import scala.collection.{Seq, Set, mutable} import scala.jdk.CollectionConverters._ import kafka.cluster.{Broker, EndPoint} @@ -31,6 +29,7 @@ import kafka.server.{BrokerFeatures, CachedControllerId, KRaftCachedControllerId import kafka.utils.CoreUtils._ import kafka.utils.Logging import kafka.utils.Implicits._ +import org.apache.kafka.admin.BrokerMetadata import org.apache.kafka.common.internals.Topic import org.apache.kafka.common.message.UpdateMetadataRequestData.UpdateMetadataPartitionState import org.apache.kafka.common.{Cluster, Node, PartitionInfo, TopicPartition, Uuid} @@ -254,7 +253,7 @@ class ZkMetadataCache( override def hasAliveBroker(brokerId: Int): Boolean = metadataSnapshot.aliveBrokers.contains(brokerId) override def getAliveBrokers(): Iterable[BrokerMetadata] = { - metadataSnapshot.aliveBrokers.values.map(b => new BrokerMetadata(b.id, b.rack)) + metadataSnapshot.aliveBrokers.values.map(b => new BrokerMetadata(b.id, Optional.ofNullable(b.rack.orNull))) } override def getAliveBrokerNode(brokerId: Int, listenerName: ListenerName): Option[Node] = { diff --git a/core/src/main/scala/kafka/zk/AdminZkClient.scala b/core/src/main/scala/kafka/zk/AdminZkClient.scala index 07ca35cc9ae06..d72a4594d8a88 100644 --- a/core/src/main/scala/kafka/zk/AdminZkClient.scala +++ b/core/src/main/scala/kafka/zk/AdminZkClient.scala @@ -16,13 +16,14 @@ */ package kafka.zk -import java.util.Properties -import kafka.admin.{AdminUtils, BrokerMetadata, RackAwareMode} +import java.util.{Optional, Properties} +import kafka.admin.{AdminUtils, RackAwareMode} import kafka.common.TopicAlreadyMarkedForDeletionException import kafka.controller.ReplicaAssignment import kafka.server.{ConfigEntityName, ConfigType, DynamicConfig} import kafka.utils._ import kafka.utils.Implicits._ +import org.apache.kafka.admin.BrokerMetadata import org.apache.kafka.common.{TopicPartition, Uuid} import org.apache.kafka.common.errors._ import org.apache.kafka.common.internals.Topic @@ -77,10 +78,10 @@ class AdminZkClient(zkClient: KafkaZkClient) extends Logging { " to make replica assignment without rack information.") } val brokerMetadatas = rackAwareMode match { - case RackAwareMode.Disabled => brokers.map(broker => BrokerMetadata(broker.id, None)) + case RackAwareMode.Disabled => brokers.map(broker => new BrokerMetadata(broker.id, Optional.empty())) case RackAwareMode.Safe if brokersWithRack.size < brokers.size => - brokers.map(broker => BrokerMetadata(broker.id, None)) - case _ => brokers.map(broker => BrokerMetadata(broker.id, broker.rack)) + brokers.map(broker => new BrokerMetadata(broker.id, Optional.empty())) + case _ => brokers.map(broker => new BrokerMetadata(broker.id, Optional.ofNullable(broker.rack.orNull))) } brokerMetadatas.sortBy(_.id) } diff --git a/core/src/test/scala/unit/kafka/admin/AdminRackAwareTest.scala b/core/src/test/scala/unit/kafka/admin/AdminRackAwareTest.scala index d2d665bef8a54..d430c686e0b48 100644 --- a/core/src/test/scala/unit/kafka/admin/AdminRackAwareTest.scala +++ b/core/src/test/scala/unit/kafka/admin/AdminRackAwareTest.scala @@ -17,10 +17,12 @@ package kafka.admin import kafka.utils.Logging +import org.apache.kafka.admin.BrokerMetadata import org.apache.kafka.common.errors.InvalidReplicationFactorException import org.junit.jupiter.api.Assertions._ import org.junit.jupiter.api.Test +import java.util.Optional import scala.collection.Map class AdminRackAwareTest extends RackAwareTest with Logging { @@ -196,7 +198,7 @@ class AdminRackAwareTest extends RackAwareTest with Logging { @Test def testReplicaAssignment(): Unit = { - val brokerMetadatas = (0 to 4).map(new BrokerMetadata(_, None)) + val brokerMetadatas = (0 to 4).map(new BrokerMetadata(_, Optional.empty())) // test 0 replication factor assertThrows(classOf[InvalidReplicationFactorException], diff --git a/core/src/test/scala/unit/kafka/admin/RackAwareTest.scala b/core/src/test/scala/unit/kafka/admin/RackAwareTest.scala index 6ce4f7ba55d9a..c436f9066e86d 100644 --- a/core/src/test/scala/unit/kafka/admin/RackAwareTest.scala +++ b/core/src/test/scala/unit/kafka/admin/RackAwareTest.scala @@ -16,9 +16,13 @@ */ package kafka.admin +import org.apache.kafka.admin.BrokerMetadata + import scala.collection.{Map, Seq, mutable} import org.junit.jupiter.api.Assertions._ +import java.util.Optional + trait RackAwareTest { def checkReplicaDistribution(assignment: Map[Int, Seq[Int]], @@ -75,9 +79,9 @@ trait RackAwareTest { def toBrokerMetadata(rackMap: Map[Int, String], brokersWithoutRack: Seq[Int] = Seq.empty): Seq[BrokerMetadata] = rackMap.toSeq.map { case (brokerId, rack) => - BrokerMetadata(brokerId, Some(rack)) + new BrokerMetadata(brokerId, Optional.of(rack)) } ++ brokersWithoutRack.map { brokerId => - BrokerMetadata(brokerId, None) + new BrokerMetadata(brokerId, Optional.empty()) }.sortBy(_.id) } diff --git a/core/src/test/scala/unit/kafka/admin/ReassignPartitionsUnitTest.scala b/core/src/test/scala/unit/kafka/admin/ReassignPartitionsUnitTest.scala index fab7907e17be7..7aa0f8eec23e7 100644 --- a/core/src/test/scala/unit/kafka/admin/ReassignPartitionsUnitTest.scala +++ b/core/src/test/scala/unit/kafka/admin/ReassignPartitionsUnitTest.scala @@ -18,9 +18,10 @@ package kafka.admin import java.util.concurrent.ExecutionException -import java.util.{Arrays, Collections} +import java.util.{Arrays, Collections, Optional} import kafka.admin.ReassignPartitionsCommand._ import kafka.utils.Exit +import org.apache.kafka.admin.BrokerMetadata import org.apache.kafka.clients.admin.{Config, MockAdminClient, PartitionReassignment} import org.apache.kafka.common.config.ConfigResource import org.apache.kafka.common.errors.{InvalidReplicationFactorException, UnknownTopicOrPartitionException} @@ -230,19 +231,19 @@ class ReassignPartitionsUnitTest { build() try { assertEquals(Seq( - BrokerMetadata(0, Some("rack0")), - BrokerMetadata(1, Some("rack1")) + new BrokerMetadata(0, Optional.of("rack0")), + new BrokerMetadata(1, Optional.of("rack1")) ), getBrokerMetadata(adminClient, Seq(0, 1), true)) assertEquals(Seq( - BrokerMetadata(0, None), - BrokerMetadata(1, None) + new BrokerMetadata(0, Optional.empty()), + new BrokerMetadata(1, Optional.empty()) ), getBrokerMetadata(adminClient, Seq(0, 1), false)) assertStartsWith("Not all brokers have rack information", assertThrows(classOf[AdminOperationException], () => getBrokerMetadata(adminClient, Seq(1, 2), true)).getMessage) assertEquals(Seq( - BrokerMetadata(1, None), - BrokerMetadata(2, None) + new BrokerMetadata(1, Optional.empty()), + new BrokerMetadata(2, Optional.empty()) ), getBrokerMetadata(adminClient, Seq(1, 2), false)) } finally { adminClient.close() diff --git a/core/src/test/scala/unit/kafka/utils/TestUtils.scala b/core/src/test/scala/unit/kafka/utils/TestUtils.scala index e587c8cb6e8f6..d5b56ac0c5cea 100755 --- a/core/src/test/scala/unit/kafka/utils/TestUtils.scala +++ b/core/src/test/scala/unit/kafka/utils/TestUtils.scala @@ -41,6 +41,7 @@ import kafka.server.checkpoints.OffsetCheckpointFile import kafka.server.metadata.{ConfigRepository, MockConfigRepository} import kafka.utils.Implicits._ import kafka.zk._ +import org.apache.kafka.admin.BrokerMetadata import org.apache.kafka.clients.{ClientResponse, CommonClientConfigs} import org.apache.kafka.clients.admin.AlterConfigOp.OpType import org.apache.kafka.clients.admin._ @@ -893,14 +894,14 @@ object TestUtils extends Logging { } def createBrokersInZk(zkClient: KafkaZkClient, ids: Seq[Int]): Seq[Broker] = - createBrokersInZk(ids.map(kafka.admin.BrokerMetadata(_, None)), zkClient) + createBrokersInZk(ids.map(new BrokerMetadata(_, Optional.empty())), zkClient) - def createBrokersInZk(brokerMetadatas: Seq[kafka.admin.BrokerMetadata], zkClient: KafkaZkClient): Seq[Broker] = { + def createBrokersInZk(brokerMetadatas: Seq[BrokerMetadata], zkClient: KafkaZkClient): Seq[Broker] = { zkClient.makeSurePersistentPathExists(BrokerIdsZNode.path) val brokers = brokerMetadatas.map { b => val protocol = SecurityProtocol.PLAINTEXT val listenerName = ListenerName.forSecurityProtocol(protocol) - Broker(b.id, Seq(EndPoint("localhost", 6667, listenerName, protocol)), b.rack) + Broker(b.id, Seq(EndPoint("localhost", 6667, listenerName, protocol)), if (b.rack.isPresent) Some(b.rack.get) else None) } brokers.foreach(b => zkClient.registerBroker(BrokerInfo(Broker(b.id, b.endPoints, rack = b.rack), MetadataVersion.latest, jmxPort = -1))) diff --git a/core/src/test/scala/unit/kafka/zk/AdminZkClientTest.scala b/core/src/test/scala/unit/kafka/zk/AdminZkClientTest.scala index ba0a2583598db..3ff640fa76037 100644 --- a/core/src/test/scala/unit/kafka/zk/AdminZkClientTest.scala +++ b/core/src/test/scala/unit/kafka/zk/AdminZkClientTest.scala @@ -17,7 +17,7 @@ package kafka.admin import java.util -import java.util.Properties +import java.util.{Optional, Properties} import kafka.controller.ReplicaAssignment import kafka.server.DynamicConfig.Broker._ import kafka.server.KafkaConfig._ @@ -340,18 +340,18 @@ class AdminZkClientTest extends QuorumTestHarness with Logging with RackAwareTes val processedMetadatas1 = adminZkClient.getBrokerMetadatas(RackAwareMode.Disabled) assertEquals(brokerList, processedMetadatas1.map(_.id)) - assertEquals(List.fill(brokerList.size)(None), processedMetadatas1.map(_.rack)) + assertEquals(List.fill(brokerList.size)(Optional.empty()), processedMetadatas1.map(_.rack)) val processedMetadatas2 = adminZkClient.getBrokerMetadatas(RackAwareMode.Safe) assertEquals(brokerList, processedMetadatas2.map(_.id)) - assertEquals(List.fill(brokerList.size)(None), processedMetadatas2.map(_.rack)) + assertEquals(List.fill(brokerList.size)(Optional.empty()), processedMetadatas2.map(_.rack)) assertThrows(classOf[AdminOperationException], () => adminZkClient.getBrokerMetadatas(RackAwareMode.Enforced)) val partialList = List(0, 1, 2, 3, 5) val processedMetadatas3 = adminZkClient.getBrokerMetadatas(RackAwareMode.Enforced, Some(partialList)) assertEquals(partialList, processedMetadatas3.map(_.id)) - assertEquals(partialList.map(rackInfo), processedMetadatas3.flatMap(_.rack)) + assertEquals(partialList.map(rackInfo), processedMetadatas3.map(_.rack.get())) val numPartitions = 3 adminZkClient.createTopic("foo", numPartitions, 2, rackAwareMode = RackAwareMode.Safe) diff --git a/server-common/src/main/java/org/apache/kafka/admin/AdminUtils.java b/server-common/src/main/java/org/apache/kafka/admin/AdminUtils.java index 13ffa807fc203..84aa02251d404 100644 --- a/server-common/src/main/java/org/apache/kafka/admin/AdminUtils.java +++ b/server-common/src/main/java/org/apache/kafka/admin/AdminUtils.java @@ -35,6 +35,8 @@ public class AdminUtils { static final Random RAND = new Random(); + public static final String ADMIN_CLIENT_ID = "__admin_client"; + public static Map> assignReplicasToBrokers(Collection brokerMetadatas, int nPartitions, int replicationFactor) { diff --git a/server-common/src/main/java/org/apache/kafka/admin/BrokerMetadata.java b/server-common/src/main/java/org/apache/kafka/admin/BrokerMetadata.java index de7bca7e5749e..98216fa0858c7 100644 --- a/server-common/src/main/java/org/apache/kafka/admin/BrokerMetadata.java +++ b/server-common/src/main/java/org/apache/kafka/admin/BrokerMetadata.java @@ -16,15 +16,37 @@ */ package org.apache.kafka.admin; +import java.util.Objects; import java.util.Optional; +/** + * Broker metadata used by admin tools. + */ public class BrokerMetadata { public final int id; public final Optional rack; + /** + * @param id an integer that uniquely identifies this broker + * @param rack the rack of the broker, which is used to in rack aware partition assignment for fault tolerance. + * Examples: "RACK1", "us-east-1d" + */ public BrokerMetadata(int id, Optional rack) { this.id = id; this.rack = rack; } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + BrokerMetadata that = (BrokerMetadata) o; + return id == that.id && Objects.equals(rack, that.rack); + } + + @Override + public int hashCode() { + return Objects.hash(id, rack); + } } From 804d08e71c61888cd385ad32debbdb8b552d239d Mon Sep 17 00:00:00 2001 From: nizhikov Date: Tue, 25 Jul 2023 19:40:31 +0300 Subject: [PATCH 16/78] KAFKA-14595 scala BrokerMetadata removed --- checkstyle/import-control-server-common.xml | 3 + .../main/scala/kafka/admin/AdminUtils.scala | 9 +- .../scala/kafka/admin/BrokerMetadata.scala | 23 -- .../admin/ReassignPartitionsCommand.scala | 7 +- .../main/scala/kafka/server/KafkaApis.scala | 4 +- .../scala/kafka/server/MetadataCache.scala | 2 +- .../server/metadata/KRaftMetadataCache.scala | 4 +- .../server/metadata/ZkMetadataCache.scala | 7 +- .../main/scala/kafka/zk/AdminZkClient.scala | 11 +- .../unit/kafka/admin/AdminRackAwareTest.scala | 4 +- .../unit/kafka/admin/RackAwareTest.scala | 8 +- .../admin/ReassignPartitionsUnitTest.scala | 15 +- .../scala/unit/kafka/utils/TestUtils.scala | 7 +- .../unit/kafka/zk/AdminZkClientTest.scala | 8 +- .../org/apache/kafka/admin/AdminUtils.java | 257 ++++++++++++++++++ .../apache/kafka/admin/BrokerMetadata.java | 52 ++++ 16 files changed, 360 insertions(+), 61 deletions(-) delete mode 100644 core/src/main/scala/kafka/admin/BrokerMetadata.scala create mode 100644 server-common/src/main/java/org/apache/kafka/admin/AdminUtils.java create mode 100644 server-common/src/main/java/org/apache/kafka/admin/BrokerMetadata.java diff --git a/checkstyle/import-control-server-common.xml b/checkstyle/import-control-server-common.xml index ebf9984c7a066..a8d032c622113 100644 --- a/checkstyle/import-control-server-common.xml +++ b/checkstyle/import-control-server-common.xml @@ -101,4 +101,7 @@ + + + diff --git a/core/src/main/scala/kafka/admin/AdminUtils.scala b/core/src/main/scala/kafka/admin/AdminUtils.scala index 5ac09ab534895..1ab4643397d71 100644 --- a/core/src/main/scala/kafka/admin/AdminUtils.scala +++ b/core/src/main/scala/kafka/admin/AdminUtils.scala @@ -19,6 +19,7 @@ package kafka.admin import java.util.Random import kafka.utils.Logging +import org.apache.kafka.admin.BrokerMetadata import org.apache.kafka.common.errors.{InvalidPartitionsException, InvalidReplicationFactorException} import org.apache.kafka.server.common.AdminOperationException @@ -111,11 +112,11 @@ object AdminUtils extends Logging { throw new InvalidReplicationFactorException("Replication factor must be larger than 0.") if (replicationFactor > brokerMetadatas.size) throw new InvalidReplicationFactorException(s"Replication factor: $replicationFactor larger than available brokers: ${brokerMetadatas.size}.") - if (brokerMetadatas.forall(_.rack.isEmpty)) + if (brokerMetadatas.forall(!_.rack.isPresent)) assignReplicasToBrokersRackUnaware(nPartitions, replicationFactor, brokerMetadatas.map(_.id), fixedStartIndex, startPartitionId) else { - if (brokerMetadatas.exists(_.rack.isEmpty)) + if (brokerMetadatas.exists(!_.rack.isPresent)) throw new AdminOperationException("Not all brokers have rack information for replica rack aware assignment.") assignReplicasToBrokersRackAware(nPartitions, replicationFactor, brokerMetadatas, fixedStartIndex, startPartitionId) @@ -150,8 +151,8 @@ object AdminUtils extends Logging { brokerMetadatas: Iterable[BrokerMetadata], fixedStartIndex: Int, startPartitionId: Int): Map[Int, Seq[Int]] = { - val brokerRackMap = brokerMetadatas.collect { case BrokerMetadata(id, Some(rack)) => - id -> rack + val brokerRackMap = brokerMetadatas.collect { bm => + bm.id -> bm.rack.get() }.toMap val numRacks = brokerRackMap.values.toSet.size val arrangedBrokerList = getRackAlternatedBrokerList(brokerRackMap) diff --git a/core/src/main/scala/kafka/admin/BrokerMetadata.scala b/core/src/main/scala/kafka/admin/BrokerMetadata.scala deleted file mode 100644 index 86831e376e5bd..0000000000000 --- a/core/src/main/scala/kafka/admin/BrokerMetadata.scala +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE - * file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file - * to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -package kafka.admin - -/** - * Broker metadata used by admin tools. - * - * @param id an integer that uniquely identifies this broker - * @param rack the rack of the broker, which is used to in rack aware partition assignment for fault tolerance. - * Examples: "RACK1", "us-east-1d" - */ -case class BrokerMetadata(id: Int, rack: Option[String]) diff --git a/core/src/main/scala/kafka/admin/ReassignPartitionsCommand.scala b/core/src/main/scala/kafka/admin/ReassignPartitionsCommand.scala index 40f688c7085e2..22ae9c281b0ed 100755 --- a/core/src/main/scala/kafka/admin/ReassignPartitionsCommand.scala +++ b/core/src/main/scala/kafka/admin/ReassignPartitionsCommand.scala @@ -23,6 +23,7 @@ import kafka.server.DynamicConfig import kafka.utils.{CoreUtils, Exit, Json, Logging} import kafka.utils.Implicits._ import kafka.utils.json.JsonValue +import org.apache.kafka.admin.BrokerMetadata import org.apache.kafka.clients.admin.AlterConfigOp.OpType import org.apache.kafka.clients.admin.{Admin, AdminClientConfig, AlterConfigOp, ConfigEntry, NewPartitionReassignment, PartitionReassignment, TopicDescription} import org.apache.kafka.common.config.ConfigResource @@ -688,12 +689,12 @@ object ReassignPartitionsCommand extends Logging { filter(node => brokerSet.contains(node.id)). map { node => if (enableRackAwareness && node.rack != null) { - BrokerMetadata(node.id, Some(node.rack)) + new BrokerMetadata(node.id, Optional.of(node.rack)) } else { - BrokerMetadata(node.id, None) + new BrokerMetadata(node.id, Optional.empty()) } }.toSeq - val numRackless = results.count(_.rack.isEmpty) + val numRackless = results.count(!_.rack.isPresent) if (enableRackAwareness && numRackless != 0 && numRackless != results.size) { throw new AdminOperationException("Not all brokers have rack information. Add " + "--disable-rack-aware in command line to make replica assignment without rack " + diff --git a/core/src/main/scala/kafka/server/KafkaApis.scala b/core/src/main/scala/kafka/server/KafkaApis.scala index 5d6233d110005..15c4e356bdce2 100644 --- a/core/src/main/scala/kafka/server/KafkaApis.scala +++ b/core/src/main/scala/kafka/server/KafkaApis.scala @@ -17,7 +17,6 @@ package kafka.server -import kafka.admin.AdminUtils import kafka.api.ElectLeadersRequestOps import kafka.controller.ReplicaAssignment import kafka.coordinator.transaction.{InitProducerIdResult, TransactionCoordinator} @@ -26,6 +25,7 @@ import kafka.server.QuotaFactory.{QuotaManagers, UnboundedQuota} import kafka.server.metadata.ConfigRepository import kafka.utils.Implicits._ import kafka.utils.{CoreUtils, Logging} +import org.apache.kafka.admin.AdminUtils import org.apache.kafka.clients.admin.AlterConfigOp.OpType import org.apache.kafka.clients.admin.{AlterConfigOp, ConfigEntry} import org.apache.kafka.common.acl.AclOperation._ @@ -668,7 +668,7 @@ class KafkaApis(val requestChannel: RequestChannel, if (authorizedRequestInfo.isEmpty) sendResponseCallback(Map.empty) else { - val internalTopicsAllowed = request.header.clientId == AdminUtils.AdminClientId + val internalTopicsAllowed = request.header.clientId == AdminUtils.ADMIN_CLIENT_ID val transactionStatePartition = if (produceRequest.transactionalId() == null) diff --git a/core/src/main/scala/kafka/server/MetadataCache.scala b/core/src/main/scala/kafka/server/MetadataCache.scala index ecd64c17b91f3..2e60ce2ba7bd9 100755 --- a/core/src/main/scala/kafka/server/MetadataCache.scala +++ b/core/src/main/scala/kafka/server/MetadataCache.scala @@ -17,8 +17,8 @@ package kafka.server -import kafka.admin.BrokerMetadata import kafka.server.metadata.{KRaftMetadataCache, ZkMetadataCache} +import org.apache.kafka.admin.BrokerMetadata import org.apache.kafka.common.message.{MetadataResponseData, UpdateMetadataRequestData} import org.apache.kafka.common.network.ListenerName import org.apache.kafka.common.{Cluster, Node, TopicPartition, Uuid} diff --git a/core/src/main/scala/kafka/server/metadata/KRaftMetadataCache.scala b/core/src/main/scala/kafka/server/metadata/KRaftMetadataCache.scala index 5e3ed11022b4f..559ccb2e9179a 100644 --- a/core/src/main/scala/kafka/server/metadata/KRaftMetadataCache.scala +++ b/core/src/main/scala/kafka/server/metadata/KRaftMetadataCache.scala @@ -20,6 +20,7 @@ package kafka.server.metadata import kafka.controller.StateChangeLogger import kafka.server.{CachedControllerId, KRaftCachedControllerId, MetadataCache} import kafka.utils.Logging +import org.apache.kafka.admin.BrokerMetadata import org.apache.kafka.common.internals.Topic import org.apache.kafka.common.message.MetadataResponseData.{MetadataResponsePartition, MetadataResponseTopic} import org.apache.kafka.common.{Cluster, Node, PartitionInfo, TopicPartition, Uuid} @@ -32,7 +33,6 @@ import org.apache.kafka.image.MetadataImage import java.util import java.util.{Collections, Properties} import java.util.concurrent.ThreadLocalRandom -import kafka.admin.BrokerMetadata import org.apache.kafka.common.config.ConfigResource import org.apache.kafka.common.message.{DescribeClientQuotasRequestData, DescribeClientQuotasResponseData} import org.apache.kafka.common.message.{DescribeUserScramCredentialsRequestData, DescribeUserScramCredentialsResponseData} @@ -215,7 +215,7 @@ class KRaftMetadataCache(val brokerId: Int) extends MetadataCache with Logging w private def getAliveBrokers(image: MetadataImage): Iterable[BrokerMetadata] = { image.cluster().brokers().values().asScala.filterNot(_.fenced()). - map(b => BrokerMetadata(b.id, b.rack.asScala)) + map(b => new BrokerMetadata(b.id, b.rack)) } override def getAliveBrokerNode(brokerId: Int, listenerName: ListenerName): Option[Node] = { diff --git a/core/src/main/scala/kafka/server/metadata/ZkMetadataCache.scala b/core/src/main/scala/kafka/server/metadata/ZkMetadataCache.scala index 9159b7918809b..302d3fbf8de06 100755 --- a/core/src/main/scala/kafka/server/metadata/ZkMetadataCache.scala +++ b/core/src/main/scala/kafka/server/metadata/ZkMetadataCache.scala @@ -18,10 +18,8 @@ package kafka.server.metadata import java.util -import java.util.Collections +import java.util.{Collections, Optional} import java.util.concurrent.locks.{ReentrantLock, ReentrantReadWriteLock} -import kafka.admin.BrokerMetadata - import scala.collection.{Seq, Set, mutable} import scala.jdk.CollectionConverters._ import kafka.cluster.{Broker, EndPoint} @@ -31,6 +29,7 @@ import kafka.server.{BrokerFeatures, CachedControllerId, KRaftCachedControllerId import kafka.utils.CoreUtils._ import kafka.utils.Logging import kafka.utils.Implicits._ +import org.apache.kafka.admin.BrokerMetadata import org.apache.kafka.common.internals.Topic import org.apache.kafka.common.message.UpdateMetadataRequestData.UpdateMetadataPartitionState import org.apache.kafka.common.{Cluster, Node, PartitionInfo, TopicPartition, Uuid} @@ -254,7 +253,7 @@ class ZkMetadataCache( override def hasAliveBroker(brokerId: Int): Boolean = metadataSnapshot.aliveBrokers.contains(brokerId) override def getAliveBrokers(): Iterable[BrokerMetadata] = { - metadataSnapshot.aliveBrokers.values.map(b => new BrokerMetadata(b.id, b.rack)) + metadataSnapshot.aliveBrokers.values.map(b => new BrokerMetadata(b.id, Optional.ofNullable(b.rack.orNull))) } override def getAliveBrokerNode(brokerId: Int, listenerName: ListenerName): Option[Node] = { diff --git a/core/src/main/scala/kafka/zk/AdminZkClient.scala b/core/src/main/scala/kafka/zk/AdminZkClient.scala index 07ca35cc9ae06..d72a4594d8a88 100644 --- a/core/src/main/scala/kafka/zk/AdminZkClient.scala +++ b/core/src/main/scala/kafka/zk/AdminZkClient.scala @@ -16,13 +16,14 @@ */ package kafka.zk -import java.util.Properties -import kafka.admin.{AdminUtils, BrokerMetadata, RackAwareMode} +import java.util.{Optional, Properties} +import kafka.admin.{AdminUtils, RackAwareMode} import kafka.common.TopicAlreadyMarkedForDeletionException import kafka.controller.ReplicaAssignment import kafka.server.{ConfigEntityName, ConfigType, DynamicConfig} import kafka.utils._ import kafka.utils.Implicits._ +import org.apache.kafka.admin.BrokerMetadata import org.apache.kafka.common.{TopicPartition, Uuid} import org.apache.kafka.common.errors._ import org.apache.kafka.common.internals.Topic @@ -77,10 +78,10 @@ class AdminZkClient(zkClient: KafkaZkClient) extends Logging { " to make replica assignment without rack information.") } val brokerMetadatas = rackAwareMode match { - case RackAwareMode.Disabled => brokers.map(broker => BrokerMetadata(broker.id, None)) + case RackAwareMode.Disabled => brokers.map(broker => new BrokerMetadata(broker.id, Optional.empty())) case RackAwareMode.Safe if brokersWithRack.size < brokers.size => - brokers.map(broker => BrokerMetadata(broker.id, None)) - case _ => brokers.map(broker => BrokerMetadata(broker.id, broker.rack)) + brokers.map(broker => new BrokerMetadata(broker.id, Optional.empty())) + case _ => brokers.map(broker => new BrokerMetadata(broker.id, Optional.ofNullable(broker.rack.orNull))) } brokerMetadatas.sortBy(_.id) } diff --git a/core/src/test/scala/unit/kafka/admin/AdminRackAwareTest.scala b/core/src/test/scala/unit/kafka/admin/AdminRackAwareTest.scala index d2d665bef8a54..d430c686e0b48 100644 --- a/core/src/test/scala/unit/kafka/admin/AdminRackAwareTest.scala +++ b/core/src/test/scala/unit/kafka/admin/AdminRackAwareTest.scala @@ -17,10 +17,12 @@ package kafka.admin import kafka.utils.Logging +import org.apache.kafka.admin.BrokerMetadata import org.apache.kafka.common.errors.InvalidReplicationFactorException import org.junit.jupiter.api.Assertions._ import org.junit.jupiter.api.Test +import java.util.Optional import scala.collection.Map class AdminRackAwareTest extends RackAwareTest with Logging { @@ -196,7 +198,7 @@ class AdminRackAwareTest extends RackAwareTest with Logging { @Test def testReplicaAssignment(): Unit = { - val brokerMetadatas = (0 to 4).map(new BrokerMetadata(_, None)) + val brokerMetadatas = (0 to 4).map(new BrokerMetadata(_, Optional.empty())) // test 0 replication factor assertThrows(classOf[InvalidReplicationFactorException], diff --git a/core/src/test/scala/unit/kafka/admin/RackAwareTest.scala b/core/src/test/scala/unit/kafka/admin/RackAwareTest.scala index 6ce4f7ba55d9a..c436f9066e86d 100644 --- a/core/src/test/scala/unit/kafka/admin/RackAwareTest.scala +++ b/core/src/test/scala/unit/kafka/admin/RackAwareTest.scala @@ -16,9 +16,13 @@ */ package kafka.admin +import org.apache.kafka.admin.BrokerMetadata + import scala.collection.{Map, Seq, mutable} import org.junit.jupiter.api.Assertions._ +import java.util.Optional + trait RackAwareTest { def checkReplicaDistribution(assignment: Map[Int, Seq[Int]], @@ -75,9 +79,9 @@ trait RackAwareTest { def toBrokerMetadata(rackMap: Map[Int, String], brokersWithoutRack: Seq[Int] = Seq.empty): Seq[BrokerMetadata] = rackMap.toSeq.map { case (brokerId, rack) => - BrokerMetadata(brokerId, Some(rack)) + new BrokerMetadata(brokerId, Optional.of(rack)) } ++ brokersWithoutRack.map { brokerId => - BrokerMetadata(brokerId, None) + new BrokerMetadata(brokerId, Optional.empty()) }.sortBy(_.id) } diff --git a/core/src/test/scala/unit/kafka/admin/ReassignPartitionsUnitTest.scala b/core/src/test/scala/unit/kafka/admin/ReassignPartitionsUnitTest.scala index fab7907e17be7..7aa0f8eec23e7 100644 --- a/core/src/test/scala/unit/kafka/admin/ReassignPartitionsUnitTest.scala +++ b/core/src/test/scala/unit/kafka/admin/ReassignPartitionsUnitTest.scala @@ -18,9 +18,10 @@ package kafka.admin import java.util.concurrent.ExecutionException -import java.util.{Arrays, Collections} +import java.util.{Arrays, Collections, Optional} import kafka.admin.ReassignPartitionsCommand._ import kafka.utils.Exit +import org.apache.kafka.admin.BrokerMetadata import org.apache.kafka.clients.admin.{Config, MockAdminClient, PartitionReassignment} import org.apache.kafka.common.config.ConfigResource import org.apache.kafka.common.errors.{InvalidReplicationFactorException, UnknownTopicOrPartitionException} @@ -230,19 +231,19 @@ class ReassignPartitionsUnitTest { build() try { assertEquals(Seq( - BrokerMetadata(0, Some("rack0")), - BrokerMetadata(1, Some("rack1")) + new BrokerMetadata(0, Optional.of("rack0")), + new BrokerMetadata(1, Optional.of("rack1")) ), getBrokerMetadata(adminClient, Seq(0, 1), true)) assertEquals(Seq( - BrokerMetadata(0, None), - BrokerMetadata(1, None) + new BrokerMetadata(0, Optional.empty()), + new BrokerMetadata(1, Optional.empty()) ), getBrokerMetadata(adminClient, Seq(0, 1), false)) assertStartsWith("Not all brokers have rack information", assertThrows(classOf[AdminOperationException], () => getBrokerMetadata(adminClient, Seq(1, 2), true)).getMessage) assertEquals(Seq( - BrokerMetadata(1, None), - BrokerMetadata(2, None) + new BrokerMetadata(1, Optional.empty()), + new BrokerMetadata(2, Optional.empty()) ), getBrokerMetadata(adminClient, Seq(1, 2), false)) } finally { adminClient.close() diff --git a/core/src/test/scala/unit/kafka/utils/TestUtils.scala b/core/src/test/scala/unit/kafka/utils/TestUtils.scala index e587c8cb6e8f6..048489a7cebe9 100755 --- a/core/src/test/scala/unit/kafka/utils/TestUtils.scala +++ b/core/src/test/scala/unit/kafka/utils/TestUtils.scala @@ -41,6 +41,7 @@ import kafka.server.checkpoints.OffsetCheckpointFile import kafka.server.metadata.{ConfigRepository, MockConfigRepository} import kafka.utils.Implicits._ import kafka.zk._ +import org.apache.kafka.admin.BrokerMetadata import org.apache.kafka.clients.{ClientResponse, CommonClientConfigs} import org.apache.kafka.clients.admin.AlterConfigOp.OpType import org.apache.kafka.clients.admin._ @@ -893,14 +894,14 @@ object TestUtils extends Logging { } def createBrokersInZk(zkClient: KafkaZkClient, ids: Seq[Int]): Seq[Broker] = - createBrokersInZk(ids.map(kafka.admin.BrokerMetadata(_, None)), zkClient) + createBrokersInZk(ids.map(new BrokerMetadata(_, Optional.empty())), zkClient) - def createBrokersInZk(brokerMetadatas: Seq[kafka.admin.BrokerMetadata], zkClient: KafkaZkClient): Seq[Broker] = { + def createBrokersInZk(brokerMetadatas: Seq[BrokerMetadata], zkClient: KafkaZkClient): Seq[Broker] = { zkClient.makeSurePersistentPathExists(BrokerIdsZNode.path) val brokers = brokerMetadatas.map { b => val protocol = SecurityProtocol.PLAINTEXT val listenerName = ListenerName.forSecurityProtocol(protocol) - Broker(b.id, Seq(EndPoint("localhost", 6667, listenerName, protocol)), b.rack) + Broker(b.id, Seq(EndPoint("localhost", 6667, listenerName, protocol)), if (b.rack.isPresent) Some(b.rack.get()) else None) } brokers.foreach(b => zkClient.registerBroker(BrokerInfo(Broker(b.id, b.endPoints, rack = b.rack), MetadataVersion.latest, jmxPort = -1))) diff --git a/core/src/test/scala/unit/kafka/zk/AdminZkClientTest.scala b/core/src/test/scala/unit/kafka/zk/AdminZkClientTest.scala index ba0a2583598db..3ff640fa76037 100644 --- a/core/src/test/scala/unit/kafka/zk/AdminZkClientTest.scala +++ b/core/src/test/scala/unit/kafka/zk/AdminZkClientTest.scala @@ -17,7 +17,7 @@ package kafka.admin import java.util -import java.util.Properties +import java.util.{Optional, Properties} import kafka.controller.ReplicaAssignment import kafka.server.DynamicConfig.Broker._ import kafka.server.KafkaConfig._ @@ -340,18 +340,18 @@ class AdminZkClientTest extends QuorumTestHarness with Logging with RackAwareTes val processedMetadatas1 = adminZkClient.getBrokerMetadatas(RackAwareMode.Disabled) assertEquals(brokerList, processedMetadatas1.map(_.id)) - assertEquals(List.fill(brokerList.size)(None), processedMetadatas1.map(_.rack)) + assertEquals(List.fill(brokerList.size)(Optional.empty()), processedMetadatas1.map(_.rack)) val processedMetadatas2 = adminZkClient.getBrokerMetadatas(RackAwareMode.Safe) assertEquals(brokerList, processedMetadatas2.map(_.id)) - assertEquals(List.fill(brokerList.size)(None), processedMetadatas2.map(_.rack)) + assertEquals(List.fill(brokerList.size)(Optional.empty()), processedMetadatas2.map(_.rack)) assertThrows(classOf[AdminOperationException], () => adminZkClient.getBrokerMetadatas(RackAwareMode.Enforced)) val partialList = List(0, 1, 2, 3, 5) val processedMetadatas3 = adminZkClient.getBrokerMetadatas(RackAwareMode.Enforced, Some(partialList)) assertEquals(partialList, processedMetadatas3.map(_.id)) - assertEquals(partialList.map(rackInfo), processedMetadatas3.flatMap(_.rack)) + assertEquals(partialList.map(rackInfo), processedMetadatas3.map(_.rack.get())) val numPartitions = 3 adminZkClient.createTopic("foo", numPartitions, 2, rackAwareMode = RackAwareMode.Safe) diff --git a/server-common/src/main/java/org/apache/kafka/admin/AdminUtils.java b/server-common/src/main/java/org/apache/kafka/admin/AdminUtils.java new file mode 100644 index 0000000000000..84aa02251d404 --- /dev/null +++ b/server-common/src/main/java/org/apache/kafka/admin/AdminUtils.java @@ -0,0 +1,257 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.admin; + +import org.apache.kafka.common.errors.InvalidPartitionsException; +import org.apache.kafka.common.errors.InvalidReplicationFactorException; +import org.apache.kafka.server.common.AdminOperationException; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Set; +import java.util.stream.Collectors; + +public class AdminUtils { + static final Random RAND = new Random(); + + public static final String ADMIN_CLIENT_ID = "__admin_client"; + + public static Map> assignReplicasToBrokers(Collection brokerMetadatas, + int nPartitions, + int replicationFactor) { + return assignReplicasToBrokers(brokerMetadatas, nPartitions, replicationFactor, -1, -1); + } + + /** + * There are 3 goals of replica assignment: + * + *
    + *
  1. Spread the replicas evenly among brokers.
    2. + *
  2. For partitions assigned to a particular broker, their other replicas are spread over the other brokers.
    3. + *
  3. If all brokers have rack information, assign the replicas for each partition to different racks if possible
    4. + *
+ * + * To achieve this goal for replica assignment without considering racks, we: + *
    + *
  1. Assign the first replica of each partition by round-robin, starting from a random position in the broker list.
    2. + *
  2. Assign the remaining replicas of each partition with an increasing shift.
    3. + *
+ * + * Here is an example of assigning + * + * + * + * + * + * + * + * + *
broker-0broker-1broker-2broker-3broker-4 
p0 p1 p2 p3 p4 (1st replica)
p5 p6 p7 p8 p9 (1st replica)
p4 p0 p1 p2 p3 (2nd replica)
p8 p9 p5 p6 p7 (2nd replica)
p3 p4 p0 p1 p2 (3nd replica)
p7 p8 p9 p5 p6 (3nd replica)
+ * + *

+ * To create rack aware assignment, this API will first create a rack alternated broker list. For example, + * from this brokerID -> rack mapping:

+ * 0 -> "rack1", 1 -> "rack3", 2 -> "rack3", 3 -> "rack2", 4 -> "rack2", 5 -> "rack1" + *

+ *

+ * The rack alternated list will be: + *

+ * 0, 3, 1, 5, 4, 2 + *

+ *

+ * Then an easy round-robin assignment can be applied. Assume 6 partitions with replication factor of 3, the assignment + * will be: + *

+ * 0 -> 0,3,1
+ * 1 -> 3,1,5
+ * 2 -> 1,5,4
+ * 3 -> 5,4,2
+ * 4 -> 4,2,0
+ * 5 -> 2,0,3
+ *
+ *

+ * Once it has completed the first round-robin, if there are more partitions to assign, the algorithm will start + * shifting the followers. This is to ensure we will not always get the same set of sequences. + * In this case, if there is another partition to assign (partition #6), the assignment will be: + *

+ * 6 -> 0,4,2 (instead of repeating 0,3,1 as partition 0) + *

+ *

+ * The rack aware assignment always chooses the 1st replica of the partition using round robin on the rack alternated + * broker list. For rest of the replicas, it will be biased towards brokers on racks that do not have + * any replica assignment, until every rack has a replica. Then the assignment will go back to round-robin on + * the broker list. + *

+ *
+ *

+ * As the result, if the number of replicas is equal to or greater than the number of racks, it will ensure that + * each rack will get at least one replica. Otherwise, each rack will get at most one replica. In a perfect + * situation where the number of replicas is the same as the number of racks and each rack has the same number of + * brokers, it guarantees that the replica distribution is even across brokers and racks. + *

+ * @return a Map from partition id to replica ids + * @throws AdminOperationException If rack information is supplied but it is incomplete, or if it is not possible to + * assign each replica to a unique rack. + * + */ + public static Map> assignReplicasToBrokers(Collection brokerMetadatas, + int nPartitions, + int replicationFactor, + int fixedStartIndex, + int startPartitionId) { + if (nPartitions <= 0) + throw new InvalidPartitionsException("Number of partitions must be larger than 0."); + if (replicationFactor <= 0) + throw new InvalidReplicationFactorException("Replication factor must be larger than 0."); + if (replicationFactor > brokerMetadatas.size()) + throw new InvalidReplicationFactorException("Replication factor: " + replicationFactor + " larger than available brokers: " + brokerMetadatas.size() + "."); + if (brokerMetadatas.stream().noneMatch(b -> b.rack.isPresent())) + return assignReplicasToBrokersRackUnaware(nPartitions, replicationFactor, brokerMetadatas.stream().map(b -> b.id).collect(Collectors.toList()), fixedStartIndex, + startPartitionId); + else { + if (brokerMetadatas.stream().anyMatch(b -> !b.rack.isPresent())) + throw new AdminOperationException("Not all brokers have rack information for replica rack aware assignment."); + return assignReplicasToBrokersRackAware(nPartitions, replicationFactor, brokerMetadatas, fixedStartIndex, + startPartitionId); + } + } + + private static Map> assignReplicasToBrokersRackUnaware(int nPartitions, + int replicationFactor, + Collection brokerList, + int fixedStartIndex, + int startPartitionId) { + Map> ret = new HashMap<>(); + Integer[] brokerArray = brokerList.toArray(new Integer[0]); + int startIndex = fixedStartIndex >= 0 ? fixedStartIndex : RAND.nextInt(brokerArray.length); + int currentPartitionId = Math.max(0, startPartitionId); + int nextReplicaShift = fixedStartIndex >= 0 ? fixedStartIndex : RAND.nextInt(brokerArray.length); + for (int i = 0; i < nPartitions; i++) { + if (currentPartitionId > 0 && (currentPartitionId % brokerArray.length == 0)) + nextReplicaShift += 1; + int firstReplicaIndex = (currentPartitionId + startIndex) % brokerArray.length; + List replicaBuffer = new ArrayList<>(); + replicaBuffer.add(brokerArray[firstReplicaIndex]); + for (int j = 0; j < replicationFactor - 1; j++) + replicaBuffer.add(brokerArray[replicaIndex(firstReplicaIndex, nextReplicaShift, j, brokerArray.length)]); + ret.put(currentPartitionId, replicaBuffer); + currentPartitionId += 1; + } + return ret; + } + + private static Map> assignReplicasToBrokersRackAware(int nPartitions, + int replicationFactor, + Collection brokerMetadatas, + int fixedStartIndex, + int startPartitionId) { + Map brokerRackMap = new HashMap<>(); + brokerMetadatas.forEach(m -> brokerRackMap.put(m.id, m.rack.get())); + int numRacks = new HashSet<>(brokerRackMap.values()).size(); + List arrangedBrokerList = getRackAlternatedBrokerList(brokerRackMap); + int numBrokers = arrangedBrokerList.size(); + Map> ret = new HashMap<>(); + int startIndex = fixedStartIndex >= 0 ? fixedStartIndex : RAND.nextInt(arrangedBrokerList.size()); + int currentPartitionId = Math.max(0, startPartitionId); + int nextReplicaShift = fixedStartIndex >= 0 ? fixedStartIndex : RAND.nextInt(arrangedBrokerList.size()); + for (int i = 0; i < nPartitions; i++) { + if (currentPartitionId > 0 && (currentPartitionId % arrangedBrokerList.size() == 0)) + nextReplicaShift += 1; + int firstReplicaIndex = (currentPartitionId + startIndex) % arrangedBrokerList.size(); + int leader = arrangedBrokerList.get(firstReplicaIndex); + List replicaBuffer = new ArrayList<>(); + replicaBuffer.add(leader); + Set racksWithReplicas = new HashSet<>(); + racksWithReplicas.add(brokerRackMap.get(leader)); + Set brokersWithReplicas = new HashSet<>(); + brokersWithReplicas.add(leader); + int k = 0; + for (int j = 0; j < replicationFactor - 1; j++) { + boolean done = false; + while (!done) { + Integer broker = arrangedBrokerList.get(replicaIndex(firstReplicaIndex, nextReplicaShift * numRacks, k, arrangedBrokerList.size())); + String rack = brokerRackMap.get(broker); + // Skip this broker if + // 1. there is already a broker in the same rack that has assigned a replica AND there is one or more racks + // that do not have any replica, or + // 2. the broker has already assigned a replica AND there is one or more brokers that do not have replica assigned + if ((!racksWithReplicas.contains(rack) || racksWithReplicas.size() == numRacks) + && (!brokersWithReplicas.contains(broker) || brokersWithReplicas.size() == numBrokers)) { + replicaBuffer.add(broker); + racksWithReplicas.add(rack); + brokersWithReplicas.add(broker); + done = true; + } + k += 1; + } + } + ret.put(currentPartitionId, replicaBuffer); + currentPartitionId += 1; + } + return ret; + } + + /** + * Given broker and rack information, returns a list of brokers alternated by the rack. Assume + * this is the rack and its brokers: + * + * rack1: 0, 1, 2 + * rack2: 3, 4, 5 + * rack3: 6, 7, 8 + * + * This API would return the list of 0, 3, 6, 1, 4, 7, 2, 5, 8 + * + * This is essential to make sure that the assignReplicasToBrokers API can use such list and + * assign replicas to brokers in a simple round-robin fashion, while ensuring an even + * distribution of leader and replica counts on each broker and that replicas are + * distributed to all racks. + */ + static List getRackAlternatedBrokerList(Map brokerRackMap) { + Map> brokersIteratorByRack = new HashMap<>(); + getInverseMap(brokerRackMap).forEach((rack, brokers) -> brokersIteratorByRack.put(rack, brokers.iterator())); + String[] racks = brokersIteratorByRack.keySet().toArray(new String[0]); + Arrays.sort(racks); + List results = new ArrayList<>(); + int rackIndex = 0; + while (results.size() < brokerRackMap.size()) { + Iterator rackIterator = brokersIteratorByRack.get(racks[rackIndex]); + if (rackIterator.hasNext()) + results.add(rackIterator.next()); + rackIndex = (rackIndex + 1) % racks.length; + } + return results; + } + + static Map> getInverseMap(Map brokerRackMap) { + Map> results = new HashMap<>(); + brokerRackMap.forEach((id, rack) -> results.computeIfAbsent(rack, key -> new ArrayList<>()).add(id)); + results.forEach((rack, rackAndIdList) -> rackAndIdList.sort(Integer::compareTo)); + return results; + } + + static int replicaIndex(int firstReplicaIndex, int secondReplicaShift, int replicaIndex, int nBrokers) { + long shift = 1 + (secondReplicaShift + replicaIndex) % (nBrokers - 1); + return (int) ((firstReplicaIndex + shift) % nBrokers); + } +} diff --git a/server-common/src/main/java/org/apache/kafka/admin/BrokerMetadata.java b/server-common/src/main/java/org/apache/kafka/admin/BrokerMetadata.java new file mode 100644 index 0000000000000..98216fa0858c7 --- /dev/null +++ b/server-common/src/main/java/org/apache/kafka/admin/BrokerMetadata.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.admin; + +import java.util.Objects; +import java.util.Optional; + +/** + * Broker metadata used by admin tools. + */ +public class BrokerMetadata { + public final int id; + + public final Optional rack; + + /** + * @param id an integer that uniquely identifies this broker + * @param rack the rack of the broker, which is used to in rack aware partition assignment for fault tolerance. + * Examples: "RACK1", "us-east-1d" + */ + public BrokerMetadata(int id, Optional rack) { + this.id = id; + this.rack = rack; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + BrokerMetadata that = (BrokerMetadata) o; + return id == that.id && Objects.equals(rack, that.rack); + } + + @Override + public int hashCode() { + return Objects.hash(id, rack); + } +} From 948958fda9ebbf86c0bf7fc370b69637900fb72f Mon Sep 17 00:00:00 2001 From: nizhikov Date: Tue, 25 Jul 2023 22:40:56 +0300 Subject: [PATCH 17/78] KAFKA-14595 scala AdminUtils removed --- .../main/scala/kafka/admin/AdminUtils.scala | 240 ------------------ .../admin/ReassignPartitionsCommand.scala | 5 +- .../scala/kafka/server/ZkAdminManager.scala | 6 +- .../main/scala/kafka/utils/CoreUtils.scala | 5 + .../main/scala/kafka/zk/AdminZkClient.scala | 13 +- .../unit/kafka/admin/AdminRackAwareTest.scala | 63 ++--- .../unit/kafka/admin/RackAwareTest.scala | 9 +- .../unit/kafka/zk/AdminZkClientTest.scala | 2 +- .../org/apache/kafka/admin/AdminUtils.java | 2 +- 9 files changed, 59 insertions(+), 286 deletions(-) delete mode 100644 core/src/main/scala/kafka/admin/AdminUtils.scala diff --git a/core/src/main/scala/kafka/admin/AdminUtils.scala b/core/src/main/scala/kafka/admin/AdminUtils.scala deleted file mode 100644 index 1ab4643397d71..0000000000000 --- a/core/src/main/scala/kafka/admin/AdminUtils.scala +++ /dev/null @@ -1,240 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package kafka.admin - -import java.util.Random -import kafka.utils.Logging -import org.apache.kafka.admin.BrokerMetadata -import org.apache.kafka.common.errors.{InvalidPartitionsException, InvalidReplicationFactorException} -import org.apache.kafka.server.common.AdminOperationException - -import collection.{Map, mutable, _} - -object AdminUtils extends Logging { - val rand = new Random - val AdminClientId = "__admin_client" - - /** - * There are 3 goals of replica assignment: - * - *
    - *
  1. Spread the replicas evenly among brokers.
    2. - *
  2. For partitions assigned to a particular broker, their other replicas are spread over the other brokers.
    3. - *
  3. If all brokers have rack information, assign the replicas for each partition to different racks if possible
    4. - *
- * - * To achieve this goal for replica assignment without considering racks, we: - *
    - *
  1. Assign the first replica of each partition by round-robin, starting from a random position in the broker list.
    2. - *
  2. Assign the remaining replicas of each partition with an increasing shift.
    3. - *
- * - * Here is an example of assigning - * - * - * - * - * - * - * - * - *
broker-0broker-1broker-2broker-3broker-4 
p0 p1 p2 p3 p4 (1st replica)
p5 p6 p7 p8 p9 (1st replica)
p4 p0 p1 p2 p3 (2nd replica)
p8 p9 p5 p6 p7 (2nd replica)
p3 p4 p0 p1 p2 (3nd replica)
p7 p8 p9 p5 p6 (3nd replica)
- * - *

- * To create rack aware assignment, this API will first create a rack alternated broker list. For example, - * from this brokerID -> rack mapping:

- * 0 -> "rack1", 1 -> "rack3", 2 -> "rack3", 3 -> "rack2", 4 -> "rack2", 5 -> "rack1" - *

- *

- * The rack alternated list will be: - *

- * 0, 3, 1, 5, 4, 2 - *

- *

- * Then an easy round-robin assignment can be applied. Assume 6 partitions with replication factor of 3, the assignment - * will be: - *

- * 0 -> 0,3,1
- * 1 -> 3,1,5
- * 2 -> 1,5,4
- * 3 -> 5,4,2
- * 4 -> 4,2,0
- * 5 -> 2,0,3
- *
- *

- * Once it has completed the first round-robin, if there are more partitions to assign, the algorithm will start - * shifting the followers. This is to ensure we will not always get the same set of sequences. - * In this case, if there is another partition to assign (partition #6), the assignment will be: - *

- * 6 -> 0,4,2 (instead of repeating 0,3,1 as partition 0) - *

- *

- * The rack aware assignment always chooses the 1st replica of the partition using round robin on the rack alternated - * broker list. For rest of the replicas, it will be biased towards brokers on racks that do not have - * any replica assignment, until every rack has a replica. Then the assignment will go back to round-robin on - * the broker list. - *

- *
- *

- * As the result, if the number of replicas is equal to or greater than the number of racks, it will ensure that - * each rack will get at least one replica. Otherwise, each rack will get at most one replica. In a perfect - * situation where the number of replicas is the same as the number of racks and each rack has the same number of - * brokers, it guarantees that the replica distribution is even across brokers and racks. - *

- * @return a Map from partition id to replica ids - * @throws AdminOperationException If rack information is supplied but it is incomplete, or if it is not possible to - * assign each replica to a unique rack. - * - */ - def assignReplicasToBrokers(brokerMetadatas: Iterable[BrokerMetadata], - nPartitions: Int, - replicationFactor: Int, - fixedStartIndex: Int = -1, - startPartitionId: Int = -1): Map[Int, Seq[Int]] = { - if (nPartitions <= 0) - throw new InvalidPartitionsException("Number of partitions must be larger than 0.") - if (replicationFactor <= 0) - throw new InvalidReplicationFactorException("Replication factor must be larger than 0.") - if (replicationFactor > brokerMetadatas.size) - throw new InvalidReplicationFactorException(s"Replication factor: $replicationFactor larger than available brokers: ${brokerMetadatas.size}.") - if (brokerMetadatas.forall(!_.rack.isPresent)) - assignReplicasToBrokersRackUnaware(nPartitions, replicationFactor, brokerMetadatas.map(_.id), fixedStartIndex, - startPartitionId) - else { - if (brokerMetadatas.exists(!_.rack.isPresent)) - throw new AdminOperationException("Not all brokers have rack information for replica rack aware assignment.") - assignReplicasToBrokersRackAware(nPartitions, replicationFactor, brokerMetadatas, fixedStartIndex, - startPartitionId) - } - } - - private def assignReplicasToBrokersRackUnaware(nPartitions: Int, - replicationFactor: Int, - brokerList: Iterable[Int], - fixedStartIndex: Int, - startPartitionId: Int): Map[Int, Seq[Int]] = { - val ret = mutable.Map[Int, Seq[Int]]() - val brokerArray = brokerList.toArray - val startIndex = if (fixedStartIndex >= 0) fixedStartIndex else rand.nextInt(brokerArray.length) - var currentPartitionId = math.max(0, startPartitionId) - var nextReplicaShift = if (fixedStartIndex >= 0) fixedStartIndex else rand.nextInt(brokerArray.length) - for (_ <- 0 until nPartitions) { - if (currentPartitionId > 0 && (currentPartitionId % brokerArray.length == 0)) - nextReplicaShift += 1 - val firstReplicaIndex = (currentPartitionId + startIndex) % brokerArray.length - val replicaBuffer = mutable.ArrayBuffer(brokerArray(firstReplicaIndex)) - for (j <- 0 until replicationFactor - 1) - replicaBuffer += brokerArray(replicaIndex(firstReplicaIndex, nextReplicaShift, j, brokerArray.length)) - ret.put(currentPartitionId, replicaBuffer) - currentPartitionId += 1 - } - ret - } - - private def assignReplicasToBrokersRackAware(nPartitions: Int, - replicationFactor: Int, - brokerMetadatas: Iterable[BrokerMetadata], - fixedStartIndex: Int, - startPartitionId: Int): Map[Int, Seq[Int]] = { - val brokerRackMap = brokerMetadatas.collect { bm => - bm.id -> bm.rack.get() - }.toMap - val numRacks = brokerRackMap.values.toSet.size - val arrangedBrokerList = getRackAlternatedBrokerList(brokerRackMap) - val numBrokers = arrangedBrokerList.size - val ret = mutable.Map[Int, Seq[Int]]() - val startIndex = if (fixedStartIndex >= 0) fixedStartIndex else rand.nextInt(arrangedBrokerList.size) - var currentPartitionId = math.max(0, startPartitionId) - var nextReplicaShift = if (fixedStartIndex >= 0) fixedStartIndex else rand.nextInt(arrangedBrokerList.size) - for (_ <- 0 until nPartitions) { - if (currentPartitionId > 0 && (currentPartitionId % arrangedBrokerList.size == 0)) - nextReplicaShift += 1 - val firstReplicaIndex = (currentPartitionId + startIndex) % arrangedBrokerList.size - val leader = arrangedBrokerList(firstReplicaIndex) - val replicaBuffer = mutable.ArrayBuffer(leader) - val racksWithReplicas = mutable.Set(brokerRackMap(leader)) - val brokersWithReplicas = mutable.Set(leader) - var k = 0 - for (_ <- 0 until replicationFactor - 1) { - var done = false - while (!done) { - val broker = arrangedBrokerList(replicaIndex(firstReplicaIndex, nextReplicaShift * numRacks, k, arrangedBrokerList.size)) - val rack = brokerRackMap(broker) - // Skip this broker if - // 1. there is already a broker in the same rack that has assigned a replica AND there is one or more racks - // that do not have any replica, or - // 2. the broker has already assigned a replica AND there is one or more brokers that do not have replica assigned - if ((!racksWithReplicas.contains(rack) || racksWithReplicas.size == numRacks) - && (!brokersWithReplicas.contains(broker) || brokersWithReplicas.size == numBrokers)) { - replicaBuffer += broker - racksWithReplicas += rack - brokersWithReplicas += broker - done = true - } - k += 1 - } - } - ret.put(currentPartitionId, replicaBuffer) - currentPartitionId += 1 - } - ret - } - - /** - * Given broker and rack information, returns a list of brokers alternated by the rack. Assume - * this is the rack and its brokers: - * - * rack1: 0, 1, 2 - * rack2: 3, 4, 5 - * rack3: 6, 7, 8 - * - * This API would return the list of 0, 3, 6, 1, 4, 7, 2, 5, 8 - * - * This is essential to make sure that the assignReplicasToBrokers API can use such list and - * assign replicas to brokers in a simple round-robin fashion, while ensuring an even - * distribution of leader and replica counts on each broker and that replicas are - * distributed to all racks. - */ - private[admin] def getRackAlternatedBrokerList(brokerRackMap: Map[Int, String]): IndexedSeq[Int] = { - val brokersIteratorByRack = getInverseMap(brokerRackMap).map { case (rack, brokers) => - (rack, brokers.iterator) - } - val racks = brokersIteratorByRack.keys.toArray.sorted - val result = new mutable.ArrayBuffer[Int] - var rackIndex = 0 - while (result.size < brokerRackMap.size) { - val rackIterator = brokersIteratorByRack(racks(rackIndex)) - if (rackIterator.hasNext) - result += rackIterator.next() - rackIndex = (rackIndex + 1) % racks.length - } - result - } - - private[admin] def getInverseMap(brokerRackMap: Map[Int, String]): Map[String, Seq[Int]] = { - brokerRackMap.toSeq.map { case (id, rack) => (rack, id) } - .groupBy { case (rack, _) => rack } - .map { case (rack, rackAndIdList) => (rack, rackAndIdList.map { case (_, id) => id }.sorted) } - } - - private def replicaIndex(firstReplicaIndex: Int, secondReplicaShift: Int, replicaIndex: Int, nBrokers: Int): Int = { - val shift = 1 + (secondReplicaShift + replicaIndex) % (nBrokers - 1) - (firstReplicaIndex + shift) % nBrokers - } - -} diff --git a/core/src/main/scala/kafka/admin/ReassignPartitionsCommand.scala b/core/src/main/scala/kafka/admin/ReassignPartitionsCommand.scala index 22ae9c281b0ed..20a324dcf2c73 100755 --- a/core/src/main/scala/kafka/admin/ReassignPartitionsCommand.scala +++ b/core/src/main/scala/kafka/admin/ReassignPartitionsCommand.scala @@ -23,7 +23,7 @@ import kafka.server.DynamicConfig import kafka.utils.{CoreUtils, Exit, Json, Logging} import kafka.utils.Implicits._ import kafka.utils.json.JsonValue -import org.apache.kafka.admin.BrokerMetadata +import org.apache.kafka.admin.{AdminUtils, BrokerMetadata} import org.apache.kafka.clients.admin.AlterConfigOp.OpType import org.apache.kafka.clients.admin.{Admin, AdminClientConfig, AlterConfigOp, ConfigEntry, NewPartitionReassignment, PartitionReassignment, TopicDescription} import org.apache.kafka.common.config.ConfigResource @@ -608,7 +608,8 @@ object ReassignPartitionsCommand extends Logging { groupedByTopic.forKeyValue { (topic, assignment) => val (_, replicas) = assignment.head val assignedReplicas = AdminUtils. - assignReplicasToBrokers(brokerMetadatas, assignment.size, replicas.size) + assignReplicasToBrokers(brokerMetadatas.asJavaCollection, assignment.size, replicas.size). + asScala.map(e => (e._1.asInstanceOf[Int], e._2.asScala.map(_.asInstanceOf[Int]))) proposedAssignments ++= assignedReplicas.map { case (partition, replicas) => new TopicPartition(topic, partition) -> replicas } diff --git a/core/src/main/scala/kafka/server/ZkAdminManager.scala b/core/src/main/scala/kafka/server/ZkAdminManager.scala index 31ab40430d7a9..d9518f625eebf 100644 --- a/core/src/main/scala/kafka/server/ZkAdminManager.scala +++ b/core/src/main/scala/kafka/server/ZkAdminManager.scala @@ -18,7 +18,6 @@ package kafka.server import java.util import java.util.Properties -import kafka.admin.AdminUtils import kafka.common.TopicAlreadyMarkedForDeletionException import kafka.server.ConfigAdminManager.{prepareIncrementalConfigs, toLoggableProps} import kafka.server.DynamicConfig.QuotaConfigs @@ -26,6 +25,7 @@ import kafka.server.metadata.ZkConfigRepository import kafka.utils._ import kafka.utils.Implicits._ import kafka.zk.{AdminZkClient, KafkaZkClient} +import org.apache.kafka.admin.AdminUtils import org.apache.kafka.clients.admin.{AlterConfigOp, ScramMechanism} import org.apache.kafka.common.Uuid import org.apache.kafka.common.config.{ConfigDef, ConfigException, ConfigResource} @@ -184,8 +184,8 @@ class ZkAdminManager(val config: KafkaConfig, defaultReplicationFactor else topic.replicationFactor val assignments = if (topic.assignments.isEmpty) { - AdminUtils.assignReplicasToBrokers( - brokers, resolvedNumPartitions, resolvedReplicationFactor) + CoreUtils.asScala(AdminUtils.assignReplicasToBrokers( + brokers.asJavaCollection, resolvedNumPartitions, resolvedReplicationFactor)) } else { val assignments = new mutable.HashMap[Int, Seq[Int]] // Note: we don't check that replicaAssignment contains unknown brokers - unlike in add-partitions case, diff --git a/core/src/main/scala/kafka/utils/CoreUtils.scala b/core/src/main/scala/kafka/utils/CoreUtils.scala index 149d2a78374ad..667eda5b069ad 100755 --- a/core/src/main/scala/kafka/utils/CoreUtils.scala +++ b/core/src/main/scala/kafka/utils/CoreUtils.scala @@ -34,7 +34,9 @@ import org.apache.kafka.common.security.auth.SecurityProtocol import org.apache.kafka.common.utils.Utils import org.slf4j.event.Level +import java.util import scala.annotation.nowarn +import scala.jdk.CollectionConverters._ /** * General helper functions! @@ -315,4 +317,7 @@ object CoreUtils { elements.groupMapReduce(key)(f)(reduce) } + def asScala(map: util.Map[Integer, util.List[Integer]]): Map[Int, Seq[Int]] = { + map.asScala.map(e => (e._1.asInstanceOf[Int], e._2.asScala.map(_.asInstanceOf[Int]))) + } } diff --git a/core/src/main/scala/kafka/zk/AdminZkClient.scala b/core/src/main/scala/kafka/zk/AdminZkClient.scala index d72a4594d8a88..7b3ae70900ec7 100644 --- a/core/src/main/scala/kafka/zk/AdminZkClient.scala +++ b/core/src/main/scala/kafka/zk/AdminZkClient.scala @@ -17,13 +17,13 @@ package kafka.zk import java.util.{Optional, Properties} -import kafka.admin.{AdminUtils, RackAwareMode} +import kafka.admin.RackAwareMode import kafka.common.TopicAlreadyMarkedForDeletionException import kafka.controller.ReplicaAssignment import kafka.server.{ConfigEntityName, ConfigType, DynamicConfig} import kafka.utils._ import kafka.utils.Implicits._ -import org.apache.kafka.admin.BrokerMetadata +import org.apache.kafka.admin.{AdminUtils, BrokerMetadata} import org.apache.kafka.common.{TopicPartition, Uuid} import org.apache.kafka.common.errors._ import org.apache.kafka.common.internals.Topic @@ -31,6 +31,7 @@ import org.apache.kafka.server.common.AdminOperationException import org.apache.kafka.storage.internals.log.LogConfig import org.apache.zookeeper.KeeperException.NodeExistsException +import scala.jdk.CollectionConverters._ import scala.collection.{Map, Seq} /** @@ -56,8 +57,8 @@ class AdminZkClient(zkClient: KafkaZkClient) extends Logging { topicConfig: Properties = new Properties, rackAwareMode: RackAwareMode = RackAwareMode.Enforced, usesTopicId: Boolean = false): Unit = { - val brokerMetadatas = getBrokerMetadatas(rackAwareMode) - val replicaAssignment = AdminUtils.assignReplicasToBrokers(brokerMetadatas, partitions, replicationFactor) + val brokerMetadatas = getBrokerMetadatas(rackAwareMode).asJava + val replicaAssignment = CoreUtils.asScala(AdminUtils.assignReplicasToBrokers(brokerMetadatas, partitions, replicationFactor)) createTopicWithAssignment(topic, topicConfig, replicaAssignment, usesTopicId = usesTopicId) } @@ -269,8 +270,8 @@ class AdminZkClient(zkClient: KafkaZkClient) extends Logging { val proposedAssignmentForNewPartitions = replicaAssignment.getOrElse { val startIndex = math.max(0, allBrokers.indexWhere(_.id >= existingAssignmentPartition0.head)) - AdminUtils.assignReplicasToBrokers(allBrokers, partitionsToAdd, existingAssignmentPartition0.size, - startIndex, existingAssignment.size) + CoreUtils.asScala(AdminUtils.assignReplicasToBrokers(allBrokers.asJava, partitionsToAdd, existingAssignmentPartition0.size, + startIndex, existingAssignment.size)) } proposedAssignmentForNewPartitions.map { case (tp, replicas) => diff --git a/core/src/test/scala/unit/kafka/admin/AdminRackAwareTest.scala b/core/src/test/scala/unit/kafka/admin/AdminRackAwareTest.scala index d430c686e0b48..deadd6fd4d938 100644 --- a/core/src/test/scala/unit/kafka/admin/AdminRackAwareTest.scala +++ b/core/src/test/scala/unit/kafka/admin/AdminRackAwareTest.scala @@ -16,25 +16,26 @@ */ package kafka.admin -import kafka.utils.Logging -import org.apache.kafka.admin.BrokerMetadata +import kafka.utils.{CoreUtils, Logging} +import org.apache.kafka.admin.{AdminUtils, BrokerMetadata} import org.apache.kafka.common.errors.InvalidReplicationFactorException import org.junit.jupiter.api.Assertions._ import org.junit.jupiter.api.Test import java.util.Optional import scala.collection.Map +import scala.jdk.CollectionConverters._ class AdminRackAwareTest extends RackAwareTest with Logging { @Test def testGetRackAlternatedBrokerListAndAssignReplicasToBrokers(): Unit = { val rackMap = Map(0 -> "rack1", 1 -> "rack3", 2 -> "rack3", 3 -> "rack2", 4 -> "rack2", 5 -> "rack1") - val newList = AdminUtils.getRackAlternatedBrokerList(rackMap) + val newList = AdminUtils.getRackAlternatedBrokerList(rackMap.map(e => (e._1.asInstanceOf[Integer], e._2)).asJava) assertEquals(List(0, 3, 1, 5, 4, 2), newList) - val anotherList = AdminUtils.getRackAlternatedBrokerList(rackMap.toMap - 5) + val anotherList = AdminUtils.getRackAlternatedBrokerList((rackMap.toMap - 5).map(e => (e._1.asInstanceOf[Integer], e._2)).asJava) assertEquals(List(0, 3, 1, 4, 2), anotherList) - val assignment = AdminUtils.assignReplicasToBrokers(toBrokerMetadata(rackMap), 7, 3, 0, 0) + val assignment = CoreUtils.asScala(AdminUtils.assignReplicasToBrokers(toBrokerMetadata(rackMap), 7, 3, 0, 0)) val expected = Map(0 -> List(0, 3, 1), 1 -> List(3, 1, 5), 2 -> List(1, 5, 4), @@ -50,8 +51,8 @@ class AdminRackAwareTest extends RackAwareTest with Logging { val brokerRackMapping = Map(0 -> "rack1", 1 -> "rack2", 2 -> "rack2", 3 -> "rack3", 4 -> "rack3", 5 -> "rack1") val numPartitions = 6 val replicationFactor = 3 - val assignment = AdminUtils.assignReplicasToBrokers(toBrokerMetadata(brokerRackMapping), numPartitions, - replicationFactor, 2, 0) + val assignment = CoreUtils.asScala(AdminUtils.assignReplicasToBrokers(toBrokerMetadata(brokerRackMapping), numPartitions, + replicationFactor, 2, 0)) checkReplicaDistribution(assignment, brokerRackMapping, brokerRackMapping.size, numPartitions, replicationFactor) } @@ -61,8 +62,8 @@ class AdminRackAwareTest extends RackAwareTest with Logging { val brokerRackMapping = Map(0 -> "rack1", 1 -> "rack2", 2 -> "rack2", 3 -> "rack3", 4 -> "rack3", 5 -> "rack1") val numPartitions = 6 val replicationFactor = 3 - val assignment = AdminUtils.assignReplicasToBrokers(toBrokerMetadata(brokerRackMapping), numPartitions, - replicationFactor) + val assignment = CoreUtils.asScala(AdminUtils.assignReplicasToBrokers(toBrokerMetadata(brokerRackMapping), numPartitions, + replicationFactor)) checkReplicaDistribution(assignment, brokerRackMapping, brokerRackMapping.size, numPartitions, replicationFactor) } @@ -72,8 +73,8 @@ class AdminRackAwareTest extends RackAwareTest with Logging { val brokerRackMapping = Map(0 -> "rack1", 1 -> "rack2", 2 -> "rack2", 3 -> "rack3", 4 -> "rack3", 5 -> "rack1") val numPartitions = 13 val replicationFactor = 3 - val assignment = AdminUtils.assignReplicasToBrokers(toBrokerMetadata(brokerRackMapping), numPartitions, - replicationFactor, 0, 0) + val assignment = CoreUtils.asScala(AdminUtils.assignReplicasToBrokers(toBrokerMetadata(brokerRackMapping), numPartitions, + replicationFactor, 0, 0)) checkReplicaDistribution(assignment, brokerRackMapping, brokerRackMapping.size, numPartitions, replicationFactor, verifyLeaderDistribution = false, verifyReplicasDistribution = false) } @@ -83,8 +84,8 @@ class AdminRackAwareTest extends RackAwareTest with Logging { val brokerRackMapping = Map(0 -> "rack1", 1 -> "rack1", 2 -> "rack2", 3 -> "rack3", 4 -> "rack3", 5 -> "rack1") val numPartitions = 12 val replicationFactor = 3 - val assignment = AdminUtils.assignReplicasToBrokers(toBrokerMetadata(brokerRackMapping), numPartitions, - replicationFactor) + val assignment = CoreUtils.asScala(AdminUtils.assignReplicasToBrokers(toBrokerMetadata(brokerRackMapping), numPartitions, + replicationFactor)) checkReplicaDistribution(assignment, brokerRackMapping, brokerRackMapping.size, numPartitions, replicationFactor, verifyReplicasDistribution = false) } @@ -94,8 +95,8 @@ class AdminRackAwareTest extends RackAwareTest with Logging { val brokerRackMapping = Map(0 -> "rack1", 1 -> "rack2", 2 -> "rack2", 3 -> "rack3", 4 -> "rack3", 5 -> "rack1") val numPartitions = 12 val replicationFactor = 2 - val assignment = AdminUtils.assignReplicasToBrokers(toBrokerMetadata(brokerRackMapping), numPartitions, - replicationFactor) + val assignment = CoreUtils.asScala(AdminUtils.assignReplicasToBrokers(toBrokerMetadata(brokerRackMapping), numPartitions, + replicationFactor)) checkReplicaDistribution(assignment, brokerRackMapping, brokerRackMapping.size, numPartitions, replicationFactor) } @@ -105,8 +106,8 @@ class AdminRackAwareTest extends RackAwareTest with Logging { val brokerRackMapping = Map(6 -> "rack1", 7 -> "rack2", 8 -> "rack2", 9 -> "rack3", 10 -> "rack3", 11 -> "rack1") val numPartitions = 12 val replicationFactor = 2 - val assignment = AdminUtils.assignReplicasToBrokers(toBrokerMetadata(brokerRackMapping), numPartitions, - replicationFactor, startPartitionId = 12) + val assignment = CoreUtils.asScala(AdminUtils.assignReplicasToBrokers(toBrokerMetadata(brokerRackMapping), numPartitions, + replicationFactor, -1, 12)) checkReplicaDistribution(assignment, brokerRackMapping, brokerRackMapping.size, numPartitions, replicationFactor) } @@ -116,8 +117,8 @@ class AdminRackAwareTest extends RackAwareTest with Logging { val brokerRackMapping = Map(0 -> "rack1", 1 -> "rack2", 2 -> "rack2", 3 -> "rack3", 4 -> "rack3", 5 -> "rack1") val numPartitions = 6 val replicationFactor = 2 - val assignment = AdminUtils.assignReplicasToBrokers(toBrokerMetadata(brokerRackMapping), numPartitions, - replicationFactor) + val assignment = CoreUtils.asScala(AdminUtils.assignReplicasToBrokers(toBrokerMetadata(brokerRackMapping), numPartitions, + replicationFactor)) checkReplicaDistribution(assignment, brokerRackMapping, brokerRackMapping.size, numPartitions, replicationFactor) } @@ -127,7 +128,7 @@ class AdminRackAwareTest extends RackAwareTest with Logging { val brokerRackMapping = Map(0 -> "rack1", 1 -> "rack2", 4 -> "rack3") val numPartitions = 3 val replicationFactor = 2 - val assignment = AdminUtils.assignReplicasToBrokers(toBrokerMetadata(brokerRackMapping), numPartitions, replicationFactor) + val assignment = CoreUtils.asScala(AdminUtils.assignReplicasToBrokers(toBrokerMetadata(brokerRackMapping), numPartitions, replicationFactor)) checkReplicaDistribution(assignment, brokerRackMapping, brokerRackMapping.size, numPartitions, replicationFactor) } @@ -137,8 +138,8 @@ class AdminRackAwareTest extends RackAwareTest with Logging { val replicationFactor = 3 val brokerRackMapping = Map(0 -> "rack1", 1 -> "rack2", 2 -> "rack2", 3 -> "rack3", 4 -> "rack3", 5 -> "rack1", 6 -> "rack1", 7 -> "rack2", 8 -> "rack2", 9 -> "rack3", 10 -> "rack1", 11 -> "rack3") - val assignment = AdminUtils.assignReplicasToBrokers(toBrokerMetadata(brokerRackMapping), numPartitions, - replicationFactor) + val assignment = CoreUtils.asScala(AdminUtils.assignReplicasToBrokers(toBrokerMetadata(brokerRackMapping), numPartitions, + replicationFactor)) checkReplicaDistribution(assignment, brokerRackMapping, brokerRackMapping.size, numPartitions, replicationFactor) } @@ -148,7 +149,7 @@ class AdminRackAwareTest extends RackAwareTest with Logging { val numPartitions = 6 val replicationFactor = 5 val brokerRackMapping = Map(0 -> "rack1", 1 -> "rack2", 2 -> "rack2", 3 -> "rack3", 4 -> "rack3", 5 -> "rack2") - val assignment = AdminUtils.assignReplicasToBrokers(toBrokerMetadata(brokerRackMapping), numPartitions, replicationFactor) + val assignment = CoreUtils.asScala(AdminUtils.assignReplicasToBrokers(toBrokerMetadata(brokerRackMapping), numPartitions, replicationFactor)) assertEquals(List.fill(assignment.size)(replicationFactor), assignment.values.toIndexedSeq.map(_.size)) val distribution = getReplicaDistribution(assignment, brokerRackMapping) for (partition <- 0 until numPartitions) @@ -160,8 +161,8 @@ class AdminRackAwareTest extends RackAwareTest with Logging { val numPartitions = 6 val replicationFactor = 2 val brokerRackMapping = Map(0 -> "rack1", 1 -> "rack2", 2 -> "rack2", 3 -> "rack3", 4 -> "rack3", 5 -> "rack2") - val assignment = AdminUtils.assignReplicasToBrokers(toBrokerMetadata(brokerRackMapping), numPartitions, - replicationFactor) + val assignment = CoreUtils.asScala(AdminUtils.assignReplicasToBrokers(toBrokerMetadata(brokerRackMapping), numPartitions, + replicationFactor)) assertEquals(List.fill(assignment.size)(replicationFactor), assignment.values.toIndexedSeq.map(_.size)) val distribution = getReplicaDistribution(assignment, brokerRackMapping) for (partition <- 0 to 5) @@ -173,7 +174,7 @@ class AdminRackAwareTest extends RackAwareTest with Logging { val numPartitions = 6 val replicationFactor = 3 val brokerRackMapping = Map(0 -> "rack1", 1 -> "rack1", 2 -> "rack1", 3 -> "rack1", 4 -> "rack1", 5 -> "rack1") - val assignment = AdminUtils.assignReplicasToBrokers(toBrokerMetadata(brokerRackMapping), numPartitions, replicationFactor) + val assignment = CoreUtils.asScala(AdminUtils.assignReplicasToBrokers(toBrokerMetadata(brokerRackMapping), numPartitions, replicationFactor)) assertEquals(List.fill(assignment.size)(replicationFactor), assignment.values.toIndexedSeq.map(_.size)) val distribution = getReplicaDistribution(assignment, brokerRackMapping) for (partition <- 0 until numPartitions) @@ -189,16 +190,16 @@ class AdminRackAwareTest extends RackAwareTest with Logging { val numPartitions = 6 val replicationFactor = 4 val brokerMetadatas = toBrokerMetadata(rackInfo) - assertEquals(brokerList, brokerMetadatas.map(_.id)) - val assignment = AdminUtils.assignReplicasToBrokers(brokerMetadatas, numPartitions, replicationFactor, - fixedStartIndex = 2) + assertEquals(brokerList, brokerMetadatas.asScala.map(_.id)) + val assignment = CoreUtils.asScala(AdminUtils.assignReplicasToBrokers(brokerMetadatas, numPartitions, replicationFactor, + 2, -1)) checkReplicaDistribution(assignment, rackInfo, 5, 6, 4, verifyRackAware = false, verifyLeaderDistribution = false, verifyReplicasDistribution = false) } @Test def testReplicaAssignment(): Unit = { - val brokerMetadatas = (0 to 4).map(new BrokerMetadata(_, Optional.empty())) + val brokerMetadatas = (0 to 4).map(new BrokerMetadata(_, Optional.empty())).asJava // test 0 replication factor assertThrows(classOf[InvalidReplicationFactorException], @@ -221,7 +222,7 @@ class AdminRackAwareTest extends RackAwareTest with Logging { 8 -> List(3, 0, 1), 9 -> List(4, 1, 2)) - val actualAssignment = AdminUtils.assignReplicasToBrokers(brokerMetadatas, 10, 3, 0) + val actualAssignment = AdminUtils.assignReplicasToBrokers(brokerMetadatas, 10, 3, 0, -1) assertEquals(expectedAssignment, actualAssignment) } } diff --git a/core/src/test/scala/unit/kafka/admin/RackAwareTest.scala b/core/src/test/scala/unit/kafka/admin/RackAwareTest.scala index c436f9066e86d..17ce08a39bfe2 100644 --- a/core/src/test/scala/unit/kafka/admin/RackAwareTest.scala +++ b/core/src/test/scala/unit/kafka/admin/RackAwareTest.scala @@ -21,7 +21,9 @@ import org.apache.kafka.admin.BrokerMetadata import scala.collection.{Map, Seq, mutable} import org.junit.jupiter.api.Assertions._ +import java.util import java.util.Optional +import scala.jdk.CollectionConverters._ trait RackAwareTest { @@ -77,13 +79,16 @@ trait RackAwareTest { ReplicaDistributions(partitionRackMap, leaderCount, partitionCount) } - def toBrokerMetadata(rackMap: Map[Int, String], brokersWithoutRack: Seq[Int] = Seq.empty): Seq[BrokerMetadata] = - rackMap.toSeq.map { case (brokerId, rack) => + def toBrokerMetadata(rackMap: Map[Int, String], brokersWithoutRack: Seq[Int] = Seq.empty): util.Collection[BrokerMetadata] = { + val res = rackMap.toSeq.map { case (brokerId, rack) => new BrokerMetadata(brokerId, Optional.of(rack)) } ++ brokersWithoutRack.map { brokerId => new BrokerMetadata(brokerId, Optional.empty()) }.sortBy(_.id) + res.asJavaCollection + } + } case class ReplicaDistributions(partitionRacks: Map[Int, Seq[String]], brokerLeaderCount: Map[Int, Int], brokerReplicasCount: Map[Int, Int]) diff --git a/core/src/test/scala/unit/kafka/zk/AdminZkClientTest.scala b/core/src/test/scala/unit/kafka/zk/AdminZkClientTest.scala index 3ff640fa76037..e3171647ff182 100644 --- a/core/src/test/scala/unit/kafka/zk/AdminZkClientTest.scala +++ b/core/src/test/scala/unit/kafka/zk/AdminZkClientTest.scala @@ -336,7 +336,7 @@ class AdminZkClientTest extends QuorumTestHarness with Logging with RackAwareTes val brokerList = 0 to 5 val rackInfo = Map(0 -> "rack1", 1 -> "rack2", 2 -> "rack2", 3 -> "rack1", 5 -> "rack3") val brokerMetadatas = toBrokerMetadata(rackInfo, brokersWithoutRack = brokerList.filterNot(rackInfo.keySet)) - TestUtils.createBrokersInZk(brokerMetadatas, zkClient) + TestUtils.createBrokersInZk(brokerMetadatas.asScala.toSeq, zkClient) val processedMetadatas1 = adminZkClient.getBrokerMetadatas(RackAwareMode.Disabled) assertEquals(brokerList, processedMetadatas1.map(_.id)) diff --git a/server-common/src/main/java/org/apache/kafka/admin/AdminUtils.java b/server-common/src/main/java/org/apache/kafka/admin/AdminUtils.java index 84aa02251d404..9a1703e9ec593 100644 --- a/server-common/src/main/java/org/apache/kafka/admin/AdminUtils.java +++ b/server-common/src/main/java/org/apache/kafka/admin/AdminUtils.java @@ -227,7 +227,7 @@ private static Map> assignReplicasToBrokersRackAware(int * distribution of leader and replica counts on each broker and that replicas are * distributed to all racks. */ - static List getRackAlternatedBrokerList(Map brokerRackMap) { + public static List getRackAlternatedBrokerList(Map brokerRackMap) { Map> brokersIteratorByRack = new HashMap<>(); getInverseMap(brokerRackMap).forEach((rack, brokers) -> brokersIteratorByRack.put(rack, brokers.iterator())); String[] racks = brokersIteratorByRack.keySet().toArray(new String[0]); From 737aa693cbf6b1f7a65dbd9923408783944fb914 Mon Sep 17 00:00:00 2001 From: nizhikov Date: Tue, 25 Jul 2023 22:46:58 +0300 Subject: [PATCH 18/78] KAFKA-14595 tests fix --- .../test/scala/unit/kafka/admin/AdminRackAwareTest.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/test/scala/unit/kafka/admin/AdminRackAwareTest.scala b/core/src/test/scala/unit/kafka/admin/AdminRackAwareTest.scala index deadd6fd4d938..ca1e70a33f460 100644 --- a/core/src/test/scala/unit/kafka/admin/AdminRackAwareTest.scala +++ b/core/src/test/scala/unit/kafka/admin/AdminRackAwareTest.scala @@ -32,9 +32,9 @@ class AdminRackAwareTest extends RackAwareTest with Logging { def testGetRackAlternatedBrokerListAndAssignReplicasToBrokers(): Unit = { val rackMap = Map(0 -> "rack1", 1 -> "rack3", 2 -> "rack3", 3 -> "rack2", 4 -> "rack2", 5 -> "rack1") val newList = AdminUtils.getRackAlternatedBrokerList(rackMap.map(e => (e._1.asInstanceOf[Integer], e._2)).asJava) - assertEquals(List(0, 3, 1, 5, 4, 2), newList) + assertEquals(List(0, 3, 1, 5, 4, 2), newList.asScala.toList) val anotherList = AdminUtils.getRackAlternatedBrokerList((rackMap.toMap - 5).map(e => (e._1.asInstanceOf[Integer], e._2)).asJava) - assertEquals(List(0, 3, 1, 4, 2), anotherList) + assertEquals(List(0, 3, 1, 4, 2), anotherList.asScala.toList) val assignment = CoreUtils.asScala(AdminUtils.assignReplicasToBrokers(toBrokerMetadata(rackMap), 7, 3, 0, 0)) val expected = Map(0 -> List(0, 3, 1), 1 -> List(3, 1, 5), @@ -222,7 +222,7 @@ class AdminRackAwareTest extends RackAwareTest with Logging { 8 -> List(3, 0, 1), 9 -> List(4, 1, 2)) - val actualAssignment = AdminUtils.assignReplicasToBrokers(brokerMetadatas, 10, 3, 0, -1) + val actualAssignment = CoreUtils.asScala(AdminUtils.assignReplicasToBrokers(brokerMetadatas, 10, 3, 0, -1)) assertEquals(expectedAssignment, actualAssignment) } } From 13f81253e4349857ea011519eb9737c743eb4907 Mon Sep 17 00:00:00 2001 From: nizhikov Date: Wed, 26 Jul 2023 10:26:49 +0300 Subject: [PATCH 19/78] KAFKA-14595 typo --- .../src/main/java/org/apache/kafka/admin/AdminUtils.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server-common/src/main/java/org/apache/kafka/admin/AdminUtils.java b/server-common/src/main/java/org/apache/kafka/admin/AdminUtils.java index 9a1703e9ec593..ac5a260575332 100644 --- a/server-common/src/main/java/org/apache/kafka/admin/AdminUtils.java +++ b/server-common/src/main/java/org/apache/kafka/admin/AdminUtils.java @@ -232,15 +232,15 @@ public static List getRackAlternatedBrokerList(Map bro getInverseMap(brokerRackMap).forEach((rack, brokers) -> brokersIteratorByRack.put(rack, brokers.iterator())); String[] racks = brokersIteratorByRack.keySet().toArray(new String[0]); Arrays.sort(racks); - List results = new ArrayList<>(); + List result = new ArrayList<>(); int rackIndex = 0; - while (results.size() < brokerRackMap.size()) { + while (result.size() < brokerRackMap.size()) { Iterator rackIterator = brokersIteratorByRack.get(racks[rackIndex]); if (rackIterator.hasNext()) - results.add(rackIterator.next()); + result.add(rackIterator.next()); rackIndex = (rackIndex + 1) % racks.length; } - return results; + return result; } static Map> getInverseMap(Map brokerRackMap) { From 5c7dc12f76b40fec3a3d5aa418d0f9119c59577d Mon Sep 17 00:00:00 2001 From: nizhikov Date: Sun, 30 Jul 2023 21:27:15 +0300 Subject: [PATCH 20/78] KAFKA-14595 ReassignPartitionsCommandArgsTest, ReassignPartitionsUnitTest rewritten in java --- bin/kafka-reassign-partitions.sh | 2 +- bin/windows/kafka-reassign-partitions.bat | 2 +- .../ReassignPartitionsCommandArgsTest.scala | 284 ------- .../admin/ReassignPartitionsUnitTest.scala | 671 ---------------- .../kafka/server/util/json/DecodeJson.java | 4 +- .../kafka/tools/reassign/ActiveMoveState.java | 15 + .../tools/reassign/CancelledMoveState.java | 15 + .../tools/reassign/CompletedMoveState.java | 15 + .../reassign/MissingLogDirMoveState.java | 15 + .../reassign/MissingReplicaMoveState.java | 15 + .../kafka/tools/reassign/PartitionMove.java | 22 + .../reassign/PartitionReassignmentState.java | 14 + ...nd.java => ReassignPartitionsCommand.java} | 161 ++-- .../ReassignPartitionsCommandArgsTest.java | 291 +++++++ .../reassign/ReassignPartitionsUnitTest.java | 747 ++++++++++++++++++ 15 files changed, 1242 insertions(+), 1031 deletions(-) delete mode 100644 core/src/test/scala/unit/kafka/admin/ReassignPartitionsCommandArgsTest.scala delete mode 100644 core/src/test/scala/unit/kafka/admin/ReassignPartitionsUnitTest.scala rename tools/src/main/java/org/apache/kafka/tools/reassign/{ReassignPartitionCommand.java => ReassignPartitionsCommand.java} (90%) create mode 100644 tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommandArgsTest.java create mode 100644 tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java diff --git a/bin/kafka-reassign-partitions.sh b/bin/kafka-reassign-partitions.sh index 4c7f1bc35e0a9..65310f90bfe4d 100755 --- a/bin/kafka-reassign-partitions.sh +++ b/bin/kafka-reassign-partitions.sh @@ -14,4 +14,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -exec $(dirname $0)/kafka-run-class.sh kafka.admin.ReassignPartitionsCommand "$@" +exec $(dirname $0)/kafka-run-class.sh org.apache.kafka.tools.reassign.ReassignPartitionsCommand "$@" diff --git a/bin/windows/kafka-reassign-partitions.bat b/bin/windows/kafka-reassign-partitions.bat index 62b710d36e080..5817671451344 100644 --- a/bin/windows/kafka-reassign-partitions.bat +++ b/bin/windows/kafka-reassign-partitions.bat @@ -14,4 +14,4 @@ rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. rem See the License for the specific language governing permissions and rem limitations under the License. -"%~dp0kafka-run-class.bat" kafka.admin.ReassignPartitionsCommand %* +"%~dp0kafka-run-class.bat" org.apache.kafka.tools.reassign.ReassignPartitionsCommand %* diff --git a/core/src/test/scala/unit/kafka/admin/ReassignPartitionsCommandArgsTest.scala b/core/src/test/scala/unit/kafka/admin/ReassignPartitionsCommandArgsTest.scala deleted file mode 100644 index 98d54629f2081..0000000000000 --- a/core/src/test/scala/unit/kafka/admin/ReassignPartitionsCommandArgsTest.scala +++ /dev/null @@ -1,284 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package kafka.admin - -import kafka.utils.Exit -import org.junit.jupiter.api.Assertions._ -import org.junit.jupiter.api.{AfterEach, BeforeEach, Test, Timeout} - -@Timeout(60) -class ReassignPartitionsCommandArgsTest { - - val missingBootstrapServerMsg = "Please specify --bootstrap-server" - - @BeforeEach - def setUp(): Unit = { - Exit.setExitProcedure((_, message) => throw new IllegalArgumentException(message.orNull)) - } - - @AfterEach - def tearDown(): Unit = { - Exit.resetExitProcedure() - } - - ///// Test valid argument parsing - @Test - def shouldCorrectlyParseValidMinimumGenerateOptions(): Unit = { - val args = Array( - "--bootstrap-server", "localhost:1234", - "--generate", - "--broker-list", "101,102", - "--topics-to-move-json-file", "myfile.json") - ReassignPartitionsCommand.validateAndParseArgs(args) - } - - @Test - def shouldCorrectlyParseValidMinimumExecuteOptions(): Unit = { - val args = Array( - "--bootstrap-server", "localhost:1234", - "--execute", - "--reassignment-json-file", "myfile.json") - ReassignPartitionsCommand.validateAndParseArgs(args) - } - - @Test - def shouldCorrectlyParseValidMinimumVerifyOptions(): Unit = { - val args = Array( - "--bootstrap-server", "localhost:1234", - "--verify", - "--reassignment-json-file", "myfile.json") - ReassignPartitionsCommand.validateAndParseArgs(args) - } - - @Test - def shouldAllowThrottleOptionOnExecute(): Unit = { - val args = Array( - "--bootstrap-server", "localhost:1234", - "--execute", - "--throttle", "100", - "--reassignment-json-file", "myfile.json") - ReassignPartitionsCommand.validateAndParseArgs(args) - } - - @Test - def shouldUseDefaultsIfEnabled(): Unit = { - val args = Array( - "--bootstrap-server", "localhost:1234", - "--execute", - "--reassignment-json-file", "myfile.json") - val opts = ReassignPartitionsCommand.validateAndParseArgs(args) - assertEquals(10000L, opts.options.valueOf(opts.timeoutOpt)) - assertEquals(-1L, opts.options.valueOf(opts.interBrokerThrottleOpt)) - } - - @Test - def testList(): Unit = { - val args = Array( - "--list", - "--bootstrap-server", "localhost:1234") - ReassignPartitionsCommand.validateAndParseArgs(args) - } - - @Test - def testCancelWithPreserveThrottlesOption(): Unit = { - val args = Array( - "--cancel", - "--bootstrap-server", "localhost:1234", - "--reassignment-json-file", "myfile.json", - "--preserve-throttles") - ReassignPartitionsCommand.validateAndParseArgs(args) - } - - ///// Test handling missing or invalid actions - @Test - def shouldFailIfNoArgs(): Unit = { - val args: Array[String]= Array() - shouldFailWith(ReassignPartitionsCommand.helpText, args) - } - - @Test - def shouldFailIfBlankArg(): Unit = { - val args = Array(" ") - shouldFailWith("Command must include exactly one action", args) - } - - @Test - def shouldFailIfMultipleActions(): Unit = { - val args = Array( - "--bootstrap-server", "localhost:1234", - "--execute", - "--verify", - "--reassignment-json-file", "myfile.json" - ) - shouldFailWith("Command must include exactly one action", args) - } - - ///// Test --execute - @Test - def shouldNotAllowExecuteWithTopicsOption(): Unit = { - val args = Array( - "--bootstrap-server", "localhost:1234", - "--execute", - "--reassignment-json-file", "myfile.json", - "--topics-to-move-json-file", "myfile.json") - shouldFailWith("Option \"[topics-to-move-json-file]\" can't be used with action \"[execute]\"", args) - } - - @Test - def shouldNotAllowExecuteWithBrokerList(): Unit = { - val args = Array( - "--bootstrap-server", "localhost:1234", - "--execute", - "--reassignment-json-file", "myfile.json", - "--broker-list", "101,102" - ) - shouldFailWith("Option \"[broker-list]\" can't be used with action \"[execute]\"", args) - } - - @Test - def shouldNotAllowExecuteWithoutReassignmentOption(): Unit = { - val args = Array( - "--bootstrap-server", "localhost:1234", - "--execute") - shouldFailWith("Missing required argument \"[reassignment-json-file]\"", args) - } - - @Test - def testMissingBootstrapServerArgumentForExecute(): Unit = { - val args = Array( - "--execute") - shouldFailWith(missingBootstrapServerMsg, args) - } - - ///// Test --generate - @Test - def shouldNotAllowGenerateWithoutBrokersAndTopicsOptions(): Unit = { - val args = Array( - "--bootstrap-server", "localhost:1234", - "--generate") - shouldFailWith("Missing required argument \"[topics-to-move-json-file]\"", args) - } - - @Test - def shouldNotAllowGenerateWithoutBrokersOption(): Unit = { - val args = Array( - "--bootstrap-server", "localhost:1234", - "--topics-to-move-json-file", "myfile.json", - "--generate") - shouldFailWith("Missing required argument \"[broker-list]\"", args) - } - - @Test - def shouldNotAllowGenerateWithoutTopicsOption(): Unit = { - val args = Array( - "--bootstrap-server", "localhost:1234", - "--broker-list", "101,102", - "--generate") - shouldFailWith("Missing required argument \"[topics-to-move-json-file]\"", args) - } - - @Test - def shouldNotAllowGenerateWithThrottleOption(): Unit = { - val args = Array( - "--bootstrap-server", "localhost:1234", - "--generate", - "--broker-list", "101,102", - "--throttle", "100", - "--topics-to-move-json-file", "myfile.json") - shouldFailWith("Option \"[throttle]\" can't be used with action \"[generate]\"", args) - } - - @Test - def shouldNotAllowGenerateWithReassignmentOption(): Unit = { - val args = Array( - "--bootstrap-server", "localhost:1234", - "--generate", - "--broker-list", "101,102", - "--topics-to-move-json-file", "myfile.json", - "--reassignment-json-file", "myfile.json") - shouldFailWith("Option \"[reassignment-json-file]\" can't be used with action \"[generate]\"", args) - } - - @Test - def shouldPrintHelpTextIfHelpArg(): Unit = { - val args: Array[String]= Array("--help") - // note, this is not actually a failed case, it's just we share the same `printUsageAndExit` method when wrong arg received - shouldFailWith(ReassignPartitionsCommand.helpText, args) - } - - ///// Test --verify - @Test - def shouldNotAllowVerifyWithoutReassignmentOption(): Unit = { - val args = Array( - "--bootstrap-server", "localhost:1234", - "--verify") - shouldFailWith("Missing required argument \"[reassignment-json-file]\"", args) - } - - @Test - def shouldNotAllowBrokersListWithVerifyOption(): Unit = { - val args = Array( - "--bootstrap-server", "localhost:1234", - "--verify", - "--broker-list", "100,101", - "--reassignment-json-file", "myfile.json") - shouldFailWith("Option \"[broker-list]\" can't be used with action \"[verify]\"", args) - } - - @Test - def shouldNotAllowThrottleWithVerifyOption(): Unit = { - val args = Array( - "--bootstrap-server", "localhost:1234", - "--verify", - "--throttle", "100", - "--reassignment-json-file", "myfile.json") - shouldFailWith("Option \"[throttle]\" can't be used with action \"[verify]\"", args) - } - - @Test - def shouldNotAllowTopicsOptionWithVerify(): Unit = { - val args = Array( - "--bootstrap-server", "localhost:1234", - "--verify", - "--reassignment-json-file", "myfile.json", - "--topics-to-move-json-file", "myfile.json") - shouldFailWith("Option \"[topics-to-move-json-file]\" can't be used with action \"[verify]\"", args) - } - - def shouldFailWith(msg: String, args: Array[String]): Unit = { - val e = assertThrows(classOf[Exception], () => ReassignPartitionsCommand.validateAndParseArgs(args), - () => s"Should have failed with [$msg] but no failure occurred.") - assertTrue(e.getMessage.startsWith(msg), s"Expected exception with message:\n[$msg]\nbut was\n[${e.getMessage}]") - } - - ///// Test --cancel - @Test - def shouldNotAllowCancelWithoutBootstrapServerOption(): Unit = { - val args = Array( - "--cancel") - shouldFailWith(missingBootstrapServerMsg, args) - } - - @Test - def shouldNotAllowCancelWithoutReassignmentJsonFile(): Unit = { - val args = Array( - "--cancel", - "--bootstrap-server", "localhost:1234", - "--preserve-throttles") - shouldFailWith("Missing required argument \"[reassignment-json-file]\"", args) - } -} diff --git a/core/src/test/scala/unit/kafka/admin/ReassignPartitionsUnitTest.scala b/core/src/test/scala/unit/kafka/admin/ReassignPartitionsUnitTest.scala deleted file mode 100644 index 7aa0f8eec23e7..0000000000000 --- a/core/src/test/scala/unit/kafka/admin/ReassignPartitionsUnitTest.scala +++ /dev/null @@ -1,671 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package kafka.admin - -import java.util.concurrent.ExecutionException -import java.util.{Arrays, Collections, Optional} -import kafka.admin.ReassignPartitionsCommand._ -import kafka.utils.Exit -import org.apache.kafka.admin.BrokerMetadata -import org.apache.kafka.clients.admin.{Config, MockAdminClient, PartitionReassignment} -import org.apache.kafka.common.config.ConfigResource -import org.apache.kafka.common.errors.{InvalidReplicationFactorException, UnknownTopicOrPartitionException} -import org.apache.kafka.common.{Node, TopicPartition, TopicPartitionInfo, TopicPartitionReplica} -import org.apache.kafka.server.common.{AdminCommandFailedException, AdminOperationException} -import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertThrows, assertTrue} -import org.junit.jupiter.api.{AfterEach, BeforeEach, Test, Timeout} - -import scala.collection.mutable -import scala.jdk.CollectionConverters._ - -@Timeout(60) -class ReassignPartitionsUnitTest { - - @BeforeEach - def setUp(): Unit = { - Exit.setExitProcedure((_, message) => throw new IllegalArgumentException(message.orNull)) - } - - @AfterEach - def tearDown(): Unit = { - Exit.resetExitProcedure() - } - - @Test - def testCompareTopicPartitions(): Unit = { - assertTrue(compareTopicPartitions(new TopicPartition("abc", 0), - new TopicPartition("abc", 1))) - assertFalse(compareTopicPartitions(new TopicPartition("def", 0), - new TopicPartition("abc", 1))) - } - - @Test - def testCompareTopicPartitionReplicas(): Unit = { - assertTrue(compareTopicPartitionReplicas(new TopicPartitionReplica("def", 0, 0), - new TopicPartitionReplica("abc", 0, 1))) - assertFalse(compareTopicPartitionReplicas(new TopicPartitionReplica("def", 0, 0), - new TopicPartitionReplica("cde", 0, 0))) - } - - @Test - def testPartitionReassignStatesToString(): Unit = { - assertEquals(Seq( - "Status of partition reassignment:", - "Reassignment of partition bar-0 is still in progress.", - "Reassignment of partition foo-0 is completed.", - "Reassignment of partition foo-1 is still in progress."). - mkString(System.lineSeparator()), - partitionReassignmentStatesToString(Map( - new TopicPartition("foo", 0) -> - PartitionReassignmentState(Seq(1, 2, 3), Seq(1, 2, 3), true), - new TopicPartition("foo", 1) -> - PartitionReassignmentState(Seq(1, 2, 3), Seq(1, 2, 4), false), - new TopicPartition("bar", 0) -> - PartitionReassignmentState(Seq(1, 2, 3), Seq(1, 2, 4), false), - ))) - } - - private def addTopics(adminClient: MockAdminClient): Unit = { - val b = adminClient.brokers() - adminClient.addTopic(false, "foo", Arrays.asList( - new TopicPartitionInfo(0, b.get(0), - Arrays.asList(b.get(0), b.get(1), b.get(2)), - Arrays.asList(b.get(0), b.get(1))), - new TopicPartitionInfo(1, b.get(1), - Arrays.asList(b.get(1), b.get(2), b.get(3)), - Arrays.asList(b.get(1), b.get(2), b.get(3))) - ), Collections.emptyMap()) - adminClient.addTopic(false, "bar", Arrays.asList( - new TopicPartitionInfo(0, b.get(2), - Arrays.asList(b.get(2), b.get(3), b.get(0)), - Arrays.asList(b.get(2), b.get(3), b.get(0))) - ), Collections.emptyMap()) - } - - @Test - def testFindPartitionReassignmentStates(): Unit = { - val adminClient = new MockAdminClient.Builder().numBrokers(4).build() - try { - addTopics(adminClient) - // Create a reassignment and test findPartitionReassignmentStates. - val reassignmentResult: Map[TopicPartition, Class[_ <: Throwable]] = alterPartitionReassignments(adminClient, Map( - new TopicPartition("foo", 0) -> Seq(0,1,3), - new TopicPartition("quux", 0) -> Seq(1,2,3))).map { case (k, v) => k -> v.getClass }.toMap - assertEquals(Map(new TopicPartition("quux", 0) -> classOf[UnknownTopicOrPartitionException]), - reassignmentResult) - assertEquals((Map( - new TopicPartition("foo", 0) -> PartitionReassignmentState(Seq(0,1,2), Seq(0,1,3), false), - new TopicPartition("foo", 1) -> PartitionReassignmentState(Seq(1,2,3), Seq(1,2,3), true) - ), true), - findPartitionReassignmentStates(adminClient, Seq( - (new TopicPartition("foo", 0), Seq(0,1,3)), - (new TopicPartition("foo", 1), Seq(1,2,3)) - ))) - // Cancel the reassignment and test findPartitionReassignmentStates again. - val cancelResult: Map[TopicPartition, Class[_ <: Throwable]] = cancelPartitionReassignments(adminClient, - Set(new TopicPartition("foo", 0), new TopicPartition("quux", 2))).map { case (k, v) => - k -> v.getClass - }.toMap - assertEquals(Map(new TopicPartition("quux", 2) -> classOf[UnknownTopicOrPartitionException]), - cancelResult) - assertEquals((Map( - new TopicPartition("foo", 0) -> PartitionReassignmentState(Seq(0,1,2), Seq(0,1,3), true), - new TopicPartition("foo", 1) -> PartitionReassignmentState(Seq(1,2,3), Seq(1,2,3), true) - ), false), - findPartitionReassignmentStates(adminClient, Seq( - (new TopicPartition("foo", 0), Seq(0,1,3)), - (new TopicPartition("foo", 1), Seq(1,2,3)) - ))) - } finally { - adminClient.close() - } - } - - @Test - def testFindLogDirMoveStates(): Unit = { - val adminClient = new MockAdminClient.Builder(). - numBrokers(4). - brokerLogDirs(Arrays.asList( - Arrays.asList("/tmp/kafka-logs0", "/tmp/kafka-logs1"), - Arrays.asList("/tmp/kafka-logs0", "/tmp/kafka-logs1"), - Arrays.asList("/tmp/kafka-logs0", "/tmp/kafka-logs1"), - Arrays.asList("/tmp/kafka-logs0", null))). - build(); - try { - addTopics(adminClient) - val b = adminClient.brokers() - adminClient.addTopic(false, "quux", Arrays.asList( - new TopicPartitionInfo(0, b.get(2), - Arrays.asList(b.get(1), b.get(2), b.get(3)), - Arrays.asList(b.get(1), b.get(2), b.get(3)))), - Collections.emptyMap()) - adminClient.alterReplicaLogDirs(Map( - new TopicPartitionReplica("foo", 0, 0) -> "/tmp/kafka-logs1", - new TopicPartitionReplica("quux", 0, 0) -> "/tmp/kafka-logs1" - ).asJava).all().get() - assertEquals(Map( - new TopicPartitionReplica("bar", 0, 0) -> new CompletedMoveState("/tmp/kafka-logs0"), - new TopicPartitionReplica("foo", 0, 0) -> new ActiveMoveState("/tmp/kafka-logs0", - "/tmp/kafka-logs1", "/tmp/kafka-logs1"), - new TopicPartitionReplica("foo", 1, 0) -> new CancelledMoveState("/tmp/kafka-logs0", - "/tmp/kafka-logs1"), - new TopicPartitionReplica("quux", 1, 0) -> new MissingLogDirMoveState("/tmp/kafka-logs1"), - new TopicPartitionReplica("quuz", 0, 0) -> new MissingReplicaMoveState("/tmp/kafka-logs0") - ), findLogDirMoveStates(adminClient, Map( - new TopicPartitionReplica("bar", 0, 0) -> "/tmp/kafka-logs0", - new TopicPartitionReplica("foo", 0, 0) -> "/tmp/kafka-logs1", - new TopicPartitionReplica("foo", 1, 0) -> "/tmp/kafka-logs1", - new TopicPartitionReplica("quux", 1, 0) -> "/tmp/kafka-logs1", - new TopicPartitionReplica("quuz", 0, 0) -> "/tmp/kafka-logs0" - ))) - } finally { - adminClient.close() - } - } - - @Test - def testReplicaMoveStatesToString(): Unit = { - assertEquals(Seq( - "Reassignment of replica bar-0-0 completed successfully.", - "Reassignment of replica foo-0-0 is still in progress.", - "Partition foo-1 on broker 0 is not being moved from log dir /tmp/kafka-logs0 to /tmp/kafka-logs1.", - "Partition quux-0 cannot be found in any live log directory on broker 0.", - "Partition quux-1 on broker 1 is being moved to log dir /tmp/kafka-logs2 instead of /tmp/kafka-logs1.", - "Partition quux-2 is not found in any live log dir on broker 1. " + - "There is likely an offline log directory on the broker.").mkString(System.lineSeparator()), - replicaMoveStatesToString(Map( - new TopicPartitionReplica("bar", 0, 0) -> CompletedMoveState("/tmp/kafka-logs0"), - new TopicPartitionReplica("foo", 0, 0) -> ActiveMoveState("/tmp/kafka-logs0", - "/tmp/kafka-logs1", "/tmp/kafka-logs1"), - new TopicPartitionReplica("foo", 1, 0) -> CancelledMoveState("/tmp/kafka-logs0", - "/tmp/kafka-logs1"), - new TopicPartitionReplica("quux", 0, 0) -> MissingReplicaMoveState("/tmp/kafka-logs1"), - new TopicPartitionReplica("quux", 1, 1) -> ActiveMoveState("/tmp/kafka-logs0", - "/tmp/kafka-logs1", "/tmp/kafka-logs2"), - new TopicPartitionReplica("quux", 2, 1) -> MissingLogDirMoveState("/tmp/kafka-logs1") - ))) - } - - @Test - def testGetReplicaAssignments(): Unit = { - val adminClient = new MockAdminClient.Builder().numBrokers(4).build() - try { - addTopics(adminClient) - assertEquals(Map( - new TopicPartition("foo", 0) -> Seq(0, 1, 2), - new TopicPartition("foo", 1) -> Seq(1, 2, 3), - ), - getReplicaAssignmentForTopics(adminClient, Seq("foo"))) - assertEquals(Map( - new TopicPartition("foo", 0) -> Seq(0, 1, 2), - new TopicPartition("bar", 0) -> Seq(2, 3, 0), - ), - getReplicaAssignmentForPartitions(adminClient, Set( - new TopicPartition("foo", 0), new TopicPartition("bar", 0)))) - } finally { - adminClient.close() - } - } - - @Test - def testGetBrokerRackInformation(): Unit = { - val adminClient = new MockAdminClient.Builder(). - brokers(Arrays.asList(new Node(0, "localhost", 9092, "rack0"), - new Node(1, "localhost", 9093, "rack1"), - new Node(2, "localhost", 9094, null))). - build() - try { - assertEquals(Seq( - new BrokerMetadata(0, Optional.of("rack0")), - new BrokerMetadata(1, Optional.of("rack1")) - ), getBrokerMetadata(adminClient, Seq(0, 1), true)) - assertEquals(Seq( - new BrokerMetadata(0, Optional.empty()), - new BrokerMetadata(1, Optional.empty()) - ), getBrokerMetadata(adminClient, Seq(0, 1), false)) - assertStartsWith("Not all brokers have rack information", - assertThrows(classOf[AdminOperationException], - () => getBrokerMetadata(adminClient, Seq(1, 2), true)).getMessage) - assertEquals(Seq( - new BrokerMetadata(1, Optional.empty()), - new BrokerMetadata(2, Optional.empty()) - ), getBrokerMetadata(adminClient, Seq(1, 2), false)) - } finally { - adminClient.close() - } - } - - @Test - def testParseGenerateAssignmentArgs(): Unit = { - assertStartsWith("Broker list contains duplicate entries", - assertThrows(classOf[AdminCommandFailedException], () => parseGenerateAssignmentArgs( - """{"topics": [{"topic": "foo"}], "version":1}""", "1,1,2"), - () => "Expected to detect duplicate broker list entries").getMessage) - assertStartsWith("Broker list contains duplicate entries", - assertThrows(classOf[AdminCommandFailedException], () => parseGenerateAssignmentArgs( - """{"topics": [{"topic": "foo"}], "version":1}""", "5,2,3,4,5"), - () => "Expected to detect duplicate broker list entries").getMessage) - assertEquals((Seq(5,2,3,4),Seq("foo")), - parseGenerateAssignmentArgs("""{"topics": [{"topic": "foo"}], "version":1}""", - "5,2,3,4")) - assertStartsWith("List of topics to reassign contains duplicate entries", - assertThrows(classOf[AdminCommandFailedException], () => parseGenerateAssignmentArgs( - """{"topics": [{"topic": "foo"},{"topic": "foo"}], "version":1}""", "5,2,3,4"), - () => "Expected to detect duplicate topic entries").getMessage) - assertEquals((Seq(5,3,4),Seq("foo","bar")), - parseGenerateAssignmentArgs( - """{"topics": [{"topic": "foo"},{"topic": "bar"}], "version":1}""", - "5,3,4")) - } - - @Test - def testGenerateAssignmentFailsWithoutEnoughReplicas(): Unit = { - val adminClient = new MockAdminClient.Builder().numBrokers(4).build() - try { - addTopics(adminClient) - assertStartsWith("Replication factor: 3 larger than available brokers: 2", - assertThrows(classOf[InvalidReplicationFactorException], - () => generateAssignment(adminClient, """{"topics":[{"topic":"foo"},{"topic":"bar"}]}""", "0,1", false), - () => "Expected generateAssignment to fail").getMessage) - } finally { - adminClient.close() - } - } - - @Test - def testGenerateAssignmentWithInvalidPartitionsFails(): Unit = { - val adminClient = new MockAdminClient.Builder().numBrokers(5).build() - try { - addTopics(adminClient) - assertStartsWith("Topic quux not found", - assertThrows(classOf[ExecutionException], - () => generateAssignment(adminClient, """{"topics":[{"topic":"foo"},{"topic":"quux"}]}""", "0,1", false), - () => "Expected generateAssignment to fail").getCause.getMessage) - } finally { - adminClient.close() - } - } - - @Test - def testGenerateAssignmentWithInconsistentRacks(): Unit = { - val adminClient = new MockAdminClient.Builder(). - brokers(Arrays.asList( - new Node(0, "localhost", 9092, "rack0"), - new Node(1, "localhost", 9093, "rack0"), - new Node(2, "localhost", 9094, null), - new Node(3, "localhost", 9095, "rack1"), - new Node(4, "localhost", 9096, "rack1"), - new Node(5, "localhost", 9097, "rack2"))). - build() - try { - addTopics(adminClient) - assertStartsWith("Not all brokers have rack information.", - assertThrows(classOf[AdminOperationException], - () => generateAssignment(adminClient, """{"topics":[{"topic":"foo"}]}""", "0,1,2,3", true), - () => "Expected generateAssignment to fail").getMessage) - // It should succeed when --disable-rack-aware is used. - val (_, current) = generateAssignment(adminClient, - """{"topics":[{"topic":"foo"}]}""", "0,1,2,3", false) - assertEquals(Map( - new TopicPartition("foo", 0) -> Seq(0, 1, 2), - new TopicPartition("foo", 1) -> Seq(1, 2, 3), - ), current) - } finally { - adminClient.close() - } - } - - @Test - def testGenerateAssignmentWithFewerBrokers(): Unit = { - val adminClient = new MockAdminClient.Builder().numBrokers(4).build() - try { - addTopics(adminClient) - val goalBrokers = Set(0,1,3) - val (proposed, current) = generateAssignment(adminClient, - """{"topics":[{"topic":"foo"},{"topic":"bar"}]}""", - goalBrokers.mkString(","), false) - assertEquals(Map( - new TopicPartition("foo", 0) -> Seq(0, 1, 2), - new TopicPartition("foo", 1) -> Seq(1, 2, 3), - new TopicPartition("bar", 0) -> Seq(2, 3, 0) - ), current) - - // The proposed assignment should only span the provided brokers - proposed.values.foreach(replicas => assertTrue(replicas.forall(goalBrokers.contains), - s"Proposed assignment $proposed puts replicas on brokers other than $goalBrokers")) - } finally { - adminClient.close() - } - } - - @Test - def testCurrentPartitionReplicaAssignmentToString(): Unit = { - assertEquals(Seq( - """Current partition replica assignment""", - """""", - """{"version":1,"partitions":""" + - """[{"topic":"bar","partition":0,"replicas":[7,8],"log_dirs":["any","any"]},""" + - """{"topic":"foo","partition":1,"replicas":[4,5,6],"log_dirs":["any","any","any"]}]""" + - """}""", - """""", - """Save this to use as the --reassignment-json-file option during rollback""" - ).mkString(System.lineSeparator()), - currentPartitionReplicaAssignmentToString(Map( - new TopicPartition("foo", 1) -> Seq(1,2,3), - new TopicPartition("bar", 0) -> Seq(7,8,9) - ), - Map( - new TopicPartition("foo", 0) -> Seq(1,2,3), - new TopicPartition("foo", 1) -> Seq(4,5,6), - new TopicPartition("bar", 0) -> Seq(7,8), - new TopicPartition("baz", 0) -> Seq(10,11,12) - ), - )) - } - - @Test - def testMoveMap(): Unit = { - // overwrite foo-0 with different reassignments - // keep old reassignments of foo-1 - // overwrite foo-2 with same reassignments - // overwrite foo-3 with new reassignments without overlap of old reassignments - // overwrite foo-4 with a subset of old reassignments - // overwrite foo-5 with a superset of old reassignments - // add new reassignments to bar-0 - val moveMap = calculateProposedMoveMap(Map( - new TopicPartition("foo", 0) -> new PartitionReassignment( - Arrays.asList(1,2,3,4), Arrays.asList(4), Arrays.asList(3)), - new TopicPartition("foo", 1) -> new PartitionReassignment( - Arrays.asList(4,5,6,7,8), Arrays.asList(7, 8), Arrays.asList(4, 5)), - new TopicPartition("foo", 2) -> new PartitionReassignment( - Arrays.asList(1,2,3,4), Arrays.asList(3,4), Arrays.asList(1,2)), - new TopicPartition("foo", 3) -> new PartitionReassignment( - Arrays.asList(1,2,3,4), Arrays.asList(3,4), Arrays.asList(1,2)), - new TopicPartition("foo", 4) -> new PartitionReassignment( - Arrays.asList(1,2,3,4), Arrays.asList(3,4), Arrays.asList(1,2)), - new TopicPartition("foo", 5) -> new PartitionReassignment( - Arrays.asList(1,2,3,4), Arrays.asList(3,4), Arrays.asList(1,2)) - ), Map( - new TopicPartition("foo", 0) -> Seq(1,2,5), - new TopicPartition("foo", 2) -> Seq(3,4), - new TopicPartition("foo", 3) -> Seq(5,6), - new TopicPartition("foo", 4) -> Seq(3), - new TopicPartition("foo", 5) -> Seq(3,4,5,6), - new TopicPartition("bar", 0) -> Seq(1,2,3) - ), Map( - new TopicPartition("foo", 0) -> Seq(1,2,3,4), - new TopicPartition("foo", 1) -> Seq(4,5,6,7,8), - new TopicPartition("foo", 2) -> Seq(1,2,3,4), - new TopicPartition("foo", 3) -> Seq(1,2,3,4), - new TopicPartition("foo", 4) -> Seq(1,2,3,4), - new TopicPartition("foo", 5) -> Seq(1,2,3,4), - new TopicPartition("bar", 0) -> Seq(2,3,4), - new TopicPartition("baz", 0) -> Seq(1,2,3) - )) - - assertEquals( - mutable.Map("foo" -> mutable.Map( - 0 -> PartitionMove(mutable.Set(1,2,3), mutable.Set(5)), - 1 -> PartitionMove(mutable.Set(4,5,6), mutable.Set(7,8)), - 2 -> PartitionMove(mutable.Set(1,2), mutable.Set(3,4)), - 3 -> PartitionMove(mutable.Set(1,2), mutable.Set(5,6)), - 4 -> PartitionMove(mutable.Set(1,2), mutable.Set(3)), - 5 -> PartitionMove(mutable.Set(1,2), mutable.Set(3,4,5,6)) - ), "bar" -> mutable.Map( - 0 -> PartitionMove(mutable.Set(2,3,4), mutable.Set(1)), - )), moveMap) - - assertEquals(Map( - "foo" -> "0:1,0:2,0:3,1:4,1:5,1:6,2:1,2:2,3:1,3:2,4:1,4:2,5:1,5:2", - "bar" -> "0:2,0:3,0:4" - ), calculateLeaderThrottles(moveMap)) - - assertEquals(Map( - "foo" -> "0:5,1:7,1:8,2:3,2:4,3:5,3:6,4:3,5:3,5:4,5:5,5:6", - "bar" -> "0:1" - ), calculateFollowerThrottles(moveMap)) - - assertEquals(Set(1,2,3,4,5,6,7,8), calculateReassigningBrokers(moveMap)) - - assertEquals(Set(0,2), calculateMovingBrokers( - Set(new TopicPartitionReplica("quux", 0, 0), - new TopicPartitionReplica("quux", 1, 2)))) - } - - @Test - def testParseExecuteAssignmentArgs(): Unit = { - assertStartsWith("Partition reassignment list cannot be empty", - assertThrows(classOf[AdminCommandFailedException], - () => parseExecuteAssignmentArgs("""{"version":1,"partitions":[]}"""), - () => "Expected to detect empty partition reassignment list").getMessage) - assertStartsWith("Partition reassignment contains duplicate topic partitions", - assertThrows(classOf[AdminCommandFailedException], () => parseExecuteAssignmentArgs( - """{"version":1,"partitions":""" + - """[{"topic":"foo","partition":0,"replicas":[0,1],"log_dirs":["any","any"]},""" + - """{"topic":"foo","partition":0,"replicas":[2,3,4],"log_dirs":["any","any","any"]}""" + - """]}"""), () => "Expected to detect a partition list with duplicate entries").getMessage) - assertStartsWith("Partition reassignment contains duplicate topic partitions", - assertThrows(classOf[AdminCommandFailedException], () => parseExecuteAssignmentArgs( - """{"version":1,"partitions":""" + - """[{"topic":"foo","partition":0,"replicas":[0,1],"log_dirs":["/abc","/def"]},""" + - """{"topic":"foo","partition":0,"replicas":[2,3],"log_dirs":["/abc","/def"]}""" + - """]}"""), () => "Expected to detect a partition replica list with duplicate entries").getMessage) - assertStartsWith("Partition replica lists may not contain duplicate entries", - assertThrows(classOf[AdminCommandFailedException], () => parseExecuteAssignmentArgs( - """{"version":1,"partitions":""" + - """[{"topic":"foo","partition":0,"replicas":[0,0],"log_dirs":["/abc","/def"]},""" + - """{"topic":"foo","partition":1,"replicas":[2,3],"log_dirs":["/abc","/def"]}""" + - """]}"""), () => "Expected to detect a partition replica list with duplicate entries").getMessage) - assertEquals((Map( - new TopicPartition("foo", 0) -> Seq(1, 2, 3), - new TopicPartition("foo", 1) -> Seq(3, 4, 5), - ), Map( - )), - parseExecuteAssignmentArgs( - """{"version":1,"partitions":""" + - """[{"topic":"foo","partition":0,"replicas":[1,2,3],"log_dirs":["any","any","any"]},""" + - """{"topic":"foo","partition":1,"replicas":[3,4,5],"log_dirs":["any","any","any"]}""" + - """]}""")) - assertEquals((Map( - new TopicPartition("foo", 0) -> Seq(1, 2, 3), - ), Map( - new TopicPartitionReplica("foo", 0, 1) -> "/tmp/a", - new TopicPartitionReplica("foo", 0, 2) -> "/tmp/b", - new TopicPartitionReplica("foo", 0, 3) -> "/tmp/c" - )), - parseExecuteAssignmentArgs( - """{"version":1,"partitions":""" + - """[{"topic":"foo","partition":0,"replicas":[1,2,3],"log_dirs":["/tmp/a","/tmp/b","/tmp/c"]}""" + - """]}""")) - } - - @Test - def testExecuteWithInvalidPartitionsFails(): Unit = { - val adminClient = new MockAdminClient.Builder().numBrokers(5).build() - try { - addTopics(adminClient) - assertStartsWith("Topic quux not found", - assertThrows(classOf[ExecutionException], () => executeAssignment(adminClient, false, - """{"version":1,"partitions":""" + - """[{"topic":"foo","partition":0,"replicas":[0,1],"log_dirs":["any","any"]},""" + - """{"topic":"quux","partition":0,"replicas":[2,3,4],"log_dirs":["any","any","any"]}""" + - """]}"""), () => "Expected reassignment with non-existent topic to fail").getCause.getMessage) - } finally { - adminClient.close() - } - } - - @Test - def testExecuteWithInvalidBrokerIdFails(): Unit = { - val adminClient = new MockAdminClient.Builder().numBrokers(4).build() - try { - addTopics(adminClient) - assertStartsWith("Unknown broker id 4", - assertThrows(classOf[AdminCommandFailedException], () => executeAssignment(adminClient, false, - """{"version":1,"partitions":""" + - """[{"topic":"foo","partition":0,"replicas":[0,1],"log_dirs":["any","any"]},""" + - """{"topic":"foo","partition":1,"replicas":[2,3,4],"log_dirs":["any","any","any"]}""" + - """]}"""), () => "Expected reassignment with non-existent broker id to fail").getMessage) - } finally { - adminClient.close() - } - } - - @Test - def testModifyBrokerInterBrokerThrottle(): Unit = { - val adminClient = new MockAdminClient.Builder().numBrokers(4).build() - try { - modifyInterBrokerThrottle(adminClient, Set(0, 1, 2), 1000) - modifyInterBrokerThrottle(adminClient, Set(0, 3), 100) - val brokers = Seq(0, 1, 2, 3).map( - id => new ConfigResource(ConfigResource.Type.BROKER, id.toString)) - val results = adminClient.describeConfigs(brokers.asJava).all().get() - verifyBrokerThrottleResults(results.get(brokers(0)), 100, -1) - verifyBrokerThrottleResults(results.get(brokers(1)), 1000, -1) - verifyBrokerThrottleResults(results.get(brokers(2)), 1000, -1) - verifyBrokerThrottleResults(results.get(brokers(3)), 100, -1) - } finally { - adminClient.close() - } - } - - @Test - def testModifyLogDirThrottle(): Unit = { - val adminClient = new MockAdminClient.Builder().numBrokers(4).build() - try { - modifyLogDirThrottle(adminClient, Set(0, 1, 2), 2000) - modifyLogDirThrottle(adminClient, Set(0, 3), -1) - val brokers = Seq(0, 1, 2, 3).map( - id => new ConfigResource(ConfigResource.Type.BROKER, id.toString)) - val results = adminClient.describeConfigs(brokers.asJava).all().get() - verifyBrokerThrottleResults(results.get(brokers(0)), -1, 2000) - verifyBrokerThrottleResults(results.get(brokers(1)), -1, 2000) - verifyBrokerThrottleResults(results.get(brokers(2)), -1, 2000) - verifyBrokerThrottleResults(results.get(brokers(3)), -1, -1) - } finally { - adminClient.close() - } - } - - @Test - def testCurReassignmentsToString(): Unit = { - val adminClient = new MockAdminClient.Builder().numBrokers(4).build() - try { - addTopics(adminClient) - assertEquals("No partition reassignments found.", curReassignmentsToString(adminClient)) - val reassignmentResult: Map[TopicPartition, Class[_ <: Throwable]] = alterPartitionReassignments(adminClient, - Map( - new TopicPartition("foo", 1) -> Seq(4,5,3), - new TopicPartition("foo", 0) -> Seq(0,1,4,2), - new TopicPartition("bar", 0) -> Seq(2,3) - ) - ).map { case (k, v) => k -> v.getClass }.toMap - assertEquals(Map(), reassignmentResult) - assertEquals(Seq("Current partition reassignments:", - "bar-0: replicas: 2,3,0. removing: 0.", - "foo-0: replicas: 0,1,2. adding: 4.", - "foo-1: replicas: 1,2,3. adding: 4,5. removing: 1,2.").mkString(System.lineSeparator()), - curReassignmentsToString(adminClient)) - } finally { - adminClient.close() - } - } - - private def verifyBrokerThrottleResults(config: Config, - expectedInterBrokerThrottle: Long, - expectedReplicaAlterLogDirsThrottle: Long): Unit = { - val configs = new mutable.HashMap[String, String] - config.entries.forEach(entry => configs.put(entry.name, entry.value)) - if (expectedInterBrokerThrottle >= 0) { - assertEquals(expectedInterBrokerThrottle.toString, - configs.getOrElse(brokerLevelLeaderThrottle, "")) - assertEquals(expectedInterBrokerThrottle.toString, - configs.getOrElse(brokerLevelFollowerThrottle, "")) - } - if (expectedReplicaAlterLogDirsThrottle >= 0) { - assertEquals(expectedReplicaAlterLogDirsThrottle.toString, - configs.getOrElse(brokerLevelLogDirThrottle, "")) - } - } - - @Test - def testModifyTopicThrottles(): Unit = { - val adminClient = new MockAdminClient.Builder().numBrokers(4).build() - try { - addTopics(adminClient) - modifyTopicThrottles(adminClient, - Map("foo" -> "leaderFoo", "bar" -> "leaderBar"), - Map("bar" -> "followerBar")) - val topics = Seq("bar", "foo").map( - id => new ConfigResource(ConfigResource.Type.TOPIC, id)) - val results = adminClient.describeConfigs(topics.asJava).all().get() - verifyTopicThrottleResults(results.get(topics(0)), "leaderBar", "followerBar") - verifyTopicThrottleResults(results.get(topics(1)), "leaderFoo", "") - } finally { - adminClient.close() - } - } - - private def verifyTopicThrottleResults(config: Config, - expectedLeaderThrottle: String, - expectedFollowerThrottle: String): Unit = { - val configs = new mutable.HashMap[String, String] - config.entries.forEach(entry => configs.put(entry.name, entry.value)) - assertEquals(expectedLeaderThrottle, - configs.getOrElse(topicLevelLeaderThrottle, "")) - assertEquals(expectedFollowerThrottle, - configs.getOrElse(topicLevelFollowerThrottle, "")) - } - - @Test - def testAlterReplicaLogDirs(): Unit = { - val adminClient = new MockAdminClient.Builder(). - numBrokers(4). - brokerLogDirs(Collections.nCopies(4, - Arrays.asList("/tmp/kafka-logs0", "/tmp/kafka-logs1"))). - build() - try { - addTopics(adminClient) - assertEquals(Set( - new TopicPartitionReplica("foo", 0, 0) - ), - alterReplicaLogDirs(adminClient, Map( - new TopicPartitionReplica("foo", 0, 0) -> "/tmp/kafka-logs1", - new TopicPartitionReplica("quux", 1, 0) -> "/tmp/kafka-logs1" - ))) - } finally { - adminClient.close() - } - } - - def assertStartsWith(prefix: String, str: String): Unit = { - assertTrue(str.startsWith(prefix), "Expected the string to start with %s, but it was %s".format(prefix, str)) - } - - @Test - def testPropagateInvalidJsonError(): Unit = { - val adminClient = new MockAdminClient.Builder().numBrokers(4).build() - try { - addTopics(adminClient) - assertStartsWith("Unexpected character", - assertThrows(classOf[AdminOperationException], () => executeAssignment(adminClient, additional = false, "{invalid_json")).getMessage) - } finally { - adminClient.close() - } - } -} diff --git a/server-common/src/main/java/org/apache/kafka/server/util/json/DecodeJson.java b/server-common/src/main/java/org/apache/kafka/server/util/json/DecodeJson.java index dbc28548a3427..e46c85c4d4892 100644 --- a/server-common/src/main/java/org/apache/kafka/server/util/json/DecodeJson.java +++ b/server-common/src/main/java/org/apache/kafka/server/util/json/DecodeJson.java @@ -96,7 +96,7 @@ static DecodeJson> decodeOptional(DecodeJson decodeJson) { }; } - static DecodeJson> decodeList(DecodeJson decodeJson) throws JsonMappingException { + static DecodeJson> decodeList(DecodeJson decodeJson) { return node -> { if (node.isArray()) { List result = new ArrayList<>(); @@ -110,7 +110,7 @@ static DecodeJson> decodeList(DecodeJson decodeJson) throws JsonM }; } - static DecodeJson> decodeMap(DecodeJson decodeJson) throws JsonMappingException { + static DecodeJson> decodeMap(DecodeJson decodeJson) { return node -> { if (node.isObject()) { Map result = new HashMap<>(); diff --git a/tools/src/main/java/org/apache/kafka/tools/reassign/ActiveMoveState.java b/tools/src/main/java/org/apache/kafka/tools/reassign/ActiveMoveState.java index fa9af20e59378..842d46ec58741 100644 --- a/tools/src/main/java/org/apache/kafka/tools/reassign/ActiveMoveState.java +++ b/tools/src/main/java/org/apache/kafka/tools/reassign/ActiveMoveState.java @@ -17,6 +17,8 @@ package org.apache.kafka.tools.reassign; +import java.util.Objects; + /** * A replica log directory move state where the move is in progress. */ @@ -42,4 +44,17 @@ public ActiveMoveState(String currentLogDir, String targetLogDir, String futureL public boolean done() { return false; } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ActiveMoveState that = (ActiveMoveState) o; + return Objects.equals(currentLogDir, that.currentLogDir) && Objects.equals(targetLogDir, that.targetLogDir) && Objects.equals(futureLogDir, that.futureLogDir); + } + + @Override + public int hashCode() { + return Objects.hash(currentLogDir, targetLogDir, futureLogDir); + } } diff --git a/tools/src/main/java/org/apache/kafka/tools/reassign/CancelledMoveState.java b/tools/src/main/java/org/apache/kafka/tools/reassign/CancelledMoveState.java index 5c4fbf98f5312..f405eedd4038b 100644 --- a/tools/src/main/java/org/apache/kafka/tools/reassign/CancelledMoveState.java +++ b/tools/src/main/java/org/apache/kafka/tools/reassign/CancelledMoveState.java @@ -17,6 +17,8 @@ package org.apache.kafka.tools.reassign; +import java.util.Objects; + /** * A replica log directory move state where there is no move in progress, but we did not * reach the target log directory. @@ -39,4 +41,17 @@ public CancelledMoveState(String currentLogDir, String targetLogDir) { public boolean done() { return true; } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + CancelledMoveState that = (CancelledMoveState) o; + return Objects.equals(currentLogDir, that.currentLogDir) && Objects.equals(targetLogDir, that.targetLogDir); + } + + @Override + public int hashCode() { + return Objects.hash(currentLogDir, targetLogDir); + } } diff --git a/tools/src/main/java/org/apache/kafka/tools/reassign/CompletedMoveState.java b/tools/src/main/java/org/apache/kafka/tools/reassign/CompletedMoveState.java index af18195db4542..df5fb89014868 100644 --- a/tools/src/main/java/org/apache/kafka/tools/reassign/CompletedMoveState.java +++ b/tools/src/main/java/org/apache/kafka/tools/reassign/CompletedMoveState.java @@ -17,6 +17,8 @@ package org.apache.kafka.tools.reassign; +import java.util.Objects; + /** * The completed replica log directory move state. */ @@ -34,4 +36,17 @@ public CompletedMoveState(String targetLogDir) { public boolean done() { return true; } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + CompletedMoveState that = (CompletedMoveState) o; + return Objects.equals(targetLogDir, that.targetLogDir); + } + + @Override + public int hashCode() { + return Objects.hash(targetLogDir); + } } diff --git a/tools/src/main/java/org/apache/kafka/tools/reassign/MissingLogDirMoveState.java b/tools/src/main/java/org/apache/kafka/tools/reassign/MissingLogDirMoveState.java index d647e045ef3a8..eb3f592841c22 100644 --- a/tools/src/main/java/org/apache/kafka/tools/reassign/MissingLogDirMoveState.java +++ b/tools/src/main/java/org/apache/kafka/tools/reassign/MissingLogDirMoveState.java @@ -17,6 +17,8 @@ package org.apache.kafka.tools.reassign; +import java.util.Objects; + /** * A replica log directory move state where the source replica is missing. */ @@ -34,4 +36,17 @@ public MissingLogDirMoveState(String targetLogDir) { public boolean done() { return false; } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + MissingLogDirMoveState that = (MissingLogDirMoveState) o; + return Objects.equals(targetLogDir, that.targetLogDir); + } + + @Override + public int hashCode() { + return Objects.hash(targetLogDir); + } } diff --git a/tools/src/main/java/org/apache/kafka/tools/reassign/MissingReplicaMoveState.java b/tools/src/main/java/org/apache/kafka/tools/reassign/MissingReplicaMoveState.java index 04c978f6b79df..eda9c22b829a8 100644 --- a/tools/src/main/java/org/apache/kafka/tools/reassign/MissingReplicaMoveState.java +++ b/tools/src/main/java/org/apache/kafka/tools/reassign/MissingReplicaMoveState.java @@ -17,6 +17,8 @@ package org.apache.kafka.tools.reassign; +import java.util.Objects; + /** * A replica log directory move state where the source log directory is missing. */ @@ -34,4 +36,17 @@ public MissingReplicaMoveState(String targetLogDir) { public boolean done() { return false; } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + MissingReplicaMoveState that = (MissingReplicaMoveState) o; + return Objects.equals(targetLogDir, that.targetLogDir); + } + + @Override + public int hashCode() { + return Objects.hash(targetLogDir); + } } diff --git a/tools/src/main/java/org/apache/kafka/tools/reassign/PartitionMove.java b/tools/src/main/java/org/apache/kafka/tools/reassign/PartitionMove.java index 76c467ccf1859..27fc72923df29 100644 --- a/tools/src/main/java/org/apache/kafka/tools/reassign/PartitionMove.java +++ b/tools/src/main/java/org/apache/kafka/tools/reassign/PartitionMove.java @@ -17,6 +17,7 @@ package org.apache.kafka.tools.reassign; +import java.util.Objects; import java.util.Set; /** @@ -35,4 +36,25 @@ public PartitionMove(Set sources, Set destinations) { this.sources = sources; this.destinations = destinations; } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + PartitionMove that = (PartitionMove) o; + return Objects.equals(sources, that.sources) && Objects.equals(destinations, that.destinations); + } + + @Override + public int hashCode() { + return Objects.hash(sources, destinations); + } + + @Override + public String toString() { + return "PartitionMove{" + + "sources=" + sources + + ", destinations=" + destinations + + '}'; + } } diff --git a/tools/src/main/java/org/apache/kafka/tools/reassign/PartitionReassignmentState.java b/tools/src/main/java/org/apache/kafka/tools/reassign/PartitionReassignmentState.java index a7e0f81086f8f..53ae960f863db 100644 --- a/tools/src/main/java/org/apache/kafka/tools/reassign/PartitionReassignmentState.java +++ b/tools/src/main/java/org/apache/kafka/tools/reassign/PartitionReassignmentState.java @@ -18,6 +18,7 @@ package org.apache.kafka.tools.reassign; import java.util.List; +import java.util.Objects; /** * The state of a partition reassignment. The current replicas and target replicas @@ -40,4 +41,17 @@ public PartitionReassignmentState(List currentReplicas, List t this.targetReplicas = targetReplicas; this.done = done; } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + PartitionReassignmentState state = (PartitionReassignmentState) o; + return done == state.done && Objects.equals(currentReplicas, state.currentReplicas) && Objects.equals(targetReplicas, state.targetReplicas); + } + + @Override + public int hashCode() { + return Objects.hash(currentReplicas, targetReplicas, done); + } } diff --git a/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionCommand.java b/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommand.java similarity index 90% rename from tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionCommand.java rename to tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommand.java index 2393fefe8aa9a..87d70845ad487 100644 --- a/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionCommand.java +++ b/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommand.java @@ -68,15 +68,19 @@ import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; -public class ReassignPartitionCommand { +public class ReassignPartitionsCommand { private static final String ANY_LOG_DIR = "any"; - private static final String HELP_TEXT = "This tool helps to move topic partitions between replicas."; + static final String HELP_TEXT = "This tool helps to move topic partitions between replicas."; private static final DecodeJson.DecodeInteger INT = new DecodeJson.DecodeInteger(); private static final DecodeJson.DecodeString STRING = new DecodeJson.DecodeString(); + private static final DecodeJson> INT_LIST = DecodeJson.decodeList(INT); + + private static final DecodeJson> STRING_LIST = DecodeJson.decodeList(STRING); + /** * The earliest version of the partition reassignment JSON. We will default to this * version if no other version number is given. @@ -91,11 +95,11 @@ public class ReassignPartitionCommand { // Throttles that are set at the level of an individual broker. //DynamicConfig.Broker.LeaderReplicationThrottledRateProp - private static final String BROKER_LEVEL_LEADER_THROTTLE = "leader.replication.throttled.rate"; + static final String BROKER_LEVEL_LEADER_THROTTLE = "leader.replication.throttled.rate"; //DynamicConfig.Broker.FollowerReplicationThrottledRateProp - private static final String BROKER_LEVEL_FOLLOWER_THROTTLE = "follower.replication.throttled.rate"; + static final String BROKER_LEVEL_FOLLOWER_THROTTLE = "follower.replication.throttled.rate"; //DynamicConfig.Broker.ReplicaAlterLogDirsIoMaxBytesPerSecondProp - private static final String BROKER_LEVEL_LOG_DIR_THROTTLE = "replica.alter.log.dirs.io.max.bytes.per.second"; + static final String BROKER_LEVEL_LOG_DIR_THROTTLE = "replica.alter.log.dirs.io.max.bytes.per.second"; private static final List BROKER_LEVEL_THROTTLES = Arrays.asList( BROKER_LEVEL_LEADER_THROTTLE, BROKER_LEVEL_FOLLOWER_THROTTLE, @@ -104,9 +108,9 @@ public class ReassignPartitionCommand { // Throttles that are set at the level of an individual topic. //LogConfig.LEADER_REPLICATION_THROTTLED_REPLICAS_CONFIG - private static final String TOPIC_LEVEL_LEADER_THROTTLE = "leader.replication.throttled.replicas"; + static final String TOPIC_LEVEL_LEADER_THROTTLE = "leader.replication.throttled.replicas"; //LogConfig.FOLLOWER_REPLICATION_THROTTLED_REPLICAS_CONFIG - private static final String TOPIC_LEVEL_FOLLOWER_THROTTLE = "follower.replication.throttled.replicas"; + static final String TOPIC_LEVEL_FOLLOWER_THROTTLE = "follower.replication.throttled.replicas"; private static final List TOPIC_LEVEL_THROTTLES = Arrays.asList( TOPIC_LEVEL_LEADER_THROTTLE, TOPIC_LEVEL_FOLLOWER_THROTTLE @@ -241,12 +245,12 @@ private static Tuple, Boolean> v return new Tuple<>(partStates, partsOngoing); } - private static int compareTopicPartitions(TopicPartition a, TopicPartition b) { + static int compareTopicPartitions(TopicPartition a, TopicPartition b) { int topicOrder = Objects.compare(a.topic(), b.topic(), String::compareTo); return topicOrder == 0 ? Integer.compare(a.partition(), b.partition()) : topicOrder; } - private static int compareTopicPartitionReplicas(TopicPartitionReplica a, TopicPartitionReplica b) { + static int compareTopicPartitionReplicas(TopicPartitionReplica a, TopicPartitionReplica b) { int brokerOrder = Integer.compare(a.brokerId(), b.brokerId()); if (brokerOrder != 0) @@ -262,10 +266,10 @@ private static int compareTopicPartitionReplicas(TopicPartitionReplica a, TopicP * @param states A map from topic partitions to states. * @return A string summarizing the partition reassignment states. */ - private static String partitionReassignmentStatesToString(Map states) { + static String partitionReassignmentStatesToString(Map states) { List bld = new ArrayList<>(); bld.add("Status of partition reassignment:"); - states.keySet().stream().sorted(ReassignPartitionCommand::compareTopicPartitions).forEach(topicPartition -> { + states.keySet().stream().sorted(ReassignPartitionsCommand::compareTopicPartitions).forEach(topicPartition -> { PartitionReassignmentState state = states.get(topicPartition); if (state.done) { if (state.currentReplicas.equals(state.targetReplicas)) { @@ -294,8 +298,8 @@ private static String partitionReassignmentStatesToString(Map, Boolean> findPartitionReassignmentStates(Admin adminClient, - List>> targetReassignments) throws ExecutionException, InterruptedException { + static Tuple, Boolean> findPartitionReassignmentStates(Admin adminClient, + List>> targetReassignments) throws ExecutionException, InterruptedException { Map currentReassignments = adminClient. listPartitionReassignments().reassignments().get(); @@ -303,7 +307,7 @@ private static Tuple, Boolean> f List>> notFoundReassignments = new ArrayList<>(); targetReassignments.forEach(reassignment -> { - if (targetReassignments.contains(reassignment)) + if (currentReassignments.containsKey(reassignment.v1)) foundReassignments.add(reassignment); else notFoundReassignments.add(reassignment); @@ -404,8 +408,8 @@ private static Tuple, Boolean> verif * * @return The states for each replica movement. */ - private static Map findLogDirMoveStates(Admin adminClient, - Map targetMoves) throws ExecutionException, InterruptedException { + static Map findLogDirMoveStates(Admin adminClient, + Map targetMoves) throws ExecutionException, InterruptedException { Map replicaLogDirInfos = adminClient. describeReplicaLogDirs(targetMoves.keySet()).all().get(); @@ -448,9 +452,9 @@ private static Map findLogDirMoveStates( * @return A tuple of a summary string, and a boolean describing * whether there are any active replica moves. */ - private static String replicaMoveStatesToString(Map states) { + static String replicaMoveStatesToString(Map states) { List bld = new ArrayList<>(); - states.keySet().stream().sorted(ReassignPartitionCommand::compareTopicPartitionReplicas).forEach(replica -> { + states.keySet().stream().sorted(ReassignPartitionsCommand::compareTopicPartitionReplicas).forEach(replica -> { LogDirMoveState state = states.get(replica); if (state instanceof MissingLogDirMoveState) { bld.add("Partition " + replica.topic() + "-" + replica.partition() + " is not found " + @@ -548,10 +552,10 @@ private static void clearTopicLevelThrottles(Admin adminClient, Set topi * @return A tuple containing the proposed assignment and the * current assignment. */ - private static Tuple>, Map>> generateAssignment(Admin adminClient, - String reassignmentJson, - String brokerListString, - Boolean enableRackAwareness) throws ExecutionException, InterruptedException, JsonProcessingException { + static Tuple>, Map>> generateAssignment(Admin adminClient, + String reassignmentJson, + String brokerListString, + Boolean enableRackAwareness) throws ExecutionException, InterruptedException, JsonProcessingException { Tuple, List> t0 = parseGenerateAssignmentArgs(reassignmentJson, brokerListString); List brokersToReassign = t0.v1; @@ -618,8 +622,8 @@ private static Map describeTopics(Admin adminClient, * @return A map from partitions to broker assignments. * If any topic can't be found, an exception will be thrown. */ - private static Map> getReplicaAssignmentForTopics(Admin adminClient, - List topics) throws ExecutionException, InterruptedException { + static Map> getReplicaAssignmentForTopics(Admin adminClient, + List topics) throws ExecutionException, InterruptedException { Map> res = new HashMap<>(); describeTopics(adminClient, new HashSet<>(topics)).forEach((topicName, topicDescription) -> @@ -639,7 +643,7 @@ private static Map> getReplicaAssignmentForTopics( * @return A map from partitions to broker assignments. * If any topic can't be found, an exception will be thrown. */ - private static Map> getReplicaAssignmentForPartitions(Admin adminClient, + static Map> getReplicaAssignmentForPartitions(Admin adminClient, Set partitions) throws ExecutionException, InterruptedException { Map> res = new HashMap<>(); describeTopics(adminClient, partitions.stream().map(TopicPartition::topic).collect(Collectors.toSet())).forEach((topicName, topicDescription) -> { @@ -663,7 +667,7 @@ private static Map> getReplicaAssignmentForPartiti * @return The metadata for each broker that was found. * Brokers that were not found will be omitted. */ - private static List getBrokerMetadata(Admin adminClient, List brokers, boolean enableRackAwareness) throws ExecutionException, InterruptedException { + static List getBrokerMetadata(Admin adminClient, List brokers, boolean enableRackAwareness) throws ExecutionException, InterruptedException { Set brokerSet = new HashSet<>(brokers); List results = adminClient.describeCluster().nodes().get().stream(). filter(node -> brokerSet.contains(node.id())). @@ -690,7 +694,7 @@ private static List getBrokerMetadata(Admin adminClient, List, List> parseGenerateAssignmentArgs(String reassignmentJson, + static Tuple, List> parseGenerateAssignmentArgs(String reassignmentJson, String brokerList) throws JsonMappingException { List brokerListToReassign = Arrays.stream(brokerList.split(",")).map(Integer::parseInt).collect(Collectors.toList()); Set duplicateReassignments = ToolsUtils.duplicates(brokerListToReassign); @@ -718,7 +722,7 @@ private static Tuple, List> parseGenerateAssignmentArgs(St * replica assignment to begin. * @param time The Time object to use. */ - private static void executeAssignment(Admin adminClient, + static void executeAssignment(Admin adminClient, Boolean additional, String reassignmentJson, Long interBrokerThrottle, @@ -763,14 +767,14 @@ private static void executeAssignment(Admin adminClient, throw new TerseReassignmentFailureException( String.format("Error reassigning partition(s):%n%s", errors.keySet().stream(). - sorted(ReassignPartitionCommand::compareTopicPartitions). + sorted(ReassignPartitionsCommand::compareTopicPartitions). map(part -> part + ": " + errors.get(part).getMessage()). collect(Collectors.joining(System.lineSeparator())))); } System.out.printf("Successfully started partition reassignment%s for %s%n", proposedParts.size() == 1 ? "" : "s", proposedParts.keySet().stream(). - sorted(ReassignPartitionCommand::compareTopicPartitions). + sorted(ReassignPartitionsCommand::compareTopicPartitions). map(Objects::toString). collect(Collectors.joining(","))); if (!proposedReplicas.isEmpty()) { @@ -802,7 +806,7 @@ private static void executeMoves(Admin adminClient, System.out.printf("Successfully started log directory move%s for: %s", completed.size() == 1 ? "" : "s", completed.stream(). - sorted(ReassignPartitionCommand::compareTopicPartitionReplicas). + sorted(ReassignPartitionsCommand::compareTopicPartitionReplicas). map(Object::toString). collect(Collectors.joining(","))); } @@ -814,7 +818,7 @@ private static void executeMoves(Admin adminClient, "Timed out before log directory move%s could be started for: %s", pendingReplicas.size() == 1 ? "" : "s", pendingReplicas.keySet().stream(). - sorted(ReassignPartitionCommand::compareTopicPartitionReplicas). + sorted(ReassignPartitionsCommand::compareTopicPartitionReplicas). map(Object::toString). collect(Collectors.joining(",")))); } else { @@ -843,10 +847,10 @@ private static void listReassignments(Admin adminClient) throws ExecutionExcepti * @param adminClient The AdminClient to use. * @return A string describing the current partition reassignments. */ - private static String curReassignmentsToString(Admin adminClient) throws ExecutionException, InterruptedException { + static String curReassignmentsToString(Admin adminClient) throws ExecutionException, InterruptedException { Map currentReassignments = adminClient. listPartitionReassignments().reassignments().get(); - String text = currentReassignments.keySet().stream().sorted(ReassignPartitionCommand::compareTopicPartitions).map(part -> { + String text = currentReassignments.keySet().stream().sorted(ReassignPartitionsCommand::compareTopicPartitions).map(part -> { PartitionReassignment reassignment = currentReassignments.get(part); List replicas = reassignment.replicas(); List addingReplicas = reassignment.addingReplicas(); @@ -893,8 +897,8 @@ private static void verifyBrokerIds(Admin adminClient, Set brokers) thr * @return The string to print. We will only print information about * partitions that appear in the proposed partition assignment. */ - private static String currentPartitionReplicaAssignmentToString(Map> proposedParts, - Map> currentParts) throws JsonProcessingException { + static String currentPartitionReplicaAssignmentToString(Map> proposedParts, + Map> currentParts) throws JsonProcessingException { Map> partitionsToBeReassigned = currentParts.entrySet().stream(). filter(e -> proposedParts.containsKey(e.getKey())). collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); @@ -911,8 +915,8 @@ private static String currentPartitionReplicaAssignmentToString(Map alterPartitionReassignments(Admin adminClient, - Map> reassignments) throws InterruptedException { + static Map alterPartitionReassignments(Admin adminClient, + Map> reassignments) throws InterruptedException { Map> args = new HashMap<>(); reassignments.entrySet().forEach(e -> { TopicPartition part = e.getKey(); @@ -939,7 +943,7 @@ private static Map alterPartitionReassignments(Admin * @param reassignments The partition reassignments to cancel. * @return A map from partition objects to error strings. */ - private static Map cancelPartitionReassignments(Admin adminClient, + static Map cancelPartitionReassignments(Admin adminClient, Set reassignments) throws InterruptedException { Map> args = new HashMap<>(); reassignments.forEach(part -> args.put(part, Optional.empty())); @@ -966,6 +970,18 @@ private static Map> calculateCurrentMoveMap( currentReassignments.entrySet().forEach(e -> { TopicPartition part = e.getKey(); PartitionReassignment reassignment = e.getValue(); + + List allReplicas = reassignment.replicas(); + List addingReplicas = reassignment.addingReplicas(); + + // The addingReplicas is included in the replicas during reassignment + Set sources = new HashSet<>(allReplicas); + sources.removeAll(addingReplicas); + + Set destinations = new HashSet<>(addingReplicas); + + Map partMoves = moveMap.computeIfAbsent(part.topic(), k -> new HashMap<>()); + partMoves.put(part.partition(), new PartitionMove(sources, destinations)); }); return moveMap; } @@ -981,9 +997,9 @@ private static Map> calculateCurrentMoveMap( * The partition map is keyed on partition index and contains * the movements for that partition. */ - private static Map> calculateProposedMoveMap(Map currentReassignments, - Map> proposedParts, - Map> currentParts) { + static Map> calculateProposedMoveMap(Map currentReassignments, + Map> proposedParts, + Map> currentParts) { Map> moveMap = calculateCurrentMoveMap(currentReassignments); for (Map.Entry> e : proposedParts.entrySet()) { TopicPartition part = e.getKey(); @@ -997,7 +1013,9 @@ private static Map> calculateProposedMoveMap if (partMoves.containsKey(part.partition())) { PartitionMove move = partMoves.get(part.partition()); sources.addAll(move.sources); - } else if (!currentParts.containsKey(part)) + } else if (currentParts.containsKey(part)) + sources.addAll(currentParts.get(part)); + else throw new RuntimeException("Trying to reassign a topic partition " + part + " with 0 replicas"); Set destinations = new HashSet<>(replicas); @@ -1014,7 +1032,7 @@ private static Map> calculateProposedMoveMap * @param moveMap The movements. * @return A map from topic names to leader throttle configurations. */ - private static Map calculateLeaderThrottles(Map> moveMap) { + static Map calculateLeaderThrottles(Map> moveMap) { Map results = new HashMap<>(); moveMap.forEach((topicName, partMoveMap) -> { Set components = new TreeSet<>(); @@ -1031,7 +1049,7 @@ private static Map calculateLeaderThrottles(Map calculateFollowerThrottles(Map> moveMap) { + static Map calculateFollowerThrottles(Map> moveMap) { Map results = new HashMap<>(); moveMap.forEach((topicName, partMoveMap) -> { Set components = new TreeSet<>(); @@ -1054,7 +1072,7 @@ private static Map calculateFollowerThrottles(Map calculateReassigningBrokers(Map> moveMap) { + static Set calculateReassigningBrokers(Map> moveMap) { Set reassigningBrokers = new TreeSet<>(); moveMap.values().forEach(partMoveMap -> { partMoveMap.values().forEach(partMove -> { @@ -1071,7 +1089,7 @@ private static Set calculateReassigningBrokers(Map calculateMovingBrokers(Set replicaMoves) { + static Set calculateMovingBrokers(Set replicaMoves) { return replicaMoves.stream().map(TopicPartitionReplica::brokerId).collect(Collectors.toSet()); } @@ -1082,9 +1100,9 @@ private static Set calculateMovingBrokers(Set re * @param leaderThrottles A map from topic names to leader throttle configurations. * @param followerThrottles A map from topic names to follower throttle configurations. */ - private static void modifyTopicThrottles(Admin adminClient, - Map leaderThrottles, - Map followerThrottles) throws ExecutionException, InterruptedException { + static void modifyTopicThrottles(Admin adminClient, + Map leaderThrottles, + Map followerThrottles) throws ExecutionException, InterruptedException { Map> configs = new HashMap<>(); Set topicNames = new HashSet<>(leaderThrottles.keySet()); topicNames.addAll(followerThrottles.keySet()); @@ -1094,7 +1112,7 @@ private static void modifyTopicThrottles(Admin adminClient, ops.add(new AlterConfigOp(new ConfigEntry(TOPIC_LEVEL_LEADER_THROTTLE, leaderThrottles.get(topicName)), AlterConfigOp.OpType.SET)); } if (followerThrottles.containsKey(topicName)) { - ops.add(new AlterConfigOp(new ConfigEntry(TOPIC_LEVEL_FOLLOWER_THROTTLE, leaderThrottles.get(topicName)), AlterConfigOp.OpType.SET)); + ops.add(new AlterConfigOp(new ConfigEntry(TOPIC_LEVEL_FOLLOWER_THROTTLE, followerThrottles.get(topicName)), AlterConfigOp.OpType.SET)); } if (!ops.isEmpty()) { configs.put(new ConfigResource(ConfigResource.Type.TOPIC, topicName), ops); @@ -1119,9 +1137,9 @@ private static void modifyReassignmentThrottle(Admin admin, Map reassigningBrokers, - long interBrokerThrottle) throws ExecutionException, InterruptedException { + static void modifyInterBrokerThrottle(Admin adminClient, + Set reassigningBrokers, + long interBrokerThrottle) throws ExecutionException, InterruptedException { if (interBrokerThrottle >= 0) { Map> configs = new HashMap<>(); reassigningBrokers.forEach(brokerId -> { @@ -1144,9 +1162,9 @@ private static void modifyInterBrokerThrottle(Admin adminClient, * @param movingBrokers The set of broker to alter the throttle of * @param logDirThrottle The new throttle (ignored if less than 0) */ - private static void modifyLogDirThrottle(Admin admin, - Set movingBrokers, - long logDirThrottle) throws ExecutionException, InterruptedException { + static void modifyLogDirThrottle(Admin admin, + Set movingBrokers, + long logDirThrottle) throws ExecutionException, InterruptedException { if (logDirThrottle >= 0) { Map> configs = new HashMap<>(); movingBrokers.forEach(brokerId -> { @@ -1166,7 +1184,7 @@ private static void modifyLogDirThrottle(Admin admin, * @return A tuple of the partitions to be reassigned and the replicas * to be reassigned. */ - private static Tuple>, Map> parseExecuteAssignmentArgs(String reassignmentJson) throws JsonProcessingException { + static Tuple>, Map> parseExecuteAssignmentArgs(String reassignmentJson) throws JsonProcessingException { Tuple>>, Map> t0 = parsePartitionReassignmentData(reassignmentJson); List>> partitionsToBeReassigned = t0.v1; @@ -1177,7 +1195,7 @@ private static Tuple>, Map t.v2.isEmpty())) { throw new AdminCommandFailedException("Partition replica list cannot be empty"); } - Set duplicateReassignedPartitions = ToolsUtils.duplicates(partitionsToBeReassigned.stream().map(t -> t.v1).collect(Collectors.toSet())); + Set duplicateReassignedPartitions = ToolsUtils.duplicates(partitionsToBeReassigned.stream().map(t -> t.v1).collect(Collectors.toList())); if (!duplicateReassignedPartitions.isEmpty()) { throw new AdminCommandFailedException(String.format( "Partition reassignment contains duplicate topic partitions: %s", @@ -1234,13 +1252,13 @@ private static Tuple, Set> cancelAssi "Error cancelling partition reassignment%s for:%n%s", errors.size() == 1 ? "" : "s", errors.keySet().stream(). - sorted(ReassignPartitionCommand::compareTopicPartitions). + sorted(ReassignPartitionsCommand::compareTopicPartitions). map(part -> part + ": " + errors.get(part).getMessage()).collect(Collectors.joining(System.lineSeparator()))) ); } System.out.printf("Successfully cancelled partition reassignment%s for: %s", curReassigningParts.size() == 1 ? "" : "s", - curReassigningParts.stream().sorted(ReassignPartitionCommand::compareTopicPartitions).map(Object::toString).collect(Collectors.joining(",")) + curReassigningParts.stream().sorted(ReassignPartitionsCommand::compareTopicPartitions).map(Object::toString).collect(Collectors.joining(",")) ); } else { System.out.println("None of the specified partition reassignments are active."); @@ -1263,8 +1281,8 @@ private static Tuple, Set> cancelAssi private static String formatAsReassignmentJson(Map> partitionsToBeReassigned, Map replicaLogDirAssignment) throws JsonProcessingException { - Map> partitions = new LinkedHashMap<>(); - partitionsToBeReassigned.keySet().stream().sorted(ReassignPartitionCommand::compareTopicPartitions).forEach(tp -> { + List> partitions = new ArrayList<>(); + partitionsToBeReassigned.keySet().stream().sorted(ReassignPartitionsCommand::compareTopicPartitions).forEach(tp -> { List replicas = partitionsToBeReassigned.get(tp); Map data = new LinkedHashMap<>(); @@ -1275,10 +1293,10 @@ private static String formatAsReassignmentJson(Map map(r -> replicaLogDirAssignment.getOrDefault(new TopicPartitionReplica(tp.topic(), tp.partition(), r), ANY_LOG_DIR)). collect(Collectors.toList())); - partitions.put(tp, data); + partitions.add(data); }); - Map results = new HashMap<>(); + Map results = new LinkedHashMap<>(); results.put("version", 1); results.put("partitions", partitions); @@ -1333,18 +1351,18 @@ private static Tuple>>, Map>> partitionAssignment = new ArrayList<>(); Map replicaAssignment = new HashMap<>(); - Optional partitionsSeq = jsonData.asJsonObject().get("topics"); + Optional partitionsSeq = jsonData.asJsonObject().get("partitions"); if (partitionsSeq.isPresent()) { Iterator iter = partitionsSeq.get().asJsonArray().iterator(); while (iter.hasNext()) { JsonObject partitionFields = iter.next().asJsonObject(); String topic = partitionFields.apply("topic").to(STRING); int partition = partitionFields.apply("partition").to(INT); - List newReplicas = partitionFields.apply("replicas").to(null); + List newReplicas = partitionFields.apply("replicas").to(INT_LIST); Optional logDirsOpts = partitionFields.get("log_dirs"); List newLogDirs; if (logDirsOpts.isPresent()) - newLogDirs = logDirsOpts.get().to(null); + newLogDirs = logDirsOpts.get().to(STRING_LIST); else newLogDirs = newReplicas.stream().map(r -> ANY_LOG_DIR).collect(Collectors.toList()); if (newReplicas.size() != newLogDirs.size()) @@ -1370,8 +1388,7 @@ private static Tuple>>, Map alterReplicaLogDirs(Admin adminClient, - Map assignment) throws InterruptedException { + static Set alterReplicaLogDirs(Admin adminClient, + Map assignment) throws InterruptedException { Set results = new HashSet<>(); Map> values = adminClient.alterReplicaLogDirs(assignment).values(); diff --git a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommandArgsTest.java b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommandArgsTest.java new file mode 100644 index 0000000000000..c248796d25f07 --- /dev/null +++ b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommandArgsTest.java @@ -0,0 +1,291 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.tools.reassign; + +import org.apache.kafka.common.utils.Exit; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@Timeout(60) +public class ReassignPartitionsCommandArgsTest { + public static final String MISSING_BOOTSTRAP_SERVER_MSG = "Please specify --bootstrap-server"; + + @BeforeEach + public void setUp() { + Exit.setExitProcedure((statusCode, message) -> { + throw new IllegalArgumentException(message); + }); + } + + @AfterEach + public void tearDown() { + Exit.resetExitProcedure(); + } + + ///// Test valid argument parsing + @Test + public void shouldCorrectlyParseValidMinimumGenerateOptions() { + String[] args = new String[] { + "--bootstrap-server", "localhost:1234", + "--generate", + "--broker-list", "101,102", + "--topics-to-move-json-file", "myfile.json"}; + ReassignPartitionsCommand.validateAndParseArgs(args); + } + + @Test + public void shouldCorrectlyParseValidMinimumExecuteOptions() { + String[] args = new String[] { + "--bootstrap-server", "localhost:1234", + "--execute", + "--reassignment-json-file", "myfile.json"}; + ReassignPartitionsCommand.validateAndParseArgs(args); + } + + @Test + public void shouldCorrectlyParseValidMinimumVerifyOptions() { + String[] args = new String[] { + "--bootstrap-server", "localhost:1234", + "--verify", + "--reassignment-json-file", "myfile.json"}; + ReassignPartitionsCommand.validateAndParseArgs(args); + } + + @Test + public void shouldAllowThrottleOptionOnExecute() { + String[] args = new String[] { + "--bootstrap-server", "localhost:1234", + "--execute", + "--throttle", "100", + "--reassignment-json-file", "myfile.json"}; + ReassignPartitionsCommand.validateAndParseArgs(args); + } + + @Test + public void shouldUseDefaultsIfEnabled() { + String[] args = new String[] { + "--bootstrap-server", "localhost:1234", + "--execute", + "--reassignment-json-file", "myfile.json"}; + ReassignPartitionsCommandOptions opts = ReassignPartitionsCommand.validateAndParseArgs(args); + assertEquals(10000L, opts.options.valueOf(opts.timeoutOpt)); + assertEquals(-1L, opts.options.valueOf(opts.interBrokerThrottleOpt)); + } + + @Test + public void testList() { + String[] args = new String[] { + "--list", + "--bootstrap-server", "localhost:1234"}; + ReassignPartitionsCommand.validateAndParseArgs(args); + } + + @Test + public void testCancelWithPreserveThrottlesOption() { + String[] args = new String[] { + "--cancel", + "--bootstrap-server", "localhost:1234", + "--reassignment-json-file", "myfile.json", + "--preserve-throttles"}; + ReassignPartitionsCommand.validateAndParseArgs(args); + } + + ///// Test handling missing or invalid actions + @Test + public void shouldFailIfNoArgs() { + String[] args = new String[0]; + shouldFailWith(ReassignPartitionsCommand.HELP_TEXT, args); + } + + @Test + public void shouldFailIfBlankArg() { + String[] args = new String[] {" "}; + shouldFailWith("Command must include exactly one action", args); + } + + @Test + public void shouldFailIfMultipleActions() { + String[] args = new String[] { + "--bootstrap-server", "localhost:1234", + "--execute", + "--verify", + "--reassignment-json-file", "myfile.json" + }; + shouldFailWith("Command must include exactly one action", args); + } + + ///// Test --execute + @Test + public void shouldNotAllowExecuteWithTopicsOption() { + String[] args = new String[] { + "--bootstrap-server", "localhost:1234", + "--execute", + "--reassignment-json-file", "myfile.json", + "--topics-to-move-json-file", "myfile.json"}; + shouldFailWith("Option \"[topics-to-move-json-file]\" can't be used with action \"[execute]\"", args); + } + + @Test + public void shouldNotAllowExecuteWithBrokerList() { + String[] args = new String[] { + "--bootstrap-server", "localhost:1234", + "--execute", + "--reassignment-json-file", "myfile.json", + "--broker-list", "101,102" + }; + shouldFailWith("Option \"[broker-list]\" can't be used with action \"[execute]\"", args); + } + + @Test + public void shouldNotAllowExecuteWithoutReassignmentOption() { + String[] args = new String[] { + "--bootstrap-server", "localhost:1234", + "--execute"}; + shouldFailWith("Missing required argument \"[reassignment-json-file]\"", args); + } + + @Test + public void testMissingBootstrapServerArgumentForExecute() { + String[] args = new String[] { + "--execute"}; + shouldFailWith(MISSING_BOOTSTRAP_SERVER_MSG, args); + } + + ///// Test --generate + @Test + public void shouldNotAllowGenerateWithoutBrokersAndTopicsOptions() { + String[] args = new String[] { + "--bootstrap-server", "localhost:1234", + "--generate"}; + shouldFailWith("Missing required argument \"[topics-to-move-json-file]\"", args); + } + + @Test + public void shouldNotAllowGenerateWithoutBrokersOption() { + String[] args = new String[] { + "--bootstrap-server", "localhost:1234", + "--topics-to-move-json-file", "myfile.json", + "--generate"}; + shouldFailWith("Missing required argument \"[broker-list]\"", args); + } + + @Test + public void shouldNotAllowGenerateWithoutTopicsOption() { + String[] args = new String[] { + "--bootstrap-server", "localhost:1234", + "--broker-list", "101,102", + "--generate"}; + shouldFailWith("Missing required argument \"[topics-to-move-json-file]\"", args); + } + + @Test + public void shouldNotAllowGenerateWithThrottleOption() { + String[] args = new String[] { + "--bootstrap-server", "localhost:1234", + "--generate", + "--broker-list", "101,102", + "--throttle", "100", + "--topics-to-move-json-file", "myfile.json"}; + shouldFailWith("Option \"[throttle]\" can't be used with action \"[generate]\"", args); + } + + @Test + public void shouldNotAllowGenerateWithReassignmentOption() { + String[] args = new String[] { + "--bootstrap-server", "localhost:1234", + "--generate", + "--broker-list", "101,102", + "--topics-to-move-json-file", "myfile.json", + "--reassignment-json-file", "myfile.json"}; + shouldFailWith("Option \"[reassignment-json-file]\" can't be used with action \"[generate]\"", args); + } + + @Test + public void shouldPrintHelpTextIfHelpArg() { + String[] args = new String[] {"--help"}; + // note, this is not actually a failed case, it's just we share the same `printUsageAndExit` method when wrong arg received + shouldFailWith(ReassignPartitionsCommand.HELP_TEXT, args); + } + + ///// Test --verify + @Test + public void shouldNotAllowVerifyWithoutReassignmentOption() { + String[] args = new String[] { + "--bootstrap-server", "localhost:1234", + "--verify"}; + shouldFailWith("Missing required argument \"[reassignment-json-file]\"", args); + } + + @Test + public void shouldNotAllowBrokersListWithVerifyOption() { + String[] args = new String[] { + "--bootstrap-server", "localhost:1234", + "--verify", + "--broker-list", "100,101", + "--reassignment-json-file", "myfile.json"}; + shouldFailWith("Option \"[broker-list]\" can't be used with action \"[verify]\"", args); + } + + @Test + public void shouldNotAllowThrottleWithVerifyOption() { + String[] args = new String[] { + "--bootstrap-server", "localhost:1234", + "--verify", + "--throttle", "100", + "--reassignment-json-file", "myfile.json"}; + shouldFailWith("Option \"[throttle]\" can't be used with action \"[verify]\"", args); + } + + @Test + public void shouldNotAllowTopicsOptionWithVerify() { + String[] args = new String[] { + "--bootstrap-server", "localhost:1234", + "--verify", + "--reassignment-json-file", "myfile.json", + "--topics-to-move-json-file", "myfile.json"}; + shouldFailWith("Option \"[topics-to-move-json-file]\" can't be used with action \"[verify]\"", args); + } + + private void shouldFailWith(String msg, String[] args) { + Throwable e = assertThrows(Exception.class, () -> ReassignPartitionsCommand.validateAndParseArgs(args), + () -> "Should have failed with [" + msg + "] but no failure occurred."); + assertTrue(e.getMessage().startsWith(msg), "Expected exception with message:\n[" + msg + "]\nbut was\n[" + e.getMessage() + "]"); + } + + ///// Test --cancel + @Test + public void shouldNotAllowCancelWithoutBootstrapServerOption() { + String[] args = new String[] { + "--cancel"}; + shouldFailWith(MISSING_BOOTSTRAP_SERVER_MSG, args); + } + + @Test + public void shouldNotAllowCancelWithoutReassignmentJsonFile() { + String[] args = new String[] { + "--cancel", + "--bootstrap-server", "localhost:1234", + "--preserve-throttles"}; + shouldFailWith("Missing required argument \"[reassignment-json-file]\"", args); + } +} diff --git a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java new file mode 100644 index 0000000000000..05f241e8acc2b --- /dev/null +++ b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java @@ -0,0 +1,747 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.tools.reassign; + +import org.apache.kafka.admin.BrokerMetadata; +import org.apache.kafka.clients.admin.Config; +import org.apache.kafka.clients.admin.MockAdminClient; +import org.apache.kafka.clients.admin.PartitionReassignment; +import org.apache.kafka.common.Node; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.TopicPartitionInfo; +import org.apache.kafka.common.TopicPartitionReplica; +import org.apache.kafka.common.config.ConfigResource; +import org.apache.kafka.common.errors.InvalidReplicationFactorException; +import org.apache.kafka.common.errors.UnknownTopicOrPartitionException; +import org.apache.kafka.common.utils.Exit; +import org.apache.kafka.common.utils.Time; +import org.apache.kafka.server.common.AdminCommandFailedException; +import org.apache.kafka.server.common.AdminOperationException; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.apache.kafka.tools.reassign.ReassignPartitionsCommand.BROKER_LEVEL_FOLLOWER_THROTTLE; +import static org.apache.kafka.tools.reassign.ReassignPartitionsCommand.BROKER_LEVEL_LEADER_THROTTLE; +import static org.apache.kafka.tools.reassign.ReassignPartitionsCommand.BROKER_LEVEL_LOG_DIR_THROTTLE; +import static org.apache.kafka.tools.reassign.ReassignPartitionsCommand.TOPIC_LEVEL_FOLLOWER_THROTTLE; +import static org.apache.kafka.tools.reassign.ReassignPartitionsCommand.TOPIC_LEVEL_LEADER_THROTTLE; +import static org.apache.kafka.tools.reassign.ReassignPartitionsCommand.alterPartitionReassignments; +import static org.apache.kafka.tools.reassign.ReassignPartitionsCommand.alterReplicaLogDirs; +import static org.apache.kafka.tools.reassign.ReassignPartitionsCommand.calculateFollowerThrottles; +import static org.apache.kafka.tools.reassign.ReassignPartitionsCommand.calculateLeaderThrottles; +import static org.apache.kafka.tools.reassign.ReassignPartitionsCommand.calculateMovingBrokers; +import static org.apache.kafka.tools.reassign.ReassignPartitionsCommand.calculateProposedMoveMap; +import static org.apache.kafka.tools.reassign.ReassignPartitionsCommand.calculateReassigningBrokers; +import static org.apache.kafka.tools.reassign.ReassignPartitionsCommand.cancelPartitionReassignments; +import static org.apache.kafka.tools.reassign.ReassignPartitionsCommand.compareTopicPartitionReplicas; +import static org.apache.kafka.tools.reassign.ReassignPartitionsCommand.compareTopicPartitions; +import static org.apache.kafka.tools.reassign.ReassignPartitionsCommand.curReassignmentsToString; +import static org.apache.kafka.tools.reassign.ReassignPartitionsCommand.currentPartitionReplicaAssignmentToString; +import static org.apache.kafka.tools.reassign.ReassignPartitionsCommand.executeAssignment; +import static org.apache.kafka.tools.reassign.ReassignPartitionsCommand.findLogDirMoveStates; +import static org.apache.kafka.tools.reassign.ReassignPartitionsCommand.findPartitionReassignmentStates; +import static org.apache.kafka.tools.reassign.ReassignPartitionsCommand.generateAssignment; +import static org.apache.kafka.tools.reassign.ReassignPartitionsCommand.getBrokerMetadata; +import static org.apache.kafka.tools.reassign.ReassignPartitionsCommand.getReplicaAssignmentForPartitions; +import static org.apache.kafka.tools.reassign.ReassignPartitionsCommand.getReplicaAssignmentForTopics; +import static org.apache.kafka.tools.reassign.ReassignPartitionsCommand.modifyInterBrokerThrottle; +import static org.apache.kafka.tools.reassign.ReassignPartitionsCommand.modifyLogDirThrottle; +import static org.apache.kafka.tools.reassign.ReassignPartitionsCommand.modifyTopicThrottles; +import static org.apache.kafka.tools.reassign.ReassignPartitionsCommand.parseExecuteAssignmentArgs; +import static org.apache.kafka.tools.reassign.ReassignPartitionsCommand.parseGenerateAssignmentArgs; +import static org.apache.kafka.tools.reassign.ReassignPartitionsCommand.partitionReassignmentStatesToString; +import static org.apache.kafka.tools.reassign.ReassignPartitionsCommand.replicaMoveStatesToString; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@Timeout(60) +public class ReassignPartitionsUnitTest { + @BeforeEach + public void setUp() { + Exit.setExitProcedure((statusCode, message) -> { + throw new IllegalArgumentException(message); + }); + } + + @AfterEach + public void tearDown() { + Exit.resetExitProcedure(); + } + + @Test + public void testCompareTopicPartitions() { + assertTrue(compareTopicPartitions(new TopicPartition("abc", 0), + new TopicPartition("abc", 1)) < 0); + assertTrue(compareTopicPartitions(new TopicPartition("def", 0), + new TopicPartition("abc", 1)) > 0); + } + + @Test + public void testCompareTopicPartitionReplicas() { + assertTrue(compareTopicPartitionReplicas(new TopicPartitionReplica("def", 0, 0), + new TopicPartitionReplica("abc", 0, 1)) < 0); + assertTrue(compareTopicPartitionReplicas(new TopicPartitionReplica("def", 0, 0), + new TopicPartitionReplica("cde", 0, 0)) > 0); + } + + @Test + public void testPartitionReassignStatesToString() { + Map states = new HashMap<>(); + + states.put(new TopicPartition("foo", 0), new PartitionReassignmentState(Arrays.asList(1, 2, 3), Arrays.asList(1, 2, 3), true)); + states.put(new TopicPartition("foo", 1), new PartitionReassignmentState(Arrays.asList(1, 2, 3), Arrays.asList(1, 2, 4), false)); + states.put(new TopicPartition("bar", 0), new PartitionReassignmentState(Arrays.asList(1, 2, 3), Arrays.asList(1, 2, 4), false)); + + assertEquals(String.join(System.lineSeparator(), Arrays.asList( + "Status of partition reassignment:", + "Reassignment of partition bar-0 is still in progress.", + "Reassignment of partition foo-0 is completed.", + "Reassignment of partition foo-1 is still in progress.")), + partitionReassignmentStatesToString(states)); + } + + private void addTopics(MockAdminClient adminClient) { + List b = adminClient.brokers(); + adminClient.addTopic(false, "foo", Arrays.asList( + new TopicPartitionInfo(0, b.get(0), + Arrays.asList(b.get(0), b.get(1), b.get(2)), + Arrays.asList(b.get(0), b.get(1))), + new TopicPartitionInfo(1, b.get(1), + Arrays.asList(b.get(1), b.get(2), b.get(3)), + Arrays.asList(b.get(1), b.get(2), b.get(3))) + ), Collections.emptyMap()); + adminClient.addTopic(false, "bar", Collections.singletonList( + new TopicPartitionInfo(0, b.get(2), + Arrays.asList(b.get(2), b.get(3), b.get(0)), + Arrays.asList(b.get(2), b.get(3), b.get(0))) + ), Collections.emptyMap()); + } + + @Test + public void testFindPartitionReassignmentStates() throws Exception { + try (MockAdminClient adminClient = new MockAdminClient.Builder().numBrokers(4).build()) { + addTopics(adminClient); + // Create a reassignment and test findPartitionReassignmentStates. + Map> reassignments = new HashMap<>(); + + reassignments.put(new TopicPartition("foo", 0), Arrays.asList(0, 1, 3)); + reassignments.put(new TopicPartition("quux", 0), Arrays.asList(1, 2, 3)); + + Map> reassignmentResult = alterPartitionReassignments(adminClient, reassignments) + .entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().getClass())); + + assertEquals(Collections.singletonMap(new TopicPartition("quux", 0), UnknownTopicOrPartitionException.class), reassignmentResult); + + Map partitionReassignmentStates = new HashMap<>(); + + partitionReassignmentStates.put(new TopicPartition("foo", 0), new PartitionReassignmentState(Arrays.asList(0, 1, 2), Arrays.asList(0, 1, 3), false)); + partitionReassignmentStates.put(new TopicPartition("foo", 1), new PartitionReassignmentState(Arrays.asList(1, 2, 3), Arrays.asList(1, 2, 3), true)); + + assertEquals( + new Tuple<>(partitionReassignmentStates, true), + findPartitionReassignmentStates(adminClient, Arrays.asList( + new Tuple<>(new TopicPartition("foo", 0), Arrays.asList(0, 1, 3)), + new Tuple<>(new TopicPartition("foo", 1), Arrays.asList(1, 2, 3)) + ))); + + // Cancel the reassignment and test findPartitionReassignmentStates again. + Map> cancelResult = cancelPartitionReassignments(adminClient, new HashSet<>(Arrays.asList(new TopicPartition("foo", 0), new TopicPartition("quux", 2)))) + .entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().getClass())); + + assertEquals(Collections.singletonMap(new TopicPartition("quux", 2), UnknownTopicOrPartitionException.class), cancelResult); + + partitionReassignmentStates.clear(); + + partitionReassignmentStates.put(new TopicPartition("foo", 0), new PartitionReassignmentState(Arrays.asList(0, 1, 2), Arrays.asList(0, 1, 3), true)); + partitionReassignmentStates.put(new TopicPartition("foo", 1), new PartitionReassignmentState(Arrays.asList(1, 2, 3), Arrays.asList(1, 2, 3), true)); + + assertEquals( + new Tuple<>(partitionReassignmentStates, false), + findPartitionReassignmentStates(adminClient, Arrays.asList( + new Tuple<>(new TopicPartition("foo", 0), Arrays.asList(0, 1, 3)), + new Tuple<>(new TopicPartition("foo", 1), Arrays.asList(1, 2, 3)) + ))); + } + } + + @Test + public void testFindLogDirMoveStates() throws Exception { + try (MockAdminClient adminClient = new MockAdminClient.Builder(). + numBrokers(4). + brokerLogDirs(Arrays.asList( + Arrays.asList("/tmp/kafka-logs0", "/tmp/kafka-logs1"), + Arrays.asList("/tmp/kafka-logs0", "/tmp/kafka-logs1"), + Arrays.asList("/tmp/kafka-logs0", "/tmp/kafka-logs1"), + Arrays.asList("/tmp/kafka-logs0", null))). + build()) { + + addTopics(adminClient); + List b = adminClient.brokers(); + adminClient.addTopic(false, "quux", Collections.singletonList( + new TopicPartitionInfo(0, b.get(2), + Arrays.asList(b.get(1), b.get(2), b.get(3)), + Arrays.asList(b.get(1), b.get(2), b.get(3)))), + Collections.emptyMap()); + + Map replicaAssignment = new HashMap<>(); + + replicaAssignment.put(new TopicPartitionReplica("foo", 0, 0), "/tmp/kafka-logs1"); + replicaAssignment.put(new TopicPartitionReplica("quux", 0, 0), "/tmp/kafka-logs1"); + + adminClient.alterReplicaLogDirs(replicaAssignment).all().get(); + + Map expLogDirMoveStates = new HashMap<>(); + + expLogDirMoveStates.put(new TopicPartitionReplica("bar", 0, 0), new CompletedMoveState("/tmp/kafka-logs0")); + expLogDirMoveStates.put(new TopicPartitionReplica("foo", 0, 0), new ActiveMoveState("/tmp/kafka-logs0", "/tmp/kafka-logs1", "/tmp/kafka-logs1")); + expLogDirMoveStates.put(new TopicPartitionReplica("foo", 1, 0), new CancelledMoveState("/tmp/kafka-logs0", "/tmp/kafka-logs1")); + expLogDirMoveStates.put(new TopicPartitionReplica("quux", 1, 0), new MissingLogDirMoveState("/tmp/kafka-logs1")); + expLogDirMoveStates.put(new TopicPartitionReplica("quuz", 0, 0), new MissingReplicaMoveState("/tmp/kafka-logs0")); + + Map targetMoves = new HashMap<>(); + + targetMoves.put(new TopicPartitionReplica("bar", 0, 0), "/tmp/kafka-logs0"); + targetMoves.put(new TopicPartitionReplica("foo", 0, 0), "/tmp/kafka-logs1"); + targetMoves.put(new TopicPartitionReplica("foo", 1, 0), "/tmp/kafka-logs1"); + targetMoves.put(new TopicPartitionReplica("quux", 1, 0), "/tmp/kafka-logs1"); + targetMoves.put(new TopicPartitionReplica("quuz", 0, 0), "/tmp/kafka-logs0"); + + assertEquals(expLogDirMoveStates, findLogDirMoveStates(adminClient, targetMoves)); + } + } + + @Test + public void testReplicaMoveStatesToString() { + Map states = new HashMap<>(); + + states.put(new TopicPartitionReplica("bar", 0, 0), new CompletedMoveState("/tmp/kafka-logs0")); + states.put(new TopicPartitionReplica("foo", 0, 0), new ActiveMoveState("/tmp/kafka-logs0", + "/tmp/kafka-logs1", "/tmp/kafka-logs1")); + states.put(new TopicPartitionReplica("foo", 1, 0), new CancelledMoveState("/tmp/kafka-logs0", + "/tmp/kafka-logs1")); + states.put(new TopicPartitionReplica("quux", 0, 0), new MissingReplicaMoveState("/tmp/kafka-logs1")); + states.put(new TopicPartitionReplica("quux", 1, 1), new ActiveMoveState("/tmp/kafka-logs0", + "/tmp/kafka-logs1", "/tmp/kafka-logs2")); + states.put(new TopicPartitionReplica("quux", 2, 1), new MissingLogDirMoveState("/tmp/kafka-logs1")); + + assertEquals(String.join(System.lineSeparator(), Arrays.asList( + "Reassignment of replica bar-0-0 completed successfully.", + "Reassignment of replica foo-0-0 is still in progress.", + "Partition foo-1 on broker 0 is not being moved from log dir /tmp/kafka-logs0 to /tmp/kafka-logs1.", + "Partition quux-0 cannot be found in any live log directory on broker 0.", + "Partition quux-1 on broker 1 is being moved to log dir /tmp/kafka-logs2 instead of /tmp/kafka-logs1.", + "Partition quux-2 is not found in any live log dir on broker 1. " + + "There is likely an offline log directory on the broker.")), + replicaMoveStatesToString(states)); + } + + @Test + public void testGetReplicaAssignments() throws Exception { + try (MockAdminClient adminClient = new MockAdminClient.Builder().numBrokers(4).build()) { + addTopics(adminClient); + + Map> assignments = new HashMap<>(); + + assignments.put(new TopicPartition("foo", 0), Arrays.asList(0, 1, 2)); + assignments.put(new TopicPartition("foo", 1), Arrays.asList(1, 2, 3)); + + assertEquals(assignments, + getReplicaAssignmentForTopics(adminClient, Collections.singletonList("foo"))); + + assignments.clear(); + + assignments.put(new TopicPartition("foo", 0), Arrays.asList(0, 1, 2)); + assignments.put(new TopicPartition("bar", 0), Arrays.asList(2, 3, 0)); + + assertEquals(assignments, + getReplicaAssignmentForPartitions(adminClient, new HashSet<>(Arrays.asList( + new TopicPartition("foo", 0), new TopicPartition("bar", 0))))); + } + } + + @Test + public void testGetBrokerRackInformation() throws Exception { + try (MockAdminClient adminClient = new MockAdminClient.Builder(). + brokers(Arrays.asList(new Node(0, "localhost", 9092, "rack0"), + new Node(1, "localhost", 9093, "rack1"), + new Node(2, "localhost", 9094, null))). + build()) { + + assertEquals(Arrays.asList( + new BrokerMetadata(0, Optional.of("rack0")), + new BrokerMetadata(1, Optional.of("rack1")) + ), getBrokerMetadata(adminClient, Arrays.asList(0, 1), true)); + assertEquals(Arrays.asList( + new BrokerMetadata(0, Optional.empty()), + new BrokerMetadata(1, Optional.empty()) + ), getBrokerMetadata(adminClient, Arrays.asList(0, 1), false)); + assertStartsWith("Not all brokers have rack information", + assertThrows(AdminOperationException.class, + () -> getBrokerMetadata(adminClient, Arrays.asList(1, 2), true)).getMessage()); + assertEquals(Arrays.asList( + new BrokerMetadata(1, Optional.empty()), + new BrokerMetadata(2, Optional.empty()) + ), getBrokerMetadata(adminClient, Arrays.asList(1, 2), false)); + } + } + + @Test + public void testParseGenerateAssignmentArgs() throws Exception { + assertStartsWith("Broker list contains duplicate entries", + assertThrows(AdminCommandFailedException.class, () -> parseGenerateAssignmentArgs( + "{\"topics\": [{\"topic\": \"foo\"}], \"version\":1}", "1,1,2"), + "Expected to detect duplicate broker list entries").getMessage()); + assertStartsWith("Broker list contains duplicate entries", + assertThrows(AdminCommandFailedException.class, () -> parseGenerateAssignmentArgs( + "{\"topics\": [{\"topic\": \"foo\"}], \"version\":1}", "5,2,3,4,5"), + "Expected to detect duplicate broker list entries").getMessage()); + assertEquals(new Tuple<>(Arrays.asList(5, 2, 3, 4), Collections.singletonList("foo")), + parseGenerateAssignmentArgs("{\"topics\": [{\"topic\": \"foo\"}], \"version\":1}", + "5,2,3,4")); + assertStartsWith("List of topics to reassign contains duplicate entries", + assertThrows(AdminCommandFailedException.class, () -> parseGenerateAssignmentArgs( + "{\"topics\": [{\"topic\": \"foo\"},{\"topic\": \"foo\"}], \"version\":1}", "5,2,3,4"), + "Expected to detect duplicate topic entries").getMessage()); + assertEquals(new Tuple<>(Arrays.asList(5, 3, 4), Arrays.asList("foo", "bar")), + parseGenerateAssignmentArgs( + "{\"topics\": [{\"topic\": \"foo\"},{\"topic\": \"bar\"}], \"version\":1}", + "5,3,4")); + } + + @Test + public void testGenerateAssignmentFailsWithoutEnoughReplicas() { + try (MockAdminClient adminClient = new MockAdminClient.Builder().numBrokers(4).build()) { + addTopics(adminClient); + assertStartsWith("Replication factor: 3 larger than available brokers: 2", + assertThrows(InvalidReplicationFactorException.class, + () -> generateAssignment(adminClient, "{\"topics\":[{\"topic\":\"foo\"},{\"topic\":\"bar\"}]}", "0,1", false), + "Expected generateAssignment to fail").getMessage()); + } + } + + @Test + public void testGenerateAssignmentWithInvalidPartitionsFails() { + try (MockAdminClient adminClient = new MockAdminClient.Builder().numBrokers(5).build()) { + addTopics(adminClient); + assertStartsWith("Topic quux not found", + assertThrows(ExecutionException.class, + () -> generateAssignment(adminClient, "{\"topics\":[{\"topic\":\"foo\"},{\"topic\":\"quux\"}]}", "0,1", false), + "Expected generateAssignment to fail").getCause().getMessage()); + } + } + + @Test + public void testGenerateAssignmentWithInconsistentRacks() throws Exception { + try (MockAdminClient adminClient = new MockAdminClient.Builder(). + brokers(Arrays.asList( + new Node(0, "localhost", 9092, "rack0"), + new Node(1, "localhost", 9093, "rack0"), + new Node(2, "localhost", 9094, null), + new Node(3, "localhost", 9095, "rack1"), + new Node(4, "localhost", 9096, "rack1"), + new Node(5, "localhost", 9097, "rack2"))). + build()) { + + addTopics(adminClient); + assertStartsWith("Not all brokers have rack information.", + assertThrows(AdminOperationException.class, + () -> generateAssignment(adminClient, "{\"topics\":[{\"topic\":\"foo\"}]}", "0,1,2,3", true), + "Expected generateAssignment to fail").getMessage()); + // It should succeed when --disable-rack-aware is used. + Tuple>, Map>> proposedCurrent = generateAssignment(adminClient, + "{\"topics\":[{\"topic\":\"foo\"}]}", "0,1,2,3", false); + + Map> expCurrent = new HashMap<>(); + + expCurrent.put(new TopicPartition("foo", 0), Arrays.asList(0, 1, 2)); + expCurrent.put(new TopicPartition("foo", 1), Arrays.asList(1, 2, 3)); + + assertEquals(expCurrent, proposedCurrent.v2); + } + } + + @Test + public void testGenerateAssignmentWithFewerBrokers() throws Exception { + try (MockAdminClient adminClient = new MockAdminClient.Builder().numBrokers(4).build()) { + addTopics(adminClient); + List goalBrokers = Arrays.asList(0, 1, 3); + + Tuple>, Map>> proposedCurrent = + generateAssignment(adminClient, + "{\"topics\":[{\"topic\":\"foo\"},{\"topic\":\"bar\"}]}", + goalBrokers.stream().map(Object::toString).collect(Collectors.joining(",")), false); + + Map> expCurrent = new HashMap<>(); + + expCurrent.put(new TopicPartition("foo", 0), Arrays.asList(0, 1, 2)); + expCurrent.put(new TopicPartition("foo", 1), Arrays.asList(1, 2, 3)); + expCurrent.put(new TopicPartition("bar", 0), Arrays.asList(2, 3, 0)); + + assertEquals(expCurrent, proposedCurrent.v2); + + // The proposed assignment should only span the provided brokers + proposedCurrent.v1.values().forEach(replicas -> assertTrue(goalBrokers.containsAll(replicas), + "Proposed assignment " + proposedCurrent.v1 + " puts replicas on brokers other than " + goalBrokers)); + } + } + + @Test + public void testCurrentPartitionReplicaAssignmentToString() throws Exception { + Map> proposedParts = new HashMap<>(); + + proposedParts.put(new TopicPartition("foo", 1), Arrays.asList(1, 2, 3)); + proposedParts.put(new TopicPartition("bar", 0), Arrays.asList(7, 8, 9)); + + Map> currentParts = new HashMap<>(); + + currentParts.put(new TopicPartition("foo", 0), Arrays.asList(1, 2, 3)); + currentParts.put(new TopicPartition("foo", 1), Arrays.asList(4, 5, 6)); + currentParts.put(new TopicPartition("bar", 0), Arrays.asList(7, 8)); + currentParts.put(new TopicPartition("baz", 0), Arrays.asList(10, 11, 12)); + + assertEquals(String.join(System.lineSeparator(), Arrays.asList( + "Current partition replica assignment", + "", + "{\"version\":1,\"partitions\":" + + "[{\"topic\":\"bar\",\"partition\":0,\"replicas\":[7,8],\"log_dirs\":[\"any\",\"any\"]}," + + "{\"topic\":\"foo\",\"partition\":1,\"replicas\":[4,5,6],\"log_dirs\":[\"any\",\"any\",\"any\"]}]" + + "}", + "", + "Save this to use as the --reassignment-json-file option during rollback" + )), + currentPartitionReplicaAssignmentToString(proposedParts, currentParts) + ); + } + + @Test + public void testMoveMap() { + // overwrite foo-0 with different reassignments + // keep old reassignments of foo-1 + // overwrite foo-2 with same reassignments + // overwrite foo-3 with new reassignments without overlap of old reassignments + // overwrite foo-4 with a subset of old reassignments + // overwrite foo-5 with a superset of old reassignments + // add new reassignments to bar-0 + Map currentReassignments = new HashMap<>(); + + currentReassignments.put(new TopicPartition("foo", 0), new PartitionReassignment( + Arrays.asList(1, 2, 3, 4), Collections.singletonList(4), Collections.singletonList(3))); + currentReassignments.put(new TopicPartition("foo", 1), new PartitionReassignment( + Arrays.asList(4, 5, 6, 7, 8), Arrays.asList(7, 8), Arrays.asList(4, 5))); + currentReassignments.put(new TopicPartition("foo", 2), new PartitionReassignment( + Arrays.asList(1, 2, 3, 4), Arrays.asList(3, 4), Arrays.asList(1, 2))); + currentReassignments.put(new TopicPartition("foo", 3), new PartitionReassignment( + Arrays.asList(1, 2, 3, 4), Arrays.asList(3, 4), Arrays.asList(1, 2))); + currentReassignments.put(new TopicPartition("foo", 4), new PartitionReassignment( + Arrays.asList(1, 2, 3, 4), Arrays.asList(3, 4), Arrays.asList(1, 2))); + currentReassignments.put(new TopicPartition("foo", 5), new PartitionReassignment( + Arrays.asList(1, 2, 3, 4), Arrays.asList(3, 4), Arrays.asList(1, 2))); + + Map> proposedParts = new HashMap<>(); + + proposedParts.put(new TopicPartition("foo", 0), Arrays.asList(1, 2, 5)); + proposedParts.put(new TopicPartition("foo", 2), Arrays.asList(3, 4)); + proposedParts.put(new TopicPartition("foo", 3), Arrays.asList(5, 6)); + proposedParts.put(new TopicPartition("foo", 4), Collections.singletonList(3)); + proposedParts.put(new TopicPartition("foo", 5), Arrays.asList(3, 4, 5, 6)); + proposedParts.put(new TopicPartition("bar", 0), Arrays.asList(1, 2, 3)); + + Map> currentParts = new HashMap<>(); + + currentParts.put(new TopicPartition("foo", 0), Arrays.asList(1, 2, 3, 4)); + currentParts.put(new TopicPartition("foo", 1), Arrays.asList(4, 5, 6, 7, 8)); + currentParts.put(new TopicPartition("foo", 2), Arrays.asList(1, 2, 3, 4)); + currentParts.put(new TopicPartition("foo", 3), Arrays.asList(1, 2, 3, 4)); + currentParts.put(new TopicPartition("foo", 4), Arrays.asList(1, 2, 3, 4)); + currentParts.put(new TopicPartition("foo", 5), Arrays.asList(1, 2, 3, 4)); + currentParts.put(new TopicPartition("bar", 0), Arrays.asList(2, 3, 4)); + currentParts.put(new TopicPartition("baz", 0), Arrays.asList(1, 2, 3)); + + Map> moveMap = calculateProposedMoveMap(currentReassignments, proposedParts, currentParts); + + Map> expMoveMap = new HashMap<>(); + + Map fooMoves = new HashMap<>(); + + fooMoves.put(0, new PartitionMove(new HashSet<>(Arrays.asList(1, 2, 3)), new HashSet<>(Collections.singletonList(5)))); + fooMoves.put(1, new PartitionMove(new HashSet<>(Arrays.asList(4, 5, 6)), new HashSet<>(Arrays.asList(7, 8)))); + fooMoves.put(2, new PartitionMove(new HashSet<>(Arrays.asList(1, 2)), new HashSet<>(Arrays.asList(3, 4)))); + fooMoves.put(3, new PartitionMove(new HashSet<>(Arrays.asList(1, 2)), new HashSet<>(Arrays.asList(5, 6)))); + fooMoves.put(4, new PartitionMove(new HashSet<>(Arrays.asList(1, 2)), new HashSet<>(Collections.singletonList(3)))); + fooMoves.put(5, new PartitionMove(new HashSet<>(Arrays.asList(1, 2)), new HashSet<>(Arrays.asList(3, 4, 5, 6)))); + + expMoveMap.put("foo", fooMoves); + expMoveMap.put("bar", Collections.singletonMap(0, new PartitionMove(new HashSet<>(Arrays.asList(2, 3, 4)), new HashSet<>(Collections.singletonList(1))))); + + assertEquals(expMoveMap, moveMap); + + Map expLeaderThrottle = new HashMap<>(); + + expLeaderThrottle.put("foo", "0:1,0:2,0:3,1:4,1:5,1:6,2:1,2:2,3:1,3:2,4:1,4:2,5:1,5:2"); + expLeaderThrottle.put("bar", "0:2,0:3,0:4"); + + assertEquals(expLeaderThrottle, calculateLeaderThrottles(moveMap)); + + Map expFollowerThrottle = new HashMap<>(); + + expFollowerThrottle.put("foo", "0:5,1:7,1:8,2:3,2:4,3:5,3:6,4:3,5:3,5:4,5:5,5:6"); + expFollowerThrottle.put("bar", "0:1"); + + assertEquals(expFollowerThrottle, calculateFollowerThrottles(moveMap)); + + assertEquals(new HashSet<>(Arrays.asList(1, 2, 3, 4, 5, 6, 7, 8)), calculateReassigningBrokers(moveMap)); + + assertEquals(new HashSet<>(Arrays.asList(0, 2)), calculateMovingBrokers(new HashSet<>(Arrays.asList( + new TopicPartitionReplica("quux", 0, 0), + new TopicPartitionReplica("quux", 1, 2))))); + } + + @Test + public void testParseExecuteAssignmentArgs() throws Exception { + assertStartsWith("Partition reassignment list cannot be empty", + assertThrows(AdminCommandFailedException.class, + () -> parseExecuteAssignmentArgs("{\"version\":1,\"partitions\":[]}"), + "Expected to detect empty partition reassignment list").getMessage()); + assertStartsWith("Partition reassignment contains duplicate topic partitions", + assertThrows(AdminCommandFailedException.class, () -> parseExecuteAssignmentArgs( + "{\"version\":1,\"partitions\":" + + "[{\"topic\":\"foo\",\"partition\":0,\"replicas\":[0,1],\"log_dirs\":[\"any\",\"any\"]}," + + "{\"topic\":\"foo\",\"partition\":0,\"replicas\":[2,3,4],\"log_dirs\":[\"any\",\"any\",\"any\"]}" + + "]}"), "Expected to detect a partition list with duplicate entries").getMessage()); + assertStartsWith("Partition reassignment contains duplicate topic partitions", + assertThrows(AdminCommandFailedException.class, () -> parseExecuteAssignmentArgs( + "{\"version\":1,\"partitions\":" + + "[{\"topic\":\"foo\",\"partition\":0,\"replicas\":[0,1],\"log_dirs\":[\"/abc\",\"/def\"]}," + + "{\"topic\":\"foo\",\"partition\":0,\"replicas\":[2,3],\"log_dirs\":[\"/abc\",\"/def\"]}" + + "]}"), "Expected to detect a partition replica list with duplicate entries").getMessage()); + assertStartsWith("Partition replica lists may not contain duplicate entries", + assertThrows(AdminCommandFailedException.class, () -> parseExecuteAssignmentArgs( + "{\"version\":1,\"partitions\":" + + "[{\"topic\":\"foo\",\"partition\":0,\"replicas\":[0,0],\"log_dirs\":[\"/abc\",\"/def\"]}," + + "{\"topic\":\"foo\",\"partition\":1,\"replicas\":[2,3],\"log_dirs\":[\"/abc\",\"/def\"]}" + + "]}"), "Expected to detect a partition replica list with duplicate entries").getMessage()); + + Map> partitionsToBeReassigned = new HashMap<>(); + + partitionsToBeReassigned.put(new TopicPartition("foo", 0), Arrays.asList(1, 2, 3)); + partitionsToBeReassigned.put(new TopicPartition("foo", 1), Arrays.asList(3, 4, 5)); + + assertEquals(new Tuple<>(partitionsToBeReassigned, Collections.emptyMap()), + parseExecuteAssignmentArgs( + "{\"version\":1,\"partitions\":" + + "[{\"topic\":\"foo\",\"partition\":0,\"replicas\":[1,2,3],\"log_dirs\":[\"any\",\"any\",\"any\"]}," + + "{\"topic\":\"foo\",\"partition\":1,\"replicas\":[3,4,5],\"log_dirs\":[\"any\",\"any\",\"any\"]}" + + "]}")); + + Map replicaAssignment = new HashMap<>(); + + replicaAssignment.put(new TopicPartitionReplica("foo", 0, 1), "/tmp/a"); + replicaAssignment.put(new TopicPartitionReplica("foo", 0, 2), "/tmp/b"); + replicaAssignment.put(new TopicPartitionReplica("foo", 0, 3), "/tmp/c"); + + assertEquals( + new Tuple<>(Collections.singletonMap(new TopicPartition("foo", 0), Arrays.asList(1, 2, 3)), replicaAssignment), + parseExecuteAssignmentArgs( + "{\"version\":1,\"partitions\":" + + "[{\"topic\":\"foo\",\"partition\":0,\"replicas\":[1,2,3],\"log_dirs\":[\"/tmp/a\",\"/tmp/b\",\"/tmp/c\"]}" + + "]}")); + } + + @Test + public void testExecuteWithInvalidPartitionsFails() { + try (MockAdminClient adminClient = new MockAdminClient.Builder().numBrokers(5).build()) { + addTopics(adminClient); + assertStartsWith("Topic quux not found", + assertThrows(ExecutionException.class, () -> executeAssignment(adminClient, false, + "{\"version\":1,\"partitions\":" + + "[{\"topic\":\"foo\",\"partition\":0,\"replicas\":[0,1],\"log_dirs\":[\"any\",\"any\"]}," + + "{\"topic\":\"quux\",\"partition\":0,\"replicas\":[2,3,4],\"log_dirs\":[\"any\",\"any\",\"any\"]}" + + "]}", -1L, -1L, 10000L, Time.SYSTEM), "Expected reassignment with non-existent topic to fail").getCause().getMessage()); + } + } + + @Test + public void testExecuteWithInvalidBrokerIdFails() { + try (MockAdminClient adminClient = new MockAdminClient.Builder().numBrokers(4).build()) { + addTopics(adminClient); + assertStartsWith("Unknown broker id 4", + assertThrows(AdminCommandFailedException.class, () -> executeAssignment(adminClient, false, + "{\"version\":1,\"partitions\":" + + "[{\"topic\":\"foo\",\"partition\":0,\"replicas\":[0,1],\"log_dirs\":[\"any\",\"any\"]}," + + "{\"topic\":\"foo\",\"partition\":1,\"replicas\":[2,3,4],\"log_dirs\":[\"any\",\"any\",\"any\"]}" + + "]}", -1L, -1L, 10000L, Time.SYSTEM), "Expected reassignment with non-existent broker id to fail").getMessage()); + } + } + + @Test + public void testModifyBrokerInterBrokerThrottle() throws Exception { + try (MockAdminClient adminClient = new MockAdminClient.Builder().numBrokers(4).build()) { + modifyInterBrokerThrottle(adminClient, new HashSet<>(Arrays.asList(0, 1, 2)), 1000); + modifyInterBrokerThrottle(adminClient, new HashSet<>(Arrays.asList(0, 3)), 100); + List brokers = new ArrayList<>(); + for (int i = 0; i < 4; i++) + brokers.add(new ConfigResource(ConfigResource.Type.BROKER, Integer.toString(i))); + Map results = adminClient.describeConfigs(brokers).all().get(); + verifyBrokerThrottleResults(results.get(brokers.get(0)), 100, -1); + verifyBrokerThrottleResults(results.get(brokers.get(1)), 1000, -1); + verifyBrokerThrottleResults(results.get(brokers.get(2)), 1000, -1); + verifyBrokerThrottleResults(results.get(brokers.get(3)), 100, -1); + } + } + + @Test + public void testModifyLogDirThrottle() throws Exception { + try (MockAdminClient adminClient = new MockAdminClient.Builder().numBrokers(4).build()) { + modifyLogDirThrottle(adminClient, new HashSet<>(Arrays.asList(0, 1, 2)), 2000); + modifyLogDirThrottle(adminClient, new HashSet<>(Arrays.asList(0, 3)), -1); + + List brokers = new ArrayList<>(); + for (int i = 0; i < 4; i++) + brokers.add(new ConfigResource(ConfigResource.Type.BROKER, Integer.toString(i))); + + Map results = adminClient.describeConfigs(brokers).all().get(); + + verifyBrokerThrottleResults(results.get(brokers.get(0)), -1, 2000); + verifyBrokerThrottleResults(results.get(brokers.get(1)), -1, 2000); + verifyBrokerThrottleResults(results.get(brokers.get(2)), -1, 2000); + verifyBrokerThrottleResults(results.get(brokers.get(3)), -1, -1); + } + } + + @Test + public void testCurReassignmentsToString() throws Exception { + try (MockAdminClient adminClient = new MockAdminClient.Builder().numBrokers(4).build()) { + addTopics(adminClient); + assertEquals("No partition reassignments found.", curReassignmentsToString(adminClient)); + + Map> reassignments = new HashMap<>(); + + reassignments.put(new TopicPartition("foo", 1), Arrays.asList(4, 5, 3)); + reassignments.put(new TopicPartition("foo", 0), Arrays.asList(0, 1, 4, 2)); + reassignments.put(new TopicPartition("bar", 0), Arrays.asList(2, 3)); + + Map reassignmentResult = alterPartitionReassignments(adminClient, reassignments); + + assertTrue(reassignmentResult.isEmpty()); + assertEquals(String.join(System.lineSeparator(), Arrays.asList("Current partition reassignments:", + "bar-0: replicas: 2,3,0. removing: 0.", + "foo-0: replicas: 0,1,2. adding: 4.", + "foo-1: replicas: 1,2,3. adding: 4,5. removing: 1,2.")), + curReassignmentsToString(adminClient)); + } + } + + private void verifyBrokerThrottleResults(Config config, + long expectedInterBrokerThrottle, + long expectedReplicaAlterLogDirsThrottle) { + Map configs = new HashMap<>(); + config.entries().forEach(entry -> configs.put(entry.name(), entry.value())); + if (expectedInterBrokerThrottle >= 0) { + assertEquals(Long.toString(expectedInterBrokerThrottle), + configs.getOrDefault(BROKER_LEVEL_LEADER_THROTTLE, "")); + assertEquals(Long.toString(expectedInterBrokerThrottle), + configs.getOrDefault(BROKER_LEVEL_FOLLOWER_THROTTLE, "")); + } + if (expectedReplicaAlterLogDirsThrottle >= 0) { + assertEquals(Long.toString(expectedReplicaAlterLogDirsThrottle), + configs.getOrDefault(BROKER_LEVEL_LOG_DIR_THROTTLE, "")); + } + } + + @Test + public void testModifyTopicThrottles() throws Exception { + try (MockAdminClient adminClient = new MockAdminClient.Builder().numBrokers(4).build()) { + addTopics(adminClient); + + Map leaderThrottles = new HashMap<>(); + + leaderThrottles.put("foo", "leaderFoo"); + leaderThrottles.put("bar", "leaderBar"); + + modifyTopicThrottles(adminClient, + leaderThrottles, + Collections.singletonMap("bar", "followerBar")); + List topics = Stream.of("bar", "foo").map( + id -> new ConfigResource(ConfigResource.Type.TOPIC, id)).collect(Collectors.toList()); + Map results = adminClient.describeConfigs(topics).all().get(); + verifyTopicThrottleResults(results.get(topics.get(0)), "leaderBar", "followerBar"); + verifyTopicThrottleResults(results.get(topics.get(1)), "leaderFoo", ""); + } + } + + private void verifyTopicThrottleResults(Config config, + String expectedLeaderThrottle, + String expectedFollowerThrottle) { + Map configs = new HashMap<>(); + config.entries().forEach(entry -> configs.put(entry.name(), entry.value())); + assertEquals(expectedLeaderThrottle, + configs.getOrDefault(TOPIC_LEVEL_LEADER_THROTTLE, "")); + assertEquals(expectedFollowerThrottle, + configs.getOrDefault(TOPIC_LEVEL_FOLLOWER_THROTTLE, "")); + } + + @Test + public void testAlterReplicaLogDirs() throws Exception { + try (MockAdminClient adminClient = new MockAdminClient.Builder(). + numBrokers(4). + brokerLogDirs(Collections.nCopies(4, + Arrays.asList("/tmp/kafka-logs0", "/tmp/kafka-logs1"))). + build()) { + + addTopics(adminClient); + + Map assignment = new HashMap<>(); + + assignment.put(new TopicPartitionReplica("foo", 0, 0), "/tmp/kafka-logs1"); + assignment.put(new TopicPartitionReplica("quux", 1, 0), "/tmp/kafka-logs1"); + + assertEquals(new HashSet<>(Collections.singletonList( + new TopicPartitionReplica("foo", 0, 0) + )), + alterReplicaLogDirs(adminClient, assignment)); + } + } + + public void assertStartsWith(String prefix, String str) { + assertTrue(str.startsWith(prefix), String.format("Expected the string to start with %s, but it was %s", prefix, str)); + } + + @Test + public void testPropagateInvalidJsonError() { + try (MockAdminClient adminClient = new MockAdminClient.Builder().numBrokers(4).build()) { + addTopics(adminClient); + assertStartsWith("Unexpected character", + assertThrows(AdminOperationException.class, () -> executeAssignment(adminClient, false, "{invalid_json", -1L, -1L, 10000L, Time.SYSTEM)).getMessage()); + } + } +} + From fe06e63f898dd9e5b78b30a7bb309a91e5f1472a Mon Sep 17 00:00:00 2001 From: nizhikov Date: Mon, 31 Jul 2023 20:16:16 +0300 Subject: [PATCH 21/78] KAFKA-14595 ReassignPartitionsCommandArgsTest, ReassignPartitionsUnitTest rewritten in java --- .../reassign/ReassignPartitionsCommand.java | 2 +- .../ReassignPartitionsIntegrationTest.java | 256 ++++++++++++++++++ 2 files changed, 257 insertions(+), 1 deletion(-) create mode 100644 tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsIntegrationTest.java diff --git a/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommand.java b/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommand.java index 87d70845ad487..8b1191a203e65 100644 --- a/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommand.java +++ b/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommand.java @@ -100,7 +100,7 @@ public class ReassignPartitionsCommand { static final String BROKER_LEVEL_FOLLOWER_THROTTLE = "follower.replication.throttled.rate"; //DynamicConfig.Broker.ReplicaAlterLogDirsIoMaxBytesPerSecondProp static final String BROKER_LEVEL_LOG_DIR_THROTTLE = "replica.alter.log.dirs.io.max.bytes.per.second"; - private static final List BROKER_LEVEL_THROTTLES = Arrays.asList( + static final List BROKER_LEVEL_THROTTLES = Arrays.asList( BROKER_LEVEL_LEADER_THROTTLE, BROKER_LEVEL_FOLLOWER_THROTTLE, BROKER_LEVEL_LOG_DIR_THROTTLE diff --git a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsIntegrationTest.java b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsIntegrationTest.java new file mode 100644 index 0000000000000..8c0d46c6571a7 --- /dev/null +++ b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsIntegrationTest.java @@ -0,0 +1,256 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.tools.reassign; + +import kafka.server.IsrChangePropagationConfig; +import kafka.server.KafkaBroker; +import kafka.server.KafkaConfig; +import kafka.server.QuorumTestHarness; +import kafka.server.ZkAlterPartitionManager; +import kafka.utils.TestInfoUtils; +import kafka.utils.TestUtils; +import org.apache.kafka.clients.admin.Admin; +import org.apache.kafka.clients.admin.AdminClientConfig; +import org.apache.kafka.clients.admin.NewTopic; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.common.utils.Time; +import org.apache.kafka.common.utils.Utils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import scala.Some$; +import scala.collection.JavaConverters; +import scala.collection.Seq; + +import java.io.Closeable; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static org.apache.kafka.server.common.MetadataVersion.IBP_2_7_IV1; +import static org.apache.kafka.tools.reassign.ReassignPartitionsCommand.BROKER_LEVEL_THROTTLES; + +@Timeout(300) +public class ReassignPartitionsIntegrationTest extends QuorumTestHarness { + ReassignPartitionsTestCluster cluster; + + @AfterEach + @Override + public void tearDown() { + Utils.closeQuietly(cluster, "ReassignPartitionsTestCluster"); + super.tearDown(); + } + + private final Map> unthrottledBrokerConfigs = new HashMap<>(); { + IntStream.range(0, 4).forEach(brokerId -> + unthrottledBrokerConfigs.put(brokerId, BROKER_LEVEL_THROTTLES.stream() + .collect(Collectors.toMap(throttle -> throttle, throttle -> -1L)))); + } + + @ParameterizedTest(name = "{displayName}.quorum={0}") + @ValueSource(strings = {"zk", "kraft"}) + public void testReassignment(String quorum) throws Exception { + ReassignPartitionsTestCluster cluster = new ReassignPartitionsTestCluster(Collections.emptyMap(), Collections.emptyMap()); + cluster.setup(); + executeAndVerifyReassignment(); + } + + @ParameterizedTest(name = "{displayName}.quorum={0}") + @ValueSource(strings = "zk") // Note: KRaft requires AlterPartition + public void testReassignmentWithAlterPartitionDisabled(String quorum) throws Exception { + // Test reassignment when the IBP is on an older version which does not use + // the `AlterPartition` API. In this case, the controller will register individual + // watches for each reassigning partition so that the reassignment can be + // completed as soon as the ISR is expanded. + Map configOverrides = Collections.singletonMap(KafkaConfig.InterBrokerProtocolVersionProp(), IBP_2_7_IV1.version()); + cluster = new ReassignPartitionsTestCluster(configOverrides, Collections.emptyMap()); + cluster.setup(); + executeAndVerifyReassignment(); + } + + @ParameterizedTest(name = "{displayName}.quorum={0}") + @ValueSource(strings = "zk") // Note: KRaft requires AlterPartition + public void testReassignmentCompletionDuringPartialUpgrade(String quorum) throws Exception { + // Test reassignment during a partial upgrade when some brokers are relying on + // `AlterPartition` and some rely on the old notification logic through Zookeeper. + // In this test case, broker 0 starts up first on the latest IBP and is typically + // elected as controller. The three remaining brokers start up on the older IBP. + // We want to ensure that reassignment can still complete through the ISR change + // notification path even though the controller expects `AlterPartition`. + + // Override change notification settings so that test is not delayed by ISR + // change notification delay + ZkAlterPartitionManager.DefaultIsrPropagationConfig_$eq(new IsrChangePropagationConfig(500, 100, 500)); + + Map oldIbpConfig = Collections.singletonMap(KafkaConfig.InterBrokerProtocolVersionProp(), IBP_2_7_IV1.version()); + Map> brokerConfigOverrides = new HashMap<>(); + brokerConfigOverrides.put(1, oldIbpConfig); + brokerConfigOverrides.put(2, oldIbpConfig); + brokerConfigOverrides.put(3, oldIbpConfig); + + cluster = new ReassignPartitionsTestCluster(Collections.emptyMap(), brokerConfigOverrides); + cluster.setup(); + + executeAndVerifyReassignment(); + } + + private void executeAndVerifyReassignment() { + // TODO: fixme. + } + + class ReassignPartitionsTestCluster implements Closeable { + private final Map configOverrides; + + private final Map> brokerConfigOverrides; + + private final List brokerConfigs = new ArrayList<>(); + + private final Map brokers = new HashMap<>(); { + brokers.put(0, "rack0"); + brokers.put(1, "rack0"); + brokers.put(2, "rack1"); + brokers.put(3, "rack1"); + brokers.put(4, "rack1"); + } + + private final Map>> topics = new HashMap<>(); { + topics.put("foo", Arrays.asList(Arrays.asList(0, 1, 2), Arrays.asList(1, 2, 3))); + topics.put("bar", Arrays.asList(Arrays.asList(3, 2, 1))); + topics.put("baz", Arrays.asList(Arrays.asList(1, 0, 2), Arrays.asList(2, 0, 1), Arrays.asList(0, 2, 1))); + } + + private List servers = new ArrayList<>(); + + private String brokerList; + + private Admin adminClient; + + public ReassignPartitionsTestCluster(Map configOverrides, Map> brokerConfigOverrides) { + this.configOverrides = configOverrides; + this.brokerConfigOverrides = brokerConfigOverrides; + + brokers.forEach((brokerId, rack) -> { + Properties config = TestUtils.createBrokerConfig( + brokerId, + zkConnectOrNull(), + false, // shorten test time + true, + TestUtils.RandomPort(), + scala.None$.empty(), + scala.None$.empty(), + scala.None$.empty(), + true, + false, + TestUtils.RandomPort(), + false, + TestUtils.RandomPort(), + false, + TestUtils.RandomPort(), + Some$.MODULE$.apply(rack), + 3, + false, + 1, + (short)1, + false); + // shorter backoff to reduce test durations when no active partitions are eligible for fetching due to throttling + config.setProperty(KafkaConfig.ReplicaFetchBackoffMsProp(), "100"); + // Don't move partition leaders automatically. + config.setProperty(KafkaConfig.AutoLeaderRebalanceEnableProp(), "false"); + config.setProperty(KafkaConfig.ReplicaLagTimeMaxMsProp(), "1000"); + configOverrides.forEach(config::setProperty); + brokerConfigOverrides.getOrDefault(brokerId, Collections.emptyMap()).forEach(config::setProperty); + + brokerConfigs.add(new KafkaConfig(config)); + }); + } + + public void setup() throws ExecutionException, InterruptedException { + createServers(); + createTopics(); + } + + public void createServers() { + brokers.keySet().forEach(brokerId -> + servers.add(createBroker(brokerConfigs.get(brokerId), Time.SYSTEM, true, scala.None$.empty())) + ); + } + + public void createTopics() throws ExecutionException, InterruptedException { + TestUtils.waitUntilBrokerMetadataIsPropagated(toSeq(servers), org.apache.kafka.test.TestUtils.DEFAULT_MAX_WAIT_MS); + brokerList = TestUtils.plaintextBootstrapServers(toSeq(servers)); + + adminClient = Admin.create(Collections.singletonMap(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, brokerList)); + + adminClient.createTopics(topics.entrySet().stream().map(e -> { + Map> partMap = new HashMap<>(); + + Iterator> partsIter = e.getValue().iterator(); + int index = 0; + while (partsIter.hasNext()) { + partMap.put(index, partsIter.next()); + index++; + } + return new NewTopic(e.getKey(), partMap); + }).collect(Collectors.toList())).all().get(); + topics.forEach((topicName, parts) -> { + TestUtils.waitForAllPartitionsMetadata(toSeq(servers), topicName, parts.size()); + }); + + if (isKRaftTest()) { + TestUtils.ensureConsistentKRaftMetadata( + toSeq(cluster.servers), + controllerServer(), + "Timeout waiting for controller metadata propagating to brokers" + ); + } + } + + public void produceMessages(String topic, int partition, int numMessages) { + List> records = IntStream.range(0, numMessages).mapToObj(i -> + new ProducerRecord(topic, partition, + null, new byte[10000])).collect(Collectors.toList()); + TestUtils.produceMessages(toSeq(servers), toSeq(records), -1); + } + + @Override + public void close() throws IOException { + brokerList = null; + Utils.closeQuietly(adminClient, "adminClient"); + adminClient = null; + try { + TestUtils.shutdownServers(toSeq(servers), true); + } finally { + servers.clear(); + } + } + } + + private static Seq toSeq(List list) { + return JavaConverters.asScalaIteratorConverter(list.iterator()).asScala().toSeq(); + } +} From a7ad31602ba0b5370773a45c078566f43059057c Mon Sep 17 00:00:00 2001 From: nizhikov Date: Tue, 1 Aug 2023 02:15:44 +0300 Subject: [PATCH 22/78] KAFKA-14595 ReassignPartitionsCommandArgsTest, ReassignPartitionsUnitTest rewritten in java --- .../reassign/ReassignPartitionsCommand.java | 12 +- .../ReassignPartitionsIntegrationTest.java | 170 +++++++++++++++++- 2 files changed, 173 insertions(+), 9 deletions(-) diff --git a/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommand.java b/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommand.java index 8b1191a203e65..472aa6acb1306 100644 --- a/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommand.java +++ b/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommand.java @@ -196,7 +196,7 @@ private static void handleAction(Admin adminClient, ReassignPartitionsCommandOpt * * @return A result that is useful for testing. */ - private static VerifyAssignmentResult verifyAssignment(Admin adminClient, String jsonString, Boolean preserveThrottles) throws ExecutionException, InterruptedException, JsonProcessingException { + static VerifyAssignmentResult verifyAssignment(Admin adminClient, String jsonString, Boolean preserveThrottles) throws ExecutionException, InterruptedException, JsonProcessingException { Tuple>>, Map> t0 = parsePartitionReassignmentData(jsonString); List>> targetParts = t0.v1; @@ -1230,11 +1230,11 @@ static Tuple>, Map, Set> cancelAssignment(Admin adminClient, - String jsonString, - Boolean preserveThrottles, - Long timeoutMs, - Time time) throws ExecutionException, InterruptedException, JsonProcessingException { + static Tuple, Set> cancelAssignment(Admin adminClient, + String jsonString, + Boolean preserveThrottles, + Long timeoutMs, + Time time) throws ExecutionException, InterruptedException, JsonProcessingException { Tuple>>, Map> t0 = parsePartitionReassignmentData(jsonString); List>> targetParts = t0.v1; diff --git a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsIntegrationTest.java b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsIntegrationTest.java index 8c0d46c6571a7..6e70b76028d79 100644 --- a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsIntegrationTest.java +++ b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsIntegrationTest.java @@ -17,17 +17,19 @@ package org.apache.kafka.tools.reassign; +import com.fasterxml.jackson.core.JsonProcessingException; import kafka.server.IsrChangePropagationConfig; import kafka.server.KafkaBroker; import kafka.server.KafkaConfig; import kafka.server.QuorumTestHarness; import kafka.server.ZkAlterPartitionManager; -import kafka.utils.TestInfoUtils; import kafka.utils.TestUtils; import org.apache.kafka.clients.admin.Admin; import org.apache.kafka.clients.admin.AdminClientConfig; +import org.apache.kafka.clients.admin.DescribeLogDirsResult; import org.apache.kafka.clients.admin.NewTopic; import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.utils.Time; import org.apache.kafka.common.utils.Utils; import org.junit.jupiter.api.AfterEach; @@ -44,16 +46,24 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Properties; +import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; import java.util.stream.IntStream; import static org.apache.kafka.server.common.MetadataVersion.IBP_2_7_IV1; import static org.apache.kafka.tools.reassign.ReassignPartitionsCommand.BROKER_LEVEL_THROTTLES; +import static org.apache.kafka.tools.reassign.ReassignPartitionsCommand.cancelAssignment; +import static org.apache.kafka.tools.reassign.ReassignPartitionsCommand.executeAssignment; +import static org.apache.kafka.tools.reassign.ReassignPartitionsCommand.verifyAssignment; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; @Timeout(300) public class ReassignPartitionsIntegrationTest extends QuorumTestHarness { @@ -119,8 +129,162 @@ public void testReassignmentCompletionDuringPartialUpgrade(String quorum) throws executeAndVerifyReassignment(); } - private void executeAndVerifyReassignment() { - // TODO: fixme. + private void executeAndVerifyReassignment() throws ExecutionException, InterruptedException, JsonProcessingException { + String assignment = "{\"version\":1,\"partitions\":" + + "[{\"topic\":\"foo\",\"partition\":0,\"replicas\":[0,1,3],\"log_dirs\":[\"any\",\"any\",\"any\"]}," + + "{\"topic\":\"bar\",\"partition\":0,\"replicas\":[3,2,0],\"log_dirs\":[\"any\",\"any\",\"any\"]}" + + "]}"; + + TopicPartition foo0 = new TopicPartition("foo", 0); + TopicPartition bar0 = new TopicPartition("bar", 0); + + // Check that the assignment has not yet been started yet. + Map initialAssignment = new HashMap<>(); + + initialAssignment.put(foo0, new PartitionReassignmentState(Arrays.asList(0, 1, 2), Arrays.asList(0, 1, 3), true)); + initialAssignment.put(bar0, new PartitionReassignmentState(Arrays.asList(3, 2, 1), Arrays.asList(3, 2, 0), true)); + + waitForVerifyAssignment(cluster.adminClient, assignment, false, + new VerifyAssignmentResult(initialAssignment)); + +/* + // Execute the assignment + runExecuteAssignment(cluster.adminClient, false, assignment, -1L, -1L); + assertEquals(unthrottledBrokerConfigs, describeBrokerLevelThrottles(unthrottledBrokerConfigs.keySet())); + val finalAssignment = Map( + foo0 -> PartitionReassignmentState(Seq(0, 1, 3), Seq(0, 1, 3), true), + bar0 -> PartitionReassignmentState(Seq(3, 2, 0), Seq(3, 2, 0), true) + ) + + val verifyAssignmentResult = runVerifyAssignment(cluster.adminClient, assignment, false) + assertFalse(verifyAssignmentResult.movesOngoing) + + // Wait for the assignment to complete + waitForVerifyAssignment(cluster.adminClient, assignment, false, + VerifyAssignmentResult(finalAssignment)) + + assertEquals(unthrottledBrokerConfigs, + describeBrokerLevelThrottles(unthrottledBrokerConfigs.keySet.toSeq)) + + // Verify that partitions are removed from brokers no longer assigned + verifyReplicaDeleted(topicPartition = foo0, replicaId = 2) + verifyReplicaDeleted(topicPartition = bar0, replicaId = 1) +*/ + } + + class LogDirReassignment { + final String json; + final String currentDir; + final String targetDir; + + public LogDirReassignment(String json, String currentDir, String targetDir) { + this.json = json; + this.currentDir = currentDir; + this.targetDir = targetDir; + } + } + + private LogDirReassignment buildLogDirReassignment(TopicPartition topicPartition, + int brokerId, + List replicas) throws ExecutionException, InterruptedException { + + DescribeLogDirsResult describeLogDirsResult = cluster.adminClient.describeLogDirs( + IntStream.range(0, 4).boxed().collect(Collectors.toList())); + + BrokerDirs logDirInfo = new BrokerDirs(describeLogDirsResult, brokerId); + assertTrue(logDirInfo.futureLogDirs.isEmpty()); + + String currentDir = logDirInfo.curLogDirs.get(topicPartition); + String newDir = logDirInfo.logDirs.stream().filter(dir -> !dir.equals(currentDir)).findFirst().get(); + + List logDirs = replicas.stream().map(replicaId -> { + if (replicaId == brokerId) + return "\"" + newDir + "\""; + else + return "\"any\""; + }).collect(Collectors.toList()); + + String reassignmentJson = + " { \"version\": 1," + + " \"partitions\": [" + + " {" + + " \"topic\": \"" + topicPartition.topic() + "\"," + + " \"partition\": " + topicPartition.partition() + "," + + " \"replicas\": [" + replicas.stream().map(Object::toString).collect(Collectors.joining(",")) + "]," + + " \"log_dirs\": [" + String.join(",", logDirs) + "]" + + " }" + + " ]" + + " }"; + + return new LogDirReassignment(reassignmentJson, currentDir, newDir); + } + + + + private VerifyAssignmentResult runVerifyAssignment(Admin adminClient, String jsonString, + Boolean preserveThrottles) throws ExecutionException, InterruptedException, JsonProcessingException { + System.out.println("==> verifyAssignment(adminClient, jsonString=" + jsonString); + return verifyAssignment(adminClient, jsonString, preserveThrottles); + } + + private void waitForVerifyAssignment(Admin adminClient, + String jsonString, + Boolean preserveThrottles, + VerifyAssignmentResult expectedResult) { + final VerifyAssignmentResult[] latestResult = {null}; + TestUtils.waitUntilTrue( + () -> { + try { + latestResult[0] = runVerifyAssignment(adminClient, jsonString, preserveThrottles); + } catch (ExecutionException | InterruptedException | JsonProcessingException e) { + throw new RuntimeException(e); + } + return expectedResult.equals(latestResult[0]); + }, () -> "Timed out waiting for verifyAssignment result " + expectedResult + ". " + + "The latest result was " + latestResult[0], org.apache.kafka.test.TestUtils.DEFAULT_MAX_WAIT_MS, 10L); + } + + private void runExecuteAssignment(Admin adminClient, + Boolean additional, + String reassignmentJson, + Long interBrokerThrottle, + Long replicaAlterLogDirsThrottle) throws ExecutionException, InterruptedException, JsonProcessingException { + System.out.println("==> executeAssignment(adminClient, additional=" + additional + ", " + + "reassignmentJson=" + reassignmentJson + ", " + + "interBrokerThrottle=" + interBrokerThrottle + ", " + + "replicaAlterLogDirsThrottle=" + replicaAlterLogDirsThrottle + "))"); + executeAssignment(adminClient, additional, reassignmentJson, + interBrokerThrottle, replicaAlterLogDirsThrottle, 10000L, Time.SYSTEM); + } + + private void runCancelAssignment(Admin adminClient, String jsonString, Boolean preserveThrottles) throws ExecutionException, InterruptedException, JsonProcessingException { + System.out.println("==> cancelAssignment(adminClient, jsonString=" + jsonString); + cancelAssignment(adminClient, jsonString, preserveThrottles, 10000L, Time.SYSTEM); + } + + class BrokerDirs { + final DescribeLogDirsResult result; + final int brokerId; + + final Set logDirs = new HashSet<>(); + final Map curLogDirs = new HashMap<>(); + final Map futureLogDirs = new HashMap<>(); + + public BrokerDirs(DescribeLogDirsResult result, int brokerId) throws ExecutionException, InterruptedException { + this.result = result; + this.brokerId = brokerId; + + result.descriptions().get(brokerId).get().forEach((logDirName, logDirInfo) -> { + logDirs.add(logDirName); + logDirInfo.replicaInfos().forEach((part, info) -> { + if (info.isFuture()) { + futureLogDirs.put(part, logDirName); + } else { + curLogDirs.put(part, logDirName); + } + }); + }); + } } class ReassignPartitionsTestCluster implements Closeable { From 26c76fc7c6717bfeefc57f05e83bf500ceddddb0 Mon Sep 17 00:00:00 2001 From: nizhikov Date: Tue, 1 Aug 2023 20:57:57 +0300 Subject: [PATCH 23/78] KAFKA-14595 ReassignPartitionsIntegrationTest rewritten in java --- .../ReassignPartitionsIntegrationTest.java | 469 +++++++++++++++++- 1 file changed, 453 insertions(+), 16 deletions(-) diff --git a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsIntegrationTest.java b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsIntegrationTest.java index 6e70b76028d79..c7aa39723e48d 100644 --- a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsIntegrationTest.java +++ b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsIntegrationTest.java @@ -18,6 +18,9 @@ package org.apache.kafka.tools.reassign; import com.fasterxml.jackson.core.JsonProcessingException; +import kafka.cluster.Partition; +import kafka.log.UnifiedLog; +import kafka.server.HostedPartition; import kafka.server.IsrChangePropagationConfig; import kafka.server.KafkaBroker; import kafka.server.KafkaConfig; @@ -26,16 +29,26 @@ import kafka.utils.TestUtils; import org.apache.kafka.clients.admin.Admin; import org.apache.kafka.clients.admin.AdminClientConfig; +import org.apache.kafka.clients.admin.AlterConfigOp; +import org.apache.kafka.clients.admin.Config; +import org.apache.kafka.clients.admin.ConfigEntry; import org.apache.kafka.clients.admin.DescribeLogDirsResult; import org.apache.kafka.clients.admin.NewTopic; +import org.apache.kafka.clients.consumer.Consumer; import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.TopicPartitionReplica; +import org.apache.kafka.common.config.ConfigResource; +import org.apache.kafka.common.security.auth.SecurityProtocol; +import org.apache.kafka.common.serialization.ByteArrayDeserializer; import org.apache.kafka.common.utils.Time; import org.apache.kafka.common.utils.Utils; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Timeout; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; +import scala.None$; +import scala.Option; import scala.Some$; import scala.collection.JavaConverters; import scala.collection.Seq; @@ -44,12 +57,14 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Properties; import java.util.Set; import java.util.concurrent.ExecutionException; @@ -147,29 +162,447 @@ private void executeAndVerifyReassignment() throws ExecutionException, Interrupt waitForVerifyAssignment(cluster.adminClient, assignment, false, new VerifyAssignmentResult(initialAssignment)); -/* // Execute the assignment runExecuteAssignment(cluster.adminClient, false, assignment, -1L, -1L); assertEquals(unthrottledBrokerConfigs, describeBrokerLevelThrottles(unthrottledBrokerConfigs.keySet())); - val finalAssignment = Map( - foo0 -> PartitionReassignmentState(Seq(0, 1, 3), Seq(0, 1, 3), true), - bar0 -> PartitionReassignmentState(Seq(3, 2, 0), Seq(3, 2, 0), true) - ) + Map finalAssignment = new HashMap<>(); + finalAssignment.put(foo0, new PartitionReassignmentState(Arrays.asList(0, 1, 3), Arrays.asList(0, 1, 3), true)); + finalAssignment.put(bar0, new PartitionReassignmentState(Arrays.asList(3, 2, 0), Arrays.asList(3, 2, 0), true)); - val verifyAssignmentResult = runVerifyAssignment(cluster.adminClient, assignment, false) - assertFalse(verifyAssignmentResult.movesOngoing) + VerifyAssignmentResult verifyAssignmentResult = runVerifyAssignment(cluster.adminClient, assignment, false); + assertFalse(verifyAssignmentResult.movesOngoing); // Wait for the assignment to complete waitForVerifyAssignment(cluster.adminClient, assignment, false, - VerifyAssignmentResult(finalAssignment)) + new VerifyAssignmentResult(finalAssignment)); assertEquals(unthrottledBrokerConfigs, - describeBrokerLevelThrottles(unthrottledBrokerConfigs.keySet.toSeq)) + describeBrokerLevelThrottles(unthrottledBrokerConfigs.keySet())); // Verify that partitions are removed from brokers no longer assigned - verifyReplicaDeleted(topicPartition = foo0, replicaId = 2) - verifyReplicaDeleted(topicPartition = bar0, replicaId = 1) -*/ + verifyReplicaDeleted(foo0, 2); + verifyReplicaDeleted(bar0, 1); + } + + @ParameterizedTest(name = "{displayName}.quorum={0}") + @ValueSource(strings = {"zk", "kraft"}) + public void testHighWaterMarkAfterPartitionReassignment(String quorum) throws Exception { + cluster = new ReassignPartitionsTestCluster(Collections.emptyMap(), Collections.emptyMap()); + cluster.setup(); + String assignment = "{\"version\":1,\"partitions\":" + + "[{\"topic\":\"foo\",\"partition\":0,\"replicas\":[3,1,2],\"log_dirs\":[\"any\",\"any\",\"any\"]}" + + "]}"; + + // Set the high water mark of foo-0 to 123 on its leader. + TopicPartition part = new TopicPartition("foo", 0); + cluster.servers.get(0).replicaManager().logManager().truncateFullyAndStartAt(part, 123L, false); + + // Execute the assignment + runExecuteAssignment(cluster.adminClient, false, assignment, -1L, -1L); + Map finalAssignment = Collections.singletonMap(part, + new PartitionReassignmentState(Arrays.asList(3, 1, 2), Arrays.asList(3, 1, 2), true)); + + // Wait for the assignment to complete + waitForVerifyAssignment(cluster.adminClient, assignment, false, + new VerifyAssignmentResult(finalAssignment)); + + TestUtils.waitUntilTrue(() -> + cluster.servers.get(3).replicaManager().onlinePartition(part). + map(Partition::leaderLogIfLocal).isDefined() + , () -> "broker 3 should be the new leader", org.apache.kafka.test.TestUtils.DEFAULT_MAX_WAIT_MS, 10L); + assertEquals(123L, cluster.servers.get(3).replicaManager().localLogOrException(part).highWatermark(), + "Expected broker 3 to have the correct high water mark for the partition."); + } + + @ParameterizedTest(name = "{displayName}.quorum={0}") + @ValueSource(strings = {"zk", "kraft"}) + public void testAlterReassignmentThrottle(String quorum) throws Exception { + cluster = new ReassignPartitionsTestCluster(Collections.emptyMap(), Collections.emptyMap()); + cluster.setup(); + cluster.produceMessages("foo", 0, 50); + cluster.produceMessages("baz", 2, 60); + String assignment = "{\"version\":1,\"partitions\":" + + "[{\"topic\":\"foo\",\"partition\":0,\"replicas\":[0,3,2],\"log_dirs\":[\"any\",\"any\",\"any\"]}," + + "{\"topic\":\"baz\",\"partition\":2,\"replicas\":[3,2,1],\"log_dirs\":[\"any\",\"any\",\"any\"]}" + + "]}"; + + // Execute the assignment with a low throttle + long initialThrottle = 1L; + runExecuteAssignment(cluster.adminClient, false, assignment, initialThrottle, -1L); + waitForInterBrokerThrottle(Arrays.asList(0, 1, 2, 3), initialThrottle); + + // Now update the throttle and verify the reassignment completes + long updatedThrottle = 300000L; + runExecuteAssignment(cluster.adminClient, true, assignment, updatedThrottle, -1L); + waitForInterBrokerThrottle(Arrays.asList(0, 1, 2, 3), updatedThrottle); + + Map finalAssignment = new HashMap<>(); + finalAssignment.put(new TopicPartition("foo", 0), + new PartitionReassignmentState(Arrays.asList(0, 3, 2), Arrays.asList(0, 3, 2), true)); + finalAssignment.put(new TopicPartition("baz", 2), + new PartitionReassignmentState(Arrays.asList(3, 2, 1), Arrays.asList(3, 2, 1), true)); + + // Now remove the throttles. + waitForVerifyAssignment(cluster.adminClient, assignment, false, + new VerifyAssignmentResult(finalAssignment)); + waitForBrokerLevelThrottles(unthrottledBrokerConfigs); + } + + /** + * Test running a reassignment with the interBrokerThrottle set. + */ + @ParameterizedTest(name = "{displayName}.quorum={0}") + @ValueSource(strings = {"zk", "kraft"}) + public void testThrottledReassignment(String quorum) throws Exception { + cluster = new ReassignPartitionsTestCluster(Collections.emptyMap(), Collections.emptyMap()); + cluster.setup(); + cluster.produceMessages("foo", 0, 50); + cluster.produceMessages("baz", 2, 60); + String assignment = "{\"version\":1,\"partitions\":" + + "[{\"topic\":\"foo\",\"partition\":0,\"replicas\":[0,3,2],\"log_dirs\":[\"any\",\"any\",\"any\"]}," + + "{\"topic\":\"baz\",\"partition\":2,\"replicas\":[3,2,1],\"log_dirs\":[\"any\",\"any\",\"any\"]}" + + "]}"; + + // Check that the assignment has not yet been started yet. + Map initialAssignment = new HashMap<>(); + initialAssignment.put(new TopicPartition("foo", 0), + new PartitionReassignmentState(Arrays.asList(0, 1, 2), Arrays.asList(0, 3, 2), true)); + initialAssignment.put(new TopicPartition("baz", 2), + new PartitionReassignmentState(Arrays.asList(0, 2, 1), Arrays.asList(3, 2, 1), true)); + assertEquals(new VerifyAssignmentResult(initialAssignment), runVerifyAssignment(cluster.adminClient, assignment, false)); + assertEquals(unthrottledBrokerConfigs, describeBrokerLevelThrottles(unthrottledBrokerConfigs.keySet())); + + // Execute the assignment + long interBrokerThrottle = 300000L; + runExecuteAssignment(cluster.adminClient, false, assignment, interBrokerThrottle, -1L); + waitForInterBrokerThrottle(Arrays.asList(0, 1, 2, 3), interBrokerThrottle); + + Map finalAssignment = new HashMap<>(); + finalAssignment.put(new TopicPartition("foo", 0), + new PartitionReassignmentState(Arrays.asList(0, 3, 2), Arrays.asList(0, 3, 2), true)); + finalAssignment.put(new TopicPartition("baz", 2), + new PartitionReassignmentState(Arrays.asList(3, 2, 1), Arrays.asList(3, 2, 1), true)); + + // Wait for the assignment to complete + TestUtils.waitUntilTrue( + () -> { + try { + // Check the reassignment status. + VerifyAssignmentResult result = runVerifyAssignment(cluster.adminClient, assignment, true); + + if (!result.partsOngoing) { + return true; + } else { + assertFalse(result.partStates.values().stream().allMatch(v -> v.done), "Expected at least one partition reassignment to be ongoing when result = " + result); + assertEquals(Arrays.asList(0, 3, 2), result.partStates.get(new TopicPartition("foo", 0)).targetReplicas); + assertEquals(Arrays.asList(3, 2, 1), result.partStates.get(new TopicPartition("baz", 2)).targetReplicas); + System.out.println("Current result: " + result); + waitForInterBrokerThrottle(Arrays.asList(0, 1, 2, 3), interBrokerThrottle); + return false; + } + } catch (ExecutionException | InterruptedException | JsonProcessingException e) { + throw new RuntimeException(e); + } + }, () -> "Expected reassignment to complete.", org.apache.kafka.test.TestUtils.DEFAULT_MAX_WAIT_MS, 100L); + waitForVerifyAssignment(cluster.adminClient, assignment, true, + new VerifyAssignmentResult(finalAssignment)); + // The throttles should still have been preserved, since we ran with --preserve-throttles + waitForInterBrokerThrottle(Arrays.asList(0, 1, 2, 3), interBrokerThrottle); + // Now remove the throttles. + waitForVerifyAssignment(cluster.adminClient, assignment, false, + new VerifyAssignmentResult(finalAssignment)); + waitForBrokerLevelThrottles(unthrottledBrokerConfigs); + } + + @ParameterizedTest(name = "{displayName}.quorum={0}") + @ValueSource(strings = {"zk", "kraft"}) + public void testProduceAndConsumeWithReassignmentInProgress(String quorum) throws Exception { + cluster = new ReassignPartitionsTestCluster(Collections.emptyMap(), Collections.emptyMap()); + cluster.setup(); + cluster.produceMessages("baz", 2, 60); + String assignment = "{\"version\":1,\"partitions\":" + + "[{\"topic\":\"baz\",\"partition\":2,\"replicas\":[3,2,1],\"log_dirs\":[\"any\",\"any\",\"any\"]}" + + "]}"; + runExecuteAssignment(cluster.adminClient, false, assignment, 300L, -1L); + cluster.produceMessages("baz", 2, 100); + Consumer consumer = TestUtils.createConsumer(cluster.brokerList, + "group", + "earliest", + true, + false, + 500, + SecurityProtocol.PLAINTEXT, + None$.empty(), + None$.empty(), + new ByteArrayDeserializer(), + new ByteArrayDeserializer() + ); + + TopicPartition part = new TopicPartition("baz", 2); + try { + consumer.assign(Collections.singleton(part)); + TestUtils.pollUntilAtLeastNumRecords(consumer, 100, org.apache.kafka.test.TestUtils.DEFAULT_MAX_WAIT_MS); + } finally { + consumer.close(); + } + TestUtils.removeReplicationThrottleForPartitions(cluster.adminClient, toSeq(Arrays.asList(0,1,2,3)), toSet(Collections.singleton(part))); + Map finalAssignment = Collections.singletonMap(part, + new PartitionReassignmentState(Arrays.asList(3, 2, 1), Arrays.asList(3, 2, 1), true)); + waitForVerifyAssignment(cluster.adminClient, assignment, false, + new VerifyAssignmentResult(finalAssignment)); + } + + /** + * Test running a reassignment and then cancelling it. + */ + @ParameterizedTest(name = "{displayName}.quorum={0}") + @ValueSource(strings = {"zk", "kraft"}) + public void testCancellation(String quorum) throws Exception { + TopicPartition foo0 = new TopicPartition("foo", 0); + TopicPartition baz1 = new TopicPartition("baz", 1); + + cluster = new ReassignPartitionsTestCluster(Collections.emptyMap(), Collections.emptyMap()); + cluster.setup(); + cluster.produceMessages(foo0.topic(), foo0.partition(), 200); + cluster.produceMessages(baz1.topic(), baz1.partition(), 200); + String assignment = "{\"version\":1,\"partitions\":" + + "[{\"topic\":\"foo\",\"partition\":0,\"replicas\":[0,1,3],\"log_dirs\":[\"any\",\"any\",\"any\"]}," + + "{\"topic\":\"baz\",\"partition\":1,\"replicas\":[0,2,3],\"log_dirs\":[\"any\",\"any\",\"any\"]}" + + "]}"; + assertEquals(unthrottledBrokerConfigs, + describeBrokerLevelThrottles(unthrottledBrokerConfigs.keySet())); + long interBrokerThrottle = 1L; + runExecuteAssignment(cluster.adminClient, false, assignment, interBrokerThrottle, -1L); + waitForInterBrokerThrottle(Arrays.asList(0, 1, 2, 3), interBrokerThrottle); + + Map partStates = new HashMap<>(); + + partStates.put(foo0, new PartitionReassignmentState(Arrays.asList(0, 1, 3, 2), Arrays.asList(0, 1, 3), false)); + partStates.put(baz1, new PartitionReassignmentState(Arrays.asList(0, 2, 3, 1), Arrays.asList(0, 2, 3), false)); + + // Verify that the reassignment is running. The very low throttle should keep it + // from completing before this runs. + waitForVerifyAssignment(cluster.adminClient, assignment, true, + new VerifyAssignmentResult(partStates, true, Collections.emptyMap(), false)); + // Cancel the reassignment. + assertEquals(new Tuple<>(new HashSet<>(Arrays.asList(foo0, baz1)), Collections.emptySet()), runCancelAssignment(cluster.adminClient, assignment, true)); + // Broker throttles are still active because we passed --preserve-throttles + waitForInterBrokerThrottle(Arrays.asList(0, 1, 2, 3), interBrokerThrottle); + // Cancelling the reassignment again should reveal nothing to cancel. + assertEquals(new Tuple<>(Collections.emptySet(), Collections.emptySet()), runCancelAssignment(cluster.adminClient, assignment, false)); + // This time, the broker throttles were removed. + waitForBrokerLevelThrottles(unthrottledBrokerConfigs); + // Verify that there are no ongoing reassignments. + assertFalse(runVerifyAssignment(cluster.adminClient, assignment, false).partsOngoing); + // Verify that the partition is removed from cancelled replicas + verifyReplicaDeleted(foo0, 3); + verifyReplicaDeleted(baz1, 3); + } + + @ParameterizedTest(name = "{displayName}.quorum={0}") + @ValueSource(strings = {"zk", "kraft"}) + public void testCancellationWithAddingReplicaInIsr(String quorum) throws Exception { + TopicPartition foo0 = new TopicPartition("foo", 0); + + cluster = new ReassignPartitionsTestCluster(Collections.emptyMap(), Collections.emptyMap()); + cluster.setup(); + cluster.produceMessages(foo0.topic(), foo0.partition(), 200); + + // The reassignment will bring replicas 3 and 4 into the replica set and remove 1 and 2. + String assignment = "{\"version\":1,\"partitions\":" + + "[{\"topic\":\"foo\",\"partition\":0,\"replicas\":[0,3,4],\"log_dirs\":[\"any\",\"any\",\"any\"]}" + + "]}"; + + // We will throttle replica 4 so that only replica 3 joins the ISR + TestUtils.setReplicationThrottleForPartitions( + cluster.adminClient, + toSeq(Collections.singleton(4)), + toSet(Collections.singleton(foo0)), + 1 + ); + + // Execute the assignment and wait for replica 3 (only) to join the ISR + runExecuteAssignment( + cluster.adminClient, + false, + assignment, + -1L, + -1L + ); + TestUtils.waitUntilTrue( + () -> JavaConverters.asJava(TestUtils.currentIsr(cluster.adminClient, foo0)).equals(new HashSet<>(Arrays.asList(0, 1, 2, 3))), + () -> "Timed out while waiting for replica 3 to join the ISR", + org.apache.kafka.test.TestUtils.DEFAULT_MAX_WAIT_MS, 100L + ); + + // Now cancel the assignment and verify that the partition is removed from cancelled replicas + assertEquals(new Tuple<>(Collections.singleton(foo0), Collections.emptySet()), runCancelAssignment(cluster.adminClient, assignment, true)); + verifyReplicaDeleted(foo0, 3); + verifyReplicaDeleted(foo0, 4); + } + + private void verifyReplicaDeleted( + TopicPartition topicPartition, + Integer replicaId + ) { + TestUtils.waitUntilTrue(() -> { + KafkaBroker server = cluster.servers.get(replicaId); + HostedPartition partition = server.replicaManager().getPartition(topicPartition); + Option log = server.logManager().getLog(topicPartition, false); + return partition == HostedPartition.None$.MODULE$ && log.isEmpty(); + }, () -> "Timed out waiting for replica " + replicaId + " of " + topicPartition + " to be deleted", + org.apache.kafka.test.TestUtils.DEFAULT_MAX_WAIT_MS, 100L); + } + + private void waitForLogDirThrottle(Set throttledBrokers, Long logDirThrottle) { + Map throttledConfigMap = new HashMap<>(); + throttledConfigMap.put(ReassignPartitionsCommand.BROKER_LEVEL_LEADER_THROTTLE, -1L); + throttledConfigMap.put(ReassignPartitionsCommand.BROKER_LEVEL_FOLLOWER_THROTTLE, -1L); + throttledConfigMap.put(ReassignPartitionsCommand.BROKER_LEVEL_LOG_DIR_THROTTLE, logDirThrottle); + waitForBrokerThrottles(throttledBrokers, throttledConfigMap); + } + + private void waitForInterBrokerThrottle(List throttledBrokers, Long interBrokerThrottle) { + Map throttledConfigMap = new HashMap<>(); + throttledConfigMap.put(ReassignPartitionsCommand.BROKER_LEVEL_LEADER_THROTTLE, interBrokerThrottle); + throttledConfigMap.put(ReassignPartitionsCommand.BROKER_LEVEL_FOLLOWER_THROTTLE, interBrokerThrottle); + throttledConfigMap.put(ReassignPartitionsCommand.BROKER_LEVEL_LOG_DIR_THROTTLE, -1L); + waitForBrokerThrottles(throttledBrokers, throttledConfigMap); + } + + private void waitForBrokerThrottles(Collection throttledBrokers, Map throttleConfig) { + Map> throttledBrokerConfigs = new HashMap<>(); + unthrottledBrokerConfigs.forEach((brokerId, unthrottledConfig) -> { + Map expectedThrottleConfig = throttledBrokers.contains(brokerId) + ? throttleConfig + : unthrottledConfig; + throttledBrokerConfigs.put(brokerId, expectedThrottleConfig); + }); + waitForBrokerLevelThrottles(throttledBrokerConfigs); + } + + private void waitForBrokerLevelThrottles(Map> targetThrottles) { + Map>[] curThrottles = new Map[]{Collections.emptyMap()}; + TestUtils.waitUntilTrue(() -> { + try { + curThrottles[0] = describeBrokerLevelThrottles(targetThrottles.keySet()); + return targetThrottles.equals(curThrottles[0]); + } catch (ExecutionException | InterruptedException e) { + throw new RuntimeException(e); + } + }, () -> "timed out waiting for broker throttle to become " + targetThrottles + ". " + + "Latest throttles were " + curThrottles[0], org.apache.kafka.test.TestUtils.DEFAULT_MAX_WAIT_MS, 25); + } + + /** + * Describe the broker-level throttles in the cluster. + * + * @return A map whose keys are broker IDs and whose values are throttle + * information. The nested maps are keyed on throttle name. + */ + private Map> describeBrokerLevelThrottles(Collection brokerIds) throws ExecutionException, InterruptedException { + Map> results = new HashMap<>(); + for (Integer brokerId : brokerIds) { + ConfigResource brokerResource = new ConfigResource(ConfigResource.Type.BROKER, brokerId.toString()); + Config brokerConfigs = cluster.adminClient.describeConfigs(Collections.singleton(brokerResource)).values() + .get(brokerResource) + .get(); + + Map throttles = new HashMap<>(); + BROKER_LEVEL_THROTTLES.stream().forEach(throttleName -> { + String configValue = Optional.ofNullable(brokerConfigs.get(throttleName)).map(ConfigEntry::value).orElse("-1"); + throttles.put(throttleName, Long.parseLong(configValue)); + }); + results.put(brokerId, throttles); + } + return results; + } + + /** + * Test moving partitions between directories. + */ + @ParameterizedTest(name = "{displayName}.quorum={0}") + @ValueSource(strings = "zk") // JBOD not yet implemented for KRaft + public void testLogDirReassignment(String quorum) throws Exception { + TopicPartition topicPartition = new TopicPartition("foo", 0); + + cluster = new ReassignPartitionsTestCluster(Collections.emptyMap(), Collections.emptyMap()); + cluster.setup(); + cluster.produceMessages(topicPartition.topic(), topicPartition.partition(), 700); + + int targetBrokerId = 0; + List replicas = Arrays.asList(0, 1, 2); + LogDirReassignment reassignment = buildLogDirReassignment(topicPartition, targetBrokerId, replicas); + + // Start the replica move, but throttle it to be very slow so that it can't complete + // before our next checks happen. + long logDirThrottle = 1L; + runExecuteAssignment(cluster.adminClient, false, reassignment.json, + -1L, logDirThrottle); + + // Check the output of --verify + waitForVerifyAssignment(cluster.adminClient, reassignment.json, true, + new VerifyAssignmentResult(Collections.singletonMap( + topicPartition, new PartitionReassignmentState(Arrays.asList(0, 1, 2), Arrays.asList(0, 1, 2), true) + ), false, Collections.singletonMap( + new TopicPartitionReplica(topicPartition.topic(), topicPartition.partition(), 0), + new ActiveMoveState(reassignment.currentDir, reassignment.targetDir, reassignment.targetDir) + ), true)); + waitForLogDirThrottle(Collections.singleton(0), logDirThrottle); + + // Remove the throttle + cluster.adminClient.incrementalAlterConfigs(Collections.singletonMap( + new ConfigResource(ConfigResource.Type.BROKER, "0"), + Collections.singletonList(new AlterConfigOp( + new ConfigEntry(ReassignPartitionsCommand.BROKER_LEVEL_LOG_DIR_THROTTLE, ""), AlterConfigOp.OpType.DELETE)))) + .all().get(); + waitForBrokerLevelThrottles(unthrottledBrokerConfigs); + + // Wait for the directory movement to complete. + waitForVerifyAssignment(cluster.adminClient, reassignment.json, true, + new VerifyAssignmentResult(Collections.singletonMap( + topicPartition, new PartitionReassignmentState(Arrays.asList(0, 1, 2), Arrays.asList(0, 1, 2), true) + ), false, Collections.singletonMap( + new TopicPartitionReplica(topicPartition.topic(), topicPartition.partition(), 0), + new CompletedMoveState(reassignment.targetDir) + ), false)); + + BrokerDirs info1 = new BrokerDirs(cluster.adminClient.describeLogDirs(IntStream.range(0, 4).boxed().collect(Collectors.toList())), 0); + assertEquals(reassignment.targetDir, info1.curLogDirs.getOrDefault(topicPartition, "")); + } + + @ParameterizedTest(name = "{displayName}.quorum={0}") + @ValueSource(strings = "zk") // JBOD not yet implemented for KRaft + public void testAlterLogDirReassignmentThrottle(String quorum) throws Exception { + TopicPartition topicPartition = new TopicPartition("foo", 0); + + cluster = new ReassignPartitionsTestCluster(Collections.emptyMap(), Collections.emptyMap()); + cluster.setup(); + cluster.produceMessages(topicPartition.topic(), topicPartition.partition(), 700); + + int targetBrokerId = 0; + List replicas = Arrays.asList(0, 1, 2); + LogDirReassignment reassignment = buildLogDirReassignment(topicPartition, targetBrokerId, replicas); + + // Start the replica move with a low throttle so it does not complete + long initialLogDirThrottle = 1L; + runExecuteAssignment(cluster.adminClient, false, reassignment.json, + -1L, initialLogDirThrottle); + waitForLogDirThrottle(new HashSet<>(Collections.singletonList(0)), initialLogDirThrottle); + + // Now increase the throttle and verify that the log dir movement completes + long updatedLogDirThrottle = 3000000L; + runExecuteAssignment(cluster.adminClient, true, reassignment.json, + -1L, updatedLogDirThrottle); + waitForLogDirThrottle(Collections.singleton(0), updatedLogDirThrottle); + + waitForVerifyAssignment(cluster.adminClient, reassignment.json, true, + new VerifyAssignmentResult(Collections.singletonMap( + topicPartition, new PartitionReassignmentState(Arrays.asList(0, 1, 2), Arrays.asList(0, 1, 2), true) + ), false, Collections.singletonMap( + new TopicPartitionReplica(topicPartition.topic(), topicPartition.partition(), targetBrokerId), + new CompletedMoveState(reassignment.targetDir) + ), false)); } class LogDirReassignment { @@ -257,9 +690,9 @@ private void runExecuteAssignment(Admin adminClient, interBrokerThrottle, replicaAlterLogDirsThrottle, 10000L, Time.SYSTEM); } - private void runCancelAssignment(Admin adminClient, String jsonString, Boolean preserveThrottles) throws ExecutionException, InterruptedException, JsonProcessingException { + private Tuple, Set> runCancelAssignment(Admin adminClient, String jsonString, Boolean preserveThrottles) throws ExecutionException, InterruptedException, JsonProcessingException { System.out.println("==> cancelAssignment(adminClient, jsonString=" + jsonString); - cancelAssignment(adminClient, jsonString, preserveThrottles, 10000L, Time.SYSTEM); + return cancelAssignment(adminClient, jsonString, preserveThrottles, 10000L, Time.SYSTEM); } class BrokerDirs { @@ -414,7 +847,11 @@ public void close() throws IOException { } } - private static Seq toSeq(List list) { - return JavaConverters.asScalaIteratorConverter(list.iterator()).asScala().toSeq(); + private static Seq toSeq(Collection col) { + return JavaConverters.asScalaIteratorConverter(col.iterator()).asScala().toSeq(); + } + + private static scala.collection.immutable.Set toSet(Collection set) { + return JavaConverters.asScalaIteratorConverter(set.iterator()).asScala().toSet(); } } From 741e5788f4f0894f2318f1c356ff514a38ff6cba Mon Sep 17 00:00:00 2001 From: nizhikov Date: Wed, 2 Aug 2023 01:48:56 +0300 Subject: [PATCH 24/78] KAFKA-14595 ReassignPartitionsIntegrationTest rewritten in java --- .../ReassignPartitionsIntegrationTest.scala | 740 ------------------ .../reassign/ReassignPartitionsCommand.java | 2 +- .../reassign/VerifyAssignmentResult.java | 14 + .../ReassignPartitionsIntegrationTest.java | 11 +- 4 files changed, 21 insertions(+), 746 deletions(-) delete mode 100644 core/src/test/scala/integration/kafka/admin/ReassignPartitionsIntegrationTest.scala diff --git a/core/src/test/scala/integration/kafka/admin/ReassignPartitionsIntegrationTest.scala b/core/src/test/scala/integration/kafka/admin/ReassignPartitionsIntegrationTest.scala deleted file mode 100644 index d3a04da4d1344..0000000000000 --- a/core/src/test/scala/integration/kafka/admin/ReassignPartitionsIntegrationTest.scala +++ /dev/null @@ -1,740 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package kafka.admin - -import kafka.admin.ReassignPartitionsCommand._ -import kafka.server._ -import kafka.utils.Implicits._ -import kafka.utils.{TestInfoUtils, TestUtils} -import org.apache.kafka.clients.admin._ -import org.apache.kafka.clients.producer.ProducerRecord -import org.apache.kafka.common.config.ConfigResource -import org.apache.kafka.common.utils.Utils -import org.apache.kafka.common.{TopicPartition, TopicPartitionReplica} -import org.apache.kafka.server.common.MetadataVersion.IBP_2_7_IV1 -import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertTrue} -import org.junit.jupiter.api.{AfterEach, Timeout} -import org.junit.jupiter.params.ParameterizedTest -import org.junit.jupiter.params.provider.ValueSource - -import java.io.Closeable -import java.util.{Collections, HashMap, List} -import scala.collection.{Map, Seq, mutable} -import scala.jdk.CollectionConverters._ - -@Timeout(300) -class ReassignPartitionsIntegrationTest extends QuorumTestHarness { - - var cluster: ReassignPartitionsTestCluster = _ - - @AfterEach - override def tearDown(): Unit = { - Utils.closeQuietly(cluster, "ReassignPartitionsTestCluster") - super.tearDown() - } - - val unthrottledBrokerConfigs = - 0.to(4).map { brokerId => - brokerId -> brokerLevelThrottles.map(throttle => (throttle, -1L)).toMap - }.toMap - - - @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumName) - @ValueSource(strings = Array("zk", "kraft")) - def testReassignment(quorum: String): Unit = { - cluster = new ReassignPartitionsTestCluster() - cluster.setup() - executeAndVerifyReassignment() - } - - @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumName) - @ValueSource(strings = Array("zk")) // Note: KRaft requires AlterPartition - def testReassignmentWithAlterPartitionDisabled(quorum: String): Unit = { - // Test reassignment when the IBP is on an older version which does not use - // the `AlterPartition` API. In this case, the controller will register individual - // watches for each reassigning partition so that the reassignment can be - // completed as soon as the ISR is expanded. - val configOverrides = Map(KafkaConfig.InterBrokerProtocolVersionProp -> IBP_2_7_IV1.version) - cluster = new ReassignPartitionsTestCluster(configOverrides = configOverrides) - cluster.setup() - executeAndVerifyReassignment() - } - - @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumName) - @ValueSource(strings = Array("zk")) // Note: KRaft requires AlterPartition - def testReassignmentCompletionDuringPartialUpgrade(quorum: String): Unit = { - // Test reassignment during a partial upgrade when some brokers are relying on - // `AlterPartition` and some rely on the old notification logic through Zookeeper. - // In this test case, broker 0 starts up first on the latest IBP and is typically - // elected as controller. The three remaining brokers start up on the older IBP. - // We want to ensure that reassignment can still complete through the ISR change - // notification path even though the controller expects `AlterPartition`. - - // Override change notification settings so that test is not delayed by ISR - // change notification delay - ZkAlterPartitionManager.DefaultIsrPropagationConfig = IsrChangePropagationConfig( - checkIntervalMs = 500, - lingerMs = 100, - maxDelayMs = 500 - ) - - val oldIbpConfig = Map(KafkaConfig.InterBrokerProtocolVersionProp -> IBP_2_7_IV1.version) - val brokerConfigOverrides = Map(1 -> oldIbpConfig, 2 -> oldIbpConfig, 3 -> oldIbpConfig) - - cluster = new ReassignPartitionsTestCluster(brokerConfigOverrides = brokerConfigOverrides) - cluster.setup() - - executeAndVerifyReassignment() - } - - private def executeAndVerifyReassignment(): Unit = { - val assignment = """{"version":1,"partitions":""" + - """[{"topic":"foo","partition":0,"replicas":[0,1,3],"log_dirs":["any","any","any"]},""" + - """{"topic":"bar","partition":0,"replicas":[3,2,0],"log_dirs":["any","any","any"]}""" + - """]}""" - - val foo0 = new TopicPartition("foo", 0) - val bar0 = new TopicPartition("bar", 0) - - // Check that the assignment has not yet been started yet. - val initialAssignment = Map( - foo0 -> PartitionReassignmentState(Seq(0, 1, 2), Seq(0, 1, 3), true), - bar0 -> PartitionReassignmentState(Seq(3, 2, 1), Seq(3, 2, 0), true) - ) - waitForVerifyAssignment(cluster.adminClient, assignment, false, - VerifyAssignmentResult(initialAssignment)) - - // Execute the assignment - runExecuteAssignment(cluster.adminClient, false, assignment, -1L, -1L) - assertEquals(unthrottledBrokerConfigs, describeBrokerLevelThrottles(unthrottledBrokerConfigs.keySet.toSeq)) - val finalAssignment = Map( - foo0 -> PartitionReassignmentState(Seq(0, 1, 3), Seq(0, 1, 3), true), - bar0 -> PartitionReassignmentState(Seq(3, 2, 0), Seq(3, 2, 0), true) - ) - - val verifyAssignmentResult = runVerifyAssignment(cluster.adminClient, assignment, false) - assertFalse(verifyAssignmentResult.movesOngoing) - - // Wait for the assignment to complete - waitForVerifyAssignment(cluster.adminClient, assignment, false, - VerifyAssignmentResult(finalAssignment)) - - assertEquals(unthrottledBrokerConfigs, - describeBrokerLevelThrottles(unthrottledBrokerConfigs.keySet.toSeq)) - - // Verify that partitions are removed from brokers no longer assigned - verifyReplicaDeleted(topicPartition = foo0, replicaId = 2) - verifyReplicaDeleted(topicPartition = bar0, replicaId = 1) - } - - @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumName) - @ValueSource(strings = Array("zk", "kraft")) - def testHighWaterMarkAfterPartitionReassignment(quorum: String): Unit = { - cluster = new ReassignPartitionsTestCluster() - cluster.setup() - val assignment = """{"version":1,"partitions":""" + - """[{"topic":"foo","partition":0,"replicas":[3,1,2],"log_dirs":["any","any","any"]}""" + - """]}""" - - // Set the high water mark of foo-0 to 123 on its leader. - val part = new TopicPartition("foo", 0) - cluster.servers(0).replicaManager.logManager.truncateFullyAndStartAt(part, 123L, false) - - // Execute the assignment - runExecuteAssignment(cluster.adminClient, false, assignment, -1L, -1L) - val finalAssignment = Map(part -> - PartitionReassignmentState(Seq(3, 1, 2), Seq(3, 1, 2), true)) - - // Wait for the assignment to complete - waitForVerifyAssignment(cluster.adminClient, assignment, false, - VerifyAssignmentResult(finalAssignment)) - - TestUtils.waitUntilTrue(() => { - cluster.servers(3).replicaManager.onlinePartition(part). - flatMap(_.leaderLogIfLocal).isDefined - }, "broker 3 should be the new leader", pause = 10L) - assertEquals(123L, cluster.servers(3).replicaManager.localLogOrException(part).highWatermark, - s"Expected broker 3 to have the correct high water mark for the partition.") - } - - @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumName) - @ValueSource(strings = Array("zk", "kraft")) - def testAlterReassignmentThrottle(quorum: String): Unit = { - cluster = new ReassignPartitionsTestCluster() - cluster.setup() - cluster.produceMessages("foo", 0, 50) - cluster.produceMessages("baz", 2, 60) - val assignment = """{"version":1,"partitions": - [{"topic":"foo","partition":0,"replicas":[0,3,2],"log_dirs":["any","any","any"]}, - {"topic":"baz","partition":2,"replicas":[3,2,1],"log_dirs":["any","any","any"]} - ]}""" - - // Execute the assignment with a low throttle - val initialThrottle = 1L - runExecuteAssignment(cluster.adminClient, false, assignment, initialThrottle, -1L) - waitForInterBrokerThrottle(Set(0, 1, 2, 3), initialThrottle) - - // Now update the throttle and verify the reassignment completes - val updatedThrottle = 300000L - runExecuteAssignment(cluster.adminClient, additional = true, assignment, updatedThrottle, -1L) - waitForInterBrokerThrottle(Set(0, 1, 2, 3), updatedThrottle) - - val finalAssignment = Map( - new TopicPartition("foo", 0) -> - PartitionReassignmentState(Seq(0, 3, 2), Seq(0, 3, 2), true), - new TopicPartition("baz", 2) -> - PartitionReassignmentState(Seq(3, 2, 1), Seq(3, 2, 1), true)) - - // Now remove the throttles. - waitForVerifyAssignment(cluster.adminClient, assignment, false, - VerifyAssignmentResult(finalAssignment)) - waitForBrokerLevelThrottles(unthrottledBrokerConfigs) - } - - /** - * Test running a reassignment with the interBrokerThrottle set. - */ - @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumName) - @ValueSource(strings = Array("zk", "kraft")) - def testThrottledReassignment(quorum: String): Unit = { - cluster = new ReassignPartitionsTestCluster() - cluster.setup() - cluster.produceMessages("foo", 0, 50) - cluster.produceMessages("baz", 2, 60) - val assignment = """{"version":1,"partitions":""" + - """[{"topic":"foo","partition":0,"replicas":[0,3,2],"log_dirs":["any","any","any"]},""" + - """{"topic":"baz","partition":2,"replicas":[3,2,1],"log_dirs":["any","any","any"]}""" + - """]}""" - - // Check that the assignment has not yet been started yet. - val initialAssignment = Map( - new TopicPartition("foo", 0) -> - PartitionReassignmentState(Seq(0, 1, 2), Seq(0, 3, 2), true), - new TopicPartition("baz", 2) -> - PartitionReassignmentState(Seq(0, 2, 1), Seq(3, 2, 1), true)) - assertEquals(VerifyAssignmentResult(initialAssignment), runVerifyAssignment(cluster.adminClient, assignment, false)) - assertEquals(unthrottledBrokerConfigs, describeBrokerLevelThrottles(unthrottledBrokerConfigs.keySet.toSeq)) - - // Execute the assignment - val interBrokerThrottle = 300000L - runExecuteAssignment(cluster.adminClient, false, assignment, interBrokerThrottle, -1L) - waitForInterBrokerThrottle(Set(0, 1, 2, 3), interBrokerThrottle) - - val finalAssignment = Map( - new TopicPartition("foo", 0) -> - PartitionReassignmentState(Seq(0, 3, 2), Seq(0, 3, 2), true), - new TopicPartition("baz", 2) -> - PartitionReassignmentState(Seq(3, 2, 1), Seq(3, 2, 1), true)) - - // Wait for the assignment to complete - TestUtils.waitUntilTrue( - () => { - // Check the reassignment status. - val result = runVerifyAssignment(cluster.adminClient, assignment, true) - if (!result.partsOngoing) { - true - } else { - assertFalse(result.partStates.forall(_._2.done), s"Expected at least one partition reassignment to be ongoing when result = $result") - assertEquals(Seq(0, 3, 2), result.partStates(new TopicPartition("foo", 0)).targetReplicas) - assertEquals(Seq(3, 2, 1), result.partStates(new TopicPartition("baz", 2)).targetReplicas) - logger.info(s"Current result: ${result}") - waitForInterBrokerThrottle(Set(0, 1, 2, 3), interBrokerThrottle) - false - } - }, "Expected reassignment to complete.") - waitForVerifyAssignment(cluster.adminClient, assignment, true, - VerifyAssignmentResult(finalAssignment)) - // The throttles should still have been preserved, since we ran with --preserve-throttles - waitForInterBrokerThrottle(Set(0, 1, 2, 3), interBrokerThrottle) - // Now remove the throttles. - waitForVerifyAssignment(cluster.adminClient, assignment, false, - VerifyAssignmentResult(finalAssignment)) - waitForBrokerLevelThrottles(unthrottledBrokerConfigs) - } - - @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumName) - @ValueSource(strings = Array("zk", "kraft")) - def testProduceAndConsumeWithReassignmentInProgress(quorum: String): Unit = { - cluster = new ReassignPartitionsTestCluster() - cluster.setup() - cluster.produceMessages("baz", 2, 60) - val assignment = """{"version":1,"partitions":""" + - """[{"topic":"baz","partition":2,"replicas":[3,2,1],"log_dirs":["any","any","any"]}""" + - """]}""" - runExecuteAssignment(cluster.adminClient, false, assignment, 300L, -1L) - cluster.produceMessages("baz", 2, 100) - val consumer = TestUtils.createConsumer(cluster.brokerList) - val part = new TopicPartition("baz", 2) - try { - consumer.assign(Seq(part).asJava) - TestUtils.pollUntilAtLeastNumRecords(consumer, numRecords = 100) - } finally { - consumer.close() - } - TestUtils.removeReplicationThrottleForPartitions(cluster.adminClient, Seq(0,1,2,3), Set(part)) - val finalAssignment = Map(part -> - PartitionReassignmentState(Seq(3, 2, 1), Seq(3, 2, 1), true)) - waitForVerifyAssignment(cluster.adminClient, assignment, false, - VerifyAssignmentResult(finalAssignment)) - } - - /** - * Test running a reassignment and then cancelling it. - */ - @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumName) - @ValueSource(strings = Array("zk", "kraft")) - def testCancellation(quorum: String): Unit = { - val foo0 = new TopicPartition("foo", 0) - val baz1 = new TopicPartition("baz", 1) - - cluster = new ReassignPartitionsTestCluster() - cluster.setup() - cluster.produceMessages(foo0.topic, foo0.partition, 200) - cluster.produceMessages(baz1.topic, baz1.partition, 200) - val assignment = """{"version":1,"partitions":""" + - """[{"topic":"foo","partition":0,"replicas":[0,1,3],"log_dirs":["any","any","any"]},""" + - """{"topic":"baz","partition":1,"replicas":[0,2,3],"log_dirs":["any","any","any"]}""" + - """]}""" - assertEquals(unthrottledBrokerConfigs, - describeBrokerLevelThrottles(unthrottledBrokerConfigs.keySet.toSeq)) - val interBrokerThrottle = 1L - runExecuteAssignment(cluster.adminClient, false, assignment, interBrokerThrottle, -1L) - waitForInterBrokerThrottle(Set(0, 1, 2, 3), interBrokerThrottle) - - // Verify that the reassignment is running. The very low throttle should keep it - // from completing before this runs. - waitForVerifyAssignment(cluster.adminClient, assignment, true, - VerifyAssignmentResult(Map( - foo0 -> PartitionReassignmentState(Seq(0, 1, 3, 2), Seq(0, 1, 3), false), - baz1 -> PartitionReassignmentState(Seq(0, 2, 3, 1), Seq(0, 2, 3), false)), - true, Map(), false)) - // Cancel the reassignment. - assertEquals((Set(foo0, baz1), Set()), runCancelAssignment(cluster.adminClient, assignment, true)) - // Broker throttles are still active because we passed --preserve-throttles - waitForInterBrokerThrottle(Set(0, 1, 2, 3), interBrokerThrottle) - // Cancelling the reassignment again should reveal nothing to cancel. - assertEquals((Set(), Set()), runCancelAssignment(cluster.adminClient, assignment, false)) - // This time, the broker throttles were removed. - waitForBrokerLevelThrottles(unthrottledBrokerConfigs) - // Verify that there are no ongoing reassignments. - assertFalse(runVerifyAssignment(cluster.adminClient, assignment, false).partsOngoing) - // Verify that the partition is removed from cancelled replicas - verifyReplicaDeleted(topicPartition = foo0, replicaId = 3) - verifyReplicaDeleted(topicPartition = baz1, replicaId = 3) - } - - @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumName) - @ValueSource(strings = Array("zk", "kraft")) - def testCancellationWithAddingReplicaInIsr(quorum: String): Unit = { - val foo0 = new TopicPartition("foo", 0) - - cluster = new ReassignPartitionsTestCluster() - cluster.setup() - cluster.produceMessages(foo0.topic, foo0.partition, 200) - - // The reassignment will bring replicas 3 and 4 into the replica set and remove 1 and 2. - val assignment = """{"version":1,"partitions":""" + - """[{"topic":"foo","partition":0,"replicas":[0,3,4],"log_dirs":["any","any","any"]}""" + - """]}""" - - // We will throttle replica 4 so that only replica 3 joins the ISR - TestUtils.setReplicationThrottleForPartitions( - cluster.adminClient, - brokerIds = Seq(4), - partitions = Set(foo0), - throttleBytes = 1 - ) - - // Execute the assignment and wait for replica 3 (only) to join the ISR - runExecuteAssignment( - cluster.adminClient, - additional = false, - reassignmentJson = assignment - ) - TestUtils.waitUntilTrue( - () => TestUtils.currentIsr(cluster.adminClient, foo0) == Set(0, 1, 2, 3), - msg = "Timed out while waiting for replica 3 to join the ISR" - ) - - // Now cancel the assignment and verify that the partition is removed from cancelled replicas - assertEquals((Set(foo0), Set()), runCancelAssignment(cluster.adminClient, assignment, preserveThrottles = true)) - verifyReplicaDeleted(topicPartition = foo0, replicaId = 3) - verifyReplicaDeleted(topicPartition = foo0, replicaId = 4) - } - - private def verifyReplicaDeleted( - topicPartition: TopicPartition, - replicaId: Int - ): Unit = { - def isReplicaStoppedAndDeleted(): Boolean = { - val server = cluster.servers(replicaId) - val partition = server.replicaManager.getPartition(topicPartition) - val log = server.logManager.getLog(topicPartition) - partition == HostedPartition.None && log.isEmpty - } - TestUtils.waitUntilTrue(isReplicaStoppedAndDeleted, - msg = s"Timed out waiting for replica $replicaId of $topicPartition to be deleted") - } - - private def waitForLogDirThrottle(throttledBrokers: Set[Int], logDirThrottle: Long): Unit = { - val throttledConfigMap = Map[String, Long]( - brokerLevelLeaderThrottle -> -1, - brokerLevelFollowerThrottle -> -1, - brokerLevelLogDirThrottle -> logDirThrottle) - waitForBrokerThrottles(throttledBrokers, throttledConfigMap) - } - - private def waitForInterBrokerThrottle(throttledBrokers: Set[Int], interBrokerThrottle: Long): Unit = { - val throttledConfigMap = Map[String, Long]( - brokerLevelLeaderThrottle -> interBrokerThrottle, - brokerLevelFollowerThrottle -> interBrokerThrottle, - brokerLevelLogDirThrottle -> -1L) - waitForBrokerThrottles(throttledBrokers, throttledConfigMap) - } - - private def waitForBrokerThrottles(throttledBrokers: Set[Int], throttleConfig: Map[String, Long]): Unit = { - val throttledBrokerConfigs = unthrottledBrokerConfigs.map { case (brokerId, unthrottledConfig) => - val expectedThrottleConfig = if (throttledBrokers.contains(brokerId)) { - throttleConfig - } else { - unthrottledConfig - } - brokerId -> expectedThrottleConfig - } - waitForBrokerLevelThrottles(throttledBrokerConfigs) - } - - private def waitForBrokerLevelThrottles(targetThrottles: Map[Int, Map[String, Long]]): Unit = { - var curThrottles: Map[Int, Map[String, Long]] = Map.empty - TestUtils.waitUntilTrue(() => { - curThrottles = describeBrokerLevelThrottles(targetThrottles.keySet.toSeq) - targetThrottles.equals(curThrottles) - }, s"timed out waiting for broker throttle to become ${targetThrottles}. " + - s"Latest throttles were ${curThrottles}", pause = 25) - } - - /** - * Describe the broker-level throttles in the cluster. - * - * @return A map whose keys are broker IDs and whose values are throttle - * information. The nested maps are keyed on throttle name. - */ - private def describeBrokerLevelThrottles(brokerIds: Seq[Int]): Map[Int, Map[String, Long]] = { - brokerIds.map { brokerId => - val brokerResource = new ConfigResource(ConfigResource.Type.BROKER, brokerId.toString) - val brokerConfigs = cluster.adminClient.describeConfigs(Collections.singleton(brokerResource)).values() - .get(brokerResource) - .get() - - val throttles = brokerLevelThrottles.map { throttleName => - val configValue = Option(brokerConfigs.get(throttleName)) - .map(_.value) - .getOrElse("-1") - (throttleName, configValue.toLong) - }.toMap - brokerId -> throttles - }.toMap - } - - /** - * Test moving partitions between directories. - */ - @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumName) - @ValueSource(strings = Array("zk")) // JBOD not yet implemented for KRaft - def testLogDirReassignment(quorum: String): Unit = { - val topicPartition = new TopicPartition("foo", 0) - - cluster = new ReassignPartitionsTestCluster() - cluster.setup() - cluster.produceMessages(topicPartition.topic, topicPartition.partition, 700) - - val targetBrokerId = 0 - val replicas = Seq(0, 1, 2) - val reassignment = buildLogDirReassignment(topicPartition, targetBrokerId, replicas) - - // Start the replica move, but throttle it to be very slow so that it can't complete - // before our next checks happen. - val logDirThrottle = 1L - runExecuteAssignment(cluster.adminClient, additional = false, reassignment.json, - interBrokerThrottle = -1L, logDirThrottle) - - // Check the output of --verify - waitForVerifyAssignment(cluster.adminClient, reassignment.json, true, - VerifyAssignmentResult(Map( - topicPartition -> PartitionReassignmentState(Seq(0, 1, 2), Seq(0, 1, 2), true) - ), false, Map( - new TopicPartitionReplica(topicPartition.topic, topicPartition.partition, 0) -> - ActiveMoveState(reassignment.currentDir, reassignment.targetDir, reassignment.targetDir) - ), true)) - waitForLogDirThrottle(Set(0), logDirThrottle) - - // Remove the throttle - cluster.adminClient.incrementalAlterConfigs(Collections.singletonMap( - new ConfigResource(ConfigResource.Type.BROKER, "0"), - Collections.singletonList(new AlterConfigOp( - new ConfigEntry(brokerLevelLogDirThrottle, ""), AlterConfigOp.OpType.DELETE)))) - .all().get() - waitForBrokerLevelThrottles(unthrottledBrokerConfigs) - - // Wait for the directory movement to complete. - waitForVerifyAssignment(cluster.adminClient, reassignment.json, true, - VerifyAssignmentResult(Map( - topicPartition -> PartitionReassignmentState(Seq(0, 1, 2), Seq(0, 1, 2), true) - ), false, Map( - new TopicPartitionReplica(topicPartition.topic, topicPartition.partition, 0) -> - CompletedMoveState(reassignment.targetDir) - ), false)) - - val info1 = new BrokerDirs(cluster.adminClient.describeLogDirs(0.to(4). - map(_.asInstanceOf[Integer]).asJavaCollection), 0) - assertEquals(reassignment.targetDir, info1.curLogDirs.getOrElse(topicPartition, "")) - } - - @ParameterizedTest(name = TestInfoUtils.TestWithParameterizedQuorumName) - @ValueSource(strings = Array("zk")) // JBOD not yet implemented for KRaft - def testAlterLogDirReassignmentThrottle(quorum: String): Unit = { - val topicPartition = new TopicPartition("foo", 0) - - cluster = new ReassignPartitionsTestCluster() - cluster.setup() - cluster.produceMessages(topicPartition.topic, topicPartition.partition, 700) - - val targetBrokerId = 0 - val replicas = Seq(0, 1, 2) - val reassignment = buildLogDirReassignment(topicPartition, targetBrokerId, replicas) - - // Start the replica move with a low throttle so it does not complete - val initialLogDirThrottle = 1L - runExecuteAssignment(cluster.adminClient, false, reassignment.json, - interBrokerThrottle = -1L, initialLogDirThrottle) - waitForLogDirThrottle(Set(0), initialLogDirThrottle) - - // Now increase the throttle and verify that the log dir movement completes - val updatedLogDirThrottle = 3000000L - runExecuteAssignment(cluster.adminClient, additional = true, reassignment.json, - interBrokerThrottle = -1L, replicaAlterLogDirsThrottle = updatedLogDirThrottle) - waitForLogDirThrottle(Set(0), updatedLogDirThrottle) - - waitForVerifyAssignment(cluster.adminClient, reassignment.json, true, - VerifyAssignmentResult(Map( - topicPartition -> PartitionReassignmentState(Seq(0, 1, 2), Seq(0, 1, 2), true) - ), false, Map( - new TopicPartitionReplica(topicPartition.topic, topicPartition.partition, targetBrokerId) -> - CompletedMoveState(reassignment.targetDir) - ), false)) - } - - case class LogDirReassignment(json: String, currentDir: String, targetDir: String) - - private def buildLogDirReassignment(topicPartition: TopicPartition, - brokerId: Int, - replicas: Seq[Int]): LogDirReassignment = { - - val describeLogDirsResult = cluster.adminClient.describeLogDirs( - 0.to(4).map(_.asInstanceOf[Integer]).asJavaCollection) - - val logDirInfo = new BrokerDirs(describeLogDirsResult, brokerId) - assertTrue(logDirInfo.futureLogDirs.isEmpty) - - val currentDir = logDirInfo.curLogDirs(topicPartition) - val newDir = logDirInfo.logDirs.find(!_.equals(currentDir)).get - - val logDirs = replicas.map { replicaId => - if (replicaId == brokerId) - s""""$newDir"""" - else - "\"any\"" - } - - val reassignmentJson = - s""" - | { "version": 1, - | "partitions": [ - | { - | "topic": "${topicPartition.topic}", - | "partition": ${topicPartition.partition}, - | "replicas": [${replicas.mkString(",")}], - | "log_dirs": [${logDirs.mkString(",")}] - | } - | ] - | } - |""".stripMargin - - LogDirReassignment(reassignmentJson, currentDir = currentDir, targetDir = newDir) - } - - private def runVerifyAssignment(adminClient: Admin, jsonString: String, - preserveThrottles: Boolean) = { - println(s"==> verifyAssignment(adminClient, jsonString=${jsonString})") - verifyAssignment(adminClient, jsonString, preserveThrottles) - } - - private def waitForVerifyAssignment(adminClient: Admin, - jsonString: String, - preserveThrottles: Boolean, - expectedResult: VerifyAssignmentResult): Unit = { - var latestResult: VerifyAssignmentResult = null - TestUtils.waitUntilTrue( - () => { - latestResult = runVerifyAssignment(adminClient, jsonString, preserveThrottles) - expectedResult.equals(latestResult) - }, s"Timed out waiting for verifyAssignment result ${expectedResult}. " + - s"The latest result was ${latestResult}", pause = 10L) - } - - private def runExecuteAssignment(adminClient: Admin, - additional: Boolean, - reassignmentJson: String, - interBrokerThrottle: Long = -1, - replicaAlterLogDirsThrottle: Long = -1) = { - println(s"==> executeAssignment(adminClient, additional=${additional}, " + - s"reassignmentJson=${reassignmentJson}, " + - s"interBrokerThrottle=${interBrokerThrottle}, " + - s"replicaAlterLogDirsThrottle=${replicaAlterLogDirsThrottle}))") - executeAssignment(adminClient, additional, reassignmentJson, - interBrokerThrottle, replicaAlterLogDirsThrottle) - } - - private def runCancelAssignment(adminClient: Admin, jsonString: String, - preserveThrottles: Boolean) = { - println(s"==> cancelAssignment(adminClient, jsonString=${jsonString})") - cancelAssignment(adminClient, jsonString, preserveThrottles) - } - - class BrokerDirs(result: DescribeLogDirsResult, val brokerId: Int) { - val logDirs = new mutable.HashSet[String] - val curLogDirs = new mutable.HashMap[TopicPartition, String] - val futureLogDirs = new mutable.HashMap[TopicPartition, String] - result.descriptions.get(brokerId).get().forEach { - case (logDirName, logDirInfo) => { - logDirs.add(logDirName) - logDirInfo.replicaInfos.forEach { - case (part, info) => - if (info.isFuture) { - futureLogDirs.put(part, logDirName) - } else { - curLogDirs.put(part, logDirName) - } - } - } - } - } - - class ReassignPartitionsTestCluster( - configOverrides: Map[String, String] = Map.empty, - brokerConfigOverrides: Map[Int, Map[String, String]] = Map.empty - ) extends Closeable { - val brokers = Map( - 0 -> "rack0", - 1 -> "rack0", - 2 -> "rack1", - 3 -> "rack1", - 4 -> "rack1" - ) - - val topics = Map( - "foo" -> Seq(Seq(0, 1, 2), Seq(1, 2, 3)), - "bar" -> Seq(Seq(3, 2, 1)), - "baz" -> Seq(Seq(1, 0, 2), Seq(2, 0, 1), Seq(0, 2, 1)) - ) - - val brokerConfigs = brokers.map { - case (brokerId, rack) => - val config = TestUtils.createBrokerConfig( - nodeId = brokerId, - zkConnect = zkConnectOrNull, - rack = Some(rack), - enableControlledShutdown = false, // shorten test time - logDirCount = 3) - // shorter backoff to reduce test durations when no active partitions are eligible for fetching due to throttling - config.setProperty(KafkaConfig.ReplicaFetchBackoffMsProp, "100") - // Don't move partition leaders automatically. - config.setProperty(KafkaConfig.AutoLeaderRebalanceEnableProp, "false") - config.setProperty(KafkaConfig.ReplicaLagTimeMaxMsProp, "1000") - configOverrides.forKeyValue(config.setProperty) - - brokerConfigOverrides.get(brokerId).foreach { overrides => - overrides.forKeyValue(config.setProperty) - } - - new KafkaConfig(config) - }.toBuffer - - var servers = new mutable.ArrayBuffer[KafkaBroker] - - var brokerList: String = _ - - var adminClient: Admin = _ - - def setup(): Unit = { - createServers() - createTopics() - } - - def createServers(): Unit = { - brokers.keySet.foreach { brokerId => - servers += createBroker(brokerConfigs(brokerId)) - } - } - - def createTopics(): Unit = { - TestUtils.waitUntilBrokerMetadataIsPropagated(servers) - brokerList = TestUtils.plaintextBootstrapServers(servers) - adminClient = Admin.create(Map[String, Object]( - AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG -> brokerList - ).asJava) - adminClient.createTopics(topics.map { - case (topicName, parts) => - val partMap = new HashMap[Integer, List[Integer]]() - parts.zipWithIndex.foreach { - case (part, index) => partMap.put(index, part.map(Integer.valueOf).asJava) - } - new NewTopic(topicName, partMap) - }.toList.asJava).all().get() - topics.foreach { - case (topicName, parts) => - TestUtils.waitForAllPartitionsMetadata(servers, topicName, parts.size) - } - - if (isKRaftTest()) { - TestUtils.ensureConsistentKRaftMetadata( - cluster.servers, - controllerServer - ) - } - } - - def produceMessages(topic: String, partition: Int, numMessages: Int): Unit = { - val records = (0 until numMessages).map(_ => - new ProducerRecord[Array[Byte], Array[Byte]](topic, partition, - null, new Array[Byte](10000))) - TestUtils.produceMessages(servers, records, -1) - } - - override def close(): Unit = { - brokerList = null - Utils.closeQuietly(adminClient, "adminClient") - adminClient = null - try { - TestUtils.shutdownServers(servers) - } finally { - servers.clear() - } - } - } -} diff --git a/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommand.java b/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommand.java index 472aa6acb1306..e128cf179da21 100644 --- a/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommand.java +++ b/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommand.java @@ -395,7 +395,7 @@ private static Tuple, Boolean> verif Map targetReassignments) throws ExecutionException, InterruptedException { Map moveStates = findLogDirMoveStates(adminClient, targetReassignments); System.out.println(replicaMoveStatesToString(moveStates)); - return new Tuple<>(moveStates, moveStates.values().stream().noneMatch(LogDirMoveState::done)); + return new Tuple<>(moveStates, !moveStates.values().stream().allMatch(LogDirMoveState::done)); } /** diff --git a/tools/src/main/java/org/apache/kafka/tools/reassign/VerifyAssignmentResult.java b/tools/src/main/java/org/apache/kafka/tools/reassign/VerifyAssignmentResult.java index 7721066ab0024..c20cfd1029c33 100644 --- a/tools/src/main/java/org/apache/kafka/tools/reassign/VerifyAssignmentResult.java +++ b/tools/src/main/java/org/apache/kafka/tools/reassign/VerifyAssignmentResult.java @@ -22,6 +22,7 @@ import java.util.Collections; import java.util.Map; +import java.util.Objects; /** * A result returned from verifyAssignment. @@ -53,4 +54,17 @@ public VerifyAssignmentResult( this.moveStates = moveStates; this.movesOngoing = movesOngoing; } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + VerifyAssignmentResult that = (VerifyAssignmentResult) o; + return partsOngoing == that.partsOngoing && movesOngoing == that.movesOngoing && Objects.equals(partStates, that.partStates) && Objects.equals(moveStates, that.moveStates); + } + + @Override + public int hashCode() { + return Objects.hash(partStates, partsOngoing, moveStates, movesOngoing); + } } diff --git a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsIntegrationTest.java b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsIntegrationTest.java index c7aa39723e48d..05106563f1c60 100644 --- a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsIntegrationTest.java +++ b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsIntegrationTest.java @@ -68,6 +68,7 @@ import java.util.Properties; import java.util.Set; import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -100,7 +101,7 @@ public void tearDown() { @ParameterizedTest(name = "{displayName}.quorum={0}") @ValueSource(strings = {"zk", "kraft"}) public void testReassignment(String quorum) throws Exception { - ReassignPartitionsTestCluster cluster = new ReassignPartitionsTestCluster(Collections.emptyMap(), Collections.emptyMap()); + cluster = new ReassignPartitionsTestCluster(Collections.emptyMap(), Collections.emptyMap()); cluster.setup(); executeAndVerifyReassignment(); } @@ -482,16 +483,16 @@ private void waitForBrokerThrottles(Collection throttledBrokers, Map> targetThrottles) { - Map>[] curThrottles = new Map[]{Collections.emptyMap()}; + AtomicReference>> curThrottles = new AtomicReference<>(new HashMap<>()); TestUtils.waitUntilTrue(() -> { try { - curThrottles[0] = describeBrokerLevelThrottles(targetThrottles.keySet()); - return targetThrottles.equals(curThrottles[0]); + curThrottles.set(describeBrokerLevelThrottles(targetThrottles.keySet())); + return targetThrottles.equals(curThrottles.get()); } catch (ExecutionException | InterruptedException e) { throw new RuntimeException(e); } }, () -> "timed out waiting for broker throttle to become " + targetThrottles + ". " + - "Latest throttles were " + curThrottles[0], org.apache.kafka.test.TestUtils.DEFAULT_MAX_WAIT_MS, 25); + "Latest throttles were " + curThrottles.get(), org.apache.kafka.test.TestUtils.DEFAULT_MAX_WAIT_MS, 25); } /** From 568f6626ab61bf537c46a976c9453351977683d3 Mon Sep 17 00:00:00 2001 From: nizhikov Date: Wed, 2 Aug 2023 02:22:33 +0300 Subject: [PATCH 25/78] KAFKA-14595 ReassignPartitionsIntegrationTest rewritten in java --- .../ReassignPartitionsIntegrationTest.java | 42 +++++++++---------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsIntegrationTest.java b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsIntegrationTest.java index 05106563f1c60..819c0dc5f111d 100644 --- a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsIntegrationTest.java +++ b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsIntegrationTest.java @@ -1,12 +1,12 @@ /* * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with + * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at + * the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -209,8 +209,8 @@ public void testHighWaterMarkAfterPartitionReassignment(String quorum) throws Ex TestUtils.waitUntilTrue(() -> cluster.servers.get(3).replicaManager().onlinePartition(part). - map(Partition::leaderLogIfLocal).isDefined() - , () -> "broker 3 should be the new leader", org.apache.kafka.test.TestUtils.DEFAULT_MAX_WAIT_MS, 10L); + map(Partition::leaderLogIfLocal).isDefined(), + () -> "broker 3 should be the new leader", org.apache.kafka.test.TestUtils.DEFAULT_MAX_WAIT_MS, 10L); assertEquals(123L, cluster.servers.get(3).replicaManager().localLogOrException(part).highWatermark(), "Expected broker 3 to have the correct high water mark for the partition."); } @@ -223,9 +223,9 @@ public void testAlterReassignmentThrottle(String quorum) throws Exception { cluster.produceMessages("foo", 0, 50); cluster.produceMessages("baz", 2, 60); String assignment = "{\"version\":1,\"partitions\":" + - "[{\"topic\":\"foo\",\"partition\":0,\"replicas\":[0,3,2],\"log_dirs\":[\"any\",\"any\",\"any\"]}," + - "{\"topic\":\"baz\",\"partition\":2,\"replicas\":[3,2,1],\"log_dirs\":[\"any\",\"any\",\"any\"]}" + - "]}"; + "[{\"topic\":\"foo\",\"partition\":0,\"replicas\":[0,3,2],\"log_dirs\":[\"any\",\"any\",\"any\"]}," + + "{\"topic\":\"baz\",\"partition\":2,\"replicas\":[3,2,1],\"log_dirs\":[\"any\",\"any\",\"any\"]}" + + "]}"; // Execute the assignment with a low throttle long initialThrottle = 1L; @@ -346,7 +346,7 @@ public void testProduceAndConsumeWithReassignmentInProgress(String quorum) throw } finally { consumer.close(); } - TestUtils.removeReplicationThrottleForPartitions(cluster.adminClient, toSeq(Arrays.asList(0,1,2,3)), toSet(Collections.singleton(part))); + TestUtils.removeReplicationThrottleForPartitions(cluster.adminClient, toSeq(Arrays.asList(0, 1, 2, 3)), toSet(Collections.singleton(part))); Map finalAssignment = Collections.singletonMap(part, new PartitionReassignmentState(Arrays.asList(3, 2, 1), Arrays.asList(3, 2, 1), true)); waitForVerifyAssignment(cluster.adminClient, assignment, false, @@ -639,16 +639,16 @@ private LogDirReassignment buildLogDirReassignment(TopicPartition topicPartition }).collect(Collectors.toList()); String reassignmentJson = - " { \"version\": 1," + - " \"partitions\": [" + - " {" + - " \"topic\": \"" + topicPartition.topic() + "\"," + - " \"partition\": " + topicPartition.partition() + "," + - " \"replicas\": [" + replicas.stream().map(Object::toString).collect(Collectors.joining(",")) + "]," + - " \"log_dirs\": [" + String.join(",", logDirs) + "]" + - " }" + - " ]" + - " }"; + " { \"version\": 1," + + " \"partitions\": [" + + " {" + + " \"topic\": \"" + topicPartition.topic() + "\"," + + " \"partition\": " + topicPartition.partition() + "," + + " \"replicas\": [" + replicas.stream().map(Object::toString).collect(Collectors.joining(",")) + "]," + + " \"log_dirs\": [" + String.join(",", logDirs) + "]" + + " }" + + " ]" + + " }"; return new LogDirReassignment(reassignmentJson, currentDir, newDir); } @@ -773,7 +773,7 @@ public ReassignPartitionsTestCluster(Map configOverrides, Map { - TestUtils.waitForAllPartitionsMetadata(toSeq(servers), topicName, parts.size()); + TestUtils.waitForAllPartitionsMetadata(toSeq(servers), topicName, parts.size()); }); if (isKRaftTest()) { From 794b57c49d6a0638bfc1363f57f531e0344be210 Mon Sep 17 00:00:00 2001 From: nizhikov Date: Wed, 2 Aug 2023 02:24:35 +0300 Subject: [PATCH 26/78] KAFKA-14595 ReassignPartitionsIntegrationTest rewritten in java --- checkstyle/import-control.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/checkstyle/import-control.xml b/checkstyle/import-control.xml index a334d58ce56ef..6b9f87cb00e10 100644 --- a/checkstyle/import-control.xml +++ b/checkstyle/import-control.xml @@ -316,6 +316,12 @@ + + + + + + From e567030e8fb3c7138f9717b9c6afe9c4a4b07535 Mon Sep 17 00:00:00 2001 From: nizhikov Date: Wed, 2 Aug 2023 02:26:53 +0300 Subject: [PATCH 27/78] KAFKA-14595 ReassignPartitionsIntegrationTest rewritten in java --- .../admin/ReassignPartitionsCommand.scala | 1502 ----------------- .../kafka/ReplicationQuotasTestRig.scala | 341 ---- 2 files changed, 1843 deletions(-) delete mode 100755 core/src/main/scala/kafka/admin/ReassignPartitionsCommand.scala delete mode 100644 core/src/test/scala/other/kafka/ReplicationQuotasTestRig.scala diff --git a/core/src/main/scala/kafka/admin/ReassignPartitionsCommand.scala b/core/src/main/scala/kafka/admin/ReassignPartitionsCommand.scala deleted file mode 100755 index 20a324dcf2c73..0000000000000 --- a/core/src/main/scala/kafka/admin/ReassignPartitionsCommand.scala +++ /dev/null @@ -1,1502 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package kafka.admin - -import java.util -import java.util.Optional -import java.util.concurrent.ExecutionException -import kafka.server.DynamicConfig -import kafka.utils.{CoreUtils, Exit, Json, Logging} -import kafka.utils.Implicits._ -import kafka.utils.json.JsonValue -import org.apache.kafka.admin.{AdminUtils, BrokerMetadata} -import org.apache.kafka.clients.admin.AlterConfigOp.OpType -import org.apache.kafka.clients.admin.{Admin, AdminClientConfig, AlterConfigOp, ConfigEntry, NewPartitionReassignment, PartitionReassignment, TopicDescription} -import org.apache.kafka.common.config.ConfigResource -import org.apache.kafka.common.errors.{ReplicaNotAvailableException, UnknownTopicOrPartitionException} -import org.apache.kafka.common.utils.{Time, Utils} -import org.apache.kafka.common.{KafkaException, KafkaFuture, TopicPartition, TopicPartitionReplica} -import org.apache.kafka.server.common.{AdminCommandFailedException, AdminOperationException} -import org.apache.kafka.server.util.{CommandDefaultOptions, CommandLineUtils} -import org.apache.kafka.storage.internals.log.LogConfig - -import scala.jdk.CollectionConverters._ -import scala.collection.{Map, Seq, mutable} -import scala.math.Ordered.orderingToOrdered - - -object ReassignPartitionsCommand extends Logging { - private[admin] val AnyLogDir = "any" - - val helpText = "This tool helps to move topic partitions between replicas." - - /** - * The earliest version of the partition reassignment JSON. We will default to this - * version if no other version number is given. - */ - private[admin] val EarliestVersion = 1 - - /** - * The earliest version of the JSON for each partition reassignment topic. We will - * default to this version if no other version number is given. - */ - private[admin] val EarliestTopicsJsonVersion = 1 - - // Throttles that are set at the level of an individual broker. - private[admin] val brokerLevelLeaderThrottle = - DynamicConfig.Broker.LeaderReplicationThrottledRateProp - private[admin] val brokerLevelFollowerThrottle = - DynamicConfig.Broker.FollowerReplicationThrottledRateProp - private[admin] val brokerLevelLogDirThrottle = - DynamicConfig.Broker.ReplicaAlterLogDirsIoMaxBytesPerSecondProp - private[admin] val brokerLevelThrottles = Seq( - brokerLevelLeaderThrottle, - brokerLevelFollowerThrottle, - brokerLevelLogDirThrottle - ) - - // Throttles that are set at the level of an individual topic. - private[admin] val topicLevelLeaderThrottle = - LogConfig.LEADER_REPLICATION_THROTTLED_REPLICAS_CONFIG - private[admin] val topicLevelFollowerThrottle = - LogConfig.FOLLOWER_REPLICATION_THROTTLED_REPLICAS_CONFIG - private[admin] val topicLevelThrottles = Seq( - topicLevelLeaderThrottle, - topicLevelFollowerThrottle - ) - - private[admin] val cannotExecuteBecauseOfExistingMessage = "Cannot execute because " + - "there is an existing partition assignment. Use --additional to override this and " + - "create a new partition assignment in addition to the existing one. The --additional " + - "flag can also be used to change the throttle by resubmitting the current reassignment." - - private[admin] val youMustRunVerifyPeriodicallyMessage = "Warning: You must run " + - "--verify periodically, until the reassignment completes, to ensure the throttle " + - "is removed." - - /** - * A map from topic names to partition movements. - */ - type MoveMap = mutable.Map[String, mutable.Map[Int, PartitionMove]] - - /** - * A partition movement. The source and destination brokers may overlap. - * - * @param sources The source brokers. - * @param destinations The destination brokers. - */ - sealed case class PartitionMove(sources: mutable.Set[Int], - destinations: mutable.Set[Int]) { } - - /** - * The state of a partition reassignment. The current replicas and target replicas - * may overlap. - * - * @param currentReplicas The current replicas. - * @param targetReplicas The target replicas. - * @param done True if the reassignment is done. - */ - sealed case class PartitionReassignmentState(currentReplicas: Seq[Int], - targetReplicas: Seq[Int], - done: Boolean) {} - - /** - * The state of a replica log directory movement. - */ - sealed trait LogDirMoveState { - /** - * True if the move is done without errors. - */ - def done: Boolean - } - - /** - * A replica log directory move state where the source log directory is missing. - * - * @param targetLogDir The log directory that we wanted the replica to move to. - */ - sealed case class MissingReplicaMoveState(targetLogDir: String) - extends LogDirMoveState { - override def done = false - } - - /** - * A replica log directory move state where the source replica is missing. - * - * @param targetLogDir The log directory that we wanted the replica to move to. - */ - sealed case class MissingLogDirMoveState(targetLogDir: String) - extends LogDirMoveState { - override def done = false - } - - /** - * A replica log directory move state where the move is in progress. - * - * @param currentLogDir The current log directory. - * @param futureLogDir The log directory that the replica is moving to. - * @param targetLogDir The log directory that we wanted the replica to move to. - */ - sealed case class ActiveMoveState(currentLogDir: String, - targetLogDir: String, - futureLogDir: String) - extends LogDirMoveState { - override def done = false - } - - /** - * A replica log directory move state where there is no move in progress, but we did not - * reach the target log directory. - * - * @param currentLogDir The current log directory. - * @param targetLogDir The log directory that we wanted the replica to move to. - */ - sealed case class CancelledMoveState(currentLogDir: String, - targetLogDir: String) - extends LogDirMoveState { - override def done = true - } - - /** - * The completed replica log directory move state. - * - * @param targetLogDir The log directory that we wanted the replica to move to. - */ - sealed case class CompletedMoveState(targetLogDir: String) - extends LogDirMoveState { - override def done = true - } - - /** - * An exception thrown to indicate that the command has failed, but we don't want to - * print a stack trace. - * - * @param message The message to print out before exiting. A stack trace will not - * be printed. - */ - class TerseReassignmentFailureException(message: String) extends KafkaException(message) { - } - - def main(args: Array[String]): Unit = { - val opts = validateAndParseArgs(args) - var failed = true - var adminClient: Admin = null - - try { - val props = if (opts.options.has(opts.commandConfigOpt)) - Utils.loadProps(opts.options.valueOf(opts.commandConfigOpt)) - else - new util.Properties() - props.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, opts.options.valueOf(opts.bootstrapServerOpt)) - props.putIfAbsent(AdminClientConfig.CLIENT_ID_CONFIG, "reassign-partitions-tool") - adminClient = Admin.create(props) - handleAction(adminClient, opts) - failed = false - } catch { - case e: TerseReassignmentFailureException => - println(e.getMessage) - case e: Throwable => - println("Error: " + e.getMessage) - println(Utils.stackTrace(e)) - } finally { - // It's good to do this after printing any error stack trace. - if (adminClient != null) { - adminClient.close() - } - } - // If the command failed, exit with a non-zero exit code. - if (failed) { - Exit.exit(1) - } - } - - private def handleAction(adminClient: Admin, - opts: ReassignPartitionsCommandOptions): Unit = { - if (opts.options.has(opts.verifyOpt)) { - verifyAssignment(adminClient, - Utils.readFileAsString(opts.options.valueOf(opts.reassignmentJsonFileOpt)), - opts.options.has(opts.preserveThrottlesOpt)) - } else if (opts.options.has(opts.generateOpt)) { - generateAssignment(adminClient, - Utils.readFileAsString(opts.options.valueOf(opts.topicsToMoveJsonFileOpt)), - opts.options.valueOf(opts.brokerListOpt), - !opts.options.has(opts.disableRackAware)) - } else if (opts.options.has(opts.executeOpt)) { - executeAssignment(adminClient, - opts.options.has(opts.additionalOpt), - Utils.readFileAsString(opts.options.valueOf(opts.reassignmentJsonFileOpt)), - opts.options.valueOf(opts.interBrokerThrottleOpt), - opts.options.valueOf(opts.replicaAlterLogDirsThrottleOpt), - opts.options.valueOf(opts.timeoutOpt)) - } else if (opts.options.has(opts.cancelOpt)) { - cancelAssignment(adminClient, - Utils.readFileAsString(opts.options.valueOf(opts.reassignmentJsonFileOpt)), - opts.options.has(opts.preserveThrottlesOpt), - opts.options.valueOf(opts.timeoutOpt)) - } else if (opts.options.has(opts.listOpt)) { - listReassignments(adminClient) - } else { - throw new RuntimeException("Unsupported action.") - } - } - - /** - * A result returned from verifyAssignment. - * - * @param partStates A map from partitions to reassignment states. - * @param partsOngoing True if there are any ongoing partition reassignments. - * @param moveStates A map from log directories to movement states. - * @param movesOngoing True if there are any ongoing moves that we know about. - */ - case class VerifyAssignmentResult(partStates: Map[TopicPartition, PartitionReassignmentState], - partsOngoing: Boolean = false, - moveStates: Map[TopicPartitionReplica, LogDirMoveState] = Map.empty, - movesOngoing: Boolean = false) - - /** - * The entry point for the --verify command. - * - * @param adminClient The AdminClient to use. - * @param jsonString The JSON string to use for the topics and partitions to verify. - * @param preserveThrottles True if we should avoid changing topic or broker throttles. - * - * @return A result that is useful for testing. - */ - def verifyAssignment(adminClient: Admin, jsonString: String, preserveThrottles: Boolean) - : VerifyAssignmentResult = { - val (targetParts, targetLogDirs) = parsePartitionReassignmentData(jsonString) - val (partStates, partsOngoing) = verifyPartitionAssignments(adminClient, targetParts) - val (moveStates, movesOngoing) = verifyReplicaMoves(adminClient, targetLogDirs) - if (!partsOngoing && !movesOngoing && !preserveThrottles) { - // If the partition assignments and replica assignments are done, clear any throttles - // that were set. We have to clear all throttles, because we don't have enough - // information to know all of the source brokers that might have been involved in the - // previous reassignments. - clearAllThrottles(adminClient, targetParts) - } - VerifyAssignmentResult(partStates, partsOngoing, moveStates, movesOngoing) - } - - /** - * Verify the partition reassignments specified by the user. - * - * @param adminClient The AdminClient to use. - * @param targets The partition reassignments specified by the user. - * - * @return A tuple of the partition reassignment states, and a - * boolean which is true if there are no ongoing - * reassignments (including reassignments not described - * in the JSON file.) - */ - def verifyPartitionAssignments(adminClient: Admin, - targets: Seq[(TopicPartition, Seq[Int])]) - : (Map[TopicPartition, PartitionReassignmentState], Boolean) = { - val (partStates, partsOngoing) = findPartitionReassignmentStates(adminClient, targets) - println(partitionReassignmentStatesToString(partStates)) - (partStates, partsOngoing) - } - - def compareTopicPartitions(a: TopicPartition, b: TopicPartition): Boolean = { - (a.topic(), a.partition()) < (b.topic(), b.partition()) - } - - def compareTopicPartitionReplicas(a: TopicPartitionReplica, b: TopicPartitionReplica): Boolean = { - (a.brokerId(), a.topic(), a.partition()) < (b.brokerId(), b.topic(), b.partition()) - } - - /** - * Convert partition reassignment states to a human-readable string. - * - * @param states A map from topic partitions to states. - * @return A string summarizing the partition reassignment states. - */ - def partitionReassignmentStatesToString(states: Map[TopicPartition, PartitionReassignmentState]) - : String = { - val bld = new mutable.ArrayBuffer[String]() - bld.append("Status of partition reassignment:") - states.keySet.toBuffer.sortWith(compareTopicPartitions).foreach { topicPartition => - val state = states(topicPartition) - if (state.done) { - if (state.currentReplicas.equals(state.targetReplicas)) { - bld.append("Reassignment of partition %s is completed.". - format(topicPartition.toString)) - } else { - bld.append(s"There is no active reassignment of partition ${topicPartition}, " + - s"but replica set is ${state.currentReplicas.mkString(",")} rather than " + - s"${state.targetReplicas.mkString(",")}.") - } - } else { - bld.append("Reassignment of partition %s is still in progress.".format(topicPartition)) - } - } - bld.mkString(System.lineSeparator()) - } - - /** - * Find the state of the specified partition reassignments. - * - * @param adminClient The Admin client to use. - * @param targetReassignments The reassignments we want to learn about. - * - * @return A tuple containing the reassignment states for each topic - * partition, plus whether there are any ongoing reassignments. - */ - def findPartitionReassignmentStates(adminClient: Admin, - targetReassignments: Seq[(TopicPartition, Seq[Int])]) - : (Map[TopicPartition, PartitionReassignmentState], Boolean) = { - val currentReassignments = adminClient. - listPartitionReassignments.reassignments.get().asScala - val (foundReassignments, notFoundReassignments) = targetReassignments.partition { - case (part, _) => currentReassignments.contains(part) - } - val foundResults = foundReassignments.map { - case (part, targetReplicas) => (part, - PartitionReassignmentState( - currentReassignments(part).replicas. - asScala.map(i => i.asInstanceOf[Int]), - targetReplicas, - false)) - } - val topicNamesToLookUp = new mutable.HashSet[String]() - notFoundReassignments.foreach { case (part, _) => - if (!currentReassignments.contains(part)) - topicNamesToLookUp.add(part.topic) - } - val topicDescriptions = adminClient. - describeTopics(topicNamesToLookUp.asJava).topicNameValues().asScala - val notFoundResults = notFoundReassignments.map { - case (part, targetReplicas) => - currentReassignments.get(part) match { - case Some(reassignment) => (part, - PartitionReassignmentState( - reassignment.replicas.asScala.map(_.asInstanceOf[Int]), - targetReplicas, - false)) - case None => - (part, topicDescriptionFutureToState(part.partition, - topicDescriptions(part.topic), targetReplicas)) - } - } - val allResults = foundResults ++ notFoundResults - (allResults.toMap, currentReassignments.nonEmpty) - } - - private def topicDescriptionFutureToState(partition: Int, - future: KafkaFuture[TopicDescription], - targetReplicas: Seq[Int]): PartitionReassignmentState = { - try { - val topicDescription = future.get() - if (topicDescription.partitions().size() < partition) { - throw new ExecutionException("Too few partitions found", new UnknownTopicOrPartitionException()) - } - PartitionReassignmentState( - topicDescription.partitions.get(partition).replicas.asScala.map(_.id), - targetReplicas, - true) - } catch { - case t: ExecutionException if t.getCause.isInstanceOf[UnknownTopicOrPartitionException] => - PartitionReassignmentState(Seq(), targetReplicas, true) - } - } - - /** - * Verify the replica reassignments specified by the user. - * - * @param adminClient The AdminClient to use. - * @param targetReassignments The replica reassignments specified by the user. - * - * @return A tuple of the replica states, and a boolean which is true - * if there are any ongoing replica moves. - * - * Note: Unlike in verifyPartitionAssignments, we will - * return false here even if there are unrelated ongoing - * reassignments. (We don't have an efficient API that - * returns all ongoing replica reassignments.) - */ - def verifyReplicaMoves(adminClient: Admin, - targetReassignments: Map[TopicPartitionReplica, String]) - : (Map[TopicPartitionReplica, LogDirMoveState], Boolean) = { - val moveStates = findLogDirMoveStates(adminClient, targetReassignments) - println(replicaMoveStatesToString(moveStates)) - (moveStates, !moveStates.values.forall(_.done)) - } - - /** - * Find the state of the specified partition reassignments. - * - * @param adminClient The AdminClient to use. - * @param targetMoves The movements we want to learn about. The map is keyed - * by TopicPartitionReplica, and its values are target log - * directories. - * - * @return The states for each replica movement. - */ - def findLogDirMoveStates(adminClient: Admin, - targetMoves: Map[TopicPartitionReplica, String]) - : Map[TopicPartitionReplica, LogDirMoveState] = { - val replicaLogDirInfos = adminClient.describeReplicaLogDirs( - targetMoves.keySet.asJava).all().get().asScala - targetMoves.map { case (replica, targetLogDir) => - val moveState = replicaLogDirInfos.get(replica) match { - case None => MissingReplicaMoveState(targetLogDir) - case Some(info) => if (info.getCurrentReplicaLogDir == null) { - MissingLogDirMoveState(targetLogDir) - } else if (info.getFutureReplicaLogDir == null) { - if (info.getCurrentReplicaLogDir.equals(targetLogDir)) { - CompletedMoveState(targetLogDir) - } else { - CancelledMoveState(info.getCurrentReplicaLogDir, targetLogDir) - } - } else { - ActiveMoveState(info.getCurrentReplicaLogDir(), - targetLogDir, - info.getFutureReplicaLogDir) - } - } - (replica, moveState) - } - } - - /** - * Convert replica move states to a human-readable string. - * - * @param states A map from topic partition replicas to states. - * @return A tuple of a summary string, and a boolean describing - * whether there are any active replica moves. - */ - def replicaMoveStatesToString(states: Map[TopicPartitionReplica, LogDirMoveState]) - : String = { - val bld = new mutable.ArrayBuffer[String] - states.keySet.toBuffer.sortWith(compareTopicPartitionReplicas).foreach { replica => - val state = states(replica) - state match { - case MissingLogDirMoveState(_) => - bld.append(s"Partition ${replica.topic}-${replica.partition} is not found " + - s"in any live log dir on broker ${replica.brokerId}. There is likely an " + - s"offline log directory on the broker.") - case MissingReplicaMoveState(_) => - bld.append(s"Partition ${replica.topic}-${replica.partition} cannot be found " + - s"in any live log directory on broker ${replica.brokerId}.") - case ActiveMoveState(_, targetLogDir, futureLogDir) => - if (targetLogDir.equals(futureLogDir)) { - bld.append(s"Reassignment of replica $replica is still in progress.") - } else { - bld.append(s"Partition ${replica.topic}-${replica.partition} on broker " + - s"${replica.brokerId} is being moved to log dir $futureLogDir " + - s"instead of $targetLogDir.") - } - case CancelledMoveState(currentLogDir, targetLogDir) => - bld.append(s"Partition ${replica.topic}-${replica.partition} on broker " + - s"${replica.brokerId} is not being moved from log dir $currentLogDir to " + - s"$targetLogDir.") - case CompletedMoveState(_) => - bld.append(s"Reassignment of replica $replica completed successfully.") - } - } - bld.mkString(System.lineSeparator()) - } - - /** - * Clear all topic-level and broker-level throttles. - * - * @param adminClient The AdminClient to use. - * @param targetParts The target partitions loaded from the JSON file. - */ - def clearAllThrottles(adminClient: Admin, - targetParts: Seq[(TopicPartition, Seq[Int])]): Unit = { - val activeBrokers = adminClient.describeCluster().nodes().get().asScala.map(_.id()).toSet - val brokers = activeBrokers ++ targetParts.flatMap(_._2).toSet - println("Clearing broker-level throttles on broker%s %s".format( - if (brokers.size == 1) "" else "s", brokers.mkString(","))) - clearBrokerLevelThrottles(adminClient, brokers) - - val topics = targetParts.map(_._1.topic()).toSet - println("Clearing topic-level throttles on topic%s %s".format( - if (topics.size == 1) "" else "s", topics.mkString(","))) - clearTopicLevelThrottles(adminClient, topics) - } - - /** - * Clear all throttles which have been set at the broker level. - * - * @param adminClient The AdminClient to use. - * @param brokers The brokers to clear the throttles for. - */ - def clearBrokerLevelThrottles(adminClient: Admin, brokers: Set[Int]): Unit = { - val configOps = new util.HashMap[ConfigResource, util.Collection[AlterConfigOp]]() - brokers.foreach { brokerId => - configOps.put( - new ConfigResource(ConfigResource.Type.BROKER, brokerId.toString), - brokerLevelThrottles.map(throttle => new AlterConfigOp( - new ConfigEntry(throttle, null), OpType.DELETE)).asJava) - } - adminClient.incrementalAlterConfigs(configOps).all().get() - } - - /** - * Clear the reassignment throttles for the specified topics. - * - * @param adminClient The AdminClient to use. - * @param topics The topics to clear the throttles for. - */ - def clearTopicLevelThrottles(adminClient: Admin, topics: Set[String]): Unit = { - val configOps = new util.HashMap[ConfigResource, util.Collection[AlterConfigOp]]() - topics.foreach { - topicName => configOps.put( - new ConfigResource(ConfigResource.Type.TOPIC, topicName), - topicLevelThrottles.map(throttle => new AlterConfigOp(new ConfigEntry(throttle, null), - OpType.DELETE)).asJava) - } - adminClient.incrementalAlterConfigs(configOps).all().get() - } - - /** - * The entry point for the --generate command. - * - * @param adminClient The AdminClient to use. - * @param reassignmentJson The JSON string to use for the topics to reassign. - * @param brokerListString The comma-separated string of broker IDs to use. - * @param enableRackAwareness True if rack-awareness should be enabled. - * - * @return A tuple containing the proposed assignment and the - * current assignment. - */ - def generateAssignment(adminClient: Admin, - reassignmentJson: String, - brokerListString: String, - enableRackAwareness: Boolean) - : (Map[TopicPartition, Seq[Int]], Map[TopicPartition, Seq[Int]]) = { - val (brokersToReassign, topicsToReassign) = - parseGenerateAssignmentArgs(reassignmentJson, brokerListString) - val currentAssignments = getReplicaAssignmentForTopics(adminClient, topicsToReassign) - val brokerMetadatas = getBrokerMetadata(adminClient, brokersToReassign, enableRackAwareness) - val proposedAssignments = calculateAssignment(currentAssignments, brokerMetadatas) - println("Current partition replica assignment\n%s\n". - format(formatAsReassignmentJson(currentAssignments, Map.empty))) - println("Proposed partition reassignment configuration\n%s". - format(formatAsReassignmentJson(proposedAssignments, Map.empty))) - (proposedAssignments, currentAssignments) - } - - /** - * Calculate the new partition assignments to suggest in --generate. - * - * @param currentAssignment The current partition assignments. - * @param brokerMetadatas The rack information for each broker. - * - * @return A map from partitions to the proposed assignments for each. - */ - def calculateAssignment(currentAssignment: Map[TopicPartition, Seq[Int]], - brokerMetadatas: Seq[BrokerMetadata]) - : Map[TopicPartition, Seq[Int]] = { - val groupedByTopic = currentAssignment.groupBy { case (tp, _) => tp.topic } - val proposedAssignments = mutable.Map[TopicPartition, Seq[Int]]() - groupedByTopic.forKeyValue { (topic, assignment) => - val (_, replicas) = assignment.head - val assignedReplicas = AdminUtils. - assignReplicasToBrokers(brokerMetadatas.asJavaCollection, assignment.size, replicas.size). - asScala.map(e => (e._1.asInstanceOf[Int], e._2.asScala.map(_.asInstanceOf[Int]))) - proposedAssignments ++= assignedReplicas.map { case (partition, replicas) => - new TopicPartition(topic, partition) -> replicas - } - } - proposedAssignments - } - - private def describeTopics(adminClient: Admin, - topics: Set[String]) - : Map[String, TopicDescription] = { - adminClient.describeTopics(topics.asJava).topicNameValues().asScala.map { case (topicName, topicDescriptionFuture) => - try topicName -> topicDescriptionFuture.get - catch { - case t: ExecutionException if t.getCause.isInstanceOf[UnknownTopicOrPartitionException] => - throw new ExecutionException( - new UnknownTopicOrPartitionException(s"Topic $topicName not found.")) - } - } - } - - /** - * Get the current replica assignments for some topics. - * - * @param adminClient The AdminClient to use. - * @param topics The topics to get information about. - * @return A map from partitions to broker assignments. - * If any topic can't be found, an exception will be thrown. - */ - def getReplicaAssignmentForTopics(adminClient: Admin, - topics: Seq[String]) - : Map[TopicPartition, Seq[Int]] = { - describeTopics(adminClient, topics.toSet).flatMap { - case (topicName, topicDescription) => topicDescription.partitions.asScala.map { info => - (new TopicPartition(topicName, info.partition), info.replicas.asScala.map(_.id)) - } - } - } - - /** - * Get the current replica assignments for some partitions. - * - * @param adminClient The AdminClient to use. - * @param partitions The partitions to get information about. - * @return A map from partitions to broker assignments. - * If any topic can't be found, an exception will be thrown. - */ - def getReplicaAssignmentForPartitions(adminClient: Admin, - partitions: Set[TopicPartition]) - : Map[TopicPartition, Seq[Int]] = { - describeTopics(adminClient, partitions.map(_.topic)).flatMap { - case (topicName, topicDescription) => topicDescription.partitions.asScala.flatMap { info => - val tp = new TopicPartition(topicName, info.partition) - if (partitions.contains(tp)) { - Some(tp, info.replicas.asScala.map(_.id)) - } else { - None - } - } - } - } - - /** - * Find the rack information for some brokers. - * - * @param adminClient The AdminClient object. - * @param brokers The brokers to gather metadata about. - * @param enableRackAwareness True if we should return rack information, and throw an - * exception if it is inconsistent. - * - * @return The metadata for each broker that was found. - * Brokers that were not found will be omitted. - */ - def getBrokerMetadata(adminClient: Admin, - brokers: Seq[Int], - enableRackAwareness: Boolean): Seq[BrokerMetadata] = { - val brokerSet = brokers.toSet - val results = adminClient.describeCluster().nodes.get().asScala. - filter(node => brokerSet.contains(node.id)). - map { - node => if (enableRackAwareness && node.rack != null) { - new BrokerMetadata(node.id, Optional.of(node.rack)) - } else { - new BrokerMetadata(node.id, Optional.empty()) - } - }.toSeq - val numRackless = results.count(!_.rack.isPresent) - if (enableRackAwareness && numRackless != 0 && numRackless != results.size) { - throw new AdminOperationException("Not all brokers have rack information. Add " + - "--disable-rack-aware in command line to make replica assignment without rack " + - "information.") - } - results - } - - /** - * Parse and validate data gathered from the command-line for --generate - * In particular, we parse the JSON and validate that duplicate brokers and - * topics don't appear. - * - * @param reassignmentJson The JSON passed to --generate . - * @param brokerList A list of brokers passed to --generate. - * - * @return A tuple of brokers to reassign, topics to reassign - */ - def parseGenerateAssignmentArgs(reassignmentJson: String, - brokerList: String): (Seq[Int], Seq[String]) = { - val brokerListToReassign = brokerList.split(',').map(_.toInt) - val duplicateReassignments = CoreUtils.duplicates(brokerListToReassign) - if (duplicateReassignments.nonEmpty) - throw new AdminCommandFailedException("Broker list contains duplicate entries: %s". - format(duplicateReassignments.mkString(","))) - val topicsToReassign = parseTopicsData(reassignmentJson) - val duplicateTopicsToReassign = CoreUtils.duplicates(topicsToReassign) - if (duplicateTopicsToReassign.nonEmpty) - throw new AdminCommandFailedException("List of topics to reassign contains duplicate entries: %s". - format(duplicateTopicsToReassign.mkString(","))) - (brokerListToReassign, topicsToReassign) - } - - /** - * The entry point for the --execute and --execute-additional commands. - * - * @param adminClient The AdminClient to use. - * @param additional Whether --additional was passed. - * @param reassignmentJson The JSON string to use for the topics to reassign. - * @param interBrokerThrottle The inter-broker throttle to use, or a negative - * number to skip using a throttle. - * @param logDirThrottle The replica log directory throttle to use, or a - * negative number to skip using a throttle. - * @param timeoutMs The maximum time in ms to wait for log directory - * replica assignment to begin. - * @param time The Time object to use. - */ - def executeAssignment(adminClient: Admin, - additional: Boolean, - reassignmentJson: String, - interBrokerThrottle: Long = -1L, - logDirThrottle: Long = -1L, - timeoutMs: Long = 10000L, - time: Time = Time.SYSTEM): Unit = { - val (proposedParts, proposedReplicas) = parseExecuteAssignmentArgs(reassignmentJson) - val currentReassignments = adminClient. - listPartitionReassignments().reassignments().get().asScala - // If there is an existing assignment, check for --additional before proceeding. - // This helps avoid surprising users. - if (!additional && currentReassignments.nonEmpty) { - throw new TerseReassignmentFailureException(cannotExecuteBecauseOfExistingMessage) - } - verifyBrokerIds(adminClient, proposedParts.values.flatten.toSet) - val currentParts = getReplicaAssignmentForPartitions(adminClient, proposedParts.keySet.toSet) - println(currentPartitionReplicaAssignmentToString(proposedParts, currentParts)) - - if (interBrokerThrottle >= 0 || logDirThrottle >= 0) { - println(youMustRunVerifyPeriodicallyMessage) - - if (interBrokerThrottle >= 0) { - val moveMap = calculateProposedMoveMap(currentReassignments, proposedParts, currentParts) - modifyReassignmentThrottle(adminClient, moveMap, interBrokerThrottle) - } - - if (logDirThrottle >= 0) { - val movingBrokers = calculateMovingBrokers(proposedReplicas.keySet.toSet) - modifyLogDirThrottle(adminClient, movingBrokers, logDirThrottle) - } - } - - // Execute the partition reassignments. - val errors = alterPartitionReassignments(adminClient, proposedParts) - if (errors.nonEmpty) { - throw new TerseReassignmentFailureException( - "Error reassigning partition(s):%n%s".format( - errors.keySet.toBuffer.sortWith(compareTopicPartitions).map { part => - s"$part: ${errors(part).getMessage}" - }.mkString(System.lineSeparator()))) - } - println("Successfully started partition reassignment%s for %s".format( - if (proposedParts.size == 1) "" else "s", - proposedParts.keySet.toBuffer.sortWith(compareTopicPartitions).mkString(","))) - if (proposedReplicas.nonEmpty) { - executeMoves(adminClient, proposedReplicas, timeoutMs, time) - } - } - - /** - * Execute some partition log directory movements. - * - * @param adminClient The AdminClient to use. - * @param proposedReplicas A map from TopicPartitionReplicas to the - * directories to move them to. - * @param timeoutMs The maximum time in ms to wait for log directory - * replica assignment to begin. - * @param time The Time object to use. - */ - def executeMoves(adminClient: Admin, - proposedReplicas: Map[TopicPartitionReplica, String], - timeoutMs: Long, - time: Time): Unit = { - val startTimeMs = time.milliseconds() - val pendingReplicas = new mutable.HashMap[TopicPartitionReplica, String]() - pendingReplicas ++= proposedReplicas - var done = false - do { - val completed = alterReplicaLogDirs(adminClient, pendingReplicas) - if (completed.nonEmpty) { - println("Successfully started log directory move%s for: %s".format( - if (completed.size == 1) "" else "s", - completed.toBuffer.sortWith(compareTopicPartitionReplicas).mkString(","))) - } - pendingReplicas --= completed - if (pendingReplicas.isEmpty) { - done = true - } else if (time.milliseconds() >= startTimeMs + timeoutMs) { - throw new TerseReassignmentFailureException( - "Timed out before log directory move%s could be started for: %s".format( - if (pendingReplicas.size == 1) "" else "s", - pendingReplicas.keySet.toBuffer.sortWith(compareTopicPartitionReplicas). - mkString(","))) - } else { - // If a replica has been moved to a new host and we also specified a particular - // log directory, we will have to keep retrying the alterReplicaLogDirs - // call. It can't take effect until the replica is moved to that host. - time.sleep(100) - } - } while (!done) - } - - /** - * Entry point for the --list command. - * - * @param adminClient The AdminClient to use. - */ - def listReassignments(adminClient: Admin): Unit = { - println(curReassignmentsToString(adminClient)) - } - - /** - * Convert the current partition reassignments to text. - * - * @param adminClient The AdminClient to use. - * @return A string describing the current partition reassignments. - */ - def curReassignmentsToString(adminClient: Admin): String = { - val currentReassignments = adminClient. - listPartitionReassignments().reassignments().get().asScala - val text = currentReassignments.keySet.toBuffer.sortWith(compareTopicPartitions).map { part => - val reassignment = currentReassignments(part) - val replicas = reassignment.replicas.asScala - val addingReplicas = reassignment.addingReplicas.asScala - val removingReplicas = reassignment.removingReplicas.asScala - "%s: replicas: %s.%s%s".format(part, replicas.mkString(","), - if (addingReplicas.isEmpty) "" else - " adding: %s.".format(addingReplicas.mkString(",")), - if (removingReplicas.isEmpty) "" else - " removing: %s.".format(removingReplicas.mkString(","))) - }.mkString(System.lineSeparator()) - if (text.isEmpty) { - "No partition reassignments found." - } else { - "Current partition reassignments:%n%s".format(text) - } - } - - /** - * Verify that all the brokers in an assignment exist. - * - * @param adminClient The AdminClient to use. - * @param brokers The broker IDs to verify. - */ - def verifyBrokerIds(adminClient: Admin, brokers: Set[Int]): Unit = { - val allNodeIds = adminClient.describeCluster().nodes().get().asScala.map(_.id).toSet - brokers.find(!allNodeIds.contains(_)).map { - id => throw new AdminCommandFailedException(s"Unknown broker id ${id}") - } - } - - /** - * Return the string which we want to print to describe the current partition assignment. - * - * @param proposedParts The proposed partition assignment. - * @param currentParts The current partition assignment. - * - * @return The string to print. We will only print information about - * partitions that appear in the proposed partition assignment. - */ - def currentPartitionReplicaAssignmentToString(proposedParts: Map[TopicPartition, Seq[Int]], - currentParts: Map[TopicPartition, Seq[Int]]): String = { - "Current partition replica assignment%n%n%s%n%nSave this to use as the %s". - format(formatAsReassignmentJson(currentParts.filter { case (k, _) => proposedParts.contains(k) }.toMap, Map.empty), - "--reassignment-json-file option during rollback") - } - - /** - * Execute the given partition reassignments. - * - * @param adminClient The admin client object to use. - * @param reassignments A map from topic names to target replica assignments. - * @return A map from partition objects to error strings. - */ - def alterPartitionReassignments(adminClient: Admin, - reassignments: Map[TopicPartition, Seq[Int]]): Map[TopicPartition, Throwable] = { - val results = adminClient.alterPartitionReassignments(reassignments.map { case (part, replicas) => - (part, Optional.of(new NewPartitionReassignment(replicas.map(Integer.valueOf).asJava))) - }.asJava).values().asScala - results.flatMap { - case (part, future) => { - try { - future.get() - None - } catch { - case t: ExecutionException => Some(part, t.getCause()) - } - } - } - } - - /** - * Cancel the given partition reassignments. - * - * @param adminClient The admin client object to use. - * @param reassignments The partition reassignments to cancel. - * @return A map from partition objects to error strings. - */ - def cancelPartitionReassignments(adminClient: Admin, - reassignments: Set[TopicPartition]) - : Map[TopicPartition, Throwable] = { - val results = adminClient.alterPartitionReassignments(reassignments.map { - (_, Optional.empty[NewPartitionReassignment]()) - }.toMap.asJava).values().asScala - results.flatMap { case (part, future) => - try { - future.get() - None - } catch { - case t: ExecutionException => Some(part, t.getCause()) - } - } - } - - /** - * Compute the in progress partition move from the current reassignments. - * @param currentReassignments All replicas, adding replicas and removing replicas of target partitions - */ - private def calculateCurrentMoveMap(currentReassignments: Map[TopicPartition, PartitionReassignment]): MoveMap = { - val moveMap = new mutable.HashMap[String, mutable.Map[Int, PartitionMove]]() - // Add the current reassignments to the move map. - currentReassignments.forKeyValue { (part, reassignment) => - val allReplicas = reassignment.replicas().asScala.map(Int.unbox) - val addingReplicas = reassignment.addingReplicas.asScala.map(Int.unbox) - - // The addingReplicas is included in the replicas during reassignment - val sources = mutable.Set[Int]() ++ allReplicas.diff(addingReplicas) - val destinations = mutable.Set[Int]() ++ addingReplicas - - val partMoves = moveMap.getOrElseUpdate(part.topic, new mutable.HashMap[Int, PartitionMove]) - partMoves.put(part.partition, PartitionMove(sources, destinations)) - } - moveMap - } - - /** - * Calculate the global map of all partitions that are moving. - * - * @param currentReassignments The currently active reassignments. - * @param proposedParts The proposed location of the partitions (destinations replicas only). - * @param currentParts The current location of the partitions that we are - * proposing to move. - * @return A map from topic name to partition map. - * The partition map is keyed on partition index and contains - * the movements for that partition. - */ - def calculateProposedMoveMap(currentReassignments: Map[TopicPartition, PartitionReassignment], - proposedParts: Map[TopicPartition, Seq[Int]], - currentParts: Map[TopicPartition, Seq[Int]]): MoveMap = { - val moveMap = calculateCurrentMoveMap(currentReassignments) - - proposedParts.forKeyValue { (part, replicas) => - val partMoves = moveMap.getOrElseUpdate(part.topic, new mutable.HashMap[Int, PartitionMove]) - - // If there is a reassignment in progress, use the sources from moveMap, otherwise - // use the sources from currentParts - val sources = mutable.Set[Int]() ++ (partMoves.get(part.partition) match { - case Some(move) => move.sources.toSeq - case None => currentParts.getOrElse(part, - throw new RuntimeException(s"Trying to reassign a topic partition $part with 0 replicas")) - }) - val destinations = mutable.Set[Int]() ++ replicas.diff(sources.toSeq) - - partMoves.put(part.partition, - PartitionMove(sources, destinations)) - } - moveMap - } - - /** - * Calculate the leader throttle configurations to use. - * - * @param moveMap The movements. - * @return A map from topic names to leader throttle configurations. - */ - def calculateLeaderThrottles(moveMap: MoveMap): Map[String, String] = { - moveMap.map { - case (topicName, partMoveMap) => { - val components = new mutable.TreeSet[String] - partMoveMap.forKeyValue { (partId, move) => - move.sources.foreach(source => components.add("%d:%d".format(partId, source))) - } - (topicName, components.mkString(",")) - } - } - } - - /** - * Calculate the follower throttle configurations to use. - * - * @param moveMap The movements. - * @return A map from topic names to follower throttle configurations. - */ - def calculateFollowerThrottles(moveMap: MoveMap): Map[String, String] = { - moveMap.map { - case (topicName, partMoveMap) => { - val components = new mutable.TreeSet[String] - partMoveMap.forKeyValue { (partId, move) => - move.destinations.foreach(destination => - if (!move.sources.contains(destination)) { - components.add("%d:%d".format(partId, destination)) - }) - } - (topicName, components.mkString(",")) - } - } - } - - /** - * Calculate all the brokers which are involved in the given partition reassignments. - * - * @param moveMap The partition movements. - * @return A set of all the brokers involved. - */ - def calculateReassigningBrokers(moveMap: MoveMap): Set[Int] = { - val reassigningBrokers = new mutable.TreeSet[Int] - moveMap.values.foreach { - _.values.foreach { - partMove => - partMove.sources.foreach(reassigningBrokers.add) - partMove.destinations.foreach(reassigningBrokers.add) - } - } - reassigningBrokers.toSet - } - - /** - * Calculate all the brokers which are involved in the given directory movements. - * - * @param replicaMoves The replica movements. - * @return A set of all the brokers involved. - */ - def calculateMovingBrokers(replicaMoves: Set[TopicPartitionReplica]): Set[Int] = { - replicaMoves.map(_.brokerId()) - } - - /** - * Modify the topic configurations that control inter-broker throttling. - * - * @param adminClient The adminClient object to use. - * @param leaderThrottles A map from topic names to leader throttle configurations. - * @param followerThrottles A map from topic names to follower throttle configurations. - */ - def modifyTopicThrottles(adminClient: Admin, - leaderThrottles: Map[String, String], - followerThrottles: Map[String, String]): Unit = { - val configs = new util.HashMap[ConfigResource, util.Collection[AlterConfigOp]]() - val topicNames = leaderThrottles.keySet ++ followerThrottles.keySet - topicNames.foreach { topicName => - val ops = new util.ArrayList[AlterConfigOp] - leaderThrottles.get(topicName).foreach { value => - ops.add(new AlterConfigOp(new ConfigEntry(topicLevelLeaderThrottle, value), OpType.SET)) - } - followerThrottles.get(topicName).foreach { value => - ops.add(new AlterConfigOp(new ConfigEntry(topicLevelFollowerThrottle, value), OpType.SET)) - } - if (!ops.isEmpty) { - configs.put(new ConfigResource(ConfigResource.Type.TOPIC, topicName), ops) - } - } - adminClient.incrementalAlterConfigs(configs).all().get() - } - - private def modifyReassignmentThrottle(admin: Admin, moveMap: MoveMap, interBrokerThrottle: Long): Unit = { - val leaderThrottles = calculateLeaderThrottles(moveMap) - val followerThrottles = calculateFollowerThrottles(moveMap) - modifyTopicThrottles(admin, leaderThrottles, followerThrottles) - - val reassigningBrokers = calculateReassigningBrokers(moveMap) - modifyInterBrokerThrottle(admin, reassigningBrokers, interBrokerThrottle) - } - - /** - * Modify the leader/follower replication throttles for a set of brokers. - * - * @param adminClient The Admin instance to use - * @param reassigningBrokers The set of brokers involved in the reassignment - * @param interBrokerThrottle The new throttle (ignored if less than 0) - */ - def modifyInterBrokerThrottle(adminClient: Admin, - reassigningBrokers: Set[Int], - interBrokerThrottle: Long): Unit = { - if (interBrokerThrottle >= 0) { - val configs = new util.HashMap[ConfigResource, util.Collection[AlterConfigOp]]() - reassigningBrokers.foreach { brokerId => - val ops = new util.ArrayList[AlterConfigOp] - ops.add(new AlterConfigOp(new ConfigEntry(brokerLevelLeaderThrottle, - interBrokerThrottle.toString), OpType.SET)) - ops.add(new AlterConfigOp(new ConfigEntry(brokerLevelFollowerThrottle, - interBrokerThrottle.toString), OpType.SET)) - configs.put(new ConfigResource(ConfigResource.Type.BROKER, brokerId.toString), ops) - } - adminClient.incrementalAlterConfigs(configs).all().get() - println(s"The inter-broker throttle limit was set to $interBrokerThrottle B/s") - } - } - - /** - * Modify the log dir reassignment throttle for a set of brokers. - * - * @param admin The Admin instance to use - * @param movingBrokers The set of broker to alter the throttle of - * @param logDirThrottle The new throttle (ignored if less than 0) - */ - def modifyLogDirThrottle(admin: Admin, - movingBrokers: Set[Int], - logDirThrottle: Long): Unit = { - if (logDirThrottle >= 0) { - val configs = new util.HashMap[ConfigResource, util.Collection[AlterConfigOp]]() - movingBrokers.foreach { brokerId => - val ops = new util.ArrayList[AlterConfigOp] - ops.add(new AlterConfigOp(new ConfigEntry(brokerLevelLogDirThrottle, logDirThrottle.toString), OpType.SET)) - configs.put(new ConfigResource(ConfigResource.Type.BROKER, brokerId.toString), ops) - } - admin.incrementalAlterConfigs(configs).all().get() - println(s"The replica-alter-dir throttle limit was set to $logDirThrottle B/s") - } - } - - /** - * Parse the reassignment JSON string passed to the --execute command. - * - * @param reassignmentJson The JSON string. - * @return A tuple of the partitions to be reassigned and the replicas - * to be reassigned. - */ - def parseExecuteAssignmentArgs(reassignmentJson: String) - : (Map[TopicPartition, Seq[Int]], Map[TopicPartitionReplica, String]) = { - val (partitionsToBeReassigned, replicaAssignment) = parsePartitionReassignmentData(reassignmentJson) - if (partitionsToBeReassigned.isEmpty) - throw new AdminCommandFailedException("Partition reassignment list cannot be empty") - if (partitionsToBeReassigned.exists(_._2.isEmpty)) { - throw new AdminCommandFailedException("Partition replica list cannot be empty") - } - val duplicateReassignedPartitions = CoreUtils.duplicates(partitionsToBeReassigned.map { case (tp, _) => tp }) - if (duplicateReassignedPartitions.nonEmpty) - throw new AdminCommandFailedException("Partition reassignment contains duplicate topic partitions: %s".format(duplicateReassignedPartitions.mkString(","))) - val duplicateEntries = partitionsToBeReassigned - .map { case (tp, replicas) => (tp, CoreUtils.duplicates(replicas))} - .filter { case (_, duplicatedReplicas) => duplicatedReplicas.nonEmpty } - if (duplicateEntries.nonEmpty) { - val duplicatesMsg = duplicateEntries - .map { case (tp, duplicateReplicas) => "%s contains multiple entries for %s".format(tp, duplicateReplicas.mkString(",")) } - .mkString(". ") - throw new AdminCommandFailedException("Partition replica lists may not contain duplicate entries: %s".format(duplicatesMsg)) - } - (partitionsToBeReassigned.toMap, replicaAssignment) - } - - /** - * The entry point for the --cancel command. - * - * @param adminClient The AdminClient to use. - * @param jsonString The JSON string to use for the topics and partitions to cancel. - * @param preserveThrottles True if we should avoid changing topic or broker throttles. - * @param timeoutMs The maximum time in ms to wait for log directory - * replica assignment to begin. - * @param time The Time object to use. - * - * @return A tuple of the partition reassignments that were cancelled, - * and the replica movements that were cancelled. - */ - def cancelAssignment(adminClient: Admin, - jsonString: String, - preserveThrottles: Boolean, - timeoutMs: Long = 10000L, - time: Time = Time.SYSTEM) - : (Set[TopicPartition], Set[TopicPartitionReplica]) = { - val (targetParts, targetReplicas) = parsePartitionReassignmentData(jsonString) - val targetPartsSet = targetParts.map(_._1).toSet - val curReassigningParts = adminClient.listPartitionReassignments(targetPartsSet.asJava). - reassignments().get().asScala.flatMap { - case (part, reassignment) => if (!reassignment.addingReplicas().isEmpty || - !reassignment.removingReplicas().isEmpty) { - Some(part) - } else { - None - } - }.toSet - if (curReassigningParts.nonEmpty) { - val errors = cancelPartitionReassignments(adminClient, curReassigningParts) - if (errors.nonEmpty) { - throw new TerseReassignmentFailureException( - "Error cancelling partition reassignment%s for:%n%s".format( - if (errors.size == 1) "" else "s", - errors.keySet.toBuffer.sortWith(compareTopicPartitions).map { - part => s"${part}: ${errors(part).getMessage}" - }.mkString(System.lineSeparator()))) - } - println("Successfully cancelled partition reassignment%s for: %s".format( - if (curReassigningParts.size == 1) "" else "s", - s"${curReassigningParts.toBuffer.sortWith(compareTopicPartitions).mkString(",")}")) - } else { - println("None of the specified partition reassignments are active.") - } - val curMovingParts = findLogDirMoveStates(adminClient, targetReplicas).flatMap { - case (part, moveState) => moveState match { - case state: ActiveMoveState => Some(part, state.currentLogDir) - case _ => None - } - }.toMap - if (curMovingParts.isEmpty) { - println("None of the specified partition moves are active.") - } else { - executeMoves(adminClient, curMovingParts, timeoutMs, time) - } - if (!preserveThrottles) { - clearAllThrottles(adminClient, targetParts) - } - (curReassigningParts, curMovingParts.keySet) - } - - def formatAsReassignmentJson(partitionsToBeReassigned: Map[TopicPartition, Seq[Int]], - replicaLogDirAssignment: Map[TopicPartitionReplica, String]): String = { - Json.encodeAsString(Map( - "version" -> 1, - "partitions" -> partitionsToBeReassigned.keySet.toBuffer.sortWith(compareTopicPartitions).map { - tp => - val replicas = partitionsToBeReassigned(tp) - Map( - "topic" -> tp.topic, - "partition" -> tp.partition, - "replicas" -> replicas.asJava, - "log_dirs" -> replicas.map(r => replicaLogDirAssignment.getOrElse(new TopicPartitionReplica(tp.topic, tp.partition, r), AnyLogDir)).asJava - ).asJava - }.asJava - ).asJava) - } - - def parseTopicsData(jsonData: String): Seq[String] = { - Json.parseFull(jsonData) match { - case Some(js) => - val version = js.asJsonObject.get("version") match { - case Some(jsonValue) => jsonValue.to[Int] - case None => EarliestTopicsJsonVersion - } - parseTopicsData(version, js) - case None => throw new AdminOperationException("The input string is not a valid JSON") - } - } - - def parseTopicsData(version: Int, js: JsonValue): Seq[String] = { - version match { - case 1 => - for { - partitionsSeq <- js.asJsonObject.get("topics").toSeq - p <- partitionsSeq.asJsonArray.iterator - } yield p.asJsonObject("topic").to[String] - case _ => throw new AdminOperationException(s"Not supported version field value $version") - } - } - - def parsePartitionReassignmentData(jsonData: String): (Seq[(TopicPartition, Seq[Int])], Map[TopicPartitionReplica, String]) = { - Json.tryParseFull(jsonData) match { - case Right(js) => - val version = js.asJsonObject.get("version") match { - case Some(jsonValue) => jsonValue.to[Int] - case None => EarliestVersion - } - parsePartitionReassignmentData(version, js) - case Left(f) => - throw new AdminOperationException(f) - } - } - - // Parses without deduplicating keys so the data can be checked before allowing reassignment to proceed - def parsePartitionReassignmentData(version:Int, jsonData: JsonValue): (Seq[(TopicPartition, Seq[Int])], Map[TopicPartitionReplica, String]) = { - version match { - case 1 => - val partitionAssignment = mutable.ListBuffer.empty[(TopicPartition, Seq[Int])] - val replicaAssignment = mutable.Map.empty[TopicPartitionReplica, String] - for { - partitionsSeq <- jsonData.asJsonObject.get("partitions").toSeq - p <- partitionsSeq.asJsonArray.iterator - } { - val partitionFields = p.asJsonObject - val topic = partitionFields("topic").to[String] - val partition = partitionFields("partition").to[Int] - val newReplicas = partitionFields("replicas").to[Seq[Int]] - val newLogDirs = partitionFields.get("log_dirs") match { - case Some(jsonValue) => jsonValue.to[Seq[String]] - case None => newReplicas.map(_ => AnyLogDir) - } - if (newReplicas.size != newLogDirs.size) - throw new AdminCommandFailedException(s"Size of replicas list $newReplicas is different from " + - s"size of log dirs list $newLogDirs for partition ${new TopicPartition(topic, partition)}") - partitionAssignment += (new TopicPartition(topic, partition) -> newReplicas) - replicaAssignment ++= newReplicas.zip(newLogDirs).map { case (replica, logDir) => - new TopicPartitionReplica(topic, partition, replica) -> logDir - }.filter(_._2 != AnyLogDir) - } - (partitionAssignment, replicaAssignment) - case _ => throw new AdminOperationException(s"Not supported version field value $version") - } - } - - def validateAndParseArgs(args: Array[String]): ReassignPartitionsCommandOptions = { - val opts = new ReassignPartitionsCommandOptions(args) - - CommandLineUtils.maybePrintHelpOrVersion(opts, helpText) - - // Determine which action we should perform. - val validActions = Seq(opts.generateOpt, opts.executeOpt, opts.verifyOpt, - opts.cancelOpt, opts.listOpt) - val allActions = validActions.filter(opts.options.has _) - if (allActions.size != 1) { - CommandLineUtils.printUsageAndExit(opts.parser, "Command must include exactly one action: %s".format( - validActions.map("--" + _.options().get(0)).mkString(", "))) - } - val action = allActions.head - - if (!opts.options.has(opts.bootstrapServerOpt)) - CommandLineUtils.printUsageAndExit(opts.parser, "Please specify --bootstrap-server") - - // Make sure that we have all the required arguments for our action. - val requiredArgs = Map( - opts.verifyOpt -> collection.immutable.Seq( - opts.reassignmentJsonFileOpt - ), - opts.generateOpt -> collection.immutable.Seq( - opts.topicsToMoveJsonFileOpt, - opts.brokerListOpt - ), - opts.executeOpt -> collection.immutable.Seq( - opts.reassignmentJsonFileOpt - ), - opts.cancelOpt -> collection.immutable.Seq( - opts.reassignmentJsonFileOpt - ), - opts.listOpt -> collection.immutable.Seq.empty - ) - CommandLineUtils.checkRequiredArgs(opts.parser, opts.options, requiredArgs(action): _*) - - // Make sure that we didn't specify any arguments that are incompatible with our chosen action. - val permittedArgs = Map( - opts.verifyOpt -> Seq( - opts.bootstrapServerOpt, - opts.commandConfigOpt, - opts.preserveThrottlesOpt, - ), - opts.generateOpt -> Seq( - opts.bootstrapServerOpt, - opts.brokerListOpt, - opts.commandConfigOpt, - opts.disableRackAware, - ), - opts.executeOpt -> Seq( - opts.additionalOpt, - opts.bootstrapServerOpt, - opts.commandConfigOpt, - opts.interBrokerThrottleOpt, - opts.replicaAlterLogDirsThrottleOpt, - opts.timeoutOpt, - ), - opts.cancelOpt -> Seq( - opts.bootstrapServerOpt, - opts.commandConfigOpt, - opts.preserveThrottlesOpt, - opts.timeoutOpt - ), - opts.listOpt -> Seq( - opts.bootstrapServerOpt, - opts.commandConfigOpt - ) - ) - opts.options.specs.forEach(opt => { - if (!opt.equals(action) && - !requiredArgs(action).contains(opt) && - !permittedArgs(action).contains(opt)) { - CommandLineUtils.printUsageAndExit(opts.parser, - """Option "%s" can't be used with action "%s"""".format(opt, action)) - } - }) - - opts - } - - def alterReplicaLogDirs(adminClient: Admin, - assignment: Map[TopicPartitionReplica, String]) - : Set[TopicPartitionReplica] = { - adminClient.alterReplicaLogDirs(assignment.asJava).values().asScala.flatMap { - case (replica, future) => { - try { - future.get() - Some(replica) - } catch { - case t: ExecutionException => - t.getCause match { - // Ignore ReplicaNotAvailableException. It is OK if the replica is not - // available at this moment. - case _: ReplicaNotAvailableException => None - case e: Throwable => - throw new AdminCommandFailedException(s"Failed to alter dir for $replica", e) - } - } - } - }.toSet - } - - sealed class ReassignPartitionsCommandOptions(args: Array[String]) extends CommandDefaultOptions(args) { - // Actions - val verifyOpt = parser.accepts("verify", "Verify if the reassignment completed as specified by the " + - "--reassignment-json-file option. If there is a throttle engaged for the replicas specified, and the rebalance has completed, the throttle will be removed") - val generateOpt = parser.accepts("generate", "Generate a candidate partition reassignment configuration." + - " Note that this only generates a candidate assignment, it does not execute it.") - val executeOpt = parser.accepts("execute", "Kick off the reassignment as specified by the --reassignment-json-file option.") - val cancelOpt = parser.accepts("cancel", "Cancel an active reassignment.") - val listOpt = parser.accepts("list", "List all active partition reassignments.") - - // Arguments - val bootstrapServerOpt = parser.accepts("bootstrap-server", "REQUIRED: the server(s) to use for bootstrapping.") - .withRequiredArg - .describedAs("Server(s) to use for bootstrapping") - .ofType(classOf[String]) - - val commandConfigOpt = parser.accepts("command-config", "Property file containing configs to be passed to Admin Client.") - .withRequiredArg - .describedAs("Admin client property file") - .ofType(classOf[String]) - - val reassignmentJsonFileOpt = parser.accepts("reassignment-json-file", "The JSON file with the partition reassignment configuration" + - "The format to use is - \n" + - "{\"partitions\":\n\t[{\"topic\": \"foo\",\n\t \"partition\": 1,\n\t \"replicas\": [1,2,3],\n\t \"log_dirs\": [\"dir1\",\"dir2\",\"dir3\"] }],\n\"version\":1\n}\n" + - "Note that \"log_dirs\" is optional. When it is specified, its length must equal the length of the replicas list. The value in this list " + - "can be either \"any\" or the absolution path of the log directory on the broker. If absolute log directory path is specified, the replica will be moved to the specified log directory on the broker.") - .withRequiredArg - .describedAs("manual assignment json file path") - .ofType(classOf[String]) - val topicsToMoveJsonFileOpt = parser.accepts("topics-to-move-json-file", "Generate a reassignment configuration to move the partitions" + - " of the specified topics to the list of brokers specified by the --broker-list option. The format to use is - \n" + - "{\"topics\":\n\t[{\"topic\": \"foo\"},{\"topic\": \"foo1\"}],\n\"version\":1\n}") - .withRequiredArg - .describedAs("topics to reassign json file path") - .ofType(classOf[String]) - val brokerListOpt = parser.accepts("broker-list", "The list of brokers to which the partitions need to be reassigned" + - " in the form \"0,1,2\". This is required if --topics-to-move-json-file is used to generate reassignment configuration") - .withRequiredArg - .describedAs("brokerlist") - .ofType(classOf[String]) - val disableRackAware = parser.accepts("disable-rack-aware", "Disable rack aware replica assignment") - val interBrokerThrottleOpt = parser.accepts("throttle", "The movement of partitions between brokers will be throttled to this value (bytes/sec). " + - "This option can be included with --execute when a reassignment is started, and it can be altered by resubmitting the current reassignment " + - "along with the --additional flag. The throttle rate should be at least 1 KB/s.") - .withRequiredArg() - .describedAs("throttle") - .ofType(classOf[Long]) - .defaultsTo(-1) - val replicaAlterLogDirsThrottleOpt = parser.accepts("replica-alter-log-dirs-throttle", - "The movement of replicas between log directories on the same broker will be throttled to this value (bytes/sec). " + - "This option can be included with --execute when a reassignment is started, and it can be altered by resubmitting the current reassignment " + - "along with the --additional flag. The throttle rate should be at least 1 KB/s.") - .withRequiredArg() - .describedAs("replicaAlterLogDirsThrottle") - .ofType(classOf[Long]) - .defaultsTo(-1) - val timeoutOpt = parser.accepts("timeout", "The maximum time in ms to wait for log directory replica assignment to begin.") - .withRequiredArg() - .describedAs("timeout") - .ofType(classOf[Long]) - .defaultsTo(10000) - val additionalOpt = parser.accepts("additional", "Execute this reassignment in addition to any " + - "other ongoing ones. This option can also be used to change the throttle of an ongoing reassignment.") - val preserveThrottlesOpt = parser.accepts("preserve-throttles", "Do not modify broker or topic throttles.") - options = parser.parse(args : _*) - } -} diff --git a/core/src/test/scala/other/kafka/ReplicationQuotasTestRig.scala b/core/src/test/scala/other/kafka/ReplicationQuotasTestRig.scala deleted file mode 100644 index dddd33f2d54ec..0000000000000 --- a/core/src/test/scala/other/kafka/ReplicationQuotasTestRig.scala +++ /dev/null @@ -1,341 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package kafka - -import java.io.{File, PrintWriter} -import java.nio.charset.StandardCharsets -import java.nio.file.{Files, StandardOpenOption} - -import javax.imageio.ImageIO -import kafka.admin.ReassignPartitionsCommand -import kafka.server.{KafkaConfig, KafkaServer, QuorumTestHarness, QuotaType} -import kafka.utils.TestUtils._ -import kafka.utils.{EmptyTestInfo, Exit, Logging, TestUtils} -import kafka.zk.ReassignPartitionsZNode -import org.apache.kafka.clients.admin.{Admin, AdminClientConfig} -import org.apache.kafka.clients.producer.ProducerRecord -import org.apache.kafka.common.TopicPartition -import org.apache.kafka.common.utils.Utils -import org.jfree.chart.plot.PlotOrientation -import org.jfree.chart.{ChartFactory, ChartFrame, JFreeChart} -import org.jfree.data.xy.{XYSeries, XYSeriesCollection} - -import scala.jdk.CollectionConverters._ -import scala.collection.{Map, Seq, mutable} - -/** - * Test rig for measuring throttling performance. Configure the parameters for a set of experiments, then execute them - * and view the html output file, with charts, that are produced. You can also render the charts to the screen if - * you wish. - * - * Currently you'll need about 40GB of disk space to run these experiments (largest data written x2). Tune the msgSize - * & #partitions and throttle to adjust this. - */ -object ReplicationQuotasTestRig { - new File("Experiments").mkdir() - private val dir = "Experiments/Run" + System.currentTimeMillis().toString.substring(8) - new File(dir).mkdir() - val k = 1000 * 1000 - - - def main(args: Array[String]): Unit = { - val displayChartsOnScreen = if (args.length > 0 && args(0) == "show-gui") true else false - val journal = new Journal() - - val experiments = Seq( - //1GB total data written, will take 210s - new ExperimentDef("Experiment1", brokers = 5, partitions = 20, throttle = 1 * k, msgsPerPartition = 500, msgSize = 100 * 1000), - //5GB total data written, will take 110s - new ExperimentDef("Experiment2", brokers = 5, partitions = 50, throttle = 10 * k, msgsPerPartition = 1000, msgSize = 100 * 1000), - //5GB total data written, will take 110s - new ExperimentDef("Experiment3", brokers = 50, partitions = 50, throttle = 2 * k, msgsPerPartition = 1000, msgSize = 100 * 1000), - //10GB total data written, will take 110s - new ExperimentDef("Experiment4", brokers = 25, partitions = 100, throttle = 4 * k, msgsPerPartition = 1000, msgSize = 100 * 1000), - //10GB total data written, will take 80s - new ExperimentDef("Experiment5", brokers = 5, partitions = 50, throttle = 50 * k, msgsPerPartition = 4000, msgSize = 100 * 1000) - ) - experiments.foreach(run(_, journal, displayChartsOnScreen)) - - if (!displayChartsOnScreen) - Exit.exit(0) - } - - def run(config: ExperimentDef, journal: Journal, displayChartsOnScreen: Boolean): Unit = { - val experiment = new Experiment() - try { - experiment.setUp(new EmptyTestInfo()) - experiment.run(config, journal, displayChartsOnScreen) - journal.footer() - } - catch { - case e: Exception => e.printStackTrace() - } - finally { - experiment.tearDown() - } - } - - case class ExperimentDef(name: String, brokers: Int, partitions: Int, throttle: Long, msgsPerPartition: Int, msgSize: Int) { - val targetBytesPerBrokerMB: Long = msgsPerPartition.toLong * msgSize.toLong * partitions.toLong / brokers.toLong / 1000000 - } - - class Experiment extends QuorumTestHarness with Logging { - val topicName = "my-topic" - var experimentName = "unset" - val partitionId = 0 - var servers: Seq[KafkaServer] = _ - val leaderRates = mutable.Map[Int, Array[Double]]() - val followerRates = mutable.Map[Int, Array[Double]]() - var adminClient: Admin = _ - - def startBrokers(brokerIds: Seq[Int]): Unit = { - println("Starting Brokers") - servers = brokerIds.map(i => createBrokerConfig(i, zkConnect)) - .map(c => createServer(KafkaConfig.fromProps(c))) - - TestUtils.waitUntilBrokerMetadataIsPropagated(servers) - val brokerList = TestUtils.plaintextBootstrapServers(servers) - adminClient = Admin.create(Map[String, Object]( - AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG -> brokerList - ).asJava) - } - - override def tearDown(): Unit = { - Utils.closeQuietly(adminClient, "adminClient") - TestUtils.shutdownServers(servers) - super.tearDown() - } - - def run(config: ExperimentDef, journal: Journal, displayChartsOnScreen: Boolean): Unit = { - experimentName = config.name - val brokers = (100 to 100 + config.brokers) - var count = 0 - val shift = Math.round(config.brokers / 2f) - - def nextReplicaRoundRobin(): Int = { - count = count + 1 - 100 + (count + shift) % config.brokers - } - val replicas = (0 to config.partitions).map(partition => partition -> Seq(nextReplicaRoundRobin())).toMap - - startBrokers(brokers) - createTopic(zkClient, topicName, replicas, servers) - - println("Writing Data") - val producer = TestUtils.createProducer(TestUtils.plaintextBootstrapServers(servers), acks = 0) - (0 until config.msgsPerPartition).foreach { x => - (0 until config.partitions).foreach { partition => - producer.send(new ProducerRecord(topicName, partition, null, new Array[Byte](config.msgSize))) - } - } - - println("Generating Reassignment") - val (newAssignment, _) = ReassignPartitionsCommand.generateAssignment(adminClient, - json(topicName), brokers.mkString(","), true) - - println("Starting Reassignment") - val start = System.currentTimeMillis() - ReassignPartitionsCommand.executeAssignment(adminClient, false, - new String(ReassignPartitionsZNode.encode(newAssignment), StandardCharsets.UTF_8), - config.throttle) - - //Await completion - waitForReassignmentToComplete() - println(s"Reassignment took ${(System.currentTimeMillis() - start)/1000}s") - - validateAllOffsetsMatch(config) - - journal.appendToJournal(config) - renderChart(leaderRates, "Leader", journal, displayChartsOnScreen) - renderChart(followerRates, "Follower", journal, displayChartsOnScreen) - logOutput(config, replicas, newAssignment) - - println("Output can be found here: " + journal.path()) - } - - def validateAllOffsetsMatch(config: ExperimentDef): Unit = { - //Validate that offsets are correct in all brokers - for (broker <- servers) { - (0 until config.partitions).foreach { partitionId => - val offset = broker.getLogManager.getLog(new TopicPartition(topicName, partitionId)).map(_.logEndOffset).getOrElse(-1L) - if (offset >= 0 && offset != config.msgsPerPartition) { - throw new RuntimeException(s"Run failed as offsets did not match for partition $partitionId on broker ${broker.config.brokerId}. Expected ${config.msgsPerPartition} but was $offset.") - } - } - } - } - - def logOutput(config: ExperimentDef, replicas: Map[Int, Seq[Int]], newAssignment: Map[TopicPartition, Seq[Int]]): Unit = { - val actual = zkClient.getPartitionAssignmentForTopics(Set(topicName))(topicName) - - //Long stats - println("The replicas are " + replicas.toSeq.sortBy(_._1).map("\n" + _)) - println("This is the current replica assignment:\n" + actual.map { case (k, v) => k -> v.replicas }) - println("proposed assignment is: \n" + newAssignment) - println("This is the assignment we ended up with" + actual.map { case (k, v) => k -> v.replicas }) - - //Test Stats - println(s"numBrokers: ${config.brokers}") - println(s"numPartitions: ${config.partitions}") - println(s"throttle: ${config.throttle}") - println(s"numMessagesPerPartition: ${config.msgsPerPartition}") - println(s"msgSize: ${config.msgSize}") - println(s"We will write ${config.targetBytesPerBrokerMB}MB of data per broker") - println(s"Worst case duration is ${config.targetBytesPerBrokerMB * 1000 * 1000/ config.throttle}") - } - - def waitForReassignmentToComplete(): Unit = { - waitUntilTrue(() => { - printRateMetrics() - adminClient.listPartitionReassignments().reassignments().get().isEmpty - }, s"Partition reassignments didn't complete.", 60 * 60 * 1000, pause = 1000L) - } - - def renderChart(data: mutable.Map[Int, Array[Double]], name: String, journal: Journal, displayChartsOnScreen: Boolean): Unit = { - val dataset = addDataToChart(data) - val chart = createChart(name, dataset) - - writeToFile(name, journal, chart) - maybeDisplayOnScreen(displayChartsOnScreen, chart) - println(s"Chart generated for $name") - } - - def maybeDisplayOnScreen(displayChartsOnScreen: Boolean, chart: JFreeChart): Unit = { - if (displayChartsOnScreen) { - val frame = new ChartFrame(experimentName, chart) - frame.pack() - frame.setVisible(true) - } - } - - def writeToFile(name: String, journal: Journal, chart: JFreeChart): Unit = { - val file = new File(dir, experimentName + "-" + name + ".png") - ImageIO.write(chart.createBufferedImage(1000, 700), "png", file) - journal.appendChart(file.getAbsolutePath, name.eq("Leader")) - } - - def createChart(name: String, dataset: XYSeriesCollection): JFreeChart = { - val chart: JFreeChart = ChartFactory.createXYLineChart( - experimentName + " - " + name + " Throttling Performance", - "Time (s)", - "Throttle Throughput (B/s)", - dataset - , PlotOrientation.VERTICAL, false, true, false - ) - chart - } - - def addDataToChart(data: mutable.Map[Int, Array[Double]]): XYSeriesCollection = { - val dataset = new XYSeriesCollection - data.foreach { case (broker, values) => - val series = new XYSeries("Broker:" + broker) - var x = 0 - values.foreach { value => - series.add(x, value) - x += 1 - } - dataset.addSeries(series) - } - dataset - } - - def record(rates: mutable.Map[Int, Array[Double]], brokerId: Int, currentRate: Double) = { - var leaderRatesBroker: Array[Double] = rates.getOrElse(brokerId, Array[Double]()) - leaderRatesBroker = leaderRatesBroker ++ Array(currentRate) - rates.put(brokerId, leaderRatesBroker) - } - - def printRateMetrics(): Unit = { - for (broker <- servers) { - val leaderRate: Double = measuredRate(broker, QuotaType.LeaderReplication) - if (broker.config.brokerId == 100) - info("waiting... Leader rate on 101 is " + leaderRate) - record(leaderRates, broker.config.brokerId, leaderRate) - if (leaderRate > 0) - trace("Leader Rate on " + broker.config.brokerId + " is " + leaderRate) - - val followerRate: Double = measuredRate(broker, QuotaType.FollowerReplication) - record(followerRates, broker.config.brokerId, followerRate) - if (followerRate > 0) - trace("Follower Rate on " + broker.config.brokerId + " is " + followerRate) - } - } - - private def measuredRate(broker: KafkaServer, repType: QuotaType): Double = { - val metricName = broker.metrics.metricName("byte-rate", repType.toString) - if (broker.metrics.metrics.asScala.contains(metricName)) - broker.metrics.metrics.asScala(metricName).metricValue.asInstanceOf[Double] - else -1 - } - - def json(topic: String*): String = { - val topicStr = topic.map { - t => "{\"topic\": \"" + t + "\"}" - }.mkString(",") - s"""{"topics": [$topicStr],"version":1}""" - } - } - - class Journal { - private val log = new File(dir, "Log.html") - header() - - def appendToJournal(config: ExperimentDef): Unit = { - val message = s"\n\n

${config.name}

" + - s"

- BrokerCount: ${config.brokers}" + - s"

- PartitionCount: ${config.partitions}" + - f"

- Throttle: ${config.throttle.toDouble}%,.0f MB/s" + - f"

- MsgCount: ${config.msgsPerPartition}%,.0f " + - f"

- MsgSize: ${config.msgSize}%,.0f" + - s"

- TargetBytesPerBrokerMB: ${config.targetBytesPerBrokerMB}

" - append(message) - } - - def appendChart(path: String, first: Boolean): Unit = { - val message = new StringBuilder - if (first) - message.append("

") - message.append("\"Chart\"") - if (!first) - message.append("

") - append(message.toString()) - } - - def header(): Unit = { - append("

Replication Quotas Test Rig

") - } - - def footer(): Unit = { - append("") - } - - def append(message: String): Unit = { - val stream = Files.newOutputStream(log.toPath, StandardOpenOption.CREATE, StandardOpenOption.APPEND) - new PrintWriter(stream) { - append(message) - close - } - } - - def path(): String = { - log.getAbsolutePath - } - } - -} - From ac1705a3a6655b98672fb95975805c6bd35446f3 Mon Sep 17 00:00:00 2001 From: nizhikov Date: Thu, 3 Aug 2023 13:00:54 +0300 Subject: [PATCH 28/78] KAFKA-14595 ReassignPartitionsIntegrationTest rewritten in java --- .../tools/reassign/ReassignPartitionsIntegrationTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsIntegrationTest.java b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsIntegrationTest.java index 819c0dc5f111d..196357151aa9d 100644 --- a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsIntegrationTest.java +++ b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsIntegrationTest.java @@ -64,6 +64,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Properties; import java.util.Set; @@ -431,7 +432,7 @@ public void testCancellationWithAddingReplicaInIsr(String quorum) throws Excepti -1L ); TestUtils.waitUntilTrue( - () -> JavaConverters.asJava(TestUtils.currentIsr(cluster.adminClient, foo0)).equals(new HashSet<>(Arrays.asList(0, 1, 2, 3))), + () -> Objects.equals(TestUtils.currentIsr(cluster.adminClient, foo0), toSet(Arrays.asList(0, 1, 2, 3))), () -> "Timed out while waiting for replica 3 to join the ISR", org.apache.kafka.test.TestUtils.DEFAULT_MAX_WAIT_MS, 100L ); From d80dc6953a637e5f53af1c98a7a57e243add7576 Mon Sep 17 00:00:00 2001 From: nizhikov Date: Fri, 11 Aug 2023 16:12:35 +0300 Subject: [PATCH 29/78] KAFKA-14595 Remove TerseReassignmentFailureException exception --- .../reassign/ReassignPartitionsCommand.java | 19 ++++++----- .../TerseReassignmentFailureException.java | 34 ------------------- 2 files changed, 10 insertions(+), 43 deletions(-) delete mode 100644 tools/src/main/java/org/apache/kafka/tools/reassign/TerseReassignmentFailureException.java diff --git a/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommand.java b/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommand.java index e128cf179da21..535503de4b8da 100644 --- a/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommand.java +++ b/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommand.java @@ -47,6 +47,7 @@ import org.apache.kafka.server.util.json.DecodeJson; import org.apache.kafka.server.util.json.JsonObject; import org.apache.kafka.server.util.json.JsonValue; +import org.apache.kafka.tools.TerseException; import org.apache.kafka.tools.ToolsUtils; import java.io.IOException; @@ -139,7 +140,7 @@ public static void main(String[] args) { adminClient = Admin.create(props); handleAction(adminClient, opts); failed = false; - } catch (TerseReassignmentFailureException e) { + } catch (TerseException e) { System.out.println(e.getMessage()); } catch (Throwable e) { System.out.println("Error: " + e.getMessage()); @@ -156,7 +157,7 @@ public static void main(String[] args) { } } - private static void handleAction(Admin adminClient, ReassignPartitionsCommandOptions opts) throws IOException, ExecutionException, InterruptedException { + private static void handleAction(Admin adminClient, ReassignPartitionsCommandOptions opts) throws IOException, ExecutionException, InterruptedException, TerseException { if (opts.options.has(opts.verifyOpt)) { verifyAssignment(adminClient, Utils.readFileAsString(opts.options.valueOf(opts.reassignmentJsonFileOpt)), @@ -728,7 +729,7 @@ static void executeAssignment(Admin adminClient, Long interBrokerThrottle, Long logDirThrottle, Long timeoutMs, - Time time) throws ExecutionException, InterruptedException, JsonProcessingException { + Time time) throws ExecutionException, InterruptedException, JsonProcessingException, TerseException { Tuple>, Map> t0 = parseExecuteAssignmentArgs(reassignmentJson); Map> proposedParts = t0.v1; @@ -738,7 +739,7 @@ static void executeAssignment(Admin adminClient, // If there is an existing assignment, check for --additional before proceeding. // This helps avoid surprising users. if (!additional && !currentReassignments.isEmpty()) { - throw new TerseReassignmentFailureException(CANNOT_EXECUTE_BECAUSE_OF_EXISTING_MESSAGE); + throw new TerseException(CANNOT_EXECUTE_BECAUSE_OF_EXISTING_MESSAGE); } Set brokers = new HashSet<>(); proposedParts.values().forEach(brokers::addAll); @@ -764,7 +765,7 @@ static void executeAssignment(Admin adminClient, // Execute the partition reassignments. Map errors = alterPartitionReassignments(adminClient, proposedParts); if (!errors.isEmpty()) { - throw new TerseReassignmentFailureException( + throw new TerseException( String.format("Error reassigning partition(s):%n%s", errors.keySet().stream(). sorted(ReassignPartitionsCommand::compareTopicPartitions). @@ -795,7 +796,7 @@ static void executeAssignment(Admin adminClient, private static void executeMoves(Admin adminClient, Map proposedReplicas, Long timeoutMs, - Time time) throws InterruptedException { + Time time) throws InterruptedException, TerseException { long startTimeMs = time.milliseconds(); Map pendingReplicas = new HashMap<>(); pendingReplicas.putAll(proposedReplicas); @@ -814,7 +815,7 @@ private static void executeMoves(Admin adminClient, if (pendingReplicas.isEmpty()) { done = true; } else if (time.milliseconds() >= startTimeMs + timeoutMs) { - throw new TerseReassignmentFailureException(String.format( + throw new TerseException(String.format( "Timed out before log directory move%s could be started for: %s", pendingReplicas.size() == 1 ? "" : "s", pendingReplicas.keySet().stream(). @@ -1234,7 +1235,7 @@ static Tuple, Set> cancelAssignment(A String jsonString, Boolean preserveThrottles, Long timeoutMs, - Time time) throws ExecutionException, InterruptedException, JsonProcessingException { + Time time) throws ExecutionException, InterruptedException, JsonProcessingException, TerseException { Tuple>>, Map> t0 = parsePartitionReassignmentData(jsonString); List>> targetParts = t0.v1; @@ -1248,7 +1249,7 @@ static Tuple, Set> cancelAssignment(A if (!curReassigningParts.isEmpty()) { Map errors = cancelPartitionReassignments(adminClient, curReassigningParts); if (!errors.isEmpty()) { - throw new TerseReassignmentFailureException(String.format( + throw new TerseException(String.format( "Error cancelling partition reassignment%s for:%n%s", errors.size() == 1 ? "" : "s", errors.keySet().stream(). diff --git a/tools/src/main/java/org/apache/kafka/tools/reassign/TerseReassignmentFailureException.java b/tools/src/main/java/org/apache/kafka/tools/reassign/TerseReassignmentFailureException.java deleted file mode 100644 index 524e25ff53cf8..0000000000000 --- a/tools/src/main/java/org/apache/kafka/tools/reassign/TerseReassignmentFailureException.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kafka.tools.reassign; - -import org.apache.kafka.common.KafkaException; - -/** - * An exception thrown to indicate that the command has failed, but we don't want to - * print a stack trace. - */ -public class TerseReassignmentFailureException extends KafkaException { - /** - * @param message The message to print out before exiting. A stack trace will not - * be printed. - */ - public TerseReassignmentFailureException(String message) { - super(message); - } -} From 145dd4c311324daea259c3d30a18a96a98e6bf9f Mon Sep 17 00:00:00 2001 From: nizhikov Date: Fri, 11 Aug 2023 16:22:08 +0300 Subject: [PATCH 30/78] KAFKA-14595 Compilation fix --- .../tools/reassign/ReassignPartitionsIntegrationTest.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsIntegrationTest.java b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsIntegrationTest.java index 196357151aa9d..9939f1e2a2f81 100644 --- a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsIntegrationTest.java +++ b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsIntegrationTest.java @@ -43,6 +43,7 @@ import org.apache.kafka.common.serialization.ByteArrayDeserializer; import org.apache.kafka.common.utils.Time; import org.apache.kafka.common.utils.Utils; +import org.apache.kafka.tools.TerseException; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Timeout; import org.junit.jupiter.params.ParameterizedTest; @@ -146,7 +147,7 @@ public void testReassignmentCompletionDuringPartialUpgrade(String quorum) throws executeAndVerifyReassignment(); } - private void executeAndVerifyReassignment() throws ExecutionException, InterruptedException, JsonProcessingException { + private void executeAndVerifyReassignment() throws ExecutionException, InterruptedException, JsonProcessingException, TerseException { String assignment = "{\"version\":1,\"partitions\":" + "[{\"topic\":\"foo\",\"partition\":0,\"replicas\":[0,1,3],\"log_dirs\":[\"any\",\"any\",\"any\"]}," + "{\"topic\":\"bar\",\"partition\":0,\"replicas\":[3,2,0],\"log_dirs\":[\"any\",\"any\",\"any\"]}" + @@ -683,7 +684,7 @@ private void runExecuteAssignment(Admin adminClient, Boolean additional, String reassignmentJson, Long interBrokerThrottle, - Long replicaAlterLogDirsThrottle) throws ExecutionException, InterruptedException, JsonProcessingException { + Long replicaAlterLogDirsThrottle) throws ExecutionException, InterruptedException, JsonProcessingException, TerseException { System.out.println("==> executeAssignment(adminClient, additional=" + additional + ", " + "reassignmentJson=" + reassignmentJson + ", " + "interBrokerThrottle=" + interBrokerThrottle + ", " + @@ -692,7 +693,7 @@ private void runExecuteAssignment(Admin adminClient, interBrokerThrottle, replicaAlterLogDirsThrottle, 10000L, Time.SYSTEM); } - private Tuple, Set> runCancelAssignment(Admin adminClient, String jsonString, Boolean preserveThrottles) throws ExecutionException, InterruptedException, JsonProcessingException { + private Tuple, Set> runCancelAssignment(Admin adminClient, String jsonString, Boolean preserveThrottles) throws ExecutionException, InterruptedException, JsonProcessingException, TerseException { System.out.println("==> cancelAssignment(adminClient, jsonString=" + jsonString); return cancelAssignment(adminClient, jsonString, preserveThrottles, 10000L, Time.SYSTEM); } From 996e30b29d20b5735ba7bcc764fee513715e9008 Mon Sep 17 00:00:00 2001 From: nizhikov Date: Tue, 15 Aug 2023 20:57:14 +0300 Subject: [PATCH 31/78] KAFKA-14595 Separate test move --- checkstyle/import-control.xml | 7 + .../admin/ReassignPartitionsCommand.scala | 14 +- .../ReassignPartitionsCommandArgsTest.scala | 284 ------ .../admin/ReassignPartitionsUnitTest.scala | 671 ------------- .../ReassignPartitionsCommandArgsTest.java | 292 ++++++ .../reassign/ReassignPartitionsUnitTest.java | 925 ++++++++++++++++++ 6 files changed, 1231 insertions(+), 962 deletions(-) delete mode 100644 core/src/test/scala/unit/kafka/admin/ReassignPartitionsCommandArgsTest.scala delete mode 100644 core/src/test/scala/unit/kafka/admin/ReassignPartitionsUnitTest.scala create mode 100644 tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommandArgsTest.java create mode 100644 tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java diff --git a/checkstyle/import-control.xml b/checkstyle/import-control.xml index 44673899772c5..9e1854efdf713 100644 --- a/checkstyle/import-control.xml +++ b/checkstyle/import-control.xml @@ -300,9 +300,11 @@ + + @@ -319,6 +321,11 @@ + + + + + diff --git a/core/src/main/scala/kafka/admin/ReassignPartitionsCommand.scala b/core/src/main/scala/kafka/admin/ReassignPartitionsCommand.scala index 2fc36648981d3..205eb9ded935e 100755 --- a/core/src/main/scala/kafka/admin/ReassignPartitionsCommand.scala +++ b/core/src/main/scala/kafka/admin/ReassignPartitionsCommand.scala @@ -57,24 +57,24 @@ object ReassignPartitionsCommand extends Logging { private[admin] val EarliestTopicsJsonVersion = 1 // Throttles that are set at the level of an individual broker. - private[admin] val brokerLevelLeaderThrottle = + val brokerLevelLeaderThrottle = DynamicConfig.Broker.LeaderReplicationThrottledRateProp - private[admin] val brokerLevelFollowerThrottle = + val brokerLevelFollowerThrottle = DynamicConfig.Broker.FollowerReplicationThrottledRateProp - private[admin] val brokerLevelLogDirThrottle = + val brokerLevelLogDirThrottle = DynamicConfig.Broker.ReplicaAlterLogDirsIoMaxBytesPerSecondProp - private[admin] val brokerLevelThrottles = Seq( + val brokerLevelThrottles = Seq( brokerLevelLeaderThrottle, brokerLevelFollowerThrottle, brokerLevelLogDirThrottle ) // Throttles that are set at the level of an individual topic. - private[admin] val topicLevelLeaderThrottle = + val topicLevelLeaderThrottle = LogConfig.LEADER_REPLICATION_THROTTLED_REPLICAS_CONFIG - private[admin] val topicLevelFollowerThrottle = + val topicLevelFollowerThrottle = LogConfig.FOLLOWER_REPLICATION_THROTTLED_REPLICAS_CONFIG - private[admin] val topicLevelThrottles = Seq( + val topicLevelThrottles = Seq( topicLevelLeaderThrottle, topicLevelFollowerThrottle ) diff --git a/core/src/test/scala/unit/kafka/admin/ReassignPartitionsCommandArgsTest.scala b/core/src/test/scala/unit/kafka/admin/ReassignPartitionsCommandArgsTest.scala deleted file mode 100644 index 98d54629f2081..0000000000000 --- a/core/src/test/scala/unit/kafka/admin/ReassignPartitionsCommandArgsTest.scala +++ /dev/null @@ -1,284 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package kafka.admin - -import kafka.utils.Exit -import org.junit.jupiter.api.Assertions._ -import org.junit.jupiter.api.{AfterEach, BeforeEach, Test, Timeout} - -@Timeout(60) -class ReassignPartitionsCommandArgsTest { - - val missingBootstrapServerMsg = "Please specify --bootstrap-server" - - @BeforeEach - def setUp(): Unit = { - Exit.setExitProcedure((_, message) => throw new IllegalArgumentException(message.orNull)) - } - - @AfterEach - def tearDown(): Unit = { - Exit.resetExitProcedure() - } - - ///// Test valid argument parsing - @Test - def shouldCorrectlyParseValidMinimumGenerateOptions(): Unit = { - val args = Array( - "--bootstrap-server", "localhost:1234", - "--generate", - "--broker-list", "101,102", - "--topics-to-move-json-file", "myfile.json") - ReassignPartitionsCommand.validateAndParseArgs(args) - } - - @Test - def shouldCorrectlyParseValidMinimumExecuteOptions(): Unit = { - val args = Array( - "--bootstrap-server", "localhost:1234", - "--execute", - "--reassignment-json-file", "myfile.json") - ReassignPartitionsCommand.validateAndParseArgs(args) - } - - @Test - def shouldCorrectlyParseValidMinimumVerifyOptions(): Unit = { - val args = Array( - "--bootstrap-server", "localhost:1234", - "--verify", - "--reassignment-json-file", "myfile.json") - ReassignPartitionsCommand.validateAndParseArgs(args) - } - - @Test - def shouldAllowThrottleOptionOnExecute(): Unit = { - val args = Array( - "--bootstrap-server", "localhost:1234", - "--execute", - "--throttle", "100", - "--reassignment-json-file", "myfile.json") - ReassignPartitionsCommand.validateAndParseArgs(args) - } - - @Test - def shouldUseDefaultsIfEnabled(): Unit = { - val args = Array( - "--bootstrap-server", "localhost:1234", - "--execute", - "--reassignment-json-file", "myfile.json") - val opts = ReassignPartitionsCommand.validateAndParseArgs(args) - assertEquals(10000L, opts.options.valueOf(opts.timeoutOpt)) - assertEquals(-1L, opts.options.valueOf(opts.interBrokerThrottleOpt)) - } - - @Test - def testList(): Unit = { - val args = Array( - "--list", - "--bootstrap-server", "localhost:1234") - ReassignPartitionsCommand.validateAndParseArgs(args) - } - - @Test - def testCancelWithPreserveThrottlesOption(): Unit = { - val args = Array( - "--cancel", - "--bootstrap-server", "localhost:1234", - "--reassignment-json-file", "myfile.json", - "--preserve-throttles") - ReassignPartitionsCommand.validateAndParseArgs(args) - } - - ///// Test handling missing or invalid actions - @Test - def shouldFailIfNoArgs(): Unit = { - val args: Array[String]= Array() - shouldFailWith(ReassignPartitionsCommand.helpText, args) - } - - @Test - def shouldFailIfBlankArg(): Unit = { - val args = Array(" ") - shouldFailWith("Command must include exactly one action", args) - } - - @Test - def shouldFailIfMultipleActions(): Unit = { - val args = Array( - "--bootstrap-server", "localhost:1234", - "--execute", - "--verify", - "--reassignment-json-file", "myfile.json" - ) - shouldFailWith("Command must include exactly one action", args) - } - - ///// Test --execute - @Test - def shouldNotAllowExecuteWithTopicsOption(): Unit = { - val args = Array( - "--bootstrap-server", "localhost:1234", - "--execute", - "--reassignment-json-file", "myfile.json", - "--topics-to-move-json-file", "myfile.json") - shouldFailWith("Option \"[topics-to-move-json-file]\" can't be used with action \"[execute]\"", args) - } - - @Test - def shouldNotAllowExecuteWithBrokerList(): Unit = { - val args = Array( - "--bootstrap-server", "localhost:1234", - "--execute", - "--reassignment-json-file", "myfile.json", - "--broker-list", "101,102" - ) - shouldFailWith("Option \"[broker-list]\" can't be used with action \"[execute]\"", args) - } - - @Test - def shouldNotAllowExecuteWithoutReassignmentOption(): Unit = { - val args = Array( - "--bootstrap-server", "localhost:1234", - "--execute") - shouldFailWith("Missing required argument \"[reassignment-json-file]\"", args) - } - - @Test - def testMissingBootstrapServerArgumentForExecute(): Unit = { - val args = Array( - "--execute") - shouldFailWith(missingBootstrapServerMsg, args) - } - - ///// Test --generate - @Test - def shouldNotAllowGenerateWithoutBrokersAndTopicsOptions(): Unit = { - val args = Array( - "--bootstrap-server", "localhost:1234", - "--generate") - shouldFailWith("Missing required argument \"[topics-to-move-json-file]\"", args) - } - - @Test - def shouldNotAllowGenerateWithoutBrokersOption(): Unit = { - val args = Array( - "--bootstrap-server", "localhost:1234", - "--topics-to-move-json-file", "myfile.json", - "--generate") - shouldFailWith("Missing required argument \"[broker-list]\"", args) - } - - @Test - def shouldNotAllowGenerateWithoutTopicsOption(): Unit = { - val args = Array( - "--bootstrap-server", "localhost:1234", - "--broker-list", "101,102", - "--generate") - shouldFailWith("Missing required argument \"[topics-to-move-json-file]\"", args) - } - - @Test - def shouldNotAllowGenerateWithThrottleOption(): Unit = { - val args = Array( - "--bootstrap-server", "localhost:1234", - "--generate", - "--broker-list", "101,102", - "--throttle", "100", - "--topics-to-move-json-file", "myfile.json") - shouldFailWith("Option \"[throttle]\" can't be used with action \"[generate]\"", args) - } - - @Test - def shouldNotAllowGenerateWithReassignmentOption(): Unit = { - val args = Array( - "--bootstrap-server", "localhost:1234", - "--generate", - "--broker-list", "101,102", - "--topics-to-move-json-file", "myfile.json", - "--reassignment-json-file", "myfile.json") - shouldFailWith("Option \"[reassignment-json-file]\" can't be used with action \"[generate]\"", args) - } - - @Test - def shouldPrintHelpTextIfHelpArg(): Unit = { - val args: Array[String]= Array("--help") - // note, this is not actually a failed case, it's just we share the same `printUsageAndExit` method when wrong arg received - shouldFailWith(ReassignPartitionsCommand.helpText, args) - } - - ///// Test --verify - @Test - def shouldNotAllowVerifyWithoutReassignmentOption(): Unit = { - val args = Array( - "--bootstrap-server", "localhost:1234", - "--verify") - shouldFailWith("Missing required argument \"[reassignment-json-file]\"", args) - } - - @Test - def shouldNotAllowBrokersListWithVerifyOption(): Unit = { - val args = Array( - "--bootstrap-server", "localhost:1234", - "--verify", - "--broker-list", "100,101", - "--reassignment-json-file", "myfile.json") - shouldFailWith("Option \"[broker-list]\" can't be used with action \"[verify]\"", args) - } - - @Test - def shouldNotAllowThrottleWithVerifyOption(): Unit = { - val args = Array( - "--bootstrap-server", "localhost:1234", - "--verify", - "--throttle", "100", - "--reassignment-json-file", "myfile.json") - shouldFailWith("Option \"[throttle]\" can't be used with action \"[verify]\"", args) - } - - @Test - def shouldNotAllowTopicsOptionWithVerify(): Unit = { - val args = Array( - "--bootstrap-server", "localhost:1234", - "--verify", - "--reassignment-json-file", "myfile.json", - "--topics-to-move-json-file", "myfile.json") - shouldFailWith("Option \"[topics-to-move-json-file]\" can't be used with action \"[verify]\"", args) - } - - def shouldFailWith(msg: String, args: Array[String]): Unit = { - val e = assertThrows(classOf[Exception], () => ReassignPartitionsCommand.validateAndParseArgs(args), - () => s"Should have failed with [$msg] but no failure occurred.") - assertTrue(e.getMessage.startsWith(msg), s"Expected exception with message:\n[$msg]\nbut was\n[${e.getMessage}]") - } - - ///// Test --cancel - @Test - def shouldNotAllowCancelWithoutBootstrapServerOption(): Unit = { - val args = Array( - "--cancel") - shouldFailWith(missingBootstrapServerMsg, args) - } - - @Test - def shouldNotAllowCancelWithoutReassignmentJsonFile(): Unit = { - val args = Array( - "--cancel", - "--bootstrap-server", "localhost:1234", - "--preserve-throttles") - shouldFailWith("Missing required argument \"[reassignment-json-file]\"", args) - } -} diff --git a/core/src/test/scala/unit/kafka/admin/ReassignPartitionsUnitTest.scala b/core/src/test/scala/unit/kafka/admin/ReassignPartitionsUnitTest.scala deleted file mode 100644 index 7aa0f8eec23e7..0000000000000 --- a/core/src/test/scala/unit/kafka/admin/ReassignPartitionsUnitTest.scala +++ /dev/null @@ -1,671 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package kafka.admin - -import java.util.concurrent.ExecutionException -import java.util.{Arrays, Collections, Optional} -import kafka.admin.ReassignPartitionsCommand._ -import kafka.utils.Exit -import org.apache.kafka.admin.BrokerMetadata -import org.apache.kafka.clients.admin.{Config, MockAdminClient, PartitionReassignment} -import org.apache.kafka.common.config.ConfigResource -import org.apache.kafka.common.errors.{InvalidReplicationFactorException, UnknownTopicOrPartitionException} -import org.apache.kafka.common.{Node, TopicPartition, TopicPartitionInfo, TopicPartitionReplica} -import org.apache.kafka.server.common.{AdminCommandFailedException, AdminOperationException} -import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertThrows, assertTrue} -import org.junit.jupiter.api.{AfterEach, BeforeEach, Test, Timeout} - -import scala.collection.mutable -import scala.jdk.CollectionConverters._ - -@Timeout(60) -class ReassignPartitionsUnitTest { - - @BeforeEach - def setUp(): Unit = { - Exit.setExitProcedure((_, message) => throw new IllegalArgumentException(message.orNull)) - } - - @AfterEach - def tearDown(): Unit = { - Exit.resetExitProcedure() - } - - @Test - def testCompareTopicPartitions(): Unit = { - assertTrue(compareTopicPartitions(new TopicPartition("abc", 0), - new TopicPartition("abc", 1))) - assertFalse(compareTopicPartitions(new TopicPartition("def", 0), - new TopicPartition("abc", 1))) - } - - @Test - def testCompareTopicPartitionReplicas(): Unit = { - assertTrue(compareTopicPartitionReplicas(new TopicPartitionReplica("def", 0, 0), - new TopicPartitionReplica("abc", 0, 1))) - assertFalse(compareTopicPartitionReplicas(new TopicPartitionReplica("def", 0, 0), - new TopicPartitionReplica("cde", 0, 0))) - } - - @Test - def testPartitionReassignStatesToString(): Unit = { - assertEquals(Seq( - "Status of partition reassignment:", - "Reassignment of partition bar-0 is still in progress.", - "Reassignment of partition foo-0 is completed.", - "Reassignment of partition foo-1 is still in progress."). - mkString(System.lineSeparator()), - partitionReassignmentStatesToString(Map( - new TopicPartition("foo", 0) -> - PartitionReassignmentState(Seq(1, 2, 3), Seq(1, 2, 3), true), - new TopicPartition("foo", 1) -> - PartitionReassignmentState(Seq(1, 2, 3), Seq(1, 2, 4), false), - new TopicPartition("bar", 0) -> - PartitionReassignmentState(Seq(1, 2, 3), Seq(1, 2, 4), false), - ))) - } - - private def addTopics(adminClient: MockAdminClient): Unit = { - val b = adminClient.brokers() - adminClient.addTopic(false, "foo", Arrays.asList( - new TopicPartitionInfo(0, b.get(0), - Arrays.asList(b.get(0), b.get(1), b.get(2)), - Arrays.asList(b.get(0), b.get(1))), - new TopicPartitionInfo(1, b.get(1), - Arrays.asList(b.get(1), b.get(2), b.get(3)), - Arrays.asList(b.get(1), b.get(2), b.get(3))) - ), Collections.emptyMap()) - adminClient.addTopic(false, "bar", Arrays.asList( - new TopicPartitionInfo(0, b.get(2), - Arrays.asList(b.get(2), b.get(3), b.get(0)), - Arrays.asList(b.get(2), b.get(3), b.get(0))) - ), Collections.emptyMap()) - } - - @Test - def testFindPartitionReassignmentStates(): Unit = { - val adminClient = new MockAdminClient.Builder().numBrokers(4).build() - try { - addTopics(adminClient) - // Create a reassignment and test findPartitionReassignmentStates. - val reassignmentResult: Map[TopicPartition, Class[_ <: Throwable]] = alterPartitionReassignments(adminClient, Map( - new TopicPartition("foo", 0) -> Seq(0,1,3), - new TopicPartition("quux", 0) -> Seq(1,2,3))).map { case (k, v) => k -> v.getClass }.toMap - assertEquals(Map(new TopicPartition("quux", 0) -> classOf[UnknownTopicOrPartitionException]), - reassignmentResult) - assertEquals((Map( - new TopicPartition("foo", 0) -> PartitionReassignmentState(Seq(0,1,2), Seq(0,1,3), false), - new TopicPartition("foo", 1) -> PartitionReassignmentState(Seq(1,2,3), Seq(1,2,3), true) - ), true), - findPartitionReassignmentStates(adminClient, Seq( - (new TopicPartition("foo", 0), Seq(0,1,3)), - (new TopicPartition("foo", 1), Seq(1,2,3)) - ))) - // Cancel the reassignment and test findPartitionReassignmentStates again. - val cancelResult: Map[TopicPartition, Class[_ <: Throwable]] = cancelPartitionReassignments(adminClient, - Set(new TopicPartition("foo", 0), new TopicPartition("quux", 2))).map { case (k, v) => - k -> v.getClass - }.toMap - assertEquals(Map(new TopicPartition("quux", 2) -> classOf[UnknownTopicOrPartitionException]), - cancelResult) - assertEquals((Map( - new TopicPartition("foo", 0) -> PartitionReassignmentState(Seq(0,1,2), Seq(0,1,3), true), - new TopicPartition("foo", 1) -> PartitionReassignmentState(Seq(1,2,3), Seq(1,2,3), true) - ), false), - findPartitionReassignmentStates(adminClient, Seq( - (new TopicPartition("foo", 0), Seq(0,1,3)), - (new TopicPartition("foo", 1), Seq(1,2,3)) - ))) - } finally { - adminClient.close() - } - } - - @Test - def testFindLogDirMoveStates(): Unit = { - val adminClient = new MockAdminClient.Builder(). - numBrokers(4). - brokerLogDirs(Arrays.asList( - Arrays.asList("/tmp/kafka-logs0", "/tmp/kafka-logs1"), - Arrays.asList("/tmp/kafka-logs0", "/tmp/kafka-logs1"), - Arrays.asList("/tmp/kafka-logs0", "/tmp/kafka-logs1"), - Arrays.asList("/tmp/kafka-logs0", null))). - build(); - try { - addTopics(adminClient) - val b = adminClient.brokers() - adminClient.addTopic(false, "quux", Arrays.asList( - new TopicPartitionInfo(0, b.get(2), - Arrays.asList(b.get(1), b.get(2), b.get(3)), - Arrays.asList(b.get(1), b.get(2), b.get(3)))), - Collections.emptyMap()) - adminClient.alterReplicaLogDirs(Map( - new TopicPartitionReplica("foo", 0, 0) -> "/tmp/kafka-logs1", - new TopicPartitionReplica("quux", 0, 0) -> "/tmp/kafka-logs1" - ).asJava).all().get() - assertEquals(Map( - new TopicPartitionReplica("bar", 0, 0) -> new CompletedMoveState("/tmp/kafka-logs0"), - new TopicPartitionReplica("foo", 0, 0) -> new ActiveMoveState("/tmp/kafka-logs0", - "/tmp/kafka-logs1", "/tmp/kafka-logs1"), - new TopicPartitionReplica("foo", 1, 0) -> new CancelledMoveState("/tmp/kafka-logs0", - "/tmp/kafka-logs1"), - new TopicPartitionReplica("quux", 1, 0) -> new MissingLogDirMoveState("/tmp/kafka-logs1"), - new TopicPartitionReplica("quuz", 0, 0) -> new MissingReplicaMoveState("/tmp/kafka-logs0") - ), findLogDirMoveStates(adminClient, Map( - new TopicPartitionReplica("bar", 0, 0) -> "/tmp/kafka-logs0", - new TopicPartitionReplica("foo", 0, 0) -> "/tmp/kafka-logs1", - new TopicPartitionReplica("foo", 1, 0) -> "/tmp/kafka-logs1", - new TopicPartitionReplica("quux", 1, 0) -> "/tmp/kafka-logs1", - new TopicPartitionReplica("quuz", 0, 0) -> "/tmp/kafka-logs0" - ))) - } finally { - adminClient.close() - } - } - - @Test - def testReplicaMoveStatesToString(): Unit = { - assertEquals(Seq( - "Reassignment of replica bar-0-0 completed successfully.", - "Reassignment of replica foo-0-0 is still in progress.", - "Partition foo-1 on broker 0 is not being moved from log dir /tmp/kafka-logs0 to /tmp/kafka-logs1.", - "Partition quux-0 cannot be found in any live log directory on broker 0.", - "Partition quux-1 on broker 1 is being moved to log dir /tmp/kafka-logs2 instead of /tmp/kafka-logs1.", - "Partition quux-2 is not found in any live log dir on broker 1. " + - "There is likely an offline log directory on the broker.").mkString(System.lineSeparator()), - replicaMoveStatesToString(Map( - new TopicPartitionReplica("bar", 0, 0) -> CompletedMoveState("/tmp/kafka-logs0"), - new TopicPartitionReplica("foo", 0, 0) -> ActiveMoveState("/tmp/kafka-logs0", - "/tmp/kafka-logs1", "/tmp/kafka-logs1"), - new TopicPartitionReplica("foo", 1, 0) -> CancelledMoveState("/tmp/kafka-logs0", - "/tmp/kafka-logs1"), - new TopicPartitionReplica("quux", 0, 0) -> MissingReplicaMoveState("/tmp/kafka-logs1"), - new TopicPartitionReplica("quux", 1, 1) -> ActiveMoveState("/tmp/kafka-logs0", - "/tmp/kafka-logs1", "/tmp/kafka-logs2"), - new TopicPartitionReplica("quux", 2, 1) -> MissingLogDirMoveState("/tmp/kafka-logs1") - ))) - } - - @Test - def testGetReplicaAssignments(): Unit = { - val adminClient = new MockAdminClient.Builder().numBrokers(4).build() - try { - addTopics(adminClient) - assertEquals(Map( - new TopicPartition("foo", 0) -> Seq(0, 1, 2), - new TopicPartition("foo", 1) -> Seq(1, 2, 3), - ), - getReplicaAssignmentForTopics(adminClient, Seq("foo"))) - assertEquals(Map( - new TopicPartition("foo", 0) -> Seq(0, 1, 2), - new TopicPartition("bar", 0) -> Seq(2, 3, 0), - ), - getReplicaAssignmentForPartitions(adminClient, Set( - new TopicPartition("foo", 0), new TopicPartition("bar", 0)))) - } finally { - adminClient.close() - } - } - - @Test - def testGetBrokerRackInformation(): Unit = { - val adminClient = new MockAdminClient.Builder(). - brokers(Arrays.asList(new Node(0, "localhost", 9092, "rack0"), - new Node(1, "localhost", 9093, "rack1"), - new Node(2, "localhost", 9094, null))). - build() - try { - assertEquals(Seq( - new BrokerMetadata(0, Optional.of("rack0")), - new BrokerMetadata(1, Optional.of("rack1")) - ), getBrokerMetadata(adminClient, Seq(0, 1), true)) - assertEquals(Seq( - new BrokerMetadata(0, Optional.empty()), - new BrokerMetadata(1, Optional.empty()) - ), getBrokerMetadata(adminClient, Seq(0, 1), false)) - assertStartsWith("Not all brokers have rack information", - assertThrows(classOf[AdminOperationException], - () => getBrokerMetadata(adminClient, Seq(1, 2), true)).getMessage) - assertEquals(Seq( - new BrokerMetadata(1, Optional.empty()), - new BrokerMetadata(2, Optional.empty()) - ), getBrokerMetadata(adminClient, Seq(1, 2), false)) - } finally { - adminClient.close() - } - } - - @Test - def testParseGenerateAssignmentArgs(): Unit = { - assertStartsWith("Broker list contains duplicate entries", - assertThrows(classOf[AdminCommandFailedException], () => parseGenerateAssignmentArgs( - """{"topics": [{"topic": "foo"}], "version":1}""", "1,1,2"), - () => "Expected to detect duplicate broker list entries").getMessage) - assertStartsWith("Broker list contains duplicate entries", - assertThrows(classOf[AdminCommandFailedException], () => parseGenerateAssignmentArgs( - """{"topics": [{"topic": "foo"}], "version":1}""", "5,2,3,4,5"), - () => "Expected to detect duplicate broker list entries").getMessage) - assertEquals((Seq(5,2,3,4),Seq("foo")), - parseGenerateAssignmentArgs("""{"topics": [{"topic": "foo"}], "version":1}""", - "5,2,3,4")) - assertStartsWith("List of topics to reassign contains duplicate entries", - assertThrows(classOf[AdminCommandFailedException], () => parseGenerateAssignmentArgs( - """{"topics": [{"topic": "foo"},{"topic": "foo"}], "version":1}""", "5,2,3,4"), - () => "Expected to detect duplicate topic entries").getMessage) - assertEquals((Seq(5,3,4),Seq("foo","bar")), - parseGenerateAssignmentArgs( - """{"topics": [{"topic": "foo"},{"topic": "bar"}], "version":1}""", - "5,3,4")) - } - - @Test - def testGenerateAssignmentFailsWithoutEnoughReplicas(): Unit = { - val adminClient = new MockAdminClient.Builder().numBrokers(4).build() - try { - addTopics(adminClient) - assertStartsWith("Replication factor: 3 larger than available brokers: 2", - assertThrows(classOf[InvalidReplicationFactorException], - () => generateAssignment(adminClient, """{"topics":[{"topic":"foo"},{"topic":"bar"}]}""", "0,1", false), - () => "Expected generateAssignment to fail").getMessage) - } finally { - adminClient.close() - } - } - - @Test - def testGenerateAssignmentWithInvalidPartitionsFails(): Unit = { - val adminClient = new MockAdminClient.Builder().numBrokers(5).build() - try { - addTopics(adminClient) - assertStartsWith("Topic quux not found", - assertThrows(classOf[ExecutionException], - () => generateAssignment(adminClient, """{"topics":[{"topic":"foo"},{"topic":"quux"}]}""", "0,1", false), - () => "Expected generateAssignment to fail").getCause.getMessage) - } finally { - adminClient.close() - } - } - - @Test - def testGenerateAssignmentWithInconsistentRacks(): Unit = { - val adminClient = new MockAdminClient.Builder(). - brokers(Arrays.asList( - new Node(0, "localhost", 9092, "rack0"), - new Node(1, "localhost", 9093, "rack0"), - new Node(2, "localhost", 9094, null), - new Node(3, "localhost", 9095, "rack1"), - new Node(4, "localhost", 9096, "rack1"), - new Node(5, "localhost", 9097, "rack2"))). - build() - try { - addTopics(adminClient) - assertStartsWith("Not all brokers have rack information.", - assertThrows(classOf[AdminOperationException], - () => generateAssignment(adminClient, """{"topics":[{"topic":"foo"}]}""", "0,1,2,3", true), - () => "Expected generateAssignment to fail").getMessage) - // It should succeed when --disable-rack-aware is used. - val (_, current) = generateAssignment(adminClient, - """{"topics":[{"topic":"foo"}]}""", "0,1,2,3", false) - assertEquals(Map( - new TopicPartition("foo", 0) -> Seq(0, 1, 2), - new TopicPartition("foo", 1) -> Seq(1, 2, 3), - ), current) - } finally { - adminClient.close() - } - } - - @Test - def testGenerateAssignmentWithFewerBrokers(): Unit = { - val adminClient = new MockAdminClient.Builder().numBrokers(4).build() - try { - addTopics(adminClient) - val goalBrokers = Set(0,1,3) - val (proposed, current) = generateAssignment(adminClient, - """{"topics":[{"topic":"foo"},{"topic":"bar"}]}""", - goalBrokers.mkString(","), false) - assertEquals(Map( - new TopicPartition("foo", 0) -> Seq(0, 1, 2), - new TopicPartition("foo", 1) -> Seq(1, 2, 3), - new TopicPartition("bar", 0) -> Seq(2, 3, 0) - ), current) - - // The proposed assignment should only span the provided brokers - proposed.values.foreach(replicas => assertTrue(replicas.forall(goalBrokers.contains), - s"Proposed assignment $proposed puts replicas on brokers other than $goalBrokers")) - } finally { - adminClient.close() - } - } - - @Test - def testCurrentPartitionReplicaAssignmentToString(): Unit = { - assertEquals(Seq( - """Current partition replica assignment""", - """""", - """{"version":1,"partitions":""" + - """[{"topic":"bar","partition":0,"replicas":[7,8],"log_dirs":["any","any"]},""" + - """{"topic":"foo","partition":1,"replicas":[4,5,6],"log_dirs":["any","any","any"]}]""" + - """}""", - """""", - """Save this to use as the --reassignment-json-file option during rollback""" - ).mkString(System.lineSeparator()), - currentPartitionReplicaAssignmentToString(Map( - new TopicPartition("foo", 1) -> Seq(1,2,3), - new TopicPartition("bar", 0) -> Seq(7,8,9) - ), - Map( - new TopicPartition("foo", 0) -> Seq(1,2,3), - new TopicPartition("foo", 1) -> Seq(4,5,6), - new TopicPartition("bar", 0) -> Seq(7,8), - new TopicPartition("baz", 0) -> Seq(10,11,12) - ), - )) - } - - @Test - def testMoveMap(): Unit = { - // overwrite foo-0 with different reassignments - // keep old reassignments of foo-1 - // overwrite foo-2 with same reassignments - // overwrite foo-3 with new reassignments without overlap of old reassignments - // overwrite foo-4 with a subset of old reassignments - // overwrite foo-5 with a superset of old reassignments - // add new reassignments to bar-0 - val moveMap = calculateProposedMoveMap(Map( - new TopicPartition("foo", 0) -> new PartitionReassignment( - Arrays.asList(1,2,3,4), Arrays.asList(4), Arrays.asList(3)), - new TopicPartition("foo", 1) -> new PartitionReassignment( - Arrays.asList(4,5,6,7,8), Arrays.asList(7, 8), Arrays.asList(4, 5)), - new TopicPartition("foo", 2) -> new PartitionReassignment( - Arrays.asList(1,2,3,4), Arrays.asList(3,4), Arrays.asList(1,2)), - new TopicPartition("foo", 3) -> new PartitionReassignment( - Arrays.asList(1,2,3,4), Arrays.asList(3,4), Arrays.asList(1,2)), - new TopicPartition("foo", 4) -> new PartitionReassignment( - Arrays.asList(1,2,3,4), Arrays.asList(3,4), Arrays.asList(1,2)), - new TopicPartition("foo", 5) -> new PartitionReassignment( - Arrays.asList(1,2,3,4), Arrays.asList(3,4), Arrays.asList(1,2)) - ), Map( - new TopicPartition("foo", 0) -> Seq(1,2,5), - new TopicPartition("foo", 2) -> Seq(3,4), - new TopicPartition("foo", 3) -> Seq(5,6), - new TopicPartition("foo", 4) -> Seq(3), - new TopicPartition("foo", 5) -> Seq(3,4,5,6), - new TopicPartition("bar", 0) -> Seq(1,2,3) - ), Map( - new TopicPartition("foo", 0) -> Seq(1,2,3,4), - new TopicPartition("foo", 1) -> Seq(4,5,6,7,8), - new TopicPartition("foo", 2) -> Seq(1,2,3,4), - new TopicPartition("foo", 3) -> Seq(1,2,3,4), - new TopicPartition("foo", 4) -> Seq(1,2,3,4), - new TopicPartition("foo", 5) -> Seq(1,2,3,4), - new TopicPartition("bar", 0) -> Seq(2,3,4), - new TopicPartition("baz", 0) -> Seq(1,2,3) - )) - - assertEquals( - mutable.Map("foo" -> mutable.Map( - 0 -> PartitionMove(mutable.Set(1,2,3), mutable.Set(5)), - 1 -> PartitionMove(mutable.Set(4,5,6), mutable.Set(7,8)), - 2 -> PartitionMove(mutable.Set(1,2), mutable.Set(3,4)), - 3 -> PartitionMove(mutable.Set(1,2), mutable.Set(5,6)), - 4 -> PartitionMove(mutable.Set(1,2), mutable.Set(3)), - 5 -> PartitionMove(mutable.Set(1,2), mutable.Set(3,4,5,6)) - ), "bar" -> mutable.Map( - 0 -> PartitionMove(mutable.Set(2,3,4), mutable.Set(1)), - )), moveMap) - - assertEquals(Map( - "foo" -> "0:1,0:2,0:3,1:4,1:5,1:6,2:1,2:2,3:1,3:2,4:1,4:2,5:1,5:2", - "bar" -> "0:2,0:3,0:4" - ), calculateLeaderThrottles(moveMap)) - - assertEquals(Map( - "foo" -> "0:5,1:7,1:8,2:3,2:4,3:5,3:6,4:3,5:3,5:4,5:5,5:6", - "bar" -> "0:1" - ), calculateFollowerThrottles(moveMap)) - - assertEquals(Set(1,2,3,4,5,6,7,8), calculateReassigningBrokers(moveMap)) - - assertEquals(Set(0,2), calculateMovingBrokers( - Set(new TopicPartitionReplica("quux", 0, 0), - new TopicPartitionReplica("quux", 1, 2)))) - } - - @Test - def testParseExecuteAssignmentArgs(): Unit = { - assertStartsWith("Partition reassignment list cannot be empty", - assertThrows(classOf[AdminCommandFailedException], - () => parseExecuteAssignmentArgs("""{"version":1,"partitions":[]}"""), - () => "Expected to detect empty partition reassignment list").getMessage) - assertStartsWith("Partition reassignment contains duplicate topic partitions", - assertThrows(classOf[AdminCommandFailedException], () => parseExecuteAssignmentArgs( - """{"version":1,"partitions":""" + - """[{"topic":"foo","partition":0,"replicas":[0,1],"log_dirs":["any","any"]},""" + - """{"topic":"foo","partition":0,"replicas":[2,3,4],"log_dirs":["any","any","any"]}""" + - """]}"""), () => "Expected to detect a partition list with duplicate entries").getMessage) - assertStartsWith("Partition reassignment contains duplicate topic partitions", - assertThrows(classOf[AdminCommandFailedException], () => parseExecuteAssignmentArgs( - """{"version":1,"partitions":""" + - """[{"topic":"foo","partition":0,"replicas":[0,1],"log_dirs":["/abc","/def"]},""" + - """{"topic":"foo","partition":0,"replicas":[2,3],"log_dirs":["/abc","/def"]}""" + - """]}"""), () => "Expected to detect a partition replica list with duplicate entries").getMessage) - assertStartsWith("Partition replica lists may not contain duplicate entries", - assertThrows(classOf[AdminCommandFailedException], () => parseExecuteAssignmentArgs( - """{"version":1,"partitions":""" + - """[{"topic":"foo","partition":0,"replicas":[0,0],"log_dirs":["/abc","/def"]},""" + - """{"topic":"foo","partition":1,"replicas":[2,3],"log_dirs":["/abc","/def"]}""" + - """]}"""), () => "Expected to detect a partition replica list with duplicate entries").getMessage) - assertEquals((Map( - new TopicPartition("foo", 0) -> Seq(1, 2, 3), - new TopicPartition("foo", 1) -> Seq(3, 4, 5), - ), Map( - )), - parseExecuteAssignmentArgs( - """{"version":1,"partitions":""" + - """[{"topic":"foo","partition":0,"replicas":[1,2,3],"log_dirs":["any","any","any"]},""" + - """{"topic":"foo","partition":1,"replicas":[3,4,5],"log_dirs":["any","any","any"]}""" + - """]}""")) - assertEquals((Map( - new TopicPartition("foo", 0) -> Seq(1, 2, 3), - ), Map( - new TopicPartitionReplica("foo", 0, 1) -> "/tmp/a", - new TopicPartitionReplica("foo", 0, 2) -> "/tmp/b", - new TopicPartitionReplica("foo", 0, 3) -> "/tmp/c" - )), - parseExecuteAssignmentArgs( - """{"version":1,"partitions":""" + - """[{"topic":"foo","partition":0,"replicas":[1,2,3],"log_dirs":["/tmp/a","/tmp/b","/tmp/c"]}""" + - """]}""")) - } - - @Test - def testExecuteWithInvalidPartitionsFails(): Unit = { - val adminClient = new MockAdminClient.Builder().numBrokers(5).build() - try { - addTopics(adminClient) - assertStartsWith("Topic quux not found", - assertThrows(classOf[ExecutionException], () => executeAssignment(adminClient, false, - """{"version":1,"partitions":""" + - """[{"topic":"foo","partition":0,"replicas":[0,1],"log_dirs":["any","any"]},""" + - """{"topic":"quux","partition":0,"replicas":[2,3,4],"log_dirs":["any","any","any"]}""" + - """]}"""), () => "Expected reassignment with non-existent topic to fail").getCause.getMessage) - } finally { - adminClient.close() - } - } - - @Test - def testExecuteWithInvalidBrokerIdFails(): Unit = { - val adminClient = new MockAdminClient.Builder().numBrokers(4).build() - try { - addTopics(adminClient) - assertStartsWith("Unknown broker id 4", - assertThrows(classOf[AdminCommandFailedException], () => executeAssignment(adminClient, false, - """{"version":1,"partitions":""" + - """[{"topic":"foo","partition":0,"replicas":[0,1],"log_dirs":["any","any"]},""" + - """{"topic":"foo","partition":1,"replicas":[2,3,4],"log_dirs":["any","any","any"]}""" + - """]}"""), () => "Expected reassignment with non-existent broker id to fail").getMessage) - } finally { - adminClient.close() - } - } - - @Test - def testModifyBrokerInterBrokerThrottle(): Unit = { - val adminClient = new MockAdminClient.Builder().numBrokers(4).build() - try { - modifyInterBrokerThrottle(adminClient, Set(0, 1, 2), 1000) - modifyInterBrokerThrottle(adminClient, Set(0, 3), 100) - val brokers = Seq(0, 1, 2, 3).map( - id => new ConfigResource(ConfigResource.Type.BROKER, id.toString)) - val results = adminClient.describeConfigs(brokers.asJava).all().get() - verifyBrokerThrottleResults(results.get(brokers(0)), 100, -1) - verifyBrokerThrottleResults(results.get(brokers(1)), 1000, -1) - verifyBrokerThrottleResults(results.get(brokers(2)), 1000, -1) - verifyBrokerThrottleResults(results.get(brokers(3)), 100, -1) - } finally { - adminClient.close() - } - } - - @Test - def testModifyLogDirThrottle(): Unit = { - val adminClient = new MockAdminClient.Builder().numBrokers(4).build() - try { - modifyLogDirThrottle(adminClient, Set(0, 1, 2), 2000) - modifyLogDirThrottle(adminClient, Set(0, 3), -1) - val brokers = Seq(0, 1, 2, 3).map( - id => new ConfigResource(ConfigResource.Type.BROKER, id.toString)) - val results = adminClient.describeConfigs(brokers.asJava).all().get() - verifyBrokerThrottleResults(results.get(brokers(0)), -1, 2000) - verifyBrokerThrottleResults(results.get(brokers(1)), -1, 2000) - verifyBrokerThrottleResults(results.get(brokers(2)), -1, 2000) - verifyBrokerThrottleResults(results.get(brokers(3)), -1, -1) - } finally { - adminClient.close() - } - } - - @Test - def testCurReassignmentsToString(): Unit = { - val adminClient = new MockAdminClient.Builder().numBrokers(4).build() - try { - addTopics(adminClient) - assertEquals("No partition reassignments found.", curReassignmentsToString(adminClient)) - val reassignmentResult: Map[TopicPartition, Class[_ <: Throwable]] = alterPartitionReassignments(adminClient, - Map( - new TopicPartition("foo", 1) -> Seq(4,5,3), - new TopicPartition("foo", 0) -> Seq(0,1,4,2), - new TopicPartition("bar", 0) -> Seq(2,3) - ) - ).map { case (k, v) => k -> v.getClass }.toMap - assertEquals(Map(), reassignmentResult) - assertEquals(Seq("Current partition reassignments:", - "bar-0: replicas: 2,3,0. removing: 0.", - "foo-0: replicas: 0,1,2. adding: 4.", - "foo-1: replicas: 1,2,3. adding: 4,5. removing: 1,2.").mkString(System.lineSeparator()), - curReassignmentsToString(adminClient)) - } finally { - adminClient.close() - } - } - - private def verifyBrokerThrottleResults(config: Config, - expectedInterBrokerThrottle: Long, - expectedReplicaAlterLogDirsThrottle: Long): Unit = { - val configs = new mutable.HashMap[String, String] - config.entries.forEach(entry => configs.put(entry.name, entry.value)) - if (expectedInterBrokerThrottle >= 0) { - assertEquals(expectedInterBrokerThrottle.toString, - configs.getOrElse(brokerLevelLeaderThrottle, "")) - assertEquals(expectedInterBrokerThrottle.toString, - configs.getOrElse(brokerLevelFollowerThrottle, "")) - } - if (expectedReplicaAlterLogDirsThrottle >= 0) { - assertEquals(expectedReplicaAlterLogDirsThrottle.toString, - configs.getOrElse(brokerLevelLogDirThrottle, "")) - } - } - - @Test - def testModifyTopicThrottles(): Unit = { - val adminClient = new MockAdminClient.Builder().numBrokers(4).build() - try { - addTopics(adminClient) - modifyTopicThrottles(adminClient, - Map("foo" -> "leaderFoo", "bar" -> "leaderBar"), - Map("bar" -> "followerBar")) - val topics = Seq("bar", "foo").map( - id => new ConfigResource(ConfigResource.Type.TOPIC, id)) - val results = adminClient.describeConfigs(topics.asJava).all().get() - verifyTopicThrottleResults(results.get(topics(0)), "leaderBar", "followerBar") - verifyTopicThrottleResults(results.get(topics(1)), "leaderFoo", "") - } finally { - adminClient.close() - } - } - - private def verifyTopicThrottleResults(config: Config, - expectedLeaderThrottle: String, - expectedFollowerThrottle: String): Unit = { - val configs = new mutable.HashMap[String, String] - config.entries.forEach(entry => configs.put(entry.name, entry.value)) - assertEquals(expectedLeaderThrottle, - configs.getOrElse(topicLevelLeaderThrottle, "")) - assertEquals(expectedFollowerThrottle, - configs.getOrElse(topicLevelFollowerThrottle, "")) - } - - @Test - def testAlterReplicaLogDirs(): Unit = { - val adminClient = new MockAdminClient.Builder(). - numBrokers(4). - brokerLogDirs(Collections.nCopies(4, - Arrays.asList("/tmp/kafka-logs0", "/tmp/kafka-logs1"))). - build() - try { - addTopics(adminClient) - assertEquals(Set( - new TopicPartitionReplica("foo", 0, 0) - ), - alterReplicaLogDirs(adminClient, Map( - new TopicPartitionReplica("foo", 0, 0) -> "/tmp/kafka-logs1", - new TopicPartitionReplica("quux", 1, 0) -> "/tmp/kafka-logs1" - ))) - } finally { - adminClient.close() - } - } - - def assertStartsWith(prefix: String, str: String): Unit = { - assertTrue(str.startsWith(prefix), "Expected the string to start with %s, but it was %s".format(prefix, str)) - } - - @Test - def testPropagateInvalidJsonError(): Unit = { - val adminClient = new MockAdminClient.Builder().numBrokers(4).build() - try { - addTopics(adminClient) - assertStartsWith("Unexpected character", - assertThrows(classOf[AdminOperationException], () => executeAssignment(adminClient, additional = false, "{invalid_json")).getMessage) - } finally { - adminClient.close() - } - } -} diff --git a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommandArgsTest.java b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommandArgsTest.java new file mode 100644 index 0000000000000..7b267646d496b --- /dev/null +++ b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommandArgsTest.java @@ -0,0 +1,292 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.tools.reassign; + +import kafka.admin.ReassignPartitionsCommand; +import org.apache.kafka.common.utils.Exit; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@Timeout(60) +public class ReassignPartitionsCommandArgsTest { + public static final String MISSING_BOOTSTRAP_SERVER_MSG = "Please specify --bootstrap-server"; + + @BeforeEach + public void setUp() { + Exit.setExitProcedure((statusCode, message) -> { + throw new IllegalArgumentException(message); + }); + } + + @AfterEach + public void tearDown() { + Exit.resetExitProcedure(); + } + + ///// Test valid argument parsing + @Test + public void shouldCorrectlyParseValidMinimumGenerateOptions() { + String[] args = new String[] { + "--bootstrap-server", "localhost:1234", + "--generate", + "--broker-list", "101,102", + "--topics-to-move-json-file", "myfile.json"}; + ReassignPartitionsCommand.validateAndParseArgs(args); + } + + @Test + public void shouldCorrectlyParseValidMinimumExecuteOptions() { + String[] args = new String[] { + "--bootstrap-server", "localhost:1234", + "--execute", + "--reassignment-json-file", "myfile.json"}; + ReassignPartitionsCommand.validateAndParseArgs(args); + } + + @Test + public void shouldCorrectlyParseValidMinimumVerifyOptions() { + String[] args = new String[] { + "--bootstrap-server", "localhost:1234", + "--verify", + "--reassignment-json-file", "myfile.json"}; + ReassignPartitionsCommand.validateAndParseArgs(args); + } + + @Test + public void shouldAllowThrottleOptionOnExecute() { + String[] args = new String[] { + "--bootstrap-server", "localhost:1234", + "--execute", + "--throttle", "100", + "--reassignment-json-file", "myfile.json"}; + ReassignPartitionsCommand.validateAndParseArgs(args); + } + + @Test + public void shouldUseDefaultsIfEnabled() { + String[] args = new String[] { + "--bootstrap-server", "localhost:1234", + "--execute", + "--reassignment-json-file", "myfile.json"}; + ReassignPartitionsCommand.ReassignPartitionsCommandOptions opts = ReassignPartitionsCommand.validateAndParseArgs(args); + assertEquals(10000L, opts.options.valueOf(opts.timeoutOpt())); + assertEquals(-1L, opts.options.valueOf(opts.interBrokerThrottleOpt())); + } + + @Test + public void testList() { + String[] args = new String[] { + "--list", + "--bootstrap-server", "localhost:1234"}; + ReassignPartitionsCommand.validateAndParseArgs(args); + } + + @Test + public void testCancelWithPreserveThrottlesOption() { + String[] args = new String[] { + "--cancel", + "--bootstrap-server", "localhost:1234", + "--reassignment-json-file", "myfile.json", + "--preserve-throttles"}; + ReassignPartitionsCommand.validateAndParseArgs(args); + } + + ///// Test handling missing or invalid actions + @Test + public void shouldFailIfNoArgs() { + String[] args = new String[0]; + shouldFailWith(ReassignPartitionsCommand.helpText(), args); + } + + @Test + public void shouldFailIfBlankArg() { + String[] args = new String[] {" "}; + shouldFailWith("Command must include exactly one action", args); + } + + @Test + public void shouldFailIfMultipleActions() { + String[] args = new String[] { + "--bootstrap-server", "localhost:1234", + "--execute", + "--verify", + "--reassignment-json-file", "myfile.json" + }; + shouldFailWith("Command must include exactly one action", args); + } + + ///// Test --execute + @Test + public void shouldNotAllowExecuteWithTopicsOption() { + String[] args = new String[] { + "--bootstrap-server", "localhost:1234", + "--execute", + "--reassignment-json-file", "myfile.json", + "--topics-to-move-json-file", "myfile.json"}; + shouldFailWith("Option \"[topics-to-move-json-file]\" can't be used with action \"[execute]\"", args); + } + + @Test + public void shouldNotAllowExecuteWithBrokerList() { + String[] args = new String[] { + "--bootstrap-server", "localhost:1234", + "--execute", + "--reassignment-json-file", "myfile.json", + "--broker-list", "101,102" + }; + shouldFailWith("Option \"[broker-list]\" can't be used with action \"[execute]\"", args); + } + + @Test + public void shouldNotAllowExecuteWithoutReassignmentOption() { + String[] args = new String[] { + "--bootstrap-server", "localhost:1234", + "--execute"}; + shouldFailWith("Missing required argument \"[reassignment-json-file]\"", args); + } + + @Test + public void testMissingBootstrapServerArgumentForExecute() { + String[] args = new String[] { + "--execute"}; + shouldFailWith(MISSING_BOOTSTRAP_SERVER_MSG, args); + } + + ///// Test --generate + @Test + public void shouldNotAllowGenerateWithoutBrokersAndTopicsOptions() { + String[] args = new String[] { + "--bootstrap-server", "localhost:1234", + "--generate"}; + shouldFailWith("Missing required argument \"[topics-to-move-json-file]\"", args); + } + + @Test + public void shouldNotAllowGenerateWithoutBrokersOption() { + String[] args = new String[] { + "--bootstrap-server", "localhost:1234", + "--topics-to-move-json-file", "myfile.json", + "--generate"}; + shouldFailWith("Missing required argument \"[broker-list]\"", args); + } + + @Test + public void shouldNotAllowGenerateWithoutTopicsOption() { + String[] args = new String[] { + "--bootstrap-server", "localhost:1234", + "--broker-list", "101,102", + "--generate"}; + shouldFailWith("Missing required argument \"[topics-to-move-json-file]\"", args); + } + + @Test + public void shouldNotAllowGenerateWithThrottleOption() { + String[] args = new String[] { + "--bootstrap-server", "localhost:1234", + "--generate", + "--broker-list", "101,102", + "--throttle", "100", + "--topics-to-move-json-file", "myfile.json"}; + shouldFailWith("Option \"[throttle]\" can't be used with action \"[generate]\"", args); + } + + @Test + public void shouldNotAllowGenerateWithReassignmentOption() { + String[] args = new String[] { + "--bootstrap-server", "localhost:1234", + "--generate", + "--broker-list", "101,102", + "--topics-to-move-json-file", "myfile.json", + "--reassignment-json-file", "myfile.json"}; + shouldFailWith("Option \"[reassignment-json-file]\" can't be used with action \"[generate]\"", args); + } + + @Test + public void shouldPrintHelpTextIfHelpArg() { + String[] args = new String[] {"--help"}; + // note, this is not actually a failed case, it's just we share the same `printUsageAndExit` method when wrong arg received + shouldFailWith(ReassignPartitionsCommand.helpText(), args); + } + + ///// Test --verify + @Test + public void shouldNotAllowVerifyWithoutReassignmentOption() { + String[] args = new String[] { + "--bootstrap-server", "localhost:1234", + "--verify"}; + shouldFailWith("Missing required argument \"[reassignment-json-file]\"", args); + } + + @Test + public void shouldNotAllowBrokersListWithVerifyOption() { + String[] args = new String[] { + "--bootstrap-server", "localhost:1234", + "--verify", + "--broker-list", "100,101", + "--reassignment-json-file", "myfile.json"}; + shouldFailWith("Option \"[broker-list]\" can't be used with action \"[verify]\"", args); + } + + @Test + public void shouldNotAllowThrottleWithVerifyOption() { + String[] args = new String[] { + "--bootstrap-server", "localhost:1234", + "--verify", + "--throttle", "100", + "--reassignment-json-file", "myfile.json"}; + shouldFailWith("Option \"[throttle]\" can't be used with action \"[verify]\"", args); + } + + @Test + public void shouldNotAllowTopicsOptionWithVerify() { + String[] args = new String[] { + "--bootstrap-server", "localhost:1234", + "--verify", + "--reassignment-json-file", "myfile.json", + "--topics-to-move-json-file", "myfile.json"}; + shouldFailWith("Option \"[topics-to-move-json-file]\" can't be used with action \"[verify]\"", args); + } + + private void shouldFailWith(String msg, String[] args) { + Throwable e = assertThrows(Exception.class, () -> ReassignPartitionsCommand.validateAndParseArgs(args), + () -> "Should have failed with [" + msg + "] but no failure occurred."); + assertTrue(e.getMessage().startsWith(msg), "Expected exception with message:\n[" + msg + "]\nbut was\n[" + e.getMessage() + "]"); + } + + ///// Test --cancel + @Test + public void shouldNotAllowCancelWithoutBootstrapServerOption() { + String[] args = new String[] { + "--cancel"}; + shouldFailWith(MISSING_BOOTSTRAP_SERVER_MSG, args); + } + + @Test + public void shouldNotAllowCancelWithoutReassignmentJsonFile() { + String[] args = new String[] { + "--cancel", + "--bootstrap-server", "localhost:1234", + "--preserve-throttles"}; + shouldFailWith("Missing required argument \"[reassignment-json-file]\"", args); + } +} diff --git a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java new file mode 100644 index 0000000000000..2f13129a62b1f --- /dev/null +++ b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java @@ -0,0 +1,925 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.tools.reassign; + +import kafka.admin.ReassignPartitionsCommand; +import org.apache.kafka.admin.BrokerMetadata; +import org.apache.kafka.clients.admin.Config; +import org.apache.kafka.clients.admin.MockAdminClient; +import org.apache.kafka.clients.admin.PartitionReassignment; +import org.apache.kafka.common.Node; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.TopicPartitionInfo; +import org.apache.kafka.common.TopicPartitionReplica; +import org.apache.kafka.common.config.ConfigResource; +import org.apache.kafka.common.errors.InvalidReplicationFactorException; +import org.apache.kafka.common.errors.UnknownTopicOrPartitionException; +import org.apache.kafka.common.utils.Exit; +import org.apache.kafka.common.utils.Time; +import org.apache.kafka.server.common.AdminCommandFailedException; +import org.apache.kafka.server.common.AdminOperationException; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static kafka.admin.ReassignPartitionsCommand.alterPartitionReassignments; +import static kafka.admin.ReassignPartitionsCommand.alterReplicaLogDirs; +import static kafka.admin.ReassignPartitionsCommand.brokerLevelFollowerThrottle; +import static kafka.admin.ReassignPartitionsCommand.brokerLevelLeaderThrottle; +import static kafka.admin.ReassignPartitionsCommand.brokerLevelLogDirThrottle; +import static kafka.admin.ReassignPartitionsCommand.calculateFollowerThrottles; +import static kafka.admin.ReassignPartitionsCommand.calculateLeaderThrottles; +import static kafka.admin.ReassignPartitionsCommand.calculateMovingBrokers; +import static kafka.admin.ReassignPartitionsCommand.calculateProposedMoveMap; +import static kafka.admin.ReassignPartitionsCommand.calculateReassigningBrokers; +import static kafka.admin.ReassignPartitionsCommand.cancelPartitionReassignments; +import static kafka.admin.ReassignPartitionsCommand.compareTopicPartitionReplicas; +import static kafka.admin.ReassignPartitionsCommand.compareTopicPartitions; +import static kafka.admin.ReassignPartitionsCommand.curReassignmentsToString; +import static kafka.admin.ReassignPartitionsCommand.currentPartitionReplicaAssignmentToString; +import static kafka.admin.ReassignPartitionsCommand.executeAssignment; +import static kafka.admin.ReassignPartitionsCommand.findLogDirMoveStates; +import static kafka.admin.ReassignPartitionsCommand.findPartitionReassignmentStates; +import static kafka.admin.ReassignPartitionsCommand.generateAssignment; +import static kafka.admin.ReassignPartitionsCommand.getBrokerMetadata; +import static kafka.admin.ReassignPartitionsCommand.getReplicaAssignmentForPartitions; +import static kafka.admin.ReassignPartitionsCommand.getReplicaAssignmentForTopics; +import static kafka.admin.ReassignPartitionsCommand.modifyInterBrokerThrottle; +import static kafka.admin.ReassignPartitionsCommand.modifyLogDirThrottle; +import static kafka.admin.ReassignPartitionsCommand.modifyTopicThrottles; +import static kafka.admin.ReassignPartitionsCommand.parseExecuteAssignmentArgs; +import static kafka.admin.ReassignPartitionsCommand.parseGenerateAssignmentArgs; +import static kafka.admin.ReassignPartitionsCommand.partitionReassignmentStatesToString; +import static kafka.admin.ReassignPartitionsCommand.replicaMoveStatesToString; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import scala.Tuple2; +import scala.collection.JavaConverters; +import scala.collection.Seq; +import scala.collection.mutable.Map$; + + +@Timeout(60) +public class ReassignPartitionsUnitTest { + @BeforeEach + public void setUp() { + Exit.setExitProcedure((statusCode, message) -> { + throw new IllegalArgumentException(message); + }); + } + + @AfterEach + public void tearDown() { + Exit.resetExitProcedure(); + } + + @Test + public void testCompareTopicPartitions() { + assertTrue(compareTopicPartitions(new TopicPartition("abc", 0), + new TopicPartition("abc", 1))); + assertFalse(compareTopicPartitions(new TopicPartition("def", 0), + new TopicPartition("abc", 1))); + } + + @Test + public void testCompareTopicPartitionReplicas() { + assertTrue(compareTopicPartitionReplicas(new TopicPartitionReplica("def", 0, 0), + new TopicPartitionReplica("abc", 0, 1))); + assertFalse(compareTopicPartitionReplicas(new TopicPartitionReplica("def", 0, 0), + new TopicPartitionReplica("cde", 0, 0))); + } + + @Test + public void testPartitionReassignStatesToString() { + Map states = new HashMap<>(); + + states.put(new TopicPartition("foo", 0), new PartitionReassignmentState(Arrays.asList(1, 2, 3), Arrays.asList(1, 2, 3), true)); + states.put(new TopicPartition("foo", 1), new PartitionReassignmentState(Arrays.asList(1, 2, 3), Arrays.asList(1, 2, 4), false)); + states.put(new TopicPartition("bar", 0), new PartitionReassignmentState(Arrays.asList(1, 2, 3), Arrays.asList(1, 2, 4), false)); + + assertEquals(String.join(System.lineSeparator(), Arrays.asList( + "Status of partition reassignment:", + "Reassignment of partition bar-0 is still in progress.", + "Reassignment of partition foo-0 is completed.", + "Reassignment of partition foo-1 is still in progress.")), + partitionReassignmentStatesToString(toScala(states, this::toScala))); + } + + private void addTopics(MockAdminClient adminClient) { + List b = adminClient.brokers(); + adminClient.addTopic(false, "foo", Arrays.asList( + new TopicPartitionInfo(0, b.get(0), + Arrays.asList(b.get(0), b.get(1), b.get(2)), + Arrays.asList(b.get(0), b.get(1))), + new TopicPartitionInfo(1, b.get(1), + Arrays.asList(b.get(1), b.get(2), b.get(3)), + Arrays.asList(b.get(1), b.get(2), b.get(3))) + ), Collections.emptyMap()); + adminClient.addTopic(false, "bar", Collections.singletonList( + new TopicPartitionInfo(0, b.get(2), + Arrays.asList(b.get(2), b.get(3), b.get(0)), + Arrays.asList(b.get(2), b.get(3), b.get(0))) + ), Collections.emptyMap()); + } + + @Test + public void testFindPartitionReassignmentStates() throws Exception { + try (MockAdminClient adminClient = new MockAdminClient.Builder().numBrokers(4).build()) { + addTopics(adminClient); + // Create a reassignment and test findPartitionReassignmentStates. + Map> reassignments = new HashMap<>(); + + reassignments.put(new TopicPartition("foo", 0), Arrays.asList(0, 1, 3)); + reassignments.put(new TopicPartition("quux", 0), Arrays.asList(1, 2, 3)); + + Map> reassignmentResult = javaMap(alterPartitionReassignments(adminClient, toScala(reassignments, this::toRawSeq))) + .entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().getClass())); + + assertEquals(Collections.singletonMap(new TopicPartition("quux", 0), UnknownTopicOrPartitionException.class), reassignmentResult); + + Map partitionReassignmentStates = new HashMap<>(); + + partitionReassignmentStates.put(new TopicPartition("foo", 0), new PartitionReassignmentState(Arrays.asList(0, 1, 2), Arrays.asList(0, 1, 3), false)); + partitionReassignmentStates.put(new TopicPartition("foo", 1), new PartitionReassignmentState(Arrays.asList(1, 2, 3), Arrays.asList(1, 2, 3), true)); + + Tuple2, Boolean> expected = new Tuple2<>(partitionReassignmentStates, true); + Tuple2, Object> actual = + findPartitionReassignmentStates(adminClient, toSeq(Arrays.asList( + new Tuple2<>(new TopicPartition("foo", 0), Arrays.asList(0, 1, 3)), + new Tuple2<>(new TopicPartition("foo", 1), Arrays.asList(1, 2, 3)) + ), this::toScala)); + + assertEquals(expected._1, javaMap(actual._1, this::toJava)); + assertEquals(expected._2, actual._2); + + // Cancel the reassignment and test findPartitionReassignmentStates again. + Map> cancelResult = javaMap(cancelPartitionReassignments(adminClient, toSet(new HashSet<>(Arrays.asList(new TopicPartition("foo", 0), new TopicPartition("quux", 2)))))) + .entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().getClass())); + + assertEquals(Collections.singletonMap(new TopicPartition("quux", 2), UnknownTopicOrPartitionException.class), cancelResult); + + partitionReassignmentStates.clear(); + + partitionReassignmentStates.put(new TopicPartition("foo", 0), new PartitionReassignmentState(Arrays.asList(0, 1, 2), Arrays.asList(0, 1, 3), true)); + partitionReassignmentStates.put(new TopicPartition("foo", 1), new PartitionReassignmentState(Arrays.asList(1, 2, 3), Arrays.asList(1, 2, 3), true)); + + expected = new Tuple2<>(partitionReassignmentStates, false); + actual = findPartitionReassignmentStates(adminClient, toSeq(Arrays.asList( + new Tuple2<>(new TopicPartition("foo", 0), Arrays.asList(0, 1, 3)), + new Tuple2<>(new TopicPartition("foo", 1), Arrays.asList(1, 2, 3)) + ), this::toScala)); + + assertEquals(expected._1, javaMap(actual._1, this::toJava)); + assertEquals(expected._2, actual._2); + } + } + + @Test + public void testFindLogDirMoveStates() throws Exception { + try (MockAdminClient adminClient = new MockAdminClient.Builder(). + numBrokers(4). + brokerLogDirs(Arrays.asList( + Arrays.asList("/tmp/kafka-logs0", "/tmp/kafka-logs1"), + Arrays.asList("/tmp/kafka-logs0", "/tmp/kafka-logs1"), + Arrays.asList("/tmp/kafka-logs0", "/tmp/kafka-logs1"), + Arrays.asList("/tmp/kafka-logs0", null))). + build()) { + + addTopics(adminClient); + List b = adminClient.brokers(); + adminClient.addTopic(false, "quux", Collections.singletonList( + new TopicPartitionInfo(0, b.get(2), + Arrays.asList(b.get(1), b.get(2), b.get(3)), + Arrays.asList(b.get(1), b.get(2), b.get(3)))), + Collections.emptyMap()); + + Map replicaAssignment = new HashMap<>(); + + replicaAssignment.put(new TopicPartitionReplica("foo", 0, 0), "/tmp/kafka-logs1"); + replicaAssignment.put(new TopicPartitionReplica("quux", 0, 0), "/tmp/kafka-logs1"); + + adminClient.alterReplicaLogDirs(replicaAssignment).all().get(); + + Map expLogDirMoveStates = new HashMap<>(); + + expLogDirMoveStates.put(new TopicPartitionReplica("bar", 0, 0), new CompletedMoveState("/tmp/kafka-logs0")); + expLogDirMoveStates.put(new TopicPartitionReplica("foo", 0, 0), new ActiveMoveState("/tmp/kafka-logs0", "/tmp/kafka-logs1", "/tmp/kafka-logs1")); + expLogDirMoveStates.put(new TopicPartitionReplica("foo", 1, 0), new CancelledMoveState("/tmp/kafka-logs0", "/tmp/kafka-logs1")); + expLogDirMoveStates.put(new TopicPartitionReplica("quux", 1, 0), new MissingLogDirMoveState("/tmp/kafka-logs1")); + expLogDirMoveStates.put(new TopicPartitionReplica("quuz", 0, 0), new MissingReplicaMoveState("/tmp/kafka-logs0")); + + Map targetMoves = new HashMap<>(); + + targetMoves.put(new TopicPartitionReplica("bar", 0, 0), "/tmp/kafka-logs0"); + targetMoves.put(new TopicPartitionReplica("foo", 0, 0), "/tmp/kafka-logs1"); + targetMoves.put(new TopicPartitionReplica("foo", 1, 0), "/tmp/kafka-logs1"); + targetMoves.put(new TopicPartitionReplica("quux", 1, 0), "/tmp/kafka-logs1"); + targetMoves.put(new TopicPartitionReplica("quuz", 0, 0), "/tmp/kafka-logs0"); + + assertEquals( + expLogDirMoveStates, + javaMap(findLogDirMoveStates(adminClient, toScala(targetMoves)), this::toJava) + ); + } + } + + @Test + public void testReplicaMoveStatesToString() { + Map states = new HashMap<>(); + + states.put(new TopicPartitionReplica("bar", 0, 0), new CompletedMoveState("/tmp/kafka-logs0")); + states.put(new TopicPartitionReplica("foo", 0, 0), new ActiveMoveState("/tmp/kafka-logs0", + "/tmp/kafka-logs1", "/tmp/kafka-logs1")); + states.put(new TopicPartitionReplica("foo", 1, 0), new CancelledMoveState("/tmp/kafka-logs0", + "/tmp/kafka-logs1")); + states.put(new TopicPartitionReplica("quux", 0, 0), new MissingReplicaMoveState("/tmp/kafka-logs1")); + states.put(new TopicPartitionReplica("quux", 1, 1), new ActiveMoveState("/tmp/kafka-logs0", + "/tmp/kafka-logs1", "/tmp/kafka-logs2")); + states.put(new TopicPartitionReplica("quux", 2, 1), new MissingLogDirMoveState("/tmp/kafka-logs1")); + + assertEquals(String.join(System.lineSeparator(), Arrays.asList( + "Reassignment of replica bar-0-0 completed successfully.", + "Reassignment of replica foo-0-0 is still in progress.", + "Partition foo-1 on broker 0 is not being moved from log dir /tmp/kafka-logs0 to /tmp/kafka-logs1.", + "Partition quux-0 cannot be found in any live log directory on broker 0.", + "Partition quux-1 on broker 1 is being moved to log dir /tmp/kafka-logs2 instead of /tmp/kafka-logs1.", + "Partition quux-2 is not found in any live log dir on broker 1. " + + "There is likely an offline log directory on the broker.")), + replicaMoveStatesToString(toScala(states, this::toScala))); + } + + @Test + public void testGetReplicaAssignments() throws Exception { + try (MockAdminClient adminClient = new MockAdminClient.Builder().numBrokers(4).build()) { + addTopics(adminClient); + + Map> assignments = new HashMap<>(); + + assignments.put(new TopicPartition("foo", 0), Arrays.asList(0, 1, 2)); + assignments.put(new TopicPartition("foo", 1), Arrays.asList(1, 2, 3)); + + assertEquals(assignments, + javaMap3(getReplicaAssignmentForTopics(adminClient, toSeq(Collections.singletonList("foo"))))); + + assignments.clear(); + + assignments.put(new TopicPartition("foo", 0), Arrays.asList(0, 1, 2)); + assignments.put(new TopicPartition("bar", 0), Arrays.asList(2, 3, 0)); + + assertEquals(assignments, + javaMap(getReplicaAssignmentForPartitions(adminClient, toSet(new HashSet<>(Arrays.asList( + new TopicPartition("foo", 0), new TopicPartition("bar", 0))))), this::toJavaList)); + } + } + + @Test + public void testGetBrokerRackInformation() throws Exception { + try (MockAdminClient adminClient = new MockAdminClient.Builder(). + brokers(Arrays.asList(new Node(0, "localhost", 9092, "rack0"), + new Node(1, "localhost", 9093, "rack1"), + new Node(2, "localhost", 9094, null))). + build()) { + + assertEquals(Arrays.asList( + new BrokerMetadata(0, Optional.of("rack0")), + new BrokerMetadata(1, Optional.of("rack1")) + ), toJavaList(getBrokerMetadata(adminClient, toSeq(Arrays.asList(0, 1)), true))); + assertEquals(Arrays.asList( + new BrokerMetadata(0, Optional.empty()), + new BrokerMetadata(1, Optional.empty()) + ), toJavaList(getBrokerMetadata(adminClient, toSeq(Arrays.asList(0, 1)), false))); + assertStartsWith("Not all brokers have rack information", + assertThrows(AdminOperationException.class, + () -> getBrokerMetadata(adminClient, toSeq(Arrays.asList(1, 2)), true)).getMessage()); + assertEquals(Arrays.asList( + new BrokerMetadata(1, Optional.empty()), + new BrokerMetadata(2, Optional.empty()) + ), toJavaList(getBrokerMetadata(adminClient, toSeq(Arrays.asList(1, 2)), false))); + } + } + + @Test + public void testParseGenerateAssignmentArgs() throws Exception { + assertStartsWith("Broker list contains duplicate entries", + assertThrows(AdminCommandFailedException.class, () -> parseGenerateAssignmentArgs( + "{\"topics\": [{\"topic\": \"foo\"}], \"version\":1}", "1,1,2"), + "Expected to detect duplicate broker list entries").getMessage()); + assertStartsWith("Broker list contains duplicate entries", + assertThrows(AdminCommandFailedException.class, () -> parseGenerateAssignmentArgs( + "{\"topics\": [{\"topic\": \"foo\"}], \"version\":1}", "5,2,3,4,5"), + "Expected to detect duplicate broker list entries").getMessage()); + assertEquals(new Tuple2<>(Arrays.asList(5, 2, 3, 4), Collections.singletonList("foo")), + mapTuple(parseGenerateAssignmentArgs("{\"topics\": [{\"topic\": \"foo\"}], \"version\":1}", + "5,2,3,4"), this::toJavaList, this::toJavaList)); + assertStartsWith("List of topics to reassign contains duplicate entries", + assertThrows(AdminCommandFailedException.class, () -> parseGenerateAssignmentArgs( + "{\"topics\": [{\"topic\": \"foo\"},{\"topic\": \"foo\"}], \"version\":1}", "5,2,3,4"), + "Expected to detect duplicate topic entries").getMessage()); + assertEquals(new Tuple2<>(Arrays.asList(5, 3, 4), Arrays.asList("foo", "bar")), + mapTuple(parseGenerateAssignmentArgs( + "{\"topics\": [{\"topic\": \"foo\"},{\"topic\": \"bar\"}], \"version\":1}", + "5,3,4"), this::toJavaList, this::toJavaList)); + } + + @Test + public void testGenerateAssignmentFailsWithoutEnoughReplicas() { + try (MockAdminClient adminClient = new MockAdminClient.Builder().numBrokers(4).build()) { + addTopics(adminClient); + assertStartsWith("Replication factor: 3 larger than available brokers: 2", + assertThrows(InvalidReplicationFactorException.class, + () -> generateAssignment(adminClient, "{\"topics\":[{\"topic\":\"foo\"},{\"topic\":\"bar\"}]}", "0,1", false), + "Expected generateAssignment to fail").getMessage()); + } + } + + @Test + public void testGenerateAssignmentWithInvalidPartitionsFails() { + try (MockAdminClient adminClient = new MockAdminClient.Builder().numBrokers(5).build()) { + addTopics(adminClient); + assertStartsWith("Topic quux not found", + assertThrows(ExecutionException.class, + () -> generateAssignment(adminClient, "{\"topics\":[{\"topic\":\"foo\"},{\"topic\":\"quux\"}]}", "0,1", false), + "Expected generateAssignment to fail").getCause().getMessage()); + } + } + + @Test + public void testGenerateAssignmentWithInconsistentRacks() throws Exception { + try (MockAdminClient adminClient = new MockAdminClient.Builder(). + brokers(Arrays.asList( + new Node(0, "localhost", 9092, "rack0"), + new Node(1, "localhost", 9093, "rack0"), + new Node(2, "localhost", 9094, null), + new Node(3, "localhost", 9095, "rack1"), + new Node(4, "localhost", 9096, "rack1"), + new Node(5, "localhost", 9097, "rack2"))). + build()) { + + addTopics(adminClient); + assertStartsWith("Not all brokers have rack information.", + assertThrows(AdminOperationException.class, + () -> generateAssignment(adminClient, "{\"topics\":[{\"topic\":\"foo\"}]}", "0,1,2,3", true), + "Expected generateAssignment to fail").getMessage()); + // It should succeed when --disable-rack-aware is used. + Tuple2>, Map>> proposedCurrent = mapTuple(generateAssignment(adminClient, + "{\"topics\":[{\"topic\":\"foo\"}]}", "0,1,2,3", false), this::javaMap3, this::javaMap3); + + Map> expCurrent = new HashMap<>(); + + expCurrent.put(new TopicPartition("foo", 0), Arrays.asList(0, 1, 2)); + expCurrent.put(new TopicPartition("foo", 1), Arrays.asList(1, 2, 3)); + + assertEquals(expCurrent, proposedCurrent._2()); + } + } + + @Test + public void testGenerateAssignmentWithFewerBrokers() throws Exception { + try (MockAdminClient adminClient = new MockAdminClient.Builder().numBrokers(4).build()) { + addTopics(adminClient); + List goalBrokers = Arrays.asList(0, 1, 3); + + Tuple2>, Map>> proposedCurrent = + mapTuple(generateAssignment(adminClient, + "{\"topics\":[{\"topic\":\"foo\"},{\"topic\":\"bar\"}]}", + goalBrokers.stream().map(Object::toString).collect(Collectors.joining(",")), false), this::javaMap3, this::javaMap3); + + Map> expCurrent = new HashMap<>(); + + expCurrent.put(new TopicPartition("foo", 0), Arrays.asList(0, 1, 2)); + expCurrent.put(new TopicPartition("foo", 1), Arrays.asList(1, 2, 3)); + expCurrent.put(new TopicPartition("bar", 0), Arrays.asList(2, 3, 0)); + + assertEquals(expCurrent, proposedCurrent._2()); + + // The proposed assignment should only span the provided brokers + proposedCurrent._1().values().forEach(replicas -> assertTrue(goalBrokers.containsAll(replicas), + "Proposed assignment " + proposedCurrent._1() + " puts replicas on brokers other than " + goalBrokers)); + } + } + + @Test + public void testCurrentPartitionReplicaAssignmentToString() throws Exception { + Map> proposedParts = new HashMap<>(); + + proposedParts.put(new TopicPartition("foo", 1), Arrays.asList(1, 2, 3)); + proposedParts.put(new TopicPartition("bar", 0), Arrays.asList(7, 8, 9)); + + Map> currentParts = new HashMap<>(); + + currentParts.put(new TopicPartition("foo", 0), Arrays.asList(1, 2, 3)); + currentParts.put(new TopicPartition("foo", 1), Arrays.asList(4, 5, 6)); + currentParts.put(new TopicPartition("bar", 0), Arrays.asList(7, 8)); + currentParts.put(new TopicPartition("baz", 0), Arrays.asList(10, 11, 12)); + + assertEquals(String.join(System.lineSeparator(), Arrays.asList( + "Current partition replica assignment", + "", + "{\"version\":1,\"partitions\":" + + "[{\"topic\":\"bar\",\"partition\":0,\"replicas\":[7,8],\"log_dirs\":[\"any\",\"any\"]}," + + "{\"topic\":\"foo\",\"partition\":1,\"replicas\":[4,5,6],\"log_dirs\":[\"any\",\"any\",\"any\"]}]" + + "}", + "", + "Save this to use as the --reassignment-json-file option during rollback" + )), + currentPartitionReplicaAssignmentToString(toScala(proposedParts, this::toRawSeq), toScala(currentParts, this::toRawSeq)) + ); + } + + @Test + public void testMoveMap() { + // overwrite foo-0 with different reassignments + // keep old reassignments of foo-1 + // overwrite foo-2 with same reassignments + // overwrite foo-3 with new reassignments without overlap of old reassignments + // overwrite foo-4 with a subset of old reassignments + // overwrite foo-5 with a superset of old reassignments + // add new reassignments to bar-0 + Map currentReassignments = new HashMap<>(); + + currentReassignments.put(new TopicPartition("foo", 0), new PartitionReassignment( + Arrays.asList(1, 2, 3, 4), Collections.singletonList(4), Collections.singletonList(3))); + currentReassignments.put(new TopicPartition("foo", 1), new PartitionReassignment( + Arrays.asList(4, 5, 6, 7, 8), Arrays.asList(7, 8), Arrays.asList(4, 5))); + currentReassignments.put(new TopicPartition("foo", 2), new PartitionReassignment( + Arrays.asList(1, 2, 3, 4), Arrays.asList(3, 4), Arrays.asList(1, 2))); + currentReassignments.put(new TopicPartition("foo", 3), new PartitionReassignment( + Arrays.asList(1, 2, 3, 4), Arrays.asList(3, 4), Arrays.asList(1, 2))); + currentReassignments.put(new TopicPartition("foo", 4), new PartitionReassignment( + Arrays.asList(1, 2, 3, 4), Arrays.asList(3, 4), Arrays.asList(1, 2))); + currentReassignments.put(new TopicPartition("foo", 5), new PartitionReassignment( + Arrays.asList(1, 2, 3, 4), Arrays.asList(3, 4), Arrays.asList(1, 2))); + + Map> proposedParts = new HashMap<>(); + + proposedParts.put(new TopicPartition("foo", 0), Arrays.asList(1, 2, 5)); + proposedParts.put(new TopicPartition("foo", 2), Arrays.asList(3, 4)); + proposedParts.put(new TopicPartition("foo", 3), Arrays.asList(5, 6)); + proposedParts.put(new TopicPartition("foo", 4), Collections.singletonList(3)); + proposedParts.put(new TopicPartition("foo", 5), Arrays.asList(3, 4, 5, 6)); + proposedParts.put(new TopicPartition("bar", 0), Arrays.asList(1, 2, 3)); + + Map> currentParts = new HashMap<>(); + + currentParts.put(new TopicPartition("foo", 0), Arrays.asList(1, 2, 3, 4)); + currentParts.put(new TopicPartition("foo", 1), Arrays.asList(4, 5, 6, 7, 8)); + currentParts.put(new TopicPartition("foo", 2), Arrays.asList(1, 2, 3, 4)); + currentParts.put(new TopicPartition("foo", 3), Arrays.asList(1, 2, 3, 4)); + currentParts.put(new TopicPartition("foo", 4), Arrays.asList(1, 2, 3, 4)); + currentParts.put(new TopicPartition("foo", 5), Arrays.asList(1, 2, 3, 4)); + currentParts.put(new TopicPartition("bar", 0), Arrays.asList(2, 3, 4)); + currentParts.put(new TopicPartition("baz", 0), Arrays.asList(1, 2, 3)); + + Map> moveMap = javaMap(calculateProposedMoveMap(toScala(currentReassignments), toScala(proposedParts, this::toRawSeq), toScala(currentParts, this::toRawSeq)), this::javaMap2); + + Map> expMoveMap = new HashMap<>(); + + Map fooMoves = new HashMap<>(); + + fooMoves.put(0, new PartitionMove(new HashSet<>(Arrays.asList(1, 2, 3)), new HashSet<>(Collections.singletonList(5)))); + fooMoves.put(1, new PartitionMove(new HashSet<>(Arrays.asList(4, 5, 6)), new HashSet<>(Arrays.asList(7, 8)))); + fooMoves.put(2, new PartitionMove(new HashSet<>(Arrays.asList(1, 2)), new HashSet<>(Arrays.asList(3, 4)))); + fooMoves.put(3, new PartitionMove(new HashSet<>(Arrays.asList(1, 2)), new HashSet<>(Arrays.asList(5, 6)))); + fooMoves.put(4, new PartitionMove(new HashSet<>(Arrays.asList(1, 2)), new HashSet<>(Collections.singletonList(3)))); + fooMoves.put(5, new PartitionMove(new HashSet<>(Arrays.asList(1, 2)), new HashSet<>(Arrays.asList(3, 4, 5, 6)))); + + expMoveMap.put("foo", fooMoves); + expMoveMap.put("bar", Collections.singletonMap(0, new PartitionMove(new HashSet<>(Arrays.asList(2, 3, 4)), new HashSet<>(Collections.singletonList(1))))); + + assertEquals(expMoveMap, moveMap); + + Map expLeaderThrottle = new HashMap<>(); + + expLeaderThrottle.put("foo", "0:1,0:2,0:3,1:4,1:5,1:6,2:1,2:2,3:1,3:2,4:1,4:2,5:1,5:2"); + expLeaderThrottle.put("bar", "0:2,0:3,0:4"); + + assertEquals(expLeaderThrottle, javaMap(calculateLeaderThrottles(mutableMap(toScala(moveMap, this::toMoveMap))))); + + Map expFollowerThrottle = new HashMap<>(); + + expFollowerThrottle.put("foo", "0:5,1:7,1:8,2:3,2:4,3:5,3:6,4:3,5:3,5:4,5:5,5:6"); + expFollowerThrottle.put("bar", "0:1"); + + assertEquals(expFollowerThrottle, javaMap(calculateFollowerThrottles(mutableMap(toScala(moveMap, this::toMoveMap))))); + + assertEquals(new HashSet<>(Arrays.asList(1, 2, 3, 4, 5, 6, 7, 8)), toJavaSet(calculateReassigningBrokers(mutableMap(toScala(moveMap, this::toMoveMap))))); + + assertEquals(new HashSet<>(Arrays.asList(0, 2)), toJavaSet(calculateMovingBrokers(toSet(new HashSet<>(Arrays.asList( + new TopicPartitionReplica("quux", 0, 0), + new TopicPartitionReplica("quux", 1, 2))))))); + } + + @Test + public void testParseExecuteAssignmentArgs() throws Exception { + assertStartsWith("Partition reassignment list cannot be empty", + assertThrows(AdminCommandFailedException.class, + () -> parseExecuteAssignmentArgs("{\"version\":1,\"partitions\":[]}"), + "Expected to detect empty partition reassignment list").getMessage()); + assertStartsWith("Partition reassignment contains duplicate topic partitions", + assertThrows(AdminCommandFailedException.class, () -> parseExecuteAssignmentArgs( + "{\"version\":1,\"partitions\":" + + "[{\"topic\":\"foo\",\"partition\":0,\"replicas\":[0,1],\"log_dirs\":[\"any\",\"any\"]}," + + "{\"topic\":\"foo\",\"partition\":0,\"replicas\":[2,3,4],\"log_dirs\":[\"any\",\"any\",\"any\"]}" + + "]}"), "Expected to detect a partition list with duplicate entries").getMessage()); + assertStartsWith("Partition reassignment contains duplicate topic partitions", + assertThrows(AdminCommandFailedException.class, () -> parseExecuteAssignmentArgs( + "{\"version\":1,\"partitions\":" + + "[{\"topic\":\"foo\",\"partition\":0,\"replicas\":[0,1],\"log_dirs\":[\"/abc\",\"/def\"]}," + + "{\"topic\":\"foo\",\"partition\":0,\"replicas\":[2,3],\"log_dirs\":[\"/abc\",\"/def\"]}" + + "]}"), "Expected to detect a partition replica list with duplicate entries").getMessage()); + assertStartsWith("Partition replica lists may not contain duplicate entries", + assertThrows(AdminCommandFailedException.class, () -> parseExecuteAssignmentArgs( + "{\"version\":1,\"partitions\":" + + "[{\"topic\":\"foo\",\"partition\":0,\"replicas\":[0,0],\"log_dirs\":[\"/abc\",\"/def\"]}," + + "{\"topic\":\"foo\",\"partition\":1,\"replicas\":[2,3],\"log_dirs\":[\"/abc\",\"/def\"]}" + + "]}"), "Expected to detect a partition replica list with duplicate entries").getMessage()); + + Map> partitionsToBeReassigned = new HashMap<>(); + + partitionsToBeReassigned.put(new TopicPartition("foo", 0), Arrays.asList(1, 2, 3)); + partitionsToBeReassigned.put(new TopicPartition("foo", 1), Arrays.asList(3, 4, 5)); + + Tuple2>, scala.collection.Map> actual = parseExecuteAssignmentArgs( + "{\"version\":1,\"partitions\":" + + "[{\"topic\":\"foo\",\"partition\":0,\"replicas\":[1,2,3],\"log_dirs\":[\"any\",\"any\",\"any\"]}," + + "{\"topic\":\"foo\",\"partition\":1,\"replicas\":[3,4,5],\"log_dirs\":[\"any\",\"any\",\"any\"]}" + + "]}"); + + assertEquals(partitionsToBeReassigned, javaMap(actual._1, this::toJavaList)); + assertTrue(actual._2.isEmpty()); + + Map replicaAssignment = new HashMap<>(); + + replicaAssignment.put(new TopicPartitionReplica("foo", 0, 1), "/tmp/a"); + replicaAssignment.put(new TopicPartitionReplica("foo", 0, 2), "/tmp/b"); + replicaAssignment.put(new TopicPartitionReplica("foo", 0, 3), "/tmp/c"); + + actual = parseExecuteAssignmentArgs( + "{\"version\":1,\"partitions\":" + + "[{\"topic\":\"foo\",\"partition\":0,\"replicas\":[1,2,3],\"log_dirs\":[\"/tmp/a\",\"/tmp/b\",\"/tmp/c\"]}" + + "]}"); + + assertEquals(Collections.singletonMap(new TopicPartition("foo", 0), Arrays.asList(1, 2, 3)), javaMap(actual._1, this::toJavaList)); + assertEquals(replicaAssignment, javaMap(actual._2)); + } + + @Test + public void testExecuteWithInvalidPartitionsFails() { + try (MockAdminClient adminClient = new MockAdminClient.Builder().numBrokers(5).build()) { + addTopics(adminClient); + assertStartsWith("Topic quux not found", + assertThrows(ExecutionException.class, () -> executeAssignment(adminClient, false, + "{\"version\":1,\"partitions\":" + + "[{\"topic\":\"foo\",\"partition\":0,\"replicas\":[0,1],\"log_dirs\":[\"any\",\"any\"]}," + + "{\"topic\":\"quux\",\"partition\":0,\"replicas\":[2,3,4],\"log_dirs\":[\"any\",\"any\",\"any\"]}" + + "]}", -1L, -1L, 10000L, Time.SYSTEM), "Expected reassignment with non-existent topic to fail").getCause().getMessage()); + } + } + + @Test + public void testExecuteWithInvalidBrokerIdFails() { + try (MockAdminClient adminClient = new MockAdminClient.Builder().numBrokers(4).build()) { + addTopics(adminClient); + assertStartsWith("Unknown broker id 4", + assertThrows(AdminCommandFailedException.class, () -> executeAssignment(adminClient, false, + "{\"version\":1,\"partitions\":" + + "[{\"topic\":\"foo\",\"partition\":0,\"replicas\":[0,1],\"log_dirs\":[\"any\",\"any\"]}," + + "{\"topic\":\"foo\",\"partition\":1,\"replicas\":[2,3,4],\"log_dirs\":[\"any\",\"any\",\"any\"]}" + + "]}", -1L, -1L, 10000L, Time.SYSTEM), "Expected reassignment with non-existent broker id to fail").getMessage()); + } + } + + @Test + public void testModifyBrokerInterBrokerThrottle() throws Exception { + try (MockAdminClient adminClient = new MockAdminClient.Builder().numBrokers(4).build()) { + modifyInterBrokerThrottle(adminClient, toSet(Arrays.asList(0, 1, 2)), 1000); + modifyInterBrokerThrottle(adminClient, toSet(Arrays.asList(0, 3)), 100); + List brokers = new ArrayList<>(); + for (int i = 0; i < 4; i++) + brokers.add(new ConfigResource(ConfigResource.Type.BROKER, Integer.toString(i))); + Map results = adminClient.describeConfigs(brokers).all().get(); + verifyBrokerThrottleResults(results.get(brokers.get(0)), 100, -1); + verifyBrokerThrottleResults(results.get(brokers.get(1)), 1000, -1); + verifyBrokerThrottleResults(results.get(brokers.get(2)), 1000, -1); + verifyBrokerThrottleResults(results.get(brokers.get(3)), 100, -1); + } + } + + @Test + public void testModifyLogDirThrottle() throws Exception { + try (MockAdminClient adminClient = new MockAdminClient.Builder().numBrokers(4).build()) { + modifyLogDirThrottle(adminClient, toSet(Arrays.asList(0, 1, 2)), 2000); + modifyLogDirThrottle(adminClient, toSet(Arrays.asList(0, 3)), -1); + + List brokers = new ArrayList<>(); + for (int i = 0; i < 4; i++) + brokers.add(new ConfigResource(ConfigResource.Type.BROKER, Integer.toString(i))); + + Map results = adminClient.describeConfigs(brokers).all().get(); + + verifyBrokerThrottleResults(results.get(brokers.get(0)), -1, 2000); + verifyBrokerThrottleResults(results.get(brokers.get(1)), -1, 2000); + verifyBrokerThrottleResults(results.get(brokers.get(2)), -1, 2000); + verifyBrokerThrottleResults(results.get(brokers.get(3)), -1, -1); + } + } + + @Test + public void testCurReassignmentsToString() throws Exception { + try (MockAdminClient adminClient = new MockAdminClient.Builder().numBrokers(4).build()) { + addTopics(adminClient); + assertEquals("No partition reassignments found.", curReassignmentsToString(adminClient)); + + Map> reassignments = new HashMap<>(); + + reassignments.put(new TopicPartition("foo", 1), Arrays.asList(4, 5, 3)); + reassignments.put(new TopicPartition("foo", 0), Arrays.asList(0, 1, 4, 2)); + reassignments.put(new TopicPartition("bar", 0), Arrays.asList(2, 3)); + + Map reassignmentResult = javaMap(alterPartitionReassignments(adminClient, toScala(reassignments, this::toRawSeq))); + + assertTrue(reassignmentResult.isEmpty()); + assertEquals(String.join(System.lineSeparator(), Arrays.asList("Current partition reassignments:", + "bar-0: replicas: 2,3,0. removing: 0.", + "foo-0: replicas: 0,1,2. adding: 4.", + "foo-1: replicas: 1,2,3. adding: 4,5. removing: 1,2.")), + curReassignmentsToString(adminClient)); + } + } + + private void verifyBrokerThrottleResults(Config config, + long expectedInterBrokerThrottle, + long expectedReplicaAlterLogDirsThrottle) { + Map configs = new HashMap<>(); + config.entries().forEach(entry -> configs.put(entry.name(), entry.value())); + if (expectedInterBrokerThrottle >= 0) { + assertEquals(Long.toString(expectedInterBrokerThrottle), + configs.getOrDefault(brokerLevelLeaderThrottle(), "")); + assertEquals(Long.toString(expectedInterBrokerThrottle), + configs.getOrDefault(brokerLevelFollowerThrottle(), "")); + } + if (expectedReplicaAlterLogDirsThrottle >= 0) { + assertEquals(Long.toString(expectedReplicaAlterLogDirsThrottle), + configs.getOrDefault(brokerLevelLogDirThrottle(), "")); + } + } + + @Test + public void testModifyTopicThrottles() throws Exception { + try (MockAdminClient adminClient = new MockAdminClient.Builder().numBrokers(4).build()) { + addTopics(adminClient); + + Map leaderThrottles = new HashMap<>(); + + leaderThrottles.put("foo", "leaderFoo"); + leaderThrottles.put("bar", "leaderBar"); + + modifyTopicThrottles(adminClient, + toScala(leaderThrottles), + toScala(Collections.singletonMap("bar", "followerBar"))); + List topics = Stream.of("bar", "foo").map( + id -> new ConfigResource(ConfigResource.Type.TOPIC, id)).collect(Collectors.toList()); + Map results = adminClient.describeConfigs(topics).all().get(); + verifyTopicThrottleResults(results.get(topics.get(0)), "leaderBar", "followerBar"); + verifyTopicThrottleResults(results.get(topics.get(1)), "leaderFoo", ""); + } + } + + private void verifyTopicThrottleResults(Config config, + String expectedLeaderThrottle, + String expectedFollowerThrottle) { + Map configs = new HashMap<>(); + config.entries().forEach(entry -> configs.put(entry.name(), entry.value())); + assertEquals(expectedLeaderThrottle, + configs.getOrDefault(ReassignPartitionsCommand.topicLevelLeaderThrottle(), "")); + assertEquals(expectedFollowerThrottle, + configs.getOrDefault(ReassignPartitionsCommand.topicLevelFollowerThrottle(), "")); + } + + @Test + public void testAlterReplicaLogDirs() throws Exception { + try (MockAdminClient adminClient = new MockAdminClient.Builder(). + numBrokers(4). + brokerLogDirs(Collections.nCopies(4, + Arrays.asList("/tmp/kafka-logs0", "/tmp/kafka-logs1"))). + build()) { + + addTopics(adminClient); + + Map assignment = new HashMap<>(); + + assignment.put(new TopicPartitionReplica("foo", 0, 0), "/tmp/kafka-logs1"); + assignment.put(new TopicPartitionReplica("quux", 1, 0), "/tmp/kafka-logs1"); + + assertEquals(new HashSet<>(Collections.singletonList( + new TopicPartitionReplica("foo", 0, 0) + )), + toJavaSet(alterReplicaLogDirs(adminClient, toScala(assignment)))); + } + } + + public void assertStartsWith(String prefix, String str) { + assertTrue(str.startsWith(prefix), String.format("Expected the string to start with %s, but it was %s", prefix, str)); + } + + @Test + public void testPropagateInvalidJsonError() { + try (MockAdminClient adminClient = new MockAdminClient.Builder().numBrokers(4).build()) { + addTopics(adminClient); + assertStartsWith("Unexpected character", + assertThrows(AdminOperationException.class, () -> executeAssignment(adminClient, false, "{invalid_json", -1L, -1L, 10000L, Time.SYSTEM)).getMessage()); + } + } + + private scala.collection.immutable.Set toSet(Collection set) { + return toMutableSet(set).toSet(); + } + + private scala.collection.mutable.Set toMutableSet(Collection set) { + return JavaConverters.asScala(new HashSet<>(set)); + } + + @SuppressWarnings("unchecked") + private Seq toRawSeq(Collection col) { + return (Seq) toSeq(col); + } + + private Seq toSeq(Collection col) { + return toSeq(col, Function.identity()); + } + + private Seq toSeq(Collection col, Function mapper) { + return JavaConverters.asScala(col.stream().map(mapper).collect(Collectors.toList())).toSeq(); + } + + @SuppressWarnings("unchecked") + private scala.collection.mutable.Map toMoveMap(Map moveMap) { + scala.collection.mutable.Map res = + (scala.collection.mutable.Map) Map$.MODULE$.mapFactory().newBuilder().result(); + + moveMap.forEach((k, v) -> + res.addOne(new Tuple2<>(k, new ReassignPartitionsCommand.PartitionMove(toMutableSet(v.sources), toMutableSet(v.destinations))))); + + return res; + } + + private scala.collection.Map toScala(Map map) { + return toScala(map, Function.identity()); + } + + private scala.collection.Map toScala(Map map, Function mapper) { + return JavaConverters.asScala(map.entrySet().stream(). + collect(Collectors.toMap(Map.Entry::getKey, e -> mapper.apply(e.getValue())))); + } + + @SuppressWarnings("unchecked") + private scala.collection.mutable.Map mutableMap(scala.collection.Map map) { + return (scala.collection.mutable.Map) map.to(Map$.MODULE$.mapFactory()); + } + + private ReassignPartitionsCommand.LogDirMoveState toScala(LogDirMoveState state) { + if (state instanceof ActiveMoveState) { + ActiveMoveState s = (ActiveMoveState) state; + return new ReassignPartitionsCommand.ActiveMoveState(s.currentLogDir, s.targetLogDir, s.futureLogDir); + } else if (state instanceof CancelledMoveState) { + CancelledMoveState s = (CancelledMoveState) state; + return new ReassignPartitionsCommand.CancelledMoveState(s.currentLogDir, s.targetLogDir); + } else if (state instanceof CompletedMoveState) { + CompletedMoveState s = (CompletedMoveState) state; + return new ReassignPartitionsCommand.CompletedMoveState(s.targetLogDir); + } else if (state instanceof MissingLogDirMoveState) { + MissingLogDirMoveState s = (MissingLogDirMoveState) state; + return new ReassignPartitionsCommand.MissingLogDirMoveState(s.targetLogDir); + } else if (state instanceof MissingReplicaMoveState) { + MissingReplicaMoveState s = (MissingReplicaMoveState) state; + return new ReassignPartitionsCommand.MissingReplicaMoveState(s.targetLogDir); + } + + throw new IllegalArgumentException("Unknown state " + state); + } + + private ReassignPartitionsCommand.PartitionReassignmentState toScala(PartitionReassignmentState state) { + return new ReassignPartitionsCommand.PartitionReassignmentState( + toRawSeq(state.currentReplicas), + toRawSeq(state.targetReplicas), + state.done); + } + + private LogDirMoveState toJava(ReassignPartitionsCommand.LogDirMoveState state) { + if (state instanceof ReassignPartitionsCommand.ActiveMoveState) { + ReassignPartitionsCommand.ActiveMoveState s = (ReassignPartitionsCommand.ActiveMoveState) state; + return new ActiveMoveState(s.currentLogDir(), s.targetLogDir(), s.futureLogDir()); + } else if (state instanceof ReassignPartitionsCommand.CancelledMoveState) { + ReassignPartitionsCommand.CancelledMoveState s = (ReassignPartitionsCommand.CancelledMoveState) state; + return new CancelledMoveState(s.currentLogDir(), s.targetLogDir()); + } else if (state instanceof ReassignPartitionsCommand.CompletedMoveState) { + ReassignPartitionsCommand.CompletedMoveState s = (ReassignPartitionsCommand.CompletedMoveState) state; + return new CompletedMoveState(s.targetLogDir()); + } else if (state instanceof ReassignPartitionsCommand.MissingLogDirMoveState) { + ReassignPartitionsCommand.MissingLogDirMoveState s = (ReassignPartitionsCommand.MissingLogDirMoveState) state; + return new MissingLogDirMoveState(s.targetLogDir()); + } else if (state instanceof ReassignPartitionsCommand.MissingReplicaMoveState) { + ReassignPartitionsCommand.MissingReplicaMoveState s = (ReassignPartitionsCommand.MissingReplicaMoveState) state; + return new MissingReplicaMoveState(s.targetLogDir()); + } + + throw new IllegalArgumentException("Unknown state " + state); + } + + private PartitionReassignmentState toJava(ReassignPartitionsCommand.PartitionReassignmentState state) { + return new PartitionReassignmentState( + toJavaList(state.currentReplicas(), o -> (Integer) o), + toJavaList(state.targetReplicas(), o -> (Integer) o), + state.done()); + } + + private Tuple2> toScala(Tuple2> t) { + return mapTuple(t, Function.identity(), this::toRawSeq); + } + + private Map javaMap(scala.collection.Map map) { + return javaMap(map, Function.identity()); + } + + private Map javaMap(scala.collection.Map map, Function mapper) { + Map res = new HashMap<>(); + map.foreach(t -> res.put(t._1, mapper.apply(t._2))); + return res; + } + + private Map javaMap2(scala.collection.mutable.Map map) { + Map res = new HashMap<>(); + map.foreach(t -> res.put((Integer) t._1, new PartitionMove(toJavaSet(t._2.sources()), toJavaSet(t._2.destinations())))); + return res; + } + + @SuppressWarnings("unchecked") + private Map> javaMap3(scala.collection.Map> map) { + Map> res = new HashMap<>(); + map.foreach(e -> res.put(e._1, (List) toJavaList(e._2))); + return res; + } + + @SuppressWarnings("unchecked") + private Set toJavaSet(scala.collection.mutable.Set set) { + Set res = new HashSet<>(); + set.foreach(e -> res.add((T) e)); + return res; + } + + @SuppressWarnings("unchecked") + private Set toJavaSet(scala.collection.Set set) { + Set res = new HashSet<>(); + set.foreach(e -> res.add((T) e)); + return res; + } + + private List toJavaList(Seq seq) { + List res = new ArrayList<>(); + seq.foreach(res::add); + return res; + } + + private List toJavaList(Seq seq, Function mapper) { + List res = new ArrayList<>(); + seq.foreach(e -> res.add(mapper.apply(e))); + return res; + } + + private Tuple2 mapTuple(Tuple2 t, Function mapper1, Function mapper2) { + return new Tuple2<>(mapper1.apply(t._1), mapper2.apply(t._2)); + } +} From dcb8b2bcf2d1a7efbe13882fa41ecce8adb94d85 Mon Sep 17 00:00:00 2001 From: nizhikov Date: Thu, 7 Sep 2023 20:35:37 +0300 Subject: [PATCH 32/78] KAFKA-14595 Merge trunk --- .../kafka/tools/reassign/ReassignPartitionsIntegrationTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsIntegrationTest.java b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsIntegrationTest.java index 9939f1e2a2f81..1243bda032fb4 100644 --- a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsIntegrationTest.java +++ b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsIntegrationTest.java @@ -198,7 +198,7 @@ public void testHighWaterMarkAfterPartitionReassignment(String quorum) throws Ex // Set the high water mark of foo-0 to 123 on its leader. TopicPartition part = new TopicPartition("foo", 0); - cluster.servers.get(0).replicaManager().logManager().truncateFullyAndStartAt(part, 123L, false); + cluster.servers.get(0).replicaManager().logManager().truncateFullyAndStartAt(part, 123L, false, None$.empty()); // Execute the assignment runExecuteAssignment(cluster.adminClient, false, assignment, -1L, -1L); From 2d76eca1bb1661111e044d24750c3d19bb454503 Mon Sep 17 00:00:00 2001 From: nizhikov Date: Thu, 7 Sep 2023 20:37:03 +0300 Subject: [PATCH 33/78] KAFKA-14595 Merge trunk --- .../kafka/tools/reassign/ReassignPartitionsCommandArgsTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommandArgsTest.java b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommandArgsTest.java index 0354647bd7092..c248796d25f07 100644 --- a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommandArgsTest.java +++ b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommandArgsTest.java @@ -16,7 +16,6 @@ */ package org.apache.kafka.tools.reassign; -import kafka.admin.ReassignPartitionsCommand; import org.apache.kafka.common.utils.Exit; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; From 3b6d212829c506998e27db134f7a23e7fc153e81 Mon Sep 17 00:00:00 2001 From: nizhikov Date: Thu, 7 Sep 2023 22:42:34 +0300 Subject: [PATCH 34/78] KAFKA-14595 Improving test --- .../reassign/ReassignPartitionsUnitTest.java | 175 ++++++++---------- 1 file changed, 76 insertions(+), 99 deletions(-) diff --git a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java index 2f13129a62b1f..18a7b12da1f23 100644 --- a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java +++ b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java @@ -36,6 +36,10 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; +import scala.Tuple2; +import scala.collection.Seq; +import scala.collection.mutable.Map$; +import scala.jdk.javaapi.CollectionConverters; import java.util.ArrayList; import java.util.Arrays; @@ -86,11 +90,6 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -import scala.Tuple2; -import scala.collection.JavaConverters; -import scala.collection.Seq; -import scala.collection.mutable.Map$; - @Timeout(60) public class ReassignPartitionsUnitTest { @@ -126,16 +125,19 @@ public void testCompareTopicPartitionReplicas() { public void testPartitionReassignStatesToString() { Map states = new HashMap<>(); - states.put(new TopicPartition("foo", 0), new PartitionReassignmentState(Arrays.asList(1, 2, 3), Arrays.asList(1, 2, 3), true)); - states.put(new TopicPartition("foo", 1), new PartitionReassignmentState(Arrays.asList(1, 2, 3), Arrays.asList(1, 2, 4), false)); - states.put(new TopicPartition("bar", 0), new PartitionReassignmentState(Arrays.asList(1, 2, 3), Arrays.asList(1, 2, 4), false)); + states.put(new TopicPartition("foo", 0), + new PartitionReassignmentState(Arrays.asList(1, 2, 3), Arrays.asList(1, 2, 3), true)); + states.put(new TopicPartition("foo", 1), + new PartitionReassignmentState(Arrays.asList(1, 2, 3), Arrays.asList(1, 2, 4), false)); + states.put(new TopicPartition("bar", 0), + new PartitionReassignmentState(Arrays.asList(1, 2, 3), Arrays.asList(1, 2, 4), false)); assertEquals(String.join(System.lineSeparator(), Arrays.asList( "Status of partition reassignment:", "Reassignment of partition bar-0 is still in progress.", "Reassignment of partition foo-0 is completed.", "Reassignment of partition foo-1 is still in progress.")), - partitionReassignmentStatesToString(toScala(states, this::toScala))); + partitionReassignmentStatesToString(asScala(states, this::asScala))); } private void addTopics(MockAdminClient adminClient) { @@ -156,7 +158,7 @@ private void addTopics(MockAdminClient adminClient) { } @Test - public void testFindPartitionReassignmentStates() throws Exception { + public void testFindPartitionReassignmentStates() { try (MockAdminClient adminClient = new MockAdminClient.Builder().numBrokers(4).build()) { addTopics(adminClient); // Create a reassignment and test findPartitionReassignmentStates. @@ -165,7 +167,7 @@ public void testFindPartitionReassignmentStates() throws Exception { reassignments.put(new TopicPartition("foo", 0), Arrays.asList(0, 1, 3)); reassignments.put(new TopicPartition("quux", 0), Arrays.asList(1, 2, 3)); - Map> reassignmentResult = javaMap(alterPartitionReassignments(adminClient, toScala(reassignments, this::toRawSeq))) + Map> reassignmentResult = CollectionConverters.asJava(alterPartitionReassignments(adminClient, asScala(reassignments, this::toRawSeq))) .entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().getClass())); assertEquals(Collections.singletonMap(new TopicPartition("quux", 0), UnknownTopicOrPartitionException.class), reassignmentResult); @@ -180,13 +182,13 @@ public void testFindPartitionReassignmentStates() throws Exception { findPartitionReassignmentStates(adminClient, toSeq(Arrays.asList( new Tuple2<>(new TopicPartition("foo", 0), Arrays.asList(0, 1, 3)), new Tuple2<>(new TopicPartition("foo", 1), Arrays.asList(1, 2, 3)) - ), this::toScala)); + ), this::asScala)); - assertEquals(expected._1, javaMap(actual._1, this::toJava)); + assertEquals(expected._1, javaMap(actual._1, this::asJava)); assertEquals(expected._2, actual._2); // Cancel the reassignment and test findPartitionReassignmentStates again. - Map> cancelResult = javaMap(cancelPartitionReassignments(adminClient, toSet(new HashSet<>(Arrays.asList(new TopicPartition("foo", 0), new TopicPartition("quux", 2)))))) + Map> cancelResult = CollectionConverters.asJava(cancelPartitionReassignments(adminClient, toSet(new HashSet<>(Arrays.asList(new TopicPartition("foo", 0), new TopicPartition("quux", 2)))))) .entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().getClass())); assertEquals(Collections.singletonMap(new TopicPartition("quux", 2), UnknownTopicOrPartitionException.class), cancelResult); @@ -200,9 +202,9 @@ public void testFindPartitionReassignmentStates() throws Exception { actual = findPartitionReassignmentStates(adminClient, toSeq(Arrays.asList( new Tuple2<>(new TopicPartition("foo", 0), Arrays.asList(0, 1, 3)), new Tuple2<>(new TopicPartition("foo", 1), Arrays.asList(1, 2, 3)) - ), this::toScala)); + ), this::asScala)); - assertEquals(expected._1, javaMap(actual._1, this::toJava)); + assertEquals(expected._1, javaMap(actual._1, this::asJava)); assertEquals(expected._2, actual._2); } } @@ -251,7 +253,7 @@ public void testFindLogDirMoveStates() throws Exception { assertEquals( expLogDirMoveStates, - javaMap(findLogDirMoveStates(adminClient, toScala(targetMoves)), this::toJava) + javaMap(findLogDirMoveStates(adminClient, CollectionConverters.asScala(targetMoves)), this::asJava) ); } } @@ -278,11 +280,11 @@ public void testReplicaMoveStatesToString() { "Partition quux-1 on broker 1 is being moved to log dir /tmp/kafka-logs2 instead of /tmp/kafka-logs1.", "Partition quux-2 is not found in any live log dir on broker 1. " + "There is likely an offline log directory on the broker.")), - replicaMoveStatesToString(toScala(states, this::toScala))); + replicaMoveStatesToString(asScala(states, this::asScala))); } @Test - public void testGetReplicaAssignments() throws Exception { + public void testGetReplicaAssignments() { try (MockAdminClient adminClient = new MockAdminClient.Builder().numBrokers(4).build()) { addTopics(adminClient); @@ -292,7 +294,7 @@ public void testGetReplicaAssignments() throws Exception { assignments.put(new TopicPartition("foo", 1), Arrays.asList(1, 2, 3)); assertEquals(assignments, - javaMap3(getReplicaAssignmentForTopics(adminClient, toSeq(Collections.singletonList("foo"))))); + asJava(getReplicaAssignmentForTopics(adminClient, CollectionConverters.asScala(Collections.singletonList("foo"))))); assignments.clear(); @@ -301,12 +303,12 @@ public void testGetReplicaAssignments() throws Exception { assertEquals(assignments, javaMap(getReplicaAssignmentForPartitions(adminClient, toSet(new HashSet<>(Arrays.asList( - new TopicPartition("foo", 0), new TopicPartition("bar", 0))))), this::toJavaList)); + new TopicPartition("foo", 0), new TopicPartition("bar", 0))))), CollectionConverters::asJava)); } } @Test - public void testGetBrokerRackInformation() throws Exception { + public void testGetBrokerRackInformation() { try (MockAdminClient adminClient = new MockAdminClient.Builder(). brokers(Arrays.asList(new Node(0, "localhost", 9092, "rack0"), new Node(1, "localhost", 9093, "rack1"), @@ -316,23 +318,23 @@ public void testGetBrokerRackInformation() throws Exception { assertEquals(Arrays.asList( new BrokerMetadata(0, Optional.of("rack0")), new BrokerMetadata(1, Optional.of("rack1")) - ), toJavaList(getBrokerMetadata(adminClient, toSeq(Arrays.asList(0, 1)), true))); + ), CollectionConverters.asJava(getBrokerMetadata(adminClient, CollectionConverters.asScala(Arrays.asList(0, 1)), true))); assertEquals(Arrays.asList( new BrokerMetadata(0, Optional.empty()), new BrokerMetadata(1, Optional.empty()) - ), toJavaList(getBrokerMetadata(adminClient, toSeq(Arrays.asList(0, 1)), false))); + ), CollectionConverters.asJava(getBrokerMetadata(adminClient, CollectionConverters.asScala(Arrays.asList(0, 1)), false))); assertStartsWith("Not all brokers have rack information", assertThrows(AdminOperationException.class, - () -> getBrokerMetadata(adminClient, toSeq(Arrays.asList(1, 2)), true)).getMessage()); + () -> getBrokerMetadata(adminClient, CollectionConverters.asScala(Arrays.asList(1, 2)), true)).getMessage()); assertEquals(Arrays.asList( new BrokerMetadata(1, Optional.empty()), new BrokerMetadata(2, Optional.empty()) - ), toJavaList(getBrokerMetadata(adminClient, toSeq(Arrays.asList(1, 2)), false))); + ), CollectionConverters.asJava(getBrokerMetadata(adminClient, CollectionConverters.asScala(Arrays.asList(1, 2)), false))); } } @Test - public void testParseGenerateAssignmentArgs() throws Exception { + public void testParseGenerateAssignmentArgs() { assertStartsWith("Broker list contains duplicate entries", assertThrows(AdminCommandFailedException.class, () -> parseGenerateAssignmentArgs( "{\"topics\": [{\"topic\": \"foo\"}], \"version\":1}", "1,1,2"), @@ -342,16 +344,16 @@ public void testParseGenerateAssignmentArgs() throws Exception { "{\"topics\": [{\"topic\": \"foo\"}], \"version\":1}", "5,2,3,4,5"), "Expected to detect duplicate broker list entries").getMessage()); assertEquals(new Tuple2<>(Arrays.asList(5, 2, 3, 4), Collections.singletonList("foo")), - mapTuple(parseGenerateAssignmentArgs("{\"topics\": [{\"topic\": \"foo\"}], \"version\":1}", - "5,2,3,4"), this::toJavaList, this::toJavaList)); + map(parseGenerateAssignmentArgs("{\"topics\": [{\"topic\": \"foo\"}], \"version\":1}", + "5,2,3,4"), CollectionConverters::asJava, CollectionConverters::asJava)); assertStartsWith("List of topics to reassign contains duplicate entries", assertThrows(AdminCommandFailedException.class, () -> parseGenerateAssignmentArgs( "{\"topics\": [{\"topic\": \"foo\"},{\"topic\": \"foo\"}], \"version\":1}", "5,2,3,4"), "Expected to detect duplicate topic entries").getMessage()); assertEquals(new Tuple2<>(Arrays.asList(5, 3, 4), Arrays.asList("foo", "bar")), - mapTuple(parseGenerateAssignmentArgs( + map(parseGenerateAssignmentArgs( "{\"topics\": [{\"topic\": \"foo\"},{\"topic\": \"bar\"}], \"version\":1}", - "5,3,4"), this::toJavaList, this::toJavaList)); + "5,3,4"), CollectionConverters::asJava, CollectionConverters::asJava)); } @Test @@ -377,7 +379,7 @@ public void testGenerateAssignmentWithInvalidPartitionsFails() { } @Test - public void testGenerateAssignmentWithInconsistentRacks() throws Exception { + public void testGenerateAssignmentWithInconsistentRacks() { try (MockAdminClient adminClient = new MockAdminClient.Builder(). brokers(Arrays.asList( new Node(0, "localhost", 9092, "rack0"), @@ -394,8 +396,8 @@ public void testGenerateAssignmentWithInconsistentRacks() throws Exception { () -> generateAssignment(adminClient, "{\"topics\":[{\"topic\":\"foo\"}]}", "0,1,2,3", true), "Expected generateAssignment to fail").getMessage()); // It should succeed when --disable-rack-aware is used. - Tuple2>, Map>> proposedCurrent = mapTuple(generateAssignment(adminClient, - "{\"topics\":[{\"topic\":\"foo\"}]}", "0,1,2,3", false), this::javaMap3, this::javaMap3); + Tuple2>, Map>> proposedCurrent = map(generateAssignment(adminClient, + "{\"topics\":[{\"topic\":\"foo\"}]}", "0,1,2,3", false), this::asJava, this::asJava); Map> expCurrent = new HashMap<>(); @@ -407,15 +409,15 @@ public void testGenerateAssignmentWithInconsistentRacks() throws Exception { } @Test - public void testGenerateAssignmentWithFewerBrokers() throws Exception { + public void testGenerateAssignmentWithFewerBrokers() { try (MockAdminClient adminClient = new MockAdminClient.Builder().numBrokers(4).build()) { addTopics(adminClient); List goalBrokers = Arrays.asList(0, 1, 3); Tuple2>, Map>> proposedCurrent = - mapTuple(generateAssignment(adminClient, + map(generateAssignment(adminClient, "{\"topics\":[{\"topic\":\"foo\"},{\"topic\":\"bar\"}]}", - goalBrokers.stream().map(Object::toString).collect(Collectors.joining(",")), false), this::javaMap3, this::javaMap3); + goalBrokers.stream().map(Object::toString).collect(Collectors.joining(",")), false), this::asJava, this::asJava); Map> expCurrent = new HashMap<>(); @@ -432,7 +434,7 @@ public void testGenerateAssignmentWithFewerBrokers() throws Exception { } @Test - public void testCurrentPartitionReplicaAssignmentToString() throws Exception { + public void testCurrentPartitionReplicaAssignmentToString() { Map> proposedParts = new HashMap<>(); proposedParts.put(new TopicPartition("foo", 1), Arrays.asList(1, 2, 3)); @@ -455,7 +457,7 @@ public void testCurrentPartitionReplicaAssignmentToString() throws Exception { "", "Save this to use as the --reassignment-json-file option during rollback" )), - currentPartitionReplicaAssignmentToString(toScala(proposedParts, this::toRawSeq), toScala(currentParts, this::toRawSeq)) + currentPartitionReplicaAssignmentToString(asScala(proposedParts, this::toRawSeq), asScala(currentParts, this::toRawSeq)) ); } @@ -503,7 +505,7 @@ public void testMoveMap() { currentParts.put(new TopicPartition("bar", 0), Arrays.asList(2, 3, 4)); currentParts.put(new TopicPartition("baz", 0), Arrays.asList(1, 2, 3)); - Map> moveMap = javaMap(calculateProposedMoveMap(toScala(currentReassignments), toScala(proposedParts, this::toRawSeq), toScala(currentParts, this::toRawSeq)), this::javaMap2); + Map> moveMap = javaMap(calculateProposedMoveMap(CollectionConverters.asScala(currentReassignments), asScala(proposedParts, this::toRawSeq), asScala(currentParts, this::toRawSeq)), this::asJava); Map> expMoveMap = new HashMap<>(); @@ -526,24 +528,24 @@ public void testMoveMap() { expLeaderThrottle.put("foo", "0:1,0:2,0:3,1:4,1:5,1:6,2:1,2:2,3:1,3:2,4:1,4:2,5:1,5:2"); expLeaderThrottle.put("bar", "0:2,0:3,0:4"); - assertEquals(expLeaderThrottle, javaMap(calculateLeaderThrottles(mutableMap(toScala(moveMap, this::toMoveMap))))); + assertEquals(expLeaderThrottle, CollectionConverters.asJava(calculateLeaderThrottles(mutableMap(asScala(moveMap, this::toMoveMap))))); Map expFollowerThrottle = new HashMap<>(); expFollowerThrottle.put("foo", "0:5,1:7,1:8,2:3,2:4,3:5,3:6,4:3,5:3,5:4,5:5,5:6"); expFollowerThrottle.put("bar", "0:1"); - assertEquals(expFollowerThrottle, javaMap(calculateFollowerThrottles(mutableMap(toScala(moveMap, this::toMoveMap))))); + assertEquals(expFollowerThrottle, CollectionConverters.asJava(calculateFollowerThrottles(mutableMap(asScala(moveMap, this::toMoveMap))))); - assertEquals(new HashSet<>(Arrays.asList(1, 2, 3, 4, 5, 6, 7, 8)), toJavaSet(calculateReassigningBrokers(mutableMap(toScala(moveMap, this::toMoveMap))))); + assertEquals(new HashSet<>(Arrays.asList(1, 2, 3, 4, 5, 6, 7, 8)), CollectionConverters.asJava(calculateReassigningBrokers(mutableMap(asScala(moveMap, this::toMoveMap))))); - assertEquals(new HashSet<>(Arrays.asList(0, 2)), toJavaSet(calculateMovingBrokers(toSet(new HashSet<>(Arrays.asList( + assertEquals(new HashSet<>(Arrays.asList(0, 2)), CollectionConverters.asJava(calculateMovingBrokers(toSet(new HashSet<>(Arrays.asList( new TopicPartitionReplica("quux", 0, 0), new TopicPartitionReplica("quux", 1, 2))))))); } @Test - public void testParseExecuteAssignmentArgs() throws Exception { + public void testParseExecuteAssignmentArgs() { assertStartsWith("Partition reassignment list cannot be empty", assertThrows(AdminCommandFailedException.class, () -> parseExecuteAssignmentArgs("{\"version\":1,\"partitions\":[]}"), @@ -578,7 +580,7 @@ public void testParseExecuteAssignmentArgs() throws Exception { "{\"topic\":\"foo\",\"partition\":1,\"replicas\":[3,4,5],\"log_dirs\":[\"any\",\"any\",\"any\"]}" + "]}"); - assertEquals(partitionsToBeReassigned, javaMap(actual._1, this::toJavaList)); + assertEquals(partitionsToBeReassigned, javaMap(actual._1, CollectionConverters::asJava)); assertTrue(actual._2.isEmpty()); Map replicaAssignment = new HashMap<>(); @@ -592,8 +594,8 @@ public void testParseExecuteAssignmentArgs() throws Exception { "[{\"topic\":\"foo\",\"partition\":0,\"replicas\":[1,2,3],\"log_dirs\":[\"/tmp/a\",\"/tmp/b\",\"/tmp/c\"]}" + "]}"); - assertEquals(Collections.singletonMap(new TopicPartition("foo", 0), Arrays.asList(1, 2, 3)), javaMap(actual._1, this::toJavaList)); - assertEquals(replicaAssignment, javaMap(actual._2)); + assertEquals(Collections.singletonMap(new TopicPartition("foo", 0), Arrays.asList(1, 2, 3)), javaMap(actual._1, CollectionConverters::asJava)); + assertEquals(replicaAssignment, CollectionConverters.asJava(actual._2)); } @Test @@ -658,7 +660,7 @@ public void testModifyLogDirThrottle() throws Exception { } @Test - public void testCurReassignmentsToString() throws Exception { + public void testCurReassignmentsToString() { try (MockAdminClient adminClient = new MockAdminClient.Builder().numBrokers(4).build()) { addTopics(adminClient); assertEquals("No partition reassignments found.", curReassignmentsToString(adminClient)); @@ -669,7 +671,7 @@ public void testCurReassignmentsToString() throws Exception { reassignments.put(new TopicPartition("foo", 0), Arrays.asList(0, 1, 4, 2)); reassignments.put(new TopicPartition("bar", 0), Arrays.asList(2, 3)); - Map reassignmentResult = javaMap(alterPartitionReassignments(adminClient, toScala(reassignments, this::toRawSeq))); + Map reassignmentResult = CollectionConverters.asJava(alterPartitionReassignments(adminClient, asScala(reassignments, this::toRawSeq))); assertTrue(reassignmentResult.isEmpty()); assertEquals(String.join(System.lineSeparator(), Arrays.asList("Current partition reassignments:", @@ -708,8 +710,8 @@ public void testModifyTopicThrottles() throws Exception { leaderThrottles.put("bar", "leaderBar"); modifyTopicThrottles(adminClient, - toScala(leaderThrottles), - toScala(Collections.singletonMap("bar", "followerBar"))); + CollectionConverters.asScala(leaderThrottles), + CollectionConverters.asScala(Collections.singletonMap("bar", "followerBar"))); List topics = Stream.of("bar", "foo").map( id -> new ConfigResource(ConfigResource.Type.TOPIC, id)).collect(Collectors.toList()); Map results = adminClient.describeConfigs(topics).all().get(); @@ -730,7 +732,7 @@ private void verifyTopicThrottleResults(Config config, } @Test - public void testAlterReplicaLogDirs() throws Exception { + public void testAlterReplicaLogDirs() { try (MockAdminClient adminClient = new MockAdminClient.Builder(). numBrokers(4). brokerLogDirs(Collections.nCopies(4, @@ -747,7 +749,7 @@ public void testAlterReplicaLogDirs() throws Exception { assertEquals(new HashSet<>(Collections.singletonList( new TopicPartitionReplica("foo", 0, 0) )), - toJavaSet(alterReplicaLogDirs(adminClient, toScala(assignment)))); + CollectionConverters.asJava(alterReplicaLogDirs(adminClient, CollectionConverters.asScala(assignment)))); } } @@ -769,20 +771,16 @@ private scala.collection.immutable.Set toSet(Collection set) { } private scala.collection.mutable.Set toMutableSet(Collection set) { - return JavaConverters.asScala(new HashSet<>(set)); + return CollectionConverters.asScala(new HashSet<>(set)); } @SuppressWarnings("unchecked") private Seq toRawSeq(Collection col) { - return (Seq) toSeq(col); - } - - private Seq toSeq(Collection col) { - return toSeq(col, Function.identity()); + return (Seq) toSeq(col, Function.identity()); } private Seq toSeq(Collection col, Function mapper) { - return JavaConverters.asScala(col.stream().map(mapper).collect(Collectors.toList())).toSeq(); + return CollectionConverters.asScala(col.stream().map(mapper).collect(Collectors.toList())).toSeq(); } @SuppressWarnings("unchecked") @@ -796,12 +794,8 @@ private scala.collection.mutable.Map scala.collection.Map toScala(Map map) { - return toScala(map, Function.identity()); - } - - private scala.collection.Map toScala(Map map, Function mapper) { - return JavaConverters.asScala(map.entrySet().stream(). + private scala.collection.Map asScala(Map map, Function mapper) { + return CollectionConverters.asScala(map.entrySet().stream(). collect(Collectors.toMap(Map.Entry::getKey, e -> mapper.apply(e.getValue())))); } @@ -810,7 +804,7 @@ private scala.collection.mutable.Map mutableMap(scala.collection.Ma return (scala.collection.mutable.Map) map.to(Map$.MODULE$.mapFactory()); } - private ReassignPartitionsCommand.LogDirMoveState toScala(LogDirMoveState state) { + private ReassignPartitionsCommand.LogDirMoveState asScala(LogDirMoveState state) { if (state instanceof ActiveMoveState) { ActiveMoveState s = (ActiveMoveState) state; return new ReassignPartitionsCommand.ActiveMoveState(s.currentLogDir, s.targetLogDir, s.futureLogDir); @@ -831,14 +825,14 @@ private ReassignPartitionsCommand.LogDirMoveState toScala(LogDirMoveState state) throw new IllegalArgumentException("Unknown state " + state); } - private ReassignPartitionsCommand.PartitionReassignmentState toScala(PartitionReassignmentState state) { + private ReassignPartitionsCommand.PartitionReassignmentState asScala(PartitionReassignmentState state) { return new ReassignPartitionsCommand.PartitionReassignmentState( toRawSeq(state.currentReplicas), toRawSeq(state.targetReplicas), state.done); } - private LogDirMoveState toJava(ReassignPartitionsCommand.LogDirMoveState state) { + private LogDirMoveState asJava(ReassignPartitionsCommand.LogDirMoveState state) { if (state instanceof ReassignPartitionsCommand.ActiveMoveState) { ReassignPartitionsCommand.ActiveMoveState s = (ReassignPartitionsCommand.ActiveMoveState) state; return new ActiveMoveState(s.currentLogDir(), s.targetLogDir(), s.futureLogDir()); @@ -859,19 +853,15 @@ private LogDirMoveState toJava(ReassignPartitionsCommand.LogDirMoveState state) throw new IllegalArgumentException("Unknown state " + state); } - private PartitionReassignmentState toJava(ReassignPartitionsCommand.PartitionReassignmentState state) { + private PartitionReassignmentState asJava(ReassignPartitionsCommand.PartitionReassignmentState state) { return new PartitionReassignmentState( - toJavaList(state.currentReplicas(), o -> (Integer) o), - toJavaList(state.targetReplicas(), o -> (Integer) o), + asJava(state.currentReplicas(), o -> (Integer) o), + asJava(state.targetReplicas(), o -> (Integer) o), state.done()); } - private Tuple2> toScala(Tuple2> t) { - return mapTuple(t, Function.identity(), this::toRawSeq); - } - - private Map javaMap(scala.collection.Map map) { - return javaMap(map, Function.identity()); + private Tuple2> asScala(Tuple2> t) { + return map(t, Function.identity(), this::toRawSeq); } private Map javaMap(scala.collection.Map map, Function mapper) { @@ -880,46 +870,33 @@ private Map javaMap(scala.collection.Map map, Function javaMap2(scala.collection.mutable.Map map) { + private Map asJava(scala.collection.mutable.Map map) { Map res = new HashMap<>(); - map.foreach(t -> res.put((Integer) t._1, new PartitionMove(toJavaSet(t._2.sources()), toJavaSet(t._2.destinations())))); + map.foreach(t -> res.put((Integer) t._1, new PartitionMove(asJava(t._2.sources()), asJava(t._2.destinations())))); return res; } @SuppressWarnings("unchecked") - private Map> javaMap3(scala.collection.Map> map) { + private Map> asJava(scala.collection.Map> map) { Map> res = new HashMap<>(); - map.foreach(e -> res.put(e._1, (List) toJavaList(e._2))); + map.foreach(e -> res.put(e._1, (List) CollectionConverters.asJava(e._2))); return res; } @SuppressWarnings("unchecked") - private Set toJavaSet(scala.collection.mutable.Set set) { + private Set asJava(scala.collection.mutable.Set set) { Set res = new HashSet<>(); set.foreach(e -> res.add((T) e)); return res; } - @SuppressWarnings("unchecked") - private Set toJavaSet(scala.collection.Set set) { - Set res = new HashSet<>(); - set.foreach(e -> res.add((T) e)); - return res; - } - - private List toJavaList(Seq seq) { - List res = new ArrayList<>(); - seq.foreach(res::add); - return res; - } - - private List toJavaList(Seq seq, Function mapper) { + private List asJava(Seq seq, Function mapper) { List res = new ArrayList<>(); seq.foreach(e -> res.add(mapper.apply(e))); return res; } - private Tuple2 mapTuple(Tuple2 t, Function mapper1, Function mapper2) { + private Tuple2 map(Tuple2 t, Function mapper1, Function mapper2) { return new Tuple2<>(mapper1.apply(t._1), mapper2.apply(t._2)); } } From 97fb3f434ba08fc2015e4c9d8d3a44f5dac46d63 Mon Sep 17 00:00:00 2001 From: nizhikov Date: Thu, 7 Sep 2023 23:23:54 +0300 Subject: [PATCH 35/78] KAFKA-14595 Improving test --- checkstyle/import-control.xml | 3 +-- .../kafka/tools/reassign/ReassignPartitionsUnitTest.java | 3 ++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/checkstyle/import-control.xml b/checkstyle/import-control.xml index c069d69f587cd..f4b5decdd88c2 100644 --- a/checkstyle/import-control.xml +++ b/checkstyle/import-control.xml @@ -271,11 +271,9 @@ - - @@ -294,6 +292,7 @@ + diff --git a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java index 18a7b12da1f23..d648ba72688e1 100644 --- a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java +++ b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java @@ -870,9 +870,10 @@ private Map javaMap(scala.collection.Map map, Function asJava(scala.collection.mutable.Map map) { Map res = new HashMap<>(); - map.foreach(t -> res.put((Integer) t._1, new PartitionMove(asJava(t._2.sources()), asJava(t._2.destinations())))); + map.foreach(t -> res.put((Integer) t._1, new PartitionMove(CollectionConverters.asJava(t._2.sources().toSet()), asJava(t._2.destinations())))); return res; } From 1372fcb243499c88aa5dd521d0b5d06e58e9821d Mon Sep 17 00:00:00 2001 From: nizhikov Date: Thu, 7 Sep 2023 23:26:20 +0300 Subject: [PATCH 36/78] KAFKA-14595 Improving test --- .../tools/reassign/ReassignPartitionsUnitTest.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java index d648ba72688e1..6846569779cf1 100644 --- a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java +++ b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java @@ -315,21 +315,21 @@ public void testGetBrokerRackInformation() { new Node(2, "localhost", 9094, null))). build()) { - assertEquals(Arrays.asList( + assertEquals(CollectionConverters.asScala(Arrays.asList( new BrokerMetadata(0, Optional.of("rack0")), new BrokerMetadata(1, Optional.of("rack1")) - ), CollectionConverters.asJava(getBrokerMetadata(adminClient, CollectionConverters.asScala(Arrays.asList(0, 1)), true))); - assertEquals(Arrays.asList( + )), getBrokerMetadata(adminClient, CollectionConverters.asScala(Arrays.asList(0, 1)), true)); + assertEquals(CollectionConverters.asScala(Arrays.asList( new BrokerMetadata(0, Optional.empty()), new BrokerMetadata(1, Optional.empty()) - ), CollectionConverters.asJava(getBrokerMetadata(adminClient, CollectionConverters.asScala(Arrays.asList(0, 1)), false))); + )), getBrokerMetadata(adminClient, CollectionConverters.asScala(Arrays.asList(0, 1)), false)); assertStartsWith("Not all brokers have rack information", assertThrows(AdminOperationException.class, () -> getBrokerMetadata(adminClient, CollectionConverters.asScala(Arrays.asList(1, 2)), true)).getMessage()); - assertEquals(Arrays.asList( + assertEquals(CollectionConverters.asScala(Arrays.asList( new BrokerMetadata(1, Optional.empty()), new BrokerMetadata(2, Optional.empty()) - ), CollectionConverters.asJava(getBrokerMetadata(adminClient, CollectionConverters.asScala(Arrays.asList(1, 2)), false))); + )), getBrokerMetadata(adminClient, CollectionConverters.asScala(Arrays.asList(1, 2)), false)); } } From 928ba4201cd92f768637756b46d0657762c139db Mon Sep 17 00:00:00 2001 From: nizhikov Date: Thu, 7 Sep 2023 23:56:33 +0300 Subject: [PATCH 37/78] KAFKA-14595 Improving test --- .../reassign/ReassignPartitionsUnitTest.java | 44 ++++++++----------- 1 file changed, 19 insertions(+), 25 deletions(-) diff --git a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java index 6846569779cf1..7795860a45ebe 100644 --- a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java +++ b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java @@ -123,21 +123,21 @@ public void testCompareTopicPartitionReplicas() { @Test public void testPartitionReassignStatesToString() { - Map states = new HashMap<>(); + Map states = new HashMap<>(); states.put(new TopicPartition("foo", 0), - new PartitionReassignmentState(Arrays.asList(1, 2, 3), Arrays.asList(1, 2, 3), true)); + asScala(new PartitionReassignmentState(Arrays.asList(1, 2, 3), Arrays.asList(1, 2, 3), true))); states.put(new TopicPartition("foo", 1), - new PartitionReassignmentState(Arrays.asList(1, 2, 3), Arrays.asList(1, 2, 4), false)); + asScala(new PartitionReassignmentState(Arrays.asList(1, 2, 3), Arrays.asList(1, 2, 4), false))); states.put(new TopicPartition("bar", 0), - new PartitionReassignmentState(Arrays.asList(1, 2, 3), Arrays.asList(1, 2, 4), false)); + asScala(new PartitionReassignmentState(Arrays.asList(1, 2, 3), Arrays.asList(1, 2, 4), false))); assertEquals(String.join(System.lineSeparator(), Arrays.asList( "Status of partition reassignment:", "Reassignment of partition bar-0 is still in progress.", "Reassignment of partition foo-0 is completed.", "Reassignment of partition foo-1 is still in progress.")), - partitionReassignmentStatesToString(asScala(states, this::asScala))); + partitionReassignmentStatesToString(CollectionConverters.asScala(states))); } private void addTopics(MockAdminClient adminClient) { @@ -260,17 +260,17 @@ public void testFindLogDirMoveStates() throws Exception { @Test public void testReplicaMoveStatesToString() { - Map states = new HashMap<>(); - - states.put(new TopicPartitionReplica("bar", 0, 0), new CompletedMoveState("/tmp/kafka-logs0")); - states.put(new TopicPartitionReplica("foo", 0, 0), new ActiveMoveState("/tmp/kafka-logs0", - "/tmp/kafka-logs1", "/tmp/kafka-logs1")); - states.put(new TopicPartitionReplica("foo", 1, 0), new CancelledMoveState("/tmp/kafka-logs0", - "/tmp/kafka-logs1")); - states.put(new TopicPartitionReplica("quux", 0, 0), new MissingReplicaMoveState("/tmp/kafka-logs1")); - states.put(new TopicPartitionReplica("quux", 1, 1), new ActiveMoveState("/tmp/kafka-logs0", - "/tmp/kafka-logs1", "/tmp/kafka-logs2")); - states.put(new TopicPartitionReplica("quux", 2, 1), new MissingLogDirMoveState("/tmp/kafka-logs1")); + Map states = new HashMap<>(); + + states.put(new TopicPartitionReplica("bar", 0, 0), asScala(new CompletedMoveState("/tmp/kafka-logs0"))); + states.put(new TopicPartitionReplica("foo", 0, 0), asScala(new ActiveMoveState("/tmp/kafka-logs0", + "/tmp/kafka-logs1", "/tmp/kafka-logs1"))); + states.put(new TopicPartitionReplica("foo", 1, 0), asScala(new CancelledMoveState("/tmp/kafka-logs0", + "/tmp/kafka-logs1"))); + states.put(new TopicPartitionReplica("quux", 0, 0), asScala(new MissingReplicaMoveState("/tmp/kafka-logs1"))); + states.put(new TopicPartitionReplica("quux", 1, 1), asScala(new ActiveMoveState("/tmp/kafka-logs0", + "/tmp/kafka-logs1", "/tmp/kafka-logs2"))); + states.put(new TopicPartitionReplica("quux", 2, 1), asScala(new MissingLogDirMoveState("/tmp/kafka-logs1"))); assertEquals(String.join(System.lineSeparator(), Arrays.asList( "Reassignment of replica bar-0-0 completed successfully.", @@ -280,7 +280,7 @@ public void testReplicaMoveStatesToString() { "Partition quux-1 on broker 1 is being moved to log dir /tmp/kafka-logs2 instead of /tmp/kafka-logs1.", "Partition quux-2 is not found in any live log dir on broker 1. " + "There is likely an offline log directory on the broker.")), - replicaMoveStatesToString(asScala(states, this::asScala))); + replicaMoveStatesToString(CollectionConverters.asScala(states))); } @Test @@ -855,8 +855,8 @@ private LogDirMoveState asJava(ReassignPartitionsCommand.LogDirMoveState state) private PartitionReassignmentState asJava(ReassignPartitionsCommand.PartitionReassignmentState state) { return new PartitionReassignmentState( - asJava(state.currentReplicas(), o -> (Integer) o), - asJava(state.targetReplicas(), o -> (Integer) o), + CollectionConverters.asJava(state.currentReplicas()).stream().map(i -> (Integer) i).collect(Collectors.toList()), + CollectionConverters.asJava(state.targetReplicas()).stream().map(i -> (Integer) i).collect(Collectors.toList()), state.done()); } @@ -891,12 +891,6 @@ private Set asJava(scala.collection.mutable.Set set) { return res; } - private List asJava(Seq seq, Function mapper) { - List res = new ArrayList<>(); - seq.foreach(e -> res.add(mapper.apply(e))); - return res; - } - private Tuple2 map(Tuple2 t, Function mapper1, Function mapper2) { return new Tuple2<>(mapper1.apply(t._1), mapper2.apply(t._2)); } From 1bbd0c344dac6194a95449b946b87bb9cf8428c8 Mon Sep 17 00:00:00 2001 From: nizhikov Date: Fri, 8 Sep 2023 11:35:38 +0300 Subject: [PATCH 38/78] KAFKA-14595 Improving test --- .../reassign/ReassignPartitionsUnitTest.java | 46 +++++-------------- 1 file changed, 12 insertions(+), 34 deletions(-) diff --git a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java index 7795860a45ebe..76a6d5cabebf0 100644 --- a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java +++ b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java @@ -235,13 +235,13 @@ public void testFindLogDirMoveStates() throws Exception { adminClient.alterReplicaLogDirs(replicaAssignment).all().get(); - Map expLogDirMoveStates = new HashMap<>(); + Map expLogDirMoveStates = new HashMap<>(); - expLogDirMoveStates.put(new TopicPartitionReplica("bar", 0, 0), new CompletedMoveState("/tmp/kafka-logs0")); - expLogDirMoveStates.put(new TopicPartitionReplica("foo", 0, 0), new ActiveMoveState("/tmp/kafka-logs0", "/tmp/kafka-logs1", "/tmp/kafka-logs1")); - expLogDirMoveStates.put(new TopicPartitionReplica("foo", 1, 0), new CancelledMoveState("/tmp/kafka-logs0", "/tmp/kafka-logs1")); - expLogDirMoveStates.put(new TopicPartitionReplica("quux", 1, 0), new MissingLogDirMoveState("/tmp/kafka-logs1")); - expLogDirMoveStates.put(new TopicPartitionReplica("quuz", 0, 0), new MissingReplicaMoveState("/tmp/kafka-logs0")); + expLogDirMoveStates.put(new TopicPartitionReplica("bar", 0, 0), asScala(new CompletedMoveState("/tmp/kafka-logs0"))); + expLogDirMoveStates.put(new TopicPartitionReplica("foo", 0, 0), asScala(new ActiveMoveState("/tmp/kafka-logs0", "/tmp/kafka-logs1", "/tmp/kafka-logs1"))); + expLogDirMoveStates.put(new TopicPartitionReplica("foo", 1, 0), asScala(new CancelledMoveState("/tmp/kafka-logs0", "/tmp/kafka-logs1"))); + expLogDirMoveStates.put(new TopicPartitionReplica("quux", 1, 0), asScala(new MissingLogDirMoveState("/tmp/kafka-logs1"))); + expLogDirMoveStates.put(new TopicPartitionReplica("quuz", 0, 0), asScala(new MissingReplicaMoveState("/tmp/kafka-logs0"))); Map targetMoves = new HashMap<>(); @@ -252,8 +252,8 @@ public void testFindLogDirMoveStates() throws Exception { targetMoves.put(new TopicPartitionReplica("quuz", 0, 0), "/tmp/kafka-logs0"); assertEquals( - expLogDirMoveStates, - javaMap(findLogDirMoveStates(adminClient, CollectionConverters.asScala(targetMoves)), this::asJava) + CollectionConverters.asScala(expLogDirMoveStates), + findLogDirMoveStates(adminClient, CollectionConverters.asScala(targetMoves)) ); } } @@ -767,10 +767,10 @@ public void testPropagateInvalidJsonError() { } private scala.collection.immutable.Set toSet(Collection set) { - return toMutableSet(set).toSet(); + return toMutableSet(new HashSet<>(set)).toSet(); } - private scala.collection.mutable.Set toMutableSet(Collection set) { + private scala.collection.mutable.Set toMutableSet(Set set) { return CollectionConverters.asScala(new HashSet<>(set)); } @@ -788,8 +788,8 @@ private scala.collection.mutable.Map res = (scala.collection.mutable.Map) Map$.MODULE$.mapFactory().newBuilder().result(); - moveMap.forEach((k, v) -> - res.addOne(new Tuple2<>(k, new ReassignPartitionsCommand.PartitionMove(toMutableSet(v.sources), toMutableSet(v.destinations))))); + moveMap.forEach((k, v) -> res.addOne(new Tuple2<>(k, + new ReassignPartitionsCommand.PartitionMove(toMutableSet(v.sources), toMutableSet(v.destinations))))); return res; } @@ -832,27 +832,6 @@ private ReassignPartitionsCommand.PartitionReassignmentState asScala(PartitionRe state.done); } - private LogDirMoveState asJava(ReassignPartitionsCommand.LogDirMoveState state) { - if (state instanceof ReassignPartitionsCommand.ActiveMoveState) { - ReassignPartitionsCommand.ActiveMoveState s = (ReassignPartitionsCommand.ActiveMoveState) state; - return new ActiveMoveState(s.currentLogDir(), s.targetLogDir(), s.futureLogDir()); - } else if (state instanceof ReassignPartitionsCommand.CancelledMoveState) { - ReassignPartitionsCommand.CancelledMoveState s = (ReassignPartitionsCommand.CancelledMoveState) state; - return new CancelledMoveState(s.currentLogDir(), s.targetLogDir()); - } else if (state instanceof ReassignPartitionsCommand.CompletedMoveState) { - ReassignPartitionsCommand.CompletedMoveState s = (ReassignPartitionsCommand.CompletedMoveState) state; - return new CompletedMoveState(s.targetLogDir()); - } else if (state instanceof ReassignPartitionsCommand.MissingLogDirMoveState) { - ReassignPartitionsCommand.MissingLogDirMoveState s = (ReassignPartitionsCommand.MissingLogDirMoveState) state; - return new MissingLogDirMoveState(s.targetLogDir()); - } else if (state instanceof ReassignPartitionsCommand.MissingReplicaMoveState) { - ReassignPartitionsCommand.MissingReplicaMoveState s = (ReassignPartitionsCommand.MissingReplicaMoveState) state; - return new MissingReplicaMoveState(s.targetLogDir()); - } - - throw new IllegalArgumentException("Unknown state " + state); - } - private PartitionReassignmentState asJava(ReassignPartitionsCommand.PartitionReassignmentState state) { return new PartitionReassignmentState( CollectionConverters.asJava(state.currentReplicas()).stream().map(i -> (Integer) i).collect(Collectors.toList()), @@ -870,7 +849,6 @@ private Map javaMap(scala.collection.Map map, Function asJava(scala.collection.mutable.Map map) { Map res = new HashMap<>(); map.foreach(t -> res.put((Integer) t._1, new PartitionMove(CollectionConverters.asJava(t._2.sources().toSet()), asJava(t._2.destinations())))); From b3c20a86bee4315865d0bd36c71c9deb9a29c618 Mon Sep 17 00:00:00 2001 From: nizhikov Date: Fri, 8 Sep 2023 13:02:31 +0300 Subject: [PATCH 39/78] KAFKA-14595 Improving test --- .../reassign/ReassignPartitionsUnitTest.java | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java index 76a6d5cabebf0..9383927c32e47 100644 --- a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java +++ b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java @@ -188,7 +188,7 @@ public void testFindPartitionReassignmentStates() { assertEquals(expected._2, actual._2); // Cancel the reassignment and test findPartitionReassignmentStates again. - Map> cancelResult = CollectionConverters.asJava(cancelPartitionReassignments(adminClient, toSet(new HashSet<>(Arrays.asList(new TopicPartition("foo", 0), new TopicPartition("quux", 2)))))) + Map> cancelResult = CollectionConverters.asJava(cancelPartitionReassignments(adminClient, toSet(new TopicPartition("foo", 0), new TopicPartition("quux", 2)))) .entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().getClass())); assertEquals(Collections.singletonMap(new TopicPartition("quux", 2), UnknownTopicOrPartitionException.class), cancelResult); @@ -302,8 +302,7 @@ public void testGetReplicaAssignments() { assignments.put(new TopicPartition("bar", 0), Arrays.asList(2, 3, 0)); assertEquals(assignments, - javaMap(getReplicaAssignmentForPartitions(adminClient, toSet(new HashSet<>(Arrays.asList( - new TopicPartition("foo", 0), new TopicPartition("bar", 0))))), CollectionConverters::asJava)); + javaMap(getReplicaAssignmentForPartitions(adminClient, toSet(new TopicPartition("foo", 0), new TopicPartition("bar", 0))), CollectionConverters::asJava)); } } @@ -539,9 +538,9 @@ public void testMoveMap() { assertEquals(new HashSet<>(Arrays.asList(1, 2, 3, 4, 5, 6, 7, 8)), CollectionConverters.asJava(calculateReassigningBrokers(mutableMap(asScala(moveMap, this::toMoveMap))))); - assertEquals(new HashSet<>(Arrays.asList(0, 2)), CollectionConverters.asJava(calculateMovingBrokers(toSet(new HashSet<>(Arrays.asList( + assertEquals(new HashSet<>(Arrays.asList(0, 2)), CollectionConverters.asJava(calculateMovingBrokers(toSet( new TopicPartitionReplica("quux", 0, 0), - new TopicPartitionReplica("quux", 1, 2))))))); + new TopicPartitionReplica("quux", 1, 2))))); } @Test @@ -627,8 +626,8 @@ public void testExecuteWithInvalidBrokerIdFails() { @Test public void testModifyBrokerInterBrokerThrottle() throws Exception { try (MockAdminClient adminClient = new MockAdminClient.Builder().numBrokers(4).build()) { - modifyInterBrokerThrottle(adminClient, toSet(Arrays.asList(0, 1, 2)), 1000); - modifyInterBrokerThrottle(adminClient, toSet(Arrays.asList(0, 3)), 100); + modifyInterBrokerThrottle(adminClient, toSet(0, 1, 2), 1000); + modifyInterBrokerThrottle(adminClient, toSet(0, 3), 100); List brokers = new ArrayList<>(); for (int i = 0; i < 4; i++) brokers.add(new ConfigResource(ConfigResource.Type.BROKER, Integer.toString(i))); @@ -643,8 +642,8 @@ public void testModifyBrokerInterBrokerThrottle() throws Exception { @Test public void testModifyLogDirThrottle() throws Exception { try (MockAdminClient adminClient = new MockAdminClient.Builder().numBrokers(4).build()) { - modifyLogDirThrottle(adminClient, toSet(Arrays.asList(0, 1, 2)), 2000); - modifyLogDirThrottle(adminClient, toSet(Arrays.asList(0, 3)), -1); + modifyLogDirThrottle(adminClient, toSet(0, 1, 2), 2000); + modifyLogDirThrottle(adminClient, toSet(0, 3), -1); List brokers = new ArrayList<>(); for (int i = 0; i < 4; i++) @@ -766,8 +765,9 @@ public void testPropagateInvalidJsonError() { } } - private scala.collection.immutable.Set toSet(Collection set) { - return toMutableSet(new HashSet<>(set)).toSet(); + @SuppressWarnings("unchecked") + private scala.collection.immutable.Set toSet(final T... set) { + return (scala.collection.immutable.Set) toMutableSet(new HashSet<>(Arrays.asList(set))).toSet(); } private scala.collection.mutable.Set toMutableSet(Set set) { From 07e1204866122abb1d52d5fc8d274d546bff616b Mon Sep 17 00:00:00 2001 From: nizhikov Date: Fri, 8 Sep 2023 13:10:03 +0300 Subject: [PATCH 40/78] KAFKA-14595 Improving test --- .../reassign/ReassignPartitionsUnitTest.java | 22 ++++--------------- 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java index 9383927c32e47..e84470de96c09 100644 --- a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java +++ b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java @@ -184,7 +184,7 @@ public void testFindPartitionReassignmentStates() { new Tuple2<>(new TopicPartition("foo", 1), Arrays.asList(1, 2, 3)) ), this::asScala)); - assertEquals(expected._1, javaMap(actual._1, this::asJava)); + assertEquals(asScala(expected._1, this::asScala), actual._1); assertEquals(expected._2, actual._2); // Cancel the reassignment and test findPartitionReassignmentStates again. @@ -204,7 +204,7 @@ public void testFindPartitionReassignmentStates() { new Tuple2<>(new TopicPartition("foo", 1), Arrays.asList(1, 2, 3)) ), this::asScala)); - assertEquals(expected._1, javaMap(actual._1, this::asJava)); + assertEquals(asScala(expected._1, this::asScala), actual._1); assertEquals(expected._2, actual._2); } } @@ -767,7 +767,7 @@ public void testPropagateInvalidJsonError() { @SuppressWarnings("unchecked") private scala.collection.immutable.Set toSet(final T... set) { - return (scala.collection.immutable.Set) toMutableSet(new HashSet<>(Arrays.asList(set))).toSet(); + return toMutableSet(new HashSet<>(Arrays.asList(set))).toSet(); } private scala.collection.mutable.Set toMutableSet(Set set) { @@ -832,13 +832,6 @@ private ReassignPartitionsCommand.PartitionReassignmentState asScala(PartitionRe state.done); } - private PartitionReassignmentState asJava(ReassignPartitionsCommand.PartitionReassignmentState state) { - return new PartitionReassignmentState( - CollectionConverters.asJava(state.currentReplicas()).stream().map(i -> (Integer) i).collect(Collectors.toList()), - CollectionConverters.asJava(state.targetReplicas()).stream().map(i -> (Integer) i).collect(Collectors.toList()), - state.done()); - } - private Tuple2> asScala(Tuple2> t) { return map(t, Function.identity(), this::toRawSeq); } @@ -851,7 +844,7 @@ private Map javaMap(scala.collection.Map map, Function asJava(scala.collection.mutable.Map map) { Map res = new HashMap<>(); - map.foreach(t -> res.put((Integer) t._1, new PartitionMove(CollectionConverters.asJava(t._2.sources().toSet()), asJava(t._2.destinations())))); + map.foreach(t -> res.put((Integer) t._1, new PartitionMove(CollectionConverters.asJava(t._2.sources().toSet()), CollectionConverters.asJava(t._2.destinations().toSet())))); return res; } @@ -862,13 +855,6 @@ private Map> asJava(scala.collection.Map Set asJava(scala.collection.mutable.Set set) { - Set res = new HashSet<>(); - set.foreach(e -> res.add((T) e)); - return res; - } - private Tuple2 map(Tuple2 t, Function mapper1, Function mapper2) { return new Tuple2<>(mapper1.apply(t._1), mapper2.apply(t._2)); } From 290133d2e22c59c79f79782894cb60e9769fb17d Mon Sep 17 00:00:00 2001 From: nizhikov Date: Fri, 8 Sep 2023 14:27:48 +0300 Subject: [PATCH 41/78] KAFKA-14595 Improving test --- .../reassign/ReassignPartitionsUnitTest.java | 97 ++++++++----------- 1 file changed, 41 insertions(+), 56 deletions(-) diff --git a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java index e84470de96c09..049777e494031 100644 --- a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java +++ b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java @@ -179,10 +179,10 @@ public void testFindPartitionReassignmentStates() { Tuple2, Boolean> expected = new Tuple2<>(partitionReassignmentStates, true); Tuple2, Object> actual = - findPartitionReassignmentStates(adminClient, toSeq(Arrays.asList( - new Tuple2<>(new TopicPartition("foo", 0), Arrays.asList(0, 1, 3)), - new Tuple2<>(new TopicPartition("foo", 1), Arrays.asList(1, 2, 3)) - ), this::asScala)); + findPartitionReassignmentStates(adminClient, CollectionConverters.asScala(Arrays.asList( + new Tuple2<>(new TopicPartition("foo", 0), toRawSeq(Arrays.asList(0, 1, 3))), + new Tuple2<>(new TopicPartition("foo", 1), toRawSeq(Arrays.asList(1, 2, 3))) + ))); assertEquals(asScala(expected._1, this::asScala), actual._1); assertEquals(expected._2, actual._2); @@ -199,10 +199,10 @@ public void testFindPartitionReassignmentStates() { partitionReassignmentStates.put(new TopicPartition("foo", 1), new PartitionReassignmentState(Arrays.asList(1, 2, 3), Arrays.asList(1, 2, 3), true)); expected = new Tuple2<>(partitionReassignmentStates, false); - actual = findPartitionReassignmentStates(adminClient, toSeq(Arrays.asList( - new Tuple2<>(new TopicPartition("foo", 0), Arrays.asList(0, 1, 3)), - new Tuple2<>(new TopicPartition("foo", 1), Arrays.asList(1, 2, 3)) - ), this::asScala)); + actual = findPartitionReassignmentStates(adminClient, CollectionConverters.asScala(Arrays.asList( + new Tuple2<>(new TopicPartition("foo", 0), toRawSeq(Arrays.asList(0, 1, 3))), + new Tuple2<>(new TopicPartition("foo", 1), toRawSeq(Arrays.asList(1, 2, 3))) + ))); assertEquals(asScala(expected._1, this::asScala), actual._1); assertEquals(expected._2, actual._2); @@ -293,16 +293,16 @@ public void testGetReplicaAssignments() { assignments.put(new TopicPartition("foo", 0), Arrays.asList(0, 1, 2)); assignments.put(new TopicPartition("foo", 1), Arrays.asList(1, 2, 3)); - assertEquals(assignments, - asJava(getReplicaAssignmentForTopics(adminClient, CollectionConverters.asScala(Collections.singletonList("foo"))))); + assertEquals(asScala(assignments, this::toRawSeq), + getReplicaAssignmentForTopics(adminClient, CollectionConverters.asScala(Collections.singletonList("foo")))); assignments.clear(); assignments.put(new TopicPartition("foo", 0), Arrays.asList(0, 1, 2)); assignments.put(new TopicPartition("bar", 0), Arrays.asList(2, 3, 0)); - assertEquals(assignments, - javaMap(getReplicaAssignmentForPartitions(adminClient, toSet(new TopicPartition("foo", 0), new TopicPartition("bar", 0))), CollectionConverters::asJava)); + assertEquals(asScala(assignments, this::toRawSeq), + getReplicaAssignmentForPartitions(adminClient, toSet(new TopicPartition("foo", 0), new TopicPartition("bar", 0)))); } } @@ -342,17 +342,15 @@ public void testParseGenerateAssignmentArgs() { assertThrows(AdminCommandFailedException.class, () -> parseGenerateAssignmentArgs( "{\"topics\": [{\"topic\": \"foo\"}], \"version\":1}", "5,2,3,4,5"), "Expected to detect duplicate broker list entries").getMessage()); - assertEquals(new Tuple2<>(Arrays.asList(5, 2, 3, 4), Collections.singletonList("foo")), - map(parseGenerateAssignmentArgs("{\"topics\": [{\"topic\": \"foo\"}], \"version\":1}", - "5,2,3,4"), CollectionConverters::asJava, CollectionConverters::asJava)); + assertEquals(new Tuple2<>(toRawSeq(Arrays.asList(5, 2, 3, 4)), toRawSeq(Collections.singletonList("foo"))), + parseGenerateAssignmentArgs("{\"topics\": [{\"topic\": \"foo\"}], \"version\":1}", "5,2,3,4")); assertStartsWith("List of topics to reassign contains duplicate entries", assertThrows(AdminCommandFailedException.class, () -> parseGenerateAssignmentArgs( "{\"topics\": [{\"topic\": \"foo\"},{\"topic\": \"foo\"}], \"version\":1}", "5,2,3,4"), "Expected to detect duplicate topic entries").getMessage()); - assertEquals(new Tuple2<>(Arrays.asList(5, 3, 4), Arrays.asList("foo", "bar")), - map(parseGenerateAssignmentArgs( - "{\"topics\": [{\"topic\": \"foo\"},{\"topic\": \"bar\"}], \"version\":1}", - "5,3,4"), CollectionConverters::asJava, CollectionConverters::asJava)); + assertEquals(new Tuple2<>(toRawSeq(Arrays.asList(5, 3, 4)), toRawSeq(Arrays.asList("foo", "bar"))), + parseGenerateAssignmentArgs( + "{\"topics\": [{\"topic\": \"foo\"},{\"topic\": \"bar\"}], \"version\":1}", "5,3,4")); } @Test @@ -395,15 +393,15 @@ public void testGenerateAssignmentWithInconsistentRacks() { () -> generateAssignment(adminClient, "{\"topics\":[{\"topic\":\"foo\"}]}", "0,1,2,3", true), "Expected generateAssignment to fail").getMessage()); // It should succeed when --disable-rack-aware is used. - Tuple2>, Map>> proposedCurrent = map(generateAssignment(adminClient, - "{\"topics\":[{\"topic\":\"foo\"}]}", "0,1,2,3", false), this::asJava, this::asJava); + Tuple2>, scala.collection.Map>> + proposedCurrent = generateAssignment(adminClient, "{\"topics\":[{\"topic\":\"foo\"}]}", "0,1,2,3", false); Map> expCurrent = new HashMap<>(); expCurrent.put(new TopicPartition("foo", 0), Arrays.asList(0, 1, 2)); expCurrent.put(new TopicPartition("foo", 1), Arrays.asList(1, 2, 3)); - assertEquals(expCurrent, proposedCurrent._2()); + assertEquals(asScala(expCurrent, this::toRawSeq), proposedCurrent._2()); } } @@ -413,10 +411,10 @@ public void testGenerateAssignmentWithFewerBrokers() { addTopics(adminClient); List goalBrokers = Arrays.asList(0, 1, 3); - Tuple2>, Map>> proposedCurrent = - map(generateAssignment(adminClient, + Tuple2>, scala.collection.Map>> + proposedCurrent = generateAssignment(adminClient, "{\"topics\":[{\"topic\":\"foo\"},{\"topic\":\"bar\"}]}", - goalBrokers.stream().map(Object::toString).collect(Collectors.joining(",")), false), this::asJava, this::asJava); + goalBrokers.stream().map(Object::toString).collect(Collectors.joining(",")), false); Map> expCurrent = new HashMap<>(); @@ -424,11 +422,14 @@ public void testGenerateAssignmentWithFewerBrokers() { expCurrent.put(new TopicPartition("foo", 1), Arrays.asList(1, 2, 3)); expCurrent.put(new TopicPartition("bar", 0), Arrays.asList(2, 3, 0)); - assertEquals(expCurrent, proposedCurrent._2()); + assertEquals(asScala(expCurrent, this::toRawSeq), proposedCurrent._2()); // The proposed assignment should only span the provided brokers - proposedCurrent._1().values().forEach(replicas -> assertTrue(goalBrokers.containsAll(replicas), - "Proposed assignment " + proposedCurrent._1() + " puts replicas on brokers other than " + goalBrokers)); + proposedCurrent._1().values().foreach(replicas -> { + assertTrue(replicas.forall(replica -> goalBrokers.contains((Integer) replica)), + "Proposed assignment " + proposedCurrent._1() + " puts replicas on brokers other than " + goalBrokers); + return null; + }); } } @@ -510,15 +511,15 @@ public void testMoveMap() { Map fooMoves = new HashMap<>(); - fooMoves.put(0, new PartitionMove(new HashSet<>(Arrays.asList(1, 2, 3)), new HashSet<>(Collections.singletonList(5)))); + fooMoves.put(0, new PartitionMove(new HashSet<>(Arrays.asList(1, 2, 3)), Collections.singleton(5))); fooMoves.put(1, new PartitionMove(new HashSet<>(Arrays.asList(4, 5, 6)), new HashSet<>(Arrays.asList(7, 8)))); fooMoves.put(2, new PartitionMove(new HashSet<>(Arrays.asList(1, 2)), new HashSet<>(Arrays.asList(3, 4)))); fooMoves.put(3, new PartitionMove(new HashSet<>(Arrays.asList(1, 2)), new HashSet<>(Arrays.asList(5, 6)))); - fooMoves.put(4, new PartitionMove(new HashSet<>(Arrays.asList(1, 2)), new HashSet<>(Collections.singletonList(3)))); + fooMoves.put(4, new PartitionMove(new HashSet<>(Arrays.asList(1, 2)), Collections.singleton(3))); fooMoves.put(5, new PartitionMove(new HashSet<>(Arrays.asList(1, 2)), new HashSet<>(Arrays.asList(3, 4, 5, 6)))); expMoveMap.put("foo", fooMoves); - expMoveMap.put("bar", Collections.singletonMap(0, new PartitionMove(new HashSet<>(Arrays.asList(2, 3, 4)), new HashSet<>(Collections.singletonList(1))))); + expMoveMap.put("bar", Collections.singletonMap(0, new PartitionMove(new HashSet<>(Arrays.asList(2, 3, 4)), Collections.singleton(1)))); assertEquals(expMoveMap, moveMap); @@ -536,11 +537,10 @@ public void testMoveMap() { assertEquals(expFollowerThrottle, CollectionConverters.asJava(calculateFollowerThrottles(mutableMap(asScala(moveMap, this::toMoveMap))))); - assertEquals(new HashSet<>(Arrays.asList(1, 2, 3, 4, 5, 6, 7, 8)), CollectionConverters.asJava(calculateReassigningBrokers(mutableMap(asScala(moveMap, this::toMoveMap))))); - - assertEquals(new HashSet<>(Arrays.asList(0, 2)), CollectionConverters.asJava(calculateMovingBrokers(toSet( + assertEquals(CollectionConverters.asScala(new HashSet<>(Arrays.asList(1, 2, 3, 4, 5, 6, 7, 8))), calculateReassigningBrokers(mutableMap(asScala(moveMap, this::toMoveMap)))); + assertEquals(CollectionConverters.asScala(new HashSet<>(Arrays.asList(0, 2))), calculateMovingBrokers(toSet( new TopicPartitionReplica("quux", 0, 0), - new TopicPartitionReplica("quux", 1, 2))))); + new TopicPartitionReplica("quux", 1, 2)))); } @Test @@ -745,10 +745,10 @@ public void testAlterReplicaLogDirs() { assignment.put(new TopicPartitionReplica("foo", 0, 0), "/tmp/kafka-logs1"); assignment.put(new TopicPartitionReplica("quux", 1, 0), "/tmp/kafka-logs1"); - assertEquals(new HashSet<>(Collections.singletonList( - new TopicPartitionReplica("foo", 0, 0) - )), - CollectionConverters.asJava(alterReplicaLogDirs(adminClient, CollectionConverters.asScala(assignment)))); + assertEquals( + CollectionConverters.asScala(Collections.singleton(new TopicPartitionReplica("foo", 0, 0))), + alterReplicaLogDirs(adminClient, CollectionConverters.asScala(assignment)) + ); } } @@ -776,11 +776,7 @@ private scala.collection.mutable.Set toMutableSet(Set set) { @SuppressWarnings("unchecked") private Seq toRawSeq(Collection col) { - return (Seq) toSeq(col, Function.identity()); - } - - private Seq toSeq(Collection col, Function mapper) { - return CollectionConverters.asScala(col.stream().map(mapper).collect(Collectors.toList())).toSeq(); + return (Seq) CollectionConverters.asScala(col).toSeq(); } @SuppressWarnings("unchecked") @@ -833,7 +829,7 @@ private ReassignPartitionsCommand.PartitionReassignmentState asScala(PartitionRe } private Tuple2> asScala(Tuple2> t) { - return map(t, Function.identity(), this::toRawSeq); + return new Tuple2<>(t._1, toRawSeq(t._2)); } private Map javaMap(scala.collection.Map map, Function mapper) { @@ -847,15 +843,4 @@ private Map asJava(scala.collection.mutable.Map res.put((Integer) t._1, new PartitionMove(CollectionConverters.asJava(t._2.sources().toSet()), CollectionConverters.asJava(t._2.destinations().toSet())))); return res; } - - @SuppressWarnings("unchecked") - private Map> asJava(scala.collection.Map> map) { - Map> res = new HashMap<>(); - map.foreach(e -> res.put(e._1, (List) CollectionConverters.asJava(e._2))); - return res; - } - - private Tuple2 map(Tuple2 t, Function mapper1, Function mapper2) { - return new Tuple2<>(mapper1.apply(t._1), mapper2.apply(t._2)); - } } From 97319a36a8448590ca90637ec31dd18172ecf114 Mon Sep 17 00:00:00 2001 From: nizhikov Date: Fri, 8 Sep 2023 15:17:19 +0300 Subject: [PATCH 42/78] KAFKA-14595 Improving test --- .../reassign/ReassignPartitionsUnitTest.java | 291 +++++++++--------- 1 file changed, 149 insertions(+), 142 deletions(-) diff --git a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java index 049777e494031..22654eec6bd2d 100644 --- a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java +++ b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java @@ -42,8 +42,6 @@ import scala.jdk.javaapi.CollectionConverters; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -56,6 +54,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import static java.util.Arrays.asList; import static kafka.admin.ReassignPartitionsCommand.alterPartitionReassignments; import static kafka.admin.ReassignPartitionsCommand.alterReplicaLogDirs; import static kafka.admin.ReassignPartitionsCommand.brokerLevelFollowerThrottle; @@ -126,13 +125,13 @@ public void testPartitionReassignStatesToString() { Map states = new HashMap<>(); states.put(new TopicPartition("foo", 0), - asScala(new PartitionReassignmentState(Arrays.asList(1, 2, 3), Arrays.asList(1, 2, 3), true))); + asScala(new PartitionReassignmentState(asList(1, 2, 3), asList(1, 2, 3), true))); states.put(new TopicPartition("foo", 1), - asScala(new PartitionReassignmentState(Arrays.asList(1, 2, 3), Arrays.asList(1, 2, 4), false))); + asScala(new PartitionReassignmentState(asList(1, 2, 3), asList(1, 2, 4), false))); states.put(new TopicPartition("bar", 0), - asScala(new PartitionReassignmentState(Arrays.asList(1, 2, 3), Arrays.asList(1, 2, 4), false))); + asScala(new PartitionReassignmentState(asList(1, 2, 3), asList(1, 2, 4), false))); - assertEquals(String.join(System.lineSeparator(), Arrays.asList( + assertEquals(String.join(System.lineSeparator(), asList( "Status of partition reassignment:", "Reassignment of partition bar-0 is still in progress.", "Reassignment of partition foo-0 is completed.", @@ -142,18 +141,18 @@ public void testPartitionReassignStatesToString() { private void addTopics(MockAdminClient adminClient) { List b = adminClient.brokers(); - adminClient.addTopic(false, "foo", Arrays.asList( + adminClient.addTopic(false, "foo", asList( new TopicPartitionInfo(0, b.get(0), - Arrays.asList(b.get(0), b.get(1), b.get(2)), - Arrays.asList(b.get(0), b.get(1))), + asList(b.get(0), b.get(1), b.get(2)), + asList(b.get(0), b.get(1))), new TopicPartitionInfo(1, b.get(1), - Arrays.asList(b.get(1), b.get(2), b.get(3)), - Arrays.asList(b.get(1), b.get(2), b.get(3))) + asList(b.get(1), b.get(2), b.get(3)), + asList(b.get(1), b.get(2), b.get(3))) ), Collections.emptyMap()); adminClient.addTopic(false, "bar", Collections.singletonList( new TopicPartitionInfo(0, b.get(2), - Arrays.asList(b.get(2), b.get(3), b.get(0)), - Arrays.asList(b.get(2), b.get(3), b.get(0))) + asList(b.get(2), b.get(3), b.get(0)), + asList(b.get(2), b.get(3), b.get(0))) ), Collections.emptyMap()); } @@ -162,46 +161,46 @@ public void testFindPartitionReassignmentStates() { try (MockAdminClient adminClient = new MockAdminClient.Builder().numBrokers(4).build()) { addTopics(adminClient); // Create a reassignment and test findPartitionReassignmentStates. - Map> reassignments = new HashMap<>(); + Map> reassignments = new HashMap<>(); - reassignments.put(new TopicPartition("foo", 0), Arrays.asList(0, 1, 3)); - reassignments.put(new TopicPartition("quux", 0), Arrays.asList(1, 2, 3)); + reassignments.put(new TopicPartition("foo", 0), seq(0, 1, 3)); + reassignments.put(new TopicPartition("quux", 0), seq(1, 2, 3)); - Map> reassignmentResult = CollectionConverters.asJava(alterPartitionReassignments(adminClient, asScala(reassignments, this::toRawSeq))) + Map> reassignmentResult = CollectionConverters.asJava(alterPartitionReassignments(adminClient, CollectionConverters.asScala(reassignments))) .entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().getClass())); assertEquals(Collections.singletonMap(new TopicPartition("quux", 0), UnknownTopicOrPartitionException.class), reassignmentResult); Map partitionReassignmentStates = new HashMap<>(); - partitionReassignmentStates.put(new TopicPartition("foo", 0), new PartitionReassignmentState(Arrays.asList(0, 1, 2), Arrays.asList(0, 1, 3), false)); - partitionReassignmentStates.put(new TopicPartition("foo", 1), new PartitionReassignmentState(Arrays.asList(1, 2, 3), Arrays.asList(1, 2, 3), true)); + partitionReassignmentStates.put(new TopicPartition("foo", 0), new PartitionReassignmentState(asList(0, 1, 2), asList(0, 1, 3), false)); + partitionReassignmentStates.put(new TopicPartition("foo", 1), new PartitionReassignmentState(asList(1, 2, 3), asList(1, 2, 3), true)); Tuple2, Boolean> expected = new Tuple2<>(partitionReassignmentStates, true); Tuple2, Object> actual = - findPartitionReassignmentStates(adminClient, CollectionConverters.asScala(Arrays.asList( - new Tuple2<>(new TopicPartition("foo", 0), toRawSeq(Arrays.asList(0, 1, 3))), - new Tuple2<>(new TopicPartition("foo", 1), toRawSeq(Arrays.asList(1, 2, 3))) + findPartitionReassignmentStates(adminClient, CollectionConverters.asScala(asList( + new Tuple2<>(new TopicPartition("foo", 0), seq(0, 1, 3)), + new Tuple2<>(new TopicPartition("foo", 1), seq(1, 2, 3)) ))); assertEquals(asScala(expected._1, this::asScala), actual._1); assertEquals(expected._2, actual._2); // Cancel the reassignment and test findPartitionReassignmentStates again. - Map> cancelResult = CollectionConverters.asJava(cancelPartitionReassignments(adminClient, toSet(new TopicPartition("foo", 0), new TopicPartition("quux", 2)))) + Map> cancelResult = CollectionConverters.asJava(cancelPartitionReassignments(adminClient, set(new TopicPartition("foo", 0), new TopicPartition("quux", 2)))) .entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().getClass())); assertEquals(Collections.singletonMap(new TopicPartition("quux", 2), UnknownTopicOrPartitionException.class), cancelResult); partitionReassignmentStates.clear(); - partitionReassignmentStates.put(new TopicPartition("foo", 0), new PartitionReassignmentState(Arrays.asList(0, 1, 2), Arrays.asList(0, 1, 3), true)); - partitionReassignmentStates.put(new TopicPartition("foo", 1), new PartitionReassignmentState(Arrays.asList(1, 2, 3), Arrays.asList(1, 2, 3), true)); + partitionReassignmentStates.put(new TopicPartition("foo", 0), new PartitionReassignmentState(asList(0, 1, 2), asList(0, 1, 3), true)); + partitionReassignmentStates.put(new TopicPartition("foo", 1), new PartitionReassignmentState(asList(1, 2, 3), asList(1, 2, 3), true)); expected = new Tuple2<>(partitionReassignmentStates, false); - actual = findPartitionReassignmentStates(adminClient, CollectionConverters.asScala(Arrays.asList( - new Tuple2<>(new TopicPartition("foo", 0), toRawSeq(Arrays.asList(0, 1, 3))), - new Tuple2<>(new TopicPartition("foo", 1), toRawSeq(Arrays.asList(1, 2, 3))) + actual = findPartitionReassignmentStates(adminClient, CollectionConverters.asScala(asList( + new Tuple2<>(new TopicPartition("foo", 0), seq(0, 1, 3)), + new Tuple2<>(new TopicPartition("foo", 1), seq(1, 2, 3)) ))); assertEquals(asScala(expected._1, this::asScala), actual._1); @@ -213,19 +212,19 @@ public void testFindPartitionReassignmentStates() { public void testFindLogDirMoveStates() throws Exception { try (MockAdminClient adminClient = new MockAdminClient.Builder(). numBrokers(4). - brokerLogDirs(Arrays.asList( - Arrays.asList("/tmp/kafka-logs0", "/tmp/kafka-logs1"), - Arrays.asList("/tmp/kafka-logs0", "/tmp/kafka-logs1"), - Arrays.asList("/tmp/kafka-logs0", "/tmp/kafka-logs1"), - Arrays.asList("/tmp/kafka-logs0", null))). + brokerLogDirs(asList( + asList("/tmp/kafka-logs0", "/tmp/kafka-logs1"), + asList("/tmp/kafka-logs0", "/tmp/kafka-logs1"), + asList("/tmp/kafka-logs0", "/tmp/kafka-logs1"), + asList("/tmp/kafka-logs0", null))). build()) { addTopics(adminClient); List b = adminClient.brokers(); adminClient.addTopic(false, "quux", Collections.singletonList( new TopicPartitionInfo(0, b.get(2), - Arrays.asList(b.get(1), b.get(2), b.get(3)), - Arrays.asList(b.get(1), b.get(2), b.get(3)))), + asList(b.get(1), b.get(2), b.get(3)), + asList(b.get(1), b.get(2), b.get(3)))), Collections.emptyMap()); Map replicaAssignment = new HashMap<>(); @@ -272,7 +271,7 @@ public void testReplicaMoveStatesToString() { "/tmp/kafka-logs1", "/tmp/kafka-logs2"))); states.put(new TopicPartitionReplica("quux", 2, 1), asScala(new MissingLogDirMoveState("/tmp/kafka-logs1"))); - assertEquals(String.join(System.lineSeparator(), Arrays.asList( + assertEquals(String.join(System.lineSeparator(), asList( "Reassignment of replica bar-0-0 completed successfully.", "Reassignment of replica foo-0-0 is still in progress.", "Partition foo-1 on broker 0 is not being moved from log dir /tmp/kafka-logs0 to /tmp/kafka-logs1.", @@ -288,47 +287,47 @@ public void testGetReplicaAssignments() { try (MockAdminClient adminClient = new MockAdminClient.Builder().numBrokers(4).build()) { addTopics(adminClient); - Map> assignments = new HashMap<>(); + Map> assignments = new HashMap<>(); - assignments.put(new TopicPartition("foo", 0), Arrays.asList(0, 1, 2)); - assignments.put(new TopicPartition("foo", 1), Arrays.asList(1, 2, 3)); + assignments.put(new TopicPartition("foo", 0), seq(0, 1, 2)); + assignments.put(new TopicPartition("foo", 1), seq(1, 2, 3)); - assertEquals(asScala(assignments, this::toRawSeq), + assertEquals(CollectionConverters.asScala(assignments), getReplicaAssignmentForTopics(adminClient, CollectionConverters.asScala(Collections.singletonList("foo")))); assignments.clear(); - assignments.put(new TopicPartition("foo", 0), Arrays.asList(0, 1, 2)); - assignments.put(new TopicPartition("bar", 0), Arrays.asList(2, 3, 0)); + assignments.put(new TopicPartition("foo", 0), seq(0, 1, 2)); + assignments.put(new TopicPartition("bar", 0), seq(2, 3, 0)); - assertEquals(asScala(assignments, this::toRawSeq), - getReplicaAssignmentForPartitions(adminClient, toSet(new TopicPartition("foo", 0), new TopicPartition("bar", 0)))); + assertEquals(CollectionConverters.asScala(assignments), + getReplicaAssignmentForPartitions(adminClient, set(new TopicPartition("foo", 0), new TopicPartition("bar", 0)))); } } @Test public void testGetBrokerRackInformation() { try (MockAdminClient adminClient = new MockAdminClient.Builder(). - brokers(Arrays.asList(new Node(0, "localhost", 9092, "rack0"), + brokers(asList(new Node(0, "localhost", 9092, "rack0"), new Node(1, "localhost", 9093, "rack1"), new Node(2, "localhost", 9094, null))). build()) { - assertEquals(CollectionConverters.asScala(Arrays.asList( + assertEquals(CollectionConverters.asScala(asList( new BrokerMetadata(0, Optional.of("rack0")), new BrokerMetadata(1, Optional.of("rack1")) - )), getBrokerMetadata(adminClient, CollectionConverters.asScala(Arrays.asList(0, 1)), true)); - assertEquals(CollectionConverters.asScala(Arrays.asList( + )), getBrokerMetadata(adminClient, CollectionConverters.asScala(asList(0, 1)), true)); + assertEquals(CollectionConverters.asScala(asList( new BrokerMetadata(0, Optional.empty()), new BrokerMetadata(1, Optional.empty()) - )), getBrokerMetadata(adminClient, CollectionConverters.asScala(Arrays.asList(0, 1)), false)); + )), getBrokerMetadata(adminClient, CollectionConverters.asScala(asList(0, 1)), false)); assertStartsWith("Not all brokers have rack information", assertThrows(AdminOperationException.class, - () -> getBrokerMetadata(adminClient, CollectionConverters.asScala(Arrays.asList(1, 2)), true)).getMessage()); - assertEquals(CollectionConverters.asScala(Arrays.asList( + () -> getBrokerMetadata(adminClient, CollectionConverters.asScala(asList(1, 2)), true)).getMessage()); + assertEquals(CollectionConverters.asScala(asList( new BrokerMetadata(1, Optional.empty()), new BrokerMetadata(2, Optional.empty()) - )), getBrokerMetadata(adminClient, CollectionConverters.asScala(Arrays.asList(1, 2)), false)); + )), getBrokerMetadata(adminClient, CollectionConverters.asScala(asList(1, 2)), false)); } } @@ -342,13 +341,13 @@ public void testParseGenerateAssignmentArgs() { assertThrows(AdminCommandFailedException.class, () -> parseGenerateAssignmentArgs( "{\"topics\": [{\"topic\": \"foo\"}], \"version\":1}", "5,2,3,4,5"), "Expected to detect duplicate broker list entries").getMessage()); - assertEquals(new Tuple2<>(toRawSeq(Arrays.asList(5, 2, 3, 4)), toRawSeq(Collections.singletonList("foo"))), + assertEquals(new Tuple2<>(seq(5, 2, 3, 4), seq("foo")), parseGenerateAssignmentArgs("{\"topics\": [{\"topic\": \"foo\"}], \"version\":1}", "5,2,3,4")); assertStartsWith("List of topics to reassign contains duplicate entries", assertThrows(AdminCommandFailedException.class, () -> parseGenerateAssignmentArgs( "{\"topics\": [{\"topic\": \"foo\"},{\"topic\": \"foo\"}], \"version\":1}", "5,2,3,4"), "Expected to detect duplicate topic entries").getMessage()); - assertEquals(new Tuple2<>(toRawSeq(Arrays.asList(5, 3, 4)), toRawSeq(Arrays.asList("foo", "bar"))), + assertEquals(new Tuple2<>(seq(5, 3, 4), seq("foo", "bar")), parseGenerateAssignmentArgs( "{\"topics\": [{\"topic\": \"foo\"},{\"topic\": \"bar\"}], \"version\":1}", "5,3,4")); } @@ -378,7 +377,7 @@ public void testGenerateAssignmentWithInvalidPartitionsFails() { @Test public void testGenerateAssignmentWithInconsistentRacks() { try (MockAdminClient adminClient = new MockAdminClient.Builder(). - brokers(Arrays.asList( + brokers(asList( new Node(0, "localhost", 9092, "rack0"), new Node(1, "localhost", 9093, "rack0"), new Node(2, "localhost", 9094, null), @@ -396,12 +395,12 @@ public void testGenerateAssignmentWithInconsistentRacks() { Tuple2>, scala.collection.Map>> proposedCurrent = generateAssignment(adminClient, "{\"topics\":[{\"topic\":\"foo\"}]}", "0,1,2,3", false); - Map> expCurrent = new HashMap<>(); + Map> expCurrent = new HashMap<>(); - expCurrent.put(new TopicPartition("foo", 0), Arrays.asList(0, 1, 2)); - expCurrent.put(new TopicPartition("foo", 1), Arrays.asList(1, 2, 3)); + expCurrent.put(new TopicPartition("foo", 0), seq(0, 1, 2)); + expCurrent.put(new TopicPartition("foo", 1), seq(1, 2, 3)); - assertEquals(asScala(expCurrent, this::toRawSeq), proposedCurrent._2()); + assertEquals(CollectionConverters.asScala(expCurrent), proposedCurrent._2()); } } @@ -409,20 +408,20 @@ public void testGenerateAssignmentWithInconsistentRacks() { public void testGenerateAssignmentWithFewerBrokers() { try (MockAdminClient adminClient = new MockAdminClient.Builder().numBrokers(4).build()) { addTopics(adminClient); - List goalBrokers = Arrays.asList(0, 1, 3); + List goalBrokers = asList(0, 1, 3); Tuple2>, scala.collection.Map>> proposedCurrent = generateAssignment(adminClient, "{\"topics\":[{\"topic\":\"foo\"},{\"topic\":\"bar\"}]}", goalBrokers.stream().map(Object::toString).collect(Collectors.joining(",")), false); - Map> expCurrent = new HashMap<>(); + Map> expCurrent = new HashMap<>(); - expCurrent.put(new TopicPartition("foo", 0), Arrays.asList(0, 1, 2)); - expCurrent.put(new TopicPartition("foo", 1), Arrays.asList(1, 2, 3)); - expCurrent.put(new TopicPartition("bar", 0), Arrays.asList(2, 3, 0)); + expCurrent.put(new TopicPartition("foo", 0), seq(0, 1, 2)); + expCurrent.put(new TopicPartition("foo", 1), seq(1, 2, 3)); + expCurrent.put(new TopicPartition("bar", 0), seq(2, 3, 0)); - assertEquals(asScala(expCurrent, this::toRawSeq), proposedCurrent._2()); + assertEquals(CollectionConverters.asScala(expCurrent), proposedCurrent._2()); // The proposed assignment should only span the provided brokers proposedCurrent._1().values().foreach(replicas -> { @@ -435,19 +434,19 @@ public void testGenerateAssignmentWithFewerBrokers() { @Test public void testCurrentPartitionReplicaAssignmentToString() { - Map> proposedParts = new HashMap<>(); + Map> proposedParts = new HashMap<>(); - proposedParts.put(new TopicPartition("foo", 1), Arrays.asList(1, 2, 3)); - proposedParts.put(new TopicPartition("bar", 0), Arrays.asList(7, 8, 9)); + proposedParts.put(new TopicPartition("foo", 1), seq(1, 2, 3)); + proposedParts.put(new TopicPartition("bar", 0), seq(7, 8, 9)); - Map> currentParts = new HashMap<>(); + Map> currentParts = new HashMap<>(); - currentParts.put(new TopicPartition("foo", 0), Arrays.asList(1, 2, 3)); - currentParts.put(new TopicPartition("foo", 1), Arrays.asList(4, 5, 6)); - currentParts.put(new TopicPartition("bar", 0), Arrays.asList(7, 8)); - currentParts.put(new TopicPartition("baz", 0), Arrays.asList(10, 11, 12)); + currentParts.put(new TopicPartition("foo", 0), seq(1, 2, 3)); + currentParts.put(new TopicPartition("foo", 1), seq(4, 5, 6)); + currentParts.put(new TopicPartition("bar", 0), seq(7, 8)); + currentParts.put(new TopicPartition("baz", 0), seq(10, 11, 12)); - assertEquals(String.join(System.lineSeparator(), Arrays.asList( + assertEquals(String.join(System.lineSeparator(), asList( "Current partition replica assignment", "", "{\"version\":1,\"partitions\":" + @@ -457,7 +456,10 @@ public void testCurrentPartitionReplicaAssignmentToString() { "", "Save this to use as the --reassignment-json-file option during rollback" )), - currentPartitionReplicaAssignmentToString(asScala(proposedParts, this::toRawSeq), asScala(currentParts, this::toRawSeq)) + currentPartitionReplicaAssignmentToString( + CollectionConverters.asScala(proposedParts), + CollectionConverters.asScala(currentParts) + ) ); } @@ -473,53 +475,57 @@ public void testMoveMap() { Map currentReassignments = new HashMap<>(); currentReassignments.put(new TopicPartition("foo", 0), new PartitionReassignment( - Arrays.asList(1, 2, 3, 4), Collections.singletonList(4), Collections.singletonList(3))); + asList(1, 2, 3, 4), Collections.singletonList(4), Collections.singletonList(3))); currentReassignments.put(new TopicPartition("foo", 1), new PartitionReassignment( - Arrays.asList(4, 5, 6, 7, 8), Arrays.asList(7, 8), Arrays.asList(4, 5))); + asList(4, 5, 6, 7, 8), asList(7, 8), asList(4, 5))); currentReassignments.put(new TopicPartition("foo", 2), new PartitionReassignment( - Arrays.asList(1, 2, 3, 4), Arrays.asList(3, 4), Arrays.asList(1, 2))); + asList(1, 2, 3, 4), asList(3, 4), asList(1, 2))); currentReassignments.put(new TopicPartition("foo", 3), new PartitionReassignment( - Arrays.asList(1, 2, 3, 4), Arrays.asList(3, 4), Arrays.asList(1, 2))); + asList(1, 2, 3, 4), asList(3, 4), asList(1, 2))); currentReassignments.put(new TopicPartition("foo", 4), new PartitionReassignment( - Arrays.asList(1, 2, 3, 4), Arrays.asList(3, 4), Arrays.asList(1, 2))); + asList(1, 2, 3, 4), asList(3, 4), asList(1, 2))); currentReassignments.put(new TopicPartition("foo", 5), new PartitionReassignment( - Arrays.asList(1, 2, 3, 4), Arrays.asList(3, 4), Arrays.asList(1, 2))); - - Map> proposedParts = new HashMap<>(); - - proposedParts.put(new TopicPartition("foo", 0), Arrays.asList(1, 2, 5)); - proposedParts.put(new TopicPartition("foo", 2), Arrays.asList(3, 4)); - proposedParts.put(new TopicPartition("foo", 3), Arrays.asList(5, 6)); - proposedParts.put(new TopicPartition("foo", 4), Collections.singletonList(3)); - proposedParts.put(new TopicPartition("foo", 5), Arrays.asList(3, 4, 5, 6)); - proposedParts.put(new TopicPartition("bar", 0), Arrays.asList(1, 2, 3)); - - Map> currentParts = new HashMap<>(); - - currentParts.put(new TopicPartition("foo", 0), Arrays.asList(1, 2, 3, 4)); - currentParts.put(new TopicPartition("foo", 1), Arrays.asList(4, 5, 6, 7, 8)); - currentParts.put(new TopicPartition("foo", 2), Arrays.asList(1, 2, 3, 4)); - currentParts.put(new TopicPartition("foo", 3), Arrays.asList(1, 2, 3, 4)); - currentParts.put(new TopicPartition("foo", 4), Arrays.asList(1, 2, 3, 4)); - currentParts.put(new TopicPartition("foo", 5), Arrays.asList(1, 2, 3, 4)); - currentParts.put(new TopicPartition("bar", 0), Arrays.asList(2, 3, 4)); - currentParts.put(new TopicPartition("baz", 0), Arrays.asList(1, 2, 3)); - - Map> moveMap = javaMap(calculateProposedMoveMap(CollectionConverters.asScala(currentReassignments), asScala(proposedParts, this::toRawSeq), asScala(currentParts, this::toRawSeq)), this::asJava); + asList(1, 2, 3, 4), asList(3, 4), asList(1, 2))); + + Map> proposedParts = new HashMap<>(); + + proposedParts.put(new TopicPartition("foo", 0), seq(1, 2, 5)); + proposedParts.put(new TopicPartition("foo", 2), seq(3, 4)); + proposedParts.put(new TopicPartition("foo", 3), seq(5, 6)); + proposedParts.put(new TopicPartition("foo", 4), seq(3)); + proposedParts.put(new TopicPartition("foo", 5), seq(3, 4, 5, 6)); + proposedParts.put(new TopicPartition("bar", 0), seq(1, 2, 3)); + + Map> currentParts = new HashMap<>(); + + currentParts.put(new TopicPartition("foo", 0), seq(1, 2, 3, 4)); + currentParts.put(new TopicPartition("foo", 1), seq(4, 5, 6, 7, 8)); + currentParts.put(new TopicPartition("foo", 2), seq(1, 2, 3, 4)); + currentParts.put(new TopicPartition("foo", 3), seq(1, 2, 3, 4)); + currentParts.put(new TopicPartition("foo", 4), seq(1, 2, 3, 4)); + currentParts.put(new TopicPartition("foo", 5), seq(1, 2, 3, 4)); + currentParts.put(new TopicPartition("bar", 0), seq(2, 3, 4)); + currentParts.put(new TopicPartition("baz", 0), seq(1, 2, 3)); + + Map> moveMap = javaMap(calculateProposedMoveMap( + CollectionConverters.asScala(currentReassignments), + CollectionConverters.asScala(proposedParts), + CollectionConverters.asScala(currentParts) + ), this::asJava); Map> expMoveMap = new HashMap<>(); Map fooMoves = new HashMap<>(); - fooMoves.put(0, new PartitionMove(new HashSet<>(Arrays.asList(1, 2, 3)), Collections.singleton(5))); - fooMoves.put(1, new PartitionMove(new HashSet<>(Arrays.asList(4, 5, 6)), new HashSet<>(Arrays.asList(7, 8)))); - fooMoves.put(2, new PartitionMove(new HashSet<>(Arrays.asList(1, 2)), new HashSet<>(Arrays.asList(3, 4)))); - fooMoves.put(3, new PartitionMove(new HashSet<>(Arrays.asList(1, 2)), new HashSet<>(Arrays.asList(5, 6)))); - fooMoves.put(4, new PartitionMove(new HashSet<>(Arrays.asList(1, 2)), Collections.singleton(3))); - fooMoves.put(5, new PartitionMove(new HashSet<>(Arrays.asList(1, 2)), new HashSet<>(Arrays.asList(3, 4, 5, 6)))); + fooMoves.put(0, new PartitionMove(new HashSet<>(asList(1, 2, 3)), Collections.singleton(5))); + fooMoves.put(1, new PartitionMove(new HashSet<>(asList(4, 5, 6)), new HashSet<>(asList(7, 8)))); + fooMoves.put(2, new PartitionMove(new HashSet<>(asList(1, 2)), new HashSet<>(asList(3, 4)))); + fooMoves.put(3, new PartitionMove(new HashSet<>(asList(1, 2)), new HashSet<>(asList(5, 6)))); + fooMoves.put(4, new PartitionMove(new HashSet<>(asList(1, 2)), Collections.singleton(3))); + fooMoves.put(5, new PartitionMove(new HashSet<>(asList(1, 2)), new HashSet<>(asList(3, 4, 5, 6)))); expMoveMap.put("foo", fooMoves); - expMoveMap.put("bar", Collections.singletonMap(0, new PartitionMove(new HashSet<>(Arrays.asList(2, 3, 4)), Collections.singleton(1)))); + expMoveMap.put("bar", Collections.singletonMap(0, new PartitionMove(new HashSet<>(asList(2, 3, 4)), Collections.singleton(1)))); assertEquals(expMoveMap, moveMap); @@ -537,8 +543,8 @@ public void testMoveMap() { assertEquals(expFollowerThrottle, CollectionConverters.asJava(calculateFollowerThrottles(mutableMap(asScala(moveMap, this::toMoveMap))))); - assertEquals(CollectionConverters.asScala(new HashSet<>(Arrays.asList(1, 2, 3, 4, 5, 6, 7, 8))), calculateReassigningBrokers(mutableMap(asScala(moveMap, this::toMoveMap)))); - assertEquals(CollectionConverters.asScala(new HashSet<>(Arrays.asList(0, 2))), calculateMovingBrokers(toSet( + assertEquals(set(1, 2, 3, 4, 5, 6, 7, 8), calculateReassigningBrokers(mutableMap(asScala(moveMap, this::toMoveMap)))); + assertEquals(set(0, 2), calculateMovingBrokers(set( new TopicPartitionReplica("quux", 0, 0), new TopicPartitionReplica("quux", 1, 2)))); } @@ -568,10 +574,10 @@ public void testParseExecuteAssignmentArgs() { "{\"topic\":\"foo\",\"partition\":1,\"replicas\":[2,3],\"log_dirs\":[\"/abc\",\"/def\"]}" + "]}"), "Expected to detect a partition replica list with duplicate entries").getMessage()); - Map> partitionsToBeReassigned = new HashMap<>(); + Map> partitionsToBeReassigned = new HashMap<>(); - partitionsToBeReassigned.put(new TopicPartition("foo", 0), Arrays.asList(1, 2, 3)); - partitionsToBeReassigned.put(new TopicPartition("foo", 1), Arrays.asList(3, 4, 5)); + partitionsToBeReassigned.put(new TopicPartition("foo", 0), seq(1, 2, 3)); + partitionsToBeReassigned.put(new TopicPartition("foo", 1), seq(3, 4, 5)); Tuple2>, scala.collection.Map> actual = parseExecuteAssignmentArgs( "{\"version\":1,\"partitions\":" + @@ -579,7 +585,7 @@ public void testParseExecuteAssignmentArgs() { "{\"topic\":\"foo\",\"partition\":1,\"replicas\":[3,4,5],\"log_dirs\":[\"any\",\"any\",\"any\"]}" + "]}"); - assertEquals(partitionsToBeReassigned, javaMap(actual._1, CollectionConverters::asJava)); + assertEquals(CollectionConverters.asScala(partitionsToBeReassigned), actual._1); assertTrue(actual._2.isEmpty()); Map replicaAssignment = new HashMap<>(); @@ -593,7 +599,7 @@ public void testParseExecuteAssignmentArgs() { "[{\"topic\":\"foo\",\"partition\":0,\"replicas\":[1,2,3],\"log_dirs\":[\"/tmp/a\",\"/tmp/b\",\"/tmp/c\"]}" + "]}"); - assertEquals(Collections.singletonMap(new TopicPartition("foo", 0), Arrays.asList(1, 2, 3)), javaMap(actual._1, CollectionConverters::asJava)); + assertEquals(Collections.singletonMap(new TopicPartition("foo", 0), asList(1, 2, 3)), javaMap(actual._1, CollectionConverters::asJava)); assertEquals(replicaAssignment, CollectionConverters.asJava(actual._2)); } @@ -626,8 +632,8 @@ public void testExecuteWithInvalidBrokerIdFails() { @Test public void testModifyBrokerInterBrokerThrottle() throws Exception { try (MockAdminClient adminClient = new MockAdminClient.Builder().numBrokers(4).build()) { - modifyInterBrokerThrottle(adminClient, toSet(0, 1, 2), 1000); - modifyInterBrokerThrottle(adminClient, toSet(0, 3), 100); + modifyInterBrokerThrottle(adminClient, set(0, 1, 2), 1000); + modifyInterBrokerThrottle(adminClient, set(0, 3), 100); List brokers = new ArrayList<>(); for (int i = 0; i < 4; i++) brokers.add(new ConfigResource(ConfigResource.Type.BROKER, Integer.toString(i))); @@ -642,8 +648,8 @@ public void testModifyBrokerInterBrokerThrottle() throws Exception { @Test public void testModifyLogDirThrottle() throws Exception { try (MockAdminClient adminClient = new MockAdminClient.Builder().numBrokers(4).build()) { - modifyLogDirThrottle(adminClient, toSet(0, 1, 2), 2000); - modifyLogDirThrottle(adminClient, toSet(0, 3), -1); + modifyLogDirThrottle(adminClient, set(0, 1, 2), 2000); + modifyLogDirThrottle(adminClient, set(0, 3), -1); List brokers = new ArrayList<>(); for (int i = 0; i < 4; i++) @@ -664,16 +670,17 @@ public void testCurReassignmentsToString() { addTopics(adminClient); assertEquals("No partition reassignments found.", curReassignmentsToString(adminClient)); - Map> reassignments = new HashMap<>(); + Map> reassignments = new HashMap<>(); - reassignments.put(new TopicPartition("foo", 1), Arrays.asList(4, 5, 3)); - reassignments.put(new TopicPartition("foo", 0), Arrays.asList(0, 1, 4, 2)); - reassignments.put(new TopicPartition("bar", 0), Arrays.asList(2, 3)); + reassignments.put(new TopicPartition("foo", 1), seq(4, 5, 3)); + reassignments.put(new TopicPartition("foo", 0), seq(0, 1, 4, 2)); + reassignments.put(new TopicPartition("bar", 0), seq(2, 3)); - Map reassignmentResult = CollectionConverters.asJava(alterPartitionReassignments(adminClient, asScala(reassignments, this::toRawSeq))); + scala.collection.Map reassignmentResult = + alterPartitionReassignments(adminClient, CollectionConverters.asScala(reassignments)); assertTrue(reassignmentResult.isEmpty()); - assertEquals(String.join(System.lineSeparator(), Arrays.asList("Current partition reassignments:", + assertEquals(String.join(System.lineSeparator(), asList("Current partition reassignments:", "bar-0: replicas: 2,3,0. removing: 0.", "foo-0: replicas: 0,1,2. adding: 4.", "foo-1: replicas: 1,2,3. adding: 4,5. removing: 1,2.")), @@ -735,7 +742,7 @@ public void testAlterReplicaLogDirs() { try (MockAdminClient adminClient = new MockAdminClient.Builder(). numBrokers(4). brokerLogDirs(Collections.nCopies(4, - Arrays.asList("/tmp/kafka-logs0", "/tmp/kafka-logs1"))). + asList("/tmp/kafka-logs0", "/tmp/kafka-logs1"))). build()) { addTopics(adminClient); @@ -746,7 +753,7 @@ public void testAlterReplicaLogDirs() { assignment.put(new TopicPartitionReplica("quux", 1, 0), "/tmp/kafka-logs1"); assertEquals( - CollectionConverters.asScala(Collections.singleton(new TopicPartitionReplica("foo", 0, 0))), + set(new TopicPartitionReplica("foo", 0, 0)), alterReplicaLogDirs(adminClient, CollectionConverters.asScala(assignment)) ); } @@ -766,17 +773,17 @@ public void testPropagateInvalidJsonError() { } @SuppressWarnings("unchecked") - private scala.collection.immutable.Set toSet(final T... set) { - return toMutableSet(new HashSet<>(Arrays.asList(set))).toSet(); + private scala.collection.immutable.Set set(final T... set) { + return toMutableSet(new HashSet<>(asList(set))).toSet(); } - private scala.collection.mutable.Set toMutableSet(Set set) { - return CollectionConverters.asScala(new HashSet<>(set)); + @SuppressWarnings("unchecked") + private Seq seq(T... col) { + return (Seq) CollectionConverters.asScala(asList(col)).toSeq(); } - @SuppressWarnings("unchecked") - private Seq toRawSeq(Collection col) { - return (Seq) CollectionConverters.asScala(col).toSeq(); + private scala.collection.mutable.Set toMutableSet(Set set) { + return CollectionConverters.asScala(new HashSet<>(set)); } @SuppressWarnings("unchecked") @@ -821,17 +828,17 @@ private ReassignPartitionsCommand.LogDirMoveState asScala(LogDirMoveState state) throw new IllegalArgumentException("Unknown state " + state); } + @SuppressWarnings("unchecked") private ReassignPartitionsCommand.PartitionReassignmentState asScala(PartitionReassignmentState state) { + List currentReplicas = state.currentReplicas; + List targetReplicas = state.targetReplicas; + return new ReassignPartitionsCommand.PartitionReassignmentState( - toRawSeq(state.currentReplicas), - toRawSeq(state.targetReplicas), + CollectionConverters.asScala(currentReplicas).toSeq(), + CollectionConverters.asScala(targetReplicas).toSeq(), state.done); } - private Tuple2> asScala(Tuple2> t) { - return new Tuple2<>(t._1, toRawSeq(t._2)); - } - private Map javaMap(scala.collection.Map map, Function mapper) { Map res = new HashMap<>(); map.foreach(t -> res.put(t._1, mapper.apply(t._2))); From c6d12759ca3f060f4ce7537ecc7ea6e5bd01cdb9 Mon Sep 17 00:00:00 2001 From: nizhikov Date: Fri, 8 Sep 2023 16:17:35 +0300 Subject: [PATCH 43/78] KAFKA-14595 Improving test --- .../reassign/ReassignPartitionsUnitTest.java | 122 +++++++----------- 1 file changed, 46 insertions(+), 76 deletions(-) diff --git a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java index 22654eec6bd2d..74d2dfae1b69d 100644 --- a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java +++ b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java @@ -38,7 +38,6 @@ import org.junit.jupiter.api.Timeout; import scala.Tuple2; import scala.collection.Seq; -import scala.collection.mutable.Map$; import scala.jdk.javaapi.CollectionConverters; import java.util.ArrayList; @@ -50,7 +49,6 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.ExecutionException; -import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -171,20 +169,19 @@ public void testFindPartitionReassignmentStates() { assertEquals(Collections.singletonMap(new TopicPartition("quux", 0), UnknownTopicOrPartitionException.class), reassignmentResult); - Map partitionReassignmentStates = new HashMap<>(); + Map partitionReassignmentStates = new HashMap<>(); - partitionReassignmentStates.put(new TopicPartition("foo", 0), new PartitionReassignmentState(asList(0, 1, 2), asList(0, 1, 3), false)); - partitionReassignmentStates.put(new TopicPartition("foo", 1), new PartitionReassignmentState(asList(1, 2, 3), asList(1, 2, 3), true)); + partitionReassignmentStates.put(new TopicPartition("foo", 0), asScala(new PartitionReassignmentState(asList(0, 1, 2), asList(0, 1, 3), false))); + partitionReassignmentStates.put(new TopicPartition("foo", 1), asScala(new PartitionReassignmentState(asList(1, 2, 3), asList(1, 2, 3), true))); - Tuple2, Boolean> expected = new Tuple2<>(partitionReassignmentStates, true); Tuple2, Object> actual = findPartitionReassignmentStates(adminClient, CollectionConverters.asScala(asList( new Tuple2<>(new TopicPartition("foo", 0), seq(0, 1, 3)), new Tuple2<>(new TopicPartition("foo", 1), seq(1, 2, 3)) ))); - assertEquals(asScala(expected._1, this::asScala), actual._1); - assertEquals(expected._2, actual._2); + assertEquals(CollectionConverters.asScala(partitionReassignmentStates), actual._1); + assertEquals(true, actual._2); // Cancel the reassignment and test findPartitionReassignmentStates again. Map> cancelResult = CollectionConverters.asJava(cancelPartitionReassignments(adminClient, set(new TopicPartition("foo", 0), new TopicPartition("quux", 2)))) @@ -194,17 +191,16 @@ public void testFindPartitionReassignmentStates() { partitionReassignmentStates.clear(); - partitionReassignmentStates.put(new TopicPartition("foo", 0), new PartitionReassignmentState(asList(0, 1, 2), asList(0, 1, 3), true)); - partitionReassignmentStates.put(new TopicPartition("foo", 1), new PartitionReassignmentState(asList(1, 2, 3), asList(1, 2, 3), true)); + partitionReassignmentStates.put(new TopicPartition("foo", 0), asScala(new PartitionReassignmentState(asList(0, 1, 2), asList(0, 1, 3), true))); + partitionReassignmentStates.put(new TopicPartition("foo", 1), asScala(new PartitionReassignmentState(asList(1, 2, 3), asList(1, 2, 3), true))); - expected = new Tuple2<>(partitionReassignmentStates, false); actual = findPartitionReassignmentStates(adminClient, CollectionConverters.asScala(asList( new Tuple2<>(new TopicPartition("foo", 0), seq(0, 1, 3)), new Tuple2<>(new TopicPartition("foo", 1), seq(1, 2, 3)) ))); - assertEquals(asScala(expected._1, this::asScala), actual._1); - assertEquals(expected._2, actual._2); + assertEquals(CollectionConverters.asScala(partitionReassignmentStates), actual._1); + assertEquals(false, actual._2); } } @@ -507,43 +503,44 @@ public void testMoveMap() { currentParts.put(new TopicPartition("bar", 0), seq(2, 3, 4)); currentParts.put(new TopicPartition("baz", 0), seq(1, 2, 3)); - Map> moveMap = javaMap(calculateProposedMoveMap( - CollectionConverters.asScala(currentReassignments), - CollectionConverters.asScala(proposedParts), - CollectionConverters.asScala(currentParts) - ), this::asJava); + scala.collection.mutable.Map> + moveMap = calculateProposedMoveMap( + CollectionConverters.asScala(currentReassignments), + CollectionConverters.asScala(proposedParts), + CollectionConverters.asScala(currentParts) + ); - Map> expMoveMap = new HashMap<>(); + Map fooMoves = new HashMap<>(); - Map fooMoves = new HashMap<>(); + fooMoves.put(0, asScala(new PartitionMove(jset(1, 2, 3), jset(5)))); + fooMoves.put(1, asScala(new PartitionMove(jset(4, 5, 6), jset(7, 8)))); + fooMoves.put(2, asScala(new PartitionMove(jset(1, 2), jset(3, 4)))); + fooMoves.put(3, asScala(new PartitionMove(jset(1, 2), jset(5, 6)))); + fooMoves.put(4, asScala(new PartitionMove(jset(1, 2), jset(3)))); + fooMoves.put(5, asScala(new PartitionMove(jset(1, 2), jset(3, 4, 5, 6)))); - fooMoves.put(0, new PartitionMove(new HashSet<>(asList(1, 2, 3)), Collections.singleton(5))); - fooMoves.put(1, new PartitionMove(new HashSet<>(asList(4, 5, 6)), new HashSet<>(asList(7, 8)))); - fooMoves.put(2, new PartitionMove(new HashSet<>(asList(1, 2)), new HashSet<>(asList(3, 4)))); - fooMoves.put(3, new PartitionMove(new HashSet<>(asList(1, 2)), new HashSet<>(asList(5, 6)))); - fooMoves.put(4, new PartitionMove(new HashSet<>(asList(1, 2)), Collections.singleton(3))); - fooMoves.put(5, new PartitionMove(new HashSet<>(asList(1, 2)), new HashSet<>(asList(3, 4, 5, 6)))); + Map barMoves = new HashMap<>(); - expMoveMap.put("foo", fooMoves); - expMoveMap.put("bar", Collections.singletonMap(0, new PartitionMove(new HashSet<>(asList(2, 3, 4)), Collections.singleton(1)))); + barMoves.put(0, asScala(new PartitionMove(jset(2, 3, 4), jset(1)))); - assertEquals(expMoveMap, moveMap); + assertEquals(CollectionConverters.asScala(fooMoves), moveMap.get("foo").get()); + assertEquals(CollectionConverters.asScala(barMoves), moveMap.get("bar").get()); Map expLeaderThrottle = new HashMap<>(); expLeaderThrottle.put("foo", "0:1,0:2,0:3,1:4,1:5,1:6,2:1,2:2,3:1,3:2,4:1,4:2,5:1,5:2"); expLeaderThrottle.put("bar", "0:2,0:3,0:4"); - assertEquals(expLeaderThrottle, CollectionConverters.asJava(calculateLeaderThrottles(mutableMap(asScala(moveMap, this::toMoveMap))))); + assertEquals(CollectionConverters.asScala(expLeaderThrottle), calculateLeaderThrottles(moveMap)); Map expFollowerThrottle = new HashMap<>(); expFollowerThrottle.put("foo", "0:5,1:7,1:8,2:3,2:4,3:5,3:6,4:3,5:3,5:4,5:5,5:6"); expFollowerThrottle.put("bar", "0:1"); - assertEquals(expFollowerThrottle, CollectionConverters.asJava(calculateFollowerThrottles(mutableMap(asScala(moveMap, this::toMoveMap))))); + assertEquals(CollectionConverters.asScala(expFollowerThrottle), calculateFollowerThrottles(moveMap)); - assertEquals(set(1, 2, 3, 4, 5, 6, 7, 8), calculateReassigningBrokers(mutableMap(asScala(moveMap, this::toMoveMap)))); + assertEquals(set(1, 2, 3, 4, 5, 6, 7, 8), calculateReassigningBrokers(moveMap)); assertEquals(set(0, 2), calculateMovingBrokers(set( new TopicPartitionReplica("quux", 0, 0), new TopicPartitionReplica("quux", 1, 2)))); @@ -599,8 +596,8 @@ public void testParseExecuteAssignmentArgs() { "[{\"topic\":\"foo\",\"partition\":0,\"replicas\":[1,2,3],\"log_dirs\":[\"/tmp/a\",\"/tmp/b\",\"/tmp/c\"]}" + "]}"); - assertEquals(Collections.singletonMap(new TopicPartition("foo", 0), asList(1, 2, 3)), javaMap(actual._1, CollectionConverters::asJava)); - assertEquals(replicaAssignment, CollectionConverters.asJava(actual._2)); + assertEquals(CollectionConverters.asScala(Collections.singletonMap(new TopicPartition("foo", 0), seq(1, 2, 3))), actual._1); + assertEquals(CollectionConverters.asScala(replicaAssignment), actual._2); } @Test @@ -774,37 +771,17 @@ public void testPropagateInvalidJsonError() { @SuppressWarnings("unchecked") private scala.collection.immutable.Set set(final T... set) { - return toMutableSet(new HashSet<>(asList(set))).toSet(); + return CollectionConverters.asScala(jset(set)).toSet(); } @SuppressWarnings("unchecked") - private Seq seq(T... col) { - return (Seq) CollectionConverters.asScala(asList(col)).toSeq(); - } - - private scala.collection.mutable.Set toMutableSet(Set set) { - return CollectionConverters.asScala(new HashSet<>(set)); + private Set jset(final T...set) { + return new HashSet<>(asList(set)); } @SuppressWarnings("unchecked") - private scala.collection.mutable.Map toMoveMap(Map moveMap) { - scala.collection.mutable.Map res = - (scala.collection.mutable.Map) Map$.MODULE$.mapFactory().newBuilder().result(); - - moveMap.forEach((k, v) -> res.addOne(new Tuple2<>(k, - new ReassignPartitionsCommand.PartitionMove(toMutableSet(v.sources), toMutableSet(v.destinations))))); - - return res; - } - - private scala.collection.Map asScala(Map map, Function mapper) { - return CollectionConverters.asScala(map.entrySet().stream(). - collect(Collectors.toMap(Map.Entry::getKey, e -> mapper.apply(e.getValue())))); - } - - @SuppressWarnings("unchecked") - private scala.collection.mutable.Map mutableMap(scala.collection.Map map) { - return (scala.collection.mutable.Map) map.to(Map$.MODULE$.mapFactory()); + private Seq seq(T... col) { + return (Seq) CollectionConverters.asScala(asList(col)).toSeq(); } private ReassignPartitionsCommand.LogDirMoveState asScala(LogDirMoveState state) { @@ -829,25 +806,18 @@ private ReassignPartitionsCommand.LogDirMoveState asScala(LogDirMoveState state) } @SuppressWarnings("unchecked") - private ReassignPartitionsCommand.PartitionReassignmentState asScala(PartitionReassignmentState state) { - List currentReplicas = state.currentReplicas; - List targetReplicas = state.targetReplicas; + private ReassignPartitionsCommand.PartitionMove asScala(PartitionMove pm) { + return new ReassignPartitionsCommand.PartitionMove( + CollectionConverters.asScala((Set) pm.sources), + CollectionConverters.asScala((Set) pm.destinations) + ); + } + @SuppressWarnings("unchecked") + private ReassignPartitionsCommand.PartitionReassignmentState asScala(PartitionReassignmentState state) { return new ReassignPartitionsCommand.PartitionReassignmentState( - CollectionConverters.asScala(currentReplicas).toSeq(), - CollectionConverters.asScala(targetReplicas).toSeq(), + CollectionConverters.asScala((List) state.currentReplicas).toSeq(), + CollectionConverters.asScala((List) state.targetReplicas).toSeq(), state.done); } - - private Map javaMap(scala.collection.Map map, Function mapper) { - Map res = new HashMap<>(); - map.foreach(t -> res.put(t._1, mapper.apply(t._2))); - return res; - } - - private Map asJava(scala.collection.mutable.Map map) { - Map res = new HashMap<>(); - map.foreach(t -> res.put((Integer) t._1, new PartitionMove(CollectionConverters.asJava(t._2.sources().toSet()), CollectionConverters.asJava(t._2.destinations().toSet())))); - return res; - } } From 9877ef700114fd4b6888864f8a797347f67db715 Mon Sep 17 00:00:00 2001 From: nizhikov Date: Fri, 8 Sep 2023 17:03:13 +0300 Subject: [PATCH 44/78] KAFKA-14595 Improving test --- .../reassign/ReassignPartitionsUnitTest.java | 165 ++++++++---------- 1 file changed, 73 insertions(+), 92 deletions(-) diff --git a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java index 74d2dfae1b69d..7a7611005dd00 100644 --- a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java +++ b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java @@ -41,6 +41,7 @@ import scala.jdk.javaapi.CollectionConverters; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -52,7 +53,6 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import static java.util.Arrays.asList; import static kafka.admin.ReassignPartitionsCommand.alterPartitionReassignments; import static kafka.admin.ReassignPartitionsCommand.alterReplicaLogDirs; import static kafka.admin.ReassignPartitionsCommand.brokerLevelFollowerThrottle; @@ -123,13 +123,13 @@ public void testPartitionReassignStatesToString() { Map states = new HashMap<>(); states.put(new TopicPartition("foo", 0), - asScala(new PartitionReassignmentState(asList(1, 2, 3), asList(1, 2, 3), true))); + asScala(new PartitionReassignmentState(Arrays.asList(1, 2, 3), Arrays.asList(1, 2, 3), true))); states.put(new TopicPartition("foo", 1), - asScala(new PartitionReassignmentState(asList(1, 2, 3), asList(1, 2, 4), false))); + asScala(new PartitionReassignmentState(Arrays.asList(1, 2, 3), Arrays.asList(1, 2, 4), false))); states.put(new TopicPartition("bar", 0), - asScala(new PartitionReassignmentState(asList(1, 2, 3), asList(1, 2, 4), false))); + asScala(new PartitionReassignmentState(Arrays.asList(1, 2, 3), Arrays.asList(1, 2, 4), false))); - assertEquals(String.join(System.lineSeparator(), asList( + assertEquals(String.join(System.lineSeparator(), Arrays.asList( "Status of partition reassignment:", "Reassignment of partition bar-0 is still in progress.", "Reassignment of partition foo-0 is completed.", @@ -139,18 +139,18 @@ public void testPartitionReassignStatesToString() { private void addTopics(MockAdminClient adminClient) { List b = adminClient.brokers(); - adminClient.addTopic(false, "foo", asList( + adminClient.addTopic(false, "foo", Arrays.asList( new TopicPartitionInfo(0, b.get(0), - asList(b.get(0), b.get(1), b.get(2)), - asList(b.get(0), b.get(1))), + Arrays.asList(b.get(0), b.get(1), b.get(2)), + Arrays.asList(b.get(0), b.get(1))), new TopicPartitionInfo(1, b.get(1), - asList(b.get(1), b.get(2), b.get(3)), - asList(b.get(1), b.get(2), b.get(3))) + Arrays.asList(b.get(1), b.get(2), b.get(3)), + Arrays.asList(b.get(1), b.get(2), b.get(3))) ), Collections.emptyMap()); - adminClient.addTopic(false, "bar", Collections.singletonList( + adminClient.addTopic(false, "bar", Arrays.asList( new TopicPartitionInfo(0, b.get(2), - asList(b.get(2), b.get(3), b.get(0)), - asList(b.get(2), b.get(3), b.get(0))) + Arrays.asList(b.get(2), b.get(3), b.get(0)), + Arrays.asList(b.get(2), b.get(3), b.get(0))) ), Collections.emptyMap()); } @@ -164,18 +164,18 @@ public void testFindPartitionReassignmentStates() { reassignments.put(new TopicPartition("foo", 0), seq(0, 1, 3)); reassignments.put(new TopicPartition("quux", 0), seq(1, 2, 3)); - Map> reassignmentResult = CollectionConverters.asJava(alterPartitionReassignments(adminClient, CollectionConverters.asScala(reassignments))) - .entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().getClass())); + scala.collection.Map reassignmentResult = alterPartitionReassignments(adminClient, CollectionConverters.asScala(reassignments)); - assertEquals(Collections.singletonMap(new TopicPartition("quux", 0), UnknownTopicOrPartitionException.class), reassignmentResult); + assertEquals(1, reassignmentResult.size()); + assertEquals(UnknownTopicOrPartitionException.class, reassignmentResult.get(new TopicPartition("quux", 0)).get().getClass()); Map partitionReassignmentStates = new HashMap<>(); - partitionReassignmentStates.put(new TopicPartition("foo", 0), asScala(new PartitionReassignmentState(asList(0, 1, 2), asList(0, 1, 3), false))); - partitionReassignmentStates.put(new TopicPartition("foo", 1), asScala(new PartitionReassignmentState(asList(1, 2, 3), asList(1, 2, 3), true))); + partitionReassignmentStates.put(new TopicPartition("foo", 0), asScala(new PartitionReassignmentState(Arrays.asList(0, 1, 2), Arrays.asList(0, 1, 3), false))); + partitionReassignmentStates.put(new TopicPartition("foo", 1), asScala(new PartitionReassignmentState(Arrays.asList(1, 2, 3), Arrays.asList(1, 2, 3), true))); Tuple2, Object> actual = - findPartitionReassignmentStates(adminClient, CollectionConverters.asScala(asList( + findPartitionReassignmentStates(adminClient, CollectionConverters.asScala(Arrays.asList( new Tuple2<>(new TopicPartition("foo", 0), seq(0, 1, 3)), new Tuple2<>(new TopicPartition("foo", 1), seq(1, 2, 3)) ))); @@ -184,17 +184,17 @@ public void testFindPartitionReassignmentStates() { assertEquals(true, actual._2); // Cancel the reassignment and test findPartitionReassignmentStates again. - Map> cancelResult = CollectionConverters.asJava(cancelPartitionReassignments(adminClient, set(new TopicPartition("foo", 0), new TopicPartition("quux", 2)))) - .entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().getClass())); + scala.collection.Map cancelResult = cancelPartitionReassignments(adminClient, set(new TopicPartition("foo", 0), new TopicPartition("quux", 2))); - assertEquals(Collections.singletonMap(new TopicPartition("quux", 2), UnknownTopicOrPartitionException.class), cancelResult); + assertEquals(1, cancelResult.size()); + assertEquals(UnknownTopicOrPartitionException.class, cancelResult.get(new TopicPartition("quux", 2)).get().getClass()); partitionReassignmentStates.clear(); - partitionReassignmentStates.put(new TopicPartition("foo", 0), asScala(new PartitionReassignmentState(asList(0, 1, 2), asList(0, 1, 3), true))); - partitionReassignmentStates.put(new TopicPartition("foo", 1), asScala(new PartitionReassignmentState(asList(1, 2, 3), asList(1, 2, 3), true))); + partitionReassignmentStates.put(new TopicPartition("foo", 0), asScala(new PartitionReassignmentState(Arrays.asList(0, 1, 2), Arrays.asList(0, 1, 3), true))); + partitionReassignmentStates.put(new TopicPartition("foo", 1), asScala(new PartitionReassignmentState(Arrays.asList(1, 2, 3), Arrays.asList(1, 2, 3), true))); - actual = findPartitionReassignmentStates(adminClient, CollectionConverters.asScala(asList( + actual = findPartitionReassignmentStates(adminClient, CollectionConverters.asScala(Arrays.asList( new Tuple2<>(new TopicPartition("foo", 0), seq(0, 1, 3)), new Tuple2<>(new TopicPartition("foo", 1), seq(1, 2, 3)) ))); @@ -208,19 +208,19 @@ public void testFindPartitionReassignmentStates() { public void testFindLogDirMoveStates() throws Exception { try (MockAdminClient adminClient = new MockAdminClient.Builder(). numBrokers(4). - brokerLogDirs(asList( - asList("/tmp/kafka-logs0", "/tmp/kafka-logs1"), - asList("/tmp/kafka-logs0", "/tmp/kafka-logs1"), - asList("/tmp/kafka-logs0", "/tmp/kafka-logs1"), - asList("/tmp/kafka-logs0", null))). + brokerLogDirs(Arrays.asList( + Arrays.asList("/tmp/kafka-logs0", "/tmp/kafka-logs1"), + Arrays.asList("/tmp/kafka-logs0", "/tmp/kafka-logs1"), + Arrays.asList("/tmp/kafka-logs0", "/tmp/kafka-logs1"), + Arrays.asList("/tmp/kafka-logs0", null))). build()) { addTopics(adminClient); List b = adminClient.brokers(); - adminClient.addTopic(false, "quux", Collections.singletonList( + adminClient.addTopic(false, "quux", Arrays.asList( new TopicPartitionInfo(0, b.get(2), - asList(b.get(1), b.get(2), b.get(3)), - asList(b.get(1), b.get(2), b.get(3)))), + Arrays.asList(b.get(1), b.get(2), b.get(3)), + Arrays.asList(b.get(1), b.get(2), b.get(3)))), Collections.emptyMap()); Map replicaAssignment = new HashMap<>(); @@ -232,11 +232,13 @@ public void testFindLogDirMoveStates() throws Exception { Map expLogDirMoveStates = new HashMap<>(); - expLogDirMoveStates.put(new TopicPartitionReplica("bar", 0, 0), asScala(new CompletedMoveState("/tmp/kafka-logs0"))); - expLogDirMoveStates.put(new TopicPartitionReplica("foo", 0, 0), asScala(new ActiveMoveState("/tmp/kafka-logs0", "/tmp/kafka-logs1", "/tmp/kafka-logs1"))); - expLogDirMoveStates.put(new TopicPartitionReplica("foo", 1, 0), asScala(new CancelledMoveState("/tmp/kafka-logs0", "/tmp/kafka-logs1"))); - expLogDirMoveStates.put(new TopicPartitionReplica("quux", 1, 0), asScala(new MissingLogDirMoveState("/tmp/kafka-logs1"))); - expLogDirMoveStates.put(new TopicPartitionReplica("quuz", 0, 0), asScala(new MissingReplicaMoveState("/tmp/kafka-logs0"))); + expLogDirMoveStates.put(new TopicPartitionReplica("bar", 0, 0), new ReassignPartitionsCommand.CompletedMoveState("/tmp/kafka-logs0")); + expLogDirMoveStates.put(new TopicPartitionReplica("foo", 0, 0), new ReassignPartitionsCommand.ActiveMoveState("/tmp/kafka-logs0", + "/tmp/kafka-logs1", "/tmp/kafka-logs1")); + expLogDirMoveStates.put(new TopicPartitionReplica("foo", 1, 0), new ReassignPartitionsCommand.CancelledMoveState("/tmp/kafka-logs0", + "/tmp/kafka-logs1")); + expLogDirMoveStates.put(new TopicPartitionReplica("quux", 1, 0), new ReassignPartitionsCommand.MissingLogDirMoveState("/tmp/kafka-logs1")); + expLogDirMoveStates.put(new TopicPartitionReplica("quuz", 0, 0), new ReassignPartitionsCommand.MissingReplicaMoveState("/tmp/kafka-logs0")); Map targetMoves = new HashMap<>(); @@ -257,17 +259,17 @@ public void testFindLogDirMoveStates() throws Exception { public void testReplicaMoveStatesToString() { Map states = new HashMap<>(); - states.put(new TopicPartitionReplica("bar", 0, 0), asScala(new CompletedMoveState("/tmp/kafka-logs0"))); - states.put(new TopicPartitionReplica("foo", 0, 0), asScala(new ActiveMoveState("/tmp/kafka-logs0", - "/tmp/kafka-logs1", "/tmp/kafka-logs1"))); - states.put(new TopicPartitionReplica("foo", 1, 0), asScala(new CancelledMoveState("/tmp/kafka-logs0", - "/tmp/kafka-logs1"))); - states.put(new TopicPartitionReplica("quux", 0, 0), asScala(new MissingReplicaMoveState("/tmp/kafka-logs1"))); - states.put(new TopicPartitionReplica("quux", 1, 1), asScala(new ActiveMoveState("/tmp/kafka-logs0", - "/tmp/kafka-logs1", "/tmp/kafka-logs2"))); - states.put(new TopicPartitionReplica("quux", 2, 1), asScala(new MissingLogDirMoveState("/tmp/kafka-logs1"))); - - assertEquals(String.join(System.lineSeparator(), asList( + states.put(new TopicPartitionReplica("bar", 0, 0), new ReassignPartitionsCommand.CompletedMoveState("/tmp/kafka-logs0")); + states.put(new TopicPartitionReplica("foo", 0, 0), new ReassignPartitionsCommand.ActiveMoveState("/tmp/kafka-logs0", + "/tmp/kafka-logs1", "/tmp/kafka-logs1")); + states.put(new TopicPartitionReplica("foo", 1, 0), new ReassignPartitionsCommand.CancelledMoveState("/tmp/kafka-logs0", + "/tmp/kafka-logs1")); + states.put(new TopicPartitionReplica("quux", 0, 0), new ReassignPartitionsCommand.MissingReplicaMoveState("/tmp/kafka-logs1")); + states.put(new TopicPartitionReplica("quux", 1, 1), new ReassignPartitionsCommand.ActiveMoveState("/tmp/kafka-logs0", + "/tmp/kafka-logs1", "/tmp/kafka-logs2")); + states.put(new TopicPartitionReplica("quux", 2, 1), new ReassignPartitionsCommand.MissingLogDirMoveState("/tmp/kafka-logs1")); + + assertEquals(String.join(System.lineSeparator(), Arrays.asList( "Reassignment of replica bar-0-0 completed successfully.", "Reassignment of replica foo-0-0 is still in progress.", "Partition foo-1 on broker 0 is not being moved from log dir /tmp/kafka-logs0 to /tmp/kafka-logs1.", @@ -289,7 +291,7 @@ public void testGetReplicaAssignments() { assignments.put(new TopicPartition("foo", 1), seq(1, 2, 3)); assertEquals(CollectionConverters.asScala(assignments), - getReplicaAssignmentForTopics(adminClient, CollectionConverters.asScala(Collections.singletonList("foo")))); + getReplicaAssignmentForTopics(adminClient, CollectionConverters.asScala(Arrays.asList("foo")))); assignments.clear(); @@ -304,26 +306,26 @@ public void testGetReplicaAssignments() { @Test public void testGetBrokerRackInformation() { try (MockAdminClient adminClient = new MockAdminClient.Builder(). - brokers(asList(new Node(0, "localhost", 9092, "rack0"), + brokers(Arrays.asList(new Node(0, "localhost", 9092, "rack0"), new Node(1, "localhost", 9093, "rack1"), new Node(2, "localhost", 9094, null))). build()) { - assertEquals(CollectionConverters.asScala(asList( + assertEquals(CollectionConverters.asScala(Arrays.asList( new BrokerMetadata(0, Optional.of("rack0")), new BrokerMetadata(1, Optional.of("rack1")) - )), getBrokerMetadata(adminClient, CollectionConverters.asScala(asList(0, 1)), true)); - assertEquals(CollectionConverters.asScala(asList( + )), getBrokerMetadata(adminClient, CollectionConverters.asScala(Arrays.asList(0, 1)), true)); + assertEquals(CollectionConverters.asScala(Arrays.asList( new BrokerMetadata(0, Optional.empty()), new BrokerMetadata(1, Optional.empty()) - )), getBrokerMetadata(adminClient, CollectionConverters.asScala(asList(0, 1)), false)); + )), getBrokerMetadata(adminClient, CollectionConverters.asScala(Arrays.asList(0, 1)), false)); assertStartsWith("Not all brokers have rack information", assertThrows(AdminOperationException.class, - () -> getBrokerMetadata(adminClient, CollectionConverters.asScala(asList(1, 2)), true)).getMessage()); - assertEquals(CollectionConverters.asScala(asList( + () -> getBrokerMetadata(adminClient, CollectionConverters.asScala(Arrays.asList(1, 2)), true)).getMessage()); + assertEquals(CollectionConverters.asScala(Arrays.asList( new BrokerMetadata(1, Optional.empty()), new BrokerMetadata(2, Optional.empty()) - )), getBrokerMetadata(adminClient, CollectionConverters.asScala(asList(1, 2)), false)); + )), getBrokerMetadata(adminClient, CollectionConverters.asScala(Arrays.asList(1, 2)), false)); } } @@ -373,7 +375,7 @@ public void testGenerateAssignmentWithInvalidPartitionsFails() { @Test public void testGenerateAssignmentWithInconsistentRacks() { try (MockAdminClient adminClient = new MockAdminClient.Builder(). - brokers(asList( + brokers(Arrays.asList( new Node(0, "localhost", 9092, "rack0"), new Node(1, "localhost", 9093, "rack0"), new Node(2, "localhost", 9094, null), @@ -404,7 +406,7 @@ public void testGenerateAssignmentWithInconsistentRacks() { public void testGenerateAssignmentWithFewerBrokers() { try (MockAdminClient adminClient = new MockAdminClient.Builder().numBrokers(4).build()) { addTopics(adminClient); - List goalBrokers = asList(0, 1, 3); + List goalBrokers = Arrays.asList(0, 1, 3); Tuple2>, scala.collection.Map>> proposedCurrent = generateAssignment(adminClient, @@ -442,7 +444,7 @@ public void testCurrentPartitionReplicaAssignmentToString() { currentParts.put(new TopicPartition("bar", 0), seq(7, 8)); currentParts.put(new TopicPartition("baz", 0), seq(10, 11, 12)); - assertEquals(String.join(System.lineSeparator(), asList( + assertEquals(String.join(System.lineSeparator(), Arrays.asList( "Current partition replica assignment", "", "{\"version\":1,\"partitions\":" + @@ -471,17 +473,17 @@ public void testMoveMap() { Map currentReassignments = new HashMap<>(); currentReassignments.put(new TopicPartition("foo", 0), new PartitionReassignment( - asList(1, 2, 3, 4), Collections.singletonList(4), Collections.singletonList(3))); + Arrays.asList(1, 2, 3, 4), Arrays.asList(4), Arrays.asList(3))); currentReassignments.put(new TopicPartition("foo", 1), new PartitionReassignment( - asList(4, 5, 6, 7, 8), asList(7, 8), asList(4, 5))); + Arrays.asList(4, 5, 6, 7, 8), Arrays.asList(7, 8), Arrays.asList(4, 5))); currentReassignments.put(new TopicPartition("foo", 2), new PartitionReassignment( - asList(1, 2, 3, 4), asList(3, 4), asList(1, 2))); + Arrays.asList(1, 2, 3, 4), Arrays.asList(3, 4), Arrays.asList(1, 2))); currentReassignments.put(new TopicPartition("foo", 3), new PartitionReassignment( - asList(1, 2, 3, 4), asList(3, 4), asList(1, 2))); + Arrays.asList(1, 2, 3, 4), Arrays.asList(3, 4), Arrays.asList(1, 2))); currentReassignments.put(new TopicPartition("foo", 4), new PartitionReassignment( - asList(1, 2, 3, 4), asList(3, 4), asList(1, 2))); + Arrays.asList(1, 2, 3, 4), Arrays.asList(3, 4), Arrays.asList(1, 2))); currentReassignments.put(new TopicPartition("foo", 5), new PartitionReassignment( - asList(1, 2, 3, 4), asList(3, 4), asList(1, 2))); + Arrays.asList(1, 2, 3, 4), Arrays.asList(3, 4), Arrays.asList(1, 2))); Map> proposedParts = new HashMap<>(); @@ -677,7 +679,7 @@ public void testCurReassignmentsToString() { alterPartitionReassignments(adminClient, CollectionConverters.asScala(reassignments)); assertTrue(reassignmentResult.isEmpty()); - assertEquals(String.join(System.lineSeparator(), asList("Current partition reassignments:", + assertEquals(String.join(System.lineSeparator(), Arrays.asList("Current partition reassignments:", "bar-0: replicas: 2,3,0. removing: 0.", "foo-0: replicas: 0,1,2. adding: 4.", "foo-1: replicas: 1,2,3. adding: 4,5. removing: 1,2.")), @@ -739,7 +741,7 @@ public void testAlterReplicaLogDirs() { try (MockAdminClient adminClient = new MockAdminClient.Builder(). numBrokers(4). brokerLogDirs(Collections.nCopies(4, - asList("/tmp/kafka-logs0", "/tmp/kafka-logs1"))). + Arrays.asList("/tmp/kafka-logs0", "/tmp/kafka-logs1"))). build()) { addTopics(adminClient); @@ -776,33 +778,12 @@ private scala.collection.immutable.Set set(final T... set) { @SuppressWarnings("unchecked") private Set jset(final T...set) { - return new HashSet<>(asList(set)); + return new HashSet<>(Arrays.asList(set)); } @SuppressWarnings("unchecked") private Seq seq(T... col) { - return (Seq) CollectionConverters.asScala(asList(col)).toSeq(); - } - - private ReassignPartitionsCommand.LogDirMoveState asScala(LogDirMoveState state) { - if (state instanceof ActiveMoveState) { - ActiveMoveState s = (ActiveMoveState) state; - return new ReassignPartitionsCommand.ActiveMoveState(s.currentLogDir, s.targetLogDir, s.futureLogDir); - } else if (state instanceof CancelledMoveState) { - CancelledMoveState s = (CancelledMoveState) state; - return new ReassignPartitionsCommand.CancelledMoveState(s.currentLogDir, s.targetLogDir); - } else if (state instanceof CompletedMoveState) { - CompletedMoveState s = (CompletedMoveState) state; - return new ReassignPartitionsCommand.CompletedMoveState(s.targetLogDir); - } else if (state instanceof MissingLogDirMoveState) { - MissingLogDirMoveState s = (MissingLogDirMoveState) state; - return new ReassignPartitionsCommand.MissingLogDirMoveState(s.targetLogDir); - } else if (state instanceof MissingReplicaMoveState) { - MissingReplicaMoveState s = (MissingReplicaMoveState) state; - return new ReassignPartitionsCommand.MissingReplicaMoveState(s.targetLogDir); - } - - throw new IllegalArgumentException("Unknown state " + state); + return (Seq) CollectionConverters.asScala(Arrays.asList(col)).toSeq(); } @SuppressWarnings("unchecked") From cffd68186e852662ba2a72ac1913d460fba39c0a Mon Sep 17 00:00:00 2001 From: nizhikov Date: Fri, 8 Sep 2023 17:15:20 +0300 Subject: [PATCH 45/78] KAFKA-14595 Improving test --- .../reassign/ReassignPartitionsUnitTest.java | 55 +++++++------------ 1 file changed, 21 insertions(+), 34 deletions(-) diff --git a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java index 7a7611005dd00..0f71dd374837f 100644 --- a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java +++ b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java @@ -48,7 +48,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -123,11 +122,11 @@ public void testPartitionReassignStatesToString() { Map states = new HashMap<>(); states.put(new TopicPartition("foo", 0), - asScala(new PartitionReassignmentState(Arrays.asList(1, 2, 3), Arrays.asList(1, 2, 3), true))); + new ReassignPartitionsCommand.PartitionReassignmentState(seq(1, 2, 3), seq(1, 2, 3), true)); states.put(new TopicPartition("foo", 1), - asScala(new PartitionReassignmentState(Arrays.asList(1, 2, 3), Arrays.asList(1, 2, 4), false))); + new ReassignPartitionsCommand.PartitionReassignmentState(seq(1, 2, 3), seq(1, 2, 4), false)); states.put(new TopicPartition("bar", 0), - asScala(new PartitionReassignmentState(Arrays.asList(1, 2, 3), Arrays.asList(1, 2, 4), false))); + new ReassignPartitionsCommand.PartitionReassignmentState(seq(1, 2, 3), seq(1, 2, 4), false)); assertEquals(String.join(System.lineSeparator(), Arrays.asList( "Status of partition reassignment:", @@ -171,8 +170,10 @@ public void testFindPartitionReassignmentStates() { Map partitionReassignmentStates = new HashMap<>(); - partitionReassignmentStates.put(new TopicPartition("foo", 0), asScala(new PartitionReassignmentState(Arrays.asList(0, 1, 2), Arrays.asList(0, 1, 3), false))); - partitionReassignmentStates.put(new TopicPartition("foo", 1), asScala(new PartitionReassignmentState(Arrays.asList(1, 2, 3), Arrays.asList(1, 2, 3), true))); + partitionReassignmentStates.put(new TopicPartition("foo", 0), + new ReassignPartitionsCommand.PartitionReassignmentState(seq(0, 1, 2), seq(0, 1, 3), false)); + partitionReassignmentStates.put(new TopicPartition("foo", 1), + new ReassignPartitionsCommand.PartitionReassignmentState(seq(1, 2, 3), seq(1, 2, 3), true)); Tuple2, Object> actual = findPartitionReassignmentStates(adminClient, CollectionConverters.asScala(Arrays.asList( @@ -191,8 +192,10 @@ public void testFindPartitionReassignmentStates() { partitionReassignmentStates.clear(); - partitionReassignmentStates.put(new TopicPartition("foo", 0), asScala(new PartitionReassignmentState(Arrays.asList(0, 1, 2), Arrays.asList(0, 1, 3), true))); - partitionReassignmentStates.put(new TopicPartition("foo", 1), asScala(new PartitionReassignmentState(Arrays.asList(1, 2, 3), Arrays.asList(1, 2, 3), true))); + partitionReassignmentStates.put(new TopicPartition("foo", 0), + new ReassignPartitionsCommand.PartitionReassignmentState(seq(0, 1, 2), seq(0, 1, 3), true)); + partitionReassignmentStates.put(new TopicPartition("foo", 1), + new ReassignPartitionsCommand.PartitionReassignmentState(seq(1, 2, 3), seq(1, 2, 3), true)); actual = findPartitionReassignmentStates(adminClient, CollectionConverters.asScala(Arrays.asList( new Tuple2<>(new TopicPartition("foo", 0), seq(0, 1, 3)), @@ -514,16 +517,16 @@ public void testMoveMap() { Map fooMoves = new HashMap<>(); - fooMoves.put(0, asScala(new PartitionMove(jset(1, 2, 3), jset(5)))); - fooMoves.put(1, asScala(new PartitionMove(jset(4, 5, 6), jset(7, 8)))); - fooMoves.put(2, asScala(new PartitionMove(jset(1, 2), jset(3, 4)))); - fooMoves.put(3, asScala(new PartitionMove(jset(1, 2), jset(5, 6)))); - fooMoves.put(4, asScala(new PartitionMove(jset(1, 2), jset(3)))); - fooMoves.put(5, asScala(new PartitionMove(jset(1, 2), jset(3, 4, 5, 6)))); + fooMoves.put(0, new ReassignPartitionsCommand.PartitionMove(mset(1, 2, 3), mset(5))); + fooMoves.put(1, new ReassignPartitionsCommand.PartitionMove(mset(4, 5, 6), mset(7, 8))); + fooMoves.put(2, new ReassignPartitionsCommand.PartitionMove(mset(1, 2), mset(3, 4))); + fooMoves.put(3, new ReassignPartitionsCommand.PartitionMove(mset(1, 2), mset(5, 6))); + fooMoves.put(4, new ReassignPartitionsCommand.PartitionMove(mset(1, 2), mset(3))); + fooMoves.put(5, new ReassignPartitionsCommand.PartitionMove(mset(1, 2), mset(3, 4, 5, 6))); Map barMoves = new HashMap<>(); - barMoves.put(0, asScala(new PartitionMove(jset(2, 3, 4), jset(1)))); + barMoves.put(0, new ReassignPartitionsCommand.PartitionMove(mset(2, 3, 4), mset(1))); assertEquals(CollectionConverters.asScala(fooMoves), moveMap.get("foo").get()); assertEquals(CollectionConverters.asScala(barMoves), moveMap.get("bar").get()); @@ -773,32 +776,16 @@ public void testPropagateInvalidJsonError() { @SuppressWarnings("unchecked") private scala.collection.immutable.Set set(final T... set) { - return CollectionConverters.asScala(jset(set)).toSet(); + return mset(set).toSet(); } @SuppressWarnings("unchecked") - private Set jset(final T...set) { - return new HashSet<>(Arrays.asList(set)); + private scala.collection.mutable.Set mset(final T...set) { + return CollectionConverters.asScala(new HashSet<>(Arrays.asList(set))); } @SuppressWarnings("unchecked") private Seq seq(T... col) { return (Seq) CollectionConverters.asScala(Arrays.asList(col)).toSeq(); } - - @SuppressWarnings("unchecked") - private ReassignPartitionsCommand.PartitionMove asScala(PartitionMove pm) { - return new ReassignPartitionsCommand.PartitionMove( - CollectionConverters.asScala((Set) pm.sources), - CollectionConverters.asScala((Set) pm.destinations) - ); - } - - @SuppressWarnings("unchecked") - private ReassignPartitionsCommand.PartitionReassignmentState asScala(PartitionReassignmentState state) { - return new ReassignPartitionsCommand.PartitionReassignmentState( - CollectionConverters.asScala((List) state.currentReplicas).toSeq(), - CollectionConverters.asScala((List) state.targetReplicas).toSeq(), - state.done); - } } From 79c4f6f57744e2d51633b995826731033898a38b Mon Sep 17 00:00:00 2001 From: nizhikov Date: Fri, 8 Sep 2023 17:33:24 +0300 Subject: [PATCH 46/78] KAFKA-14595 Improving test --- .../reassign/ReassignPartitionsUnitTest.java | 87 ++++++++++--------- 1 file changed, 44 insertions(+), 43 deletions(-) diff --git a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java index 0f71dd374837f..67e5993c9f169 100644 --- a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java +++ b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java @@ -128,11 +128,11 @@ public void testPartitionReassignStatesToString() { states.put(new TopicPartition("bar", 0), new ReassignPartitionsCommand.PartitionReassignmentState(seq(1, 2, 3), seq(1, 2, 4), false)); - assertEquals(String.join(System.lineSeparator(), Arrays.asList( - "Status of partition reassignment:", - "Reassignment of partition bar-0 is still in progress.", - "Reassignment of partition foo-0 is completed.", - "Reassignment of partition foo-1 is still in progress.")), + assertEquals( + "Status of partition reassignment:\n" + + "Reassignment of partition bar-0 is still in progress.\n" + + "Reassignment of partition foo-0 is completed.\n" + + "Reassignment of partition foo-1 is still in progress.", partitionReassignmentStatesToString(CollectionConverters.asScala(states))); } @@ -163,16 +163,17 @@ public void testFindPartitionReassignmentStates() { reassignments.put(new TopicPartition("foo", 0), seq(0, 1, 3)); reassignments.put(new TopicPartition("quux", 0), seq(1, 2, 3)); - scala.collection.Map reassignmentResult = alterPartitionReassignments(adminClient, CollectionConverters.asScala(reassignments)); + scala.collection.Map reassignmentResult = alterPartitionReassignments(adminClient, + CollectionConverters.asScala(reassignments)); assertEquals(1, reassignmentResult.size()); assertEquals(UnknownTopicOrPartitionException.class, reassignmentResult.get(new TopicPartition("quux", 0)).get().getClass()); - Map partitionReassignmentStates = new HashMap<>(); + Map expStates = new HashMap<>(); - partitionReassignmentStates.put(new TopicPartition("foo", 0), + expStates.put(new TopicPartition("foo", 0), new ReassignPartitionsCommand.PartitionReassignmentState(seq(0, 1, 2), seq(0, 1, 3), false)); - partitionReassignmentStates.put(new TopicPartition("foo", 1), + expStates.put(new TopicPartition("foo", 1), new ReassignPartitionsCommand.PartitionReassignmentState(seq(1, 2, 3), seq(1, 2, 3), true)); Tuple2, Object> actual = @@ -181,20 +182,21 @@ public void testFindPartitionReassignmentStates() { new Tuple2<>(new TopicPartition("foo", 1), seq(1, 2, 3)) ))); - assertEquals(CollectionConverters.asScala(partitionReassignmentStates), actual._1); + assertEquals(CollectionConverters.asScala(expStates), actual._1); assertEquals(true, actual._2); // Cancel the reassignment and test findPartitionReassignmentStates again. - scala.collection.Map cancelResult = cancelPartitionReassignments(adminClient, set(new TopicPartition("foo", 0), new TopicPartition("quux", 2))); + scala.collection.Map cancelResult = cancelPartitionReassignments(adminClient, + set(new TopicPartition("foo", 0), new TopicPartition("quux", 2))); assertEquals(1, cancelResult.size()); assertEquals(UnknownTopicOrPartitionException.class, cancelResult.get(new TopicPartition("quux", 2)).get().getClass()); - partitionReassignmentStates.clear(); + expStates.clear(); - partitionReassignmentStates.put(new TopicPartition("foo", 0), + expStates.put(new TopicPartition("foo", 0), new ReassignPartitionsCommand.PartitionReassignmentState(seq(0, 1, 2), seq(0, 1, 3), true)); - partitionReassignmentStates.put(new TopicPartition("foo", 1), + expStates.put(new TopicPartition("foo", 1), new ReassignPartitionsCommand.PartitionReassignmentState(seq(1, 2, 3), seq(1, 2, 3), true)); actual = findPartitionReassignmentStates(adminClient, CollectionConverters.asScala(Arrays.asList( @@ -202,7 +204,7 @@ public void testFindPartitionReassignmentStates() { new Tuple2<>(new TopicPartition("foo", 1), seq(1, 2, 3)) ))); - assertEquals(CollectionConverters.asScala(partitionReassignmentStates), actual._1); + assertEquals(CollectionConverters.asScala(expStates), actual._1); assertEquals(false, actual._2); } } @@ -233,15 +235,15 @@ public void testFindLogDirMoveStates() throws Exception { adminClient.alterReplicaLogDirs(replicaAssignment).all().get(); - Map expLogDirMoveStates = new HashMap<>(); + Map states = new HashMap<>(); - expLogDirMoveStates.put(new TopicPartitionReplica("bar", 0, 0), new ReassignPartitionsCommand.CompletedMoveState("/tmp/kafka-logs0")); - expLogDirMoveStates.put(new TopicPartitionReplica("foo", 0, 0), new ReassignPartitionsCommand.ActiveMoveState("/tmp/kafka-logs0", + states.put(new TopicPartitionReplica("bar", 0, 0), new ReassignPartitionsCommand.CompletedMoveState("/tmp/kafka-logs0")); + states.put(new TopicPartitionReplica("foo", 0, 0), new ReassignPartitionsCommand.ActiveMoveState("/tmp/kafka-logs0", "/tmp/kafka-logs1", "/tmp/kafka-logs1")); - expLogDirMoveStates.put(new TopicPartitionReplica("foo", 1, 0), new ReassignPartitionsCommand.CancelledMoveState("/tmp/kafka-logs0", + states.put(new TopicPartitionReplica("foo", 1, 0), new ReassignPartitionsCommand.CancelledMoveState("/tmp/kafka-logs0", "/tmp/kafka-logs1")); - expLogDirMoveStates.put(new TopicPartitionReplica("quux", 1, 0), new ReassignPartitionsCommand.MissingLogDirMoveState("/tmp/kafka-logs1")); - expLogDirMoveStates.put(new TopicPartitionReplica("quuz", 0, 0), new ReassignPartitionsCommand.MissingReplicaMoveState("/tmp/kafka-logs0")); + states.put(new TopicPartitionReplica("quux", 1, 0), new ReassignPartitionsCommand.MissingLogDirMoveState("/tmp/kafka-logs1")); + states.put(new TopicPartitionReplica("quuz", 0, 0), new ReassignPartitionsCommand.MissingReplicaMoveState("/tmp/kafka-logs0")); Map targetMoves = new HashMap<>(); @@ -252,7 +254,7 @@ public void testFindLogDirMoveStates() throws Exception { targetMoves.put(new TopicPartitionReplica("quuz", 0, 0), "/tmp/kafka-logs0"); assertEquals( - CollectionConverters.asScala(expLogDirMoveStates), + CollectionConverters.asScala(states), findLogDirMoveStates(adminClient, CollectionConverters.asScala(targetMoves)) ); } @@ -272,14 +274,14 @@ public void testReplicaMoveStatesToString() { "/tmp/kafka-logs1", "/tmp/kafka-logs2")); states.put(new TopicPartitionReplica("quux", 2, 1), new ReassignPartitionsCommand.MissingLogDirMoveState("/tmp/kafka-logs1")); - assertEquals(String.join(System.lineSeparator(), Arrays.asList( - "Reassignment of replica bar-0-0 completed successfully.", - "Reassignment of replica foo-0-0 is still in progress.", - "Partition foo-1 on broker 0 is not being moved from log dir /tmp/kafka-logs0 to /tmp/kafka-logs1.", - "Partition quux-0 cannot be found in any live log directory on broker 0.", - "Partition quux-1 on broker 1 is being moved to log dir /tmp/kafka-logs2 instead of /tmp/kafka-logs1.", + assertEquals( + "Reassignment of replica bar-0-0 completed successfully.\n" + + "Reassignment of replica foo-0-0 is still in progress.\n" + + "Partition foo-1 on broker 0 is not being moved from log dir /tmp/kafka-logs0 to /tmp/kafka-logs1.\n" + + "Partition quux-0 cannot be found in any live log directory on broker 0.\n" + + "Partition quux-1 on broker 1 is being moved to log dir /tmp/kafka-logs2 instead of /tmp/kafka-logs1.\n" + "Partition quux-2 is not found in any live log dir on broker 1. " + - "There is likely an offline log directory on the broker.")), + "There is likely an offline log directory on the broker.", replicaMoveStatesToString(CollectionConverters.asScala(states))); } @@ -447,16 +449,15 @@ public void testCurrentPartitionReplicaAssignmentToString() { currentParts.put(new TopicPartition("bar", 0), seq(7, 8)); currentParts.put(new TopicPartition("baz", 0), seq(10, 11, 12)); - assertEquals(String.join(System.lineSeparator(), Arrays.asList( - "Current partition replica assignment", - "", - "{\"version\":1,\"partitions\":" + - "[{\"topic\":\"bar\",\"partition\":0,\"replicas\":[7,8],\"log_dirs\":[\"any\",\"any\"]}," + - "{\"topic\":\"foo\",\"partition\":1,\"replicas\":[4,5,6],\"log_dirs\":[\"any\",\"any\",\"any\"]}]" + - "}", - "", - "Save this to use as the --reassignment-json-file option during rollback" - )), + assertEquals( + "Current partition replica assignment\n" + + "\n" + + "{\"version\":1,\"partitions\":" + + "[{\"topic\":\"bar\",\"partition\":0,\"replicas\":[7,8],\"log_dirs\":[\"any\",\"any\"]}," + + "{\"topic\":\"foo\",\"partition\":1,\"replicas\":[4,5,6],\"log_dirs\":[\"any\",\"any\",\"any\"]}]" + + "}\n" + + "\n" + + "Save this to use as the --reassignment-json-file option during rollback", currentPartitionReplicaAssignmentToString( CollectionConverters.asScala(proposedParts), CollectionConverters.asScala(currentParts) @@ -682,10 +683,10 @@ public void testCurReassignmentsToString() { alterPartitionReassignments(adminClient, CollectionConverters.asScala(reassignments)); assertTrue(reassignmentResult.isEmpty()); - assertEquals(String.join(System.lineSeparator(), Arrays.asList("Current partition reassignments:", - "bar-0: replicas: 2,3,0. removing: 0.", - "foo-0: replicas: 0,1,2. adding: 4.", - "foo-1: replicas: 1,2,3. adding: 4,5. removing: 1,2.")), + assertEquals("Current partition reassignments:\n" + + "bar-0: replicas: 2,3,0. removing: 0.\n" + + "foo-0: replicas: 0,1,2. adding: 4.\n" + + "foo-1: replicas: 1,2,3. adding: 4,5. removing: 1,2.", curReassignmentsToString(adminClient)); } } From 840fa622534edc7a6ab6eba5ff43e8caaad63f8e Mon Sep 17 00:00:00 2001 From: nizhikov Date: Fri, 8 Sep 2023 18:38:53 +0300 Subject: [PATCH 47/78] KAFKA-14595 Improving test --- .../reassign/ReassignPartitionsUnitTest.java | 102 +++++++++--------- 1 file changed, 52 insertions(+), 50 deletions(-) diff --git a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java index 67e5993c9f169..7bf7d0d643f36 100644 --- a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java +++ b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java @@ -38,13 +38,11 @@ import org.junit.jupiter.api.Timeout; import scala.Tuple2; import scala.collection.Seq; -import scala.jdk.javaapi.CollectionConverters; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -133,7 +131,7 @@ public void testPartitionReassignStatesToString() { "Reassignment of partition bar-0 is still in progress.\n" + "Reassignment of partition foo-0 is completed.\n" + "Reassignment of partition foo-1 is still in progress.", - partitionReassignmentStatesToString(CollectionConverters.asScala(states))); + partitionReassignmentStatesToString(asScala(states))); } private void addTopics(MockAdminClient adminClient) { @@ -153,6 +151,7 @@ private void addTopics(MockAdminClient adminClient) { ), Collections.emptyMap()); } + @SuppressWarnings("unchecked") @Test public void testFindPartitionReassignmentStates() { try (MockAdminClient adminClient = new MockAdminClient.Builder().numBrokers(4).build()) { @@ -164,7 +163,7 @@ public void testFindPartitionReassignmentStates() { reassignments.put(new TopicPartition("quux", 0), seq(1, 2, 3)); scala.collection.Map reassignmentResult = alterPartitionReassignments(adminClient, - CollectionConverters.asScala(reassignments)); + asScala(reassignments)); assertEquals(1, reassignmentResult.size()); assertEquals(UnknownTopicOrPartitionException.class, reassignmentResult.get(new TopicPartition("quux", 0)).get().getClass()); @@ -177,12 +176,12 @@ public void testFindPartitionReassignmentStates() { new ReassignPartitionsCommand.PartitionReassignmentState(seq(1, 2, 3), seq(1, 2, 3), true)); Tuple2, Object> actual = - findPartitionReassignmentStates(adminClient, CollectionConverters.asScala(Arrays.asList( + findPartitionReassignmentStates(adminClient, seq( new Tuple2<>(new TopicPartition("foo", 0), seq(0, 1, 3)), new Tuple2<>(new TopicPartition("foo", 1), seq(1, 2, 3)) - ))); + )); - assertEquals(CollectionConverters.asScala(expStates), actual._1); + assertEquals(asScala(expStates), actual._1); assertEquals(true, actual._2); // Cancel the reassignment and test findPartitionReassignmentStates again. @@ -199,12 +198,12 @@ public void testFindPartitionReassignmentStates() { expStates.put(new TopicPartition("foo", 1), new ReassignPartitionsCommand.PartitionReassignmentState(seq(1, 2, 3), seq(1, 2, 3), true)); - actual = findPartitionReassignmentStates(adminClient, CollectionConverters.asScala(Arrays.asList( + actual = findPartitionReassignmentStates(adminClient, seq( new Tuple2<>(new TopicPartition("foo", 0), seq(0, 1, 3)), new Tuple2<>(new TopicPartition("foo", 1), seq(1, 2, 3)) - ))); + )); - assertEquals(CollectionConverters.asScala(expStates), actual._1); + assertEquals(asScala(expStates), actual._1); assertEquals(false, actual._2); } } @@ -253,10 +252,7 @@ public void testFindLogDirMoveStates() throws Exception { targetMoves.put(new TopicPartitionReplica("quux", 1, 0), "/tmp/kafka-logs1"); targetMoves.put(new TopicPartitionReplica("quuz", 0, 0), "/tmp/kafka-logs0"); - assertEquals( - CollectionConverters.asScala(states), - findLogDirMoveStates(adminClient, CollectionConverters.asScala(targetMoves)) - ); + assertEquals(asScala(states), findLogDirMoveStates(adminClient, asScala(targetMoves))); } } @@ -282,7 +278,7 @@ public void testReplicaMoveStatesToString() { "Partition quux-1 on broker 1 is being moved to log dir /tmp/kafka-logs2 instead of /tmp/kafka-logs1.\n" + "Partition quux-2 is not found in any live log dir on broker 1. " + "There is likely an offline log directory on the broker.", - replicaMoveStatesToString(CollectionConverters.asScala(states))); + replicaMoveStatesToString(asScala(states))); } @Test @@ -295,15 +291,14 @@ public void testGetReplicaAssignments() { assignments.put(new TopicPartition("foo", 0), seq(0, 1, 2)); assignments.put(new TopicPartition("foo", 1), seq(1, 2, 3)); - assertEquals(CollectionConverters.asScala(assignments), - getReplicaAssignmentForTopics(adminClient, CollectionConverters.asScala(Arrays.asList("foo")))); + assertEquals(asScala(assignments), getReplicaAssignmentForTopics(adminClient, seq("foo"))); assignments.clear(); assignments.put(new TopicPartition("foo", 0), seq(0, 1, 2)); assignments.put(new TopicPartition("bar", 0), seq(2, 3, 0)); - assertEquals(CollectionConverters.asScala(assignments), + assertEquals(asScala(assignments), getReplicaAssignmentForPartitions(adminClient, set(new TopicPartition("foo", 0), new TopicPartition("bar", 0)))); } } @@ -316,21 +311,21 @@ public void testGetBrokerRackInformation() { new Node(2, "localhost", 9094, null))). build()) { - assertEquals(CollectionConverters.asScala(Arrays.asList( + assertEquals(seq( new BrokerMetadata(0, Optional.of("rack0")), new BrokerMetadata(1, Optional.of("rack1")) - )), getBrokerMetadata(adminClient, CollectionConverters.asScala(Arrays.asList(0, 1)), true)); - assertEquals(CollectionConverters.asScala(Arrays.asList( + ), getBrokerMetadata(adminClient, seq(0, 1), true)); + assertEquals(seq( new BrokerMetadata(0, Optional.empty()), new BrokerMetadata(1, Optional.empty()) - )), getBrokerMetadata(adminClient, CollectionConverters.asScala(Arrays.asList(0, 1)), false)); + ), getBrokerMetadata(adminClient, seq(0, 1), false)); assertStartsWith("Not all brokers have rack information", assertThrows(AdminOperationException.class, - () -> getBrokerMetadata(adminClient, CollectionConverters.asScala(Arrays.asList(1, 2)), true)).getMessage()); - assertEquals(CollectionConverters.asScala(Arrays.asList( + () -> getBrokerMetadata(adminClient, seq(1, 2), true)).getMessage()); + assertEquals(seq( new BrokerMetadata(1, Optional.empty()), new BrokerMetadata(2, Optional.empty()) - )), getBrokerMetadata(adminClient, CollectionConverters.asScala(Arrays.asList(1, 2)), false)); + ), getBrokerMetadata(adminClient, seq(1, 2), false)); } } @@ -403,7 +398,7 @@ public void testGenerateAssignmentWithInconsistentRacks() { expCurrent.put(new TopicPartition("foo", 0), seq(0, 1, 2)); expCurrent.put(new TopicPartition("foo", 1), seq(1, 2, 3)); - assertEquals(CollectionConverters.asScala(expCurrent), proposedCurrent._2()); + assertEquals(asScala(expCurrent), proposedCurrent._2()); } } @@ -424,7 +419,7 @@ public void testGenerateAssignmentWithFewerBrokers() { expCurrent.put(new TopicPartition("foo", 1), seq(1, 2, 3)); expCurrent.put(new TopicPartition("bar", 0), seq(2, 3, 0)); - assertEquals(CollectionConverters.asScala(expCurrent), proposedCurrent._2()); + assertEquals(asScala(expCurrent), proposedCurrent._2()); // The proposed assignment should only span the provided brokers proposedCurrent._1().values().foreach(replicas -> { @@ -458,10 +453,7 @@ public void testCurrentPartitionReplicaAssignmentToString() { "}\n" + "\n" + "Save this to use as the --reassignment-json-file option during rollback", - currentPartitionReplicaAssignmentToString( - CollectionConverters.asScala(proposedParts), - CollectionConverters.asScala(currentParts) - ) + currentPartitionReplicaAssignmentToString(asScala(proposedParts), asScala(currentParts)) ); } @@ -510,11 +502,7 @@ public void testMoveMap() { currentParts.put(new TopicPartition("baz", 0), seq(1, 2, 3)); scala.collection.mutable.Map> - moveMap = calculateProposedMoveMap( - CollectionConverters.asScala(currentReassignments), - CollectionConverters.asScala(proposedParts), - CollectionConverters.asScala(currentParts) - ); + moveMap = calculateProposedMoveMap(asScala(currentReassignments), asScala(proposedParts), asScala(currentParts)); Map fooMoves = new HashMap<>(); @@ -529,22 +517,22 @@ public void testMoveMap() { barMoves.put(0, new ReassignPartitionsCommand.PartitionMove(mset(2, 3, 4), mset(1))); - assertEquals(CollectionConverters.asScala(fooMoves), moveMap.get("foo").get()); - assertEquals(CollectionConverters.asScala(barMoves), moveMap.get("bar").get()); + assertEquals(asScala(fooMoves), moveMap.get("foo").get()); + assertEquals(asScala(barMoves), moveMap.get("bar").get()); Map expLeaderThrottle = new HashMap<>(); expLeaderThrottle.put("foo", "0:1,0:2,0:3,1:4,1:5,1:6,2:1,2:2,3:1,3:2,4:1,4:2,5:1,5:2"); expLeaderThrottle.put("bar", "0:2,0:3,0:4"); - assertEquals(CollectionConverters.asScala(expLeaderThrottle), calculateLeaderThrottles(moveMap)); + assertEquals(asScala(expLeaderThrottle), calculateLeaderThrottles(moveMap)); Map expFollowerThrottle = new HashMap<>(); expFollowerThrottle.put("foo", "0:5,1:7,1:8,2:3,2:4,3:5,3:6,4:3,5:3,5:4,5:5,5:6"); expFollowerThrottle.put("bar", "0:1"); - assertEquals(CollectionConverters.asScala(expFollowerThrottle), calculateFollowerThrottles(moveMap)); + assertEquals(asScala(expFollowerThrottle), calculateFollowerThrottles(moveMap)); assertEquals(set(1, 2, 3, 4, 5, 6, 7, 8), calculateReassigningBrokers(moveMap)); assertEquals(set(0, 2), calculateMovingBrokers(set( @@ -588,7 +576,7 @@ public void testParseExecuteAssignmentArgs() { "{\"topic\":\"foo\",\"partition\":1,\"replicas\":[3,4,5],\"log_dirs\":[\"any\",\"any\",\"any\"]}" + "]}"); - assertEquals(CollectionConverters.asScala(partitionsToBeReassigned), actual._1); + assertEquals(asScala(partitionsToBeReassigned), actual._1); assertTrue(actual._2.isEmpty()); Map replicaAssignment = new HashMap<>(); @@ -602,8 +590,8 @@ public void testParseExecuteAssignmentArgs() { "[{\"topic\":\"foo\",\"partition\":0,\"replicas\":[1,2,3],\"log_dirs\":[\"/tmp/a\",\"/tmp/b\",\"/tmp/c\"]}" + "]}"); - assertEquals(CollectionConverters.asScala(Collections.singletonMap(new TopicPartition("foo", 0), seq(1, 2, 3))), actual._1); - assertEquals(CollectionConverters.asScala(replicaAssignment), actual._2); + assertEquals(asScala(Collections.singletonMap(new TopicPartition("foo", 0), seq(1, 2, 3))), actual._1); + assertEquals(asScala(replicaAssignment), actual._2); } @Test @@ -680,7 +668,7 @@ public void testCurReassignmentsToString() { reassignments.put(new TopicPartition("bar", 0), seq(2, 3)); scala.collection.Map reassignmentResult = - alterPartitionReassignments(adminClient, CollectionConverters.asScala(reassignments)); + alterPartitionReassignments(adminClient, asScala(reassignments)); assertTrue(reassignmentResult.isEmpty()); assertEquals("Current partition reassignments:\n" + @@ -719,8 +707,8 @@ public void testModifyTopicThrottles() throws Exception { leaderThrottles.put("bar", "leaderBar"); modifyTopicThrottles(adminClient, - CollectionConverters.asScala(leaderThrottles), - CollectionConverters.asScala(Collections.singletonMap("bar", "followerBar"))); + asScala(leaderThrottles), + asScala(Collections.singletonMap("bar", "followerBar"))); List topics = Stream.of("bar", "foo").map( id -> new ConfigResource(ConfigResource.Type.TOPIC, id)).collect(Collectors.toList()); Map results = adminClient.describeConfigs(topics).all().get(); @@ -757,7 +745,7 @@ public void testAlterReplicaLogDirs() { assertEquals( set(new TopicPartitionReplica("foo", 0, 0)), - alterReplicaLogDirs(adminClient, CollectionConverters.asScala(assignment)) + alterReplicaLogDirs(adminClient, asScala(assignment)) ); } } @@ -782,11 +770,25 @@ private scala.collection.immutable.Set set(final T... set) { @SuppressWarnings("unchecked") private scala.collection.mutable.Set mset(final T...set) { - return CollectionConverters.asScala(new HashSet<>(Arrays.asList(set))); + scala.collection.mutable.Set res = new scala.collection.mutable.HashSet<>(); + for (T t : set) + res.add(t); + return res; + } + + @SuppressWarnings("unchecked") + private Seq seq(T... seq) { + scala.collection.mutable.Buffer res = new scala.collection.mutable.ArrayBuffer<>(); + for (T t : seq) + res.addOne(t); + return res.toSeq(); } @SuppressWarnings("unchecked") - private Seq seq(T... col) { - return (Seq) CollectionConverters.asScala(Arrays.asList(col)).toSeq(); + private scala.collection.Map asScala(Map jmap) { + scala.collection.immutable.Map res = new scala.collection.immutable.HashMap<>(); + for (Map.Entry e : jmap.entrySet()) + res = res.$plus(new Tuple2<>(e.getKey(), e.getValue())); + return res; } } From fcfb95ea372949daddfa2ff481fe2224bbe5e5ce Mon Sep 17 00:00:00 2001 From: nizhikov Date: Sat, 9 Sep 2023 12:49:52 +0300 Subject: [PATCH 48/78] KAFKA-14595 Improving test --- .../apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java index 7bf7d0d643f36..d31cff8e8ae31 100644 --- a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java +++ b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java @@ -780,7 +780,7 @@ private scala.collection.mutable.Set mset(final T...set) { private Seq seq(T... seq) { scala.collection.mutable.Buffer res = new scala.collection.mutable.ArrayBuffer<>(); for (T t : seq) - res.addOne(t); + res.$plus$eq(t); return res.toSeq(); } From d5a33fb6cddf8b83e8bce2c90e51edcea1616dfc Mon Sep 17 00:00:00 2001 From: nizhikov Date: Sat, 9 Sep 2023 19:14:30 +0300 Subject: [PATCH 49/78] KAFKA-14595 Fix scala 2.12 errors --- .../reassign/ReassignPartitionsUnitTest.java | 23 +++++++------------ 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java index d31cff8e8ae31..99674078e79d6 100644 --- a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java +++ b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java @@ -37,12 +37,14 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; import scala.Tuple2; +import scala.collection.JavaConverters; import scala.collection.Seq; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -768,27 +770,18 @@ private scala.collection.immutable.Set set(final T... set) { return mset(set).toSet(); } - @SuppressWarnings("unchecked") + @SuppressWarnings({"deprecation", "unchecked"}) private scala.collection.mutable.Set mset(final T...set) { - scala.collection.mutable.Set res = new scala.collection.mutable.HashSet<>(); - for (T t : set) - res.add(t); - return res; + return JavaConverters.asScalaSet(new HashSet<>(Arrays.asList(set))); } - @SuppressWarnings("unchecked") + @SuppressWarnings({"deprecation", "unchecked"}) private Seq seq(T... seq) { - scala.collection.mutable.Buffer res = new scala.collection.mutable.ArrayBuffer<>(); - for (T t : seq) - res.$plus$eq(t); - return res.toSeq(); + return JavaConverters.asScalaIteratorConverter(Arrays.asList(seq).iterator()).asScala().toSeq(); } - @SuppressWarnings("unchecked") + @SuppressWarnings("deprecation") private scala.collection.Map asScala(Map jmap) { - scala.collection.immutable.Map res = new scala.collection.immutable.HashMap<>(); - for (Map.Entry e : jmap.entrySet()) - res = res.$plus(new Tuple2<>(e.getKey(), e.getValue())); - return res; + return JavaConverters.mapAsScalaMap(jmap); } } From 420af7852e09bd3b2c0f0081d4c2b4d15484da70 Mon Sep 17 00:00:00 2001 From: nizhikov Date: Mon, 18 Sep 2023 13:18:12 +0300 Subject: [PATCH 50/78] KAFKA-14595 Code review fixes. --- .../reassign/ReassignPartitionsUnitTest.java | 42 +++++++++---------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java index 99674078e79d6..847c4e683208a 100644 --- a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java +++ b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java @@ -129,9 +129,9 @@ public void testPartitionReassignStatesToString() { new ReassignPartitionsCommand.PartitionReassignmentState(seq(1, 2, 3), seq(1, 2, 4), false)); assertEquals( - "Status of partition reassignment:\n" + - "Reassignment of partition bar-0 is still in progress.\n" + - "Reassignment of partition foo-0 is completed.\n" + + "Status of partition reassignment:" + System.lineSeparator() + + "Reassignment of partition bar-0 is still in progress." + System.lineSeparator() + + "Reassignment of partition foo-0 is completed." + System.lineSeparator() + "Reassignment of partition foo-1 is still in progress.", partitionReassignmentStatesToString(asScala(states))); } @@ -218,8 +218,8 @@ public void testFindLogDirMoveStates() throws Exception { Arrays.asList("/tmp/kafka-logs0", "/tmp/kafka-logs1"), Arrays.asList("/tmp/kafka-logs0", "/tmp/kafka-logs1"), Arrays.asList("/tmp/kafka-logs0", "/tmp/kafka-logs1"), - Arrays.asList("/tmp/kafka-logs0", null))). - build()) { + Arrays.asList("/tmp/kafka-logs0", null))) + .build()) { addTopics(adminClient); List b = adminClient.brokers(); @@ -273,11 +273,11 @@ public void testReplicaMoveStatesToString() { states.put(new TopicPartitionReplica("quux", 2, 1), new ReassignPartitionsCommand.MissingLogDirMoveState("/tmp/kafka-logs1")); assertEquals( - "Reassignment of replica bar-0-0 completed successfully.\n" + - "Reassignment of replica foo-0-0 is still in progress.\n" + - "Partition foo-1 on broker 0 is not being moved from log dir /tmp/kafka-logs0 to /tmp/kafka-logs1.\n" + - "Partition quux-0 cannot be found in any live log directory on broker 0.\n" + - "Partition quux-1 on broker 1 is being moved to log dir /tmp/kafka-logs2 instead of /tmp/kafka-logs1.\n" + + "Reassignment of replica bar-0-0 completed successfully." + System.lineSeparator() + + "Reassignment of replica foo-0-0 is still in progress." + System.lineSeparator() + + "Partition foo-1 on broker 0 is not being moved from log dir /tmp/kafka-logs0 to /tmp/kafka-logs1." + System.lineSeparator() + + "Partition quux-0 cannot be found in any live log directory on broker 0." + System.lineSeparator() + + "Partition quux-1 on broker 1 is being moved to log dir /tmp/kafka-logs2 instead of /tmp/kafka-logs1." + System.lineSeparator() + "Partition quux-2 is not found in any live log dir on broker 1. " + "There is likely an offline log directory on the broker.", replicaMoveStatesToString(asScala(states))); @@ -447,13 +447,13 @@ public void testCurrentPartitionReplicaAssignmentToString() { currentParts.put(new TopicPartition("baz", 0), seq(10, 11, 12)); assertEquals( - "Current partition replica assignment\n" + - "\n" + + "Current partition replica assignment" + System.lineSeparator() + + System.lineSeparator() + "{\"version\":1,\"partitions\":" + "[{\"topic\":\"bar\",\"partition\":0,\"replicas\":[7,8],\"log_dirs\":[\"any\",\"any\"]}," + "{\"topic\":\"foo\",\"partition\":1,\"replicas\":[4,5,6],\"log_dirs\":[\"any\",\"any\",\"any\"]}]" + - "}\n" + - "\n" + + "}" + System.lineSeparator() + + System.lineSeparator() + "Save this to use as the --reassignment-json-file option during rollback", currentPartitionReplicaAssignmentToString(asScala(proposedParts), asScala(currentParts)) ); @@ -673,9 +673,9 @@ public void testCurReassignmentsToString() { alterPartitionReassignments(adminClient, asScala(reassignments)); assertTrue(reassignmentResult.isEmpty()); - assertEquals("Current partition reassignments:\n" + - "bar-0: replicas: 2,3,0. removing: 0.\n" + - "foo-0: replicas: 0,1,2. adding: 4.\n" + + assertEquals("Current partition reassignments:" + System.lineSeparator() + + "bar-0: replicas: 2,3,0. removing: 0." + System.lineSeparator() + + "foo-0: replicas: 0,1,2. adding: 4." + System.lineSeparator() + "foo-1: replicas: 1,2,3. adding: 4,5. removing: 1,2.", curReassignmentsToString(adminClient)); } @@ -766,22 +766,22 @@ public void testPropagateInvalidJsonError() { } @SuppressWarnings("unchecked") - private scala.collection.immutable.Set set(final T... set) { + private static scala.collection.immutable.Set set(final T... set) { return mset(set).toSet(); } @SuppressWarnings({"deprecation", "unchecked"}) - private scala.collection.mutable.Set mset(final T...set) { + private static scala.collection.mutable.Set mset(final T...set) { return JavaConverters.asScalaSet(new HashSet<>(Arrays.asList(set))); } @SuppressWarnings({"deprecation", "unchecked"}) - private Seq seq(T... seq) { + private static Seq seq(T... seq) { return JavaConverters.asScalaIteratorConverter(Arrays.asList(seq).iterator()).asScala().toSeq(); } @SuppressWarnings("deprecation") - private scala.collection.Map asScala(Map jmap) { + private static scala.collection.Map asScala(Map jmap) { return JavaConverters.mapAsScalaMap(jmap); } } From 7d22f15ffad0ef705285e89de10cd4e3fb1ff1af Mon Sep 17 00:00:00 2001 From: nizhikov Date: Mon, 18 Sep 2023 13:33:35 +0300 Subject: [PATCH 51/78] KAFKA-14595 Code review fixes. --- .../reassign/ReassignPartitionsUnitTest.java | 45 ++++++++++--------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java index 847c4e683208a..0183c47f870e9 100644 --- a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java +++ b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java @@ -128,11 +128,11 @@ public void testPartitionReassignStatesToString() { states.put(new TopicPartition("bar", 0), new ReassignPartitionsCommand.PartitionReassignmentState(seq(1, 2, 3), seq(1, 2, 4), false)); - assertEquals( - "Status of partition reassignment:" + System.lineSeparator() + - "Reassignment of partition bar-0 is still in progress." + System.lineSeparator() + - "Reassignment of partition foo-0 is completed." + System.lineSeparator() + - "Reassignment of partition foo-1 is still in progress.", + assertEquals(String.join(System.lineSeparator(), + "Status of partition reassignment:", + "Reassignment of partition bar-0 is still in progress.", + "Reassignment of partition foo-0 is completed.", + "Reassignment of partition foo-1 is still in progress."), partitionReassignmentStatesToString(asScala(states))); } @@ -272,14 +272,14 @@ public void testReplicaMoveStatesToString() { "/tmp/kafka-logs1", "/tmp/kafka-logs2")); states.put(new TopicPartitionReplica("quux", 2, 1), new ReassignPartitionsCommand.MissingLogDirMoveState("/tmp/kafka-logs1")); - assertEquals( - "Reassignment of replica bar-0-0 completed successfully." + System.lineSeparator() + - "Reassignment of replica foo-0-0 is still in progress." + System.lineSeparator() + - "Partition foo-1 on broker 0 is not being moved from log dir /tmp/kafka-logs0 to /tmp/kafka-logs1." + System.lineSeparator() + - "Partition quux-0 cannot be found in any live log directory on broker 0." + System.lineSeparator() + - "Partition quux-1 on broker 1 is being moved to log dir /tmp/kafka-logs2 instead of /tmp/kafka-logs1." + System.lineSeparator() + + assertEquals(String.join(System.lineSeparator(), + "Reassignment of replica bar-0-0 completed successfully.", + "Reassignment of replica foo-0-0 is still in progress.", + "Partition foo-1 on broker 0 is not being moved from log dir /tmp/kafka-logs0 to /tmp/kafka-logs1.", + "Partition quux-0 cannot be found in any live log directory on broker 0.", + "Partition quux-1 on broker 1 is being moved to log dir /tmp/kafka-logs2 instead of /tmp/kafka-logs1.", "Partition quux-2 is not found in any live log dir on broker 1. " + - "There is likely an offline log directory on the broker.", + "There is likely an offline log directory on the broker."), replicaMoveStatesToString(asScala(states))); } @@ -446,15 +446,15 @@ public void testCurrentPartitionReplicaAssignmentToString() { currentParts.put(new TopicPartition("bar", 0), seq(7, 8)); currentParts.put(new TopicPartition("baz", 0), seq(10, 11, 12)); - assertEquals( - "Current partition replica assignment" + System.lineSeparator() + - System.lineSeparator() + + assertEquals(String.join(System.lineSeparator(), + "Current partition replica assignment", + "", "{\"version\":1,\"partitions\":" + "[{\"topic\":\"bar\",\"partition\":0,\"replicas\":[7,8],\"log_dirs\":[\"any\",\"any\"]}," + "{\"topic\":\"foo\",\"partition\":1,\"replicas\":[4,5,6],\"log_dirs\":[\"any\",\"any\",\"any\"]}]" + - "}" + System.lineSeparator() + - System.lineSeparator() + - "Save this to use as the --reassignment-json-file option during rollback", + "}", + "", + "Save this to use as the --reassignment-json-file option during rollback"), currentPartitionReplicaAssignmentToString(asScala(proposedParts), asScala(currentParts)) ); } @@ -673,10 +673,11 @@ public void testCurReassignmentsToString() { alterPartitionReassignments(adminClient, asScala(reassignments)); assertTrue(reassignmentResult.isEmpty()); - assertEquals("Current partition reassignments:" + System.lineSeparator() + - "bar-0: replicas: 2,3,0. removing: 0." + System.lineSeparator() + - "foo-0: replicas: 0,1,2. adding: 4." + System.lineSeparator() + - "foo-1: replicas: 1,2,3. adding: 4,5. removing: 1,2.", + assertEquals(String.join(System.lineSeparator(), + "Current partition reassignments:", + "bar-0: replicas: 2,3,0. removing: 0.", + "foo-0: replicas: 0,1,2. adding: 4.", + "foo-1: replicas: 1,2,3. adding: 4,5. removing: 1,2."), curReassignmentsToString(adminClient)); } } From bbd2f4c8fa77b98479a81933aba04507ab54b714 Mon Sep 17 00:00:00 2001 From: nizhikov Date: Mon, 18 Sep 2023 14:36:37 +0300 Subject: [PATCH 52/78] KAFKA-14595 Code review fixes. --- .../kafka/tools/reassign/ReassignPartitionsUnitTest.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java index 0183c47f870e9..cdc11cb202232 100644 --- a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java +++ b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java @@ -81,6 +81,8 @@ import static kafka.admin.ReassignPartitionsCommand.parseGenerateAssignmentArgs; import static kafka.admin.ReassignPartitionsCommand.partitionReassignmentStatesToString; import static kafka.admin.ReassignPartitionsCommand.replicaMoveStatesToString; +import static kafka.admin.ReassignPartitionsCommand.topicLevelFollowerThrottle; +import static kafka.admin.ReassignPartitionsCommand.topicLevelLeaderThrottle; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -726,9 +728,9 @@ private void verifyTopicThrottleResults(Config config, Map configs = new HashMap<>(); config.entries().forEach(entry -> configs.put(entry.name(), entry.value())); assertEquals(expectedLeaderThrottle, - configs.getOrDefault(ReassignPartitionsCommand.topicLevelLeaderThrottle(), "")); + configs.getOrDefault(topicLevelLeaderThrottle(), "")); assertEquals(expectedFollowerThrottle, - configs.getOrDefault(ReassignPartitionsCommand.topicLevelFollowerThrottle(), "")); + configs.getOrDefault(topicLevelFollowerThrottle(), "")); } @Test From 25eae9ad2f9ddf4bf8aad01515c4cceaa4990358 Mon Sep 17 00:00:00 2001 From: nizhikov Date: Tue, 19 Sep 2023 12:36:28 +0300 Subject: [PATCH 53/78] KAFKA-14595 Code review fixes. --- checkstyle/checkstyle.xml | 2 +- checkstyle/import-control.xml | 2 +- .../reassign/ReassignPartitionsCommand.java | 115 +++++++++--------- 3 files changed, 60 insertions(+), 59 deletions(-) diff --git a/checkstyle/checkstyle.xml b/checkstyle/checkstyle.xml index f79622009a79f..bf2d339da8c76 100644 --- a/checkstyle/checkstyle.xml +++ b/checkstyle/checkstyle.xml @@ -119,7 +119,7 @@ - + diff --git a/checkstyle/import-control.xml b/checkstyle/import-control.xml index 8845f3ce82afc..2944e7eab1606 100644 --- a/checkstyle/import-control.xml +++ b/checkstyle/import-control.xml @@ -294,7 +294,7 @@ - + diff --git a/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommand.java b/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommand.java index 535503de4b8da..2d791780bc9a8 100644 --- a/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommand.java +++ b/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommand.java @@ -69,6 +69,7 @@ import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; +@SuppressWarnings("ClassDataAbstractionCoupling") public class ReassignPartitionsCommand { private static final String ANY_LOG_DIR = "any"; @@ -242,7 +243,7 @@ private static Tuple, Boolean> v Map partStates = t0.v1; Boolean partsOngoing = t0.v2; - System.out.print(partitionReassignmentStatesToString(partStates)); + System.out.println(partitionReassignmentStatesToString(partStates)); return new Tuple<>(partStates, partsOngoing); } @@ -411,8 +412,8 @@ private static Tuple, Boolean> verif */ static Map findLogDirMoveStates(Admin adminClient, Map targetMoves) throws ExecutionException, InterruptedException { - Map replicaLogDirInfos = adminClient. - describeReplicaLogDirs(targetMoves.keySet()).all().get(); + Map replicaLogDirInfos = adminClient + .describeReplicaLogDirs(targetMoves.keySet()).all().get(); Map res = new HashMap<>(); @@ -500,12 +501,12 @@ private static void clearAllThrottles(Admin adminClient, List brokers.addAll(t.v2)); System.out.printf("Clearing broker-level throttles on broker%s %s%n", - brokers.size() == 1 ? "" : "s", brokers.stream().map(Object::toString).collect(Collectors.joining(","))); + brokers.size() == 1 ? "" : "s", Utils.join(brokers, ",")); clearBrokerLevelThrottles(adminClient, brokers); Set topics = targetParts.stream().map(t -> t.v1.topic()).collect(Collectors.toSet()); System.out.printf("Clearing topic-level throttles on topic%s %s%n", - topics.size() == 1 ? "" : "s", topics.stream().map(Object::toString).collect(Collectors.joining(","))); + topics.size() == 1 ? "" : "s", Utils.join(topics, ",")); clearTopicLevelThrottles(adminClient, topics); } @@ -567,7 +568,7 @@ static Tuple>, Map> proposedAssignments = calculateAssignment(currentAssignments, brokerMetadatas); System.out.printf("Current partition replica assignment%n%s%n%n", formatAsReassignmentJson(currentAssignments, Collections.emptyMap())); - System.out.printf("Proposed partition reassignment configuration%n%s", + System.out.printf("Proposed partition reassignment configuration%n%s%n", formatAsReassignmentJson(proposedAssignments, Collections.emptyMap())); return new Tuple<>(proposedAssignments, currentAssignments); } @@ -585,7 +586,7 @@ private static Map> calculateAssignment(Map>>> groupedByTopic = new HashMap<>(); for (Map.Entry> e : currentAssignment.entrySet()) groupedByTopic.computeIfAbsent(e.getKey().topic(), k -> new ArrayList<>()).add(e); - HashMap> proposedAssignments = new HashMap<>(); + Map> proposedAssignments = new HashMap<>(); groupedByTopic.forEach((topic, assignment) -> { List replicas = assignment.get(0).getValue(); Map> assignedReplicas = AdminUtils. @@ -597,7 +598,7 @@ private static Map> calculateAssignment(Map describeTopics(Admin adminClient, - Set topics) throws ExecutionException, InterruptedException { + Set topics) throws ExecutionException, InterruptedException { Map> futures = adminClient.describeTopics(topics).topicNameValues(); Map res = new HashMap<>(); for (Map.Entry> e : futures.entrySet()) { @@ -645,7 +646,7 @@ static Map> getReplicaAssignmentForTopics(Admin ad * If any topic can't be found, an exception will be thrown. */ static Map> getReplicaAssignmentForPartitions(Admin adminClient, - Set partitions) throws ExecutionException, InterruptedException { + Set partitions) throws ExecutionException, InterruptedException { Map> res = new HashMap<>(); describeTopics(adminClient, partitions.stream().map(TopicPartition::topic).collect(Collectors.toSet())).forEach((topicName, topicDescription) -> { topicDescription.partitions().forEach(info -> { @@ -670,9 +671,9 @@ static Map> getReplicaAssignmentForPartitions(Admi */ static List getBrokerMetadata(Admin adminClient, List brokers, boolean enableRackAwareness) throws ExecutionException, InterruptedException { Set brokerSet = new HashSet<>(brokers); - List results = adminClient.describeCluster().nodes().get().stream(). - filter(node -> brokerSet.contains(node.id())). - map(node -> (enableRackAwareness && node.rack() != null) + List results = adminClient.describeCluster().nodes().get().stream() + .filter(node -> brokerSet.contains(node.id())) + .map(node -> (enableRackAwareness && node.rack() != null) ? new BrokerMetadata(node.id(), Optional.of(node.rack())) : new BrokerMetadata(node.id(), Optional.empty())).collect(Collectors.toList()); @@ -696,7 +697,7 @@ static List getBrokerMetadata(Admin adminClient, List b * @return A tuple of brokers to reassign, topics to reassign */ static Tuple, List> parseGenerateAssignmentArgs(String reassignmentJson, - String brokerList) throws JsonMappingException { + String brokerList) throws JsonMappingException { List brokerListToReassign = Arrays.stream(brokerList.split(",")).map(Integer::parseInt).collect(Collectors.toList()); Set duplicateReassignments = ToolsUtils.duplicates(brokerListToReassign); if (!duplicateReassignments.isEmpty()) @@ -724,12 +725,12 @@ static Tuple, List> parseGenerateAssignmentArgs(String rea * @param time The Time object to use. */ static void executeAssignment(Admin adminClient, - Boolean additional, - String reassignmentJson, - Long interBrokerThrottle, - Long logDirThrottle, - Long timeoutMs, - Time time) throws ExecutionException, InterruptedException, JsonProcessingException, TerseException { + Boolean additional, + String reassignmentJson, + Long interBrokerThrottle, + Long logDirThrottle, + Long timeoutMs, + Time time) throws ExecutionException, InterruptedException, JsonProcessingException, TerseException { Tuple>, Map> t0 = parseExecuteAssignmentArgs(reassignmentJson); Map> proposedParts = t0.v1; @@ -767,17 +768,17 @@ static void executeAssignment(Admin adminClient, if (!errors.isEmpty()) { throw new TerseException( String.format("Error reassigning partition(s):%n%s", - errors.keySet().stream(). - sorted(ReassignPartitionsCommand::compareTopicPartitions). - map(part -> part + ": " + errors.get(part).getMessage()). - collect(Collectors.joining(System.lineSeparator())))); + errors.keySet().stream() + .sorted(ReassignPartitionsCommand::compareTopicPartitions) + .map(part -> part + ": " + errors.get(part).getMessage()) + .collect(Collectors.joining(System.lineSeparator())))); } System.out.printf("Successfully started partition reassignment%s for %s%n", proposedParts.size() == 1 ? "" : "s", - proposedParts.keySet().stream(). - sorted(ReassignPartitionsCommand::compareTopicPartitions). - map(Objects::toString). - collect(Collectors.joining(","))); + proposedParts.keySet().stream() + .sorted(ReassignPartitionsCommand::compareTopicPartitions) + .map(Objects::toString) + .collect(Collectors.joining(","))); if (!proposedReplicas.isEmpty()) { executeMoves(adminClient, proposedReplicas, timeoutMs, time); } @@ -804,12 +805,12 @@ private static void executeMoves(Admin adminClient, do { Set completed = alterReplicaLogDirs(adminClient, pendingReplicas); if (!completed.isEmpty()) { - System.out.printf("Successfully started log directory move%s for: %s", + System.out.printf("Successfully started log directory move%s for: %s%n", completed.size() == 1 ? "" : "s", - completed.stream(). - sorted(ReassignPartitionsCommand::compareTopicPartitionReplicas). - map(Object::toString). - collect(Collectors.joining(","))); + completed.stream() + .sorted(ReassignPartitionsCommand::compareTopicPartitionReplicas) + .map(Object::toString) + .collect(Collectors.joining(","))); } completed.forEach(pendingReplicas::remove); if (pendingReplicas.isEmpty()) { @@ -818,10 +819,10 @@ private static void executeMoves(Admin adminClient, throw new TerseException(String.format( "Timed out before log directory move%s could be started for: %s", pendingReplicas.size() == 1 ? "" : "s", - pendingReplicas.keySet().stream(). - sorted(ReassignPartitionsCommand::compareTopicPartitionReplicas). - map(Object::toString). - collect(Collectors.joining(",")))); + pendingReplicas.keySet().stream() + .sorted(ReassignPartitionsCommand::compareTopicPartitionReplicas) + .map(Object::toString) + .collect(Collectors.joining(",")))); } else { // If a replica has been moved to a new host and we also specified a particular // log directory, we will have to keep retrying the alterReplicaLogDirs @@ -860,16 +861,16 @@ static String curReassignmentsToString(Admin adminClient) throws ExecutionExcept return String.format("%s: replicas: %s.%s%s", part, replicas.stream().map(Object::toString).collect(Collectors.joining(",")), - addingReplicas.isEmpty() ? "" : String.format(" adding: %s.", addingReplicas.stream(). - map(Object::toString). - collect(Collectors.joining(","))), - removingReplicas.isEmpty() ? "" : String.format(" removing: %s.", removingReplicas.stream(). - map(Object::toString). - collect(Collectors.joining(","))) + addingReplicas.isEmpty() ? "" : String.format(" adding: %s.", addingReplicas.stream() + .map(Object::toString) + .collect(Collectors.joining(","))), + removingReplicas.isEmpty() ? "" : String.format(" removing: %s.", removingReplicas.stream() + .map(Object::toString) + .collect(Collectors.joining(","))) ); }).collect(Collectors.joining(System.lineSeparator())); - return text.length() == 0 + return text.isEmpty() ? "No partition reassignments found." : String.format("Current partition reassignments:%n%s", text); } @@ -882,9 +883,9 @@ static String curReassignmentsToString(Admin adminClient) throws ExecutionExcept */ private static void verifyBrokerIds(Admin adminClient, Set brokers) throws ExecutionException, InterruptedException { Set allNodeIds = adminClient.describeCluster().nodes().get().stream().map(Node::id).collect(Collectors.toSet()); - Optional unknown = brokers.stream(). - filter(i -> !allNodeIds.contains(i)). - findFirst(); + Optional unknown = brokers.stream() + .filter(brokerId -> !allNodeIds.contains(brokerId)) + .findFirst(); if (unknown.isPresent()) throw new AdminCommandFailedException("Unknown broker id " + unknown.get()); } @@ -900,9 +901,9 @@ private static void verifyBrokerIds(Admin adminClient, Set brokers) thr */ static String currentPartitionReplicaAssignmentToString(Map> proposedParts, Map> currentParts) throws JsonProcessingException { - Map> partitionsToBeReassigned = currentParts.entrySet().stream(). - filter(e -> proposedParts.containsKey(e.getKey())). - collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + Map> partitionsToBeReassigned = currentParts.entrySet().stream() + .filter(e -> proposedParts.containsKey(e.getKey())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); return String.format("Current partition replica assignment%n%n%s%n%nSave this to use as the %s", formatAsReassignmentJson(partitionsToBeReassigned, Collections.emptyMap()), @@ -945,7 +946,7 @@ static Map alterPartitionReassignments(Admin adminCli * @return A map from partition objects to error strings. */ static Map cancelPartitionReassignments(Admin adminClient, - Set reassignments) throws InterruptedException { + Set reassignments) throws InterruptedException { Map> args = new HashMap<>(); reassignments.forEach(part -> args.put(part, Optional.empty())); @@ -1252,12 +1253,12 @@ static Tuple, Set> cancelAssignment(A throw new TerseException(String.format( "Error cancelling partition reassignment%s for:%n%s", errors.size() == 1 ? "" : "s", - errors.keySet().stream(). - sorted(ReassignPartitionsCommand::compareTopicPartitions). - map(part -> part + ": " + errors.get(part).getMessage()).collect(Collectors.joining(System.lineSeparator()))) + errors.keySet().stream() + .sorted(ReassignPartitionsCommand::compareTopicPartitions) + .map(part -> part + ": " + errors.get(part).getMessage()).collect(Collectors.joining(System.lineSeparator()))) ); } - System.out.printf("Successfully cancelled partition reassignment%s for: %s", + System.out.printf("Successfully cancelled partition reassignment%s for: %s%n", curReassigningParts.size() == 1 ? "" : "s", curReassigningParts.stream().sorted(ReassignPartitionsCommand::compareTopicPartitions).map(Object::toString).collect(Collectors.joining(",")) ); @@ -1290,9 +1291,9 @@ private static String formatAsReassignmentJson(Map data.put("topic", tp.topic()); data.put("partition", tp.partition()); data.put("replicas", replicas); - data.put("log_dirs", replicas.stream(). - map(r -> replicaLogDirAssignment.getOrDefault(new TopicPartitionReplica(tp.topic(), tp.partition(), r), ANY_LOG_DIR)). - collect(Collectors.toList())); + data.put("log_dirs", replicas.stream() + .map(r -> replicaLogDirAssignment.getOrDefault(new TopicPartitionReplica(tp.topic(), tp.partition(), r), ANY_LOG_DIR)) + .collect(Collectors.toList())); partitions.add(data); }); From 8700c054c0802111dd0cc88f3e4154fea72922ba Mon Sep 17 00:00:00 2001 From: nizhikov Date: Fri, 22 Sep 2023 12:09:33 +0300 Subject: [PATCH 54/78] KAFKA-14595 Code review fixes. --- .../reassign/ReassignPartitionsUnitTest.java | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java index cdc11cb202232..4639744226d11 100644 --- a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java +++ b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java @@ -32,8 +32,8 @@ import org.apache.kafka.common.utils.Time; import org.apache.kafka.server.common.AdminCommandFailedException; import org.apache.kafka.server.common.AdminOperationException; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; import scala.Tuple2; @@ -91,15 +91,15 @@ @Timeout(60) public class ReassignPartitionsUnitTest { - @BeforeEach - public void setUp() { + @BeforeAll + public static void setUp() { Exit.setExitProcedure((statusCode, message) -> { throw new IllegalArgumentException(message); }); } - @AfterEach - public void tearDown() { + @AfterAll + public static void tearDown() { Exit.resetExitProcedure(); } @@ -186,7 +186,7 @@ public void testFindPartitionReassignmentStates() { )); assertEquals(asScala(expStates), actual._1); - assertEquals(true, actual._2); + assertTrue((Boolean) actual._2); // Cancel the reassignment and test findPartitionReassignmentStates again. scala.collection.Map cancelResult = cancelPartitionReassignments(adminClient, @@ -208,7 +208,7 @@ public void testFindPartitionReassignmentStates() { )); assertEquals(asScala(expStates), actual._1); - assertEquals(false, actual._2); + assertFalse((Boolean) actual._2); } } @@ -510,16 +510,16 @@ public void testMoveMap() { Map fooMoves = new HashMap<>(); - fooMoves.put(0, new ReassignPartitionsCommand.PartitionMove(mset(1, 2, 3), mset(5))); - fooMoves.put(1, new ReassignPartitionsCommand.PartitionMove(mset(4, 5, 6), mset(7, 8))); - fooMoves.put(2, new ReassignPartitionsCommand.PartitionMove(mset(1, 2), mset(3, 4))); - fooMoves.put(3, new ReassignPartitionsCommand.PartitionMove(mset(1, 2), mset(5, 6))); - fooMoves.put(4, new ReassignPartitionsCommand.PartitionMove(mset(1, 2), mset(3))); - fooMoves.put(5, new ReassignPartitionsCommand.PartitionMove(mset(1, 2), mset(3, 4, 5, 6))); + fooMoves.put(0, new ReassignPartitionsCommand.PartitionMove(mutableSet(1, 2, 3), mutableSet(5))); + fooMoves.put(1, new ReassignPartitionsCommand.PartitionMove(mutableSet(4, 5, 6), mutableSet(7, 8))); + fooMoves.put(2, new ReassignPartitionsCommand.PartitionMove(mutableSet(1, 2), mutableSet(3, 4))); + fooMoves.put(3, new ReassignPartitionsCommand.PartitionMove(mutableSet(1, 2), mutableSet(5, 6))); + fooMoves.put(4, new ReassignPartitionsCommand.PartitionMove(mutableSet(1, 2), mutableSet(3))); + fooMoves.put(5, new ReassignPartitionsCommand.PartitionMove(mutableSet(1, 2), mutableSet(3, 4, 5, 6))); Map barMoves = new HashMap<>(); - barMoves.put(0, new ReassignPartitionsCommand.PartitionMove(mset(2, 3, 4), mset(1))); + barMoves.put(0, new ReassignPartitionsCommand.PartitionMove(mutableSet(2, 3, 4), mutableSet(1))); assertEquals(asScala(fooMoves), moveMap.get("foo").get()); assertEquals(asScala(barMoves), moveMap.get("bar").get()); @@ -770,11 +770,11 @@ public void testPropagateInvalidJsonError() { @SuppressWarnings("unchecked") private static scala.collection.immutable.Set set(final T... set) { - return mset(set).toSet(); + return mutableSet(set).toSet(); } @SuppressWarnings({"deprecation", "unchecked"}) - private static scala.collection.mutable.Set mset(final T...set) { + private static scala.collection.mutable.Set mutableSet(final T...set) { return JavaConverters.asScalaSet(new HashSet<>(Arrays.asList(set))); } From 2dd787e41937733b4c509f524d53f7e49f2922cd Mon Sep 17 00:00:00 2001 From: nizhikov Date: Fri, 22 Sep 2023 16:58:16 +0300 Subject: [PATCH 55/78] Merge branch 'KAFKA-14595_tests' into KAFKA-14595 --- .../reassign/ReassignPartitionsUnitTest.java | 24 +++++++++---------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java index d7bf2227851de..310297b911387 100644 --- a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java +++ b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java @@ -103,7 +103,7 @@ public static void tearDown() { @Test public void testCompareTopicPartitions() { assertTrue(compareTopicPartitions(new TopicPartition("abc", 0), - new TopicPartition("abc", 1)) > 0); + new TopicPartition("abc", 1)) < 0); assertFalse(compareTopicPartitions(new TopicPartition("def", 0), new TopicPartition("abc", 1)) < 0); } @@ -111,7 +111,7 @@ public void testCompareTopicPartitions() { @Test public void testCompareTopicPartitionReplicas() { assertTrue(compareTopicPartitionReplicas(new TopicPartitionReplica("def", 0, 0), - new TopicPartitionReplica("abc", 0, 1)) > 0); + new TopicPartitionReplica("abc", 0, 1)) < 0); assertFalse(compareTopicPartitionReplicas(new TopicPartitionReplica("def", 0, 0), new TopicPartitionReplica("cde", 0, 0)) < 0); } @@ -504,16 +504,16 @@ public void testMoveMap() { Map fooMoves = new HashMap<>(); - fooMoves.put(0, new PartitionMove(mutableSet(1, 2, 3), mutableSet(5))); - fooMoves.put(1, new PartitionMove(mutableSet(4, 5, 6), mutableSet(7, 8))); - fooMoves.put(2, new PartitionMove(mutableSet(1, 2), mutableSet(3, 4))); - fooMoves.put(3, new PartitionMove(mutableSet(1, 2), mutableSet(5, 6))); - fooMoves.put(4, new PartitionMove(mutableSet(1, 2), mutableSet(3))); - fooMoves.put(5, new PartitionMove(mutableSet(1, 2), mutableSet(3, 4, 5, 6))); + fooMoves.put(0, new PartitionMove(set(1, 2, 3), set(5))); + fooMoves.put(1, new PartitionMove(set(4, 5, 6), set(7, 8))); + fooMoves.put(2, new PartitionMove(set(1, 2), set(3, 4))); + fooMoves.put(3, new PartitionMove(set(1, 2), set(5, 6))); + fooMoves.put(4, new PartitionMove(set(1, 2), set(3))); + fooMoves.put(5, new PartitionMove(set(1, 2), set(3, 4, 5, 6))); Map barMoves = new HashMap<>(); - barMoves.put(0, new PartitionMove(mutableSet(2, 3, 4), mutableSet(1))); + barMoves.put(0, new PartitionMove(set(2, 3, 4), set(1))); assertEquals(fooMoves, moveMap.get("foo")); assertEquals(barMoves, moveMap.get("bar")); @@ -761,14 +761,12 @@ public void testPropagateInvalidJsonError() { } } + @SuppressWarnings("unchecked") private static Set set(final T... set) { return new HashSet<>(Arrays.asList(set)); } - private static Set mutableSet(final T...set) { - return new HashSet<>(Arrays.asList(set)); - } - + @SuppressWarnings("unchecked") private static List seq(T... seq) { return Arrays.asList(seq); } From 95e69a2b8cc82c1fcdb38fd9b7ece9492f0181c6 Mon Sep 17 00:00:00 2001 From: nizhikov Date: Fri, 22 Sep 2023 17:01:43 +0300 Subject: [PATCH 56/78] KAFKA-14595 Code review fixes. --- .../kafka/tools/reassign/ReassignPartitionsUnitTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java index 310297b911387..f7622800dc753 100644 --- a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java +++ b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java @@ -424,8 +424,8 @@ public void testGenerateAssignmentWithFewerBrokers() throws Exception { // The proposed assignment should only span the provided brokers proposedCurrent.v1.values().forEach(replicas -> assertTrue(goalBrokers.containsAll(replicas), - "Proposed assignment " + proposedCurrent.v1 + " puts replicas on brokers other than " + goalBrokers)); - + "Proposed assignment " + proposedCurrent.v1 + " puts replicas on brokers other than " + goalBrokers) + ); } } From aa149f2930e076377f661db4747f10e336711f9e Mon Sep 17 00:00:00 2001 From: nizhikov Date: Tue, 26 Sep 2023 21:02:58 +0300 Subject: [PATCH 57/78] KAFKA-14595 Fix integration test --- build.gradle | 1 + .../test/java/kafka/test/ClusterConfig.java | 5 + .../ReassignPartitionsIntegrationTest.java | 505 +++++++++--------- 3 files changed, 252 insertions(+), 259 deletions(-) diff --git a/build.gradle b/build.gradle index bb594af3dc5e8..9ebd7963f9933 100644 --- a/build.gradle +++ b/build.gradle @@ -1879,6 +1879,7 @@ project(':tools') { dependencies { implementation project(':clients') implementation project(':server-common') + implementation project(':storage:api') implementation project(':connect:api') implementation project(':connect:runtime') implementation project(':log4j-appender') diff --git a/core/src/test/java/kafka/test/ClusterConfig.java b/core/src/test/java/kafka/test/ClusterConfig.java index 8e9f7de96abb1..c4ea8669d53a2 100644 --- a/core/src/test/java/kafka/test/ClusterConfig.java +++ b/core/src/test/java/kafka/test/ClusterConfig.java @@ -146,6 +146,11 @@ public ClusterConfig copyOf() { copy.consumerProperties.putAll(consumerProperties); copy.saslServerProperties.putAll(saslServerProperties); copy.saslClientProperties.putAll(saslClientProperties); + perBrokerOverrideProperties.forEach((brokerId, props) -> { + Properties propsCopy = new Properties(); + propsCopy.putAll(props); + copy.perBrokerOverrideProperties.put(brokerId, propsCopy); + }); return copy; } diff --git a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsIntegrationTest.java b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsIntegrationTest.java index d7be673bb2def..724ee0c54bb89 100644 --- a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsIntegrationTest.java +++ b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsIntegrationTest.java @@ -14,49 +14,52 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.kafka.tools.reassign; import com.fasterxml.jackson.core.JsonProcessingException; import kafka.cluster.Partition; import kafka.log.UnifiedLog; +import kafka.server.ControllerServer; import kafka.server.HostedPartition; import kafka.server.IsrChangePropagationConfig; import kafka.server.KafkaBroker; import kafka.server.KafkaConfig; -import kafka.server.QuorumTestHarness; import kafka.server.ZkAlterPartitionManager; +import kafka.test.ClusterConfig; +import kafka.test.ClusterGenerator; +import kafka.test.ClusterInstance; +import kafka.test.annotation.ClusterTemplate; +import kafka.test.annotation.ClusterTest; +import kafka.test.annotation.ClusterTestDefaults; +import kafka.test.annotation.Type; +import kafka.test.junit.ClusterTestExtensions; +import kafka.test.junit.RaftClusterInvocationContext.RaftClusterInstance; +import kafka.test.junit.ZkClusterInvocationContext; import kafka.utils.TestUtils; import org.apache.kafka.clients.admin.Admin; -import org.apache.kafka.clients.admin.AdminClientConfig; import org.apache.kafka.clients.admin.AlterConfigOp; import org.apache.kafka.clients.admin.Config; import org.apache.kafka.clients.admin.ConfigEntry; import org.apache.kafka.clients.admin.DescribeLogDirsResult; import org.apache.kafka.clients.admin.NewTopic; import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.TopicPartitionReplica; import org.apache.kafka.common.config.ConfigResource; import org.apache.kafka.common.security.auth.SecurityProtocol; import org.apache.kafka.common.serialization.ByteArrayDeserializer; +import org.apache.kafka.common.serialization.ByteArraySerializer; import org.apache.kafka.common.utils.Time; -import org.apache.kafka.common.utils.Utils; import org.apache.kafka.tools.TerseException; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Timeout; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.extension.ExtendWith; import scala.None$; import scala.Option; -import scala.Some$; import scala.collection.JavaConverters; import scala.collection.Seq; -import java.io.Closeable; -import java.io.IOException; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -73,7 +76,13 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; import java.util.stream.IntStream; +import java.util.stream.Stream; +import static kafka.test.annotation.Type.KRAFT; +import static kafka.test.annotation.Type.ZK; +import static org.apache.kafka.clients.CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG; +import static org.apache.kafka.clients.producer.ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG; +import static org.apache.kafka.clients.producer.ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG; import static org.apache.kafka.server.common.MetadataVersion.IBP_2_7_IV1; import static org.apache.kafka.tools.reassign.ReassignPartitionsCommand.BROKER_LEVEL_THROTTLES; import static org.apache.kafka.tools.reassign.ReassignPartitionsCommand.cancelAssignment; @@ -83,15 +92,16 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; -@Timeout(300) -public class ReassignPartitionsIntegrationTest extends QuorumTestHarness { - ReassignPartitionsTestCluster cluster; - - @AfterEach - @Override - public void tearDown() { - Utils.closeQuietly(cluster, "ReassignPartitionsTestCluster"); - super.tearDown(); +@ExtendWith(value = ClusterTestExtensions.class) +@ClusterTestDefaults(brokers = 5) +@Tag("integration") +public class ReassignPartitionsIntegrationTest { + private static final Map BROKERS = new HashMap<>(); static { + BROKERS.put(0, "rack0"); + BROKERS.put(1, "rack0"); + BROKERS.put(2, "rack1"); + BROKERS.put(3, "rack1"); + BROKERS.put(4, "rack1"); } private final Map> unthrottledBrokerConfigs = new HashMap<>(); { @@ -100,50 +110,67 @@ public void tearDown() { .collect(Collectors.toMap(throttle -> throttle, throttle -> -1L)))); } - @ParameterizedTest(name = "{displayName}.quorum={0}") - @ValueSource(strings = {"zk", "kraft"}) - public void testReassignment(String quorum) throws Exception { - cluster = new ReassignPartitionsTestCluster(Collections.emptyMap(), Collections.emptyMap()); - cluster.setup(); + private final Map>> topics = new HashMap<>(); { + topics.put("foo", Arrays.asList(Arrays.asList(0, 1, 2), Arrays.asList(1, 2, 3))); + topics.put("bar", Arrays.asList(Arrays.asList(3, 2, 1))); + topics.put("baz", Arrays.asList(Arrays.asList(1, 0, 2), Arrays.asList(2, 0, 1), Arrays.asList(0, 2, 1))); + } + + private ClusterInstance cluster; + + private Admin adminClient; + + @ClusterTemplate("zkAndKRaft") + public void testReassignment(ClusterInstance cluster) throws Exception { + this.cluster = cluster; + createTopics(); executeAndVerifyReassignment(); } - @ParameterizedTest(name = "{displayName}.quorum={0}") - @ValueSource(strings = "zk") // Note: KRaft requires AlterPartition - public void testReassignmentWithAlterPartitionDisabled(String quorum) throws Exception { + public static void zkAlterPartitionDisabled(ClusterGenerator generator) { + ClusterConfig cfg = clusterConfig(ZK); + + BROKERS.forEach((brokerId, rack) -> { + cfg.brokerServerProperties(brokerId).put(KafkaConfig.InterBrokerProtocolVersionProp(), IBP_2_7_IV1.version()); + }); + + generator.accept(cfg); + } + + @ClusterTemplate("zkAlterPartitionDisabled") + public void testReassignmentWithAlterPartitionDisabled(ClusterInstance cluster) throws Exception { + this.cluster = cluster; // Test reassignment when the IBP is on an older version which does not use // the `AlterPartition` API. In this case, the controller will register individual // watches for each reassigning partition so that the reassignment can be // completed as soon as the ISR is expanded. - Map configOverrides = Collections.singletonMap(KafkaConfig.InterBrokerProtocolVersionProp(), IBP_2_7_IV1.version()); - cluster = new ReassignPartitionsTestCluster(configOverrides, Collections.emptyMap()); - cluster.setup(); + createTopics(); executeAndVerifyReassignment(); } - @ParameterizedTest(name = "{displayName}.quorum={0}") - @ValueSource(strings = "zk") // Note: KRaft requires AlterPartition - public void testReassignmentCompletionDuringPartialUpgrade(String quorum) throws Exception { - // Test reassignment during a partial upgrade when some brokers are relying on - // `AlterPartition` and some rely on the old notification logic through Zookeeper. - // In this test case, broker 0 starts up first on the latest IBP and is typically - // elected as controller. The three remaining brokers start up on the older IBP. - // We want to ensure that reassignment can still complete through the ISR change - // notification path even though the controller expects `AlterPartition`. - + public static void zkPartialUpgrade(ClusterGenerator generator) { // Override change notification settings so that test is not delayed by ISR // change notification delay ZkAlterPartitionManager.DefaultIsrPropagationConfig_$eq(new IsrChangePropagationConfig(500, 100, 500)); + ClusterConfig cfg = clusterConfig(ZK); - Map oldIbpConfig = Collections.singletonMap(KafkaConfig.InterBrokerProtocolVersionProp(), IBP_2_7_IV1.version()); - Map> brokerConfigOverrides = new HashMap<>(); - brokerConfigOverrides.put(1, oldIbpConfig); - brokerConfigOverrides.put(2, oldIbpConfig); - brokerConfigOverrides.put(3, oldIbpConfig); + for (int brokerId = 1; brokerId < 4; brokerId++) + cfg.brokerServerProperties(brokerId).put(KafkaConfig.InterBrokerProtocolVersionProp(), IBP_2_7_IV1.version()); - cluster = new ReassignPartitionsTestCluster(Collections.emptyMap(), brokerConfigOverrides); - cluster.setup(); + generator.accept(cfg); + } + @ClusterTest(clusterType = ZK) // Note: KRaft requires AlterPartition + @ClusterTemplate("zkPartialUpgrade") + public void testReassignmentCompletionDuringPartialUpgrade(ClusterInstance cluster) throws Exception { + this.cluster = cluster; + // Test reassignment during a partial upgrade when some brokers are relying on + // `AlterPartition` and some rely on the old notification logic through Zookeeper. + // In this test case, broker 0 starts up first on the latest IBP and is typically + // elected as controller. The three remaining brokers start up on the older IBP. + // We want to ensure that reassignment can still complete through the ISR change + // notification path even though the controller expects `AlterPartition`. + createTopics(); executeAndVerifyReassignment(); } @@ -162,21 +189,21 @@ private void executeAndVerifyReassignment() throws ExecutionException, Interrupt initialAssignment.put(foo0, new PartitionReassignmentState(Arrays.asList(0, 1, 2), Arrays.asList(0, 1, 3), true)); initialAssignment.put(bar0, new PartitionReassignmentState(Arrays.asList(3, 2, 1), Arrays.asList(3, 2, 0), true)); - waitForVerifyAssignment(cluster.adminClient, assignment, false, + waitForVerifyAssignment(adminClient, assignment, false, new VerifyAssignmentResult(initialAssignment)); // Execute the assignment - runExecuteAssignment(cluster.adminClient, false, assignment, -1L, -1L); + runExecuteAssignment(adminClient, false, assignment, -1L, -1L); assertEquals(unthrottledBrokerConfigs, describeBrokerLevelThrottles(unthrottledBrokerConfigs.keySet())); Map finalAssignment = new HashMap<>(); finalAssignment.put(foo0, new PartitionReassignmentState(Arrays.asList(0, 1, 3), Arrays.asList(0, 1, 3), true)); finalAssignment.put(bar0, new PartitionReassignmentState(Arrays.asList(3, 2, 0), Arrays.asList(3, 2, 0), true)); - VerifyAssignmentResult verifyAssignmentResult = runVerifyAssignment(cluster.adminClient, assignment, false); + VerifyAssignmentResult verifyAssignmentResult = runVerifyAssignment(adminClient, assignment, false); assertFalse(verifyAssignmentResult.movesOngoing); // Wait for the assignment to complete - waitForVerifyAssignment(cluster.adminClient, assignment, false, + waitForVerifyAssignment(adminClient, assignment, false, new VerifyAssignmentResult(finalAssignment)); assertEquals(unthrottledBrokerConfigs, @@ -187,43 +214,41 @@ private void executeAndVerifyReassignment() throws ExecutionException, Interrupt verifyReplicaDeleted(bar0, 1); } - @ParameterizedTest(name = "{displayName}.quorum={0}") - @ValueSource(strings = {"zk", "kraft"}) - public void testHighWaterMarkAfterPartitionReassignment(String quorum) throws Exception { - cluster = new ReassignPartitionsTestCluster(Collections.emptyMap(), Collections.emptyMap()); - cluster.setup(); + @ClusterTemplate("zkAndKRaft") + public void testHighWaterMarkAfterPartitionReassignment(ClusterInstance cluster) throws Exception { + this.cluster = cluster; + createTopics(); String assignment = "{\"version\":1,\"partitions\":" + "[{\"topic\":\"foo\",\"partition\":0,\"replicas\":[3,1,2],\"log_dirs\":[\"any\",\"any\",\"any\"]}" + "]}"; // Set the high water mark of foo-0 to 123 on its leader. TopicPartition part = new TopicPartition("foo", 0); - cluster.servers.get(0).replicaManager().logManager().truncateFullyAndStartAt(part, 123L, false, None$.empty()); + servers().get(0).replicaManager().logManager().truncateFullyAndStartAt(part, 123L, false, None$.empty()); // Execute the assignment - runExecuteAssignment(cluster.adminClient, false, assignment, -1L, -1L); + runExecuteAssignment(cluster.createAdminClient(), false, assignment, -1L, -1L); Map finalAssignment = Collections.singletonMap(part, new PartitionReassignmentState(Arrays.asList(3, 1, 2), Arrays.asList(3, 1, 2), true)); // Wait for the assignment to complete - waitForVerifyAssignment(cluster.adminClient, assignment, false, + waitForVerifyAssignment(adminClient, assignment, false, new VerifyAssignmentResult(finalAssignment)); TestUtils.waitUntilTrue(() -> - cluster.servers.get(3).replicaManager().onlinePartition(part). - map(Partition::leaderLogIfLocal).isDefined(), + servers().get(3).replicaManager().onlinePartition(part). + map(Partition::leaderLogIfLocal).isDefined(), () -> "broker 3 should be the new leader", org.apache.kafka.test.TestUtils.DEFAULT_MAX_WAIT_MS, 10L); - assertEquals(123L, cluster.servers.get(3).replicaManager().localLogOrException(part).highWatermark(), + assertEquals(123L, servers().get(3).replicaManager().localLogOrException(part).highWatermark(), "Expected broker 3 to have the correct high water mark for the partition."); } - @ParameterizedTest(name = "{displayName}.quorum={0}") - @ValueSource(strings = {"zk", "kraft"}) - public void testAlterReassignmentThrottle(String quorum) throws Exception { - cluster = new ReassignPartitionsTestCluster(Collections.emptyMap(), Collections.emptyMap()); - cluster.setup(); - cluster.produceMessages("foo", 0, 50); - cluster.produceMessages("baz", 2, 60); + @ClusterTemplate("zkAndKRaft") + public void testAlterReassignmentThrottle(ClusterInstance cluster) throws Exception { + this.cluster = cluster; + createTopics(); + produceMessages("foo", 0, 50); + produceMessages("baz", 2, 60); String assignment = "{\"version\":1,\"partitions\":" + "[{\"topic\":\"foo\",\"partition\":0,\"replicas\":[0,3,2],\"log_dirs\":[\"any\",\"any\",\"any\"]}," + "{\"topic\":\"baz\",\"partition\":2,\"replicas\":[3,2,1],\"log_dirs\":[\"any\",\"any\",\"any\"]}" + @@ -231,12 +256,12 @@ public void testAlterReassignmentThrottle(String quorum) throws Exception { // Execute the assignment with a low throttle long initialThrottle = 1L; - runExecuteAssignment(cluster.adminClient, false, assignment, initialThrottle, -1L); + runExecuteAssignment(adminClient, false, assignment, initialThrottle, -1L); waitForInterBrokerThrottle(Arrays.asList(0, 1, 2, 3), initialThrottle); // Now update the throttle and verify the reassignment completes long updatedThrottle = 300000L; - runExecuteAssignment(cluster.adminClient, true, assignment, updatedThrottle, -1L); + runExecuteAssignment(adminClient, true, assignment, updatedThrottle, -1L); waitForInterBrokerThrottle(Arrays.asList(0, 1, 2, 3), updatedThrottle); Map finalAssignment = new HashMap<>(); @@ -246,7 +271,7 @@ public void testAlterReassignmentThrottle(String quorum) throws Exception { new PartitionReassignmentState(Arrays.asList(3, 2, 1), Arrays.asList(3, 2, 1), true)); // Now remove the throttles. - waitForVerifyAssignment(cluster.adminClient, assignment, false, + waitForVerifyAssignment(adminClient, assignment, false, new VerifyAssignmentResult(finalAssignment)); waitForBrokerLevelThrottles(unthrottledBrokerConfigs); } @@ -254,13 +279,12 @@ public void testAlterReassignmentThrottle(String quorum) throws Exception { /** * Test running a reassignment with the interBrokerThrottle set. */ - @ParameterizedTest(name = "{displayName}.quorum={0}") - @ValueSource(strings = {"zk", "kraft"}) - public void testThrottledReassignment(String quorum) throws Exception { - cluster = new ReassignPartitionsTestCluster(Collections.emptyMap(), Collections.emptyMap()); - cluster.setup(); - cluster.produceMessages("foo", 0, 50); - cluster.produceMessages("baz", 2, 60); + @ClusterTemplate("zk") + public void testThrottledReassignment(ClusterInstance cluster) throws Exception { + this.cluster = cluster; + createTopics(); + produceMessages("foo", 0, 50); + produceMessages("baz", 2, 60); String assignment = "{\"version\":1,\"partitions\":" + "[{\"topic\":\"foo\",\"partition\":0,\"replicas\":[0,3,2],\"log_dirs\":[\"any\",\"any\",\"any\"]}," + "{\"topic\":\"baz\",\"partition\":2,\"replicas\":[3,2,1],\"log_dirs\":[\"any\",\"any\",\"any\"]}" + @@ -272,12 +296,12 @@ public void testThrottledReassignment(String quorum) throws Exception { new PartitionReassignmentState(Arrays.asList(0, 1, 2), Arrays.asList(0, 3, 2), true)); initialAssignment.put(new TopicPartition("baz", 2), new PartitionReassignmentState(Arrays.asList(0, 2, 1), Arrays.asList(3, 2, 1), true)); - assertEquals(new VerifyAssignmentResult(initialAssignment), runVerifyAssignment(cluster.adminClient, assignment, false)); + assertEquals(new VerifyAssignmentResult(initialAssignment), runVerifyAssignment(adminClient, assignment, false)); assertEquals(unthrottledBrokerConfigs, describeBrokerLevelThrottles(unthrottledBrokerConfigs.keySet())); // Execute the assignment long interBrokerThrottle = 300000L; - runExecuteAssignment(cluster.adminClient, false, assignment, interBrokerThrottle, -1L); + runExecuteAssignment(adminClient, false, assignment, interBrokerThrottle, -1L); waitForInterBrokerThrottle(Arrays.asList(0, 1, 2, 3), interBrokerThrottle); Map finalAssignment = new HashMap<>(); @@ -291,7 +315,7 @@ public void testThrottledReassignment(String quorum) throws Exception { () -> { try { // Check the reassignment status. - VerifyAssignmentResult result = runVerifyAssignment(cluster.adminClient, assignment, true); + VerifyAssignmentResult result = runVerifyAssignment(adminClient, assignment, true); if (!result.partsOngoing) { return true; @@ -307,28 +331,27 @@ public void testThrottledReassignment(String quorum) throws Exception { throw new RuntimeException(e); } }, () -> "Expected reassignment to complete.", org.apache.kafka.test.TestUtils.DEFAULT_MAX_WAIT_MS, 100L); - waitForVerifyAssignment(cluster.adminClient, assignment, true, + waitForVerifyAssignment(adminClient, assignment, true, new VerifyAssignmentResult(finalAssignment)); // The throttles should still have been preserved, since we ran with --preserve-throttles waitForInterBrokerThrottle(Arrays.asList(0, 1, 2, 3), interBrokerThrottle); // Now remove the throttles. - waitForVerifyAssignment(cluster.adminClient, assignment, false, + waitForVerifyAssignment(adminClient, assignment, false, new VerifyAssignmentResult(finalAssignment)); waitForBrokerLevelThrottles(unthrottledBrokerConfigs); } - @ParameterizedTest(name = "{displayName}.quorum={0}") - @ValueSource(strings = {"zk", "kraft"}) - public void testProduceAndConsumeWithReassignmentInProgress(String quorum) throws Exception { - cluster = new ReassignPartitionsTestCluster(Collections.emptyMap(), Collections.emptyMap()); - cluster.setup(); - cluster.produceMessages("baz", 2, 60); + @ClusterTemplate("zkAndKRaft") + public void testProduceAndConsumeWithReassignmentInProgress(ClusterInstance cluster) throws Exception { + this.cluster = cluster; + createTopics(); + produceMessages("baz", 2, 60); String assignment = "{\"version\":1,\"partitions\":" + "[{\"topic\":\"baz\",\"partition\":2,\"replicas\":[3,2,1],\"log_dirs\":[\"any\",\"any\",\"any\"]}" + "]}"; - runExecuteAssignment(cluster.adminClient, false, assignment, 300L, -1L); - cluster.produceMessages("baz", 2, 100); - Consumer consumer = TestUtils.createConsumer(cluster.brokerList, + runExecuteAssignment(adminClient, false, assignment, 300L, -1L); + produceMessages("baz", 2, 100); + Consumer consumer = TestUtils.createConsumer(cluster.bootstrapServers(), "group", "earliest", true, @@ -348,26 +371,25 @@ public void testProduceAndConsumeWithReassignmentInProgress(String quorum) throw } finally { consumer.close(); } - TestUtils.removeReplicationThrottleForPartitions(cluster.adminClient, toSeq(Arrays.asList(0, 1, 2, 3)), toSet(Collections.singleton(part))); + TestUtils.removeReplicationThrottleForPartitions(adminClient, toSeq(Arrays.asList(0, 1, 2, 3)), toSet(Collections.singleton(part))); Map finalAssignment = Collections.singletonMap(part, new PartitionReassignmentState(Arrays.asList(3, 2, 1), Arrays.asList(3, 2, 1), true)); - waitForVerifyAssignment(cluster.adminClient, assignment, false, + waitForVerifyAssignment(adminClient, assignment, false, new VerifyAssignmentResult(finalAssignment)); } /** * Test running a reassignment and then cancelling it. */ - @ParameterizedTest(name = "{displayName}.quorum={0}") - @ValueSource(strings = {"zk", "kraft"}) - public void testCancellation(String quorum) throws Exception { + @ClusterTemplate("zkAndKRaft") + public void testCancellation(ClusterInstance cluster) throws Exception { + this.cluster = cluster; TopicPartition foo0 = new TopicPartition("foo", 0); TopicPartition baz1 = new TopicPartition("baz", 1); - cluster = new ReassignPartitionsTestCluster(Collections.emptyMap(), Collections.emptyMap()); - cluster.setup(); - cluster.produceMessages(foo0.topic(), foo0.partition(), 200); - cluster.produceMessages(baz1.topic(), baz1.partition(), 200); + createTopics(); + produceMessages(foo0.topic(), foo0.partition(), 200); + produceMessages(baz1.topic(), baz1.partition(), 200); String assignment = "{\"version\":1,\"partitions\":" + "[{\"topic\":\"foo\",\"partition\":0,\"replicas\":[0,1,3],\"log_dirs\":[\"any\",\"any\",\"any\"]}," + "{\"topic\":\"baz\",\"partition\":1,\"replicas\":[0,2,3],\"log_dirs\":[\"any\",\"any\",\"any\"]}" + @@ -375,7 +397,7 @@ public void testCancellation(String quorum) throws Exception { assertEquals(unthrottledBrokerConfigs, describeBrokerLevelThrottles(unthrottledBrokerConfigs.keySet())); long interBrokerThrottle = 1L; - runExecuteAssignment(cluster.adminClient, false, assignment, interBrokerThrottle, -1L); + runExecuteAssignment(adminClient, false, assignment, interBrokerThrottle, -1L); waitForInterBrokerThrottle(Arrays.asList(0, 1, 2, 3), interBrokerThrottle); Map partStates = new HashMap<>(); @@ -385,31 +407,30 @@ public void testCancellation(String quorum) throws Exception { // Verify that the reassignment is running. The very low throttle should keep it // from completing before this runs. - waitForVerifyAssignment(cluster.adminClient, assignment, true, + waitForVerifyAssignment(adminClient, assignment, true, new VerifyAssignmentResult(partStates, true, Collections.emptyMap(), false)); // Cancel the reassignment. - assertEquals(new Tuple2<>(new HashSet<>(Arrays.asList(foo0, baz1)), Collections.emptySet()), runCancelAssignment(cluster.adminClient, assignment, true)); + assertEquals(new Tuple2<>(new HashSet<>(Arrays.asList(foo0, baz1)), Collections.emptySet()), runCancelAssignment(adminClient, assignment, true)); // Broker throttles are still active because we passed --preserve-throttles waitForInterBrokerThrottle(Arrays.asList(0, 1, 2, 3), interBrokerThrottle); // Cancelling the reassignment again should reveal nothing to cancel. - assertEquals(new Tuple2<>(Collections.emptySet(), Collections.emptySet()), runCancelAssignment(cluster.adminClient, assignment, false)); + assertEquals(new Tuple2<>(Collections.emptySet(), Collections.emptySet()), runCancelAssignment(adminClient, assignment, false)); // This time, the broker throttles were removed. waitForBrokerLevelThrottles(unthrottledBrokerConfigs); // Verify that there are no ongoing reassignments. - assertFalse(runVerifyAssignment(cluster.adminClient, assignment, false).partsOngoing); + assertFalse(runVerifyAssignment(adminClient, assignment, false).partsOngoing); // Verify that the partition is removed from cancelled replicas verifyReplicaDeleted(foo0, 3); verifyReplicaDeleted(baz1, 3); } - @ParameterizedTest(name = "{displayName}.quorum={0}") - @ValueSource(strings = {"zk", "kraft"}) - public void testCancellationWithAddingReplicaInIsr(String quorum) throws Exception { + @ClusterTemplate("zkAndKRaft") + public void testCancellationWithAddingReplicaInIsr(ClusterInstance cluster) throws Exception { + this.cluster = cluster; TopicPartition foo0 = new TopicPartition("foo", 0); - cluster = new ReassignPartitionsTestCluster(Collections.emptyMap(), Collections.emptyMap()); - cluster.setup(); - cluster.produceMessages(foo0.topic(), foo0.partition(), 200); + createTopics(); + produceMessages(foo0.topic(), foo0.partition(), 200); // The reassignment will bring replicas 3 and 4 into the replica set and remove 1 and 2. String assignment = "{\"version\":1,\"partitions\":" + @@ -418,7 +439,7 @@ public void testCancellationWithAddingReplicaInIsr(String quorum) throws Excepti // We will throttle replica 4 so that only replica 3 joins the ISR TestUtils.setReplicationThrottleForPartitions( - cluster.adminClient, + adminClient, toSeq(Collections.singleton(4)), toSet(Collections.singleton(foo0)), 1 @@ -426,20 +447,20 @@ public void testCancellationWithAddingReplicaInIsr(String quorum) throws Excepti // Execute the assignment and wait for replica 3 (only) to join the ISR runExecuteAssignment( - cluster.adminClient, + adminClient, false, assignment, -1L, -1L ); TestUtils.waitUntilTrue( - () -> Objects.equals(TestUtils.currentIsr(cluster.adminClient, foo0), toSet(Arrays.asList(0, 1, 2, 3))), + () -> Objects.equals(TestUtils.currentIsr(adminClient, foo0), toSet(Arrays.asList(0, 1, 2, 3))), () -> "Timed out while waiting for replica 3 to join the ISR", org.apache.kafka.test.TestUtils.DEFAULT_MAX_WAIT_MS, 100L ); // Now cancel the assignment and verify that the partition is removed from cancelled replicas - assertEquals(new Tuple2<>(Collections.singleton(foo0), Collections.emptySet()), runCancelAssignment(cluster.adminClient, assignment, true)); + assertEquals(new Tuple2<>(Collections.singleton(foo0), Collections.emptySet()), runCancelAssignment(adminClient, assignment, true)); verifyReplicaDeleted(foo0, 3); verifyReplicaDeleted(foo0, 4); } @@ -449,7 +470,7 @@ private void verifyReplicaDeleted( Integer replicaId ) { TestUtils.waitUntilTrue(() -> { - KafkaBroker server = cluster.servers.get(replicaId); + KafkaBroker server = servers().get(replicaId); HostedPartition partition = server.replicaManager().getPartition(topicPartition); Option log = server.logManager().getLog(topicPartition, false); return partition == HostedPartition.None$.MODULE$ && log.isEmpty(); @@ -507,12 +528,12 @@ private Map> describeBrokerLevelThrottles(Collection< Map> results = new HashMap<>(); for (Integer brokerId : brokerIds) { ConfigResource brokerResource = new ConfigResource(ConfigResource.Type.BROKER, brokerId.toString()); - Config brokerConfigs = cluster.adminClient.describeConfigs(Collections.singleton(brokerResource)).values() + Config brokerConfigs = adminClient.describeConfigs(Collections.singleton(brokerResource)).values() .get(brokerResource) .get(); Map throttles = new HashMap<>(); - BROKER_LEVEL_THROTTLES.stream().forEach(throttleName -> { + BROKER_LEVEL_THROTTLES.forEach(throttleName -> { String configValue = Optional.ofNullable(brokerConfigs.get(throttleName)).map(ConfigEntry::value).orElse("-1"); throttles.put(throttleName, Long.parseLong(configValue)); }); @@ -524,14 +545,14 @@ private Map> describeBrokerLevelThrottles(Collection< /** * Test moving partitions between directories. */ - @ParameterizedTest(name = "{displayName}.quorum={0}") - @ValueSource(strings = "zk") // JBOD not yet implemented for KRaft - public void testLogDirReassignment(String quorum) throws Exception { + @ClusterTemplate("zk") // JBOD not yet implemented for KRaft + public void testLogDirReassignment(ClusterInstance cluster) throws Exception { + this.cluster = cluster; + TopicPartition topicPartition = new TopicPartition("foo", 0); - cluster = new ReassignPartitionsTestCluster(Collections.emptyMap(), Collections.emptyMap()); - cluster.setup(); - cluster.produceMessages(topicPartition.topic(), topicPartition.partition(), 700); + createTopics(); + produceMessages(topicPartition.topic(), topicPartition.partition(), 700); int targetBrokerId = 0; List replicas = Arrays.asList(0, 1, 2); @@ -540,11 +561,11 @@ public void testLogDirReassignment(String quorum) throws Exception { // Start the replica move, but throttle it to be very slow so that it can't complete // before our next checks happen. long logDirThrottle = 1L; - runExecuteAssignment(cluster.adminClient, false, reassignment.json, + runExecuteAssignment(adminClient, false, reassignment.json, -1L, logDirThrottle); // Check the output of --verify - waitForVerifyAssignment(cluster.adminClient, reassignment.json, true, + waitForVerifyAssignment(adminClient, reassignment.json, true, new VerifyAssignmentResult(Collections.singletonMap( topicPartition, new PartitionReassignmentState(Arrays.asList(0, 1, 2), Arrays.asList(0, 1, 2), true) ), false, Collections.singletonMap( @@ -554,7 +575,7 @@ topicPartition, new PartitionReassignmentState(Arrays.asList(0, 1, 2), Arrays.as waitForLogDirThrottle(Collections.singleton(0), logDirThrottle); // Remove the throttle - cluster.adminClient.incrementalAlterConfigs(Collections.singletonMap( + adminClient.incrementalAlterConfigs(Collections.singletonMap( new ConfigResource(ConfigResource.Type.BROKER, "0"), Collections.singletonList(new AlterConfigOp( new ConfigEntry(ReassignPartitionsCommand.BROKER_LEVEL_LOG_DIR_THROTTLE, ""), AlterConfigOp.OpType.DELETE)))) @@ -562,7 +583,7 @@ topicPartition, new PartitionReassignmentState(Arrays.asList(0, 1, 2), Arrays.as waitForBrokerLevelThrottles(unthrottledBrokerConfigs); // Wait for the directory movement to complete. - waitForVerifyAssignment(cluster.adminClient, reassignment.json, true, + waitForVerifyAssignment(adminClient, reassignment.json, true, new VerifyAssignmentResult(Collections.singletonMap( topicPartition, new PartitionReassignmentState(Arrays.asList(0, 1, 2), Arrays.asList(0, 1, 2), true) ), false, Collections.singletonMap( @@ -570,18 +591,18 @@ topicPartition, new PartitionReassignmentState(Arrays.asList(0, 1, 2), Arrays.as new CompletedMoveState(reassignment.targetDir) ), false)); - BrokerDirs info1 = new BrokerDirs(cluster.adminClient.describeLogDirs(IntStream.range(0, 4).boxed().collect(Collectors.toList())), 0); + BrokerDirs info1 = new BrokerDirs(adminClient.describeLogDirs(IntStream.range(0, 4).boxed().collect(Collectors.toList())), 0); assertEquals(reassignment.targetDir, info1.curLogDirs.getOrDefault(topicPartition, "")); } - @ParameterizedTest(name = "{displayName}.quorum={0}") - @ValueSource(strings = "zk") // JBOD not yet implemented for KRaft - public void testAlterLogDirReassignmentThrottle(String quorum) throws Exception { + @ClusterTemplate("zk") // JBOD not yet implemented for KRaft + public void testAlterLogDirReassignmentThrottle(ClusterInstance cluster) throws Exception { + this.cluster = cluster; + TopicPartition topicPartition = new TopicPartition("foo", 0); - cluster = new ReassignPartitionsTestCluster(Collections.emptyMap(), Collections.emptyMap()); - cluster.setup(); - cluster.produceMessages(topicPartition.topic(), topicPartition.partition(), 700); + createTopics(); + produceMessages(topicPartition.topic(), topicPartition.partition(), 700); int targetBrokerId = 0; List replicas = Arrays.asList(0, 1, 2); @@ -589,17 +610,17 @@ public void testAlterLogDirReassignmentThrottle(String quorum) throws Exception // Start the replica move with a low throttle so it does not complete long initialLogDirThrottle = 1L; - runExecuteAssignment(cluster.adminClient, false, reassignment.json, + runExecuteAssignment(adminClient, false, reassignment.json, -1L, initialLogDirThrottle); waitForLogDirThrottle(new HashSet<>(Collections.singletonList(0)), initialLogDirThrottle); // Now increase the throttle and verify that the log dir movement completes long updatedLogDirThrottle = 3000000L; - runExecuteAssignment(cluster.adminClient, true, reassignment.json, + runExecuteAssignment(adminClient, true, reassignment.json, -1L, updatedLogDirThrottle); waitForLogDirThrottle(Collections.singleton(0), updatedLogDirThrottle); - waitForVerifyAssignment(cluster.adminClient, reassignment.json, true, + waitForVerifyAssignment(adminClient, reassignment.json, true, new VerifyAssignmentResult(Collections.singletonMap( topicPartition, new PartitionReassignmentState(Arrays.asList(0, 1, 2), Arrays.asList(0, 1, 2), true) ), false, Collections.singletonMap( @@ -608,7 +629,7 @@ topicPartition, new PartitionReassignmentState(Arrays.asList(0, 1, 2), Arrays.as ), false)); } - class LogDirReassignment { + static class LogDirReassignment { final String json; final String currentDir; final String targetDir; @@ -624,7 +645,7 @@ private LogDirReassignment buildLogDirReassignment(TopicPartition topicPartition int brokerId, List replicas) throws ExecutionException, InterruptedException { - DescribeLogDirsResult describeLogDirsResult = cluster.adminClient.describeLogDirs( + DescribeLogDirsResult describeLogDirsResult = adminClient.describeLogDirs( IntStream.range(0, 4).boxed().collect(Collectors.toList())); BrokerDirs logDirInfo = new BrokerDirs(describeLogDirsResult, brokerId); @@ -655,8 +676,6 @@ private LogDirReassignment buildLogDirReassignment(TopicPartition topicPartition return new LogDirReassignment(reassignmentJson, currentDir, newDir); } - - private VerifyAssignmentResult runVerifyAssignment(Admin adminClient, String jsonString, Boolean preserveThrottles) throws ExecutionException, InterruptedException, JsonProcessingException { System.out.println("==> verifyAssignment(adminClient, jsonString=" + jsonString); @@ -664,9 +683,9 @@ private VerifyAssignmentResult runVerifyAssignment(Admin adminClient, String jso } private void waitForVerifyAssignment(Admin adminClient, - String jsonString, - Boolean preserveThrottles, - VerifyAssignmentResult expectedResult) { + String jsonString, + Boolean preserveThrottles, + VerifyAssignmentResult expectedResult) { final VerifyAssignmentResult[] latestResult = {null}; TestUtils.waitUntilTrue( () -> { @@ -698,7 +717,7 @@ private Tuple2, Set> runCancelAssignm return cancelAssignment(adminClient, jsonString, preserveThrottles, 10000L, Time.SYSTEM); } - class BrokerDirs { + static class BrokerDirs { final DescribeLogDirsResult result; final int brokerId; @@ -723,131 +742,99 @@ public BrokerDirs(DescribeLogDirsResult result, int brokerId) throws ExecutionEx } } - class ReassignPartitionsTestCluster implements Closeable { - private final Map configOverrides; + private void createTopics() throws InterruptedException, ExecutionException { + cluster.waitForReadyBrokers(); - private final Map> brokerConfigOverrides; + adminClient = cluster.createAdminClient(); - private final List brokerConfigs = new ArrayList<>(); + adminClient.createTopics(topics.entrySet().stream().map(e -> { + Map> partMap = new HashMap<>(); - private final Map brokers = new HashMap<>(); { - brokers.put(0, "rack0"); - brokers.put(1, "rack0"); - brokers.put(2, "rack1"); - brokers.put(3, "rack1"); - brokers.put(4, "rack1"); - } + Iterator> partsIter = e.getValue().iterator(); + int index = 0; + while (partsIter.hasNext()) { + partMap.put(index, partsIter.next()); + index++; + } + return new NewTopic(e.getKey(), partMap); + }).collect(Collectors.toList())).all().get(); - private final Map>> topics = new HashMap<>(); { - topics.put("foo", Arrays.asList(Arrays.asList(0, 1, 2), Arrays.asList(1, 2, 3))); - topics.put("bar", Arrays.asList(Arrays.asList(3, 2, 1))); - topics.put("baz", Arrays.asList(Arrays.asList(1, 0, 2), Arrays.asList(2, 0, 1), Arrays.asList(0, 2, 1))); - } + topics.forEach((topicName, parts) -> + TestUtils.waitForAllPartitionsMetadata(toSeq(servers()), topicName, parts.size())); - private List servers = new ArrayList<>(); - - private String brokerList; - - private Admin adminClient; - - public ReassignPartitionsTestCluster(Map configOverrides, Map> brokerConfigOverrides) { - this.configOverrides = configOverrides; - this.brokerConfigOverrides = brokerConfigOverrides; - - brokers.forEach((brokerId, rack) -> { - Properties config = TestUtils.createBrokerConfig( - brokerId, - zkConnectOrNull(), - false, // shorten test time - true, - TestUtils.RandomPort(), - scala.None$.empty(), - scala.None$.empty(), - scala.None$.empty(), - true, - false, - TestUtils.RandomPort(), - false, - TestUtils.RandomPort(), - false, - TestUtils.RandomPort(), - Some$.MODULE$.apply(rack), - 3, - false, - 1, - (short) 1, - false); - // shorter backoff to reduce test durations when no active partitions are eligible for fetching due to throttling - config.setProperty(KafkaConfig.ReplicaFetchBackoffMsProp(), "100"); - // Don't move partition leaders automatically. - config.setProperty(KafkaConfig.AutoLeaderRebalanceEnableProp(), "false"); - config.setProperty(KafkaConfig.ReplicaLagTimeMaxMsProp(), "1000"); - configOverrides.forEach(config::setProperty); - brokerConfigOverrides.getOrDefault(brokerId, Collections.emptyMap()).forEach(config::setProperty); - - brokerConfigs.add(new KafkaConfig(config)); - }); + if (cluster.isKRaftTest()) { + TestUtils.ensureConsistentKRaftMetadata( + toSeq(servers()), + controllerServer(), + "Timeout waiting for controller metadata propagating to brokers" + ); } + } - public void setup() throws ExecutionException, InterruptedException { - createServers(); - createTopics(); - } + public void produceMessages(String topic, int partition, int numMessages) { + Properties props = new Properties(); + props.put(BOOTSTRAP_SERVERS_CONFIG, cluster.bootstrapServers()); + props.put(KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class); + props.put(VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class); - public void createServers() { - brokers.keySet().forEach(brokerId -> - servers.add(createBroker(brokerConfigs.get(brokerId), Time.SYSTEM, true, scala.None$.empty())) - ); + try (KafkaProducer producer = new KafkaProducer<>(props)) { + IntStream.range(0, numMessages).forEach(i -> + producer.send(new ProducerRecord<>(topic, partition, null, new byte[10000]))); } + } + + public List servers() { + Stream brokers = cluster.isKRaftTest() + ? ((RaftClusterInstance) cluster).brokers() + : ((ZkClusterInvocationContext.ZkClusterInstance) cluster).servers(); - public void createTopics() throws ExecutionException, InterruptedException { - TestUtils.waitUntilBrokerMetadataIsPropagated(toSeq(servers), org.apache.kafka.test.TestUtils.DEFAULT_MAX_WAIT_MS); - brokerList = TestUtils.plaintextBootstrapServers(toSeq(servers)); + return brokers.collect(Collectors.toList()); + } - adminClient = Admin.create(Collections.singletonMap(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, brokerList)); + public ControllerServer controllerServer() { + return ((RaftClusterInstance) cluster).controllerServers().iterator().next(); + } - adminClient.createTopics(topics.entrySet().stream().map(e -> { - Map> partMap = new HashMap<>(); + public static void zkAndKRaft(ClusterGenerator generator) { + generator.accept(clusterConfig(ZK)); + generator.accept(clusterConfig(KRAFT)); + } - Iterator> partsIter = e.getValue().iterator(); - int index = 0; - while (partsIter.hasNext()) { - partMap.put(index, partsIter.next()); - index++; - } - return new NewTopic(e.getKey(), partMap); - }).collect(Collectors.toList())).all().get(); - topics.forEach((topicName, parts) -> { - TestUtils.waitForAllPartitionsMetadata(toSeq(servers), topicName, parts.size()); - }); + public static void zk(ClusterGenerator generator) { + generator.accept(clusterConfig(ZK)); + } - if (isKRaftTest()) { - TestUtils.ensureConsistentKRaftMetadata( - toSeq(cluster.servers), - controllerServer(), - "Timeout waiting for controller metadata propagating to brokers" - ); - } - } + private static ClusterConfig clusterConfig(Type type) { + ClusterConfig clusterConfig = ClusterConfig.defaultClusterBuilder() + .type(type) + .brokers(5) + .build(); + + BROKERS.forEach((brokerId, rack) -> { + Properties config = clusterConfig.brokerServerProperties(brokerId); + config.put(KafkaConfig.RackProp(), rack); + config.put(KafkaConfig.ControlledShutdownEnableProp(), false); // shorten test time + String logDirs = IntStream.range(1, 4).mapToObj(i -> { + // We would like to allow user to specify both relative path and absolute path as log directory for backward-compatibility reason + // We can verify this by using a mixture of relative path and absolute path as log directories in the test + return (i % 2 == 0 + ? TestUtils.tempDir() + : TestUtils.tempRelativeDir("data")).getAbsolutePath(); + }).collect(Collectors.joining(",")); + config.put(KafkaConfig.LogDirsProp(), logDirs); + // shorter backoff to reduce test durations when no active partitions are eligible for fetching due to throttling + config.setProperty(KafkaConfig.ReplicaFetchBackoffMsProp(), "100"); + // Don't move partition leaders automatically. + config.setProperty(KafkaConfig.AutoLeaderRebalanceEnableProp(), "false"); + config.setProperty(KafkaConfig.ReplicaLagTimeMaxMsProp(), "1000"); +/* + configOverrides.forEach(config::setProperty); - public void produceMessages(String topic, int partition, int numMessages) { - List> records = IntStream.range(0, numMessages).mapToObj(i -> - new ProducerRecord(topic, partition, - null, new byte[10000])).collect(Collectors.toList()); - TestUtils.produceMessages(toSeq(servers), toSeq(records), -1); - } + brokerConfigOverrides.getOrDefault(brokerId, Collections.emptyMap()).forEach(config::setProperty); +*/ + }); - @Override - public void close() throws IOException { - brokerList = null; - Utils.closeQuietly(adminClient, "adminClient"); - adminClient = null; - try { - TestUtils.shutdownServers(toSeq(servers), true); - } finally { - servers.clear(); - } - } + return clusterConfig; } private static Seq toSeq(Collection col) { From 68a0301536135b666095c4ffa13610951b9f6a9f Mon Sep 17 00:00:00 2001 From: nizhikov Date: Fri, 29 Sep 2023 19:55:10 +0300 Subject: [PATCH 58/78] KAFKA-14595 Revert Integration test to QuorumTestHarness --- .../ReassignPartitionsIntegrationTest.java | 509 +++++++++--------- 1 file changed, 261 insertions(+), 248 deletions(-) diff --git a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsIntegrationTest.java b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsIntegrationTest.java index 724ee0c54bb89..b6de697c0eb6c 100644 --- a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsIntegrationTest.java +++ b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsIntegrationTest.java @@ -19,47 +19,44 @@ import com.fasterxml.jackson.core.JsonProcessingException; import kafka.cluster.Partition; import kafka.log.UnifiedLog; -import kafka.server.ControllerServer; import kafka.server.HostedPartition; import kafka.server.IsrChangePropagationConfig; import kafka.server.KafkaBroker; import kafka.server.KafkaConfig; +import kafka.server.QuorumTestHarness; import kafka.server.ZkAlterPartitionManager; -import kafka.test.ClusterConfig; -import kafka.test.ClusterGenerator; -import kafka.test.ClusterInstance; -import kafka.test.annotation.ClusterTemplate; -import kafka.test.annotation.ClusterTest; -import kafka.test.annotation.ClusterTestDefaults; -import kafka.test.annotation.Type; -import kafka.test.junit.ClusterTestExtensions; -import kafka.test.junit.RaftClusterInvocationContext.RaftClusterInstance; -import kafka.test.junit.ZkClusterInvocationContext; import kafka.utils.TestUtils; import org.apache.kafka.clients.admin.Admin; +import org.apache.kafka.clients.admin.AdminClientConfig; import org.apache.kafka.clients.admin.AlterConfigOp; import org.apache.kafka.clients.admin.Config; import org.apache.kafka.clients.admin.ConfigEntry; import org.apache.kafka.clients.admin.DescribeLogDirsResult; import org.apache.kafka.clients.admin.NewTopic; import org.apache.kafka.clients.consumer.Consumer; -import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.TopicPartitionReplica; import org.apache.kafka.common.config.ConfigResource; import org.apache.kafka.common.security.auth.SecurityProtocol; import org.apache.kafka.common.serialization.ByteArrayDeserializer; -import org.apache.kafka.common.serialization.ByteArraySerializer; import org.apache.kafka.common.utils.Time; +import org.apache.kafka.common.utils.Utils; import org.apache.kafka.tools.TerseException; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import scala.None$; import scala.Option; +import scala.Some$; import scala.collection.JavaConverters; import scala.collection.Seq; +import java.io.Closeable; +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -76,13 +73,7 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; import java.util.stream.IntStream; -import java.util.stream.Stream; -import static kafka.test.annotation.Type.KRAFT; -import static kafka.test.annotation.Type.ZK; -import static org.apache.kafka.clients.CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG; -import static org.apache.kafka.clients.producer.ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG; -import static org.apache.kafka.clients.producer.ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG; import static org.apache.kafka.server.common.MetadataVersion.IBP_2_7_IV1; import static org.apache.kafka.tools.reassign.ReassignPartitionsCommand.BROKER_LEVEL_THROTTLES; import static org.apache.kafka.tools.reassign.ReassignPartitionsCommand.cancelAssignment; @@ -92,16 +83,15 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; -@ExtendWith(value = ClusterTestExtensions.class) -@ClusterTestDefaults(brokers = 5) -@Tag("integration") -public class ReassignPartitionsIntegrationTest { - private static final Map BROKERS = new HashMap<>(); static { - BROKERS.put(0, "rack0"); - BROKERS.put(1, "rack0"); - BROKERS.put(2, "rack1"); - BROKERS.put(3, "rack1"); - BROKERS.put(4, "rack1"); +@Timeout(300) +public class ReassignPartitionsIntegrationTest extends QuorumTestHarness { + ReassignPartitionsTestCluster cluster; + + @AfterEach + @Override + public void tearDown() { + Utils.closeQuietly(cluster, "ReassignPartitionsTestCluster"); + super.tearDown(); } private final Map> unthrottledBrokerConfigs = new HashMap<>(); { @@ -110,67 +100,50 @@ public class ReassignPartitionsIntegrationTest { .collect(Collectors.toMap(throttle -> throttle, throttle -> -1L)))); } - private final Map>> topics = new HashMap<>(); { - topics.put("foo", Arrays.asList(Arrays.asList(0, 1, 2), Arrays.asList(1, 2, 3))); - topics.put("bar", Arrays.asList(Arrays.asList(3, 2, 1))); - topics.put("baz", Arrays.asList(Arrays.asList(1, 0, 2), Arrays.asList(2, 0, 1), Arrays.asList(0, 2, 1))); - } - - private ClusterInstance cluster; - - private Admin adminClient; - - @ClusterTemplate("zkAndKRaft") - public void testReassignment(ClusterInstance cluster) throws Exception { - this.cluster = cluster; - createTopics(); + @ParameterizedTest(name = "{displayName}.quorum={0}") + @ValueSource(strings = {"zk", "kraft"}) + public void testReassignment(String quorum) throws Exception { + cluster = new ReassignPartitionsTestCluster(Collections.emptyMap(), Collections.emptyMap()); + cluster.setup(); executeAndVerifyReassignment(); } - public static void zkAlterPartitionDisabled(ClusterGenerator generator) { - ClusterConfig cfg = clusterConfig(ZK); - - BROKERS.forEach((brokerId, rack) -> { - cfg.brokerServerProperties(brokerId).put(KafkaConfig.InterBrokerProtocolVersionProp(), IBP_2_7_IV1.version()); - }); - - generator.accept(cfg); - } - - @ClusterTemplate("zkAlterPartitionDisabled") - public void testReassignmentWithAlterPartitionDisabled(ClusterInstance cluster) throws Exception { - this.cluster = cluster; + @ParameterizedTest(name = "{displayName}.quorum={0}") + @ValueSource(strings = "zk") // Note: KRaft requires AlterPartition + public void testReassignmentWithAlterPartitionDisabled(String quorum) throws Exception { // Test reassignment when the IBP is on an older version which does not use // the `AlterPartition` API. In this case, the controller will register individual // watches for each reassigning partition so that the reassignment can be // completed as soon as the ISR is expanded. - createTopics(); + Map configOverrides = Collections.singletonMap(KafkaConfig.InterBrokerProtocolVersionProp(), IBP_2_7_IV1.version()); + cluster = new ReassignPartitionsTestCluster(configOverrides, Collections.emptyMap()); + cluster.setup(); executeAndVerifyReassignment(); } - public static void zkPartialUpgrade(ClusterGenerator generator) { - // Override change notification settings so that test is not delayed by ISR - // change notification delay - ZkAlterPartitionManager.DefaultIsrPropagationConfig_$eq(new IsrChangePropagationConfig(500, 100, 500)); - ClusterConfig cfg = clusterConfig(ZK); - - for (int brokerId = 1; brokerId < 4; brokerId++) - cfg.brokerServerProperties(brokerId).put(KafkaConfig.InterBrokerProtocolVersionProp(), IBP_2_7_IV1.version()); - - generator.accept(cfg); - } - - @ClusterTest(clusterType = ZK) // Note: KRaft requires AlterPartition - @ClusterTemplate("zkPartialUpgrade") - public void testReassignmentCompletionDuringPartialUpgrade(ClusterInstance cluster) throws Exception { - this.cluster = cluster; + @ParameterizedTest(name = "{displayName}.quorum={0}") + @ValueSource(strings = "zk") // Note: KRaft requires AlterPartition + public void testReassignmentCompletionDuringPartialUpgrade(String quorum) throws Exception { // Test reassignment during a partial upgrade when some brokers are relying on // `AlterPartition` and some rely on the old notification logic through Zookeeper. // In this test case, broker 0 starts up first on the latest IBP and is typically // elected as controller. The three remaining brokers start up on the older IBP. // We want to ensure that reassignment can still complete through the ISR change // notification path even though the controller expects `AlterPartition`. - createTopics(); + + // Override change notification settings so that test is not delayed by ISR + // change notification delay + ZkAlterPartitionManager.DefaultIsrPropagationConfig_$eq(new IsrChangePropagationConfig(500, 100, 500)); + + Map oldIbpConfig = Collections.singletonMap(KafkaConfig.InterBrokerProtocolVersionProp(), IBP_2_7_IV1.version()); + Map> brokerConfigOverrides = new HashMap<>(); + brokerConfigOverrides.put(1, oldIbpConfig); + brokerConfigOverrides.put(2, oldIbpConfig); + brokerConfigOverrides.put(3, oldIbpConfig); + + cluster = new ReassignPartitionsTestCluster(Collections.emptyMap(), brokerConfigOverrides); + cluster.setup(); + executeAndVerifyReassignment(); } @@ -189,21 +162,21 @@ private void executeAndVerifyReassignment() throws ExecutionException, Interrupt initialAssignment.put(foo0, new PartitionReassignmentState(Arrays.asList(0, 1, 2), Arrays.asList(0, 1, 3), true)); initialAssignment.put(bar0, new PartitionReassignmentState(Arrays.asList(3, 2, 1), Arrays.asList(3, 2, 0), true)); - waitForVerifyAssignment(adminClient, assignment, false, + waitForVerifyAssignment(cluster.adminClient, assignment, false, new VerifyAssignmentResult(initialAssignment)); // Execute the assignment - runExecuteAssignment(adminClient, false, assignment, -1L, -1L); + runExecuteAssignment(cluster.adminClient, false, assignment, -1L, -1L); assertEquals(unthrottledBrokerConfigs, describeBrokerLevelThrottles(unthrottledBrokerConfigs.keySet())); Map finalAssignment = new HashMap<>(); finalAssignment.put(foo0, new PartitionReassignmentState(Arrays.asList(0, 1, 3), Arrays.asList(0, 1, 3), true)); finalAssignment.put(bar0, new PartitionReassignmentState(Arrays.asList(3, 2, 0), Arrays.asList(3, 2, 0), true)); - VerifyAssignmentResult verifyAssignmentResult = runVerifyAssignment(adminClient, assignment, false); + VerifyAssignmentResult verifyAssignmentResult = runVerifyAssignment(cluster.adminClient, assignment, false); assertFalse(verifyAssignmentResult.movesOngoing); // Wait for the assignment to complete - waitForVerifyAssignment(adminClient, assignment, false, + waitForVerifyAssignment(cluster.adminClient, assignment, false, new VerifyAssignmentResult(finalAssignment)); assertEquals(unthrottledBrokerConfigs, @@ -214,41 +187,43 @@ private void executeAndVerifyReassignment() throws ExecutionException, Interrupt verifyReplicaDeleted(bar0, 1); } - @ClusterTemplate("zkAndKRaft") - public void testHighWaterMarkAfterPartitionReassignment(ClusterInstance cluster) throws Exception { - this.cluster = cluster; - createTopics(); + @ParameterizedTest(name = "{displayName}.quorum={0}") + @ValueSource(strings = {"zk", "kraft"}) + public void testHighWaterMarkAfterPartitionReassignment(String quorum) throws Exception { + cluster = new ReassignPartitionsTestCluster(Collections.emptyMap(), Collections.emptyMap()); + cluster.setup(); String assignment = "{\"version\":1,\"partitions\":" + "[{\"topic\":\"foo\",\"partition\":0,\"replicas\":[3,1,2],\"log_dirs\":[\"any\",\"any\",\"any\"]}" + "]}"; // Set the high water mark of foo-0 to 123 on its leader. TopicPartition part = new TopicPartition("foo", 0); - servers().get(0).replicaManager().logManager().truncateFullyAndStartAt(part, 123L, false, None$.empty()); + cluster.servers.get(0).replicaManager().logManager().truncateFullyAndStartAt(part, 123L, false, None$.empty()); // Execute the assignment - runExecuteAssignment(cluster.createAdminClient(), false, assignment, -1L, -1L); + runExecuteAssignment(cluster.adminClient, false, assignment, -1L, -1L); Map finalAssignment = Collections.singletonMap(part, new PartitionReassignmentState(Arrays.asList(3, 1, 2), Arrays.asList(3, 1, 2), true)); // Wait for the assignment to complete - waitForVerifyAssignment(adminClient, assignment, false, + waitForVerifyAssignment(cluster.adminClient, assignment, false, new VerifyAssignmentResult(finalAssignment)); TestUtils.waitUntilTrue(() -> - servers().get(3).replicaManager().onlinePartition(part). + cluster.servers.get(3).replicaManager().onlinePartition(part). map(Partition::leaderLogIfLocal).isDefined(), () -> "broker 3 should be the new leader", org.apache.kafka.test.TestUtils.DEFAULT_MAX_WAIT_MS, 10L); - assertEquals(123L, servers().get(3).replicaManager().localLogOrException(part).highWatermark(), + assertEquals(123L, cluster.servers.get(3).replicaManager().localLogOrException(part).highWatermark(), "Expected broker 3 to have the correct high water mark for the partition."); } - @ClusterTemplate("zkAndKRaft") - public void testAlterReassignmentThrottle(ClusterInstance cluster) throws Exception { - this.cluster = cluster; - createTopics(); - produceMessages("foo", 0, 50); - produceMessages("baz", 2, 60); + @ParameterizedTest(name = "{displayName}.quorum={0}") + @ValueSource(strings = {"zk", "kraft"}) + public void testAlterReassignmentThrottle(String quorum) throws Exception { + cluster = new ReassignPartitionsTestCluster(Collections.emptyMap(), Collections.emptyMap()); + cluster.setup(); + cluster.produceMessages("foo", 0, 50); + cluster.produceMessages("baz", 2, 60); String assignment = "{\"version\":1,\"partitions\":" + "[{\"topic\":\"foo\",\"partition\":0,\"replicas\":[0,3,2],\"log_dirs\":[\"any\",\"any\",\"any\"]}," + "{\"topic\":\"baz\",\"partition\":2,\"replicas\":[3,2,1],\"log_dirs\":[\"any\",\"any\",\"any\"]}" + @@ -256,12 +231,12 @@ public void testAlterReassignmentThrottle(ClusterInstance cluster) throws Except // Execute the assignment with a low throttle long initialThrottle = 1L; - runExecuteAssignment(adminClient, false, assignment, initialThrottle, -1L); + runExecuteAssignment(cluster.adminClient, false, assignment, initialThrottle, -1L); waitForInterBrokerThrottle(Arrays.asList(0, 1, 2, 3), initialThrottle); // Now update the throttle and verify the reassignment completes long updatedThrottle = 300000L; - runExecuteAssignment(adminClient, true, assignment, updatedThrottle, -1L); + runExecuteAssignment(cluster.adminClient, true, assignment, updatedThrottle, -1L); waitForInterBrokerThrottle(Arrays.asList(0, 1, 2, 3), updatedThrottle); Map finalAssignment = new HashMap<>(); @@ -271,7 +246,7 @@ public void testAlterReassignmentThrottle(ClusterInstance cluster) throws Except new PartitionReassignmentState(Arrays.asList(3, 2, 1), Arrays.asList(3, 2, 1), true)); // Now remove the throttles. - waitForVerifyAssignment(adminClient, assignment, false, + waitForVerifyAssignment(cluster.adminClient, assignment, false, new VerifyAssignmentResult(finalAssignment)); waitForBrokerLevelThrottles(unthrottledBrokerConfigs); } @@ -279,12 +254,13 @@ public void testAlterReassignmentThrottle(ClusterInstance cluster) throws Except /** * Test running a reassignment with the interBrokerThrottle set. */ - @ClusterTemplate("zk") - public void testThrottledReassignment(ClusterInstance cluster) throws Exception { - this.cluster = cluster; - createTopics(); - produceMessages("foo", 0, 50); - produceMessages("baz", 2, 60); + @ParameterizedTest(name = "{displayName}.quorum={0}") + @ValueSource(strings = {"zk", "kraft"}) + public void testThrottledReassignment(String quorum) throws Exception { + cluster = new ReassignPartitionsTestCluster(Collections.emptyMap(), Collections.emptyMap()); + cluster.setup(); + cluster.produceMessages("foo", 0, 50); + cluster.produceMessages("baz", 2, 60); String assignment = "{\"version\":1,\"partitions\":" + "[{\"topic\":\"foo\",\"partition\":0,\"replicas\":[0,3,2],\"log_dirs\":[\"any\",\"any\",\"any\"]}," + "{\"topic\":\"baz\",\"partition\":2,\"replicas\":[3,2,1],\"log_dirs\":[\"any\",\"any\",\"any\"]}" + @@ -296,12 +272,12 @@ public void testThrottledReassignment(ClusterInstance cluster) throws Exception new PartitionReassignmentState(Arrays.asList(0, 1, 2), Arrays.asList(0, 3, 2), true)); initialAssignment.put(new TopicPartition("baz", 2), new PartitionReassignmentState(Arrays.asList(0, 2, 1), Arrays.asList(3, 2, 1), true)); - assertEquals(new VerifyAssignmentResult(initialAssignment), runVerifyAssignment(adminClient, assignment, false)); + assertEquals(new VerifyAssignmentResult(initialAssignment), runVerifyAssignment(cluster.adminClient, assignment, false)); assertEquals(unthrottledBrokerConfigs, describeBrokerLevelThrottles(unthrottledBrokerConfigs.keySet())); // Execute the assignment long interBrokerThrottle = 300000L; - runExecuteAssignment(adminClient, false, assignment, interBrokerThrottle, -1L); + runExecuteAssignment(cluster.adminClient, false, assignment, interBrokerThrottle, -1L); waitForInterBrokerThrottle(Arrays.asList(0, 1, 2, 3), interBrokerThrottle); Map finalAssignment = new HashMap<>(); @@ -315,7 +291,7 @@ public void testThrottledReassignment(ClusterInstance cluster) throws Exception () -> { try { // Check the reassignment status. - VerifyAssignmentResult result = runVerifyAssignment(adminClient, assignment, true); + VerifyAssignmentResult result = runVerifyAssignment(cluster.adminClient, assignment, true); if (!result.partsOngoing) { return true; @@ -331,35 +307,36 @@ public void testThrottledReassignment(ClusterInstance cluster) throws Exception throw new RuntimeException(e); } }, () -> "Expected reassignment to complete.", org.apache.kafka.test.TestUtils.DEFAULT_MAX_WAIT_MS, 100L); - waitForVerifyAssignment(adminClient, assignment, true, + waitForVerifyAssignment(cluster.adminClient, assignment, true, new VerifyAssignmentResult(finalAssignment)); // The throttles should still have been preserved, since we ran with --preserve-throttles waitForInterBrokerThrottle(Arrays.asList(0, 1, 2, 3), interBrokerThrottle); // Now remove the throttles. - waitForVerifyAssignment(adminClient, assignment, false, + waitForVerifyAssignment(cluster.adminClient, assignment, false, new VerifyAssignmentResult(finalAssignment)); waitForBrokerLevelThrottles(unthrottledBrokerConfigs); } - @ClusterTemplate("zkAndKRaft") - public void testProduceAndConsumeWithReassignmentInProgress(ClusterInstance cluster) throws Exception { - this.cluster = cluster; - createTopics(); - produceMessages("baz", 2, 60); + @ParameterizedTest(name = "{displayName}.quorum={0}") + @ValueSource(strings = {"zk", "kraft"}) + public void testProduceAndConsumeWithReassignmentInProgress(String quorum) throws Exception { + cluster = new ReassignPartitionsTestCluster(Collections.emptyMap(), Collections.emptyMap()); + cluster.setup(); + cluster.produceMessages("baz", 2, 60); String assignment = "{\"version\":1,\"partitions\":" + "[{\"topic\":\"baz\",\"partition\":2,\"replicas\":[3,2,1],\"log_dirs\":[\"any\",\"any\",\"any\"]}" + "]}"; - runExecuteAssignment(adminClient, false, assignment, 300L, -1L); - produceMessages("baz", 2, 100); - Consumer consumer = TestUtils.createConsumer(cluster.bootstrapServers(), + runExecuteAssignment(cluster.adminClient, false, assignment, 300L, -1L); + cluster.produceMessages("baz", 2, 100); + Consumer consumer = TestUtils.createConsumer(cluster.brokerList, "group", "earliest", true, false, 500, SecurityProtocol.PLAINTEXT, - None$.empty(), - None$.empty(), + None$.empty(), + None$.empty(), new ByteArrayDeserializer(), new ByteArrayDeserializer() ); @@ -371,25 +348,26 @@ public void testProduceAndConsumeWithReassignmentInProgress(ClusterInstance clus } finally { consumer.close(); } - TestUtils.removeReplicationThrottleForPartitions(adminClient, toSeq(Arrays.asList(0, 1, 2, 3)), toSet(Collections.singleton(part))); + TestUtils.removeReplicationThrottleForPartitions(cluster.adminClient, toSeq(Arrays.asList(0, 1, 2, 3)), toSet(Collections.singleton(part))); Map finalAssignment = Collections.singletonMap(part, new PartitionReassignmentState(Arrays.asList(3, 2, 1), Arrays.asList(3, 2, 1), true)); - waitForVerifyAssignment(adminClient, assignment, false, + waitForVerifyAssignment(cluster.adminClient, assignment, false, new VerifyAssignmentResult(finalAssignment)); } /** * Test running a reassignment and then cancelling it. */ - @ClusterTemplate("zkAndKRaft") - public void testCancellation(ClusterInstance cluster) throws Exception { - this.cluster = cluster; + @ParameterizedTest(name = "{displayName}.quorum={0}") + @ValueSource(strings = {"zk", "kraft"}) + public void testCancellation(String quorum) throws Exception { TopicPartition foo0 = new TopicPartition("foo", 0); TopicPartition baz1 = new TopicPartition("baz", 1); - createTopics(); - produceMessages(foo0.topic(), foo0.partition(), 200); - produceMessages(baz1.topic(), baz1.partition(), 200); + cluster = new ReassignPartitionsTestCluster(Collections.emptyMap(), Collections.emptyMap()); + cluster.setup(); + cluster.produceMessages(foo0.topic(), foo0.partition(), 200); + cluster.produceMessages(baz1.topic(), baz1.partition(), 200); String assignment = "{\"version\":1,\"partitions\":" + "[{\"topic\":\"foo\",\"partition\":0,\"replicas\":[0,1,3],\"log_dirs\":[\"any\",\"any\",\"any\"]}," + "{\"topic\":\"baz\",\"partition\":1,\"replicas\":[0,2,3],\"log_dirs\":[\"any\",\"any\",\"any\"]}" + @@ -397,7 +375,7 @@ public void testCancellation(ClusterInstance cluster) throws Exception { assertEquals(unthrottledBrokerConfigs, describeBrokerLevelThrottles(unthrottledBrokerConfigs.keySet())); long interBrokerThrottle = 1L; - runExecuteAssignment(adminClient, false, assignment, interBrokerThrottle, -1L); + runExecuteAssignment(cluster.adminClient, false, assignment, interBrokerThrottle, -1L); waitForInterBrokerThrottle(Arrays.asList(0, 1, 2, 3), interBrokerThrottle); Map partStates = new HashMap<>(); @@ -407,30 +385,31 @@ public void testCancellation(ClusterInstance cluster) throws Exception { // Verify that the reassignment is running. The very low throttle should keep it // from completing before this runs. - waitForVerifyAssignment(adminClient, assignment, true, + waitForVerifyAssignment(cluster.adminClient, assignment, true, new VerifyAssignmentResult(partStates, true, Collections.emptyMap(), false)); // Cancel the reassignment. - assertEquals(new Tuple2<>(new HashSet<>(Arrays.asList(foo0, baz1)), Collections.emptySet()), runCancelAssignment(adminClient, assignment, true)); + assertEquals(new Tuple2<>(new HashSet<>(Arrays.asList(foo0, baz1)), Collections.emptySet()), runCancelAssignment(cluster.adminClient, assignment, true)); // Broker throttles are still active because we passed --preserve-throttles waitForInterBrokerThrottle(Arrays.asList(0, 1, 2, 3), interBrokerThrottle); // Cancelling the reassignment again should reveal nothing to cancel. - assertEquals(new Tuple2<>(Collections.emptySet(), Collections.emptySet()), runCancelAssignment(adminClient, assignment, false)); + assertEquals(new Tuple2<>(Collections.emptySet(), Collections.emptySet()), runCancelAssignment(cluster.adminClient, assignment, false)); // This time, the broker throttles were removed. waitForBrokerLevelThrottles(unthrottledBrokerConfigs); // Verify that there are no ongoing reassignments. - assertFalse(runVerifyAssignment(adminClient, assignment, false).partsOngoing); + assertFalse(runVerifyAssignment(cluster.adminClient, assignment, false).partsOngoing); // Verify that the partition is removed from cancelled replicas verifyReplicaDeleted(foo0, 3); verifyReplicaDeleted(baz1, 3); } - @ClusterTemplate("zkAndKRaft") - public void testCancellationWithAddingReplicaInIsr(ClusterInstance cluster) throws Exception { - this.cluster = cluster; + @ParameterizedTest(name = "{displayName}.quorum={0}") + @ValueSource(strings = {"zk", "kraft"}) + public void testCancellationWithAddingReplicaInIsr(String quorum) throws Exception { TopicPartition foo0 = new TopicPartition("foo", 0); - createTopics(); - produceMessages(foo0.topic(), foo0.partition(), 200); + cluster = new ReassignPartitionsTestCluster(Collections.emptyMap(), Collections.emptyMap()); + cluster.setup(); + cluster.produceMessages(foo0.topic(), foo0.partition(), 200); // The reassignment will bring replicas 3 and 4 into the replica set and remove 1 and 2. String assignment = "{\"version\":1,\"partitions\":" + @@ -439,7 +418,7 @@ public void testCancellationWithAddingReplicaInIsr(ClusterInstance cluster) thro // We will throttle replica 4 so that only replica 3 joins the ISR TestUtils.setReplicationThrottleForPartitions( - adminClient, + cluster.adminClient, toSeq(Collections.singleton(4)), toSet(Collections.singleton(foo0)), 1 @@ -447,20 +426,20 @@ public void testCancellationWithAddingReplicaInIsr(ClusterInstance cluster) thro // Execute the assignment and wait for replica 3 (only) to join the ISR runExecuteAssignment( - adminClient, + cluster.adminClient, false, assignment, -1L, -1L ); TestUtils.waitUntilTrue( - () -> Objects.equals(TestUtils.currentIsr(adminClient, foo0), toSet(Arrays.asList(0, 1, 2, 3))), + () -> Objects.equals(TestUtils.currentIsr(cluster.adminClient, foo0), toSet(Arrays.asList(0, 1, 2, 3))), () -> "Timed out while waiting for replica 3 to join the ISR", org.apache.kafka.test.TestUtils.DEFAULT_MAX_WAIT_MS, 100L ); // Now cancel the assignment and verify that the partition is removed from cancelled replicas - assertEquals(new Tuple2<>(Collections.singleton(foo0), Collections.emptySet()), runCancelAssignment(adminClient, assignment, true)); + assertEquals(new Tuple2<>(Collections.singleton(foo0), Collections.emptySet()), runCancelAssignment(cluster.adminClient, assignment, true)); verifyReplicaDeleted(foo0, 3); verifyReplicaDeleted(foo0, 4); } @@ -470,11 +449,11 @@ private void verifyReplicaDeleted( Integer replicaId ) { TestUtils.waitUntilTrue(() -> { - KafkaBroker server = servers().get(replicaId); - HostedPartition partition = server.replicaManager().getPartition(topicPartition); - Option log = server.logManager().getLog(topicPartition, false); - return partition == HostedPartition.None$.MODULE$ && log.isEmpty(); - }, () -> "Timed out waiting for replica " + replicaId + " of " + topicPartition + " to be deleted", + KafkaBroker server = cluster.servers.get(replicaId); + HostedPartition partition = server.replicaManager().getPartition(topicPartition); + Option log = server.logManager().getLog(topicPartition, false); + return partition == HostedPartition.None$.MODULE$ && log.isEmpty(); + }, () -> "Timed out waiting for replica " + replicaId + " of " + topicPartition + " to be deleted", org.apache.kafka.test.TestUtils.DEFAULT_MAX_WAIT_MS, 100L); } @@ -528,12 +507,12 @@ private Map> describeBrokerLevelThrottles(Collection< Map> results = new HashMap<>(); for (Integer brokerId : brokerIds) { ConfigResource brokerResource = new ConfigResource(ConfigResource.Type.BROKER, brokerId.toString()); - Config brokerConfigs = adminClient.describeConfigs(Collections.singleton(brokerResource)).values() + Config brokerConfigs = cluster.adminClient.describeConfigs(Collections.singleton(brokerResource)).values() .get(brokerResource) .get(); Map throttles = new HashMap<>(); - BROKER_LEVEL_THROTTLES.forEach(throttleName -> { + BROKER_LEVEL_THROTTLES.stream().forEach(throttleName -> { String configValue = Optional.ofNullable(brokerConfigs.get(throttleName)).map(ConfigEntry::value).orElse("-1"); throttles.put(throttleName, Long.parseLong(configValue)); }); @@ -545,14 +524,14 @@ private Map> describeBrokerLevelThrottles(Collection< /** * Test moving partitions between directories. */ - @ClusterTemplate("zk") // JBOD not yet implemented for KRaft - public void testLogDirReassignment(ClusterInstance cluster) throws Exception { - this.cluster = cluster; - + @ParameterizedTest(name = "{displayName}.quorum={0}") + @ValueSource(strings = "zk") // JBOD not yet implemented for KRaft + public void testLogDirReassignment(String quorum) throws Exception { TopicPartition topicPartition = new TopicPartition("foo", 0); - createTopics(); - produceMessages(topicPartition.topic(), topicPartition.partition(), 700); + cluster = new ReassignPartitionsTestCluster(Collections.emptyMap(), Collections.emptyMap()); + cluster.setup(); + cluster.produceMessages(topicPartition.topic(), topicPartition.partition(), 700); int targetBrokerId = 0; List replicas = Arrays.asList(0, 1, 2); @@ -561,11 +540,11 @@ public void testLogDirReassignment(ClusterInstance cluster) throws Exception { // Start the replica move, but throttle it to be very slow so that it can't complete // before our next checks happen. long logDirThrottle = 1L; - runExecuteAssignment(adminClient, false, reassignment.json, + runExecuteAssignment(cluster.adminClient, false, reassignment.json, -1L, logDirThrottle); // Check the output of --verify - waitForVerifyAssignment(adminClient, reassignment.json, true, + waitForVerifyAssignment(cluster.adminClient, reassignment.json, true, new VerifyAssignmentResult(Collections.singletonMap( topicPartition, new PartitionReassignmentState(Arrays.asList(0, 1, 2), Arrays.asList(0, 1, 2), true) ), false, Collections.singletonMap( @@ -575,7 +554,7 @@ topicPartition, new PartitionReassignmentState(Arrays.asList(0, 1, 2), Arrays.as waitForLogDirThrottle(Collections.singleton(0), logDirThrottle); // Remove the throttle - adminClient.incrementalAlterConfigs(Collections.singletonMap( + cluster.adminClient.incrementalAlterConfigs(Collections.singletonMap( new ConfigResource(ConfigResource.Type.BROKER, "0"), Collections.singletonList(new AlterConfigOp( new ConfigEntry(ReassignPartitionsCommand.BROKER_LEVEL_LOG_DIR_THROTTLE, ""), AlterConfigOp.OpType.DELETE)))) @@ -583,7 +562,7 @@ topicPartition, new PartitionReassignmentState(Arrays.asList(0, 1, 2), Arrays.as waitForBrokerLevelThrottles(unthrottledBrokerConfigs); // Wait for the directory movement to complete. - waitForVerifyAssignment(adminClient, reassignment.json, true, + waitForVerifyAssignment(cluster.adminClient, reassignment.json, true, new VerifyAssignmentResult(Collections.singletonMap( topicPartition, new PartitionReassignmentState(Arrays.asList(0, 1, 2), Arrays.asList(0, 1, 2), true) ), false, Collections.singletonMap( @@ -591,18 +570,18 @@ topicPartition, new PartitionReassignmentState(Arrays.asList(0, 1, 2), Arrays.as new CompletedMoveState(reassignment.targetDir) ), false)); - BrokerDirs info1 = new BrokerDirs(adminClient.describeLogDirs(IntStream.range(0, 4).boxed().collect(Collectors.toList())), 0); + BrokerDirs info1 = new BrokerDirs(cluster.adminClient.describeLogDirs(IntStream.range(0, 4).boxed().collect(Collectors.toList())), 0); assertEquals(reassignment.targetDir, info1.curLogDirs.getOrDefault(topicPartition, "")); } - @ClusterTemplate("zk") // JBOD not yet implemented for KRaft - public void testAlterLogDirReassignmentThrottle(ClusterInstance cluster) throws Exception { - this.cluster = cluster; - + @ParameterizedTest(name = "{displayName}.quorum={0}") + @ValueSource(strings = "zk") // JBOD not yet implemented for KRaft + public void testAlterLogDirReassignmentThrottle(String quorum) throws Exception { TopicPartition topicPartition = new TopicPartition("foo", 0); - createTopics(); - produceMessages(topicPartition.topic(), topicPartition.partition(), 700); + cluster = new ReassignPartitionsTestCluster(Collections.emptyMap(), Collections.emptyMap()); + cluster.setup(); + cluster.produceMessages(topicPartition.topic(), topicPartition.partition(), 700); int targetBrokerId = 0; List replicas = Arrays.asList(0, 1, 2); @@ -610,17 +589,17 @@ public void testAlterLogDirReassignmentThrottle(ClusterInstance cluster) throws // Start the replica move with a low throttle so it does not complete long initialLogDirThrottle = 1L; - runExecuteAssignment(adminClient, false, reassignment.json, + runExecuteAssignment(cluster.adminClient, false, reassignment.json, -1L, initialLogDirThrottle); waitForLogDirThrottle(new HashSet<>(Collections.singletonList(0)), initialLogDirThrottle); // Now increase the throttle and verify that the log dir movement completes long updatedLogDirThrottle = 3000000L; - runExecuteAssignment(adminClient, true, reassignment.json, + runExecuteAssignment(cluster.adminClient, true, reassignment.json, -1L, updatedLogDirThrottle); waitForLogDirThrottle(Collections.singleton(0), updatedLogDirThrottle); - waitForVerifyAssignment(adminClient, reassignment.json, true, + waitForVerifyAssignment(cluster.adminClient, reassignment.json, true, new VerifyAssignmentResult(Collections.singletonMap( topicPartition, new PartitionReassignmentState(Arrays.asList(0, 1, 2), Arrays.asList(0, 1, 2), true) ), false, Collections.singletonMap( @@ -629,7 +608,7 @@ topicPartition, new PartitionReassignmentState(Arrays.asList(0, 1, 2), Arrays.as ), false)); } - static class LogDirReassignment { + class LogDirReassignment { final String json; final String currentDir; final String targetDir; @@ -645,7 +624,7 @@ private LogDirReassignment buildLogDirReassignment(TopicPartition topicPartition int brokerId, List replicas) throws ExecutionException, InterruptedException { - DescribeLogDirsResult describeLogDirsResult = adminClient.describeLogDirs( + DescribeLogDirsResult describeLogDirsResult = cluster.adminClient.describeLogDirs( IntStream.range(0, 4).boxed().collect(Collectors.toList())); BrokerDirs logDirInfo = new BrokerDirs(describeLogDirsResult, brokerId); @@ -676,6 +655,8 @@ private LogDirReassignment buildLogDirReassignment(TopicPartition topicPartition return new LogDirReassignment(reassignmentJson, currentDir, newDir); } + + private VerifyAssignmentResult runVerifyAssignment(Admin adminClient, String jsonString, Boolean preserveThrottles) throws ExecutionException, InterruptedException, JsonProcessingException { System.out.println("==> verifyAssignment(adminClient, jsonString=" + jsonString); @@ -717,7 +698,7 @@ private Tuple2, Set> runCancelAssignm return cancelAssignment(adminClient, jsonString, preserveThrottles, 10000L, Time.SYSTEM); } - static class BrokerDirs { + class BrokerDirs { final DescribeLogDirsResult result; final int brokerId; @@ -742,99 +723,131 @@ public BrokerDirs(DescribeLogDirsResult result, int brokerId) throws ExecutionEx } } - private void createTopics() throws InterruptedException, ExecutionException { - cluster.waitForReadyBrokers(); + class ReassignPartitionsTestCluster implements Closeable { + private final Map configOverrides; - adminClient = cluster.createAdminClient(); + private final Map> brokerConfigOverrides; - adminClient.createTopics(topics.entrySet().stream().map(e -> { - Map> partMap = new HashMap<>(); - - Iterator> partsIter = e.getValue().iterator(); - int index = 0; - while (partsIter.hasNext()) { - partMap.put(index, partsIter.next()); - index++; - } - return new NewTopic(e.getKey(), partMap); - }).collect(Collectors.toList())).all().get(); + private final List brokerConfigs = new ArrayList<>(); - topics.forEach((topicName, parts) -> - TestUtils.waitForAllPartitionsMetadata(toSeq(servers()), topicName, parts.size())); + private final Map brokers = new HashMap<>(); { + brokers.put(0, "rack0"); + brokers.put(1, "rack0"); + brokers.put(2, "rack1"); + brokers.put(3, "rack1"); + brokers.put(4, "rack1"); + } - if (cluster.isKRaftTest()) { - TestUtils.ensureConsistentKRaftMetadata( - toSeq(servers()), - controllerServer(), - "Timeout waiting for controller metadata propagating to brokers" - ); + private final Map>> topics = new HashMap<>(); { + topics.put("foo", Arrays.asList(Arrays.asList(0, 1, 2), Arrays.asList(1, 2, 3))); + topics.put("bar", Arrays.asList(Arrays.asList(3, 2, 1))); + topics.put("baz", Arrays.asList(Arrays.asList(1, 0, 2), Arrays.asList(2, 0, 1), Arrays.asList(0, 2, 1))); } - } - public void produceMessages(String topic, int partition, int numMessages) { - Properties props = new Properties(); - props.put(BOOTSTRAP_SERVERS_CONFIG, cluster.bootstrapServers()); - props.put(KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class); - props.put(VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class); + private List servers = new ArrayList<>(); + + private String brokerList; + + private Admin adminClient; + + public ReassignPartitionsTestCluster(Map configOverrides, Map> brokerConfigOverrides) { + this.configOverrides = configOverrides; + this.brokerConfigOverrides = brokerConfigOverrides; + + brokers.forEach((brokerId, rack) -> { + Properties config = TestUtils.createBrokerConfig( + brokerId, + zkConnectOrNull(), + false, // shorten test time + true, + TestUtils.RandomPort(), + scala.None$.empty(), + scala.None$.empty(), + scala.None$.empty(), + true, + false, + TestUtils.RandomPort(), + false, + TestUtils.RandomPort(), + false, + TestUtils.RandomPort(), + Some$.MODULE$.apply(rack), + 3, + false, + 1, + (short) 1, + false); + // shorter backoff to reduce test durations when no active partitions are eligible for fetching due to throttling + config.setProperty(KafkaConfig.ReplicaFetchBackoffMsProp(), "100"); + // Don't move partition leaders automatically. + config.setProperty(KafkaConfig.AutoLeaderRebalanceEnableProp(), "false"); + config.setProperty(KafkaConfig.ReplicaLagTimeMaxMsProp(), "1000"); + configOverrides.forEach(config::setProperty); + brokerConfigOverrides.getOrDefault(brokerId, Collections.emptyMap()).forEach(config::setProperty); + + brokerConfigs.add(new KafkaConfig(config)); + }); + } - try (KafkaProducer producer = new KafkaProducer<>(props)) { - IntStream.range(0, numMessages).forEach(i -> - producer.send(new ProducerRecord<>(topic, partition, null, new byte[10000]))); + public void setup() throws ExecutionException, InterruptedException { + createServers(); + createTopics(); } - } - public List servers() { - Stream brokers = cluster.isKRaftTest() - ? ((RaftClusterInstance) cluster).brokers() - : ((ZkClusterInvocationContext.ZkClusterInstance) cluster).servers(); + public void createServers() { + brokers.keySet().forEach(brokerId -> + servers.add(createBroker(brokerConfigs.get(brokerId), Time.SYSTEM, true, scala.None$.empty())) + ); + } - return brokers.collect(Collectors.toList()); - } + public void createTopics() throws ExecutionException, InterruptedException { + TestUtils.waitUntilBrokerMetadataIsPropagated(toSeq(servers), org.apache.kafka.test.TestUtils.DEFAULT_MAX_WAIT_MS); + brokerList = TestUtils.plaintextBootstrapServers(toSeq(servers)); - public ControllerServer controllerServer() { - return ((RaftClusterInstance) cluster).controllerServers().iterator().next(); - } + adminClient = Admin.create(Collections.singletonMap(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, brokerList)); - public static void zkAndKRaft(ClusterGenerator generator) { - generator.accept(clusterConfig(ZK)); - generator.accept(clusterConfig(KRAFT)); - } + adminClient.createTopics(topics.entrySet().stream().map(e -> { + Map> partMap = new HashMap<>(); - public static void zk(ClusterGenerator generator) { - generator.accept(clusterConfig(ZK)); - } + Iterator> partsIter = e.getValue().iterator(); + int index = 0; + while (partsIter.hasNext()) { + partMap.put(index, partsIter.next()); + index++; + } + return new NewTopic(e.getKey(), partMap); + }).collect(Collectors.toList())).all().get(); + topics.forEach((topicName, parts) -> { + TestUtils.waitForAllPartitionsMetadata(toSeq(servers), topicName, parts.size()); + }); - private static ClusterConfig clusterConfig(Type type) { - ClusterConfig clusterConfig = ClusterConfig.defaultClusterBuilder() - .type(type) - .brokers(5) - .build(); - - BROKERS.forEach((brokerId, rack) -> { - Properties config = clusterConfig.brokerServerProperties(brokerId); - config.put(KafkaConfig.RackProp(), rack); - config.put(KafkaConfig.ControlledShutdownEnableProp(), false); // shorten test time - String logDirs = IntStream.range(1, 4).mapToObj(i -> { - // We would like to allow user to specify both relative path and absolute path as log directory for backward-compatibility reason - // We can verify this by using a mixture of relative path and absolute path as log directories in the test - return (i % 2 == 0 - ? TestUtils.tempDir() - : TestUtils.tempRelativeDir("data")).getAbsolutePath(); - }).collect(Collectors.joining(",")); - config.put(KafkaConfig.LogDirsProp(), logDirs); - // shorter backoff to reduce test durations when no active partitions are eligible for fetching due to throttling - config.setProperty(KafkaConfig.ReplicaFetchBackoffMsProp(), "100"); - // Don't move partition leaders automatically. - config.setProperty(KafkaConfig.AutoLeaderRebalanceEnableProp(), "false"); - config.setProperty(KafkaConfig.ReplicaLagTimeMaxMsProp(), "1000"); -/* - configOverrides.forEach(config::setProperty); + if (isKRaftTest()) { + TestUtils.ensureConsistentKRaftMetadata( + toSeq(cluster.servers), + controllerServer(), + "Timeout waiting for controller metadata propagating to brokers" + ); + } + } - brokerConfigOverrides.getOrDefault(brokerId, Collections.emptyMap()).forEach(config::setProperty); -*/ - }); + public void produceMessages(String topic, int partition, int numMessages) { + List> records = IntStream.range(0, numMessages).mapToObj(i -> + new ProducerRecord(topic, partition, + null, new byte[10000])).collect(Collectors.toList()); + TestUtils.produceMessages(toSeq(servers), toSeq(records), -1); + } - return clusterConfig; + @Override + public void close() throws IOException { + brokerList = null; + Utils.closeQuietly(adminClient, "adminClient"); + adminClient = null; + try { + TestUtils.shutdownServers(toSeq(servers), true); + } finally { + servers.clear(); + } + } } private static Seq toSeq(Collection col) { From 7cf703c0c24bc42d19491c4ddef8acab86ec025a Mon Sep 17 00:00:00 2001 From: nizhikov Date: Tue, 3 Oct 2023 00:01:05 +0300 Subject: [PATCH 59/78] KAFKA-14595 Fixes after trunk merge --- .../ReassignPartitionsIntegrationTest.java | 151 +++++++----------- 1 file changed, 54 insertions(+), 97 deletions(-) diff --git a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsIntegrationTest.java b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsIntegrationTest.java index 896e4f74ffdc9..a7e40ce1f6145 100644 --- a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsIntegrationTest.java +++ b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsIntegrationTest.java @@ -16,7 +16,7 @@ */ package org.apache.kafka.tools.reassign; -import kafka.admin.ReassignPartitionsCommand; +import com.fasterxml.jackson.core.JsonProcessingException; import kafka.cluster.Partition; import kafka.log.UnifiedLog; import kafka.server.HostedPartition; @@ -42,6 +42,7 @@ import org.apache.kafka.common.serialization.ByteArrayDeserializer; import org.apache.kafka.common.utils.Time; import org.apache.kafka.common.utils.Utils; +import org.apache.kafka.tools.TerseException; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Timeout; import org.junit.jupiter.params.ParameterizedTest; @@ -49,7 +50,6 @@ import scala.None$; import scala.Option; import scala.Some$; -import scala.Tuple2; import scala.collection.JavaConverters; import scala.collection.Seq; @@ -72,16 +72,16 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; -import static kafka.admin.ReassignPartitionsCommand.brokerLevelFollowerThrottle; -import static kafka.admin.ReassignPartitionsCommand.brokerLevelLeaderThrottle; -import static kafka.admin.ReassignPartitionsCommand.brokerLevelLogDirThrottle; -import static kafka.admin.ReassignPartitionsCommand.brokerLevelThrottles; -import static kafka.admin.ReassignPartitionsCommand.cancelAssignment; -import static kafka.admin.ReassignPartitionsCommand.executeAssignment; -import static kafka.admin.ReassignPartitionsCommand.verifyAssignment; import static org.apache.kafka.server.common.MetadataVersion.IBP_2_7_IV1; import static org.apache.kafka.test.TestUtils.DEFAULT_MAX_WAIT_MS; import static org.apache.kafka.tools.ToolsTestUtils.TEST_WITH_PARAMETERIZED_QUORUM_NAME; +import static org.apache.kafka.tools.reassign.ReassignPartitionsCommand.BROKER_LEVEL_FOLLOWER_THROTTLE; +import static org.apache.kafka.tools.reassign.ReassignPartitionsCommand.BROKER_LEVEL_LEADER_THROTTLE; +import static org.apache.kafka.tools.reassign.ReassignPartitionsCommand.BROKER_LEVEL_LOG_DIR_THROTTLE; +import static org.apache.kafka.tools.reassign.ReassignPartitionsCommand.BROKER_LEVEL_THROTTLES; +import static org.apache.kafka.tools.reassign.ReassignPartitionsCommand.cancelAssignment; +import static org.apache.kafka.tools.reassign.ReassignPartitionsCommand.executeAssignment; +import static org.apache.kafka.tools.reassign.ReassignPartitionsCommand.verifyAssignment; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -102,10 +102,7 @@ public void tearDown() { IntStream.range(0, 4).forEach(brokerId -> { Map brokerConfig = new HashMap<>(); - brokerLevelThrottles().foreach(throttle -> { - brokerConfig.put(throttle, -1L); - return null; - }); + BROKER_LEVEL_THROTTLES.forEach(throttle -> brokerConfig.put(throttle, -1L)); unthrottledBrokerConfigs.put(brokerId, brokerConfig); }); @@ -183,8 +180,8 @@ private void executeAndVerifyReassignment() throws ExecutionException, Interrupt finalAssignment.put(foo0, new PartitionReassignmentState(Arrays.asList(0, 1, 3), Arrays.asList(0, 1, 3), true)); finalAssignment.put(bar0, new PartitionReassignmentState(Arrays.asList(3, 2, 0), Arrays.asList(3, 2, 0), true)); - kafka.admin.ReassignPartitionsCommand.VerifyAssignmentResult verifyAssignmentResult = runVerifyAssignment(cluster.adminClient, assignment, false); - assertFalse(verifyAssignmentResult.movesOngoing()); + VerifyAssignmentResult verifyAssignmentResult = runVerifyAssignment(cluster.adminClient, assignment, false); + assertFalse(verifyAssignmentResult.movesOngoing); // Wait for the assignment to complete waitForVerifyAssignment(cluster.adminClient, assignment, false, @@ -283,7 +280,7 @@ public void testThrottledReassignment(String quorum) throws Exception { new PartitionReassignmentState(Arrays.asList(0, 1, 2), Arrays.asList(0, 3, 2), true)); initialAssignment.put(new TopicPartition("baz", 2), new PartitionReassignmentState(Arrays.asList(0, 2, 1), Arrays.asList(3, 2, 1), true)); - assertEquals(asScala(new VerifyAssignmentResult(initialAssignment)), runVerifyAssignment(cluster.adminClient, assignment, false)); + assertEquals(new VerifyAssignmentResult(initialAssignment), runVerifyAssignment(cluster.adminClient, assignment, false)); assertEquals(unthrottledBrokerConfigs, describeBrokerLevelThrottles(unthrottledBrokerConfigs.keySet())); // Execute the assignment @@ -301,17 +298,17 @@ public void testThrottledReassignment(String quorum) throws Exception { TestUtils.waitUntilTrue( () -> { // Check the reassignment status. - kafka.admin.ReassignPartitionsCommand.VerifyAssignmentResult result = runVerifyAssignment(cluster.adminClient, assignment, true); + VerifyAssignmentResult result = runVerifyAssignment(cluster.adminClient, assignment, true); - if (!result.partsOngoing()) { + if (!result.partsOngoing) { return true; } else { assertFalse( - result.partStates().values().forall(ReassignPartitionsCommand.PartitionReassignmentState::done), + result.partStates.values().stream().allMatch(state -> state.done), "Expected at least one partition reassignment to be ongoing when result = " + result ); - assertEquals(seq(0, 3, 2), result.partStates().get(new TopicPartition("foo", 0)).get().targetReplicas()); - assertEquals(seq(3, 2, 1), result.partStates().get(new TopicPartition("baz", 2)).get().targetReplicas()); + assertEquals(Arrays.asList(0, 3, 2), result.partStates.get(new TopicPartition("foo", 0)).targetReplicas); + assertEquals(Arrays.asList(3, 2, 1), result.partStates.get(new TopicPartition("baz", 2)).targetReplicas); System.out.println("Current result: " + result); waitForInterBrokerThrottle(Arrays.asList(0, 1, 2, 3), interBrokerThrottle); return false; @@ -358,7 +355,7 @@ public void testProduceAndConsumeWithReassignmentInProgress(String quorum) throw } finally { consumer.close(); } - TestUtils.removeReplicationThrottleForPartitions(cluster.adminClient, seq(0, 1, 2, 3), set(part)); + TestUtils.removeReplicationThrottleForPartitions(cluster.adminClient, seq(Arrays.asList(0, 1, 2, 3)), mutableSet(part).toSet()); Map finalAssignment = Collections.singletonMap(part, new PartitionReassignmentState(Arrays.asList(3, 2, 1), Arrays.asList(3, 2, 1), true)); waitForVerifyAssignment(cluster.adminClient, assignment, false, @@ -406,7 +403,7 @@ public void testCancellation(String quorum) throws Exception { // This time, the broker throttles were removed. waitForBrokerLevelThrottles(unthrottledBrokerConfigs); // Verify that there are no ongoing reassignments. - assertFalse(runVerifyAssignment(cluster.adminClient, assignment, false).partsOngoing()); + assertFalse(runVerifyAssignment(cluster.adminClient, assignment, false).partsOngoing); // Verify that the partition is removed from cancelled replicas verifyReplicaDeleted(foo0, 3); verifyReplicaDeleted(baz1, 3); @@ -429,8 +426,8 @@ public void testCancellationWithAddingReplicaInIsr(String quorum) throws Excepti // We will throttle replica 4 so that only replica 3 joins the ISR TestUtils.setReplicationThrottleForPartitions( cluster.adminClient, - seq(4), - set(foo0), + seq(Arrays.asList(4)), + mutableSet(foo0).toSet(), 1 ); @@ -443,7 +440,7 @@ public void testCancellationWithAddingReplicaInIsr(String quorum) throws Excepti -1L ); TestUtils.waitUntilTrue( - () -> Objects.equals(TestUtils.currentIsr(cluster.adminClient, foo0), set(0, 1, 2, 3)), + () -> Objects.equals(TestUtils.currentIsr(cluster.adminClient, foo0), mutableSet(0, 1, 2, 3).toSet()), () -> "Timed out while waiting for replica 3 to join the ISR", DEFAULT_MAX_WAIT_MS, 100L ); @@ -473,17 +470,17 @@ private void verifyReplicaDeleted( private void waitForLogDirThrottle(Set throttledBrokers, Long logDirThrottle) { Map throttledConfigMap = new HashMap<>(); - throttledConfigMap.put(brokerLevelLeaderThrottle(), -1L); - throttledConfigMap.put(brokerLevelFollowerThrottle(), -1L); - throttledConfigMap.put(brokerLevelLogDirThrottle(), logDirThrottle); + throttledConfigMap.put(BROKER_LEVEL_LEADER_THROTTLE, -1L); + throttledConfigMap.put(BROKER_LEVEL_FOLLOWER_THROTTLE, -1L); + throttledConfigMap.put(BROKER_LEVEL_LOG_DIR_THROTTLE, logDirThrottle); waitForBrokerThrottles(throttledBrokers, throttledConfigMap); } private void waitForInterBrokerThrottle(List throttledBrokers, Long interBrokerThrottle) { Map throttledConfigMap = new HashMap<>(); - throttledConfigMap.put(brokerLevelLeaderThrottle(), interBrokerThrottle); - throttledConfigMap.put(brokerLevelFollowerThrottle(), interBrokerThrottle); - throttledConfigMap.put(brokerLevelLogDirThrottle(), -1L); + throttledConfigMap.put(BROKER_LEVEL_LEADER_THROTTLE, interBrokerThrottle); + throttledConfigMap.put(BROKER_LEVEL_FOLLOWER_THROTTLE, interBrokerThrottle); + throttledConfigMap.put(BROKER_LEVEL_LOG_DIR_THROTTLE, -1L); waitForBrokerThrottles(throttledBrokers, throttledConfigMap); } @@ -526,10 +523,9 @@ private Map> describeBrokerLevelThrottles(Collection< .get(); Map throttles = new HashMap<>(); - brokerLevelThrottles().foreach(throttleName -> { + BROKER_LEVEL_THROTTLES.forEach(throttleName -> { String configValue = Optional.ofNullable(brokerConfigs.get(throttleName)).map(ConfigEntry::value).orElse("-1"); throttles.put(throttleName, Long.parseLong(configValue)); - return null; }); results.put(brokerId, throttles); } @@ -572,7 +568,7 @@ topicPartition, new PartitionReassignmentState(Arrays.asList(0, 1, 2), Arrays.as cluster.adminClient.incrementalAlterConfigs(Collections.singletonMap( new ConfigResource(ConfigResource.Type.BROKER, "0"), Collections.singletonList(new AlterConfigOp( - new ConfigEntry(brokerLevelLogDirThrottle(), ""), AlterConfigOp.OpType.DELETE)))) + new ConfigEntry(BROKER_LEVEL_LOG_DIR_THROTTLE, ""), AlterConfigOp.OpType.DELETE)))) .all().get(); waitForBrokerLevelThrottles(unthrottledBrokerConfigs); @@ -672,22 +668,25 @@ private LogDirReassignment buildLogDirReassignment(TopicPartition topicPartition - private kafka.admin.ReassignPartitionsCommand.VerifyAssignmentResult runVerifyAssignment(Admin adminClient, String jsonString, + private VerifyAssignmentResult runVerifyAssignment(Admin adminClient, String jsonString, Boolean preserveThrottles) { System.out.println("==> verifyAssignment(adminClient, jsonString=" + jsonString); - return verifyAssignment(adminClient, jsonString, preserveThrottles); + try { + return verifyAssignment(adminClient, jsonString, preserveThrottles); + } catch (ExecutionException | InterruptedException | JsonProcessingException e) { + throw new RuntimeException(e); + } } private void waitForVerifyAssignment(Admin adminClient, String jsonString, Boolean preserveThrottles, VerifyAssignmentResult expectedResult) { - final kafka.admin.ReassignPartitionsCommand.VerifyAssignmentResult expectedResult0 = asScala(expectedResult); - final kafka.admin.ReassignPartitionsCommand.VerifyAssignmentResult[] latestResult = {null}; + final VerifyAssignmentResult[] latestResult = {null}; TestUtils.waitUntilTrue( () -> { latestResult[0] = runVerifyAssignment(adminClient, jsonString, preserveThrottles); - return expectedResult0.equals(latestResult[0]); + return expectedResult.equals(latestResult[0]); }, () -> "Timed out waiting for verifyAssignment result " + expectedResult + ". " + "The latest result was " + latestResult[0], DEFAULT_MAX_WAIT_MS, 10L); } @@ -696,22 +695,30 @@ private void runExecuteAssignment(Admin adminClient, Boolean additional, String reassignmentJson, Long interBrokerThrottle, - Long replicaAlterLogDirsThrottle) { + Long replicaAlterLogDirsThrottle) throws RuntimeException { System.out.println("==> executeAssignment(adminClient, additional=" + additional + ", " + "reassignmentJson=" + reassignmentJson + ", " + "interBrokerThrottle=" + interBrokerThrottle + ", " + "replicaAlterLogDirsThrottle=" + replicaAlterLogDirsThrottle + "))"); - executeAssignment(adminClient, additional, reassignmentJson, - interBrokerThrottle, replicaAlterLogDirsThrottle, 10000L, Time.SYSTEM); + try { + executeAssignment(adminClient, additional, reassignmentJson, + interBrokerThrottle, replicaAlterLogDirsThrottle, 10000L, Time.SYSTEM); + } catch (ExecutionException | InterruptedException | JsonProcessingException | TerseException e) { + throw new RuntimeException(e); + } } - private Tuple2, scala.collection.immutable.Set> runCancelAssignment( + private Tuple2, Set> runCancelAssignment( Admin adminClient, String jsonString, Boolean preserveThrottles ) { System.out.println("==> cancelAssignment(adminClient, jsonString=" + jsonString); - return cancelAssignment(adminClient, jsonString, preserveThrottles, 10000L, Time.SYSTEM); + try { + return cancelAssignment(adminClient, jsonString, preserveThrottles, 10000L, Time.SYSTEM); + } catch (ExecutionException | InterruptedException | JsonProcessingException | TerseException e) { + throw new RuntimeException(e); + } } static class BrokerDirs { @@ -857,49 +864,9 @@ public void close() { } } - private ReassignPartitionsCommand.VerifyAssignmentResult asScala(VerifyAssignmentResult res) { - Map partStates = new HashMap<>(); - res.partStates.forEach((tp, state) -> partStates.put(tp, asScala(state))); - - Map moveStates = new HashMap<>(); - res.moveStates.forEach((tpr, state) -> moveStates.put(tpr, asScala(state))); - - return new ReassignPartitionsCommand.VerifyAssignmentResult(asScala(partStates), res.partsOngoing, asScala(moveStates), res.movesOngoing); - } - - @SuppressWarnings({"unchecked"}) - private ReassignPartitionsCommand.PartitionReassignmentState asScala(PartitionReassignmentState state) { - return new ReassignPartitionsCommand.PartitionReassignmentState( - seq((List) state.currentReplicas), - seq((List) state.targetReplicas), - state.done - ); - } - - private ReassignPartitionsCommand.LogDirMoveState asScala(LogDirMoveState state) { - if (state instanceof ActiveMoveState) { - ActiveMoveState s = (ActiveMoveState) state; - return new ReassignPartitionsCommand.ActiveMoveState(s.currentLogDir, s.targetLogDir, s.futureLogDir); - } else if (state instanceof CancelledMoveState) { - CancelledMoveState s = (CancelledMoveState) state; - return new ReassignPartitionsCommand.CancelledMoveState(s.currentLogDir, s.targetLogDir); - } else if (state instanceof CompletedMoveState) { - CompletedMoveState s = (CompletedMoveState) state; - return new ReassignPartitionsCommand.CompletedMoveState(s.targetLogDir); - } else if (state instanceof MissingLogDirMoveState) { - MissingLogDirMoveState s = (MissingLogDirMoveState) state; - return new ReassignPartitionsCommand.MissingLogDirMoveState(s.targetLogDir); - } else if (state instanceof MissingReplicaMoveState) { - MissingReplicaMoveState s = (MissingReplicaMoveState) state; - return new ReassignPartitionsCommand.MissingReplicaMoveState(s.targetLogDir); - } - - throw new IllegalArgumentException("Unknown state " + state); - } - @SuppressWarnings("unchecked") - static scala.collection.immutable.Set set(final T... set) { - return mutableSet(set).toSet(); + static Set set(final T... set) { + return new HashSet<>(Arrays.asList(set)); } @SuppressWarnings({"deprecation", "unchecked"}) @@ -907,18 +874,8 @@ private static scala.collection.mutable.Set mutableSet(final T...set) { return JavaConverters.asScalaSet(new HashSet<>(Arrays.asList(set))); } - @SuppressWarnings({"unchecked"}) - private static Seq seq(T... seq) { - return seq(Arrays.asList(seq)); - } - @SuppressWarnings({"deprecation"}) private static Seq seq(Collection seq) { return JavaConverters.asScalaIteratorConverter(seq.iterator()).asScala().toSeq(); } - - @SuppressWarnings("deprecation") - private static scala.collection.Map asScala(Map jmap) { - return JavaConverters.mapAsScalaMap(jmap); - } } From 23d3d009f69d7584d8d930e65e9bdc9190987b0b Mon Sep 17 00:00:00 2001 From: nizhikov Date: Tue, 3 Oct 2023 00:02:33 +0300 Subject: [PATCH 60/78] KAFKA-14595 Fixes after trunk merge --- build.gradle | 1 - 1 file changed, 1 deletion(-) diff --git a/build.gradle b/build.gradle index 9f6790d777e4f..ffdbbf1ace4cf 100644 --- a/build.gradle +++ b/build.gradle @@ -1888,7 +1888,6 @@ project(':tools') { dependencies { implementation project(':clients') implementation project(':server-common') - implementation project(':storage:api') implementation project(':connect:api') implementation project(':connect:runtime') implementation project(':log4j-appender') From d8e50abfce8953b9b3a0fd17b6e71a8e2db3f9ac Mon Sep 17 00:00:00 2001 From: nizhikov Date: Tue, 3 Oct 2023 00:14:03 +0300 Subject: [PATCH 61/78] KAFKA-14595 Fixes after trunk merge --- checkstyle/import-control.xml | 2 -- 1 file changed, 2 deletions(-) diff --git a/checkstyle/import-control.xml b/checkstyle/import-control.xml index fbdca2548964c..5642f82ae02bd 100644 --- a/checkstyle/import-control.xml +++ b/checkstyle/import-control.xml @@ -271,11 +271,9 @@ - - From 717590147afb1e1cfac3832e2e382be61fec4ce1 Mon Sep 17 00:00:00 2001 From: nizhikov Date: Tue, 3 Oct 2023 00:14:41 +0300 Subject: [PATCH 62/78] KAFKA-14595 Fixes after trunk merge --- checkstyle/import-control.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/checkstyle/import-control.xml b/checkstyle/import-control.xml index 5642f82ae02bd..888e8a41ae83a 100644 --- a/checkstyle/import-control.xml +++ b/checkstyle/import-control.xml @@ -296,8 +296,8 @@ - + From be82fbcc5059c9dcf9541c305da9e23346feb68f Mon Sep 17 00:00:00 2001 From: nizhikov Date: Wed, 4 Oct 2023 12:47:36 +0300 Subject: [PATCH 63/78] KAFKA-14595 Code review fix --- tools/src/main/java/org/apache/kafka/tools/reassign/Tuple2.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/src/main/java/org/apache/kafka/tools/reassign/Tuple2.java b/tools/src/main/java/org/apache/kafka/tools/reassign/Tuple2.java index 469480f98bd04..a9e84317cf718 100644 --- a/tools/src/main/java/org/apache/kafka/tools/reassign/Tuple2.java +++ b/tools/src/main/java/org/apache/kafka/tools/reassign/Tuple2.java @@ -43,6 +43,6 @@ public int hashCode() { @Override public String toString() { - return "Tuple{v1=" + v1 + ", v2=" + v2 + '}'; + return "Tuple2{v1=" + v1 + ", v2=" + v2 + '}'; } } From c2e2356befcc6acb8195ee503217ef02b1a5912f Mon Sep 17 00:00:00 2001 From: nizhikov Date: Wed, 4 Oct 2023 14:32:41 +0300 Subject: [PATCH 64/78] KAFKA-14595 Code formatting. --- .../reassign/ReassignPartitionsCommand.java | 61 +++++++++++++------ 1 file changed, 44 insertions(+), 17 deletions(-) diff --git a/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommand.java b/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommand.java index a7b8eb924a62e..04d63dfb0040f 100644 --- a/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommand.java +++ b/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommand.java @@ -198,7 +198,10 @@ private static void handleAction(Admin adminClient, ReassignPartitionsCommandOpt * * @return A result that is useful for testing. */ - static VerifyAssignmentResult verifyAssignment(Admin adminClient, String jsonString, Boolean preserveThrottles) throws ExecutionException, InterruptedException, JsonProcessingException { + static VerifyAssignmentResult verifyAssignment(Admin adminClient, + String jsonString, + Boolean preserveThrottles + ) throws ExecutionException, InterruptedException, JsonProcessingException { Tuple2>>, Map> t0 = parsePartitionReassignmentData(jsonString); List>> targetParts = t0.v1; @@ -237,7 +240,8 @@ static VerifyAssignmentResult verifyAssignment(Admin adminClient, String jsonStr * in the JSON file.) */ private static Tuple2, Boolean> verifyPartitionAssignments(Admin adminClient, - List>> targets) throws ExecutionException, InterruptedException { + List>> targets + ) throws ExecutionException, InterruptedException { Tuple2, Boolean> t0 = findPartitionReassignmentStates(adminClient, targets); Map partStates = t0.v1; @@ -301,7 +305,9 @@ static String partitionReassignmentStatesToString(Map, Boolean> findPartitionReassignmentStates(Admin adminClient, - List>> targetReassignments) throws ExecutionException, InterruptedException { + List>> targetReassignments + ) throws ExecutionException, InterruptedException { Map currentReassignments = adminClient. listPartitionReassignments().reassignments().get(); @@ -361,7 +367,8 @@ static Tuple2, Boolean> findPart private static PartitionReassignmentState topicDescriptionFutureToState(int partition, KafkaFuture future, - List targetReplicas) throws InterruptedException, ExecutionException { + List targetReplicas + ) throws InterruptedException, ExecutionException { try { TopicDescription topicDescription = future.get(); if (topicDescription.partitions().size() < partition) { @@ -394,7 +401,8 @@ private static PartitionReassignmentState topicDescriptionFutureToState(int part * returns all ongoing replica reassignments.) */ private static Tuple2, Boolean> verifyReplicaMoves(Admin adminClient, - Map targetReassignments) throws ExecutionException, InterruptedException { + Map targetReassignments + ) throws ExecutionException, InterruptedException { Map moveStates = findLogDirMoveStates(adminClient, targetReassignments); System.out.println(replicaMoveStatesToString(moveStates)); return new Tuple2<>(moveStates, !moveStates.values().stream().allMatch(LogDirMoveState::done)); @@ -411,7 +419,8 @@ private static Tuple2, Boolean> veri * @return The states for each replica movement. */ static Map findLogDirMoveStates(Admin adminClient, - Map targetMoves) throws ExecutionException, InterruptedException { + Map targetMoves + ) throws ExecutionException, InterruptedException { Map replicaLogDirInfos = adminClient .describeReplicaLogDirs(targetMoves.keySet()).all().get(); @@ -495,7 +504,9 @@ static String replicaMoveStatesToString(Map>> targetParts) throws ExecutionException, InterruptedException { + private static void clearAllThrottles(Admin adminClient, + List>> targetParts + ) throws ExecutionException, InterruptedException { Set activeBrokers = adminClient.describeCluster().nodes().get().stream().map(Node::id).collect(Collectors.toSet()); Set brokers = new HashSet<>(activeBrokers); targetParts.forEach(t -> brokers.addAll(t.v2)); @@ -557,7 +568,8 @@ private static void clearTopicLevelThrottles(Admin adminClient, Set topi static Tuple2>, Map>> generateAssignment(Admin adminClient, String reassignmentJson, String brokerListString, - Boolean enableRackAwareness) throws ExecutionException, InterruptedException, JsonProcessingException { + Boolean enableRackAwareness + ) throws ExecutionException, InterruptedException, JsonProcessingException { Tuple2, List> t0 = parseGenerateAssignmentArgs(reassignmentJson, brokerListString); List brokersToReassign = t0.v1; @@ -625,7 +637,8 @@ private static Map describeTopics(Admin adminClient, * If any topic can't be found, an exception will be thrown. */ static Map> getReplicaAssignmentForTopics(Admin adminClient, - List topics) throws ExecutionException, InterruptedException { + List topics + ) throws ExecutionException, InterruptedException { Map> res = new HashMap<>(); describeTopics(adminClient, new HashSet<>(topics)).forEach((topicName, topicDescription) -> @@ -646,7 +659,8 @@ static Map> getReplicaAssignmentForTopics(Admin ad * If any topic can't be found, an exception will be thrown. */ static Map> getReplicaAssignmentForPartitions(Admin adminClient, - Set partitions) throws ExecutionException, InterruptedException { + Set partitions + ) throws ExecutionException, InterruptedException { Map> res = new HashMap<>(); describeTopics(adminClient, partitions.stream().map(TopicPartition::topic).collect(Collectors.toSet())).forEach((topicName, topicDescription) -> { topicDescription.partitions().forEach(info -> { @@ -730,7 +744,8 @@ static void executeAssignment(Admin adminClient, Long interBrokerThrottle, Long logDirThrottle, Long timeoutMs, - Time time) throws ExecutionException, InterruptedException, JsonProcessingException, TerseException { + Time time + ) throws ExecutionException, InterruptedException, JsonProcessingException, TerseException { Tuple2>, Map> t0 = parseExecuteAssignmentArgs(reassignmentJson); Map> proposedParts = t0.v1; @@ -797,7 +812,8 @@ static void executeAssignment(Admin adminClient, private static void executeMoves(Admin adminClient, Map proposedReplicas, Long timeoutMs, - Time time) throws InterruptedException, TerseException { + Time time + ) throws InterruptedException, TerseException { long startTimeMs = time.milliseconds(); Map pendingReplicas = new HashMap<>(); pendingReplicas.putAll(proposedReplicas); @@ -1123,7 +1139,11 @@ static void modifyTopicThrottles(Admin adminClient, adminClient.incrementalAlterConfigs(configs).all().get(); } - private static void modifyReassignmentThrottle(Admin admin, Map> moveMap, Long interBrokerThrottle) throws ExecutionException, InterruptedException { + private static void modifyReassignmentThrottle( + Admin admin, + Map> moveMap, + Long interBrokerThrottle + ) throws ExecutionException, InterruptedException { Map leaderThrottles = calculateLeaderThrottles(moveMap); Map followerThrottles = calculateFollowerThrottles(moveMap); modifyTopicThrottles(admin, leaderThrottles, followerThrottles); @@ -1186,7 +1206,9 @@ static void modifyLogDirThrottle(Admin admin, * @return A tuple of the partitions to be reassigned and the replicas * to be reassigned. */ - static Tuple2>, Map> parseExecuteAssignmentArgs(String reassignmentJson) throws JsonProcessingException { + static Tuple2>, Map> parseExecuteAssignmentArgs( + String reassignmentJson + ) throws JsonProcessingException { Tuple2>>, Map> t0 = parsePartitionReassignmentData(reassignmentJson); List>> partitionsToBeReassigned = t0.v1; @@ -1236,7 +1258,8 @@ static Tuple2, Set> cancelAssignment( String jsonString, Boolean preserveThrottles, Long timeoutMs, - Time time) throws ExecutionException, InterruptedException, JsonProcessingException, TerseException { + Time time + ) throws ExecutionException, InterruptedException, JsonProcessingException, TerseException { Tuple2>>, Map> t0 = parsePartitionReassignmentData(jsonString); List>> targetParts = t0.v1; @@ -1335,7 +1358,9 @@ private static List parseTopicsData(int version, JsonValue js) throws Js } } - private static Tuple2>>, Map> parsePartitionReassignmentData(String jsonData) throws JsonProcessingException { + private static Tuple2>>, Map> parsePartitionReassignmentData( + String jsonData + ) throws JsonProcessingException { JsonValue js; try { js = Json.tryParseFull(jsonData); @@ -1347,7 +1372,9 @@ private static Tuple2>>, Map>>, Map> parsePartitionReassignmentData(int version, JsonValue jsonData) throws JsonMappingException { + private static Tuple2>>, Map> parsePartitionReassignmentData( + int version, JsonValue jsonData + ) throws JsonMappingException { switch (version) { case 1: List>> partitionAssignment = new ArrayList<>(); From d4b30c1cebb6bb5ddec1c7b39d111de64e8eab5c Mon Sep 17 00:00:00 2001 From: nizhikov Date: Wed, 4 Oct 2023 14:48:04 +0300 Subject: [PATCH 65/78] KAFKA-14595 Code review fixes. --- .../kafka/tools/reassign/ReassignPartitionsCommand.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommand.java b/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommand.java index 04d63dfb0040f..3bf1f72e9caf4 100644 --- a/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommand.java +++ b/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommand.java @@ -243,12 +243,8 @@ private static Tuple2, Boolean> List>> targets ) throws ExecutionException, InterruptedException { Tuple2, Boolean> t0 = findPartitionReassignmentStates(adminClient, targets); - - Map partStates = t0.v1; - Boolean partsOngoing = t0.v2; - - System.out.println(partitionReassignmentStatesToString(partStates)); - return new Tuple2<>(partStates, partsOngoing); + System.out.println(partitionReassignmentStatesToString(t0.v1)); + return t0; } static int compareTopicPartitions(TopicPartition a, TopicPartition b) { From 70ddeeb538036cd0dd86e0a6d788ae62975a5023 Mon Sep 17 00:00:00 2001 From: nizhikov Date: Wed, 4 Oct 2023 15:12:13 +0300 Subject: [PATCH 66/78] KAFKA-14595 Code review fixes. --- .../tools/reassign/ReassignPartitionsCommand.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommand.java b/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommand.java index 3bf1f72e9caf4..92771e9922521 100644 --- a/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommand.java +++ b/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommand.java @@ -327,12 +327,11 @@ static Tuple2, Boolean> findPart false)); }).collect(Collectors.toList()); - Set topicNamesToLookUp = new HashSet<>(); - notFoundReassignments.forEach(e -> { - TopicPartition part = e.v1; - if (!currentReassignments.containsKey(part)) - topicNamesToLookUp.add(part.topic()); - }); + Set topicNamesToLookUp = notFoundReassignments.stream() + .map(e -> e.v1) + .filter(part -> !currentReassignments.containsKey(part)) + .map(TopicPartition::topic) + .collect(Collectors.toSet()); Map> topicDescriptions = adminClient. describeTopics(topicNamesToLookUp).topicNameValues(); From c90a6da06653c727671f6413e2f31bb4494e1c42 Mon Sep 17 00:00:00 2001 From: nizhikov Date: Wed, 4 Oct 2023 19:00:41 +0300 Subject: [PATCH 67/78] KAFKA-14595 Code review fixes. --- .../reassign/ReassignPartitionsCommand.java | 49 +++++++------------ 1 file changed, 19 insertions(+), 30 deletions(-) diff --git a/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommand.java b/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommand.java index 92771e9922521..c7f70e7e0d828 100644 --- a/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommand.java +++ b/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommand.java @@ -419,36 +419,27 @@ static Map findLogDirMoveStates(Admin ad Map replicaLogDirInfos = adminClient .describeReplicaLogDirs(targetMoves.keySet()).all().get(); - Map res = new HashMap<>(); - - targetMoves.entrySet().forEach(e -> { + return targetMoves.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> { TopicPartitionReplica replica = e.getKey(); String targetLogDir = e.getValue(); - LogDirMoveState moveState; + if (!replicaLogDirInfos.containsKey(replica)) + return new MissingReplicaMoveState(targetLogDir); - if (!replicaLogDirInfos.containsKey(replica)) { - moveState = new MissingReplicaMoveState(targetLogDir); - } else { - DescribeReplicaLogDirsResult.ReplicaLogDirInfo info = replicaLogDirInfos.get(replica); - if (info.getCurrentReplicaLogDir() == null) { - moveState = new MissingLogDirMoveState(targetLogDir); - } else if (info.getFutureReplicaLogDir() == null) { - if (info.getCurrentReplicaLogDir().equals(targetLogDir)) { - moveState = new CompletedMoveState(targetLogDir); - } else { - moveState = new CancelledMoveState(info.getCurrentReplicaLogDir(), targetLogDir); - } - } else { - moveState = new ActiveMoveState(info.getCurrentReplicaLogDir(), - targetLogDir, - info.getFutureReplicaLogDir()); - } + DescribeReplicaLogDirsResult.ReplicaLogDirInfo info = replicaLogDirInfos.get(replica); + + if (info.getCurrentReplicaLogDir() == null) + return new MissingLogDirMoveState(targetLogDir); + + if (info.getFutureReplicaLogDir() == null) { + if (info.getCurrentReplicaLogDir().equals(targetLogDir)) + return new CompletedMoveState(targetLogDir); + + return new CancelledMoveState(info.getCurrentReplicaLogDir(), targetLogDir); } - res.put(replica, moveState); - }); - return res; + return new ActiveMoveState(info.getCurrentReplicaLogDir(), targetLogDir, info.getFutureReplicaLogDir()); + })); } /** @@ -539,13 +530,12 @@ private static void clearBrokerLevelThrottles(Admin adminClient, Set br * @param topics The topics to clear the throttles for. */ private static void clearTopicLevelThrottles(Admin adminClient, Set topics) throws ExecutionException, InterruptedException { - Map> configOps = new HashMap<>(); - topics.forEach(topicName -> configOps.put( - new ConfigResource(ConfigResource.Type.TOPIC, topicName), - TOPIC_LEVEL_THROTTLES.stream().map(throttle -> new AlterConfigOp(new ConfigEntry(throttle, null), + Map> configOps = topics.stream().collect(Collectors.toMap( + topicName -> new ConfigResource(ConfigResource.Type.TOPIC, topicName), + topicName -> TOPIC_LEVEL_THROTTLES.stream().map(throttle -> new AlterConfigOp(new ConfigEntry(throttle, null), AlterConfigOp.OpType.DELETE)).collect(Collectors.toList()) - )); + adminClient.incrementalAlterConfigs(configOps).all().get(); } @@ -634,7 +624,6 @@ private static Map describeTopics(Admin adminClient, static Map> getReplicaAssignmentForTopics(Admin adminClient, List topics ) throws ExecutionException, InterruptedException { - Map> res = new HashMap<>(); describeTopics(adminClient, new HashSet<>(topics)).forEach((topicName, topicDescription) -> topicDescription.partitions().forEach(info -> res.put( From 5282f60d507ff3808b7033c986ff89dc77a8d921 Mon Sep 17 00:00:00 2001 From: nizhikov Date: Wed, 4 Oct 2023 19:20:35 +0300 Subject: [PATCH 68/78] KAFKA-14595 Code review fixes. --- .../apache/kafka/tools/reassign/ReassignPartitionsCommand.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommand.java b/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommand.java index c7f70e7e0d828..b104c3f1f1bd9 100644 --- a/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommand.java +++ b/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommand.java @@ -673,7 +673,8 @@ static List getBrokerMetadata(Admin adminClient, List b .filter(node -> brokerSet.contains(node.id())) .map(node -> (enableRackAwareness && node.rack() != null) ? new BrokerMetadata(node.id(), Optional.of(node.rack())) - : new BrokerMetadata(node.id(), Optional.empty())).collect(Collectors.toList()); + : new BrokerMetadata(node.id(), Optional.empty()) + ).collect(Collectors.toList()); long numRackless = results.stream().filter(m -> !m.rack.isPresent()).count(); if (enableRackAwareness && numRackless != 0 && numRackless != results.size()) { From 6fe1aa0b2a6bfcdf25450f4c949f682476fe3cd9 Mon Sep 17 00:00:00 2001 From: nizhikov Date: Thu, 5 Oct 2023 12:05:18 +0300 Subject: [PATCH 69/78] KAFKA-14595 Code review fixes. --- .../apache/kafka/tools/reassign/ReassignPartitionsCommand.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommand.java b/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommand.java index b104c3f1f1bd9..fc20f5655fe6d 100644 --- a/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommand.java +++ b/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommand.java @@ -830,8 +830,6 @@ private static void executeMoves(Admin adminClient, // call. It can't take effect until the replica is moved to that host. time.sleep(100); } - - } while (!done); } From f66fb6fcf98fa0d18c594c63cc17cb9f422a3c2c Mon Sep 17 00:00:00 2001 From: nizhikov Date: Thu, 5 Oct 2023 14:57:46 +0300 Subject: [PATCH 70/78] KAFKA-14595 Code review fixes. --- .../reassign/ReassignPartitionsCommand.java | 37 +++++++------------ 1 file changed, 13 insertions(+), 24 deletions(-) diff --git a/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommand.java b/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommand.java index fc20f5655fe6d..ee87008b3e5ab 100644 --- a/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommand.java +++ b/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommand.java @@ -357,7 +357,7 @@ static Tuple2, Boolean> findPart foundResults.forEach(e -> allResults.put(e.v1, e.v2)); notFoundResults.forEach(e -> allResults.put(e.v1, e.v2)); - return new Tuple2<>(allResults, currentReassignments.size() > 0); + return new Tuple2<>(allResults, !currentReassignments.isEmpty()); } private static PartitionReassignmentState topicDescriptionFutureToState(int partition, @@ -493,8 +493,7 @@ static String replicaMoveStatesToString(Map>> targetParts ) throws ExecutionException, InterruptedException { - Set activeBrokers = adminClient.describeCluster().nodes().get().stream().map(Node::id).collect(Collectors.toSet()); - Set brokers = new HashSet<>(activeBrokers); + Set brokers = adminClient.describeCluster().nodes().get().stream().map(Node::id).collect(Collectors.toSet()); targetParts.forEach(t -> brokers.addAll(t.v2)); System.out.printf("Clearing broker-level throttles on broker%s %s%n", @@ -646,13 +645,13 @@ static Map> getReplicaAssignmentForPartitions(Admi Set partitions ) throws ExecutionException, InterruptedException { Map> res = new HashMap<>(); - describeTopics(adminClient, partitions.stream().map(TopicPartition::topic).collect(Collectors.toSet())).forEach((topicName, topicDescription) -> { + describeTopics(adminClient, partitions.stream().map(TopicPartition::topic).collect(Collectors.toSet())).forEach((topicName, topicDescription) -> topicDescription.partitions().forEach(info -> { TopicPartition tp = new TopicPartition(topicName, info.partition()); if (partitions.contains(tp)) res.put(tp, info.replicas().stream().map(Node::id).collect(Collectors.toList())); - }); - }); + }) + ); return res; } @@ -800,8 +799,7 @@ private static void executeMoves(Admin adminClient, Time time ) throws InterruptedException, TerseException { long startTimeMs = time.milliseconds(); - Map pendingReplicas = new HashMap<>(); - pendingReplicas.putAll(proposedReplicas); + Map pendingReplicas = new HashMap<>(proposedReplicas); boolean done = false; do { Set completed = alterReplicaLogDirs(adminClient, pendingReplicas); @@ -919,12 +917,7 @@ static String currentPartitionReplicaAssignmentToString(Map alterPartitionReassignments(Admin adminClient, Map> reassignments) throws InterruptedException { Map> args = new HashMap<>(); - reassignments.entrySet().forEach(e -> { - TopicPartition part = e.getKey(); - List replicas = e.getValue(); - - args.put(part, Optional.of(new NewPartitionReassignment(replicas))); - }); + reassignments.forEach((part, replicas) -> args.put(part, Optional.of(new NewPartitionReassignment(replicas)))); Map> results = adminClient.alterPartitionReassignments(args).values(); Map errors = new HashMap<>(); for (Map.Entry> e : results.entrySet()) { @@ -968,16 +961,14 @@ static Map cancelPartitionReassignments(Admin adminCl private static Map> calculateCurrentMoveMap(Map currentReassignments) { Map> moveMap = new HashMap<>(); // Add the current reassignments to the move map. - currentReassignments.entrySet().forEach(e -> { - TopicPartition part = e.getKey(); - PartitionReassignment reassignment = e.getValue(); + currentReassignments.forEach((part, reassignment) -> { List allReplicas = reassignment.replicas(); List addingReplicas = reassignment.addingReplicas(); // The addingReplicas is included in the replicas during reassignment Set sources = new HashSet<>(allReplicas); - sources.removeAll(addingReplicas); + addingReplicas.forEach(sources::remove); Set destinations = new HashSet<>(addingReplicas); @@ -1075,12 +1066,10 @@ static Map calculateFollowerThrottles(Map calculateReassigningBrokers(Map> moveMap) { Set reassigningBrokers = new TreeSet<>(); - moveMap.values().forEach(partMoveMap -> { - partMoveMap.values().forEach(partMove -> { - reassigningBrokers.addAll(partMove.sources); - reassigningBrokers.addAll(partMove.destinations); - }); - }); + moveMap.values().forEach(partMoveMap -> partMoveMap.values().forEach(partMove -> { + reassigningBrokers.addAll(partMove.sources); + reassigningBrokers.addAll(partMove.destinations); + })); return reassigningBrokers; } From 85c441d739d8b4aed4f9563f141c1293cc5b5e37 Mon Sep 17 00:00:00 2001 From: nizhikov Date: Thu, 5 Oct 2023 15:37:27 +0300 Subject: [PATCH 71/78] KAFKA-14595 Code review fixes. --- .../apache/kafka/tools/reassign/ReassignPartitionsCommand.java | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommand.java b/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommand.java index ee87008b3e5ab..0665a337e8b46 100644 --- a/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommand.java +++ b/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommand.java @@ -962,7 +962,6 @@ private static Map> calculateCurrentMoveMap( Map> moveMap = new HashMap<>(); // Add the current reassignments to the move map. currentReassignments.forEach((part, reassignment) -> { - List allReplicas = reassignment.replicas(); List addingReplicas = reassignment.addingReplicas(); From f40b5a5560ff0d21a494d6aa46f8d1c2020685d7 Mon Sep 17 00:00:00 2001 From: nizhikov Date: Thu, 5 Oct 2023 16:28:47 +0300 Subject: [PATCH 72/78] KAFKA-14595 Code review fixes. --- .../ReassignPartitionsCommandOptions.java | 64 ------------------- 1 file changed, 64 deletions(-) diff --git a/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommandOptions.java b/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommandOptions.java index d0e97c036dda2..f35dfe25f73b4 100644 --- a/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommandOptions.java +++ b/tools/src/main/java/org/apache/kafka/tools/reassign/ReassignPartitionsCommandOptions.java @@ -109,68 +109,4 @@ public ReassignPartitionsCommandOptions(String[] args) { options = parser.parse(args); } - - public OptionSpec verifyOpt() { - return verifyOpt; - } - - public OptionSpec generateOpt() { - return generateOpt; - } - - public OptionSpec executeOpt() { - return executeOpt; - } - - public OptionSpec cancelOpt() { - return cancelOpt; - } - - public OptionSpec listOpt() { - return listOpt; - } - - public OptionSpec bootstrapServerOpt() { - return bootstrapServerOpt; - } - - public OptionSpec commandConfigOpt() { - return commandConfigOpt; - } - - public OptionSpec reassignmentJsonFileOpt() { - return reassignmentJsonFileOpt; - } - - public OptionSpec topicsToMoveJsonFileOpt() { - return topicsToMoveJsonFileOpt; - } - - public OptionSpec brokerListOpt() { - return brokerListOpt; - } - - public OptionSpec disableRackAware() { - return disableRackAware; - } - - public OptionSpec interBrokerThrottleOpt() { - return interBrokerThrottleOpt; - } - - public OptionSpec replicaAlterLogDirsThrottleOpt() { - return replicaAlterLogDirsThrottleOpt; - } - - public OptionSpec timeoutOpt() { - return timeoutOpt; - } - - public OptionSpec additionalOpt() { - return additionalOpt; - } - - public OptionSpec preserveThrottlesOpt() { - return preserveThrottlesOpt; - } } From ceaf2bf3d816334d008b4bac06b61ac03a247703 Mon Sep 17 00:00:00 2001 From: nizhikov Date: Mon, 9 Oct 2023 23:00:11 +0300 Subject: [PATCH 73/78] KAFKA-14595 set method removed. --- .../reassign/ReassignPartitionsIntegrationTest.java | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsIntegrationTest.java b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsIntegrationTest.java index a7e40ce1f6145..3a0cd1fc2caac 100644 --- a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsIntegrationTest.java +++ b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsIntegrationTest.java @@ -395,11 +395,11 @@ public void testCancellation(String quorum) throws Exception { waitForVerifyAssignment(cluster.adminClient, assignment, true, new VerifyAssignmentResult(partStates, true, Collections.emptyMap(), false)); // Cancel the reassignment. - assertEquals(new Tuple2<>(set(foo0, baz1), set()), runCancelAssignment(cluster.adminClient, assignment, true)); + assertEquals(new Tuple2<>(new HashSet<>(Arrays.asList(foo0, baz1)), Collections.emptySet()), runCancelAssignment(cluster.adminClient, assignment, true)); // Broker throttles are still active because we passed --preserve-throttles waitForInterBrokerThrottle(Arrays.asList(0, 1, 2, 3), interBrokerThrottle); // Cancelling the reassignment again should reveal nothing to cancel. - assertEquals(new Tuple2<>(set(), set()), runCancelAssignment(cluster.adminClient, assignment, false)); + assertEquals(new Tuple2<>(Collections.emptySet(), Collections.emptySet()), runCancelAssignment(cluster.adminClient, assignment, false)); // This time, the broker throttles were removed. waitForBrokerLevelThrottles(unthrottledBrokerConfigs); // Verify that there are no ongoing reassignments. @@ -446,7 +446,7 @@ public void testCancellationWithAddingReplicaInIsr(String quorum) throws Excepti ); // Now cancel the assignment and verify that the partition is removed from cancelled replicas - assertEquals(new Tuple2<>(set(foo0), set()), runCancelAssignment(cluster.adminClient, assignment, true)); + assertEquals(new Tuple2<>(Collections.singleton(foo0), Collections.emptySet()), runCancelAssignment(cluster.adminClient, assignment, true)); verifyReplicaDeleted(foo0, 3); verifyReplicaDeleted(foo0, 4); } @@ -864,11 +864,6 @@ public void close() { } } - @SuppressWarnings("unchecked") - static Set set(final T... set) { - return new HashSet<>(Arrays.asList(set)); - } - @SuppressWarnings({"deprecation", "unchecked"}) private static scala.collection.mutable.Set mutableSet(final T...set) { return JavaConverters.asScalaSet(new HashSet<>(Arrays.asList(set))); From b604480d592e82bc35449e5319f841091ec92741 Mon Sep 17 00:00:00 2001 From: nizhikov Date: Tue, 17 Oct 2023 20:13:04 +0300 Subject: [PATCH 74/78] KAFKA-14595 revert unnecessary change --- tools/src/main/java/org/apache/kafka/tools/ToolsUtils.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/src/main/java/org/apache/kafka/tools/ToolsUtils.java b/tools/src/main/java/org/apache/kafka/tools/ToolsUtils.java index a3fd7c5bc6e06..a3eb4ab4bcb6b 100644 --- a/tools/src/main/java/org/apache/kafka/tools/ToolsUtils.java +++ b/tools/src/main/java/org/apache/kafka/tools/ToolsUtils.java @@ -22,7 +22,6 @@ import java.io.PrintStream; import java.util.Arrays; -import java.util.Collection; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -130,7 +129,7 @@ public static void validateBootstrapServer(String hostPort) throws IllegalArgume /** * Return all duplicates in a list. A duplicated element will appear only once. */ - public static Set duplicates(Collection s) { + public static Set duplicates(List s) { Set set = new HashSet<>(); Set duplicates = new HashSet<>(); From 369e888fee0d52fbf5ea1aeaff6a957e2d3b4a68 Mon Sep 17 00:00:00 2001 From: nizhikov Date: Wed, 18 Oct 2023 17:24:12 +0300 Subject: [PATCH 75/78] KAFKA-14595 Code review fixes. --- .../reassign/ReassignPartitionsUnitTest.java | 181 +++++++++--------- 1 file changed, 88 insertions(+), 93 deletions(-) diff --git a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java index f7622800dc753..2184971efb733 100644 --- a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java +++ b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java @@ -37,7 +37,6 @@ import org.junit.jupiter.api.Timeout; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -49,6 +48,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import static java.util.Arrays.asList; import static org.apache.kafka.tools.reassign.ReassignPartitionsCommand.BROKER_LEVEL_FOLLOWER_THROTTLE; import static org.apache.kafka.tools.reassign.ReassignPartitionsCommand.BROKER_LEVEL_LEADER_THROTTLE; import static org.apache.kafka.tools.reassign.ReassignPartitionsCommand.BROKER_LEVEL_LOG_DIR_THROTTLE; @@ -121,11 +121,11 @@ public void testPartitionReassignStatesToString() { Map states = new HashMap<>(); states.put(new TopicPartition("foo", 0), - new PartitionReassignmentState(seq(1, 2, 3), seq(1, 2, 3), true)); + new PartitionReassignmentState(asList(1, 2, 3), asList(1, 2, 3), true)); states.put(new TopicPartition("foo", 1), - new PartitionReassignmentState(seq(1, 2, 3), seq(1, 2, 4), false)); + new PartitionReassignmentState(asList(1, 2, 3), asList(1, 2, 4), false)); states.put(new TopicPartition("bar", 0), - new PartitionReassignmentState(seq(1, 2, 3), seq(1, 2, 4), false)); + new PartitionReassignmentState(asList(1, 2, 3), asList(1, 2, 4), false)); assertEquals(String.join(System.lineSeparator(), "Status of partition reassignment:", @@ -137,18 +137,18 @@ public void testPartitionReassignStatesToString() { private void addTopics(MockAdminClient adminClient) { List b = adminClient.brokers(); - adminClient.addTopic(false, "foo", Arrays.asList( + adminClient.addTopic(false, "foo", asList( new TopicPartitionInfo(0, b.get(0), - Arrays.asList(b.get(0), b.get(1), b.get(2)), - Arrays.asList(b.get(0), b.get(1))), + asList(b.get(0), b.get(1), b.get(2)), + asList(b.get(0), b.get(1))), new TopicPartitionInfo(1, b.get(1), - Arrays.asList(b.get(1), b.get(2), b.get(3)), - Arrays.asList(b.get(1), b.get(2), b.get(3))) + asList(b.get(1), b.get(2), b.get(3)), + asList(b.get(1), b.get(2), b.get(3))) ), Collections.emptyMap()); - adminClient.addTopic(false, "bar", Arrays.asList( + adminClient.addTopic(false, "bar", asList( new TopicPartitionInfo(0, b.get(2), - Arrays.asList(b.get(2), b.get(3), b.get(0)), - Arrays.asList(b.get(2), b.get(3), b.get(0))) + asList(b.get(2), b.get(3), b.get(0)), + asList(b.get(2), b.get(3), b.get(0))) ), Collections.emptyMap()); } @@ -160,8 +160,8 @@ public void testFindPartitionReassignmentStates() throws Exception { // Create a reassignment and test findPartitionReassignmentStates. Map> reassignments = new HashMap<>(); - reassignments.put(new TopicPartition("foo", 0), seq(0, 1, 3)); - reassignments.put(new TopicPartition("quux", 0), seq(1, 2, 3)); + reassignments.put(new TopicPartition("foo", 0), asList(0, 1, 3)); + reassignments.put(new TopicPartition("quux", 0), asList(1, 2, 3)); Map reassignmentResult = alterPartitionReassignments(adminClient, reassignments); @@ -171,14 +171,14 @@ public void testFindPartitionReassignmentStates() throws Exception { Map expStates = new HashMap<>(); expStates.put(new TopicPartition("foo", 0), - new PartitionReassignmentState(seq(0, 1, 2), seq(0, 1, 3), false)); + new PartitionReassignmentState(asList(0, 1, 2), asList(0, 1, 3), false)); expStates.put(new TopicPartition("foo", 1), - new PartitionReassignmentState(seq(1, 2, 3), seq(1, 2, 3), true)); + new PartitionReassignmentState(asList(1, 2, 3), asList(1, 2, 3), true)); Tuple2, Boolean> actual = - findPartitionReassignmentStates(adminClient, seq( - new Tuple2<>(new TopicPartition("foo", 0), seq(0, 1, 3)), - new Tuple2<>(new TopicPartition("foo", 1), seq(1, 2, 3)) + findPartitionReassignmentStates(adminClient, asList( + new Tuple2<>(new TopicPartition("foo", 0), asList(0, 1, 3)), + new Tuple2<>(new TopicPartition("foo", 1), asList(1, 2, 3)) )); assertEquals(expStates, actual.v1); @@ -194,13 +194,13 @@ public void testFindPartitionReassignmentStates() throws Exception { expStates.clear(); expStates.put(new TopicPartition("foo", 0), - new PartitionReassignmentState(seq(0, 1, 2), seq(0, 1, 3), true)); + new PartitionReassignmentState(asList(0, 1, 2), asList(0, 1, 3), true)); expStates.put(new TopicPartition("foo", 1), - new PartitionReassignmentState(seq(1, 2, 3), seq(1, 2, 3), true)); + new PartitionReassignmentState(asList(1, 2, 3), asList(1, 2, 3), true)); - actual = findPartitionReassignmentStates(adminClient, seq( - new Tuple2<>(new TopicPartition("foo", 0), seq(0, 1, 3)), - new Tuple2<>(new TopicPartition("foo", 1), seq(1, 2, 3)) + actual = findPartitionReassignmentStates(adminClient, asList( + new Tuple2<>(new TopicPartition("foo", 0), asList(0, 1, 3)), + new Tuple2<>(new TopicPartition("foo", 1), asList(1, 2, 3)) )); assertEquals(expStates, actual.v1); @@ -212,19 +212,19 @@ public void testFindPartitionReassignmentStates() throws Exception { public void testFindLogDirMoveStates() throws Exception { try (MockAdminClient adminClient = new MockAdminClient.Builder(). numBrokers(4). - brokerLogDirs(Arrays.asList( - Arrays.asList("/tmp/kafka-logs0", "/tmp/kafka-logs1"), - Arrays.asList("/tmp/kafka-logs0", "/tmp/kafka-logs1"), - Arrays.asList("/tmp/kafka-logs0", "/tmp/kafka-logs1"), - Arrays.asList("/tmp/kafka-logs0", null))) + brokerLogDirs(asList( + asList("/tmp/kafka-logs0", "/tmp/kafka-logs1"), + asList("/tmp/kafka-logs0", "/tmp/kafka-logs1"), + asList("/tmp/kafka-logs0", "/tmp/kafka-logs1"), + asList("/tmp/kafka-logs0", null))) .build()) { addTopics(adminClient); List b = adminClient.brokers(); - adminClient.addTopic(false, "quux", Arrays.asList( + adminClient.addTopic(false, "quux", asList( new TopicPartitionInfo(0, b.get(2), - Arrays.asList(b.get(1), b.get(2), b.get(3)), - Arrays.asList(b.get(1), b.get(2), b.get(3)))), + asList(b.get(1), b.get(2), b.get(3)), + asList(b.get(1), b.get(2), b.get(3)))), Collections.emptyMap()); Map replicaAssignment = new HashMap<>(); @@ -288,15 +288,15 @@ public void testGetReplicaAssignments() throws Exception { Map> assignments = new HashMap<>(); - assignments.put(new TopicPartition("foo", 0), seq(0, 1, 2)); - assignments.put(new TopicPartition("foo", 1), seq(1, 2, 3)); + assignments.put(new TopicPartition("foo", 0), asList(0, 1, 2)); + assignments.put(new TopicPartition("foo", 1), asList(1, 2, 3)); - assertEquals(assignments, getReplicaAssignmentForTopics(adminClient, seq("foo"))); + assertEquals(assignments, getReplicaAssignmentForTopics(adminClient, asList("foo"))); assignments.clear(); - assignments.put(new TopicPartition("foo", 0), seq(0, 1, 2)); - assignments.put(new TopicPartition("bar", 0), seq(2, 3, 0)); + assignments.put(new TopicPartition("foo", 0), asList(0, 1, 2)); + assignments.put(new TopicPartition("bar", 0), asList(2, 3, 0)); assertEquals(assignments, getReplicaAssignmentForPartitions(adminClient, set(new TopicPartition("foo", 0), new TopicPartition("bar", 0)))); @@ -306,26 +306,26 @@ public void testGetReplicaAssignments() throws Exception { @Test public void testGetBrokerRackInformation() throws Exception { try (MockAdminClient adminClient = new MockAdminClient.Builder(). - brokers(Arrays.asList(new Node(0, "localhost", 9092, "rack0"), + brokers(asList(new Node(0, "localhost", 9092, "rack0"), new Node(1, "localhost", 9093, "rack1"), new Node(2, "localhost", 9094, null))). build()) { - assertEquals(seq( + assertEquals(asList( new BrokerMetadata(0, Optional.of("rack0")), new BrokerMetadata(1, Optional.of("rack1")) - ), getBrokerMetadata(adminClient, seq(0, 1), true)); - assertEquals(seq( + ), getBrokerMetadata(adminClient, asList(0, 1), true)); + assertEquals(asList( new BrokerMetadata(0, Optional.empty()), new BrokerMetadata(1, Optional.empty()) - ), getBrokerMetadata(adminClient, seq(0, 1), false)); + ), getBrokerMetadata(adminClient, asList(0, 1), false)); assertStartsWith("Not all brokers have rack information", assertThrows(AdminOperationException.class, - () -> getBrokerMetadata(adminClient, seq(1, 2), true)).getMessage()); - assertEquals(seq( + () -> getBrokerMetadata(adminClient, asList(1, 2), true)).getMessage()); + assertEquals(asList( new BrokerMetadata(1, Optional.empty()), new BrokerMetadata(2, Optional.empty()) - ), getBrokerMetadata(adminClient, seq(1, 2), false)); + ), getBrokerMetadata(adminClient, asList(1, 2), false)); } } @@ -339,13 +339,13 @@ public void testParseGenerateAssignmentArgs() throws Exception { assertThrows(AdminCommandFailedException.class, () -> parseGenerateAssignmentArgs( "{\"topics\": [{\"topic\": \"foo\"}], \"version\":1}", "5,2,3,4,5"), "Expected to detect duplicate broker list entries").getMessage()); - assertEquals(new Tuple2<>(seq(5, 2, 3, 4), seq("foo")), + assertEquals(new Tuple2<>(asList(5, 2, 3, 4), asList("foo")), parseGenerateAssignmentArgs("{\"topics\": [{\"topic\": \"foo\"}], \"version\":1}", "5,2,3,4")); assertStartsWith("List of topics to reassign contains duplicate entries", assertThrows(AdminCommandFailedException.class, () -> parseGenerateAssignmentArgs( "{\"topics\": [{\"topic\": \"foo\"},{\"topic\": \"foo\"}], \"version\":1}", "5,2,3,4"), "Expected to detect duplicate topic entries").getMessage()); - assertEquals(new Tuple2<>(seq(5, 3, 4), seq("foo", "bar")), + assertEquals(new Tuple2<>(asList(5, 3, 4), asList("foo", "bar")), parseGenerateAssignmentArgs( "{\"topics\": [{\"topic\": \"foo\"},{\"topic\": \"bar\"}], \"version\":1}", "5,3,4")); } @@ -375,7 +375,7 @@ public void testGenerateAssignmentWithInvalidPartitionsFails() { @Test public void testGenerateAssignmentWithInconsistentRacks() throws Exception { try (MockAdminClient adminClient = new MockAdminClient.Builder(). - brokers(Arrays.asList( + brokers(asList( new Node(0, "localhost", 9092, "rack0"), new Node(1, "localhost", 9093, "rack0"), new Node(2, "localhost", 9094, null), @@ -395,8 +395,8 @@ public void testGenerateAssignmentWithInconsistentRacks() throws Exception { Map> expCurrent = new HashMap<>(); - expCurrent.put(new TopicPartition("foo", 0), seq(0, 1, 2)); - expCurrent.put(new TopicPartition("foo", 1), seq(1, 2, 3)); + expCurrent.put(new TopicPartition("foo", 0), asList(0, 1, 2)); + expCurrent.put(new TopicPartition("foo", 1), asList(1, 2, 3)); assertEquals(expCurrent, proposedCurrent.v2); } @@ -406,7 +406,7 @@ public void testGenerateAssignmentWithInconsistentRacks() throws Exception { public void testGenerateAssignmentWithFewerBrokers() throws Exception { try (MockAdminClient adminClient = new MockAdminClient.Builder().numBrokers(4).build()) { addTopics(adminClient); - List goalBrokers = Arrays.asList(0, 1, 3); + List goalBrokers = asList(0, 1, 3); Tuple2>, Map>> proposedCurrent = generateAssignment(adminClient, @@ -415,9 +415,9 @@ public void testGenerateAssignmentWithFewerBrokers() throws Exception { Map> expCurrent = new HashMap<>(); - expCurrent.put(new TopicPartition("foo", 0), seq(0, 1, 2)); - expCurrent.put(new TopicPartition("foo", 1), seq(1, 2, 3)); - expCurrent.put(new TopicPartition("bar", 0), seq(2, 3, 0)); + expCurrent.put(new TopicPartition("foo", 0), asList(0, 1, 2)); + expCurrent.put(new TopicPartition("foo", 1), asList(1, 2, 3)); + expCurrent.put(new TopicPartition("bar", 0), asList(2, 3, 0)); assertEquals(expCurrent, proposedCurrent.v2); @@ -433,15 +433,15 @@ public void testGenerateAssignmentWithFewerBrokers() throws Exception { public void testCurrentPartitionReplicaAssignmentToString() throws Exception { Map> proposedParts = new HashMap<>(); - proposedParts.put(new TopicPartition("foo", 1), seq(1, 2, 3)); - proposedParts.put(new TopicPartition("bar", 0), seq(7, 8, 9)); + proposedParts.put(new TopicPartition("foo", 1), asList(1, 2, 3)); + proposedParts.put(new TopicPartition("bar", 0), asList(7, 8, 9)); Map> currentParts = new HashMap<>(); - currentParts.put(new TopicPartition("foo", 0), seq(1, 2, 3)); - currentParts.put(new TopicPartition("foo", 1), seq(4, 5, 6)); - currentParts.put(new TopicPartition("bar", 0), seq(7, 8)); - currentParts.put(new TopicPartition("baz", 0), seq(10, 11, 12)); + currentParts.put(new TopicPartition("foo", 0), asList(1, 2, 3)); + currentParts.put(new TopicPartition("foo", 1), asList(4, 5, 6)); + currentParts.put(new TopicPartition("bar", 0), asList(7, 8)); + currentParts.put(new TopicPartition("baz", 0), asList(10, 11, 12)); assertEquals(String.join(System.lineSeparator(), "Current partition replica assignment", @@ -468,37 +468,37 @@ public void testMoveMap() { Map currentReassignments = new HashMap<>(); currentReassignments.put(new TopicPartition("foo", 0), new PartitionReassignment( - Arrays.asList(1, 2, 3, 4), Arrays.asList(4), Arrays.asList(3))); + asList(1, 2, 3, 4), asList(4), asList(3))); currentReassignments.put(new TopicPartition("foo", 1), new PartitionReassignment( - Arrays.asList(4, 5, 6, 7, 8), Arrays.asList(7, 8), Arrays.asList(4, 5))); + asList(4, 5, 6, 7, 8), asList(7, 8), asList(4, 5))); currentReassignments.put(new TopicPartition("foo", 2), new PartitionReassignment( - Arrays.asList(1, 2, 3, 4), Arrays.asList(3, 4), Arrays.asList(1, 2))); + asList(1, 2, 3, 4), asList(3, 4), asList(1, 2))); currentReassignments.put(new TopicPartition("foo", 3), new PartitionReassignment( - Arrays.asList(1, 2, 3, 4), Arrays.asList(3, 4), Arrays.asList(1, 2))); + asList(1, 2, 3, 4), asList(3, 4), asList(1, 2))); currentReassignments.put(new TopicPartition("foo", 4), new PartitionReassignment( - Arrays.asList(1, 2, 3, 4), Arrays.asList(3, 4), Arrays.asList(1, 2))); + asList(1, 2, 3, 4), asList(3, 4), asList(1, 2))); currentReassignments.put(new TopicPartition("foo", 5), new PartitionReassignment( - Arrays.asList(1, 2, 3, 4), Arrays.asList(3, 4), Arrays.asList(1, 2))); + asList(1, 2, 3, 4), asList(3, 4), asList(1, 2))); Map> proposedParts = new HashMap<>(); - proposedParts.put(new TopicPartition("foo", 0), seq(1, 2, 5)); - proposedParts.put(new TopicPartition("foo", 2), seq(3, 4)); - proposedParts.put(new TopicPartition("foo", 3), seq(5, 6)); - proposedParts.put(new TopicPartition("foo", 4), seq(3)); - proposedParts.put(new TopicPartition("foo", 5), seq(3, 4, 5, 6)); - proposedParts.put(new TopicPartition("bar", 0), seq(1, 2, 3)); + proposedParts.put(new TopicPartition("foo", 0), asList(1, 2, 5)); + proposedParts.put(new TopicPartition("foo", 2), asList(3, 4)); + proposedParts.put(new TopicPartition("foo", 3), asList(5, 6)); + proposedParts.put(new TopicPartition("foo", 4), asList(3)); + proposedParts.put(new TopicPartition("foo", 5), asList(3, 4, 5, 6)); + proposedParts.put(new TopicPartition("bar", 0), asList(1, 2, 3)); Map> currentParts = new HashMap<>(); - currentParts.put(new TopicPartition("foo", 0), seq(1, 2, 3, 4)); - currentParts.put(new TopicPartition("foo", 1), seq(4, 5, 6, 7, 8)); - currentParts.put(new TopicPartition("foo", 2), seq(1, 2, 3, 4)); - currentParts.put(new TopicPartition("foo", 3), seq(1, 2, 3, 4)); - currentParts.put(new TopicPartition("foo", 4), seq(1, 2, 3, 4)); - currentParts.put(new TopicPartition("foo", 5), seq(1, 2, 3, 4)); - currentParts.put(new TopicPartition("bar", 0), seq(2, 3, 4)); - currentParts.put(new TopicPartition("baz", 0), seq(1, 2, 3)); + currentParts.put(new TopicPartition("foo", 0), asList(1, 2, 3, 4)); + currentParts.put(new TopicPartition("foo", 1), asList(4, 5, 6, 7, 8)); + currentParts.put(new TopicPartition("foo", 2), asList(1, 2, 3, 4)); + currentParts.put(new TopicPartition("foo", 3), asList(1, 2, 3, 4)); + currentParts.put(new TopicPartition("foo", 4), asList(1, 2, 3, 4)); + currentParts.put(new TopicPartition("foo", 5), asList(1, 2, 3, 4)); + currentParts.put(new TopicPartition("bar", 0), asList(2, 3, 4)); + currentParts.put(new TopicPartition("baz", 0), asList(1, 2, 3)); Map> moveMap = calculateProposedMoveMap(currentReassignments, proposedParts, currentParts); @@ -565,8 +565,8 @@ public void testParseExecuteAssignmentArgs() throws Exception { Map> partitionsToBeReassigned = new HashMap<>(); - partitionsToBeReassigned.put(new TopicPartition("foo", 0), seq(1, 2, 3)); - partitionsToBeReassigned.put(new TopicPartition("foo", 1), seq(3, 4, 5)); + partitionsToBeReassigned.put(new TopicPartition("foo", 0), asList(1, 2, 3)); + partitionsToBeReassigned.put(new TopicPartition("foo", 1), asList(3, 4, 5)); Tuple2>, Map> actual = parseExecuteAssignmentArgs( "{\"version\":1,\"partitions\":" + @@ -588,7 +588,7 @@ public void testParseExecuteAssignmentArgs() throws Exception { "[{\"topic\":\"foo\",\"partition\":0,\"replicas\":[1,2,3],\"log_dirs\":[\"/tmp/a\",\"/tmp/b\",\"/tmp/c\"]}" + "]}"); - assertEquals(Collections.singletonMap(new TopicPartition("foo", 0), seq(1, 2, 3)), actual.v1); + assertEquals(Collections.singletonMap(new TopicPartition("foo", 0), asList(1, 2, 3)), actual.v1); assertEquals(replicaAssignment, actual.v2); } @@ -661,9 +661,9 @@ public void testCurReassignmentsToString() throws Exception { Map> reassignments = new HashMap<>(); - reassignments.put(new TopicPartition("foo", 1), seq(4, 5, 3)); - reassignments.put(new TopicPartition("foo", 0), seq(0, 1, 4, 2)); - reassignments.put(new TopicPartition("bar", 0), seq(2, 3)); + reassignments.put(new TopicPartition("foo", 1), asList(4, 5, 3)); + reassignments.put(new TopicPartition("foo", 0), asList(0, 1, 4, 2)); + reassignments.put(new TopicPartition("bar", 0), asList(2, 3)); Map reassignmentResult = alterPartitionReassignments(adminClient, reassignments); @@ -731,7 +731,7 @@ public void testAlterReplicaLogDirs() throws Exception { try (MockAdminClient adminClient = new MockAdminClient.Builder(). numBrokers(4). brokerLogDirs(Collections.nCopies(4, - Arrays.asList("/tmp/kafka-logs0", "/tmp/kafka-logs1"))). + asList("/tmp/kafka-logs0", "/tmp/kafka-logs1"))). build()) { addTopics(adminClient); @@ -763,11 +763,6 @@ public void testPropagateInvalidJsonError() { @SuppressWarnings("unchecked") private static Set set(final T... set) { - return new HashSet<>(Arrays.asList(set)); - } - - @SuppressWarnings("unchecked") - private static List seq(T... seq) { - return Arrays.asList(seq); + return new HashSet<>(asList(set)); } } From 89eb55005f40108a9e9ffdacdec7bb957f9e38ba Mon Sep 17 00:00:00 2001 From: nizhikov Date: Wed, 18 Oct 2023 17:32:30 +0300 Subject: [PATCH 76/78] KAFKA-14595 static import of asList --- .../ReassignPartitionsIntegrationTest.java | 72 +++++++++---------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsIntegrationTest.java b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsIntegrationTest.java index 3a0cd1fc2caac..4290a1fab65bf 100644 --- a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsIntegrationTest.java +++ b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsIntegrationTest.java @@ -55,7 +55,6 @@ import java.io.Closeable; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -72,6 +71,7 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; +import static java.util.Arrays.asList; import static org.apache.kafka.server.common.MetadataVersion.IBP_2_7_IV1; import static org.apache.kafka.test.TestUtils.DEFAULT_MAX_WAIT_MS; import static org.apache.kafka.tools.ToolsTestUtils.TEST_WITH_PARAMETERIZED_QUORUM_NAME; @@ -167,8 +167,8 @@ private void executeAndVerifyReassignment() throws ExecutionException, Interrupt // Check that the assignment has not yet been started yet. Map initialAssignment = new HashMap<>(); - initialAssignment.put(foo0, new PartitionReassignmentState(Arrays.asList(0, 1, 2), Arrays.asList(0, 1, 3), true)); - initialAssignment.put(bar0, new PartitionReassignmentState(Arrays.asList(3, 2, 1), Arrays.asList(3, 2, 0), true)); + initialAssignment.put(foo0, new PartitionReassignmentState(asList(0, 1, 2), asList(0, 1, 3), true)); + initialAssignment.put(bar0, new PartitionReassignmentState(asList(3, 2, 1), asList(3, 2, 0), true)); waitForVerifyAssignment(cluster.adminClient, assignment, false, new VerifyAssignmentResult(initialAssignment)); @@ -177,8 +177,8 @@ private void executeAndVerifyReassignment() throws ExecutionException, Interrupt runExecuteAssignment(cluster.adminClient, false, assignment, -1L, -1L); assertEquals(unthrottledBrokerConfigs, describeBrokerLevelThrottles(unthrottledBrokerConfigs.keySet())); Map finalAssignment = new HashMap<>(); - finalAssignment.put(foo0, new PartitionReassignmentState(Arrays.asList(0, 1, 3), Arrays.asList(0, 1, 3), true)); - finalAssignment.put(bar0, new PartitionReassignmentState(Arrays.asList(3, 2, 0), Arrays.asList(3, 2, 0), true)); + finalAssignment.put(foo0, new PartitionReassignmentState(asList(0, 1, 3), asList(0, 1, 3), true)); + finalAssignment.put(bar0, new PartitionReassignmentState(asList(3, 2, 0), asList(3, 2, 0), true)); VerifyAssignmentResult verifyAssignmentResult = runVerifyAssignment(cluster.adminClient, assignment, false); assertFalse(verifyAssignmentResult.movesOngoing); @@ -211,7 +211,7 @@ public void testHighWaterMarkAfterPartitionReassignment(String quorum) throws Ex // Execute the assignment runExecuteAssignment(cluster.adminClient, false, assignment, -1L, -1L); Map finalAssignment = Collections.singletonMap(part, - new PartitionReassignmentState(Arrays.asList(3, 1, 2), Arrays.asList(3, 1, 2), true)); + new PartitionReassignmentState(asList(3, 1, 2), asList(3, 1, 2), true)); // Wait for the assignment to complete waitForVerifyAssignment(cluster.adminClient, assignment, false, @@ -240,18 +240,18 @@ public void testAlterReassignmentThrottle(String quorum) throws Exception { // Execute the assignment with a low throttle long initialThrottle = 1L; runExecuteAssignment(cluster.adminClient, false, assignment, initialThrottle, -1L); - waitForInterBrokerThrottle(Arrays.asList(0, 1, 2, 3), initialThrottle); + waitForInterBrokerThrottle(asList(0, 1, 2, 3), initialThrottle); // Now update the throttle and verify the reassignment completes long updatedThrottle = 300000L; runExecuteAssignment(cluster.adminClient, true, assignment, updatedThrottle, -1L); - waitForInterBrokerThrottle(Arrays.asList(0, 1, 2, 3), updatedThrottle); + waitForInterBrokerThrottle(asList(0, 1, 2, 3), updatedThrottle); Map finalAssignment = new HashMap<>(); finalAssignment.put(new TopicPartition("foo", 0), - new PartitionReassignmentState(Arrays.asList(0, 3, 2), Arrays.asList(0, 3, 2), true)); + new PartitionReassignmentState(asList(0, 3, 2), asList(0, 3, 2), true)); finalAssignment.put(new TopicPartition("baz", 2), - new PartitionReassignmentState(Arrays.asList(3, 2, 1), Arrays.asList(3, 2, 1), true)); + new PartitionReassignmentState(asList(3, 2, 1), asList(3, 2, 1), true)); // Now remove the throttles. waitForVerifyAssignment(cluster.adminClient, assignment, false, @@ -277,22 +277,22 @@ public void testThrottledReassignment(String quorum) throws Exception { // Check that the assignment has not yet been started yet. Map initialAssignment = new HashMap<>(); initialAssignment.put(new TopicPartition("foo", 0), - new PartitionReassignmentState(Arrays.asList(0, 1, 2), Arrays.asList(0, 3, 2), true)); + new PartitionReassignmentState(asList(0, 1, 2), asList(0, 3, 2), true)); initialAssignment.put(new TopicPartition("baz", 2), - new PartitionReassignmentState(Arrays.asList(0, 2, 1), Arrays.asList(3, 2, 1), true)); + new PartitionReassignmentState(asList(0, 2, 1), asList(3, 2, 1), true)); assertEquals(new VerifyAssignmentResult(initialAssignment), runVerifyAssignment(cluster.adminClient, assignment, false)); assertEquals(unthrottledBrokerConfigs, describeBrokerLevelThrottles(unthrottledBrokerConfigs.keySet())); // Execute the assignment long interBrokerThrottle = 300000L; runExecuteAssignment(cluster.adminClient, false, assignment, interBrokerThrottle, -1L); - waitForInterBrokerThrottle(Arrays.asList(0, 1, 2, 3), interBrokerThrottle); + waitForInterBrokerThrottle(asList(0, 1, 2, 3), interBrokerThrottle); Map finalAssignment = new HashMap<>(); finalAssignment.put(new TopicPartition("foo", 0), - new PartitionReassignmentState(Arrays.asList(0, 3, 2), Arrays.asList(0, 3, 2), true)); + new PartitionReassignmentState(asList(0, 3, 2), asList(0, 3, 2), true)); finalAssignment.put(new TopicPartition("baz", 2), - new PartitionReassignmentState(Arrays.asList(3, 2, 1), Arrays.asList(3, 2, 1), true)); + new PartitionReassignmentState(asList(3, 2, 1), asList(3, 2, 1), true)); // Wait for the assignment to complete TestUtils.waitUntilTrue( @@ -307,17 +307,17 @@ public void testThrottledReassignment(String quorum) throws Exception { result.partStates.values().stream().allMatch(state -> state.done), "Expected at least one partition reassignment to be ongoing when result = " + result ); - assertEquals(Arrays.asList(0, 3, 2), result.partStates.get(new TopicPartition("foo", 0)).targetReplicas); - assertEquals(Arrays.asList(3, 2, 1), result.partStates.get(new TopicPartition("baz", 2)).targetReplicas); + assertEquals(asList(0, 3, 2), result.partStates.get(new TopicPartition("foo", 0)).targetReplicas); + assertEquals(asList(3, 2, 1), result.partStates.get(new TopicPartition("baz", 2)).targetReplicas); System.out.println("Current result: " + result); - waitForInterBrokerThrottle(Arrays.asList(0, 1, 2, 3), interBrokerThrottle); + waitForInterBrokerThrottle(asList(0, 1, 2, 3), interBrokerThrottle); return false; } }, () -> "Expected reassignment to complete.", DEFAULT_MAX_WAIT_MS, 100L); waitForVerifyAssignment(cluster.adminClient, assignment, true, new VerifyAssignmentResult(finalAssignment)); // The throttles should still have been preserved, since we ran with --preserve-throttles - waitForInterBrokerThrottle(Arrays.asList(0, 1, 2, 3), interBrokerThrottle); + waitForInterBrokerThrottle(asList(0, 1, 2, 3), interBrokerThrottle); // Now remove the throttles. waitForVerifyAssignment(cluster.adminClient, assignment, false, new VerifyAssignmentResult(finalAssignment)); @@ -355,9 +355,9 @@ public void testProduceAndConsumeWithReassignmentInProgress(String quorum) throw } finally { consumer.close(); } - TestUtils.removeReplicationThrottleForPartitions(cluster.adminClient, seq(Arrays.asList(0, 1, 2, 3)), mutableSet(part).toSet()); + TestUtils.removeReplicationThrottleForPartitions(cluster.adminClient, seq(asList(0, 1, 2, 3)), mutableSet(part).toSet()); Map finalAssignment = Collections.singletonMap(part, - new PartitionReassignmentState(Arrays.asList(3, 2, 1), Arrays.asList(3, 2, 1), true)); + new PartitionReassignmentState(asList(3, 2, 1), asList(3, 2, 1), true)); waitForVerifyAssignment(cluster.adminClient, assignment, false, new VerifyAssignmentResult(finalAssignment)); } @@ -383,21 +383,21 @@ public void testCancellation(String quorum) throws Exception { describeBrokerLevelThrottles(unthrottledBrokerConfigs.keySet())); long interBrokerThrottle = 1L; runExecuteAssignment(cluster.adminClient, false, assignment, interBrokerThrottle, -1L); - waitForInterBrokerThrottle(Arrays.asList(0, 1, 2, 3), interBrokerThrottle); + waitForInterBrokerThrottle(asList(0, 1, 2, 3), interBrokerThrottle); Map partStates = new HashMap<>(); - partStates.put(foo0, new PartitionReassignmentState(Arrays.asList(0, 1, 3, 2), Arrays.asList(0, 1, 3), false)); - partStates.put(baz1, new PartitionReassignmentState(Arrays.asList(0, 2, 3, 1), Arrays.asList(0, 2, 3), false)); + partStates.put(foo0, new PartitionReassignmentState(asList(0, 1, 3, 2), asList(0, 1, 3), false)); + partStates.put(baz1, new PartitionReassignmentState(asList(0, 2, 3, 1), asList(0, 2, 3), false)); // Verify that the reassignment is running. The very low throttle should keep it // from completing before this runs. waitForVerifyAssignment(cluster.adminClient, assignment, true, new VerifyAssignmentResult(partStates, true, Collections.emptyMap(), false)); // Cancel the reassignment. - assertEquals(new Tuple2<>(new HashSet<>(Arrays.asList(foo0, baz1)), Collections.emptySet()), runCancelAssignment(cluster.adminClient, assignment, true)); + assertEquals(new Tuple2<>(new HashSet<>(asList(foo0, baz1)), Collections.emptySet()), runCancelAssignment(cluster.adminClient, assignment, true)); // Broker throttles are still active because we passed --preserve-throttles - waitForInterBrokerThrottle(Arrays.asList(0, 1, 2, 3), interBrokerThrottle); + waitForInterBrokerThrottle(asList(0, 1, 2, 3), interBrokerThrottle); // Cancelling the reassignment again should reveal nothing to cancel. assertEquals(new Tuple2<>(Collections.emptySet(), Collections.emptySet()), runCancelAssignment(cluster.adminClient, assignment, false)); // This time, the broker throttles were removed. @@ -426,7 +426,7 @@ public void testCancellationWithAddingReplicaInIsr(String quorum) throws Excepti // We will throttle replica 4 so that only replica 3 joins the ISR TestUtils.setReplicationThrottleForPartitions( cluster.adminClient, - seq(Arrays.asList(4)), + seq(asList(4)), mutableSet(foo0).toSet(), 1 ); @@ -545,7 +545,7 @@ public void testLogDirReassignment(String quorum) throws Exception { cluster.produceMessages(topicPartition.topic(), topicPartition.partition(), 700); int targetBrokerId = 0; - List replicas = Arrays.asList(0, 1, 2); + List replicas = asList(0, 1, 2); LogDirReassignment reassignment = buildLogDirReassignment(topicPartition, targetBrokerId, replicas); // Start the replica move, but throttle it to be very slow so that it can't complete @@ -557,7 +557,7 @@ public void testLogDirReassignment(String quorum) throws Exception { // Check the output of --verify waitForVerifyAssignment(cluster.adminClient, reassignment.json, true, new VerifyAssignmentResult(Collections.singletonMap( - topicPartition, new PartitionReassignmentState(Arrays.asList(0, 1, 2), Arrays.asList(0, 1, 2), true) + topicPartition, new PartitionReassignmentState(asList(0, 1, 2), asList(0, 1, 2), true) ), false, Collections.singletonMap( new TopicPartitionReplica(topicPartition.topic(), topicPartition.partition(), 0), new ActiveMoveState(reassignment.currentDir, reassignment.targetDir, reassignment.targetDir) @@ -575,7 +575,7 @@ topicPartition, new PartitionReassignmentState(Arrays.asList(0, 1, 2), Arrays.as // Wait for the directory movement to complete. waitForVerifyAssignment(cluster.adminClient, reassignment.json, true, new VerifyAssignmentResult(Collections.singletonMap( - topicPartition, new PartitionReassignmentState(Arrays.asList(0, 1, 2), Arrays.asList(0, 1, 2), true) + topicPartition, new PartitionReassignmentState(asList(0, 1, 2), asList(0, 1, 2), true) ), false, Collections.singletonMap( new TopicPartitionReplica(topicPartition.topic(), topicPartition.partition(), 0), new CompletedMoveState(reassignment.targetDir) @@ -595,7 +595,7 @@ public void testAlterLogDirReassignmentThrottle(String quorum) throws Exception cluster.produceMessages(topicPartition.topic(), topicPartition.partition(), 700); int targetBrokerId = 0; - List replicas = Arrays.asList(0, 1, 2); + List replicas = asList(0, 1, 2); LogDirReassignment reassignment = buildLogDirReassignment(topicPartition, targetBrokerId, replicas); // Start the replica move with a low throttle so it does not complete @@ -612,7 +612,7 @@ public void testAlterLogDirReassignmentThrottle(String quorum) throws Exception waitForVerifyAssignment(cluster.adminClient, reassignment.json, true, new VerifyAssignmentResult(Collections.singletonMap( - topicPartition, new PartitionReassignmentState(Arrays.asList(0, 1, 2), Arrays.asList(0, 1, 2), true) + topicPartition, new PartitionReassignmentState(asList(0, 1, 2), asList(0, 1, 2), true) ), false, Collections.singletonMap( new TopicPartitionReplica(topicPartition.topic(), topicPartition.partition(), targetBrokerId), new CompletedMoveState(reassignment.targetDir) @@ -758,9 +758,9 @@ class ReassignPartitionsTestCluster implements Closeable { } private final Map>> topics = new HashMap<>(); { - topics.put("foo", Arrays.asList(Arrays.asList(0, 1, 2), Arrays.asList(1, 2, 3))); - topics.put("bar", Arrays.asList(Arrays.asList(3, 2, 1))); - topics.put("baz", Arrays.asList(Arrays.asList(1, 0, 2), Arrays.asList(2, 0, 1), Arrays.asList(0, 2, 1))); + topics.put("foo", asList(asList(0, 1, 2), asList(1, 2, 3))); + topics.put("bar", asList(asList(3, 2, 1))); + topics.put("baz", asList(asList(1, 0, 2), asList(2, 0, 1), asList(0, 2, 1))); } private final List servers = new ArrayList<>(); @@ -866,7 +866,7 @@ public void close() { @SuppressWarnings({"deprecation", "unchecked"}) private static scala.collection.mutable.Set mutableSet(final T...set) { - return JavaConverters.asScalaSet(new HashSet<>(Arrays.asList(set))); + return JavaConverters.asScalaSet(new HashSet<>(asList(set))); } @SuppressWarnings({"deprecation"}) From f26a02ee7599d3fbdb4a6cae2a30b3662addbde0 Mon Sep 17 00:00:00 2001 From: nizhikov Date: Fri, 20 Oct 2023 00:26:04 +0300 Subject: [PATCH 77/78] KAFKA-15645 Remove set method --- .../reassign/ReassignPartitionsUnitTest.java | 41 ++++++++----------- 1 file changed, 17 insertions(+), 24 deletions(-) diff --git a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java index 2184971efb733..d0e1cb4baaf69 100644 --- a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java +++ b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsUnitTest.java @@ -43,7 +43,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -152,7 +151,6 @@ private void addTopics(MockAdminClient adminClient) { ), Collections.emptyMap()); } - @SuppressWarnings("unchecked") @Test public void testFindPartitionReassignmentStates() throws Exception { try (MockAdminClient adminClient = new MockAdminClient.Builder().numBrokers(4).build()) { @@ -186,7 +184,7 @@ public void testFindPartitionReassignmentStates() throws Exception { // Cancel the reassignment and test findPartitionReassignmentStates again. Map cancelResult = cancelPartitionReassignments(adminClient, - set(new TopicPartition("foo", 0), new TopicPartition("quux", 2))); + new HashSet<>(asList(new TopicPartition("foo", 0), new TopicPartition("quux", 2)))); assertEquals(1, cancelResult.size()); assertEquals(UnknownTopicOrPartitionException.class, cancelResult.get(new TopicPartition("quux", 2)).getClass()); @@ -299,7 +297,7 @@ public void testGetReplicaAssignments() throws Exception { assignments.put(new TopicPartition("bar", 0), asList(2, 3, 0)); assertEquals(assignments, - getReplicaAssignmentForPartitions(adminClient, set(new TopicPartition("foo", 0), new TopicPartition("bar", 0)))); + getReplicaAssignmentForPartitions(adminClient, new HashSet<>(asList(new TopicPartition("foo", 0), new TopicPartition("bar", 0))))); } } @@ -504,16 +502,16 @@ public void testMoveMap() { Map fooMoves = new HashMap<>(); - fooMoves.put(0, new PartitionMove(set(1, 2, 3), set(5))); - fooMoves.put(1, new PartitionMove(set(4, 5, 6), set(7, 8))); - fooMoves.put(2, new PartitionMove(set(1, 2), set(3, 4))); - fooMoves.put(3, new PartitionMove(set(1, 2), set(5, 6))); - fooMoves.put(4, new PartitionMove(set(1, 2), set(3))); - fooMoves.put(5, new PartitionMove(set(1, 2), set(3, 4, 5, 6))); + fooMoves.put(0, new PartitionMove(new HashSet<>(asList(1, 2, 3)), new HashSet<>(asList(5)))); + fooMoves.put(1, new PartitionMove(new HashSet<>(asList(4, 5, 6)), new HashSet<>(asList(7, 8)))); + fooMoves.put(2, new PartitionMove(new HashSet<>(asList(1, 2)), new HashSet<>(asList(3, 4)))); + fooMoves.put(3, new PartitionMove(new HashSet<>(asList(1, 2)), new HashSet<>(asList(5, 6)))); + fooMoves.put(4, new PartitionMove(new HashSet<>(asList(1, 2)), new HashSet<>(asList(3)))); + fooMoves.put(5, new PartitionMove(new HashSet<>(asList(1, 2)), new HashSet<>(asList(3, 4, 5, 6)))); Map barMoves = new HashMap<>(); - barMoves.put(0, new PartitionMove(set(2, 3, 4), set(1))); + barMoves.put(0, new PartitionMove(new HashSet<>(asList(2, 3, 4)), new HashSet<>(asList(1)))); assertEquals(fooMoves, moveMap.get("foo")); assertEquals(barMoves, moveMap.get("bar")); @@ -532,10 +530,10 @@ public void testMoveMap() { assertEquals(expFollowerThrottle, calculateFollowerThrottles(moveMap)); - assertEquals(set(1, 2, 3, 4, 5, 6, 7, 8), calculateReassigningBrokers(moveMap)); - assertEquals(set(0, 2), calculateMovingBrokers(set( + assertEquals(new HashSet<>(asList(1, 2, 3, 4, 5, 6, 7, 8)), calculateReassigningBrokers(moveMap)); + assertEquals(new HashSet<>(asList(0, 2)), calculateMovingBrokers(new HashSet<>(asList( new TopicPartitionReplica("quux", 0, 0), - new TopicPartitionReplica("quux", 1, 2)))); + new TopicPartitionReplica("quux", 1, 2))))); } @Test @@ -621,8 +619,8 @@ public void testExecuteWithInvalidBrokerIdFails() { @Test public void testModifyBrokerInterBrokerThrottle() throws Exception { try (MockAdminClient adminClient = new MockAdminClient.Builder().numBrokers(4).build()) { - modifyInterBrokerThrottle(adminClient, set(0, 1, 2), 1000); - modifyInterBrokerThrottle(adminClient, set(0, 3), 100); + modifyInterBrokerThrottle(adminClient, new HashSet<>(asList(0, 1, 2)), 1000); + modifyInterBrokerThrottle(adminClient, new HashSet<>(asList(0, 3)), 100); List brokers = new ArrayList<>(); for (int i = 0; i < 4; i++) brokers.add(new ConfigResource(ConfigResource.Type.BROKER, Integer.toString(i))); @@ -637,8 +635,8 @@ public void testModifyBrokerInterBrokerThrottle() throws Exception { @Test public void testModifyLogDirThrottle() throws Exception { try (MockAdminClient adminClient = new MockAdminClient.Builder().numBrokers(4).build()) { - modifyLogDirThrottle(adminClient, set(0, 1, 2), 2000); - modifyLogDirThrottle(adminClient, set(0, 3), -1); + modifyLogDirThrottle(adminClient, new HashSet<>(asList(0, 1, 2)), 2000); + modifyLogDirThrottle(adminClient, new HashSet<>(asList(0, 3)), -1); List brokers = new ArrayList<>(); for (int i = 0; i < 4; i++) @@ -742,7 +740,7 @@ public void testAlterReplicaLogDirs() throws Exception { assignment.put(new TopicPartitionReplica("quux", 1, 0), "/tmp/kafka-logs1"); assertEquals( - set(new TopicPartitionReplica("foo", 0, 0)), + new HashSet<>(asList(new TopicPartitionReplica("foo", 0, 0))), alterReplicaLogDirs(adminClient, assignment) ); } @@ -760,9 +758,4 @@ public void testPropagateInvalidJsonError() { assertThrows(AdminOperationException.class, () -> executeAssignment(adminClient, false, "{invalid_json", -1L, -1L, 10000L, Time.SYSTEM)).getMessage()); } } - - @SuppressWarnings("unchecked") - private static Set set(final T... set) { - return new HashSet<>(asList(set)); - } } From 53e767a75f2d48c4e5419fdf83033f881f21771e Mon Sep 17 00:00:00 2001 From: nizhikov Date: Fri, 20 Oct 2023 16:15:25 +0300 Subject: [PATCH 78/78] KAFKA-14595 Use Option.none instead of None$.empty --- .../ReassignPartitionsIntegrationTest.java | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsIntegrationTest.java b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsIntegrationTest.java index 4290a1fab65bf..b71331a6b233f 100644 --- a/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsIntegrationTest.java +++ b/tools/src/test/java/org/apache/kafka/tools/reassign/ReassignPartitionsIntegrationTest.java @@ -47,7 +47,6 @@ import org.junit.jupiter.api.Timeout; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; -import scala.None$; import scala.Option; import scala.Some$; import scala.collection.JavaConverters; @@ -206,7 +205,7 @@ public void testHighWaterMarkAfterPartitionReassignment(String quorum) throws Ex // Set the high water mark of foo-0 to 123 on its leader. TopicPartition part = new TopicPartition("foo", 0); - cluster.servers.get(0).replicaManager().logManager().truncateFullyAndStartAt(part, 123L, false, None$.empty()); + cluster.servers.get(0).replicaManager().logManager().truncateFullyAndStartAt(part, 123L, false, Option.empty()); // Execute the assignment runExecuteAssignment(cluster.adminClient, false, assignment, -1L, -1L); @@ -342,8 +341,8 @@ public void testProduceAndConsumeWithReassignmentInProgress(String quorum) throw false, 500, SecurityProtocol.PLAINTEXT, - None$.empty(), - None$.empty(), + Option.empty(), + Option.empty(), new ByteArrayDeserializer(), new ByteArrayDeserializer() ); @@ -777,9 +776,9 @@ public ReassignPartitionsTestCluster(Map configOverrides, Map - servers.add(createBroker(brokerConfigs.get(brokerId), Time.SYSTEM, true, scala.None$.empty())) + servers.add(createBroker(brokerConfigs.get(brokerId), Time.SYSTEM, true, Option.empty())) ); }