From f59517160a4202fd348e474c03708e51c34795d1 Mon Sep 17 00:00:00 2001 From: Sagar Rao Date: Tue, 9 Aug 2022 17:38:28 +0530 Subject: [PATCH 01/26] KAFKA-12495: Initial commit to get a balanced assignment and allow consecutive rebalance --- .../IncrementalCooperativeAssignor.java | 173 +++++++++++++++--- 1 file changed, 152 insertions(+), 21 deletions(-) diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java index 270bb5263fb81..3fb3694c6643f 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java @@ -16,7 +16,7 @@ */ package org.apache.kafka.connect.runtime.distributed; -import java.util.Arrays; +import java.util.*; import java.util.Map.Entry; import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.Time; @@ -28,18 +28,6 @@ import org.slf4j.Logger; import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.TreeSet; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -64,7 +52,7 @@ public class IncrementalCooperativeAssignor implements ConnectAssignor { private final int maxDelay; private ConnectorsAndTasks previousAssignment; private final ConnectorsAndTasks previousRevocation; - private boolean canRevoke; +// private boolean canRevoke; // visible for testing protected final Set candidateWorkersForReassignment; protected long scheduledRebalance; @@ -78,7 +66,7 @@ public IncrementalCooperativeAssignor(LogContext logContext, Time time, int maxD this.maxDelay = maxDelay; this.previousAssignment = ConnectorsAndTasks.EMPTY; this.previousRevocation = new ConnectorsAndTasks.Builder().build(); - this.canRevoke = true; +// this.canRevoke = true; this.scheduledRebalance = 0; this.candidateWorkersForReassignment = new LinkedHashSet<>(); this.delay = 0; @@ -226,7 +214,7 @@ ClusterAssignment performTaskAssignment( if (previousRevocation.connectors().stream().anyMatch(c -> activeAssignments.connectors().contains(c)) || previousRevocation.tasks().stream().anyMatch(t -> activeAssignments.tasks().contains(t))) { previousAssignment = activeAssignments; - canRevoke = true; +// canRevoke = true; } previousRevocation.connectors().clear(); previousRevocation.tasks().clear(); @@ -287,12 +275,13 @@ ClusterAssignment performTaskAssignment( // Do not revoke resources for re-assignment while a delayed rebalance is active // Also we do not revoke in two consecutive rebalances by the same leader - canRevoke = delay == 0 && canRevoke; +// canRevoke = delay == 0 && canRevoke; // Compute the connectors-and-tasks to be revoked for load balancing without taking into // account the deleted ones. - log.debug("Can leader revoke tasks in this assignment? {} (delay: {})", canRevoke, delay); - if (canRevoke) { +// log.debug("Can leader revoke tasks in this assignment? {} (delay: {})", canRevoke, delay); + log.debug("Can leader revoke tasks in this assignment? (delay: {})", delay); + if (delay == 0) { Map toExplicitlyRevoke = performTaskRevocation(activeAssignments, currentWorkerAssignment); @@ -307,9 +296,10 @@ ClusterAssignment performTaskAssignment( existing.tasks().addAll(assignment.tasks()); } ); - canRevoke = toExplicitlyRevoke.size() == 0; +// canRevoke = toExplicitlyRevoke.size() == 0; } else { - canRevoke = delay == 0; +// canRevoke = delay == 0; + log.debug("Connector and task to revoke assignments: {}", toRevoke); } assignConnectors(completeWorkerAssignment, newSubmissions.connectors()); @@ -326,6 +316,9 @@ ClusterAssignment performTaskAssignment( Map> incrementalTaskAssignments = diff(taskAssignments, currentTaskAssignments); + // Balance the assignments one final time based on round-robin assignment of connectors and tasks. + balanceAssignmentsFinally(completeWorkerAssignment, toRevoke); + previousAssignment = computePreviousAssignment(toRevoke, connectorAssignments, taskAssignments, lostAssignments); previousGenerationId = currentGenerationId; previousMembers = memberAssignments.keySet(); @@ -346,6 +339,144 @@ ClusterAssignment performTaskAssignment( ); } + private void balanceAssignmentsFinally(final List newCompleteAssignments, + final Map toRevoke) { + + if (delay == 0) { + + // First remove all connectors and tasks that we have already decided to remove + final List newCompleteAssignmentsPostRevocations = new ArrayList<>(); + int totalWorkUnits = 0; + + for (final WorkerLoad workerLoad: newCompleteAssignments) { + // if there's nothing to be revoked for this worker, then just add it + if (!toRevoke.containsKey(workerLoad.worker())) { + newCompleteAssignmentsPostRevocations.add(workerLoad); + } else { + final ConnectorsAndTasks toRevokeForThisWorker = toRevoke.get(workerLoad.worker()); + final WorkerLoad newWorkerLoad = new WorkerLoad.Builder(workerLoad.worker()).build(); + // assign connectors first + for (final String connector: workerLoad.connectors()) { + if (!toRevokeForThisWorker.connectors().contains(connector)) { + newWorkerLoad.assign(connector); + } + } + // now assign tasks + for (final ConnectorTaskId tasks: workerLoad.tasks()) { + if (!toRevokeForThisWorker.tasks().contains(tasks)) { + newWorkerLoad.assign(tasks); + } + } + newCompleteAssignmentsPostRevocations.add(newWorkerLoad); + totalWorkUnits += newWorkerLoad.connectorsSize() + newWorkerLoad.tasksSize(); + } + } + + // Sort the revised assignments list in increasing order of number of assignments per worker. + newCompleteAssignmentsPostRevocations.sort(Comparator.comparingInt(w -> w.connectorsSize() + w.tasksSize())); + + // Round-robin assignment can provide the most balanced assignments. + // Every index corresponds to the number of connector a worker should have in increasing + // order of number of assignments. + final int[] workUnitCountPerWorkerByRoundRobinAssignment = new int[newCompleteAssignmentsPostRevocations.size()]; + // These many work units should be assigned to every worker at a mininum + final int minAssignments = totalWorkUnits/newCompleteAssignmentsPostRevocations.size(); + + Arrays.fill(workUnitCountPerWorkerByRoundRobinAssignment, minAssignments); + + int extraAssignments = totalWorkUnits % newCompleteAssignmentsPostRevocations.size(); + int workerIndex = workUnitCountPerWorkerByRoundRobinAssignment.length - 1; + + while (extraAssignments > 0) { + workUnitCountPerWorkerByRoundRobinAssignment[workerIndex]++; + workerIndex--; + extraAssignments--; + } + + /* + [5, 6, 12, 12] + [8, 9, 9, 9] + + i = 3; + [6, 7, 13, 9] + + [6, 7, 9, 13] + + [7, 8, 9, 11] + [8, 9, 9, 9] + + */ + + while (!isAssignmentBalanced(newCompleteAssignmentsPostRevocations, + workUnitCountPerWorkerByRoundRobinAssignment)) { + final int lastWorkerIndex = newCompleteAssignmentsPostRevocations.size() - 1; + final WorkerLoad workerWithMaxLoad = newCompleteAssignmentsPostRevocations.get(lastWorkerIndex); + final LinkedList connectors = new LinkedList<>(workerWithMaxLoad.connectors()); + final LinkedList tasks = new LinkedList<>(workerWithMaxLoad.tasks()); + // We start with revoking connector if available and would keep toggling this switch. + boolean revokeConnector = !connectors.isEmpty(); + + // connectors and tasks reflect the running status of assignments for this worker. + // We will keep revoking until we don't hit the calculated balanced number of assignments + // for this worker + while (connectors.size() + tasks.size() > workUnitCountPerWorkerByRoundRobinAssignment[lastWorkerIndex]) { + int index = 0; + while (index < lastWorkerIndex) { + // Worker at this index already has the requisite assignment. + // No revocations needed. + if (newCompleteAssignmentsPostRevocations.get(index).connectorsSize() + + newCompleteAssignmentsPostRevocations.get(index).tasksSize() == + workUnitCountPerWorkerByRoundRobinAssignment[index]) { + index++; + } else { + // Revoke either a connector or task based on round-robin fashion + // based on availability. + if (revokeConnector) { + final String connectorToRevoke = connectors.pollLast(); + final ConnectorsAndTasks revokingForWorker = toRevoke.getOrDefault(workerWithMaxLoad.worker(), ConnectorsAndTasks.EMPTY); + revokingForWorker.connectors().add(connectorToRevoke); + toRevoke.put(workerWithMaxLoad.worker(), revokingForWorker); + newCompleteAssignmentsPostRevocations.get(index).assign(connectorToRevoke); + // Toggle only if there are task assignments left for revocation + revokeConnector = !tasks.isEmpty(); + } else { + final ConnectorTaskId taskToRevoke = tasks.pollLast(); + final ConnectorsAndTasks revokingForWorker = toRevoke.getOrDefault(workerWithMaxLoad.worker(), ConnectorsAndTasks.EMPTY); + revokingForWorker.tasks().add(taskToRevoke); + toRevoke.put(workerWithMaxLoad.worker(), revokingForWorker); + newCompleteAssignmentsPostRevocations.get(index).assign(taskToRevoke); + // Toggle only if there are connector assignments left for revocation + revokeConnector = !connectors.isEmpty(); + } + index++; + } + } + } + // Update the assignments of the given worker based on whatever is left after + // further revocations to balance the load. + newCompleteAssignmentsPostRevocations.set(lastWorkerIndex, + new WorkerLoad.Builder(workerWithMaxLoad.worker()).with(connectors, tasks).build()); + // Sort by number of assignments again + newCompleteAssignmentsPostRevocations.sort(Comparator.comparingInt(w -> w.connectorsSize() + w.tasksSize())); + } + + } else { + log.debug("Since this final balancing act may involve another round of revocations," + + "we will not do this during an active delayed rebalance"); + } + + } + + private boolean isAssignmentBalanced(final List currentAssignment, final int[] balancedAssignmentCount) { + for (int i = 0; i < balancedAssignmentCount.length; i++) { + if (currentAssignment.get(i).connectorsSize() + + currentAssignment.get(i).tasksSize() != balancedAssignmentCount[i]) { + return false; + } + } + return true; + } + private Map computeDeleted(ConnectorsAndTasks deleted, Map> connectorAssignments, Map> taskAssignments) { From f42c80b35394485e0f2ee27c2553eec4b590fc90 Mon Sep 17 00:00:00 2001 From: Sagar Rao Date: Sat, 20 Aug 2022 13:59:01 +0530 Subject: [PATCH 02/26] Some local changes --- .../IncrementalCooperativeAssignor.java | 30 +++++++++++++++---- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java index 3fb3694c6643f..e2215441a0717 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java @@ -280,7 +280,7 @@ ClusterAssignment performTaskAssignment( // Compute the connectors-and-tasks to be revoked for load balancing without taking into // account the deleted ones. // log.debug("Can leader revoke tasks in this assignment? {} (delay: {})", canRevoke, delay); - log.debug("Can leader revoke tasks in this assignment? (delay: {})", delay); +// log.debug("Can leader revoke tasks in this assignment? (delay: {})", delay); if (delay == 0) { Map toExplicitlyRevoke = performTaskRevocation(activeAssignments, currentWorkerAssignment); @@ -317,7 +317,11 @@ ClusterAssignment performTaskAssignment( diff(taskAssignments, currentTaskAssignments); // Balance the assignments one final time based on round-robin assignment of connectors and tasks. + log.debug("Final complete assignments before: {}", completeWorkerAssignment); + log.debug("toRevoke before::{}", toRevoke); balanceAssignmentsFinally(completeWorkerAssignment, toRevoke); + log.debug("toRevoke after::{}", toRevoke); + log.debug("Final complete assignments after: {}", completeWorkerAssignment); previousAssignment = computePreviousAssignment(toRevoke, connectorAssignments, taskAssignments, lostAssignments); previousGenerationId = currentGenerationId; @@ -372,6 +376,11 @@ private void balanceAssignmentsFinally(final List newCompleteAssignm } } + if (newCompleteAssignments.equals(newCompleteAssignmentsPostRevocations)) { + log.debug("The assignments are already balanced. No need for further revocations"); + return; + } + // Sort the revised assignments list in increasing order of number of assignments per worker. newCompleteAssignmentsPostRevocations.sort(Comparator.comparingInt(w -> w.connectorsSize() + w.tasksSize())); @@ -437,17 +446,26 @@ private void balanceAssignmentsFinally(final List newCompleteAssignm revokingForWorker.connectors().add(connectorToRevoke); toRevoke.put(workerWithMaxLoad.worker(), revokingForWorker); newCompleteAssignmentsPostRevocations.get(index).assign(connectorToRevoke); - // Toggle only if there are task assignments left for revocation - revokeConnector = !tasks.isEmpty(); } else { final ConnectorTaskId taskToRevoke = tasks.pollLast(); - final ConnectorsAndTasks revokingForWorker = toRevoke.getOrDefault(workerWithMaxLoad.worker(), ConnectorsAndTasks.EMPTY); + final ConnectorsAndTasks revokingForWorker = toRevoke.getOrDefault(workerWithMaxLoad.worker(), ConnectorsAndTasks.EMPTY); revokingForWorker.tasks().add(taskToRevoke); toRevoke.put(workerWithMaxLoad.worker(), revokingForWorker); newCompleteAssignmentsPostRevocations.get(index).assign(taskToRevoke); - // Toggle only if there are connector assignments left for revocation - revokeConnector = !connectors.isEmpty(); } + + if (revokeConnector) { + // toggle only if there are tasks + if (!tasks.isEmpty()) { + revokeConnector = false; + } + } else { + // toggle only if there are connectors + if (!connectors.isEmpty()) { + revokeConnector = true; + } + } + index++; } } From 73eec50e2143fb8d43419bac2cab01fc4953eb5c Mon Sep 17 00:00:00 2001 From: Sagar Rao Date: Fri, 26 Aug 2022 15:35:13 +0530 Subject: [PATCH 03/26] KAFKA-12495: Adding exponential backoff delay when worker joins after a revoking rebalance before a revoking rebalance can be triggered --- .../IncrementalCooperativeAssignor.java | 221 ++++-------------- .../IncrementalCooperativeAssignorTest.java | 77 ++++-- 2 files changed, 112 insertions(+), 186 deletions(-) diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java index e2215441a0717..0776a59b251f7 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java @@ -16,8 +16,10 @@ */ package org.apache.kafka.connect.runtime.distributed; -import java.util.*; +import java.util.Arrays; import java.util.Map.Entry; + +import org.apache.kafka.common.utils.ExponentialBackoff; import org.apache.kafka.common.utils.LogContext; import org.apache.kafka.common.utils.Time; import org.apache.kafka.connect.runtime.distributed.WorkerCoordinator.ConnectorsAndTasks; @@ -28,6 +30,18 @@ import org.slf4j.Logger; import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.TreeSet; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -52,7 +66,7 @@ public class IncrementalCooperativeAssignor implements ConnectAssignor { private final int maxDelay; private ConnectorsAndTasks previousAssignment; private final ConnectorsAndTasks previousRevocation; -// private boolean canRevoke; + private boolean canRevoke; // visible for testing protected final Set candidateWorkersForReassignment; protected long scheduledRebalance; @@ -60,18 +74,27 @@ public class IncrementalCooperativeAssignor implements ConnectAssignor { protected int previousGenerationId; protected Set previousMembers; + private final ExponentialBackoff nextSuccessiveRebalance; + + private long revokingRebalanceScheduledAt; + + private int numSuccessiveRevokingRebalances; + public IncrementalCooperativeAssignor(LogContext logContext, Time time, int maxDelay) { this.log = logContext.logger(IncrementalCooperativeAssignor.class); this.time = time; this.maxDelay = maxDelay; this.previousAssignment = ConnectorsAndTasks.EMPTY; this.previousRevocation = new ConnectorsAndTasks.Builder().build(); -// this.canRevoke = true; + this.canRevoke = true; this.scheduledRebalance = 0; this.candidateWorkersForReassignment = new LinkedHashSet<>(); this.delay = 0; + this.revokingRebalanceScheduledAt = 0; this.previousGenerationId = -1; this.previousMembers = Collections.emptySet(); + this.numSuccessiveRevokingRebalances = 0; + this.nextSuccessiveRebalance = new ExponentialBackoff(10, 20, maxDelay, 0.2); } @Override @@ -214,7 +237,7 @@ ClusterAssignment performTaskAssignment( if (previousRevocation.connectors().stream().anyMatch(c -> activeAssignments.connectors().contains(c)) || previousRevocation.tasks().stream().anyMatch(t -> activeAssignments.tasks().contains(t))) { previousAssignment = activeAssignments; -// canRevoke = true; + canRevoke = true; } previousRevocation.connectors().clear(); previousRevocation.tasks().clear(); @@ -259,7 +282,6 @@ ClusterAssignment performTaskAssignment( Map toRevoke = computeDeleted(deleted, connectorAssignments, taskAssignments); log.debug("Connector and task to delete assignments: {}", toRevoke); - // Revoking redundant connectors/tasks if the workers have duplicate assignments toRevoke.putAll(computeDuplicatedAssignments(memberAssignments, connectorAssignments, taskAssignments)); log.debug("Connector and task to revoke assignments (include duplicated assignments): {}", toRevoke); @@ -273,19 +295,14 @@ ClusterAssignment performTaskAssignment( handleLostAssignments(lostAssignments, newSubmissions, completeWorkerAssignment); - // Do not revoke resources for re-assignment while a delayed rebalance is active - // Also we do not revoke in two consecutive rebalances by the same leader -// canRevoke = delay == 0 && canRevoke; - // Compute the connectors-and-tasks to be revoked for load balancing without taking into // account the deleted ones. -// log.debug("Can leader revoke tasks in this assignment? {} (delay: {})", canRevoke, delay); -// log.debug("Can leader revoke tasks in this assignment? (delay: {})", delay); - if (delay == 0) { - Map toExplicitlyRevoke = - performTaskRevocation(activeAssignments, currentWorkerAssignment); + if (delay == 0 && canRevoke()) { + log.debug("Can leader revoke tasks in this assignment? {} (delay: {})", canRevoke, delay); - log.debug("Connector and task to revoke assignments: {}", toRevoke); + + Map toExplicitlyRevoke = + performTaskRevocation(configured, completeWorkerAssignment); toExplicitlyRevoke.forEach( (worker, assignment) -> { @@ -296,9 +313,7 @@ ClusterAssignment performTaskAssignment( existing.tasks().addAll(assignment.tasks()); } ); -// canRevoke = toExplicitlyRevoke.size() == 0; } else { -// canRevoke = delay == 0; log.debug("Connector and task to revoke assignments: {}", toRevoke); } @@ -316,13 +331,6 @@ ClusterAssignment performTaskAssignment( Map> incrementalTaskAssignments = diff(taskAssignments, currentTaskAssignments); - // Balance the assignments one final time based on round-robin assignment of connectors and tasks. - log.debug("Final complete assignments before: {}", completeWorkerAssignment); - log.debug("toRevoke before::{}", toRevoke); - balanceAssignmentsFinally(completeWorkerAssignment, toRevoke); - log.debug("toRevoke after::{}", toRevoke); - log.debug("Final complete assignments after: {}", completeWorkerAssignment); - previousAssignment = computePreviousAssignment(toRevoke, connectorAssignments, taskAssignments, lostAssignments); previousGenerationId = currentGenerationId; previousMembers = memberAssignments.keySet(); @@ -343,155 +351,28 @@ ClusterAssignment performTaskAssignment( ); } - private void balanceAssignmentsFinally(final List newCompleteAssignments, - final Map toRevoke) { - - if (delay == 0) { - - // First remove all connectors and tasks that we have already decided to remove - final List newCompleteAssignmentsPostRevocations = new ArrayList<>(); - int totalWorkUnits = 0; - - for (final WorkerLoad workerLoad: newCompleteAssignments) { - // if there's nothing to be revoked for this worker, then just add it - if (!toRevoke.containsKey(workerLoad.worker())) { - newCompleteAssignmentsPostRevocations.add(workerLoad); - } else { - final ConnectorsAndTasks toRevokeForThisWorker = toRevoke.get(workerLoad.worker()); - final WorkerLoad newWorkerLoad = new WorkerLoad.Builder(workerLoad.worker()).build(); - // assign connectors first - for (final String connector: workerLoad.connectors()) { - if (!toRevokeForThisWorker.connectors().contains(connector)) { - newWorkerLoad.assign(connector); - } - } - // now assign tasks - for (final ConnectorTaskId tasks: workerLoad.tasks()) { - if (!toRevokeForThisWorker.tasks().contains(tasks)) { - newWorkerLoad.assign(tasks); - } - } - newCompleteAssignmentsPostRevocations.add(newWorkerLoad); - totalWorkUnits += newWorkerLoad.connectorsSize() + newWorkerLoad.tasksSize(); - } - } - - if (newCompleteAssignments.equals(newCompleteAssignmentsPostRevocations)) { - log.debug("The assignments are already balanced. No need for further revocations"); - return; - } - - // Sort the revised assignments list in increasing order of number of assignments per worker. - newCompleteAssignmentsPostRevocations.sort(Comparator.comparingInt(w -> w.connectorsSize() + w.tasksSize())); - - // Round-robin assignment can provide the most balanced assignments. - // Every index corresponds to the number of connector a worker should have in increasing - // order of number of assignments. - final int[] workUnitCountPerWorkerByRoundRobinAssignment = new int[newCompleteAssignmentsPostRevocations.size()]; - // These many work units should be assigned to every worker at a mininum - final int minAssignments = totalWorkUnits/newCompleteAssignmentsPostRevocations.size(); - - Arrays.fill(workUnitCountPerWorkerByRoundRobinAssignment, minAssignments); - - int extraAssignments = totalWorkUnits % newCompleteAssignmentsPostRevocations.size(); - int workerIndex = workUnitCountPerWorkerByRoundRobinAssignment.length - 1; - - while (extraAssignments > 0) { - workUnitCountPerWorkerByRoundRobinAssignment[workerIndex]++; - workerIndex--; - extraAssignments--; - } - - /* - [5, 6, 12, 12] - [8, 9, 9, 9] - - i = 3; - [6, 7, 13, 9] - - [6, 7, 9, 13] - - [7, 8, 9, 11] - [8, 9, 9, 9] - - */ - - while (!isAssignmentBalanced(newCompleteAssignmentsPostRevocations, - workUnitCountPerWorkerByRoundRobinAssignment)) { - final int lastWorkerIndex = newCompleteAssignmentsPostRevocations.size() - 1; - final WorkerLoad workerWithMaxLoad = newCompleteAssignmentsPostRevocations.get(lastWorkerIndex); - final LinkedList connectors = new LinkedList<>(workerWithMaxLoad.connectors()); - final LinkedList tasks = new LinkedList<>(workerWithMaxLoad.tasks()); - // We start with revoking connector if available and would keep toggling this switch. - boolean revokeConnector = !connectors.isEmpty(); - - // connectors and tasks reflect the running status of assignments for this worker. - // We will keep revoking until we don't hit the calculated balanced number of assignments - // for this worker - while (connectors.size() + tasks.size() > workUnitCountPerWorkerByRoundRobinAssignment[lastWorkerIndex]) { - int index = 0; - while (index < lastWorkerIndex) { - // Worker at this index already has the requisite assignment. - // No revocations needed. - if (newCompleteAssignmentsPostRevocations.get(index).connectorsSize() + - newCompleteAssignmentsPostRevocations.get(index).tasksSize() == - workUnitCountPerWorkerByRoundRobinAssignment[index]) { - index++; - } else { - // Revoke either a connector or task based on round-robin fashion - // based on availability. - if (revokeConnector) { - final String connectorToRevoke = connectors.pollLast(); - final ConnectorsAndTasks revokingForWorker = toRevoke.getOrDefault(workerWithMaxLoad.worker(), ConnectorsAndTasks.EMPTY); - revokingForWorker.connectors().add(connectorToRevoke); - toRevoke.put(workerWithMaxLoad.worker(), revokingForWorker); - newCompleteAssignmentsPostRevocations.get(index).assign(connectorToRevoke); - } else { - final ConnectorTaskId taskToRevoke = tasks.pollLast(); - final ConnectorsAndTasks revokingForWorker = toRevoke.getOrDefault(workerWithMaxLoad.worker(), ConnectorsAndTasks.EMPTY); - revokingForWorker.tasks().add(taskToRevoke); - toRevoke.put(workerWithMaxLoad.worker(), revokingForWorker); - newCompleteAssignmentsPostRevocations.get(index).assign(taskToRevoke); - } - - if (revokeConnector) { - // toggle only if there are tasks - if (!tasks.isEmpty()) { - revokeConnector = false; - } - } else { - // toggle only if there are connectors - if (!connectors.isEmpty()) { - revokeConnector = true; - } - } - - index++; - } - } - } - // Update the assignments of the given worker based on whatever is left after - // further revocations to balance the load. - newCompleteAssignmentsPostRevocations.set(lastWorkerIndex, - new WorkerLoad.Builder(workerWithMaxLoad.worker()).with(connectors, tasks).build()); - // Sort by number of assignments again - newCompleteAssignmentsPostRevocations.sort(Comparator.comparingInt(w -> w.connectorsSize() + w.tasksSize())); - } - } else { - log.debug("Since this final balancing act may involve another round of revocations," + - "we will not do this during an active delayed rebalance"); + private boolean canRevoke() { + // First worker joining without an active delay + // We can trigger a rebalance now. + if (revokingRebalanceScheduledAt == 0) { + long revokeAfter = nextSuccessiveRebalance.backoff(++numSuccessiveRevokingRebalances); + log.debug("revokeAfter:{}, numSuccessiveRevokingRebalances:{}", revokeAfter, numSuccessiveRevokingRebalances); + revokingRebalanceScheduledAt = time.milliseconds() + revokeAfter; + return true; } - - } - - private boolean isAssignmentBalanced(final List currentAssignment, final int[] balancedAssignmentCount) { - for (int i = 0; i < balancedAssignmentCount.length; i++) { - if (currentAssignment.get(i).connectorsSize() + - currentAssignment.get(i).tasksSize() != balancedAssignmentCount[i]) { - return false; - } + long now = time.milliseconds(); + // We can't rebalance until the next scheduled successive revoking rebalance + if (now < revokingRebalanceScheduledAt) { + log.debug("Can't revoke as current time {} is less than next scheduled rebalance {}", now, revokingRebalanceScheduledAt); + return false; } + + // We crossed the last revoking rebalance time. Compute the next one by also + // including the current successful revoking rebalance. + long revokeAfter = nextSuccessiveRebalance.backoff(++numSuccessiveRevokingRebalances); + log.debug("revokeAfter:{}, numSuccessiveRevokingRebalances:{}", revokeAfter, numSuccessiveRevokingRebalances); + revokingRebalanceScheduledAt += revokeAfter; return true; } diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java index ed825312096f7..10258cb2830ed 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java @@ -30,6 +30,7 @@ import org.junit.Test; import java.nio.ByteBuffer; +import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -98,6 +99,8 @@ public void testTaskAssignmentWhenWorkerJoins() { assertBalancedAndCompleteAllocation(); // Second assignment with a second worker joining and all connectors running on previous worker + // Adding a sleep as otherwise the revocations mayn't happen due to exponential backoff + time.sleep(200); addNewEmptyWorkers("worker2"); performStandardRebalance(); assertDelay(0); @@ -118,6 +121,50 @@ public void testTaskAssignmentWhenWorkerJoins() { assertEmptyAssignment(); } + @Test + public void testAssignmentsWhenWorkerJoinsAfterReovcations() { + + // First assignment with 1 worker and 2 connectors configured but not yet assigned + performStandardRebalance(); + assertDelay(0); + assertWorkers("worker1"); + assertConnectorAllocations(2); + assertTaskAllocations(8); + assertBalancedAndCompleteAllocation(); + + // Wait for few milliseconds to pass the next scheduled rebalance delay + time.sleep(Duration.ofMillis(200).toMillis()); + // Second assignment with a second worker joining and all connectors running on previous worker + // We should revoke. + addNewEmptyWorkers("worker2"); + performStandardRebalance(); + assertDelay(0); + assertWorkers("worker1", "worker2"); + assertConnectorAllocations(0, 1); + assertTaskAllocations(0, 4); + + // Third assignment immediately after revocations, and a third worker joining + // This should not lead to a revocation due to the exponential back-off retry + addNewEmptyWorkers("worker3"); + performStandardRebalance(); + assertDelay(0); + assertWorkers("worker1", "worker2", "worker3"); + assertConnectorAllocations(0, 1, 1); + assertTaskAllocations(2, 2, 4); + + // Wait some more time before adding a new worker. + time.sleep(Duration.ofSeconds(5).toMillis()); + // Fourth assignment after revocations, and a fourth worker joining + // We should have a revocation in this case. + addNewEmptyWorkers("worker4"); + performStandardRebalance(); + assertDelay(0); + assertWorkers("worker1", "worker2", "worker3", "worker4"); + assertConnectorAllocations(0, 0, 1, 1); + assertTaskAllocations(0, 2, 2, 2); + + } + @Test public void testTaskAssignmentWhenWorkerLeavesPermanently() { // Customize assignor for this test case @@ -281,7 +328,10 @@ public void testTaskAssignmentWhenLeaderBounces() { // Third assignment with the previous leader returning as a follower. In this case, the // arrival of the previous leader is treated as an arrival of a new worker. Reassignment - // happens immediately, first with a revocation + // happens immediately, first with a revocation. + // Since the new leader may join within the next scheduled revoking rebalance, there may not be any revocations + // so adding a sleep. + time.sleep(300); addNewEmptyWorkers("worker1"); performStandardRebalance(); assertDelay(0); @@ -349,6 +399,8 @@ public void testTaskAssignmentWhenSubsequentAssignmentAttemptFails() { // Third assignment happens with members returning the same assignments (memberConfigs) // as the first time. + // wait for a few milliseconds before rebalancing to let revocations happen + time.sleep(250); performStandardRebalance(); assertDelay(0); assertConnectorAllocations(0, 1, 1); @@ -390,6 +442,8 @@ public void testTaskAssignmentWhenSubsequentAssignmentAttemptFailsOutsideTheAssi // Third assignment happens with members returning the same assignments (memberConfigs) // as the first time. + // Waiting for a few milliseconds before triggering a rebalance to let revocations happen + time.sleep(250); performRebalanceWithMismatchedGeneration(); assertDelay(0); assertConnectorAllocations(0, 1, 1); @@ -840,17 +894,11 @@ public void testTaskAssignmentWhenTasksDuplicatedInWorkerAssignment() { assertTaskAllocations(0, 4); // Third assignment after revocations - performStandardRebalance(); - assertDelay(0); - assertConnectorAllocations(1, 1); - assertTaskAllocations(2, 4); - - // fourth rebalance after revocations + // Assignments should be balanced by this round performStandardRebalance(); assertDelay(0); assertConnectorAllocations(1, 1); assertTaskAllocations(4, 4); - assertBalancedAndCompleteAllocation(); // Fifth rebalance should not change assignments performStandardRebalance(); @@ -872,27 +920,24 @@ public void testDuplicatedAssignmentHandleWhenTheDuplicatedAssignmentsDeleted() removeConnector("connector1"); // Second assignment with a second worker with duplicate assignment joining and the duplicated assignment is deleted at the same time + // The tasks should get rebalanced + // Adding a sleep of few milliseconds to let the rebalance happen + time.sleep(250); addNewWorker("worker2", newConnectors(1, 2), newTasks("connector1", 0, 4)); performStandardRebalance(); assertDelay(0); assertWorkers("worker1", "worker2"); assertConnectorAllocations(0, 1); - assertTaskAllocations(0, 4); - - // Third assignment after revocations - performStandardRebalance(); - assertDelay(0); - assertConnectorAllocations(0, 1); assertTaskAllocations(0, 2); - // fourth rebalance after revocations + // Third rebalance after revocations performStandardRebalance(); assertDelay(0); assertConnectorAllocations(0, 1); assertTaskAllocations(2, 2); assertBalancedAndCompleteAllocation(); - // Fifth rebalance should not change assignments + // Fourth rebalance should not change assignments performStandardRebalance(); assertDelay(0); assertEmptyAssignment(); From 02787b361e4c3cfc6af6b11aca77d41383784d8d Mon Sep 17 00:00:00 2001 From: Sagar Rao Date: Sun, 28 Aug 2022 12:43:57 +0530 Subject: [PATCH 04/26] Adding revokedInPrevious state to exponential backoff delay only when there are successive revoking rebalabces --- .../IncrementalCooperativeAssignor.java | 83 ++++++++++--------- .../IncrementalCooperativeAssignorTest.java | 60 ++++++++------ 2 files changed, 77 insertions(+), 66 deletions(-) diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java index 0776a59b251f7..21096eb5a82b8 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java @@ -66,17 +66,15 @@ public class IncrementalCooperativeAssignor implements ConnectAssignor { private final int maxDelay; private ConnectorsAndTasks previousAssignment; private final ConnectorsAndTasks previousRevocation; - private boolean canRevoke; // visible for testing + boolean revokedInPrevious; protected final Set candidateWorkersForReassignment; protected long scheduledRebalance; protected int delay; protected int previousGenerationId; protected Set previousMembers; - private final ExponentialBackoff nextSuccessiveRebalance; - - private long revokingRebalanceScheduledAt; + private final ExponentialBackoff consecutiveRevokingRebalancesBackoff; private int numSuccessiveRevokingRebalances; @@ -86,15 +84,14 @@ public IncrementalCooperativeAssignor(LogContext logContext, Time time, int maxD this.maxDelay = maxDelay; this.previousAssignment = ConnectorsAndTasks.EMPTY; this.previousRevocation = new ConnectorsAndTasks.Builder().build(); - this.canRevoke = true; this.scheduledRebalance = 0; + this.revokedInPrevious = false; this.candidateWorkersForReassignment = new LinkedHashSet<>(); this.delay = 0; - this.revokingRebalanceScheduledAt = 0; this.previousGenerationId = -1; this.previousMembers = Collections.emptySet(); this.numSuccessiveRevokingRebalances = 0; - this.nextSuccessiveRebalance = new ExponentialBackoff(10, 20, maxDelay, 0.2); + this.consecutiveRevokingRebalancesBackoff = new ExponentialBackoff(10, 20, maxDelay, 0.2); } @Override @@ -237,7 +234,6 @@ ClusterAssignment performTaskAssignment( if (previousRevocation.connectors().stream().anyMatch(c -> activeAssignments.connectors().contains(c)) || previousRevocation.tasks().stream().anyMatch(t -> activeAssignments.tasks().contains(t))) { previousAssignment = activeAssignments; - canRevoke = true; } previousRevocation.connectors().clear(); previousRevocation.tasks().clear(); @@ -282,6 +278,7 @@ ClusterAssignment performTaskAssignment( Map toRevoke = computeDeleted(deleted, connectorAssignments, taskAssignments); log.debug("Connector and task to delete assignments: {}", toRevoke); + // Revoking redundant connectors/tasks if the workers have duplicate assignments toRevoke.putAll(computeDuplicatedAssignments(memberAssignments, connectorAssignments, taskAssignments)); log.debug("Connector and task to revoke assignments (include duplicated assignments): {}", toRevoke); @@ -295,12 +292,9 @@ ClusterAssignment performTaskAssignment( handleLostAssignments(lostAssignments, newSubmissions, completeWorkerAssignment); - // Compute the connectors-and-tasks to be revoked for load balancing without taking into - // account the deleted ones. - if (delay == 0 && canRevoke()) { - log.debug("Can leader revoke tasks in this assignment? {} (delay: {})", canRevoke, delay); - - + log.debug("Can leader revoke tasks in this assignment? {} (delay: {})", delay == 0, delay); + // Do not revoke resources for re-assignment while a delayed rebalance is active + if (delay == 0) { Map toExplicitlyRevoke = performTaskRevocation(configured, completeWorkerAssignment); @@ -313,6 +307,29 @@ ClusterAssignment performTaskAssignment( existing.tasks().addAll(assignment.tasks()); } ); + + // If this round and the previous round involved revocation, we will do an exponential + // backoff delay to prevent rebalance storms. + if (revokedInPrevious && !toExplicitlyRevoke.isEmpty()) { + numSuccessiveRevokingRebalances++; + long processAfter = consecutiveRevokingRebalancesBackoff.backoff(numSuccessiveRevokingRebalances); + log.debug("Consecutive revoking rebalances observed. Need to wait for {} ms", processAfter); + time.sleep(processAfter); // Is it a good idea to sleep? + } else if (!toExplicitlyRevoke.isEmpty()) { + // We had a revocation in this round but not in the previous round. Let's store that state. + log.trace("Revoking rebalance. Setting the revokedInPrevious flag to true"); + revokedInPrevious = true; + } else if (revokedInPrevious) { + // No revocations in this round but the previous round had one. Probably the workers + // have converged to a balanced load. We can reset the rebalance clock + log.debug("Previous round had revocations but this round didn't. Probably, the cluster has reached a " + + "balanced load. Resetting the exponential backoff clock"); + numSuccessiveRevokingRebalances = 0; + revokedInPrevious = false; + } else { + // revokedInPrevious is false and no revocations needed in this round. no-op. + log.trace("No revocations in previous and current round."); + } } else { log.debug("Connector and task to revoke assignments: {}", toRevoke); } @@ -351,31 +368,6 @@ ClusterAssignment performTaskAssignment( ); } - - private boolean canRevoke() { - // First worker joining without an active delay - // We can trigger a rebalance now. - if (revokingRebalanceScheduledAt == 0) { - long revokeAfter = nextSuccessiveRebalance.backoff(++numSuccessiveRevokingRebalances); - log.debug("revokeAfter:{}, numSuccessiveRevokingRebalances:{}", revokeAfter, numSuccessiveRevokingRebalances); - revokingRebalanceScheduledAt = time.milliseconds() + revokeAfter; - return true; - } - long now = time.milliseconds(); - // We can't rebalance until the next scheduled successive revoking rebalance - if (now < revokingRebalanceScheduledAt) { - log.debug("Can't revoke as current time {} is less than next scheduled rebalance {}", now, revokingRebalanceScheduledAt); - return false; - } - - // We crossed the last revoking rebalance time. Compute the next one by also - // including the current successful revoking rebalance. - long revokeAfter = nextSuccessiveRebalance.backoff(++numSuccessiveRevokingRebalances); - log.debug("revokeAfter:{}, numSuccessiveRevokingRebalances:{}", revokeAfter, numSuccessiveRevokingRebalances); - revokingRebalanceScheduledAt += revokeAfter; - return true; - } - private Map computeDeleted(ConnectorsAndTasks deleted, Map> connectorAssignments, Map> taskAssignments) { @@ -614,9 +606,18 @@ private Map performTaskRevocation(ConnectorsAndTasks Collection existingWorkers = completeWorkerAssignment.stream() .filter(wl -> wl.size() > 0) .collect(Collectors.toList()); - int existingWorkersNum = existingWorkers.size(); + + // existingWorkers => workers with load > 0. + // if existingWorkers is empty, no workers have any load. + int existingWorkersNum = existingWorkers.size(); // count of workers having non-zero load. int totalWorkersNum = completeWorkerAssignment.size(); - int newWorkersNum = totalWorkersNum - existingWorkersNum; + int newWorkersNum = totalWorkersNum - existingWorkersNum; // + // if existingWorkersNum == 0, no workers have any load i.e all workers are new. + // newWorkersNum = 0 when all workers have non-zero load. + + // if newWorkersNum > 0 && existingWorkersNum == 0, then none of the workers have any load. + // We don't revoke in that situation i.e we revoke only when newWorkersNum > 0 and there are existing workers with 0 load. + // Those workers would get assigned the tasks I guess. if (log.isDebugEnabled()) { completeWorkerAssignment.forEach(wl -> log.debug( diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java index 10258cb2830ed..cadc2a723a599 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java @@ -30,7 +30,6 @@ import org.junit.Test; import java.nio.ByteBuffer; -import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -51,6 +50,7 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; import static org.mockito.ArgumentMatchers.notNull; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; @@ -99,8 +99,6 @@ public void testTaskAssignmentWhenWorkerJoins() { assertBalancedAndCompleteAllocation(); // Second assignment with a second worker joining and all connectors running on previous worker - // Adding a sleep as otherwise the revocations mayn't happen due to exponential backoff - time.sleep(200); addNewEmptyWorkers("worker2"); performStandardRebalance(); assertDelay(0); @@ -122,7 +120,7 @@ public void testTaskAssignmentWhenWorkerJoins() { } @Test - public void testAssignmentsWhenWorkerJoinsAfterReovcations() { + public void testAssignmentsWhenWorkerJoinsAfterRevocations() { // First assignment with 1 worker and 2 connectors configured but not yet assigned performStandardRebalance(); @@ -131,9 +129,9 @@ public void testAssignmentsWhenWorkerJoinsAfterReovcations() { assertConnectorAllocations(2); assertTaskAllocations(8); assertBalancedAndCompleteAllocation(); + // Flag should not be set + assertFalse(assignor.revokedInPrevious); - // Wait for few milliseconds to pass the next scheduled rebalance delay - time.sleep(Duration.ofMillis(200).toMillis()); // Second assignment with a second worker joining and all connectors running on previous worker // We should revoke. addNewEmptyWorkers("worker2"); @@ -142,18 +140,19 @@ public void testAssignmentsWhenWorkerJoinsAfterReovcations() { assertWorkers("worker1", "worker2"); assertConnectorAllocations(0, 1); assertTaskAllocations(0, 4); + // Flag should still be set + assertTrue(assignor.revokedInPrevious); // Third assignment immediately after revocations, and a third worker joining - // This should not lead to a revocation due to the exponential back-off retry addNewEmptyWorkers("worker3"); performStandardRebalance(); assertDelay(0); assertWorkers("worker1", "worker2", "worker3"); assertConnectorAllocations(0, 1, 1); - assertTaskAllocations(2, 2, 4); + assertTaskAllocations(2, 2, 3); + // Flag should still be set + assertTrue(assignor.revokedInPrevious); - // Wait some more time before adding a new worker. - time.sleep(Duration.ofSeconds(5).toMillis()); // Fourth assignment after revocations, and a fourth worker joining // We should have a revocation in this case. addNewEmptyWorkers("worker4"); @@ -161,7 +160,28 @@ public void testAssignmentsWhenWorkerJoinsAfterReovcations() { assertDelay(0); assertWorkers("worker1", "worker2", "worker3", "worker4"); assertConnectorAllocations(0, 0, 1, 1); - assertTaskAllocations(0, 2, 2, 2); + assertTaskAllocations(1, 2, 2, 2); + // Flag should still be set + assertTrue(assignor.revokedInPrevious); + + // Subsequent assignment. This should stabilise the group and reset the exponential backoff related flags + performStandardRebalance(); + assertDelay(0); + assertConnectorAllocations(0, 0, 1, 1); + assertTaskAllocations(2, 2, 2, 2); + assertBalancedAndCompleteAllocation(); + // Revoked in previous flag should be unset + assertFalse(assignor.revokedInPrevious); + + // Add another worker. No revocations seen + addNewEmptyWorkers("worker5"); + performStandardRebalance(); + assertDelay(0); + assertWorkers("worker1", "worker2", "worker3", "worker4", "worker5"); + assertConnectorAllocations(0, 0, 0, 1, 1); + assertTaskAllocations(0, 2, 2, 2, 2); + // Flag should still be unset + assertFalse(assignor.revokedInPrevious); } @@ -328,10 +348,7 @@ public void testTaskAssignmentWhenLeaderBounces() { // Third assignment with the previous leader returning as a follower. In this case, the // arrival of the previous leader is treated as an arrival of a new worker. Reassignment - // happens immediately, first with a revocation. - // Since the new leader may join within the next scheduled revoking rebalance, there may not be any revocations - // so adding a sleep. - time.sleep(300); + // happens immediately, first with a revocation addNewEmptyWorkers("worker1"); performStandardRebalance(); assertDelay(0); @@ -399,8 +416,6 @@ public void testTaskAssignmentWhenSubsequentAssignmentAttemptFails() { // Third assignment happens with members returning the same assignments (memberConfigs) // as the first time. - // wait for a few milliseconds before rebalancing to let revocations happen - time.sleep(250); performStandardRebalance(); assertDelay(0); assertConnectorAllocations(0, 1, 1); @@ -442,8 +457,6 @@ public void testTaskAssignmentWhenSubsequentAssignmentAttemptFailsOutsideTheAssi // Third assignment happens with members returning the same assignments (memberConfigs) // as the first time. - // Waiting for a few milliseconds before triggering a rebalance to let revocations happen - time.sleep(250); performRebalanceWithMismatchedGeneration(); assertDelay(0); assertConnectorAllocations(0, 1, 1); @@ -894,11 +907,11 @@ public void testTaskAssignmentWhenTasksDuplicatedInWorkerAssignment() { assertTaskAllocations(0, 4); // Third assignment after revocations - // Assignments should be balanced by this round performStandardRebalance(); assertDelay(0); assertConnectorAllocations(1, 1); assertTaskAllocations(4, 4); + assertBalancedAndCompleteAllocation(); // Fifth rebalance should not change assignments performStandardRebalance(); @@ -920,9 +933,6 @@ public void testDuplicatedAssignmentHandleWhenTheDuplicatedAssignmentsDeleted() removeConnector("connector1"); // Second assignment with a second worker with duplicate assignment joining and the duplicated assignment is deleted at the same time - // The tasks should get rebalanced - // Adding a sleep of few milliseconds to let the rebalance happen - time.sleep(250); addNewWorker("worker2", newConnectors(1, 2), newTasks("connector1", 0, 4)); performStandardRebalance(); assertDelay(0); @@ -930,14 +940,14 @@ public void testDuplicatedAssignmentHandleWhenTheDuplicatedAssignmentsDeleted() assertConnectorAllocations(0, 1); assertTaskAllocations(0, 2); - // Third rebalance after revocations + // Third assignment after revocations performStandardRebalance(); assertDelay(0); assertConnectorAllocations(0, 1); assertTaskAllocations(2, 2); assertBalancedAndCompleteAllocation(); - // Fourth rebalance should not change assignments + // Fifth rebalance should not change assignments performStandardRebalance(); assertDelay(0); assertEmptyAssignment(); From a72a83e23c98f1b4a21ed1e69058f70b3094b6f7 Mon Sep 17 00:00:00 2001 From: Sagar Rao Date: Wed, 31 Aug 2022 18:01:56 +0530 Subject: [PATCH 05/26] Removing time.sleep during successive rebalances and instead using the existing scheduled rebalanceDelay mechanism to introduce exponential backoffs during successive revoking rebalances --- .../IncrementalCooperativeAssignor.java | 30 +++---- .../IncrementalCooperativeAssignorTest.java | 89 ++++++++++++------- 2 files changed, 68 insertions(+), 51 deletions(-) diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java index 21096eb5a82b8..4be5ab72c2e1b 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java @@ -292,7 +292,9 @@ ClusterAssignment performTaskAssignment( handleLostAssignments(lostAssignments, newSubmissions, completeWorkerAssignment); - log.debug("Can leader revoke tasks in this assignment? {} (delay: {})", delay == 0, delay); + if (delay > 0) { + log.debug("Delaying {}ms for revoking tasks.", delay); + } // Do not revoke resources for re-assignment while a delayed rebalance is active if (delay == 0) { Map toExplicitlyRevoke = @@ -312,12 +314,12 @@ ClusterAssignment performTaskAssignment( // backoff delay to prevent rebalance storms. if (revokedInPrevious && !toExplicitlyRevoke.isEmpty()) { numSuccessiveRevokingRebalances++; - long processAfter = consecutiveRevokingRebalancesBackoff.backoff(numSuccessiveRevokingRebalances); - log.debug("Consecutive revoking rebalances observed. Need to wait for {} ms", processAfter); - time.sleep(processAfter); // Is it a good idea to sleep? + delay = (int) consecutiveRevokingRebalancesBackoff.backoff(numSuccessiveRevokingRebalances); + log.debug("Consecutive revoking rebalances observed. Need to wait for {} ms", delay); + scheduledRebalance = time.milliseconds() + delay; } else if (!toExplicitlyRevoke.isEmpty()) { // We had a revocation in this round but not in the previous round. Let's store that state. - log.trace("Revoking rebalance. Setting the revokedInPrevious flag to true"); + log.debug("Revoking rebalance. Setting the revokedInPrevious flag to true"); revokedInPrevious = true; } else if (revokedInPrevious) { // No revocations in this round but the previous round had one. Probably the workers @@ -328,7 +330,7 @@ ClusterAssignment performTaskAssignment( revokedInPrevious = false; } else { // revokedInPrevious is false and no revocations needed in this round. no-op. - log.trace("No revocations in previous and current round."); + log.debug("No revocations in previous and current round."); } } else { log.debug("Connector and task to revoke assignments: {}", toRevoke); @@ -483,7 +485,8 @@ private Map computeDuplicatedAssignments(Map completeWorkerAssignment) { - if (lostAssignments.isEmpty()) { + // There are no lost assignments and there have been no successive revoking rebalances + if (lostAssignments.isEmpty() && numSuccessiveRevokingRebalances == 0) { resetDelay(); return; } @@ -606,18 +609,9 @@ private Map performTaskRevocation(ConnectorsAndTasks Collection existingWorkers = completeWorkerAssignment.stream() .filter(wl -> wl.size() > 0) .collect(Collectors.toList()); - - // existingWorkers => workers with load > 0. - // if existingWorkers is empty, no workers have any load. - int existingWorkersNum = existingWorkers.size(); // count of workers having non-zero load. + int existingWorkersNum = existingWorkers.size(); int totalWorkersNum = completeWorkerAssignment.size(); - int newWorkersNum = totalWorkersNum - existingWorkersNum; // - // if existingWorkersNum == 0, no workers have any load i.e all workers are new. - // newWorkersNum = 0 when all workers have non-zero load. - - // if newWorkersNum > 0 && existingWorkersNum == 0, then none of the workers have any load. - // We don't revoke in that situation i.e we revoke only when newWorkersNum > 0 and there are existing workers with 0 load. - // Those workers would get assigned the tasks I guess. + int newWorkersNum = totalWorkersNum - existingWorkersNum; if (log.isDebugEnabled()) { completeWorkerAssignment.forEach(wl -> log.debug( diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java index cadc2a723a599..0038a92690af8 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java @@ -120,14 +120,15 @@ public void testTaskAssignmentWhenWorkerJoins() { } @Test - public void testAssignmentsWhenWorkerJoinsAfterRevocations() { + public void testAssignmentsWhenWorkerJoinsAfterRevocations() throws InterruptedException { // First assignment with 1 worker and 2 connectors configured but not yet assigned + addNewConnector("connector3", 4); performStandardRebalance(); assertDelay(0); assertWorkers("worker1"); - assertConnectorAllocations(2); - assertTaskAllocations(8); + assertConnectorAllocations(3); + assertTaskAllocations(12); assertBalancedAndCompleteAllocation(); // Flag should not be set assertFalse(assignor.revokedInPrevious); @@ -138,50 +139,70 @@ public void testAssignmentsWhenWorkerJoinsAfterRevocations() { performStandardRebalance(); assertDelay(0); assertWorkers("worker1", "worker2"); - assertConnectorAllocations(0, 1); - assertTaskAllocations(0, 4); + assertConnectorAllocations(0, 2); + assertTaskAllocations(0, 6); // Flag should still be set assertTrue(assignor.revokedInPrevious); - // Third assignment immediately after revocations, and a third worker joining + // Third assignment immediately after revocations, and a third worker joining. + // This is a successive revoking rebalance. The assignments should be balanced + // at the end of this round but this should be the start of an exponential backoff + // for any successive rebalance delays. addNewEmptyWorkers("worker3"); performStandardRebalance(); - assertDelay(0); + assertTrue(assignor.delay > 0); assertWorkers("worker1", "worker2", "worker3"); assertConnectorAllocations(0, 1, 1); - assertTaskAllocations(2, 2, 3); + assertTaskAllocations(3, 3, 4); + assertBalancedAllocation(); // Flag should still be set assertTrue(assignor.revokedInPrevious); // Fourth assignment after revocations, and a fourth worker joining - // We should have a revocation in this case. + // Since the worker is joining immediately and within the rebalance delay + // there should not be any revoking rebalance addNewEmptyWorkers("worker4"); performStandardRebalance(); - assertDelay(0); + assertTrue(assignor.delay > 0); assertWorkers("worker1", "worker2", "worker3", "worker4"); - assertConnectorAllocations(0, 0, 1, 1); - assertTaskAllocations(1, 2, 2, 2); + assertConnectorAllocations(0, 1, 1, 1); + assertTaskAllocations(2, 3, 3, 4); // Flag should still be set assertTrue(assignor.revokedInPrevious); - // Subsequent assignment. This should stabilise the group and reset the exponential backoff related flags - performStandardRebalance(); - assertDelay(0); - assertConnectorAllocations(0, 0, 1, 1); - assertTaskAllocations(2, 2, 2, 2); - assertBalancedAndCompleteAllocation(); - // Revoked in previous flag should be unset - assertFalse(assignor.revokedInPrevious); - - // Add another worker. No revocations seen + // Add new worker immediately. Since a scheduled rebalance is in progress, + // There should still not be be any revocations addNewEmptyWorkers("worker5"); performStandardRebalance(); - assertDelay(0); + assertTrue(assignor.delay > 0); assertWorkers("worker1", "worker2", "worker3", "worker4", "worker5"); - assertConnectorAllocations(0, 0, 0, 1, 1); - assertTaskAllocations(0, 2, 2, 2, 2); - // Flag should still be unset + assertConnectorAllocations(0, 0, 1, 1, 1); + assertTaskAllocations(0, 2, 3, 3, 4); + // Flag should still be set + assertTrue(assignor.revokedInPrevious); + + // Add new worker but this time after crossing the delay. + // There would be revocations allowed + time.sleep(assignor.delay); + addNewEmptyWorkers("worker6"); + performStandardRebalance(); + assertTrue(assignor.delay > 0); + assertWorkers("worker1", "worker2", "worker3", "worker4", "worker5", "worker6"); + assertConnectorAllocations(0, 0, 0, 1, 1, 1); + assertTaskAllocations(0, 0, 2, 2, 2, 2); + // Flag should be set + assertTrue(assignor.revokedInPrevious); + + // Another rebalance after crossing the scheduled rebalance delay. This is + // just to assert that we would get balanced and complete assignments. + // Rebalance clock should be reset as well + time.sleep(assignor.delay); + performStandardRebalance(); + assertWorkers("worker1", "worker2", "worker3", "worker4", "worker5", "worker6"); + assertConnectorAllocations(0, 0, 0, 1, 1, 1); + assertTaskAllocations(2, 2, 2, 2, 2, 2); assertFalse(assignor.revokedInPrevious); + assertBalancedAndCompleteAllocation(); } @@ -415,15 +436,16 @@ public void testTaskAssignmentWhenSubsequentAssignmentAttemptFails() { assertTaskAllocations(0, 4, 4); // Third assignment happens with members returning the same assignments (memberConfigs) - // as the first time. + // as the first time. Since this is a consecutive revoking rebalance, delay should be non-zero performStandardRebalance(); - assertDelay(0); + assertTrue(assignor.delay > 0); assertConnectorAllocations(0, 1, 1); assertTaskAllocations(0, 3, 3); - // Fourth assignment after revocations + // Fourth assignment after revocations. Since this is an immediate rebalance, the delay + // set due to successive revoking rebalances would still be active. performStandardRebalance(); - assertDelay(0); + assertTrue(assignor.delay > 0); assertConnectorAllocations(0, 1, 1); assertTaskAllocations(2, 3, 3); assertBalancedAndCompleteAllocation(); @@ -456,15 +478,16 @@ public void testTaskAssignmentWhenSubsequentAssignmentAttemptFailsOutsideTheAssi assertTaskAllocations(0, 4, 4); // Third assignment happens with members returning the same assignments (memberConfigs) - // as the first time. + // as the first time. Since this is a consecutive revoking rebalance, there should be delay performRebalanceWithMismatchedGeneration(); - assertDelay(0); + assertTrue(assignor.delay > 0); assertConnectorAllocations(0, 1, 1); assertTaskAllocations(0, 3, 3); // Fourth assignment after revocations performStandardRebalance(); - assertDelay(0); + // There's still an active rebalanced delay + assertTrue(assignor.delay > 0); assertConnectorAllocations(0, 1, 1); assertTaskAllocations(2, 3, 3); assertBalancedAndCompleteAllocation(); From a8a0c0c429dce4158447da5412b1b5285f7e4a81 Mon Sep 17 00:00:00 2001 From: vamossagar12 Date: Thu, 1 Sep 2022 20:18:52 +0530 Subject: [PATCH 06/26] Update connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java Co-authored-by: Chris Egerton --- .../runtime/distributed/IncrementalCooperativeAssignor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java index 4be5ab72c2e1b..14abe4e19c8c7 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java @@ -319,7 +319,7 @@ ClusterAssignment performTaskAssignment( scheduledRebalance = time.milliseconds() + delay; } else if (!toExplicitlyRevoke.isEmpty()) { // We had a revocation in this round but not in the previous round. Let's store that state. - log.debug("Revoking rebalance. Setting the revokedInPrevious flag to true"); + log.debug("Performing allocation-balancing revocation immediately as no revocations took place during the previous rebalance"); revokedInPrevious = true; } else if (revokedInPrevious) { // No revocations in this round but the previous round had one. Probably the workers From 39ab0046f5dff714008ffabc73bf781e34bcf558 Mon Sep 17 00:00:00 2001 From: Sagar Rao Date: Fri, 2 Sep 2022 17:27:59 +0530 Subject: [PATCH 07/26] Review changes --- .../IncrementalCooperativeAssignor.java | 35 ++++++++++++------- .../IncrementalCooperativeAssignorTest.java | 4 --- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java index 14abe4e19c8c7..9563486b02c1f 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java @@ -91,7 +91,7 @@ public IncrementalCooperativeAssignor(LogContext logContext, Time time, int maxD this.previousGenerationId = -1; this.previousMembers = Collections.emptySet(); this.numSuccessiveRevokingRebalances = 0; - this.consecutiveRevokingRebalancesBackoff = new ExponentialBackoff(10, 20, maxDelay, 0.2); + this.consecutiveRevokingRebalancesBackoff = new ExponentialBackoff(10, 20, maxDelay, 0); } @Override @@ -300,26 +300,23 @@ ClusterAssignment performTaskAssignment( Map toExplicitlyRevoke = performTaskRevocation(configured, completeWorkerAssignment); - toExplicitlyRevoke.forEach( - (worker, assignment) -> { - ConnectorsAndTasks existing = toRevoke.computeIfAbsent( - worker, - v -> new ConnectorsAndTasks.Builder().build()); - existing.connectors().addAll(assignment.connectors()); - existing.tasks().addAll(assignment.tasks()); - } - ); - // If this round and the previous round involved revocation, we will do an exponential // backoff delay to prevent rebalance storms. if (revokedInPrevious && !toExplicitlyRevoke.isEmpty()) { numSuccessiveRevokingRebalances++; + log.debug("Consecutive revoking rebalances observed. Computing exponential backoff delay."); delay = (int) consecutiveRevokingRebalancesBackoff.backoff(numSuccessiveRevokingRebalances); - log.debug("Consecutive revoking rebalances observed. Need to wait for {} ms", delay); - scheduledRebalance = time.milliseconds() + delay; + if (delay != 0) { + log.debug("Skipping revocations in the current round with a delay of {}ms", delay); + scheduledRebalance = time.milliseconds() + delay; + } else { + log.debug("Revoking the assignments straight-away as the exponential backoff delay is 0ms"); + revoke(toRevoke, toExplicitlyRevoke); + } } else if (!toExplicitlyRevoke.isEmpty()) { // We had a revocation in this round but not in the previous round. Let's store that state. log.debug("Performing allocation-balancing revocation immediately as no revocations took place during the previous rebalance"); + revoke(toRevoke, toExplicitlyRevoke); revokedInPrevious = true; } else if (revokedInPrevious) { // No revocations in this round but the previous round had one. Probably the workers @@ -370,6 +367,18 @@ ClusterAssignment performTaskAssignment( ); } + private void revoke(Map toRevoke, Map toExplicitlyRevoke) { + toExplicitlyRevoke.forEach( + (worker, assignment) -> { + ConnectorsAndTasks existing = toRevoke.computeIfAbsent( + worker, + v -> new ConnectorsAndTasks.Builder().build()); + existing.connectors().addAll(assignment.connectors()); + existing.tasks().addAll(assignment.tasks()); + } + ); + } + private Map computeDeleted(ConnectorsAndTasks deleted, Map> connectorAssignments, Map> taskAssignments) { diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java index 0038a92690af8..543af9a6e8225 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java @@ -130,8 +130,6 @@ public void testAssignmentsWhenWorkerJoinsAfterRevocations() throws InterruptedE assertConnectorAllocations(3); assertTaskAllocations(12); assertBalancedAndCompleteAllocation(); - // Flag should not be set - assertFalse(assignor.revokedInPrevious); // Second assignment with a second worker joining and all connectors running on previous worker // We should revoke. @@ -141,8 +139,6 @@ public void testAssignmentsWhenWorkerJoinsAfterRevocations() throws InterruptedE assertWorkers("worker1", "worker2"); assertConnectorAllocations(0, 2); assertTaskAllocations(0, 6); - // Flag should still be set - assertTrue(assignor.revokedInPrevious); // Third assignment immediately after revocations, and a third worker joining. // This is a successive revoking rebalance. The assignments should be balanced From a1da57d3e8773f75620828cd79594d2058589cdd Mon Sep 17 00:00:00 2001 From: Sagar Rao Date: Sun, 4 Sep 2022 15:50:10 +0530 Subject: [PATCH 08/26] Setting initial interval to 1 and lowering multiplier. Also always setting delay to 0 to maxDelay is 0. --- .../IncrementalCooperativeAssignor.java | 32 ++-- .../IncrementalCooperativeAssignorTest.java | 151 +++++++++++++----- 2 files changed, 130 insertions(+), 53 deletions(-) diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java index 9563486b02c1f..2fcf0cd088c4b 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java @@ -66,8 +66,7 @@ public class IncrementalCooperativeAssignor implements ConnectAssignor { private final int maxDelay; private ConnectorsAndTasks previousAssignment; private final ConnectorsAndTasks previousRevocation; - // visible for testing - boolean revokedInPrevious; + private boolean revokedInPrevious; protected final Set candidateWorkersForReassignment; protected long scheduledRebalance; protected int delay; @@ -91,7 +90,10 @@ public IncrementalCooperativeAssignor(LogContext logContext, Time time, int maxD this.previousGenerationId = -1; this.previousMembers = Collections.emptySet(); this.numSuccessiveRevokingRebalances = 0; - this.consecutiveRevokingRebalancesBackoff = new ExponentialBackoff(10, 20, maxDelay, 0); + // By default, initial interval is 1. The only corner case is when the user has set maxDelay to 0 + // in which case, the exponential backoff delay should be 0. We just set the initialInterval parameter here + // but it won't be used to compute delays. + this.consecutiveRevokingRebalancesBackoff = new ExponentialBackoff(maxDelay == 0 ? 0 : 1, 30, maxDelay, 0); } @Override @@ -292,25 +294,24 @@ ClusterAssignment performTaskAssignment( handleLostAssignments(lostAssignments, newSubmissions, completeWorkerAssignment); - if (delay > 0) { - log.debug("Delaying {}ms for revoking tasks.", delay); - } // Do not revoke resources for re-assignment while a delayed rebalance is active if (delay == 0) { Map toExplicitlyRevoke = performTaskRevocation(configured, completeWorkerAssignment); - // If this round and the previous round involved revocation, we will do an exponential - // backoff delay to prevent rebalance storms. + // If this round and the previous round involved revocation, we will calculate a delay for + // the next round when revoking rebalance would be allowed. Note that delay could be 0, in which + // case we would always revoke. if (revokedInPrevious && !toExplicitlyRevoke.isEmpty()) { numSuccessiveRevokingRebalances++; - log.debug("Consecutive revoking rebalances observed. Computing exponential backoff delay."); - delay = (int) consecutiveRevokingRebalancesBackoff.backoff(numSuccessiveRevokingRebalances); + log.debug("Consecutive revoking rebalances observed. Computing delay and next scheduled rebalance."); + delay = (maxDelay == 0) ? 0 : (int) consecutiveRevokingRebalancesBackoff.backoff(numSuccessiveRevokingRebalances); if (delay != 0) { - log.debug("Skipping revocations in the current round with a delay of {}ms", delay); scheduledRebalance = time.milliseconds() + delay; + log.debug("Skipping revocations in the current round with a delay of {}ms. Next scheduled rebalance:{}", + delay, scheduledRebalance); } else { - log.debug("Revoking the assignments straight-away as the exponential backoff delay is 0ms"); + log.debug("Revoking assignments as scheduled.rebalance.max.delay.ms is set to 0"); revoke(toRevoke, toExplicitlyRevoke); } } else if (!toExplicitlyRevoke.isEmpty()) { @@ -326,11 +327,11 @@ ClusterAssignment performTaskAssignment( numSuccessiveRevokingRebalances = 0; revokedInPrevious = false; } else { - // revokedInPrevious is false and no revocations needed in this round. no-op. + // no-op log.debug("No revocations in previous and current round."); } } else { - log.debug("Connector and task to revoke assignments: {}", toRevoke); + log.debug("Delayed rebalance is active. Delaying {}ms before revoking connectors and tasks: {}", delay, toRevoke); } assignConnectors(completeWorkerAssignment, newSubmissions.connectors()); @@ -554,6 +555,9 @@ protected void handleLostAssignments(ConnectorsAndTasks lostAssignments, newSubmissions.tasks().addAll(lostAssignments.tasks()); } resetDelay(); + // Resetting the flag as now we can permit successive revoking rebalances. + // since we have gone through the full rebalance delay + revokedInPrevious = false; } else { candidateWorkersForReassignment .addAll(candidateWorkersForReassignment(completeWorkerAssignment)); diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java index 543af9a6e8225..ea6e3de964055 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java @@ -50,7 +50,6 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; -import static org.junit.Assert.assertFalse; import static org.mockito.ArgumentMatchers.notNull; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; @@ -120,10 +119,10 @@ public void testTaskAssignmentWhenWorkerJoins() { } @Test - public void testAssignmentsWhenWorkerJoinsAfterRevocations() throws InterruptedException { + public void testAssignmentsWhenWorkersJoinAfterRevocations() { - // First assignment with 1 worker and 2 connectors configured but not yet assigned addNewConnector("connector3", 4); + // First assignment with 1 worker and 2 connectors configured but not yet assigned performStandardRebalance(); assertDelay(0); assertWorkers("worker1"); @@ -135,36 +134,28 @@ public void testAssignmentsWhenWorkerJoinsAfterRevocations() throws InterruptedE // We should revoke. addNewEmptyWorkers("worker2"); performStandardRebalance(); - assertDelay(0); assertWorkers("worker1", "worker2"); assertConnectorAllocations(0, 2); assertTaskAllocations(0, 6); // Third assignment immediately after revocations, and a third worker joining. - // This is a successive revoking rebalance. The assignments should be balanced - // at the end of this round but this should be the start of an exponential backoff - // for any successive rebalance delays. + // This is a successive revoking rebalance. We should not perform any revocations + // in this round addNewEmptyWorkers("worker3"); performStandardRebalance(); assertTrue(assignor.delay > 0); assertWorkers("worker1", "worker2", "worker3"); - assertConnectorAllocations(0, 1, 1); - assertTaskAllocations(3, 3, 4); - assertBalancedAllocation(); - // Flag should still be set - assertTrue(assignor.revokedInPrevious); + assertConnectorAllocations(0, 1, 2); + assertTaskAllocations(3, 3, 6); - // Fourth assignment after revocations, and a fourth worker joining + // Fourth assignment and a fourth worker joining // Since the worker is joining immediately and within the rebalance delay // there should not be any revoking rebalance addNewEmptyWorkers("worker4"); performStandardRebalance(); - assertTrue(assignor.delay > 0); assertWorkers("worker1", "worker2", "worker3", "worker4"); - assertConnectorAllocations(0, 1, 1, 1); - assertTaskAllocations(2, 3, 3, 4); - // Flag should still be set - assertTrue(assignor.revokedInPrevious); + assertConnectorAllocations(0, 0, 1, 2); + assertTaskAllocations(0, 3, 3, 6); // Add new worker immediately. Since a scheduled rebalance is in progress, // There should still not be be any revocations @@ -172,34 +163,105 @@ public void testAssignmentsWhenWorkerJoinsAfterRevocations() throws InterruptedE performStandardRebalance(); assertTrue(assignor.delay > 0); assertWorkers("worker1", "worker2", "worker3", "worker4", "worker5"); - assertConnectorAllocations(0, 0, 1, 1, 1); - assertTaskAllocations(0, 2, 3, 3, 4); - // Flag should still be set - assertTrue(assignor.revokedInPrevious); + assertConnectorAllocations(0, 0, 0, 1, 2); + assertTaskAllocations(0, 0, 3, 3, 6); // Add new worker but this time after crossing the delay. // There would be revocations allowed time.sleep(assignor.delay); addNewEmptyWorkers("worker6"); performStandardRebalance(); - assertTrue(assignor.delay > 0); + assertDelay(0); assertWorkers("worker1", "worker2", "worker3", "worker4", "worker5", "worker6"); - assertConnectorAllocations(0, 0, 0, 1, 1, 1); - assertTaskAllocations(0, 0, 2, 2, 2, 2); - // Flag should be set - assertTrue(assignor.revokedInPrevious); + assertConnectorAllocations(0, 0, 0, 0, 1, 1); + assertTaskAllocations(0, 0, 0, 2, 2, 2); - // Another rebalance after crossing the scheduled rebalance delay. This is - // just to assert that we would get balanced and complete assignments. - // Rebalance clock should be reset as well - time.sleep(assignor.delay); + // Follow up rebalance since there were revocations performStandardRebalance(); assertWorkers("worker1", "worker2", "worker3", "worker4", "worker5", "worker6"); assertConnectorAllocations(0, 0, 0, 1, 1, 1); assertTaskAllocations(2, 2, 2, 2, 2, 2); - assertFalse(assignor.revokedInPrevious); assertBalancedAndCompleteAllocation(); + } + + @Test + public void testImmediateRevocationsWhenMaxDelayIs0() { + + // Customize assignor for this test case + rebalanceDelay = 0; + initAssignor(); + + addNewConnector("connector3", 4); + // First assignment with 1 worker and 2 connectors configured but not yet assigned + performStandardRebalance(); + assertDelay(0); + assertWorkers("worker1"); + assertConnectorAllocations(3); + assertTaskAllocations(12); + assertBalancedAndCompleteAllocation(); + + // Second assignment with a second worker joining and all connectors running on previous worker + // We should revoke. + addNewEmptyWorkers("worker2"); + performStandardRebalance(); + assertWorkers("worker1", "worker2"); + assertConnectorAllocations(0, 2); + assertTaskAllocations(0, 6); + // Third assignment immediately after revocations, and a third worker joining. + // This is a successive revoking rebalance but we should still revoke as rebalance delay is 0 + addNewEmptyWorkers("worker3"); + performStandardRebalance(); + assertDelay(0); + assertWorkers("worker1", "worker2", "worker3"); + assertConnectorAllocations(0, 1, 1); + assertTaskAllocations(3, 3, 4); + + // Follow up rebalance post revocations + performStandardRebalance(); + assertWorkers("worker1", "worker2", "worker3"); + assertConnectorAllocations(1, 1, 1); + assertTaskAllocations(4, 4, 4); + assertBalancedAndCompleteAllocation(); + } + + @Test + public void testSuccessiveRevocationsWhenMaxDelayIsEqualToExpBackOffInitialInterval() { + + rebalanceDelay = 1; + initAssignor(); + + addNewConnector("connector3", 4); + // First assignment with 1 worker and 2 connectors configured but not yet assigned + performStandardRebalance(); + assertDelay(0); + assertWorkers("worker1"); + assertConnectorAllocations(3); + assertTaskAllocations(12); + assertBalancedAndCompleteAllocation(); + + // Second assignment with a second worker joining and all connectors running on previous worker + // We should revoke. + addNewEmptyWorkers("worker2"); + performStandardRebalance(); + assertWorkers("worker1", "worker2"); + assertConnectorAllocations(0, 2); + assertTaskAllocations(0, 6); + + // Third assignment immediately after revocations, and a third worker joining. + // This is a successive revoking rebalance. We shouldn't revoke as maxDelay is 1 ms + addNewEmptyWorkers("worker3"); + performStandardRebalance(); + assertDelay(1); + assertWorkers("worker1", "worker2", "worker3"); + assertConnectorAllocations(0, 1, 2); + assertTaskAllocations(3, 3, 6); + + // Follow up rebalance post revocations. No revocations should have happened + performStandardRebalance(); + assertWorkers("worker1", "worker2", "worker3"); + assertConnectorAllocations(0, 1, 2); + assertTaskAllocations(3, 3, 6); } @Test @@ -433,18 +495,24 @@ public void testTaskAssignmentWhenSubsequentAssignmentAttemptFails() { // Third assignment happens with members returning the same assignments (memberConfigs) // as the first time. Since this is a consecutive revoking rebalance, delay should be non-zero + // but no revoking rebalances performStandardRebalance(); assertTrue(assignor.delay > 0); assertConnectorAllocations(0, 1, 1); + assertTaskAllocations(0, 4, 4); + + // Wait for delay ms before triggering another rebalance + time.sleep(assignor.delay); + // Fourth assignment after revocations. + performStandardRebalance(); + assertDelay(0); + assertConnectorAllocations(0, 1, 1); assertTaskAllocations(0, 3, 3); - // Fourth assignment after revocations. Since this is an immediate rebalance, the delay - // set due to successive revoking rebalances would still be active. + // Follow up rebalance due to revocations performStandardRebalance(); - assertTrue(assignor.delay > 0); assertConnectorAllocations(0, 1, 1); assertTaskAllocations(2, 3, 3); - assertBalancedAndCompleteAllocation(); } @Test @@ -475,15 +543,20 @@ public void testTaskAssignmentWhenSubsequentAssignmentAttemptFailsOutsideTheAssi // Third assignment happens with members returning the same assignments (memberConfigs) // as the first time. Since this is a consecutive revoking rebalance, there should be delay + // not leading to revocations. performRebalanceWithMismatchedGeneration(); assertTrue(assignor.delay > 0); assertConnectorAllocations(0, 1, 1); - assertTaskAllocations(0, 3, 3); + assertTaskAllocations(0, 4, 4); + // Wait for delay to get revoking rebalances + time.sleep(assignor.delay); // Fourth assignment after revocations performStandardRebalance(); - // There's still an active rebalanced delay - assertTrue(assignor.delay > 0); + assertConnectorAllocations(0, 1, 1); + assertTaskAllocations(0, 3, 3); + + performStandardRebalance(); assertConnectorAllocations(0, 1, 1); assertTaskAllocations(2, 3, 3); assertBalancedAndCompleteAllocation(); From 3d6155002b337ed76f4903ff468edad0c3864443 Mon Sep 17 00:00:00 2001 From: Sagar Rao Date: Mon, 5 Sep 2022 16:28:28 +0530 Subject: [PATCH 09/26] Removing needless check while computing exponential backoff delay --- .../runtime/distributed/IncrementalCooperativeAssignor.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java index 2fcf0cd088c4b..f2a5524c85820 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java @@ -91,8 +91,7 @@ public IncrementalCooperativeAssignor(LogContext logContext, Time time, int maxD this.previousMembers = Collections.emptySet(); this.numSuccessiveRevokingRebalances = 0; // By default, initial interval is 1. The only corner case is when the user has set maxDelay to 0 - // in which case, the exponential backoff delay should be 0. We just set the initialInterval parameter here - // but it won't be used to compute delays. + // in which case, the exponential backoff delay should be 0 which would return the backoff delay to be 0 always this.consecutiveRevokingRebalancesBackoff = new ExponentialBackoff(maxDelay == 0 ? 0 : 1, 30, maxDelay, 0); } @@ -305,7 +304,7 @@ ClusterAssignment performTaskAssignment( if (revokedInPrevious && !toExplicitlyRevoke.isEmpty()) { numSuccessiveRevokingRebalances++; log.debug("Consecutive revoking rebalances observed. Computing delay and next scheduled rebalance."); - delay = (maxDelay == 0) ? 0 : (int) consecutiveRevokingRebalancesBackoff.backoff(numSuccessiveRevokingRebalances); + delay = (int) consecutiveRevokingRebalancesBackoff.backoff(numSuccessiveRevokingRebalances); if (delay != 0) { scheduledRebalance = time.milliseconds() + delay; log.debug("Skipping revocations in the current round with a delay of {}ms. Next scheduled rebalance:{}", From e42092b091b387d3f2a6ab9f4ba47a2c035dc338 Mon Sep 17 00:00:00 2001 From: Sagar Rao Date: Tue, 6 Sep 2022 20:08:59 +0530 Subject: [PATCH 10/26] Reenabling flaky test testMultipleWorkersRejoining and addressing review comments --- .../distributed/IncrementalCooperativeAssignor.java | 2 +- .../RebalanceSourceConnectorsIntegrationTest.java | 1 - .../IncrementalCooperativeAssignorTest.java | 10 +++++----- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java index f2a5524c85820..bd6a1a1d5b042 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java @@ -310,7 +310,7 @@ ClusterAssignment performTaskAssignment( log.debug("Skipping revocations in the current round with a delay of {}ms. Next scheduled rebalance:{}", delay, scheduledRebalance); } else { - log.debug("Revoking assignments as scheduled.rebalance.max.delay.ms is set to 0"); + log.debug("Revoking assignments immediately since scheduled.rebalance.max.delay.ms is set to 0"); revoke(toRevoke, toExplicitlyRevoke); } } else if (!toExplicitlyRevoke.isEmpty()) { diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/RebalanceSourceConnectorsIntegrationTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/RebalanceSourceConnectorsIntegrationTest.java index 855882c9e697b..2aaeae7237cda 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/RebalanceSourceConnectorsIntegrationTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/RebalanceSourceConnectorsIntegrationTest.java @@ -270,7 +270,6 @@ public void testRemovingWorker() throws Exception { WORKER_SETUP_DURATION_MS, "Connect and tasks are imbalanced between the workers."); } - @Ignore // TODO: To be re-enabled once we can make it less flaky (KAFKA-12495, KAFKA-12283) @Test public void testMultipleWorkersRejoining() throws Exception { // create test topic diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java index ea6e3de964055..c8feb43cd96e4 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java @@ -122,7 +122,7 @@ public void testTaskAssignmentWhenWorkerJoins() { public void testAssignmentsWhenWorkersJoinAfterRevocations() { addNewConnector("connector3", 4); - // First assignment with 1 worker and 2 connectors configured but not yet assigned + // First assignment with 1 worker and 3 connectors configured but not yet assigned performStandardRebalance(); assertDelay(0); assertWorkers("worker1"); @@ -192,7 +192,7 @@ public void testImmediateRevocationsWhenMaxDelayIs0() { initAssignor(); addNewConnector("connector3", 4); - // First assignment with 1 worker and 2 connectors configured but not yet assigned + // First assignment with 1 worker and 3 connectors configured but not yet assigned performStandardRebalance(); assertDelay(0); assertWorkers("worker1"); @@ -232,7 +232,7 @@ public void testSuccessiveRevocationsWhenMaxDelayIsEqualToExpBackOffInitialInter initAssignor(); addNewConnector("connector3", 4); - // First assignment with 1 worker and 2 connectors configured but not yet assigned + // First assignment with 1 worker and 3 connectors configured but not yet assigned performStandardRebalance(); assertDelay(0); assertWorkers("worker1"); @@ -1005,7 +1005,7 @@ public void testTaskAssignmentWhenTasksDuplicatedInWorkerAssignment() { assertTaskAllocations(4, 4); assertBalancedAndCompleteAllocation(); - // Fifth rebalance should not change assignments + // Fourth rebalance should not change assignments performStandardRebalance(); assertDelay(0); assertEmptyAssignment(); @@ -1039,7 +1039,7 @@ public void testDuplicatedAssignmentHandleWhenTheDuplicatedAssignmentsDeleted() assertTaskAllocations(2, 2); assertBalancedAndCompleteAllocation(); - // Fifth rebalance should not change assignments + // Fourth rebalance should not change assignments performStandardRebalance(); assertDelay(0); assertEmptyAssignment(); From 804374e20697cfeacbf1084327a659d5f315fae5 Mon Sep 17 00:00:00 2001 From: Sagar Rao Date: Wed, 7 Sep 2022 16:21:29 +0530 Subject: [PATCH 11/26] Removing unwanted rebalance check in testSuccessiveRevocationsWhenMaxDelayIsEqualToExpBackOffInitialInterval --- .../distributed/IncrementalCooperativeAssignorTest.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java index c8feb43cd96e4..020fa9b7bcbcb 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java @@ -256,12 +256,6 @@ public void testSuccessiveRevocationsWhenMaxDelayIsEqualToExpBackOffInitialInter assertWorkers("worker1", "worker2", "worker3"); assertConnectorAllocations(0, 1, 2); assertTaskAllocations(3, 3, 6); - - // Follow up rebalance post revocations. No revocations should have happened - performStandardRebalance(); - assertWorkers("worker1", "worker2", "worker3"); - assertConnectorAllocations(0, 1, 2); - assertTaskAllocations(3, 3, 6); } @Test From 6ef468315b246dfae6504e4cdc096fd125658950 Mon Sep 17 00:00:00 2001 From: Sagar Rao Date: Fri, 9 Sep 2022 22:12:16 +0530 Subject: [PATCH 12/26] Changing arguements of performTaskRevocation back to the old ones --- .../IncrementalCooperativeAssignor.java | 2 +- .../IncrementalCooperativeAssignorTest.java | 42 ++++++++++++++----- 2 files changed, 33 insertions(+), 11 deletions(-) diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java index bd6a1a1d5b042..d506d565ca34d 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java @@ -296,7 +296,7 @@ ClusterAssignment performTaskAssignment( // Do not revoke resources for re-assignment while a delayed rebalance is active if (delay == 0) { Map toExplicitlyRevoke = - performTaskRevocation(configured, completeWorkerAssignment); + performTaskRevocation(activeAssignments, currentWorkerAssignment); // If this round and the previous round involved revocation, we will calculate a delay for // the next round when revoking rebalance would be allowed. Note that delay could be 0, in which diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java index 020fa9b7bcbcb..0bf923d2b3534 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java @@ -107,12 +107,13 @@ public void testTaskAssignmentWhenWorkerJoins() { // Third assignment after revocations performStandardRebalance(); - assertDelay(0); + assertDelay(assignor.delay); // Sucessive revocation so delay should be non-zero assertConnectorAllocations(1, 1); assertTaskAllocations(4, 4); assertBalancedAndCompleteAllocation(); - // A fourth rebalance should not change assignments + time.sleep(assignor.delay); + // A fourth rebalance after delay should not change assignments performStandardRebalance(); assertDelay(0); assertEmptyAssignment(); @@ -215,7 +216,7 @@ public void testImmediateRevocationsWhenMaxDelayIs0() { assertDelay(0); assertWorkers("worker1", "worker2", "worker3"); assertConnectorAllocations(0, 1, 1); - assertTaskAllocations(3, 3, 4); + assertTaskAllocations(2, 3, 3); // Follow up rebalance post revocations performStandardRebalance(); @@ -344,8 +345,14 @@ public void testTaskAssignmentWhenWorkerBounces() { time.sleep(rebalanceDelay / 4); - // Fifth assignment with the same two workers. The delay has expired, so the lost - // assignments ought to be assigned to the worker that has appeared as returned. + // Fifth assignment with the same two workers. The delay has expired, so there + // should be revocations giving back the assignments to the reappearing worker + performStandardRebalance(); + assertDelay(0); + assertConnectorAllocations(1, 1); + assertTaskAllocations(2, 4); + + // Final follow up rebalance leading to balanced assignments performStandardRebalance(); assertDelay(0); assertConnectorAllocations(1, 1); @@ -431,7 +438,7 @@ public void testTaskAssignmentWhenLeaderBounces() { // Fourth assignment after revocations performStandardRebalance(); - assertDelay(0); + assertDelay(assignor.delay); // Successive revoking rebalance. Should introduce a delay assertConnectorAllocations(0, 1, 1); assertTaskAllocations(2, 3, 3); assertBalancedAndCompleteAllocation(); @@ -503,10 +510,11 @@ public void testTaskAssignmentWhenSubsequentAssignmentAttemptFails() { assertConnectorAllocations(0, 1, 1); assertTaskAllocations(0, 3, 3); - // Follow up rebalance due to revocations + // Fourth assignment after revocations performStandardRebalance(); assertConnectorAllocations(0, 1, 1); assertTaskAllocations(2, 3, 3); + assertBalancedAndCompleteAllocation(); } @Test @@ -550,6 +558,7 @@ public void testTaskAssignmentWhenSubsequentAssignmentAttemptFailsOutsideTheAssi assertConnectorAllocations(0, 1, 1); assertTaskAllocations(0, 3, 3); + // Fifth and final rebalance performStandardRebalance(); assertConnectorAllocations(0, 1, 1); assertTaskAllocations(2, 3, 3); @@ -996,10 +1005,16 @@ public void testTaskAssignmentWhenTasksDuplicatedInWorkerAssignment() { performStandardRebalance(); assertDelay(0); assertConnectorAllocations(1, 1); + assertTaskAllocations(2, 4); + + // fourth rebalance after revocations + performStandardRebalance(); + assertDelay(0); + assertConnectorAllocations(1, 1); assertTaskAllocations(4, 4); assertBalancedAndCompleteAllocation(); - // Fourth rebalance should not change assignments + // Fifth rebalance should not change assignments performStandardRebalance(); assertDelay(0); assertEmptyAssignment(); @@ -1024,16 +1039,23 @@ public void testDuplicatedAssignmentHandleWhenTheDuplicatedAssignmentsDeleted() assertDelay(0); assertWorkers("worker1", "worker2"); assertConnectorAllocations(0, 1); - assertTaskAllocations(0, 2); + assertTaskAllocations(0, 4); // Third assignment after revocations performStandardRebalance(); assertDelay(0); assertConnectorAllocations(0, 1); + assertTaskAllocations(0, 2); + + // fourth rebalance after revocations + performStandardRebalance(); + assertTrue(assignor.delay > 0); + assertConnectorAllocations(0, 1); assertTaskAllocations(2, 2); assertBalancedAndCompleteAllocation(); - // Fourth rebalance should not change assignments + time.sleep(assignor.delay); + // Fifth rebalance after delay should not change assignments performStandardRebalance(); assertDelay(0); assertEmptyAssignment(); From ebce0751781e5c92a48cb9af5b8b0d55ce3294d1 Mon Sep 17 00:00:00 2001 From: Sagar Rao Date: Wed, 14 Sep 2022 23:08:29 +0530 Subject: [PATCH 13/26] Fixing failing testcase --- connect/runtime/hs_err_pid77304.log | 676 ++++++++++++++++++ .../IncrementalCooperativeAssignorTest.java | 4 +- .../WorkerCoordinatorIncrementalTest.java | 4 +- 3 files changed, 680 insertions(+), 4 deletions(-) create mode 100644 connect/runtime/hs_err_pid77304.log diff --git a/connect/runtime/hs_err_pid77304.log b/connect/runtime/hs_err_pid77304.log new file mode 100644 index 0000000000000..e05f2de864cb2 --- /dev/null +++ b/connect/runtime/hs_err_pid77304.log @@ -0,0 +1,676 @@ +# +# A fatal error has been detected by the Java Runtime Environment: +# +# Internal Error (sharedRuntime.cpp:553), pid=77304, tid=0x0000000000028a07 +# guarantee(cb != NULL && cb->is_nmethod()) failed: safepoint polling: pc must refer to an nmethod +# +# JRE version: OpenJDK Runtime Environment (8.0_322-b06) (build 1.8.0_322-b06) +# Java VM: OpenJDK 64-Bit Server VM (25.322-b06 mixed mode bsd-amd64 compressed oops) +# Failed to write core dump. Core dumps have been disabled. To enable core dumping, try "ulimit -c unlimited" before starting Java again +# +# If you would like to submit a bug report, please visit: +# https://github.com/adoptium/adoptium-support/issues +# + +--------------- T H R E A D --------------- + +Current thread (0x00007fba832e4800): JavaThread "qtp476631259-196-acceptor-0@642d7605-http_localhost0@9be7319{HTTP/1.1, (http/1.1)}{localhost:64232}" [_thread_in_Java, id=166407, stack(0x000000031dc6a000,0x000000031e06a000)] + +Stack: [0x000000031dc6a000,0x000000031e06a000], sp=0x000000031e066740, free space=4081k +Native frames: (J=compiled Java code, j=interpreted, Vv=VM code, C=native code) +V [libjvm.dylib+0x59baae] VMError::report_and_die()+0x440 +V [libjvm.dylib+0x1d2e38] report_vm_error(char const*, int, char const*, char const*)+0x55 +V [libjvm.dylib+0x4ed6d7] SharedRuntime::get_poll_stub(unsigned char*)+0x43 +V [libjvm.dylib+0x491961] JVM_handle_bsd_signal+0x290 +V [libjvm.dylib+0x48f234] signalHandler(int, __siginfo*, void*)+0x2d +C [libsystem_platform.dylib+0x3e2d] _sigtramp+0x1d +C 0x0000000000000000 +v ~StubRoutines::call_stub +V [libjvm.dylib+0x2c9ed1] JavaCalls::call_helper(JavaValue*, methodHandle*, JavaCallArguments*, Thread*)+0x6b7 +V [libjvm.dylib+0x2fc828] jni_invoke_nonstatic(JNIEnv_*, JavaValue*, _jobject*, JNICallType, _jmethodID*, JNI_ArgumentPusher*, Thread*)+0x3f9 +V [libjvm.dylib+0x2fcd5c] jni_NewObject+0x22f +C [libjava.dylib+0xde01] JNU_NewStringPlatform+0x266 +C [libjava.dylib+0xe108] JNU_ThrowByNameWithLastError+0x4b +C [libnio.dylib+0x3b7d] Java_sun_nio_ch_ServerSocketChannelImpl_accept0+0x16f +j sun.nio.ch.ServerSocketChannelImpl.accept0(Ljava/io/FileDescriptor;Ljava/io/FileDescriptor;[Ljava/net/InetSocketAddress;)I+0 +j sun.nio.ch.ServerSocketChannelImpl.accept(Ljava/io/FileDescriptor;Ljava/io/FileDescriptor;[Ljava/net/InetSocketAddress;)I+4 +j sun.nio.ch.ServerSocketChannelImpl.accept()Ljava/nio/channels/SocketChannel;+130 +j org.eclipse.jetty.server.ServerConnector.accept(I)V+17 +j org.eclipse.jetty.server.AbstractConnector$Acceptor.run()V+237 +j org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(Ljava/lang/Runnable;)V+1 +j org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run()V+262 +j java.lang.Thread.run()V+11 +v ~StubRoutines::call_stub +V [libjvm.dylib+0x2c9ed1] JavaCalls::call_helper(JavaValue*, methodHandle*, JavaCallArguments*, Thread*)+0x6b7 +V [libjvm.dylib+0x2c8cdd] JavaCalls::call_virtual(JavaValue*, KlassHandle, Symbol*, Symbol*, JavaCallArguments*, Thread*)+0x159 +V [libjvm.dylib+0x2c8eb9] JavaCalls::call_virtual(JavaValue*, Handle, KlassHandle, Symbol*, Symbol*, Thread*)+0x57 +V [libjvm.dylib+0x340bc3] thread_entry(JavaThread*, Thread*)+0x78 +V [libjvm.dylib+0x55ee9c] JavaThread::thread_main_inner()+0x82 +V [libjvm.dylib+0x55ed61] JavaThread::run()+0x169 +V [libjvm.dylib+0x48cd9b] java_start(Thread*)+0xfa +C [libsystem_pthread.dylib+0x64f4] _pthread_start+0x7d +C [libsystem_pthread.dylib+0x200f] thread_start+0xf +C 0x0000000000000000 + + +--------------- P R O C E S S --------------- + +Java Threads: ( => current thread ) + 0x00007fba9245e800 JavaThread "Keep-Alive-Timer" daemon [_thread_blocked, id=99959, stack(0x00000003224a0000,0x00000003228a0000)] + 0x00007fba92050800 JavaThread "qtp476631259-211" [_thread_in_vm, id=100099, stack(0x000000032209d000,0x000000032249d000)] + 0x00007fba86240800 JavaThread "KafkaBasedLog Work Thread - good-config" [_thread_in_native, id=163075, stack(0x0000000321c9a000,0x000000032209a000)] + 0x00007fba8626c000 JavaThread "Connector-Scheduler-9be7319-1" [_thread_blocked, id=163383, stack(0x0000000321897000,0x0000000321c97000)] + 0x00007fba92049000 JavaThread "kafka-producer-network-thread | producer-5" daemon [_thread_in_native, id=98595, stack(0x0000000321494000,0x0000000321894000)] + 0x00007fba87039800 JavaThread "KafkaBasedLog Work Thread - good-status" [_thread_in_native, id=164355, stack(0x0000000321091000,0x0000000321491000)] + 0x00007fba9204b000 JavaThread "kafka-producer-network-thread | producer-4" daemon [_thread_in_native, id=164879, stack(0x0000000320c8e000,0x000000032108e000)] + 0x00007fba91bde000 JavaThread "KafkaBasedLog Work Thread - good-offset" [_thread_in_native, id=165123, stack(0x000000032088b000,0x0000000320c8b000)] + 0x00007fba84920800 JavaThread "kafka-producer-network-thread | producer-3" daemon [_thread_in_native, id=165799, stack(0x0000000320488000,0x0000000320888000)] + 0x00007fba9204e800 JavaThread "kafka-admin-client-thread | adminclient-15" daemon [_thread_in_native, id=94063, stack(0x0000000320085000,0x0000000320485000)] + 0x00007fba8748e000 JavaThread "Session-HouseKeeper-524f5ea5-1" [_thread_blocked, id=167299, stack(0x000000031fc82000,0x0000000320082000)] + 0x00007fba9221c000 JavaThread "DistributedHerder-connect-4-1" [_thread_in_vm, id=95795, stack(0x000000031f87f000,0x000000031fc7f000)] + 0x00007fba86e24800 JavaThread "qtp476631259-198" [_thread_blocked, id=166935, stack(0x000000031f079000,0x000000031f479000)] + 0x00007fba869ad000 JavaThread "qtp476631259-197" [_thread_blocked, id=96263, stack(0x000000031e06d000,0x000000031e46d000)] +=>0x00007fba832e4800 JavaThread "qtp476631259-196-acceptor-0@642d7605-http_localhost0@9be7319{HTTP/1.1, (http/1.1)}{localhost:64232}" [_thread_in_Java, id=166407, stack(0x000000031dc6a000,0x000000031e06a000)] + 0x00007fba9245b800 JavaThread "qtp476631259-195" [_thread_in_native, id=96519, stack(0x000000031d867000,0x000000031dc67000)] + 0x00007fba91c8a000 JavaThread "qtp476631259-194" [_thread_in_native, id=96775, stack(0x000000031d464000,0x000000031d864000)] + 0x00007fba87087800 JavaThread "qtp476631259-193" [_thread_in_native, id=96007, stack(0x000000031d061000,0x000000031d461000)] + 0x00007fba8704c000 JavaThread "qtp476631259-192" [_thread_in_native, id=94743, stack(0x000000031cc5e000,0x000000031d05e000)] + 0x00007fba923a6000 JavaThread "qtp476631259-191" [_thread_in_native, id=169219, stack(0x000000031c85b000,0x000000031cc5b000)] + 0x00007fba832e3800 JavaThread "executor-Fetch" [_thread_blocked, id=93195, stack(0x000000031c458000,0x000000031c858000)] + 0x00007fba864bc800 JavaThread "KafkaBasedLog Work Thread - good-offset" [_thread_in_native, id=170051, stack(0x000000031bc52000,0x000000031c052000)] + 0x00007fba84d0e000 JavaThread "kafka-producer-network-thread | producer-2" daemon [_thread_in_native, id=94979, stack(0x000000031f47c000,0x000000031f87c000)] + 0x00007fba91d37800 JavaThread "kafka-admin-client-thread | adminclient-13" daemon [_thread_in_native, id=169815, stack(0x000000031ec76000,0x000000031f076000)] + 0x00007fba87045800 JavaThread "kafka-admin-client-thread | adminclient-11" daemon [_thread_in_native, id=92979, stack(0x000000031e873000,0x000000031ec73000)] + 0x00007fba874d2000 JavaThread "kafka-admin-client-thread | adminclient-9" daemon [_thread_in_native, id=172815, stack(0x000000031e470000,0x000000031e870000)] + 0x00007fba90828000 JavaThread "kafka-scheduler-9" daemon [_thread_blocked, id=89615, stack(0x000000031c055000,0x000000031c455000)] + 0x00007fba85bba800 JavaThread "kafka-producer-network-thread | producer-1" daemon [_thread_in_native, id=89183, stack(0x000000031b84f000,0x000000031bc4f000)] + 0x00007fba8358c800 JavaThread "data-plane-kafka-socket-acceptor-ListenerName(PLAINTEXT)-PLAINTEXT-0" [_thread_in_native, id=64771, stack(0x000000031b44c000,0x000000031b84c000)] + 0x00007fba8358b800 JavaThread "data-plane-kafka-network-thread-0-ListenerName(PLAINTEXT)-PLAINTEXT-2" [_thread_in_native, id=65283, stack(0x000000031b049000,0x000000031b449000)] + 0x00007fba8358a800 JavaThread "data-plane-kafka-network-thread-0-ListenerName(PLAINTEXT)-PLAINTEXT-1" [_thread_in_native, id=65795, stack(0x000000031ac46000,0x000000031b046000)] + 0x00007fba83451800 JavaThread "data-plane-kafka-network-thread-0-ListenerName(PLAINTEXT)-PLAINTEXT-0" [_thread_in_native, id=66323, stack(0x000000031a843000,0x000000031ac43000)] + 0x00007fba85564800 JavaThread "/config/changes-event-process-thread" [_thread_blocked, id=66687, stack(0x000000031a440000,0x000000031a840000)] + 0x00007fba843ad000 JavaThread "kafka-scheduler-0" daemon [_thread_blocked, id=66895, stack(0x000000031a03d000,0x000000031a43d000)] + 0x00007fba85550800 JavaThread "data-plane-kafka-request-handler-7" daemon [_thread_blocked, id=62979, stack(0x0000000319c3a000,0x000000031a03a000)] + 0x00007fba8554f800 JavaThread "data-plane-kafka-request-handler-6" daemon [_thread_blocked, id=62467, stack(0x0000000319837000,0x0000000319c37000)] + 0x00007fba8554e800 JavaThread "data-plane-kafka-request-handler-5" daemon [_thread_blocked, id=67331, stack(0x0000000319434000,0x0000000319834000)] + 0x00007fba8554e000 JavaThread "data-plane-kafka-request-handler-4" daemon [_thread_blocked, id=67843, stack(0x0000000319031000,0x0000000319431000)] + 0x00007fba8555b000 JavaThread "data-plane-kafka-request-handler-3" daemon [_thread_blocked, id=68099, stack(0x0000000318c2e000,0x000000031902e000)] + 0x00007fba8555a000 JavaThread "data-plane-kafka-request-handler-2" daemon [_thread_blocked, id=68611, stack(0x000000031882b000,0x0000000318c2b000)] + 0x00007fba8506c800 JavaThread "data-plane-kafka-request-handler-1" daemon [_thread_in_vm, id=61699, stack(0x0000000318428000,0x0000000318828000)] + 0x00007fba851fa000 JavaThread "data-plane-kafka-request-handler-0" daemon [_thread_blocked, id=69499, stack(0x0000000318025000,0x0000000318425000)] + 0x00007fba86899800 JavaThread "ExpirationReaper-0-AlterAcls" [_thread_blocked, id=61375, stack(0x0000000317c22000,0x0000000318022000)] + 0x00007fba843ae800 JavaThread "Controller-0-to-broker-0-send-thread" [_thread_in_native, id=60987, stack(0x000000031781f000,0x0000000317c1f000)] + 0x00007fba85ad1000 JavaThread "TxnMarkerSenderThread-0" [_thread_in_native, id=70687, stack(0x000000031741c000,0x000000031781c000)] + 0x00007fba83056800 JavaThread "transaction-log-manager-0" daemon [_thread_blocked, id=60471, stack(0x0000000317019000,0x0000000317419000)] + 0x00007fba84891000 JavaThread "group-metadata-manager-0" daemon [_thread_blocked, id=60087, stack(0x0000000316c16000,0x0000000317016000)] + 0x00007fba8406c000 JavaThread "ExpirationReaper-0-Rebalance" [_thread_blocked, id=59151, stack(0x0000000316813000,0x0000000316c13000)] + 0x00007fba8485f800 JavaThread "ExpirationReaper-0-Heartbeat" [_thread_blocked, id=58807, stack(0x0000000316410000,0x0000000316810000)] + 0x00007fba83024800 JavaThread "ExpirationReaper-0-topic" [_thread_blocked, id=72295, stack(0x000000031600d000,0x000000031640d000)] + 0x00007fba86057800 JavaThread "controller-event-thread" [_thread_blocked, id=72851, stack(0x0000000315c0a000,0x000000031600a000)] + 0x00007fba8788a800 JavaThread "kafka-scheduler-8" daemon [_thread_blocked, id=57639, stack(0x0000000315807000,0x0000000315c07000)] + 0x00007fba8484a000 JavaThread "kafka-scheduler-7" daemon [_thread_blocked, id=57347, stack(0x0000000315404000,0x0000000315804000)] + 0x00007fba851e4800 JavaThread "LogDirFailureHandler" [_thread_blocked, id=73475, stack(0x0000000315001000,0x0000000315401000)] + 0x00007fba8402e000 JavaThread "kafka-scheduler-6" daemon [_thread_blocked, id=74299, stack(0x0000000314bfe000,0x0000000314ffe000)] + 0x00007fba83023800 JavaThread "kafka-scheduler-5" daemon [_thread_blocked, id=73987, stack(0x00000003147fb000,0x0000000314bfb000)] + 0x00007fba840a8000 JavaThread "ExpirationReaper-0-ElectLeader" [_thread_blocked, id=56091, stack(0x00000003143f8000,0x00000003147f8000)] + 0x00007fba83207800 JavaThread "ExpirationReaper-0-DeleteRecords" [_thread_blocked, id=75355, stack(0x0000000313ff5000,0x00000003143f5000)] + 0x00007fba908af000 JavaThread "ExpirationReaper-0-Fetch" [_thread_blocked, id=75531, stack(0x0000000313bf2000,0x0000000313ff2000)] + 0x00007fba83206800 JavaThread "ExpirationReaper-0-Produce" [_thread_blocked, id=55159, stack(0x00000003137ef000,0x0000000313bef000)] + 0x00007fba859e8800 JavaThread "BrokerToControllerChannelManager broker=0 name=alterPartition" [_thread_in_native, id=54391, stack(0x00000003133ec000,0x00000003137ec000)] + 0x00007fba8494b800 JavaThread "BrokerToControllerChannelManager broker=0 name=forwarding" [_thread_in_native, id=53907, stack(0x0000000312fe9000,0x00000003133e9000)] + 0x00007fba858a0800 JavaThread "feature-zk-node-event-process-thread" [_thread_blocked, id=53723, stack(0x0000000312be6000,0x0000000312fe6000)] + 0x00007fba869aa800 JavaThread "kafka-log-cleaner-thread-0" [_thread_blocked, id=77107, stack(0x00000003127e3000,0x0000000312be3000)] + 0x00007fba840cb800 JavaThread "kafka-scheduler-4" daemon [_thread_blocked, id=77571, stack(0x00000003123e0000,0x00000003127e0000)] + 0x00007fba840ca800 JavaThread "kafka-scheduler-3" daemon [_thread_blocked, id=77827, stack(0x0000000311fdd000,0x00000003123dd000)] + 0x00007fba840ca000 JavaThread "kafka-scheduler-2" daemon [_thread_blocked, id=52483, stack(0x0000000311bda000,0x0000000311fda000)] + 0x00007fba840c9000 JavaThread "kafka-scheduler-1" daemon [_thread_blocked, id=78347, stack(0x00000003117d7000,0x0000000311bd7000)] + 0x00007fba840c8000 JavaThread "kafka-scheduler-0" daemon [_thread_blocked, id=51807, stack(0x00000003113d4000,0x00000003117d4000)] + 0x00007fba858d7800 JavaThread "ThrottledChannelReaper-ControllerMutation" [_thread_blocked, id=79107, stack(0x0000000310fd1000,0x00000003113d1000)] + 0x00007fba831f5800 JavaThread "ThrottledChannelReaper-Request" [_thread_blocked, id=51203, stack(0x0000000310bce000,0x0000000310fce000)] + 0x00007fba831f7800 JavaThread "ThrottledChannelReaper-Produce" [_thread_blocked, id=80231, stack(0x00000003107cb000,0x0000000310bcb000)] + 0x00007fba87017800 JavaThread "ThrottledChannelReaper-Fetch" [_thread_blocked, id=80835, stack(0x00000003103c8000,0x00000003107c8000)] + 0x00007fba868a2800 JavaThread "SensorExpiryThread" daemon [_thread_blocked, id=80899, stack(0x000000030ffc5000,0x00000003103c5000)] + 0x00007fba871c5000 JavaThread "NIOWorkerThread-20" daemon [_thread_blocked, id=49667, stack(0x000000030fbc2000,0x000000030ffc2000)] + 0x00007fba840be800 JavaThread "NIOWorkerThread-19" daemon [_thread_blocked, id=49179, stack(0x000000030f7bf000,0x000000030fbbf000)] + 0x00007fba84944000 JavaThread "NIOWorkerThread-18" daemon [_thread_blocked, id=48411, stack(0x000000030f3bc000,0x000000030f7bc000)] + 0x00007fba86995000 JavaThread "NIOWorkerThread-17" daemon [_thread_blocked, id=48139, stack(0x000000030efb9000,0x000000030f3b9000)] + 0x00007fba84943800 JavaThread "NIOWorkerThread-16" daemon [_thread_blocked, id=82995, stack(0x000000030ebb6000,0x000000030efb6000)] + 0x00007fba84942800 JavaThread "NIOWorkerThread-15" daemon [_thread_blocked, id=46859, stack(0x000000030e7b3000,0x000000030ebb3000)] + 0x00007fba871c4000 JavaThread "NIOWorkerThread-14" daemon [_thread_blocked, id=83203, stack(0x000000030e3b0000,0x000000030e7b0000)] + 0x00007fba849ea800 JavaThread "NIOWorkerThread-13" daemon [_thread_blocked, id=84231, stack(0x000000030dfad000,0x000000030e3ad000)] + 0x00007fba8787b800 JavaThread "NIOWorkerThread-12" daemon [_thread_blocked, id=84995, stack(0x000000030dbaa000,0x000000030dfaa000)] + 0x00007fba858d0800 JavaThread "NIOWorkerThread-11" daemon [_thread_blocked, id=85771, stack(0x000000030d7a7000,0x000000030dba7000)] + 0x00007fba871c3000 JavaThread "NIOWorkerThread-10" daemon [_thread_blocked, id=46083, stack(0x000000030d3a4000,0x000000030d7a4000)] + 0x00007fba831e7800 JavaThread "NIOWorkerThread-9" daemon [_thread_blocked, id=45587, stack(0x000000030cfa1000,0x000000030d3a1000)] + 0x00007fba830a1000 JavaThread "NIOWorkerThread-8" daemon [_thread_blocked, id=45419, stack(0x000000030cb9e000,0x000000030cf9e000)] + 0x00007fba85292000 JavaThread "NIOWorkerThread-7" daemon [_thread_blocked, id=44807, stack(0x000000030c79b000,0x000000030cb9b000)] + 0x00007fba8528e800 JavaThread "NIOWorkerThread-6" daemon [_thread_blocked, id=87055, stack(0x000000030c398000,0x000000030c798000)] + 0x00007fba84178000 JavaThread "NIOWorkerThread-5" daemon [_thread_blocked, id=44079, stack(0x000000030bf95000,0x000000030c395000)] + 0x00007fba849eb800 JavaThread "NIOWorkerThread-4" daemon [_thread_blocked, id=31311, stack(0x000000030bb92000,0x000000030bf92000)] + 0x00007fba830a0800 JavaThread "NIOWorkerThread-3" daemon [_thread_blocked, id=31067, stack(0x000000030b78f000,0x000000030bb8f000)] + 0x00007fba84997800 JavaThread "NIOWorkerThread-2" daemon [_thread_blocked, id=30799, stack(0x000000030b38c000,0x000000030b78c000)] + 0x00007fba85285800 JavaThread "NIOWorkerThread-1" daemon [_thread_blocked, id=29967, stack(0x000000030af89000,0x000000030b389000)] + 0x00007fba831e7000 JavaThread "Test worker-EventThread" daemon [_thread_blocked, id=33027, stack(0x000000030ab86000,0x000000030af86000)] + 0x00007fba85a91800 JavaThread "Test worker-SendThread(127.0.0.1:63835)" daemon [_thread_in_native, id=33439, stack(0x000000030a783000,0x000000030ab83000)] + 0x00007fba8708a800 JavaThread "metrics-meter-tick-thread-2" daemon [_thread_blocked, id=29191, stack(0x000000030a380000,0x000000030a780000)] + 0x00007fba8629f800 JavaThread "metrics-meter-tick-thread-1" daemon [_thread_blocked, id=28723, stack(0x0000000309f7d000,0x000000030a37d000)] + 0x00007fba873a2800 JavaThread "RequestThrottler" [_thread_blocked, id=34403, stack(0x0000000309b7a000,0x0000000309f7a000)] + 0x00007fba873a0000 JavaThread "ProcessThread(sid:0 cport:63835):" [_thread_blocked, id=35075, stack(0x0000000309777000,0x0000000309b77000)] + 0x00007fba8628c800 JavaThread "SyncThread:0" [_thread_blocked, id=35595, stack(0x0000000309374000,0x0000000309774000)] + 0x00007fba87390800 JavaThread "SessionTracker" [_thread_blocked, id=36311, stack(0x0000000308f71000,0x0000000309371000)] + 0x00007fba85270800 JavaThread "ConnnectionExpirer" [_thread_blocked, id=27395, stack(0x0000000308b6e000,0x0000000308f6e000)] + 0x00007fba831e5000 JavaThread "NIOServerCxnFactory.AcceptThread:/127.0.0.1:0" daemon [_thread_in_native, id=36611, stack(0x000000030876b000,0x0000000308b6b000)] + 0x00007fba831e9800 JavaThread "NIOServerCxnFactory.SelectorThread-0" daemon [_thread_in_native, id=26371, stack(0x0000000308368000,0x0000000308768000)] + 0x00007fba87390000 JavaThread "NIOServerCxnFactory.SelectorThread-1" daemon [_thread_in_native, id=37123, stack(0x0000000307f65000,0x0000000308365000)] + 0x00007fba871ce000 JavaThread "/127.0.0.1:63826 to /127.0.0.1:63820 workers Thread 3" [_thread_in_native, id=39171, stack(0x0000000307b62000,0x0000000307f62000)] + 0x00007fba871cc800 JavaThread "/127.0.0.1:63826 to /127.0.0.1:63820 workers Thread 2" [_thread_in_vm, id=24887, stack(0x000000030775f000,0x0000000307b5f000)] + 0x00007fba8690e800 JavaThread "/127.0.0.1:63826 to /127.0.0.1:63820 workers" [_thread_blocked, id=24595, stack(0x000000030735c000,0x000000030775c000)] + 0x00007fba87815000 JavaThread "Service Thread" daemon [_thread_blocked, id=40963, stack(0x0000000306e56000,0x0000000307256000)] + 0x00007fba87814000 JavaThread "C1 CompilerThread3" daemon [_thread_in_vm, id=41731, stack(0x0000000306d53000,0x0000000306e53000)] + 0x00007fba87813000 JavaThread "C2 CompilerThread2" daemon [_thread_blocked, id=23043, stack(0x0000000306c50000,0x0000000306d50000)] + 0x00007fba8405b000 JavaThread "C2 CompilerThread1" daemon [_thread_blocked, id=41987, stack(0x0000000306b4d000,0x0000000306c4d000)] + 0x00007fba8400f800 JavaThread "C2 CompilerThread0" daemon [_thread_in_vm, id=42243, stack(0x0000000306a4a000,0x0000000306b4a000)] + 0x00007fba8400a800 JavaThread "Signal Dispatcher" daemon [_thread_blocked, id=22019, stack(0x0000000306647000,0x0000000306a47000)] + 0x00007fba84818000 JavaThread "Finalizer" daemon [_thread_blocked, id=15131, stack(0x0000000306134000,0x0000000306534000)] + 0x00007fba87808800 JavaThread "Reference Handler" daemon [_thread_blocked, id=14883, stack(0x0000000305d31000,0x0000000306131000)] + 0x00007fba8480b800 JavaThread "Test worker" [_thread_blocked, id=6915, stack(0x0000000304f10000,0x0000000305310000)] + +Other Threads: + 0x00007fba86847800 VMThread [stack: 0x0000000305c2e000,0x0000000305d2e000] [id=18947] + 0x00007fba90809000 WatcherThread [stack: 0x0000000307259000,0x0000000307359000] [id=40707] + +VM state:synchronizing (normal execution) + +VM Mutex/Monitor currently owned by a thread: ([mutex/lock_event]) +[0x00007fba83905db0] Safepoint_lock - owner thread: 0x00007fba86847800 +[0x00007fba83905e30] Threads_lock - owner thread: 0x00007fba86847800 + +heap address: 0x0000000740000000, size: 2048 MB, Compressed Oops mode: Zero based, Oop shift amount: 3 +Narrow klass base: 0x0000000000000000, Narrow klass shift: 3 +Compressed class space size: 1073741824 Address: 0x00000007c0000000 + +Heap: + PSYoungGen total 560640K, used 77468K [0x0000000795580000, 0x00000007c0000000, 0x00000007c0000000) + eden space 500224K, 3% used [0x0000000795580000,0x0000000796638828,0x00000007b3e00000) + from space 60416K, 99% used [0x00000007b3e00000,0x00000007b78eeb98,0x00000007b7900000) + to space 100864K, 0% used [0x00000007b9d80000,0x00000007b9d80000,0x00000007c0000000) + ParOldGen total 175104K, used 48979K [0x0000000740000000, 0x000000074ab00000, 0x0000000795580000) + object space 175104K, 27% used [0x0000000740000000,0x0000000742fd4cf8,0x000000074ab00000) + Metaspace used 66302K, capacity 71877K, committed 72152K, reserved 1112064K + class space used 8644K, capacity 9370K, committed 9472K, reserved 1048576K + +Card table byte_map: [0x000000010e3c0000,0x000000010e7c1000] byte_map_base: 0x000000010a9c0000 + +Marking Bits: (ParMarkBitMap*) 0x000000010f50bf08 + Begin Bits: [0x000000011fc51000, 0x0000000121c51000) + End Bits: [0x0000000121c51000, 0x0000000123c51000) + +Polling page: 0x0000000108c18000 + +CodeCache: size=245760Kb used=14840Kb max_used=14846Kb free=230919Kb + bounds [0x0000000110c51000, 0x0000000111b01000, 0x000000011fc51000] + total_blobs=6049 nmethods=5181 adapters=778 + compilation: enabled + +Compilation events (10 events): +Event: 53.733 Thread 0x00007fba8400f800 6469 4 java.nio.ByteBuffer::allocate (22 bytes) +Event: 53.745 Thread 0x00007fba8400f800 nmethod 6469 0x00000001115c4e10 code [0x00000001115c4f60, 0x00000001115c5138] +Event: 53.896 Thread 0x00007fba8405b000 nmethod 6463 0x000000011162ad90 code [0x000000011162b160, 0x000000011162ccd8] +Event: 53.959 Thread 0x00007fba87813000 6473 s! 4 sun.misc.URLClassPath::getLoader (243 bytes) +Event: 53.961 Thread 0x00007fba8400f800 6472 s 4 sun.misc.URLClassPath::getNextLoader (88 bytes) +Event: 53.964 Thread 0x00007fba8405b000 6475 4 java.lang.Math::max (12 bytes) +Event: 54.011 Thread 0x00007fba8400f800 nmethod 6472 0x00000001116a0610 code [0x00000001116a08e0, 0x00000001116a22c8] +Event: 54.011 Thread 0x00007fba8400f800 6479 4 org.apache.kafka.common.utils.SystemTime::milliseconds (4 bytes) +Event: 54.060 Thread 0x00007fba8405b000 nmethod 6475 0x000000011169dd90 code [0x000000011169dec0, 0x000000011169df18] +Event: 54.065 Thread 0x00007fba8405b000 6478 4 java.util.LinkedList::poll (19 bytes) + +GC Heap History (10 events): +Event: 37.292 GC heap before +{Heap before GC invocations=14 (full 3): + PSYoungGen total 614912K, used 614909K [0x0000000795580000, 0x00000007c0000000, 0x00000007c0000000) + eden space 558592K, 100% used [0x0000000795580000,0x00000007b7700000,0x00000007b7700000) + from space 56320K, 99% used [0x00000007bc900000,0x00000007bffff490,0x00000007c0000000) + to space 70144K, 0% used [0x00000007b7700000,0x00000007b7700000,0x00000007bbb80000) + ParOldGen total 175104K, used 39338K [0x0000000740000000, 0x000000074ab00000, 0x0000000795580000) + object space 175104K, 22% used [0x0000000740000000,0x000000074266abb0,0x000000074ab00000) + Metaspace used 62563K, capacity 67527K, committed 67672K, reserved 1107968K + class space used 8156K, capacity 8812K, committed 8832K, reserved 1048576K +Event: 37.693 GC heap after +Heap after GC invocations=14 (full 3): + PSYoungGen total 623104K, used 64380K [0x0000000795580000, 0x00000007c0000000, 0x00000007c0000000) + eden space 558592K, 0% used [0x0000000795580000,0x0000000795580000,0x00000007b7700000) + from space 64512K, 99% used [0x00000007b7700000,0x00000007bb5df1d0,0x00000007bb600000) + to space 75776K, 0% used [0x00000007bb600000,0x00000007bb600000,0x00000007c0000000) + ParOldGen total 175104K, used 42880K [0x0000000740000000, 0x000000074ab00000, 0x0000000795580000) + object space 175104K, 24% used [0x0000000740000000,0x00000007429e0298,0x000000074ab00000) + Metaspace used 62563K, capacity 67527K, committed 67672K, reserved 1107968K + class space used 8156K, capacity 8812K, committed 8832K, reserved 1048576K +} +Event: 40.789 GC heap before +{Heap before GC invocations=15 (full 3): + PSYoungGen total 623104K, used 622972K [0x0000000795580000, 0x00000007c0000000, 0x00000007c0000000) + eden space 558592K, 100% used [0x0000000795580000,0x00000007b7700000,0x00000007b7700000) + from space 64512K, 99% used [0x00000007b7700000,0x00000007bb5df1d0,0x00000007bb600000) + to space 75776K, 0% used [0x00000007bb600000,0x00000007bb600000,0x00000007c0000000) + ParOldGen total 175104K, used 42880K [0x0000000740000000, 0x000000074ab00000, 0x0000000795580000) + object space 175104K, 24% used [0x0000000740000000,0x00000007429e0298,0x000000074ab00000) + Metaspace used 62663K, capacity 67667K, committed 67928K, reserved 1107968K + class space used 8157K, capacity 8814K, committed 8832K, reserved 1048576K +Event: 40.976 GC heap after +Heap after GC invocations=15 (full 3): + PSYoungGen total 594432K, used 72605K [0x0000000795580000, 0x00000007c0000000, 0x00000007c0000000) + eden space 518656K, 0% used [0x0000000795580000,0x0000000795580000,0x00000007b5000000) + from space 75776K, 95% used [0x00000007bb600000,0x00000007bfce76b8,0x00000007c0000000) + to space 90112K, 0% used [0x00000007b5000000,0x00000007b5000000,0x00000007ba800000) + ParOldGen total 175104K, used 43056K [0x0000000740000000, 0x000000074ab00000, 0x0000000795580000) + object space 175104K, 24% used [0x0000000740000000,0x0000000742a0c2a8,0x000000074ab00000) + Metaspace used 62663K, capacity 67667K, committed 67928K, reserved 1107968K + class space used 8157K, capacity 8814K, committed 8832K, reserved 1048576K +} +Event: 44.128 GC heap before +{Heap before GC invocations=16 (full 3): + PSYoungGen total 594432K, used 591261K [0x0000000795580000, 0x00000007c0000000, 0x00000007c0000000) + eden space 518656K, 100% used [0x0000000795580000,0x00000007b5000000,0x00000007b5000000) + from space 75776K, 95% used [0x00000007bb600000,0x00000007bfce76b8,0x00000007c0000000) + to space 90112K, 0% used [0x00000007b5000000,0x00000007b5000000,0x00000007ba800000) + ParOldGen total 175104K, used 43056K [0x0000000740000000, 0x000000074ab00000, 0x0000000795580000) + object space 175104K, 24% used [0x0000000740000000,0x0000000742a0c2a8,0x000000074ab00000) + Metaspace used 63630K, capacity 68893K, committed 69208K, reserved 1110016K + class space used 8281K, capacity 8983K, committed 9088K, reserved 1048576K +Event: 44.346 GC heap after +Heap after GC invocations=16 (full 3): + PSYoungGen total 579072K, used 60232K [0x0000000795580000, 0x00000007c0000000, 0x00000007c0000000) + eden space 518656K, 0% used [0x0000000795580000,0x0000000795580000,0x00000007b5000000) + from space 60416K, 99% used [0x00000007b5000000,0x00000007b8ad2028,0x00000007b8b00000) + to space 94208K, 0% used [0x00000007ba400000,0x00000007ba400000,0x00000007c0000000) + ParOldGen total 175104K, used 44306K [0x0000000740000000, 0x000000074ab00000, 0x0000000795580000) + object space 175104K, 25% used [0x0000000740000000,0x0000000742b449c8,0x000000074ab00000) + Metaspace used 63630K, capacity 68893K, committed 69208K, reserved 1110016K + class space used 8281K, capacity 8983K, committed 9088K, reserved 1048576K +} +Event: 44.857 GC heap before +{Heap before GC invocations=17 (full 3): + PSYoungGen total 579072K, used 578888K [0x0000000795580000, 0x00000007c0000000, 0x00000007c0000000) + eden space 518656K, 100% used [0x0000000795580000,0x00000007b5000000,0x00000007b5000000) + from space 60416K, 99% used [0x00000007b5000000,0x00000007b8ad2028,0x00000007b8b00000) + to space 94208K, 0% used [0x00000007ba400000,0x00000007ba400000,0x00000007c0000000) + ParOldGen total 175104K, used 44306K [0x0000000740000000, 0x000000074ab00000, 0x0000000795580000) + object space 175104K, 25% used [0x0000000740000000,0x0000000742b449c8,0x000000074ab00000) + Metaspace used 63676K, capacity 68957K, committed 69208K, reserved 1110016K + class space used 8296K, capacity 9006K, committed 9088K, reserved 1048576K +Event: 45.023 GC heap after +Heap after GC invocations=17 (full 3): + PSYoungGen total 594432K, used 67076K [0x0000000795580000, 0x00000007c0000000, 0x00000007c0000000) + eden space 500224K, 0% used [0x0000000795580000,0x0000000795580000,0x00000007b3e00000) + from space 94208K, 71% used [0x00000007ba400000,0x00000007be5810c0,0x00000007c0000000) + to space 99328K, 0% used [0x00000007b3e00000,0x00000007b3e00000,0x00000007b9f00000) + ParOldGen total 175104K, used 46363K [0x0000000740000000, 0x000000074ab00000, 0x0000000795580000) + object space 175104K, 26% used [0x0000000740000000,0x0000000742d46dc0,0x000000074ab00000) + Metaspace used 63676K, capacity 68957K, committed 69208K, reserved 1110016K + class space used 8296K, capacity 9006K, committed 9088K, reserved 1048576K +} +Event: 52.822 GC heap before +{Heap before GC invocations=18 (full 3): + PSYoungGen total 594432K, used 567300K [0x0000000795580000, 0x00000007c0000000, 0x00000007c0000000) + eden space 500224K, 100% used [0x0000000795580000,0x00000007b3e00000,0x00000007b3e00000) + from space 94208K, 71% used [0x00000007ba400000,0x00000007be5810c0,0x00000007c0000000) + to space 99328K, 0% used [0x00000007b3e00000,0x00000007b3e00000,0x00000007b9f00000) + ParOldGen total 175104K, used 46363K [0x0000000740000000, 0x000000074ab00000, 0x0000000795580000) + object space 175104K, 26% used [0x0000000740000000,0x0000000742d46dc0,0x000000074ab00000) + Metaspace used 66094K, capacity 71609K, committed 71768K, reserved 1112064K + class space used 8608K, capacity 9332K, committed 9344K, reserved 1048576K +Event: 53.305 GC heap after +Heap after GC invocations=18 (full 3): + PSYoungGen total 560640K, used 60346K [0x0000000795580000, 0x00000007c0000000, 0x00000007c0000000) + eden space 500224K, 0% used [0x0000000795580000,0x0000000795580000,0x00000007b3e00000) + from space 60416K, 99% used [0x00000007b3e00000,0x00000007b78eeb98,0x00000007b7900000) + to space 100864K, 0% used [0x00000007b9d80000,0x00000007b9d80000,0x00000007c0000000) + ParOldGen total 175104K, used 48979K [0x0000000740000000, 0x000000074ab00000, 0x0000000795580000) + object space 175104K, 27% used [0x0000000740000000,0x0000000742fd4cf8,0x000000074ab00000) + Metaspace used 66094K, capacity 71609K, committed 71768K, reserved 1112064K + class space used 8608K, capacity 9332K, committed 9344K, reserved 1048576K +} + +Deoptimization events (10 events): +Event: 50.305 Thread 0x00007fba92050800 Uncommon trap: reason=unstable_if action=reinterpret pc=0x0000000111932c38 method=java.net.URI$Parser.scan(IILjava/lang/String;Ljava/lang/String;)I @ 23 +Event: 50.563 Thread 0x00007fba831e7000 Uncommon trap: reason=unstable_if action=reinterpret pc=0x0000000111669184 method=java.util.concurrent.locks.AbstractQueuedSynchronizer.doReleaseShared()V @ 24 +Event: 50.710 Thread 0x00007fba8555b000 Uncommon trap: reason=bimorphic action=maybe_recompile pc=0x000000011113be0c method=org.apache.kafka.common.utils.ImplicitLinkedHashCollection.removeFromList(Lorg/apache/kafka/common/utils/ImplicitLinkedHashCollection$Element;[Lorg/apache/kafka/common/uti +Event: 51.391 Thread 0x00007fba9221c000 Uncommon trap: reason=unstable_if action=reinterpret pc=0x0000000110db3308 method=java.lang.Throwable.getOurStackTrace()[Ljava/lang/StackTraceElement; @ 7 +Event: 51.653 Thread 0x00007fba8555b000 Uncommon trap: reason=bimorphic action=maybe_recompile pc=0x000000011113be0c method=org.apache.kafka.common.utils.ImplicitLinkedHashCollection.removeFromList(Lorg/apache/kafka/common/utils/ImplicitLinkedHashCollection$Element;[Lorg/apache/kafka/common/uti +Event: 51.912 Thread 0x00007fba8554f800 Uncommon trap: reason=bimorphic action=maybe_recompile pc=0x000000011113be0c method=org.apache.kafka.common.utils.ImplicitLinkedHashCollection.removeFromList(Lorg/apache/kafka/common/utils/ImplicitLinkedHashCollection$Element;[Lorg/apache/kafka/common/uti +Event: 52.028 Thread 0x00007fba92050800 Uncommon trap: reason=unstable_if action=reinterpret pc=0x0000000111045fc8 method=java.util.stream.StreamOpFlag.fromCharacteristics(Ljava/util/Spliterator;)I @ 10 +Event: 52.137 Thread 0x00007fba9221c000 Uncommon trap: reason=unstable_if action=reinterpret pc=0x0000000110ec16f0 method=org.apache.kafka.common.utils.ImplicitLinkedHashCollection.reseat(I)V @ 40 +Event: 53.402 Thread 0x00007fba8555b000 Uncommon trap: reason=unstable_if action=reinterpret pc=0x0000000110f689f8 method=java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(I)V @ 5 +Event: 53.477 Thread 0x00007fba8555b000 Uncommon trap: reason=unstable_if action=reinterpret pc=0x00000001112eb574 method=java.util.concurrent.locks.AbstractQueuedSynchronizer.apparentlyFirstQueuedIsExclusive()Z @ 6 + +Classes redefined (0 events): +No events + +Internal exceptions (10 events): +Event: 27.875 Thread 0x00007fba8480b800 Exception (0x000000079928e820) thrown at [/Users/jenkins/workspace/build-scripts/jobs/jdk8u/jdk8u-mac-x64-h +Event: 31.788 Thread 0x00007fba8480b800 Exception (0x000000079c323b18) thrown at [/Users/jenkins/workspace/build-scripts/jobs/jdk8u/jdk8u-mac-x64-hotspot/workspace/build/src/hotsp +Event: 33.133 Thread 0x00007fba86f5c000 Exception (0x00000007970cfcd8) thrown at [/Users/jenkins/workspace/build-scripts/jobs/jdk8u/jdk8u-mac-x64-hotspot/workspace/build/src/hotspot/src/share/vm/prims/jni.cpp, line 711] +Event: 39.208 Thread 0x00007fba91bde800 Exception (0x0000000796030d68) thrown at [/Users/jenkins/workspace/build-scripts/jobs/jdk8u/jdk8u-mac-x64-hotspot/workspace/build/src/hotspot/src/share/vm/prims/jni.cpp, line 711] +Event: 43.472 Thread 0x00007fba90826000 Exception (0x00000007b16971b0) thrown at [/Users/jenkins/workspace/build-scripts/jobs/jdk8u/jdk8u-mac-x64-hotspot/workspace/build/src/hotspot/src/share/vm/prims/jni.cpp, line 711] +Event: 50.089 Thread 0x00007fba92050800 Exception (0x00000007b2287830) thrown at [/Users/jenkins/workspace/build-scripts/jobs/jdk8u/jdk8u-mac-x64-hotspot/workspace/build/src/hotspot/src/share/vm/runtime/reflection.cpp, line 1092] +Event: 52.141 Thread 0x00007fba92050800 Exception (0x00000007b36d46f8) thrown at [/Users/jenkins/workspace/build-scripts/jobs/jdk8u/jdk8u-mac-x +Event: 52.621 Thread 0x00007fba8506c800 Exception (0x00000007b3d60cd0) thrown at [/Users/jenkins/workspace/build-scripts/jobs/jdk8u/jdk8u-mac-x64-hotspot/workspace/build/src/hotspot/src/share/vm/prims/jni.cpp, line 711] +Event: 53.703 Thread 0x00007fba8506c800 Exception (0x0000000795d4eea8) thrown at [/Users/jenkins/workspace/build-scripts/jobs/jdk8u/jdk8u-mac-x64-hotspot/workspace/build/src/hotspot/src/share/vm/prims/jni.cpp, line 711] +Event: 54.098 Thread 0x00007fba8506c800 Exception (0x0000000796561808) thrown at [/Users/jenkins/workspace/build-scripts/jobs/jdk8u/jdk8u-mac-x64-hotspot/workspace/build/src/hotspot/src/share/vm/prims/jni.cpp, line 711] + +Events (10 events): +Event: 53.997 loading class org/apache/kafka/connect/util/clusters/WorkerHandle +Event: 53.997 loading class org/apache/kafka/connect/util/clusters/WorkerHandle done +Event: 53.997 loading class org/apache/kafka/connect/util/clusters/EmbeddedConnectCluster +Event: 53.997 loading class org/apache/kafka/connect/util/clusters/EmbeddedConnectCluster done +Event: 54.022 Executing VM operation: RevokeBias +Event: 54.054 Executing VM operation: RevokeBias done +Event: 54.055 Thread 0x00007fba87814000 flushing nmethod 0x0000000110fe15d0 +Event: 54.061 Executing VM operation: RevokeBias +Event: 54.086 Executing VM operation: RevokeBias done +Event: 54.099 Executing VM operation: RevokeBias + + +Dynamic libraries: +0x00007ff830514000 /System/Library/Frameworks/Cocoa.framework/Versions/A/Cocoa +0x00007ff81be12000 /System/Library/Frameworks/AppKit.framework/Versions/C/AppKit +0x00007ff81f371000 /System/Library/Frameworks/CoreData.framework/Versions/A/CoreData +0x00007ff81a229000 /System/Library/Frameworks/Foundation.framework/Versions/C/Foundation +0x00007ff82458d000 /usr/lib/libSystem.B.dylib +0x00007ff81cca4000 /System/Library/PrivateFrameworks/UIFoundation.framework/Versions/A/UIFoundation +0x00007ff82a4c0000 /System/Library/PrivateFrameworks/RemoteViewServices.framework/Versions/A/RemoteViewServices +0x00007ff822860000 /System/Library/PrivateFrameworks/XCTTargetBootstrap.framework/Versions/A/XCTTargetBootstrap +0x00007ff826459000 /System/Library/PrivateFrameworks/InternationalSupport.framework/Versions/A/InternationalSupport +0x00007ff8264e3000 /System/Library/PrivateFrameworks/UserActivity.framework/Versions/A/UserActivity +0x00007ff825840000 /usr/lib/libspindump.dylib +0x00007ff81cf32000 /System/Library/Frameworks/UniformTypeIdentifiers.framework/Versions/A/UniformTypeIdentifiers +0x00007ff820f99000 /usr/lib/libapp_launch_measurement.dylib +0x00007ff81fd4d000 /System/Library/PrivateFrameworks/CoreAnalytics.framework/Versions/A/CoreAnalytics +0x00007ff820f9c000 /System/Library/PrivateFrameworks/CoreAutoLayout.framework/Versions/A/CoreAutoLayout +0x00007ff8221bc000 /System/Library/Frameworks/Metal.framework/Versions/A/Metal +0x00007ff822ed8000 /usr/lib/liblangid.dylib +0x00007ff822864000 /System/Library/PrivateFrameworks/CoreSVG.framework/Versions/A/CoreSVG +0x00007ff81e2f9000 /System/Library/PrivateFrameworks/SkyLight.framework/Versions/A/SkyLight +0x00007ff81e68f000 /System/Library/Frameworks/CoreGraphics.framework/Versions/A/CoreGraphics +0x00007ff82ab2f000 /System/Library/Frameworks/Accelerate.framework/Versions/A/Accelerate +0x00007ff8252bb000 /System/Library/PrivateFrameworks/IconServices.framework/Versions/A/IconServices +0x00007ff8221a1000 /System/Library/Frameworks/IOSurface.framework/Versions/A/IOSurface +0x00007ff81fd7b000 /usr/lib/libDiagnosticMessagesClient.dylib +0x00007ff8244a7000 /usr/lib/libz.1.dylib +0x00007ff82e1a0000 /System/Library/Frameworks/ApplicationServices.framework/Versions/A/ApplicationServices +0x00007ff82284c000 /System/Library/PrivateFrameworks/DFRFoundation.framework/Versions/A/DFRFoundation +0x00007ff81b703000 /usr/lib/libicucore.A.dylib +0x00007ff82717e000 /System/Library/Frameworks/AudioToolbox.framework/Versions/A/AudioToolbox +0x00007ff826465000 /System/Library/PrivateFrameworks/DataDetectorsCore.framework/Versions/A/DataDetectorsCore +0x00007ff921d57000 /System/Library/PrivateFrameworks/TextInput.framework/Versions/A/TextInput +0x00007ff81e27d000 /usr/lib/libMobileGestalt.dylib +0x00007ff822554000 /System/Library/Frameworks/Carbon.framework/Versions/A/Frameworks/HIToolbox.framework/Versions/A/HIToolbox +0x00007ff8209c6000 /System/Library/Frameworks/QuartzCore.framework/Versions/A/QuartzCore +0x00007ff81b3a3000 /System/Library/Frameworks/Security.framework/Versions/A/Security +0x00007ff82a4f8000 /System/Library/Frameworks/Carbon.framework/Versions/A/Frameworks/SpeechRecognition.framework/Versions/A/SpeechRecognition +0x00007ff820cf9000 /System/Library/PrivateFrameworks/CoreUI.framework/Versions/A/CoreUI +0x00007ff81ac7b000 /System/Library/Frameworks/CoreAudio.framework/Versions/A/CoreAudio +0x00007ff81fe69000 /System/Library/Frameworks/DiskArbitration.framework/Versions/A/DiskArbitration +0x00007ff825c23000 /System/Library/PrivateFrameworks/MultitouchSupport.framework/Versions/A/MultitouchSupport +0x00007ff81e27c000 /usr/lib/libenergytrace.dylib +0x00007ff81bcfe000 /System/Library/Frameworks/IOKit.framework/Versions/A/IOKit +0x00007ff82a8e4000 /System/Library/Frameworks/CoreServices.framework/Versions/A/CoreServices +0x00007ff820f33000 /System/Library/PrivateFrameworks/PerformanceAnalysis.framework/Versions/A/PerformanceAnalysis +0x00007ffa3183e000 /System/Library/Frameworks/OpenGL.framework/Versions/A/OpenGL +0x00007ff823ad4000 /System/Library/PrivateFrameworks/MobileKeyBag.framework/Versions/A/MobileKeyBag +0x00007ff8192c1000 /usr/lib/libobjc.A.dylib +0x00007ff81edbe000 /System/Library/Frameworks/ColorSync.framework/Versions/A/ColorSync +0x00007ff81947d000 /System/Library/Frameworks/CoreFoundation.framework/Versions/A/CoreFoundation +0x00007ff822adf000 /System/Library/Frameworks/CoreImage.framework/Versions/A/CoreImage +0x00007ff81aa91000 /System/Library/Frameworks/CoreText.framework/Versions/A/CoreText +0x00007ff822893000 /System/Library/Frameworks/ImageIO.framework/Versions/A/ImageIO +0x00007ff824593000 /System/Library/PrivateFrameworks/SoftLinking.framework/Versions/A/SoftLinking +0x00007ff820fe4000 /usr/lib/libxml2.2.dylib +0x00007ff81938a000 /usr/lib/libc++.1.dylib +0x00007ff824814000 /usr/lib/libcompression.dylib +0x00007ff82638f000 /System/Library/PrivateFrameworks/TextureIO.framework/Versions/A/TextureIO +0x00007ff824f34000 /usr/lib/libate.dylib +0x00007ff824588000 /usr/lib/system/libcache.dylib +0x00007ff824544000 /usr/lib/system/libcommonCrypto.dylib +0x00007ff82456d000 /usr/lib/system/libcompiler_rt.dylib +0x00007ff824563000 /usr/lib/system/libcopyfile.dylib +0x00007ff8191bb000 /usr/lib/system/libcorecrypto.dylib +0x00007ff81927a000 /usr/lib/system/libdispatch.dylib +0x00007ff81943c000 /usr/lib/system/libdyld.dylib +0x00007ff82457f000 /usr/lib/system/libkeymgr.dylib +0x00007ff824522000 /usr/lib/system/libmacho.dylib +0x00007ff823ba8000 /usr/lib/system/libquarantine.dylib +0x00007ff82457d000 /usr/lib/system/libremovefile.dylib +0x00007ff81e2c9000 /usr/lib/system/libsystem_asl.dylib +0x00007ff819169000 /usr/lib/system/libsystem_blocks.dylib +0x00007ff819301000 /usr/lib/system/libsystem_c.dylib +0x00007ff824575000 /usr/lib/system/libsystem_collections.dylib +0x00007ff822ec9000 /usr/lib/system/libsystem_configuration.dylib +0x00007ff822184000 /usr/lib/system/libsystem_containermanager.dylib +0x00007ff824259000 /usr/lib/system/libsystem_coreservices.dylib +0x00007ff81b98c000 /usr/lib/system/libsystem_darwin.dylib +0x00007ff824580000 /usr/lib/system/libsystem_dnssd.dylib +0x00007ff8192fe000 /usr/lib/system/libsystem_featureflags.dylib +0x00007ff819452000 /usr/lib/system/libsystem_info.dylib +0x00007ff8244ba000 /usr/lib/system/libsystem_m.dylib +0x00007ff81924e000 /usr/lib/system/libsystem_malloc.dylib +0x00007ff81e266000 /usr/lib/system/libsystem_networkextension.dylib +0x00007ff81bdb4000 /usr/lib/system/libsystem_notify.dylib +0x00007ff82a7b9000 /usr/lib/system/libsystem_product_info_filter.dylib +0x00007ff822ecd000 /usr/lib/system/libsystem_sandbox.dylib +0x00007ff82457a000 /usr/lib/system/libsystem_secinit.dylib +0x00007ff8193f9000 /usr/lib/system/libsystem_kernel.dylib +0x00007ff819448000 /usr/lib/system/libsystem_platform.dylib +0x00007ff819430000 /usr/lib/system/libsystem_pthread.dylib +0x00007ff81fb94000 /usr/lib/system/libsystem_symptoms.dylib +0x00007ff8191a2000 /usr/lib/system/libsystem_trace.dylib +0x00007ff824550000 /usr/lib/system/libunwind.dylib +0x00007ff81916b000 /usr/lib/system/libxpc.dylib +0x00007ff8193e3000 /usr/lib/libc++abi.dylib +0x00007ff82455b000 /usr/lib/liboah.dylib +0x00007ff824ced000 /usr/lib/liblzma.5.dylib +0x00007ff819f3a000 /System/Library/Frameworks/SystemConfiguration.framework/Versions/A/SystemConfiguration +0x00007ff82458f000 /usr/lib/libfakelink.dylib +0x00007ff81ddcf000 /System/Library/Frameworks/CFNetwork.framework/Versions/A/CFNetwork +0x00007ff8246ce000 /usr/lib/libarchive.2.dylib +0x00007ff822ed4000 /System/Library/PrivateFrameworks/AppleSystemInfo.framework/Versions/A/AppleSystemInfo +0x00007ff823bcf000 /usr/lib/libbsm.0.dylib +0x00007ff81d405000 /usr/lib/libnetwork.dylib +0x00007ff824528000 /usr/lib/system/libkxld.dylib +0x00007ff8237fd000 /System/Library/PrivateFrameworks/IOMobileFramebuffer.framework/Versions/A/IOMobileFramebuffer +0x00007ffb2f4aa000 /usr/lib/libCoreEntitlements.dylib +0x00007ff82423b000 /System/Library/PrivateFrameworks/AppleFSCompression.framework/Versions/A/AppleFSCompression +0x00007ff823bb7000 /usr/lib/libcoretls.dylib +0x00007ff824d07000 /usr/lib/libcoretls_cfhelpers.dylib +0x00007ff82480f000 /usr/lib/libpam.2.dylib +0x00007ff81f7ba000 /usr/lib/libsqlite3.dylib +0x00007ff824d76000 /usr/lib/libxar.1.dylib +0x00007ff82424b000 /usr/lib/libbz2.1.0.dylib +0x00007ff824594000 /usr/lib/libpcap.A.dylib +0x00007ff81fb8b000 /usr/lib/libdns_services.dylib +0x00007ff8247e1000 /usr/lib/libapple_nghttp2.dylib +0x00007ff8245cb000 /usr/lib/libiconv.2.dylib +0x00007ff824521000 /usr/lib/libcharset.1.dylib +0x00007ff820f67000 /System/Library/Frameworks/CoreServices.framework/Versions/A/Frameworks/FSEvents.framework/Versions/A/FSEvents +0x00007ff81b996000 /System/Library/Frameworks/CoreServices.framework/Versions/A/Frameworks/CarbonCore.framework/Versions/A/CarbonCore +0x00007ff81fdce000 /System/Library/Frameworks/CoreServices.framework/Versions/A/Frameworks/Metadata.framework/Versions/A/Metadata +0x00007ff82425e000 /System/Library/Frameworks/CoreServices.framework/Versions/A/Frameworks/OSServices.framework/Versions/A/OSServices +0x00007ff824755000 /System/Library/Frameworks/CoreServices.framework/Versions/A/Frameworks/SearchKit.framework/Versions/A/SearchKit +0x00007ff81fb1d000 /System/Library/Frameworks/CoreServices.framework/Versions/A/Frameworks/AE.framework/Versions/A/AE +0x00007ff81997e000 /System/Library/Frameworks/CoreServices.framework/Versions/A/Frameworks/LaunchServices.framework/Versions/A/LaunchServices +0x00007ff824c9e000 /System/Library/Frameworks/CoreServices.framework/Versions/A/Frameworks/DictionaryServices.framework/Versions/A/DictionaryServices +0x00007ff820f74000 /System/Library/Frameworks/CoreServices.framework/Versions/A/Frameworks/SharedFileList.framework/Versions/A/SharedFileList +0x00007ff823bab000 /usr/lib/libCheckFix.dylib +0x00007ff81e2e0000 /System/Library/PrivateFrameworks/TCC.framework/Versions/A/TCC +0x00007ff822eda000 /System/Library/PrivateFrameworks/CoreNLP.framework/Versions/A/CoreNLP +0x00007ff81fd7e000 /System/Library/PrivateFrameworks/MetadataUtilities.framework/Versions/A/MetadataUtilities +0x00007ff819fea000 /usr/lib/libmecabra.dylib +0x00007ff823bdf000 /usr/lib/libmecab.dylib +0x00007ff819fb5000 /usr/lib/libCRFSuite.dylib +0x00007ff823c3c000 /usr/lib/libgermantok.dylib +0x00007ff8247bc000 /usr/lib/libThaiTokenizer.dylib +0x00007ff81fe70000 /System/Library/Frameworks/Accelerate.framework/Versions/A/Frameworks/vImage.framework/Versions/A/vImage +0x00007ff82a8bb000 /System/Library/Frameworks/Accelerate.framework/Versions/A/Frameworks/vecLib.framework/Versions/A/vecLib +0x00007ff824db7000 /System/Library/Frameworks/Accelerate.framework/Versions/A/Frameworks/vecLib.framework/Versions/A/libvMisc.dylib +0x00007ff8235f1000 /System/Library/Frameworks/Accelerate.framework/Versions/A/Frameworks/vecLib.framework/Versions/A/libvDSP.dylib +0x00007ff819cf6000 /System/Library/Frameworks/Accelerate.framework/Versions/A/Frameworks/vecLib.framework/Versions/A/libBLAS.dylib +0x00007ff8248f1000 /System/Library/Frameworks/Accelerate.framework/Versions/A/Frameworks/vecLib.framework/Versions/A/libLAPACK.dylib +0x00007ff823c3f000 /System/Library/Frameworks/Accelerate.framework/Versions/A/Frameworks/vecLib.framework/Versions/A/libLinearAlgebra.dylib +0x00007ff8247fa000 /System/Library/Frameworks/Accelerate.framework/Versions/A/Frameworks/vecLib.framework/Versions/A/libSparseBLAS.dylib +0x00007ff8248eb000 /System/Library/Frameworks/Accelerate.framework/Versions/A/Frameworks/vecLib.framework/Versions/A/libQuadrature.dylib +0x00007ff822fc4000 /System/Library/Frameworks/Accelerate.framework/Versions/A/Frameworks/vecLib.framework/Versions/A/libBNNS.dylib +0x00007ff819ecb000 /System/Library/Frameworks/Accelerate.framework/Versions/A/Frameworks/vecLib.framework/Versions/A/libSparse.dylib +0x00007ff822fbd000 /System/Library/PrivateFrameworks/LinguisticData.framework/Versions/A/LinguisticData +0x00007ff819e80000 /System/Library/PrivateFrameworks/Lexicon.framework/Versions/A/Lexicon +0x00007ff824db0000 /usr/lib/libChineseTokenizer.dylib +0x00007ff81a5e4000 /System/Library/PrivateFrameworks/LanguageModeling.framework/Versions/A/LanguageModeling +0x00007ff8247be000 /System/Library/PrivateFrameworks/AppleSauce.framework/Versions/A/AppleSauce +0x00007ff8237eb000 /System/Library/PrivateFrameworks/CoreEmoji.framework/Versions/A/CoreEmoji +0x00007ff8246bd000 /usr/lib/libcmph.dylib +0x00007ff820f4b000 /System/Library/Frameworks/OpenDirectory.framework/Versions/A/Frameworks/CFOpenDirectory.framework/Versions/A/CFOpenDirectory +0x00007ff820f3f000 /System/Library/Frameworks/OpenDirectory.framework/Versions/A/OpenDirectory +0x00007ff824d09000 /System/Library/PrivateFrameworks/APFS.framework/Versions/A/APFS +0x00007ff823b0c000 /System/Library/Frameworks/SecurityFoundation.framework/Versions/A/SecurityFoundation +0x00007ff824d84000 /usr/lib/libutil.dylib +0x00007ff81bcc1000 /System/Library/PrivateFrameworks/CoreServicesStore.framework/Versions/A/CoreServicesStore +0x00007ff824d88000 /usr/lib/libxslt.1.dylib +0x00007ff823b99000 /System/Library/PrivateFrameworks/BackgroundTaskManagement.framework/Versions/A/BackgroundTaskManagement +0x00007ff8262b1000 /System/Library/Frameworks/ImageIO.framework/Versions/A/Resources/libRadiance.dylib +0x00007ff8262ba000 /System/Library/Frameworks/ImageIO.framework/Versions/A/Resources/libJPEG.dylib +0x00007ff82620b000 /System/Library/Frameworks/ImageIO.framework/Versions/A/Resources/libPng.dylib +0x00007ff82622c000 /System/Library/Frameworks/ImageIO.framework/Versions/A/Resources/libTIFF.dylib +0x00007ff826318000 /System/Library/Frameworks/ImageIO.framework/Versions/A/Resources/libGIF.dylib +0x00007ff825b40000 /System/Library/Frameworks/ImageIO.framework/Versions/A/Resources/libJP2.dylib +0x00007ff825282000 /usr/lib/libexpat.1.dylib +0x00007ff825af5000 /System/Library/PrivateFrameworks/AppleJPEG.framework/Versions/A/AppleJPEG +0x00007ff822417000 /System/Library/PrivateFrameworks/FontServices.framework/libFontParser.dylib +0x00007ff81fac7000 /System/Library/PrivateFrameworks/RunningBoardServices.framework/Versions/A/RunningBoardServices +0x00007ff830a90000 /System/Library/PrivateFrameworks/IOSurfaceAccelerator.framework/Versions/A/IOSurfaceAccelerator +0x00007ff825c1f000 /System/Library/PrivateFrameworks/WatchdogClient.framework/Versions/A/WatchdogClient +0x00007ff81a6d3000 /System/Library/Frameworks/CoreDisplay.framework/Versions/A/CoreDisplay +0x00007ff822316000 /System/Library/Frameworks/CoreMedia.framework/Versions/A/CoreMedia +0x00007ff8221b3000 /System/Library/PrivateFrameworks/IOAccelerator.framework/Versions/A/IOAccelerator +0x00007ff8210c7000 /System/Library/Frameworks/CoreVideo.framework/Versions/A/CoreVideo +0x00007ff82480d000 /System/Library/Frameworks/MetalPerformanceShaders.framework/Versions/A/MetalPerformanceShaders +0x00007ff825c5a000 /System/Library/Frameworks/VideoToolbox.framework/Versions/A/VideoToolbox +0x00007ff81fa0f000 /System/Library/PrivateFrameworks/BaseBoard.framework/Versions/A/BaseBoard +0x00007ff81f7a0000 /System/Library/PrivateFrameworks/ProtocolBuffer.framework/Versions/A/ProtocolBuffer +0x00007ff8262aa000 /System/Library/PrivateFrameworks/GPUWrangler.framework/Versions/A/GPUWrangler +0x00007ff82628d000 /System/Library/PrivateFrameworks/IOPresentment.framework/Versions/A/IOPresentment +0x00007ff8262b4000 /System/Library/PrivateFrameworks/DSExternalDisplay.framework/Versions/A/DSExternalDisplay +0x00007ffa31832000 /System/Library/Frameworks/OpenGL.framework/Versions/A/Libraries/libCoreFSCache.dylib +0x00007ffb1d71a000 /System/Library/PrivateFrameworks/GPUCompiler.framework/Versions/31001/Libraries/libGPUCompilerUtils.dylib +0x00007ff82631d000 /System/Library/PrivateFrameworks/CMCaptureCore.framework/Versions/A/CMCaptureCore +0x00007ff82582e000 /System/Library/PrivateFrameworks/AppServerSupport.framework/Versions/A/AppServerSupport +0x00007ff827a04000 /System/Library/PrivateFrameworks/perfdata.framework/Versions/A/perfdata +0x00007ff81a7ff000 /System/Library/PrivateFrameworks/AudioToolboxCore.framework/Versions/A/AudioToolboxCore +0x00007ff8222f3000 /System/Library/PrivateFrameworks/caulk.framework/Versions/A/caulk +0x00007ff827328000 /usr/lib/libAudioStatistics.dylib +0x00007ff91e5e5000 /System/Library/PrivateFrameworks/SystemPolicy.framework/Versions/A/SystemPolicy +0x00007ff82757d000 /usr/lib/libSMC.dylib +0x00007ff8303c7000 /System/Library/Frameworks/CoreMIDI.framework/Versions/A/CoreMIDI +0x00007ff8261db000 /usr/lib/libAudioToolboxUtility.dylib +0x00007ff91a9f7000 /System/Library/PrivateFrameworks/OSAServicesClient.framework/Versions/A/OSAServicesClient +0x00007ff827a11000 /usr/lib/libperfcheck.dylib +0x00007ff91f077000 /usr/lib/libmis.dylib +0x00007ffa3188e000 /System/Library/Frameworks/OpenGL.framework/Versions/A/Libraries/libGLU.dylib +0x00007ffa31851000 /System/Library/Frameworks/OpenGL.framework/Versions/A/Libraries/libGFXShared.dylib +0x00007ffa31a5f000 /System/Library/Frameworks/OpenGL.framework/Versions/A/Libraries/libGL.dylib +0x00007ffa3185a000 /System/Library/Frameworks/OpenGL.framework/Versions/A/Libraries/libGLImage.dylib +0x00007ffa3184e000 /System/Library/Frameworks/OpenGL.framework/Versions/A/Libraries/libCVMSPluginSupport.dylib +0x00007ffa31839000 /System/Library/Frameworks/OpenGL.framework/Versions/A/Libraries/libCoreVMClient.dylib +0x00007ff822e58000 /System/Library/Frameworks/MetalPerformanceShaders.framework/Versions/A/Frameworks/MPSCore.framework/Versions/A/MPSCore +0x00007ff8241a6000 /System/Library/Frameworks/MetalPerformanceShaders.framework/Versions/A/Frameworks/MPSImage.framework/Versions/A/MPSImage +0x00007ff823c55000 /System/Library/Frameworks/MetalPerformanceShaders.framework/Versions/A/Frameworks/MPSNeuralNetwork.framework/Versions/A/MPSNeuralNetwork +0x00007ff8240ca000 /System/Library/Frameworks/MetalPerformanceShaders.framework/Versions/A/Frameworks/MPSMatrix.framework/Versions/A/MPSMatrix +0x00007ff823e86000 /System/Library/Frameworks/MetalPerformanceShaders.framework/Versions/A/Frameworks/MPSRayIntersector.framework/Versions/A/MPSRayIntersector +0x00007ff824103000 /System/Library/Frameworks/MetalPerformanceShaders.framework/Versions/A/Frameworks/MPSNDArray.framework/Versions/A/MPSNDArray +0x00007ffa36453000 /System/Library/Frameworks/MetalPerformanceShaders.framework/Versions/A/Frameworks/MPSFunctions.framework/Versions/A/MPSFunctions +0x00007ff819bd9000 /System/Library/PrivateFrameworks/MetalTools.framework/Versions/A/MetalTools +0x00007ff822ed2000 /System/Library/PrivateFrameworks/AggregateDictionary.framework/Versions/A/AggregateDictionary +0x00007ff825064000 /usr/lib/libIOReport.dylib +0x00007ffa32e6e000 /System/Library/Frameworks/OpenCL.framework/Versions/A/OpenCL +0x00007ff8253c8000 /System/Library/PrivateFrameworks/FaceCore.framework/Versions/A/FaceCore +0x00007ff8253b9000 /System/Library/PrivateFrameworks/GraphVisualizer.framework/Versions/A/GraphVisualizer +0x00007ffb1d625000 /System/Library/PrivateFrameworks/FontServices.framework/Versions/A/FontServices +0x00007ff8257e5000 /System/Library/PrivateFrameworks/OTSVG.framework/Versions/A/OTSVG +0x00007ff820caf000 /System/Library/Frameworks/ApplicationServices.framework/Versions/A/Frameworks/ATS.framework/Versions/A/Resources/libFontRegistry.dylib +0x00007ff825836000 /System/Library/PrivateFrameworks/FontServices.framework/libhvf.dylib +0x00007ffb1d626000 /System/Library/PrivateFrameworks/FontServices.framework/libXTFontStaticRegistryData.dylib +0x00007ff825232000 /System/Library/PrivateFrameworks/AppleVA.framework/Versions/A/AppleVA +0x00007ff82734c000 /System/Library/Frameworks/ApplicationServices.framework/Versions/A/Frameworks/ATS.framework/Versions/A/ATS +0x00007ff81eeba000 /System/Library/Frameworks/ApplicationServices.framework/Versions/A/Frameworks/HIServices.framework/Versions/A/HIServices +0x00007ff826327000 /System/Library/Frameworks/ApplicationServices.framework/Versions/A/Frameworks/PrintCore.framework/Versions/A/PrintCore +0x00007ff8276d0000 /System/Library/Frameworks/ApplicationServices.framework/Versions/A/Frameworks/QD.framework/Versions/A/QD +0x00007ff8276c8000 /System/Library/Frameworks/ApplicationServices.framework/Versions/A/Frameworks/ColorSyncLegacy.framework/Versions/A/ColorSyncLegacy +0x00007ff82733c000 /System/Library/Frameworks/ApplicationServices.framework/Versions/A/Frameworks/SpeechSynthesis.framework/Versions/A/SpeechSynthesis +0x00007ff8262e8000 /System/Library/Frameworks/ApplicationServices.framework/Versions/A/Frameworks/ATSUI.framework/Versions/A/ATSUI +0x00007ff82765d000 /usr/lib/libcups.2.dylib +0x00007ff827a20000 /System/Library/Frameworks/Kerberos.framework/Versions/A/Kerberos +0x00007ff827a2f000 /System/Library/Frameworks/GSS.framework/Versions/A/GSS +0x00007ff8273b8000 /usr/lib/libresolv.9.dylib +0x00007ff825845000 /System/Library/PrivateFrameworks/Heimdal.framework/Versions/A/Heimdal +0x00007ff82e51d000 /System/Library/Frameworks/Kerberos.framework/Versions/A/Libraries/libHeimdalProxy.dylib +0x00007ff81fb9c000 /System/Library/Frameworks/Network.framework/Versions/A/Network +0x00007ff82529c000 /usr/lib/libheimdal-asn1.dylib +0x00007ff827a7b000 /System/Library/PrivateFrameworks/CommonAuth.framework/Versions/A/CommonAuth +0x00007ff8272b9000 /System/Library/PrivateFrameworks/AudioSession.framework/Versions/A/AudioSession +0x00007ff82532c000 /System/Library/PrivateFrameworks/MediaExperience.framework/Versions/A/MediaExperience +0x00007ff82714d000 /System/Library/PrivateFrameworks/AudioSession.framework/libSessionUtility.dylib +0x00007ff8276dc000 /System/Library/PrivateFrameworks/AudioResourceArbitration.framework/Versions/A/AudioResourceArbitration +0x00007ff82b6ce000 /System/Library/PrivateFrameworks/PowerLog.framework/Versions/A/PowerLog +0x00007ff8252a5000 /System/Library/PrivateFrameworks/IconFoundation.framework/Versions/A/IconFoundation +0x00007ff82a4e8000 /System/Library/PrivateFrameworks/SpeechRecognitionCore.framework/Versions/A/SpeechRecognitionCore +0x000000010edf6000 /Users/sagarrao/Library/Java/JavaVirtualMachines/temurin-1.8.0_322/Contents/Home/jre/lib/server/libjvm.dylib +0x0000000108c2a000 /Users/sagarrao/Library/Java/JavaVirtualMachines/temurin-1.8.0_322/Contents/Home/jre/lib/libverify.dylib +0x0000000108c95000 /Users/sagarrao/Library/Java/JavaVirtualMachines/temurin-1.8.0_322/Contents/Home/jre/lib/libjava.dylib +0x0000000108d36000 /Users/sagarrao/Library/Java/JavaVirtualMachines/temurin-1.8.0_322/Contents/Home/jre/lib/libzip.dylib +0x000000010ecee000 /Users/sagarrao/Library/Java/JavaVirtualMachines/temurin-1.8.0_322/Contents/Home/jre/lib/libnio.dylib +0x000000010ed44000 /Users/sagarrao/Library/Java/JavaVirtualMachines/temurin-1.8.0_322/Contents/Home/jre/lib/libnet.dylib +0x0000000127213000 /Users/sagarrao/Library/Java/JavaVirtualMachines/temurin-1.8.0_322/Contents/Home/jre/lib/libmanagement.dylib +0x000000013243c000 /Users/sagarrao/Library/Java/JavaVirtualMachines/temurin-1.8.0_322/Contents/Home/jre/lib/libsunec.dylib + +VM Arguments: +jvm_args: -Djava.security.manager=worker.org.gradle.process.internal.worker.child.BootstrapSecurityManager -Dorg.gradle.internal.worker.tmpdir=/Users/sagarrao/gitprojects/kafka/connect/runtime/build/tmp/test/work -Dorg.gradle.native=false -Xss4m -XX:+UseParallelGC -Xmx2g -Dfile.encoding=UTF-8 -Duser.country=IN -Duser.language=en -Duser.variant -ea +java_command: worker.org.gradle.process.internal.worker.GradleWorkerMain 'Gradle Test Executor 59' +java_class_path (initial): /Users/sagarrao/.gradle/caches/7.5.1/workerMain/gradle-worker.jar +Launcher Type: SUN_STANDARD + +Environment Variables: +PATH=/opt/homebrew/bin:/opt/homebrew/sbin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/Users/sagarrao/bin +SHELL=/bin/zsh + +Signal Handlers: +SIGSEGV: [libjvm.dylib+0x59c573], sa_mask[0]=11111111111111111111111111111111, sa_flags=SA_ONSTACK|SA_RESTART|SA_SIGINFO +SIGBUS: [libjvm.dylib+0x59c573], sa_mask[0]=11111111111111111111111111111111, sa_flags=SA_RESTART|SA_SIGINFO +SIGFPE: [libjvm.dylib+0x48f207], sa_mask[0]=11111111111111111111111111111111, sa_flags=SA_RESTART|SA_SIGINFO +SIGPIPE: [libjvm.dylib+0x48f207], sa_mask[0]=11111111111111111111111111111111, sa_flags=SA_RESTART|SA_SIGINFO +SIGXFSZ: [libjvm.dylib+0x48f207], sa_mask[0]=11111111111111111111111111111111, sa_flags=SA_RESTART|SA_SIGINFO +SIGILL: [libjvm.dylib+0x48f207], sa_mask[0]=11111111111111111111111111111111, sa_flags=SA_RESTART|SA_SIGINFO +SIGUSR1: SIG_DFL, sa_mask[0]=00000000000000000000000000000000, sa_flags=none +SIGUSR2: [libjvm.dylib+0x48fafe], sa_mask[0]=00100000000000000000000000000000, sa_flags=SA_RESTART|SA_SIGINFO +SIGHUP: [libjvm.dylib+0x48dcf1], sa_mask[0]=11111111111111111111111111111111, sa_flags=SA_RESTART|SA_SIGINFO +SIGINT: [libjvm.dylib+0x48dcf1], sa_mask[0]=11111111111111111111111111111111, sa_flags=SA_RESTART|SA_SIGINFO +SIGTERM: [libjvm.dylib+0x48dcf1], sa_mask[0]=11111111111111111111111111111111, sa_flags=SA_RESTART|SA_SIGINFO +SIGQUIT: [libjvm.dylib+0x48dcf1], sa_mask[0]=11111111111111111111111111111111, sa_flags=SA_RESTART|SA_SIGINFO + + +--------------- S Y S T E M --------------- + +OS:Bsduname:Darwin 21.3.0 Darwin Kernel Version 21.3.0: Wed Jan 5 21:37:58 PST 2022; root:xnu-8019.80.24~20/RELEASE_ARM64_T6000 x86_64 +rlimit: STACK 8176k, CORE 0k, NPROC 2666, NOFILE 10240, AS infinity +load average:22.63 15.85 14.55 + +CPU:total 10 (initial active 10) (1 cores per cpu, 1 threads per core) family 6 model 44 stepping 0, cmov, cx8, fxsr, mmx, sse, sse2, sse3, ssse3, sse4.1, sse4.2, popcnt, aes, clmul, tsc, tscinvbit, tscinv + +Memory: 4k page, physical 16777216k(18992k free) + +/proc/meminfo: + + +vm_info: OpenJDK 64-Bit Server VM (25.322-b06) for bsd-amd64 JRE (1.8.0_322-b06), built on Jan 19 2022 13:09:06 by "jenkins" with gcc 4.2.1 Compatible Apple LLVM 10.0.1 (clang-1001.0.46.4) + +time: Wed Sep 14 22:15:10 2022 +timezone: IST +elapsed time: 54.145735 seconds (0d 0h 0m 54s) + diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java index 0bf923d2b3534..7d6959356772b 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java @@ -107,7 +107,7 @@ public void testTaskAssignmentWhenWorkerJoins() { // Third assignment after revocations performStandardRebalance(); - assertDelay(assignor.delay); // Sucessive revocation so delay should be non-zero + assertTrue(assignor.delay > 0); // Sucessive revocation so delay should be non-zero assertConnectorAllocations(1, 1); assertTaskAllocations(4, 4); assertBalancedAndCompleteAllocation(); @@ -438,7 +438,7 @@ public void testTaskAssignmentWhenLeaderBounces() { // Fourth assignment after revocations performStandardRebalance(); - assertDelay(assignor.delay); // Successive revoking rebalance. Should introduce a delay + assertTrue(assignor.delay > 0); // Successive revoking rebalance. Should introduce a delay assertConnectorAllocations(0, 1, 1); assertTaskAllocations(2, 3, 3); assertBalancedAndCompleteAllocation(); diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinatorIncrementalTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinatorIncrementalTest.java index f8cf14200ca41..d8fb647201873 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinatorIncrementalTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinatorIncrementalTest.java @@ -517,13 +517,13 @@ public void testTaskAssignmentWhenWorkerBounces() { leaderAssignment = deserializeAssignment(result, leaderId); assertAssignment(leaderId, offset, Collections.emptyList(), 0, - Collections.emptyList(), 0, + Collections.emptyList(), 1, leaderAssignment); memberAssignment = deserializeAssignment(result, memberId); assertAssignment(leaderId, offset, Collections.emptyList(), 0, - Collections.emptyList(), 0, + Collections.emptyList(), 1, memberAssignment); anotherMemberAssignment = deserializeAssignment(result, anotherMemberId); From f4bfc0c04da999cdc25b069dd62cf9f25219486c Mon Sep 17 00:00:00 2001 From: Sagar Rao Date: Thu, 15 Sep 2022 08:01:19 +0530 Subject: [PATCH 14/26] Removing unwanted file --- connect/runtime/hs_err_pid77304.log | 676 ---------------------------- 1 file changed, 676 deletions(-) delete mode 100644 connect/runtime/hs_err_pid77304.log diff --git a/connect/runtime/hs_err_pid77304.log b/connect/runtime/hs_err_pid77304.log deleted file mode 100644 index e05f2de864cb2..0000000000000 --- a/connect/runtime/hs_err_pid77304.log +++ /dev/null @@ -1,676 +0,0 @@ -# -# A fatal error has been detected by the Java Runtime Environment: -# -# Internal Error (sharedRuntime.cpp:553), pid=77304, tid=0x0000000000028a07 -# guarantee(cb != NULL && cb->is_nmethod()) failed: safepoint polling: pc must refer to an nmethod -# -# JRE version: OpenJDK Runtime Environment (8.0_322-b06) (build 1.8.0_322-b06) -# Java VM: OpenJDK 64-Bit Server VM (25.322-b06 mixed mode bsd-amd64 compressed oops) -# Failed to write core dump. Core dumps have been disabled. To enable core dumping, try "ulimit -c unlimited" before starting Java again -# -# If you would like to submit a bug report, please visit: -# https://github.com/adoptium/adoptium-support/issues -# - ---------------- T H R E A D --------------- - -Current thread (0x00007fba832e4800): JavaThread "qtp476631259-196-acceptor-0@642d7605-http_localhost0@9be7319{HTTP/1.1, (http/1.1)}{localhost:64232}" [_thread_in_Java, id=166407, stack(0x000000031dc6a000,0x000000031e06a000)] - -Stack: [0x000000031dc6a000,0x000000031e06a000], sp=0x000000031e066740, free space=4081k -Native frames: (J=compiled Java code, j=interpreted, Vv=VM code, C=native code) -V [libjvm.dylib+0x59baae] VMError::report_and_die()+0x440 -V [libjvm.dylib+0x1d2e38] report_vm_error(char const*, int, char const*, char const*)+0x55 -V [libjvm.dylib+0x4ed6d7] SharedRuntime::get_poll_stub(unsigned char*)+0x43 -V [libjvm.dylib+0x491961] JVM_handle_bsd_signal+0x290 -V [libjvm.dylib+0x48f234] signalHandler(int, __siginfo*, void*)+0x2d -C [libsystem_platform.dylib+0x3e2d] _sigtramp+0x1d -C 0x0000000000000000 -v ~StubRoutines::call_stub -V [libjvm.dylib+0x2c9ed1] JavaCalls::call_helper(JavaValue*, methodHandle*, JavaCallArguments*, Thread*)+0x6b7 -V [libjvm.dylib+0x2fc828] jni_invoke_nonstatic(JNIEnv_*, JavaValue*, _jobject*, JNICallType, _jmethodID*, JNI_ArgumentPusher*, Thread*)+0x3f9 -V [libjvm.dylib+0x2fcd5c] jni_NewObject+0x22f -C [libjava.dylib+0xde01] JNU_NewStringPlatform+0x266 -C [libjava.dylib+0xe108] JNU_ThrowByNameWithLastError+0x4b -C [libnio.dylib+0x3b7d] Java_sun_nio_ch_ServerSocketChannelImpl_accept0+0x16f -j sun.nio.ch.ServerSocketChannelImpl.accept0(Ljava/io/FileDescriptor;Ljava/io/FileDescriptor;[Ljava/net/InetSocketAddress;)I+0 -j sun.nio.ch.ServerSocketChannelImpl.accept(Ljava/io/FileDescriptor;Ljava/io/FileDescriptor;[Ljava/net/InetSocketAddress;)I+4 -j sun.nio.ch.ServerSocketChannelImpl.accept()Ljava/nio/channels/SocketChannel;+130 -j org.eclipse.jetty.server.ServerConnector.accept(I)V+17 -j org.eclipse.jetty.server.AbstractConnector$Acceptor.run()V+237 -j org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(Ljava/lang/Runnable;)V+1 -j org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run()V+262 -j java.lang.Thread.run()V+11 -v ~StubRoutines::call_stub -V [libjvm.dylib+0x2c9ed1] JavaCalls::call_helper(JavaValue*, methodHandle*, JavaCallArguments*, Thread*)+0x6b7 -V [libjvm.dylib+0x2c8cdd] JavaCalls::call_virtual(JavaValue*, KlassHandle, Symbol*, Symbol*, JavaCallArguments*, Thread*)+0x159 -V [libjvm.dylib+0x2c8eb9] JavaCalls::call_virtual(JavaValue*, Handle, KlassHandle, Symbol*, Symbol*, Thread*)+0x57 -V [libjvm.dylib+0x340bc3] thread_entry(JavaThread*, Thread*)+0x78 -V [libjvm.dylib+0x55ee9c] JavaThread::thread_main_inner()+0x82 -V [libjvm.dylib+0x55ed61] JavaThread::run()+0x169 -V [libjvm.dylib+0x48cd9b] java_start(Thread*)+0xfa -C [libsystem_pthread.dylib+0x64f4] _pthread_start+0x7d -C [libsystem_pthread.dylib+0x200f] thread_start+0xf -C 0x0000000000000000 - - ---------------- P R O C E S S --------------- - -Java Threads: ( => current thread ) - 0x00007fba9245e800 JavaThread "Keep-Alive-Timer" daemon [_thread_blocked, id=99959, stack(0x00000003224a0000,0x00000003228a0000)] - 0x00007fba92050800 JavaThread "qtp476631259-211" [_thread_in_vm, id=100099, stack(0x000000032209d000,0x000000032249d000)] - 0x00007fba86240800 JavaThread "KafkaBasedLog Work Thread - good-config" [_thread_in_native, id=163075, stack(0x0000000321c9a000,0x000000032209a000)] - 0x00007fba8626c000 JavaThread "Connector-Scheduler-9be7319-1" [_thread_blocked, id=163383, stack(0x0000000321897000,0x0000000321c97000)] - 0x00007fba92049000 JavaThread "kafka-producer-network-thread | producer-5" daemon [_thread_in_native, id=98595, stack(0x0000000321494000,0x0000000321894000)] - 0x00007fba87039800 JavaThread "KafkaBasedLog Work Thread - good-status" [_thread_in_native, id=164355, stack(0x0000000321091000,0x0000000321491000)] - 0x00007fba9204b000 JavaThread "kafka-producer-network-thread | producer-4" daemon [_thread_in_native, id=164879, stack(0x0000000320c8e000,0x000000032108e000)] - 0x00007fba91bde000 JavaThread "KafkaBasedLog Work Thread - good-offset" [_thread_in_native, id=165123, stack(0x000000032088b000,0x0000000320c8b000)] - 0x00007fba84920800 JavaThread "kafka-producer-network-thread | producer-3" daemon [_thread_in_native, id=165799, stack(0x0000000320488000,0x0000000320888000)] - 0x00007fba9204e800 JavaThread "kafka-admin-client-thread | adminclient-15" daemon [_thread_in_native, id=94063, stack(0x0000000320085000,0x0000000320485000)] - 0x00007fba8748e000 JavaThread "Session-HouseKeeper-524f5ea5-1" [_thread_blocked, id=167299, stack(0x000000031fc82000,0x0000000320082000)] - 0x00007fba9221c000 JavaThread "DistributedHerder-connect-4-1" [_thread_in_vm, id=95795, stack(0x000000031f87f000,0x000000031fc7f000)] - 0x00007fba86e24800 JavaThread "qtp476631259-198" [_thread_blocked, id=166935, stack(0x000000031f079000,0x000000031f479000)] - 0x00007fba869ad000 JavaThread "qtp476631259-197" [_thread_blocked, id=96263, stack(0x000000031e06d000,0x000000031e46d000)] -=>0x00007fba832e4800 JavaThread "qtp476631259-196-acceptor-0@642d7605-http_localhost0@9be7319{HTTP/1.1, (http/1.1)}{localhost:64232}" [_thread_in_Java, id=166407, stack(0x000000031dc6a000,0x000000031e06a000)] - 0x00007fba9245b800 JavaThread "qtp476631259-195" [_thread_in_native, id=96519, stack(0x000000031d867000,0x000000031dc67000)] - 0x00007fba91c8a000 JavaThread "qtp476631259-194" [_thread_in_native, id=96775, stack(0x000000031d464000,0x000000031d864000)] - 0x00007fba87087800 JavaThread "qtp476631259-193" [_thread_in_native, id=96007, stack(0x000000031d061000,0x000000031d461000)] - 0x00007fba8704c000 JavaThread "qtp476631259-192" [_thread_in_native, id=94743, stack(0x000000031cc5e000,0x000000031d05e000)] - 0x00007fba923a6000 JavaThread "qtp476631259-191" [_thread_in_native, id=169219, stack(0x000000031c85b000,0x000000031cc5b000)] - 0x00007fba832e3800 JavaThread "executor-Fetch" [_thread_blocked, id=93195, stack(0x000000031c458000,0x000000031c858000)] - 0x00007fba864bc800 JavaThread "KafkaBasedLog Work Thread - good-offset" [_thread_in_native, id=170051, stack(0x000000031bc52000,0x000000031c052000)] - 0x00007fba84d0e000 JavaThread "kafka-producer-network-thread | producer-2" daemon [_thread_in_native, id=94979, stack(0x000000031f47c000,0x000000031f87c000)] - 0x00007fba91d37800 JavaThread "kafka-admin-client-thread | adminclient-13" daemon [_thread_in_native, id=169815, stack(0x000000031ec76000,0x000000031f076000)] - 0x00007fba87045800 JavaThread "kafka-admin-client-thread | adminclient-11" daemon [_thread_in_native, id=92979, stack(0x000000031e873000,0x000000031ec73000)] - 0x00007fba874d2000 JavaThread "kafka-admin-client-thread | adminclient-9" daemon [_thread_in_native, id=172815, stack(0x000000031e470000,0x000000031e870000)] - 0x00007fba90828000 JavaThread "kafka-scheduler-9" daemon [_thread_blocked, id=89615, stack(0x000000031c055000,0x000000031c455000)] - 0x00007fba85bba800 JavaThread "kafka-producer-network-thread | producer-1" daemon [_thread_in_native, id=89183, stack(0x000000031b84f000,0x000000031bc4f000)] - 0x00007fba8358c800 JavaThread "data-plane-kafka-socket-acceptor-ListenerName(PLAINTEXT)-PLAINTEXT-0" [_thread_in_native, id=64771, stack(0x000000031b44c000,0x000000031b84c000)] - 0x00007fba8358b800 JavaThread "data-plane-kafka-network-thread-0-ListenerName(PLAINTEXT)-PLAINTEXT-2" [_thread_in_native, id=65283, stack(0x000000031b049000,0x000000031b449000)] - 0x00007fba8358a800 JavaThread "data-plane-kafka-network-thread-0-ListenerName(PLAINTEXT)-PLAINTEXT-1" [_thread_in_native, id=65795, stack(0x000000031ac46000,0x000000031b046000)] - 0x00007fba83451800 JavaThread "data-plane-kafka-network-thread-0-ListenerName(PLAINTEXT)-PLAINTEXT-0" [_thread_in_native, id=66323, stack(0x000000031a843000,0x000000031ac43000)] - 0x00007fba85564800 JavaThread "/config/changes-event-process-thread" [_thread_blocked, id=66687, stack(0x000000031a440000,0x000000031a840000)] - 0x00007fba843ad000 JavaThread "kafka-scheduler-0" daemon [_thread_blocked, id=66895, stack(0x000000031a03d000,0x000000031a43d000)] - 0x00007fba85550800 JavaThread "data-plane-kafka-request-handler-7" daemon [_thread_blocked, id=62979, stack(0x0000000319c3a000,0x000000031a03a000)] - 0x00007fba8554f800 JavaThread "data-plane-kafka-request-handler-6" daemon [_thread_blocked, id=62467, stack(0x0000000319837000,0x0000000319c37000)] - 0x00007fba8554e800 JavaThread "data-plane-kafka-request-handler-5" daemon [_thread_blocked, id=67331, stack(0x0000000319434000,0x0000000319834000)] - 0x00007fba8554e000 JavaThread "data-plane-kafka-request-handler-4" daemon [_thread_blocked, id=67843, stack(0x0000000319031000,0x0000000319431000)] - 0x00007fba8555b000 JavaThread "data-plane-kafka-request-handler-3" daemon [_thread_blocked, id=68099, stack(0x0000000318c2e000,0x000000031902e000)] - 0x00007fba8555a000 JavaThread "data-plane-kafka-request-handler-2" daemon [_thread_blocked, id=68611, stack(0x000000031882b000,0x0000000318c2b000)] - 0x00007fba8506c800 JavaThread "data-plane-kafka-request-handler-1" daemon [_thread_in_vm, id=61699, stack(0x0000000318428000,0x0000000318828000)] - 0x00007fba851fa000 JavaThread "data-plane-kafka-request-handler-0" daemon [_thread_blocked, id=69499, stack(0x0000000318025000,0x0000000318425000)] - 0x00007fba86899800 JavaThread "ExpirationReaper-0-AlterAcls" [_thread_blocked, id=61375, stack(0x0000000317c22000,0x0000000318022000)] - 0x00007fba843ae800 JavaThread "Controller-0-to-broker-0-send-thread" [_thread_in_native, id=60987, stack(0x000000031781f000,0x0000000317c1f000)] - 0x00007fba85ad1000 JavaThread "TxnMarkerSenderThread-0" [_thread_in_native, id=70687, stack(0x000000031741c000,0x000000031781c000)] - 0x00007fba83056800 JavaThread "transaction-log-manager-0" daemon [_thread_blocked, id=60471, stack(0x0000000317019000,0x0000000317419000)] - 0x00007fba84891000 JavaThread "group-metadata-manager-0" daemon [_thread_blocked, id=60087, stack(0x0000000316c16000,0x0000000317016000)] - 0x00007fba8406c000 JavaThread "ExpirationReaper-0-Rebalance" [_thread_blocked, id=59151, stack(0x0000000316813000,0x0000000316c13000)] - 0x00007fba8485f800 JavaThread "ExpirationReaper-0-Heartbeat" [_thread_blocked, id=58807, stack(0x0000000316410000,0x0000000316810000)] - 0x00007fba83024800 JavaThread "ExpirationReaper-0-topic" [_thread_blocked, id=72295, stack(0x000000031600d000,0x000000031640d000)] - 0x00007fba86057800 JavaThread "controller-event-thread" [_thread_blocked, id=72851, stack(0x0000000315c0a000,0x000000031600a000)] - 0x00007fba8788a800 JavaThread "kafka-scheduler-8" daemon [_thread_blocked, id=57639, stack(0x0000000315807000,0x0000000315c07000)] - 0x00007fba8484a000 JavaThread "kafka-scheduler-7" daemon [_thread_blocked, id=57347, stack(0x0000000315404000,0x0000000315804000)] - 0x00007fba851e4800 JavaThread "LogDirFailureHandler" [_thread_blocked, id=73475, stack(0x0000000315001000,0x0000000315401000)] - 0x00007fba8402e000 JavaThread "kafka-scheduler-6" daemon [_thread_blocked, id=74299, stack(0x0000000314bfe000,0x0000000314ffe000)] - 0x00007fba83023800 JavaThread "kafka-scheduler-5" daemon [_thread_blocked, id=73987, stack(0x00000003147fb000,0x0000000314bfb000)] - 0x00007fba840a8000 JavaThread "ExpirationReaper-0-ElectLeader" [_thread_blocked, id=56091, stack(0x00000003143f8000,0x00000003147f8000)] - 0x00007fba83207800 JavaThread "ExpirationReaper-0-DeleteRecords" [_thread_blocked, id=75355, stack(0x0000000313ff5000,0x00000003143f5000)] - 0x00007fba908af000 JavaThread "ExpirationReaper-0-Fetch" [_thread_blocked, id=75531, stack(0x0000000313bf2000,0x0000000313ff2000)] - 0x00007fba83206800 JavaThread "ExpirationReaper-0-Produce" [_thread_blocked, id=55159, stack(0x00000003137ef000,0x0000000313bef000)] - 0x00007fba859e8800 JavaThread "BrokerToControllerChannelManager broker=0 name=alterPartition" [_thread_in_native, id=54391, stack(0x00000003133ec000,0x00000003137ec000)] - 0x00007fba8494b800 JavaThread "BrokerToControllerChannelManager broker=0 name=forwarding" [_thread_in_native, id=53907, stack(0x0000000312fe9000,0x00000003133e9000)] - 0x00007fba858a0800 JavaThread "feature-zk-node-event-process-thread" [_thread_blocked, id=53723, stack(0x0000000312be6000,0x0000000312fe6000)] - 0x00007fba869aa800 JavaThread "kafka-log-cleaner-thread-0" [_thread_blocked, id=77107, stack(0x00000003127e3000,0x0000000312be3000)] - 0x00007fba840cb800 JavaThread "kafka-scheduler-4" daemon [_thread_blocked, id=77571, stack(0x00000003123e0000,0x00000003127e0000)] - 0x00007fba840ca800 JavaThread "kafka-scheduler-3" daemon [_thread_blocked, id=77827, stack(0x0000000311fdd000,0x00000003123dd000)] - 0x00007fba840ca000 JavaThread "kafka-scheduler-2" daemon [_thread_blocked, id=52483, stack(0x0000000311bda000,0x0000000311fda000)] - 0x00007fba840c9000 JavaThread "kafka-scheduler-1" daemon [_thread_blocked, id=78347, stack(0x00000003117d7000,0x0000000311bd7000)] - 0x00007fba840c8000 JavaThread "kafka-scheduler-0" daemon [_thread_blocked, id=51807, stack(0x00000003113d4000,0x00000003117d4000)] - 0x00007fba858d7800 JavaThread "ThrottledChannelReaper-ControllerMutation" [_thread_blocked, id=79107, stack(0x0000000310fd1000,0x00000003113d1000)] - 0x00007fba831f5800 JavaThread "ThrottledChannelReaper-Request" [_thread_blocked, id=51203, stack(0x0000000310bce000,0x0000000310fce000)] - 0x00007fba831f7800 JavaThread "ThrottledChannelReaper-Produce" [_thread_blocked, id=80231, stack(0x00000003107cb000,0x0000000310bcb000)] - 0x00007fba87017800 JavaThread "ThrottledChannelReaper-Fetch" [_thread_blocked, id=80835, stack(0x00000003103c8000,0x00000003107c8000)] - 0x00007fba868a2800 JavaThread "SensorExpiryThread" daemon [_thread_blocked, id=80899, stack(0x000000030ffc5000,0x00000003103c5000)] - 0x00007fba871c5000 JavaThread "NIOWorkerThread-20" daemon [_thread_blocked, id=49667, stack(0x000000030fbc2000,0x000000030ffc2000)] - 0x00007fba840be800 JavaThread "NIOWorkerThread-19" daemon [_thread_blocked, id=49179, stack(0x000000030f7bf000,0x000000030fbbf000)] - 0x00007fba84944000 JavaThread "NIOWorkerThread-18" daemon [_thread_blocked, id=48411, stack(0x000000030f3bc000,0x000000030f7bc000)] - 0x00007fba86995000 JavaThread "NIOWorkerThread-17" daemon [_thread_blocked, id=48139, stack(0x000000030efb9000,0x000000030f3b9000)] - 0x00007fba84943800 JavaThread "NIOWorkerThread-16" daemon [_thread_blocked, id=82995, stack(0x000000030ebb6000,0x000000030efb6000)] - 0x00007fba84942800 JavaThread "NIOWorkerThread-15" daemon [_thread_blocked, id=46859, stack(0x000000030e7b3000,0x000000030ebb3000)] - 0x00007fba871c4000 JavaThread "NIOWorkerThread-14" daemon [_thread_blocked, id=83203, stack(0x000000030e3b0000,0x000000030e7b0000)] - 0x00007fba849ea800 JavaThread "NIOWorkerThread-13" daemon [_thread_blocked, id=84231, stack(0x000000030dfad000,0x000000030e3ad000)] - 0x00007fba8787b800 JavaThread "NIOWorkerThread-12" daemon [_thread_blocked, id=84995, stack(0x000000030dbaa000,0x000000030dfaa000)] - 0x00007fba858d0800 JavaThread "NIOWorkerThread-11" daemon [_thread_blocked, id=85771, stack(0x000000030d7a7000,0x000000030dba7000)] - 0x00007fba871c3000 JavaThread "NIOWorkerThread-10" daemon [_thread_blocked, id=46083, stack(0x000000030d3a4000,0x000000030d7a4000)] - 0x00007fba831e7800 JavaThread "NIOWorkerThread-9" daemon [_thread_blocked, id=45587, stack(0x000000030cfa1000,0x000000030d3a1000)] - 0x00007fba830a1000 JavaThread "NIOWorkerThread-8" daemon [_thread_blocked, id=45419, stack(0x000000030cb9e000,0x000000030cf9e000)] - 0x00007fba85292000 JavaThread "NIOWorkerThread-7" daemon [_thread_blocked, id=44807, stack(0x000000030c79b000,0x000000030cb9b000)] - 0x00007fba8528e800 JavaThread "NIOWorkerThread-6" daemon [_thread_blocked, id=87055, stack(0x000000030c398000,0x000000030c798000)] - 0x00007fba84178000 JavaThread "NIOWorkerThread-5" daemon [_thread_blocked, id=44079, stack(0x000000030bf95000,0x000000030c395000)] - 0x00007fba849eb800 JavaThread "NIOWorkerThread-4" daemon [_thread_blocked, id=31311, stack(0x000000030bb92000,0x000000030bf92000)] - 0x00007fba830a0800 JavaThread "NIOWorkerThread-3" daemon [_thread_blocked, id=31067, stack(0x000000030b78f000,0x000000030bb8f000)] - 0x00007fba84997800 JavaThread "NIOWorkerThread-2" daemon [_thread_blocked, id=30799, stack(0x000000030b38c000,0x000000030b78c000)] - 0x00007fba85285800 JavaThread "NIOWorkerThread-1" daemon [_thread_blocked, id=29967, stack(0x000000030af89000,0x000000030b389000)] - 0x00007fba831e7000 JavaThread "Test worker-EventThread" daemon [_thread_blocked, id=33027, stack(0x000000030ab86000,0x000000030af86000)] - 0x00007fba85a91800 JavaThread "Test worker-SendThread(127.0.0.1:63835)" daemon [_thread_in_native, id=33439, stack(0x000000030a783000,0x000000030ab83000)] - 0x00007fba8708a800 JavaThread "metrics-meter-tick-thread-2" daemon [_thread_blocked, id=29191, stack(0x000000030a380000,0x000000030a780000)] - 0x00007fba8629f800 JavaThread "metrics-meter-tick-thread-1" daemon [_thread_blocked, id=28723, stack(0x0000000309f7d000,0x000000030a37d000)] - 0x00007fba873a2800 JavaThread "RequestThrottler" [_thread_blocked, id=34403, stack(0x0000000309b7a000,0x0000000309f7a000)] - 0x00007fba873a0000 JavaThread "ProcessThread(sid:0 cport:63835):" [_thread_blocked, id=35075, stack(0x0000000309777000,0x0000000309b77000)] - 0x00007fba8628c800 JavaThread "SyncThread:0" [_thread_blocked, id=35595, stack(0x0000000309374000,0x0000000309774000)] - 0x00007fba87390800 JavaThread "SessionTracker" [_thread_blocked, id=36311, stack(0x0000000308f71000,0x0000000309371000)] - 0x00007fba85270800 JavaThread "ConnnectionExpirer" [_thread_blocked, id=27395, stack(0x0000000308b6e000,0x0000000308f6e000)] - 0x00007fba831e5000 JavaThread "NIOServerCxnFactory.AcceptThread:/127.0.0.1:0" daemon [_thread_in_native, id=36611, stack(0x000000030876b000,0x0000000308b6b000)] - 0x00007fba831e9800 JavaThread "NIOServerCxnFactory.SelectorThread-0" daemon [_thread_in_native, id=26371, stack(0x0000000308368000,0x0000000308768000)] - 0x00007fba87390000 JavaThread "NIOServerCxnFactory.SelectorThread-1" daemon [_thread_in_native, id=37123, stack(0x0000000307f65000,0x0000000308365000)] - 0x00007fba871ce000 JavaThread "/127.0.0.1:63826 to /127.0.0.1:63820 workers Thread 3" [_thread_in_native, id=39171, stack(0x0000000307b62000,0x0000000307f62000)] - 0x00007fba871cc800 JavaThread "/127.0.0.1:63826 to /127.0.0.1:63820 workers Thread 2" [_thread_in_vm, id=24887, stack(0x000000030775f000,0x0000000307b5f000)] - 0x00007fba8690e800 JavaThread "/127.0.0.1:63826 to /127.0.0.1:63820 workers" [_thread_blocked, id=24595, stack(0x000000030735c000,0x000000030775c000)] - 0x00007fba87815000 JavaThread "Service Thread" daemon [_thread_blocked, id=40963, stack(0x0000000306e56000,0x0000000307256000)] - 0x00007fba87814000 JavaThread "C1 CompilerThread3" daemon [_thread_in_vm, id=41731, stack(0x0000000306d53000,0x0000000306e53000)] - 0x00007fba87813000 JavaThread "C2 CompilerThread2" daemon [_thread_blocked, id=23043, stack(0x0000000306c50000,0x0000000306d50000)] - 0x00007fba8405b000 JavaThread "C2 CompilerThread1" daemon [_thread_blocked, id=41987, stack(0x0000000306b4d000,0x0000000306c4d000)] - 0x00007fba8400f800 JavaThread "C2 CompilerThread0" daemon [_thread_in_vm, id=42243, stack(0x0000000306a4a000,0x0000000306b4a000)] - 0x00007fba8400a800 JavaThread "Signal Dispatcher" daemon [_thread_blocked, id=22019, stack(0x0000000306647000,0x0000000306a47000)] - 0x00007fba84818000 JavaThread "Finalizer" daemon [_thread_blocked, id=15131, stack(0x0000000306134000,0x0000000306534000)] - 0x00007fba87808800 JavaThread "Reference Handler" daemon [_thread_blocked, id=14883, stack(0x0000000305d31000,0x0000000306131000)] - 0x00007fba8480b800 JavaThread "Test worker" [_thread_blocked, id=6915, stack(0x0000000304f10000,0x0000000305310000)] - -Other Threads: - 0x00007fba86847800 VMThread [stack: 0x0000000305c2e000,0x0000000305d2e000] [id=18947] - 0x00007fba90809000 WatcherThread [stack: 0x0000000307259000,0x0000000307359000] [id=40707] - -VM state:synchronizing (normal execution) - -VM Mutex/Monitor currently owned by a thread: ([mutex/lock_event]) -[0x00007fba83905db0] Safepoint_lock - owner thread: 0x00007fba86847800 -[0x00007fba83905e30] Threads_lock - owner thread: 0x00007fba86847800 - -heap address: 0x0000000740000000, size: 2048 MB, Compressed Oops mode: Zero based, Oop shift amount: 3 -Narrow klass base: 0x0000000000000000, Narrow klass shift: 3 -Compressed class space size: 1073741824 Address: 0x00000007c0000000 - -Heap: - PSYoungGen total 560640K, used 77468K [0x0000000795580000, 0x00000007c0000000, 0x00000007c0000000) - eden space 500224K, 3% used [0x0000000795580000,0x0000000796638828,0x00000007b3e00000) - from space 60416K, 99% used [0x00000007b3e00000,0x00000007b78eeb98,0x00000007b7900000) - to space 100864K, 0% used [0x00000007b9d80000,0x00000007b9d80000,0x00000007c0000000) - ParOldGen total 175104K, used 48979K [0x0000000740000000, 0x000000074ab00000, 0x0000000795580000) - object space 175104K, 27% used [0x0000000740000000,0x0000000742fd4cf8,0x000000074ab00000) - Metaspace used 66302K, capacity 71877K, committed 72152K, reserved 1112064K - class space used 8644K, capacity 9370K, committed 9472K, reserved 1048576K - -Card table byte_map: [0x000000010e3c0000,0x000000010e7c1000] byte_map_base: 0x000000010a9c0000 - -Marking Bits: (ParMarkBitMap*) 0x000000010f50bf08 - Begin Bits: [0x000000011fc51000, 0x0000000121c51000) - End Bits: [0x0000000121c51000, 0x0000000123c51000) - -Polling page: 0x0000000108c18000 - -CodeCache: size=245760Kb used=14840Kb max_used=14846Kb free=230919Kb - bounds [0x0000000110c51000, 0x0000000111b01000, 0x000000011fc51000] - total_blobs=6049 nmethods=5181 adapters=778 - compilation: enabled - -Compilation events (10 events): -Event: 53.733 Thread 0x00007fba8400f800 6469 4 java.nio.ByteBuffer::allocate (22 bytes) -Event: 53.745 Thread 0x00007fba8400f800 nmethod 6469 0x00000001115c4e10 code [0x00000001115c4f60, 0x00000001115c5138] -Event: 53.896 Thread 0x00007fba8405b000 nmethod 6463 0x000000011162ad90 code [0x000000011162b160, 0x000000011162ccd8] -Event: 53.959 Thread 0x00007fba87813000 6473 s! 4 sun.misc.URLClassPath::getLoader (243 bytes) -Event: 53.961 Thread 0x00007fba8400f800 6472 s 4 sun.misc.URLClassPath::getNextLoader (88 bytes) -Event: 53.964 Thread 0x00007fba8405b000 6475 4 java.lang.Math::max (12 bytes) -Event: 54.011 Thread 0x00007fba8400f800 nmethod 6472 0x00000001116a0610 code [0x00000001116a08e0, 0x00000001116a22c8] -Event: 54.011 Thread 0x00007fba8400f800 6479 4 org.apache.kafka.common.utils.SystemTime::milliseconds (4 bytes) -Event: 54.060 Thread 0x00007fba8405b000 nmethod 6475 0x000000011169dd90 code [0x000000011169dec0, 0x000000011169df18] -Event: 54.065 Thread 0x00007fba8405b000 6478 4 java.util.LinkedList::poll (19 bytes) - -GC Heap History (10 events): -Event: 37.292 GC heap before -{Heap before GC invocations=14 (full 3): - PSYoungGen total 614912K, used 614909K [0x0000000795580000, 0x00000007c0000000, 0x00000007c0000000) - eden space 558592K, 100% used [0x0000000795580000,0x00000007b7700000,0x00000007b7700000) - from space 56320K, 99% used [0x00000007bc900000,0x00000007bffff490,0x00000007c0000000) - to space 70144K, 0% used [0x00000007b7700000,0x00000007b7700000,0x00000007bbb80000) - ParOldGen total 175104K, used 39338K [0x0000000740000000, 0x000000074ab00000, 0x0000000795580000) - object space 175104K, 22% used [0x0000000740000000,0x000000074266abb0,0x000000074ab00000) - Metaspace used 62563K, capacity 67527K, committed 67672K, reserved 1107968K - class space used 8156K, capacity 8812K, committed 8832K, reserved 1048576K -Event: 37.693 GC heap after -Heap after GC invocations=14 (full 3): - PSYoungGen total 623104K, used 64380K [0x0000000795580000, 0x00000007c0000000, 0x00000007c0000000) - eden space 558592K, 0% used [0x0000000795580000,0x0000000795580000,0x00000007b7700000) - from space 64512K, 99% used [0x00000007b7700000,0x00000007bb5df1d0,0x00000007bb600000) - to space 75776K, 0% used [0x00000007bb600000,0x00000007bb600000,0x00000007c0000000) - ParOldGen total 175104K, used 42880K [0x0000000740000000, 0x000000074ab00000, 0x0000000795580000) - object space 175104K, 24% used [0x0000000740000000,0x00000007429e0298,0x000000074ab00000) - Metaspace used 62563K, capacity 67527K, committed 67672K, reserved 1107968K - class space used 8156K, capacity 8812K, committed 8832K, reserved 1048576K -} -Event: 40.789 GC heap before -{Heap before GC invocations=15 (full 3): - PSYoungGen total 623104K, used 622972K [0x0000000795580000, 0x00000007c0000000, 0x00000007c0000000) - eden space 558592K, 100% used [0x0000000795580000,0x00000007b7700000,0x00000007b7700000) - from space 64512K, 99% used [0x00000007b7700000,0x00000007bb5df1d0,0x00000007bb600000) - to space 75776K, 0% used [0x00000007bb600000,0x00000007bb600000,0x00000007c0000000) - ParOldGen total 175104K, used 42880K [0x0000000740000000, 0x000000074ab00000, 0x0000000795580000) - object space 175104K, 24% used [0x0000000740000000,0x00000007429e0298,0x000000074ab00000) - Metaspace used 62663K, capacity 67667K, committed 67928K, reserved 1107968K - class space used 8157K, capacity 8814K, committed 8832K, reserved 1048576K -Event: 40.976 GC heap after -Heap after GC invocations=15 (full 3): - PSYoungGen total 594432K, used 72605K [0x0000000795580000, 0x00000007c0000000, 0x00000007c0000000) - eden space 518656K, 0% used [0x0000000795580000,0x0000000795580000,0x00000007b5000000) - from space 75776K, 95% used [0x00000007bb600000,0x00000007bfce76b8,0x00000007c0000000) - to space 90112K, 0% used [0x00000007b5000000,0x00000007b5000000,0x00000007ba800000) - ParOldGen total 175104K, used 43056K [0x0000000740000000, 0x000000074ab00000, 0x0000000795580000) - object space 175104K, 24% used [0x0000000740000000,0x0000000742a0c2a8,0x000000074ab00000) - Metaspace used 62663K, capacity 67667K, committed 67928K, reserved 1107968K - class space used 8157K, capacity 8814K, committed 8832K, reserved 1048576K -} -Event: 44.128 GC heap before -{Heap before GC invocations=16 (full 3): - PSYoungGen total 594432K, used 591261K [0x0000000795580000, 0x00000007c0000000, 0x00000007c0000000) - eden space 518656K, 100% used [0x0000000795580000,0x00000007b5000000,0x00000007b5000000) - from space 75776K, 95% used [0x00000007bb600000,0x00000007bfce76b8,0x00000007c0000000) - to space 90112K, 0% used [0x00000007b5000000,0x00000007b5000000,0x00000007ba800000) - ParOldGen total 175104K, used 43056K [0x0000000740000000, 0x000000074ab00000, 0x0000000795580000) - object space 175104K, 24% used [0x0000000740000000,0x0000000742a0c2a8,0x000000074ab00000) - Metaspace used 63630K, capacity 68893K, committed 69208K, reserved 1110016K - class space used 8281K, capacity 8983K, committed 9088K, reserved 1048576K -Event: 44.346 GC heap after -Heap after GC invocations=16 (full 3): - PSYoungGen total 579072K, used 60232K [0x0000000795580000, 0x00000007c0000000, 0x00000007c0000000) - eden space 518656K, 0% used [0x0000000795580000,0x0000000795580000,0x00000007b5000000) - from space 60416K, 99% used [0x00000007b5000000,0x00000007b8ad2028,0x00000007b8b00000) - to space 94208K, 0% used [0x00000007ba400000,0x00000007ba400000,0x00000007c0000000) - ParOldGen total 175104K, used 44306K [0x0000000740000000, 0x000000074ab00000, 0x0000000795580000) - object space 175104K, 25% used [0x0000000740000000,0x0000000742b449c8,0x000000074ab00000) - Metaspace used 63630K, capacity 68893K, committed 69208K, reserved 1110016K - class space used 8281K, capacity 8983K, committed 9088K, reserved 1048576K -} -Event: 44.857 GC heap before -{Heap before GC invocations=17 (full 3): - PSYoungGen total 579072K, used 578888K [0x0000000795580000, 0x00000007c0000000, 0x00000007c0000000) - eden space 518656K, 100% used [0x0000000795580000,0x00000007b5000000,0x00000007b5000000) - from space 60416K, 99% used [0x00000007b5000000,0x00000007b8ad2028,0x00000007b8b00000) - to space 94208K, 0% used [0x00000007ba400000,0x00000007ba400000,0x00000007c0000000) - ParOldGen total 175104K, used 44306K [0x0000000740000000, 0x000000074ab00000, 0x0000000795580000) - object space 175104K, 25% used [0x0000000740000000,0x0000000742b449c8,0x000000074ab00000) - Metaspace used 63676K, capacity 68957K, committed 69208K, reserved 1110016K - class space used 8296K, capacity 9006K, committed 9088K, reserved 1048576K -Event: 45.023 GC heap after -Heap after GC invocations=17 (full 3): - PSYoungGen total 594432K, used 67076K [0x0000000795580000, 0x00000007c0000000, 0x00000007c0000000) - eden space 500224K, 0% used [0x0000000795580000,0x0000000795580000,0x00000007b3e00000) - from space 94208K, 71% used [0x00000007ba400000,0x00000007be5810c0,0x00000007c0000000) - to space 99328K, 0% used [0x00000007b3e00000,0x00000007b3e00000,0x00000007b9f00000) - ParOldGen total 175104K, used 46363K [0x0000000740000000, 0x000000074ab00000, 0x0000000795580000) - object space 175104K, 26% used [0x0000000740000000,0x0000000742d46dc0,0x000000074ab00000) - Metaspace used 63676K, capacity 68957K, committed 69208K, reserved 1110016K - class space used 8296K, capacity 9006K, committed 9088K, reserved 1048576K -} -Event: 52.822 GC heap before -{Heap before GC invocations=18 (full 3): - PSYoungGen total 594432K, used 567300K [0x0000000795580000, 0x00000007c0000000, 0x00000007c0000000) - eden space 500224K, 100% used [0x0000000795580000,0x00000007b3e00000,0x00000007b3e00000) - from space 94208K, 71% used [0x00000007ba400000,0x00000007be5810c0,0x00000007c0000000) - to space 99328K, 0% used [0x00000007b3e00000,0x00000007b3e00000,0x00000007b9f00000) - ParOldGen total 175104K, used 46363K [0x0000000740000000, 0x000000074ab00000, 0x0000000795580000) - object space 175104K, 26% used [0x0000000740000000,0x0000000742d46dc0,0x000000074ab00000) - Metaspace used 66094K, capacity 71609K, committed 71768K, reserved 1112064K - class space used 8608K, capacity 9332K, committed 9344K, reserved 1048576K -Event: 53.305 GC heap after -Heap after GC invocations=18 (full 3): - PSYoungGen total 560640K, used 60346K [0x0000000795580000, 0x00000007c0000000, 0x00000007c0000000) - eden space 500224K, 0% used [0x0000000795580000,0x0000000795580000,0x00000007b3e00000) - from space 60416K, 99% used [0x00000007b3e00000,0x00000007b78eeb98,0x00000007b7900000) - to space 100864K, 0% used [0x00000007b9d80000,0x00000007b9d80000,0x00000007c0000000) - ParOldGen total 175104K, used 48979K [0x0000000740000000, 0x000000074ab00000, 0x0000000795580000) - object space 175104K, 27% used [0x0000000740000000,0x0000000742fd4cf8,0x000000074ab00000) - Metaspace used 66094K, capacity 71609K, committed 71768K, reserved 1112064K - class space used 8608K, capacity 9332K, committed 9344K, reserved 1048576K -} - -Deoptimization events (10 events): -Event: 50.305 Thread 0x00007fba92050800 Uncommon trap: reason=unstable_if action=reinterpret pc=0x0000000111932c38 method=java.net.URI$Parser.scan(IILjava/lang/String;Ljava/lang/String;)I @ 23 -Event: 50.563 Thread 0x00007fba831e7000 Uncommon trap: reason=unstable_if action=reinterpret pc=0x0000000111669184 method=java.util.concurrent.locks.AbstractQueuedSynchronizer.doReleaseShared()V @ 24 -Event: 50.710 Thread 0x00007fba8555b000 Uncommon trap: reason=bimorphic action=maybe_recompile pc=0x000000011113be0c method=org.apache.kafka.common.utils.ImplicitLinkedHashCollection.removeFromList(Lorg/apache/kafka/common/utils/ImplicitLinkedHashCollection$Element;[Lorg/apache/kafka/common/uti -Event: 51.391 Thread 0x00007fba9221c000 Uncommon trap: reason=unstable_if action=reinterpret pc=0x0000000110db3308 method=java.lang.Throwable.getOurStackTrace()[Ljava/lang/StackTraceElement; @ 7 -Event: 51.653 Thread 0x00007fba8555b000 Uncommon trap: reason=bimorphic action=maybe_recompile pc=0x000000011113be0c method=org.apache.kafka.common.utils.ImplicitLinkedHashCollection.removeFromList(Lorg/apache/kafka/common/utils/ImplicitLinkedHashCollection$Element;[Lorg/apache/kafka/common/uti -Event: 51.912 Thread 0x00007fba8554f800 Uncommon trap: reason=bimorphic action=maybe_recompile pc=0x000000011113be0c method=org.apache.kafka.common.utils.ImplicitLinkedHashCollection.removeFromList(Lorg/apache/kafka/common/utils/ImplicitLinkedHashCollection$Element;[Lorg/apache/kafka/common/uti -Event: 52.028 Thread 0x00007fba92050800 Uncommon trap: reason=unstable_if action=reinterpret pc=0x0000000111045fc8 method=java.util.stream.StreamOpFlag.fromCharacteristics(Ljava/util/Spliterator;)I @ 10 -Event: 52.137 Thread 0x00007fba9221c000 Uncommon trap: reason=unstable_if action=reinterpret pc=0x0000000110ec16f0 method=org.apache.kafka.common.utils.ImplicitLinkedHashCollection.reseat(I)V @ 40 -Event: 53.402 Thread 0x00007fba8555b000 Uncommon trap: reason=unstable_if action=reinterpret pc=0x0000000110f689f8 method=java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(I)V @ 5 -Event: 53.477 Thread 0x00007fba8555b000 Uncommon trap: reason=unstable_if action=reinterpret pc=0x00000001112eb574 method=java.util.concurrent.locks.AbstractQueuedSynchronizer.apparentlyFirstQueuedIsExclusive()Z @ 6 - -Classes redefined (0 events): -No events - -Internal exceptions (10 events): -Event: 27.875 Thread 0x00007fba8480b800 Exception (0x000000079928e820) thrown at [/Users/jenkins/workspace/build-scripts/jobs/jdk8u/jdk8u-mac-x64-h -Event: 31.788 Thread 0x00007fba8480b800 Exception (0x000000079c323b18) thrown at [/Users/jenkins/workspace/build-scripts/jobs/jdk8u/jdk8u-mac-x64-hotspot/workspace/build/src/hotsp -Event: 33.133 Thread 0x00007fba86f5c000 Exception (0x00000007970cfcd8) thrown at [/Users/jenkins/workspace/build-scripts/jobs/jdk8u/jdk8u-mac-x64-hotspot/workspace/build/src/hotspot/src/share/vm/prims/jni.cpp, line 711] -Event: 39.208 Thread 0x00007fba91bde800 Exception (0x0000000796030d68) thrown at [/Users/jenkins/workspace/build-scripts/jobs/jdk8u/jdk8u-mac-x64-hotspot/workspace/build/src/hotspot/src/share/vm/prims/jni.cpp, line 711] -Event: 43.472 Thread 0x00007fba90826000 Exception (0x00000007b16971b0) thrown at [/Users/jenkins/workspace/build-scripts/jobs/jdk8u/jdk8u-mac-x64-hotspot/workspace/build/src/hotspot/src/share/vm/prims/jni.cpp, line 711] -Event: 50.089 Thread 0x00007fba92050800 Exception (0x00000007b2287830) thrown at [/Users/jenkins/workspace/build-scripts/jobs/jdk8u/jdk8u-mac-x64-hotspot/workspace/build/src/hotspot/src/share/vm/runtime/reflection.cpp, line 1092] -Event: 52.141 Thread 0x00007fba92050800 Exception (0x00000007b36d46f8) thrown at [/Users/jenkins/workspace/build-scripts/jobs/jdk8u/jdk8u-mac-x -Event: 52.621 Thread 0x00007fba8506c800 Exception (0x00000007b3d60cd0) thrown at [/Users/jenkins/workspace/build-scripts/jobs/jdk8u/jdk8u-mac-x64-hotspot/workspace/build/src/hotspot/src/share/vm/prims/jni.cpp, line 711] -Event: 53.703 Thread 0x00007fba8506c800 Exception (0x0000000795d4eea8) thrown at [/Users/jenkins/workspace/build-scripts/jobs/jdk8u/jdk8u-mac-x64-hotspot/workspace/build/src/hotspot/src/share/vm/prims/jni.cpp, line 711] -Event: 54.098 Thread 0x00007fba8506c800 Exception (0x0000000796561808) thrown at [/Users/jenkins/workspace/build-scripts/jobs/jdk8u/jdk8u-mac-x64-hotspot/workspace/build/src/hotspot/src/share/vm/prims/jni.cpp, line 711] - -Events (10 events): -Event: 53.997 loading class org/apache/kafka/connect/util/clusters/WorkerHandle -Event: 53.997 loading class org/apache/kafka/connect/util/clusters/WorkerHandle done -Event: 53.997 loading class org/apache/kafka/connect/util/clusters/EmbeddedConnectCluster -Event: 53.997 loading class org/apache/kafka/connect/util/clusters/EmbeddedConnectCluster done -Event: 54.022 Executing VM operation: RevokeBias -Event: 54.054 Executing VM operation: RevokeBias done -Event: 54.055 Thread 0x00007fba87814000 flushing nmethod 0x0000000110fe15d0 -Event: 54.061 Executing VM operation: RevokeBias -Event: 54.086 Executing VM operation: RevokeBias done -Event: 54.099 Executing VM operation: RevokeBias - - -Dynamic libraries: -0x00007ff830514000 /System/Library/Frameworks/Cocoa.framework/Versions/A/Cocoa -0x00007ff81be12000 /System/Library/Frameworks/AppKit.framework/Versions/C/AppKit -0x00007ff81f371000 /System/Library/Frameworks/CoreData.framework/Versions/A/CoreData -0x00007ff81a229000 /System/Library/Frameworks/Foundation.framework/Versions/C/Foundation -0x00007ff82458d000 /usr/lib/libSystem.B.dylib -0x00007ff81cca4000 /System/Library/PrivateFrameworks/UIFoundation.framework/Versions/A/UIFoundation -0x00007ff82a4c0000 /System/Library/PrivateFrameworks/RemoteViewServices.framework/Versions/A/RemoteViewServices -0x00007ff822860000 /System/Library/PrivateFrameworks/XCTTargetBootstrap.framework/Versions/A/XCTTargetBootstrap -0x00007ff826459000 /System/Library/PrivateFrameworks/InternationalSupport.framework/Versions/A/InternationalSupport -0x00007ff8264e3000 /System/Library/PrivateFrameworks/UserActivity.framework/Versions/A/UserActivity -0x00007ff825840000 /usr/lib/libspindump.dylib -0x00007ff81cf32000 /System/Library/Frameworks/UniformTypeIdentifiers.framework/Versions/A/UniformTypeIdentifiers -0x00007ff820f99000 /usr/lib/libapp_launch_measurement.dylib -0x00007ff81fd4d000 /System/Library/PrivateFrameworks/CoreAnalytics.framework/Versions/A/CoreAnalytics -0x00007ff820f9c000 /System/Library/PrivateFrameworks/CoreAutoLayout.framework/Versions/A/CoreAutoLayout -0x00007ff8221bc000 /System/Library/Frameworks/Metal.framework/Versions/A/Metal -0x00007ff822ed8000 /usr/lib/liblangid.dylib -0x00007ff822864000 /System/Library/PrivateFrameworks/CoreSVG.framework/Versions/A/CoreSVG -0x00007ff81e2f9000 /System/Library/PrivateFrameworks/SkyLight.framework/Versions/A/SkyLight -0x00007ff81e68f000 /System/Library/Frameworks/CoreGraphics.framework/Versions/A/CoreGraphics -0x00007ff82ab2f000 /System/Library/Frameworks/Accelerate.framework/Versions/A/Accelerate -0x00007ff8252bb000 /System/Library/PrivateFrameworks/IconServices.framework/Versions/A/IconServices -0x00007ff8221a1000 /System/Library/Frameworks/IOSurface.framework/Versions/A/IOSurface -0x00007ff81fd7b000 /usr/lib/libDiagnosticMessagesClient.dylib -0x00007ff8244a7000 /usr/lib/libz.1.dylib -0x00007ff82e1a0000 /System/Library/Frameworks/ApplicationServices.framework/Versions/A/ApplicationServices -0x00007ff82284c000 /System/Library/PrivateFrameworks/DFRFoundation.framework/Versions/A/DFRFoundation -0x00007ff81b703000 /usr/lib/libicucore.A.dylib -0x00007ff82717e000 /System/Library/Frameworks/AudioToolbox.framework/Versions/A/AudioToolbox -0x00007ff826465000 /System/Library/PrivateFrameworks/DataDetectorsCore.framework/Versions/A/DataDetectorsCore -0x00007ff921d57000 /System/Library/PrivateFrameworks/TextInput.framework/Versions/A/TextInput -0x00007ff81e27d000 /usr/lib/libMobileGestalt.dylib -0x00007ff822554000 /System/Library/Frameworks/Carbon.framework/Versions/A/Frameworks/HIToolbox.framework/Versions/A/HIToolbox -0x00007ff8209c6000 /System/Library/Frameworks/QuartzCore.framework/Versions/A/QuartzCore -0x00007ff81b3a3000 /System/Library/Frameworks/Security.framework/Versions/A/Security -0x00007ff82a4f8000 /System/Library/Frameworks/Carbon.framework/Versions/A/Frameworks/SpeechRecognition.framework/Versions/A/SpeechRecognition -0x00007ff820cf9000 /System/Library/PrivateFrameworks/CoreUI.framework/Versions/A/CoreUI -0x00007ff81ac7b000 /System/Library/Frameworks/CoreAudio.framework/Versions/A/CoreAudio -0x00007ff81fe69000 /System/Library/Frameworks/DiskArbitration.framework/Versions/A/DiskArbitration -0x00007ff825c23000 /System/Library/PrivateFrameworks/MultitouchSupport.framework/Versions/A/MultitouchSupport -0x00007ff81e27c000 /usr/lib/libenergytrace.dylib -0x00007ff81bcfe000 /System/Library/Frameworks/IOKit.framework/Versions/A/IOKit -0x00007ff82a8e4000 /System/Library/Frameworks/CoreServices.framework/Versions/A/CoreServices -0x00007ff820f33000 /System/Library/PrivateFrameworks/PerformanceAnalysis.framework/Versions/A/PerformanceAnalysis -0x00007ffa3183e000 /System/Library/Frameworks/OpenGL.framework/Versions/A/OpenGL -0x00007ff823ad4000 /System/Library/PrivateFrameworks/MobileKeyBag.framework/Versions/A/MobileKeyBag -0x00007ff8192c1000 /usr/lib/libobjc.A.dylib -0x00007ff81edbe000 /System/Library/Frameworks/ColorSync.framework/Versions/A/ColorSync -0x00007ff81947d000 /System/Library/Frameworks/CoreFoundation.framework/Versions/A/CoreFoundation -0x00007ff822adf000 /System/Library/Frameworks/CoreImage.framework/Versions/A/CoreImage -0x00007ff81aa91000 /System/Library/Frameworks/CoreText.framework/Versions/A/CoreText -0x00007ff822893000 /System/Library/Frameworks/ImageIO.framework/Versions/A/ImageIO -0x00007ff824593000 /System/Library/PrivateFrameworks/SoftLinking.framework/Versions/A/SoftLinking -0x00007ff820fe4000 /usr/lib/libxml2.2.dylib -0x00007ff81938a000 /usr/lib/libc++.1.dylib -0x00007ff824814000 /usr/lib/libcompression.dylib -0x00007ff82638f000 /System/Library/PrivateFrameworks/TextureIO.framework/Versions/A/TextureIO -0x00007ff824f34000 /usr/lib/libate.dylib -0x00007ff824588000 /usr/lib/system/libcache.dylib -0x00007ff824544000 /usr/lib/system/libcommonCrypto.dylib -0x00007ff82456d000 /usr/lib/system/libcompiler_rt.dylib -0x00007ff824563000 /usr/lib/system/libcopyfile.dylib -0x00007ff8191bb000 /usr/lib/system/libcorecrypto.dylib -0x00007ff81927a000 /usr/lib/system/libdispatch.dylib -0x00007ff81943c000 /usr/lib/system/libdyld.dylib -0x00007ff82457f000 /usr/lib/system/libkeymgr.dylib -0x00007ff824522000 /usr/lib/system/libmacho.dylib -0x00007ff823ba8000 /usr/lib/system/libquarantine.dylib -0x00007ff82457d000 /usr/lib/system/libremovefile.dylib -0x00007ff81e2c9000 /usr/lib/system/libsystem_asl.dylib -0x00007ff819169000 /usr/lib/system/libsystem_blocks.dylib -0x00007ff819301000 /usr/lib/system/libsystem_c.dylib -0x00007ff824575000 /usr/lib/system/libsystem_collections.dylib -0x00007ff822ec9000 /usr/lib/system/libsystem_configuration.dylib -0x00007ff822184000 /usr/lib/system/libsystem_containermanager.dylib -0x00007ff824259000 /usr/lib/system/libsystem_coreservices.dylib -0x00007ff81b98c000 /usr/lib/system/libsystem_darwin.dylib -0x00007ff824580000 /usr/lib/system/libsystem_dnssd.dylib -0x00007ff8192fe000 /usr/lib/system/libsystem_featureflags.dylib -0x00007ff819452000 /usr/lib/system/libsystem_info.dylib -0x00007ff8244ba000 /usr/lib/system/libsystem_m.dylib -0x00007ff81924e000 /usr/lib/system/libsystem_malloc.dylib -0x00007ff81e266000 /usr/lib/system/libsystem_networkextension.dylib -0x00007ff81bdb4000 /usr/lib/system/libsystem_notify.dylib -0x00007ff82a7b9000 /usr/lib/system/libsystem_product_info_filter.dylib -0x00007ff822ecd000 /usr/lib/system/libsystem_sandbox.dylib -0x00007ff82457a000 /usr/lib/system/libsystem_secinit.dylib -0x00007ff8193f9000 /usr/lib/system/libsystem_kernel.dylib -0x00007ff819448000 /usr/lib/system/libsystem_platform.dylib -0x00007ff819430000 /usr/lib/system/libsystem_pthread.dylib -0x00007ff81fb94000 /usr/lib/system/libsystem_symptoms.dylib -0x00007ff8191a2000 /usr/lib/system/libsystem_trace.dylib -0x00007ff824550000 /usr/lib/system/libunwind.dylib -0x00007ff81916b000 /usr/lib/system/libxpc.dylib -0x00007ff8193e3000 /usr/lib/libc++abi.dylib -0x00007ff82455b000 /usr/lib/liboah.dylib -0x00007ff824ced000 /usr/lib/liblzma.5.dylib -0x00007ff819f3a000 /System/Library/Frameworks/SystemConfiguration.framework/Versions/A/SystemConfiguration -0x00007ff82458f000 /usr/lib/libfakelink.dylib -0x00007ff81ddcf000 /System/Library/Frameworks/CFNetwork.framework/Versions/A/CFNetwork -0x00007ff8246ce000 /usr/lib/libarchive.2.dylib -0x00007ff822ed4000 /System/Library/PrivateFrameworks/AppleSystemInfo.framework/Versions/A/AppleSystemInfo -0x00007ff823bcf000 /usr/lib/libbsm.0.dylib -0x00007ff81d405000 /usr/lib/libnetwork.dylib -0x00007ff824528000 /usr/lib/system/libkxld.dylib -0x00007ff8237fd000 /System/Library/PrivateFrameworks/IOMobileFramebuffer.framework/Versions/A/IOMobileFramebuffer -0x00007ffb2f4aa000 /usr/lib/libCoreEntitlements.dylib -0x00007ff82423b000 /System/Library/PrivateFrameworks/AppleFSCompression.framework/Versions/A/AppleFSCompression -0x00007ff823bb7000 /usr/lib/libcoretls.dylib -0x00007ff824d07000 /usr/lib/libcoretls_cfhelpers.dylib -0x00007ff82480f000 /usr/lib/libpam.2.dylib -0x00007ff81f7ba000 /usr/lib/libsqlite3.dylib -0x00007ff824d76000 /usr/lib/libxar.1.dylib -0x00007ff82424b000 /usr/lib/libbz2.1.0.dylib -0x00007ff824594000 /usr/lib/libpcap.A.dylib -0x00007ff81fb8b000 /usr/lib/libdns_services.dylib -0x00007ff8247e1000 /usr/lib/libapple_nghttp2.dylib -0x00007ff8245cb000 /usr/lib/libiconv.2.dylib -0x00007ff824521000 /usr/lib/libcharset.1.dylib -0x00007ff820f67000 /System/Library/Frameworks/CoreServices.framework/Versions/A/Frameworks/FSEvents.framework/Versions/A/FSEvents -0x00007ff81b996000 /System/Library/Frameworks/CoreServices.framework/Versions/A/Frameworks/CarbonCore.framework/Versions/A/CarbonCore -0x00007ff81fdce000 /System/Library/Frameworks/CoreServices.framework/Versions/A/Frameworks/Metadata.framework/Versions/A/Metadata -0x00007ff82425e000 /System/Library/Frameworks/CoreServices.framework/Versions/A/Frameworks/OSServices.framework/Versions/A/OSServices -0x00007ff824755000 /System/Library/Frameworks/CoreServices.framework/Versions/A/Frameworks/SearchKit.framework/Versions/A/SearchKit -0x00007ff81fb1d000 /System/Library/Frameworks/CoreServices.framework/Versions/A/Frameworks/AE.framework/Versions/A/AE -0x00007ff81997e000 /System/Library/Frameworks/CoreServices.framework/Versions/A/Frameworks/LaunchServices.framework/Versions/A/LaunchServices -0x00007ff824c9e000 /System/Library/Frameworks/CoreServices.framework/Versions/A/Frameworks/DictionaryServices.framework/Versions/A/DictionaryServices -0x00007ff820f74000 /System/Library/Frameworks/CoreServices.framework/Versions/A/Frameworks/SharedFileList.framework/Versions/A/SharedFileList -0x00007ff823bab000 /usr/lib/libCheckFix.dylib -0x00007ff81e2e0000 /System/Library/PrivateFrameworks/TCC.framework/Versions/A/TCC -0x00007ff822eda000 /System/Library/PrivateFrameworks/CoreNLP.framework/Versions/A/CoreNLP -0x00007ff81fd7e000 /System/Library/PrivateFrameworks/MetadataUtilities.framework/Versions/A/MetadataUtilities -0x00007ff819fea000 /usr/lib/libmecabra.dylib -0x00007ff823bdf000 /usr/lib/libmecab.dylib -0x00007ff819fb5000 /usr/lib/libCRFSuite.dylib -0x00007ff823c3c000 /usr/lib/libgermantok.dylib -0x00007ff8247bc000 /usr/lib/libThaiTokenizer.dylib -0x00007ff81fe70000 /System/Library/Frameworks/Accelerate.framework/Versions/A/Frameworks/vImage.framework/Versions/A/vImage -0x00007ff82a8bb000 /System/Library/Frameworks/Accelerate.framework/Versions/A/Frameworks/vecLib.framework/Versions/A/vecLib -0x00007ff824db7000 /System/Library/Frameworks/Accelerate.framework/Versions/A/Frameworks/vecLib.framework/Versions/A/libvMisc.dylib -0x00007ff8235f1000 /System/Library/Frameworks/Accelerate.framework/Versions/A/Frameworks/vecLib.framework/Versions/A/libvDSP.dylib -0x00007ff819cf6000 /System/Library/Frameworks/Accelerate.framework/Versions/A/Frameworks/vecLib.framework/Versions/A/libBLAS.dylib -0x00007ff8248f1000 /System/Library/Frameworks/Accelerate.framework/Versions/A/Frameworks/vecLib.framework/Versions/A/libLAPACK.dylib -0x00007ff823c3f000 /System/Library/Frameworks/Accelerate.framework/Versions/A/Frameworks/vecLib.framework/Versions/A/libLinearAlgebra.dylib -0x00007ff8247fa000 /System/Library/Frameworks/Accelerate.framework/Versions/A/Frameworks/vecLib.framework/Versions/A/libSparseBLAS.dylib -0x00007ff8248eb000 /System/Library/Frameworks/Accelerate.framework/Versions/A/Frameworks/vecLib.framework/Versions/A/libQuadrature.dylib -0x00007ff822fc4000 /System/Library/Frameworks/Accelerate.framework/Versions/A/Frameworks/vecLib.framework/Versions/A/libBNNS.dylib -0x00007ff819ecb000 /System/Library/Frameworks/Accelerate.framework/Versions/A/Frameworks/vecLib.framework/Versions/A/libSparse.dylib -0x00007ff822fbd000 /System/Library/PrivateFrameworks/LinguisticData.framework/Versions/A/LinguisticData -0x00007ff819e80000 /System/Library/PrivateFrameworks/Lexicon.framework/Versions/A/Lexicon -0x00007ff824db0000 /usr/lib/libChineseTokenizer.dylib -0x00007ff81a5e4000 /System/Library/PrivateFrameworks/LanguageModeling.framework/Versions/A/LanguageModeling -0x00007ff8247be000 /System/Library/PrivateFrameworks/AppleSauce.framework/Versions/A/AppleSauce -0x00007ff8237eb000 /System/Library/PrivateFrameworks/CoreEmoji.framework/Versions/A/CoreEmoji -0x00007ff8246bd000 /usr/lib/libcmph.dylib -0x00007ff820f4b000 /System/Library/Frameworks/OpenDirectory.framework/Versions/A/Frameworks/CFOpenDirectory.framework/Versions/A/CFOpenDirectory -0x00007ff820f3f000 /System/Library/Frameworks/OpenDirectory.framework/Versions/A/OpenDirectory -0x00007ff824d09000 /System/Library/PrivateFrameworks/APFS.framework/Versions/A/APFS -0x00007ff823b0c000 /System/Library/Frameworks/SecurityFoundation.framework/Versions/A/SecurityFoundation -0x00007ff824d84000 /usr/lib/libutil.dylib -0x00007ff81bcc1000 /System/Library/PrivateFrameworks/CoreServicesStore.framework/Versions/A/CoreServicesStore -0x00007ff824d88000 /usr/lib/libxslt.1.dylib -0x00007ff823b99000 /System/Library/PrivateFrameworks/BackgroundTaskManagement.framework/Versions/A/BackgroundTaskManagement -0x00007ff8262b1000 /System/Library/Frameworks/ImageIO.framework/Versions/A/Resources/libRadiance.dylib -0x00007ff8262ba000 /System/Library/Frameworks/ImageIO.framework/Versions/A/Resources/libJPEG.dylib -0x00007ff82620b000 /System/Library/Frameworks/ImageIO.framework/Versions/A/Resources/libPng.dylib -0x00007ff82622c000 /System/Library/Frameworks/ImageIO.framework/Versions/A/Resources/libTIFF.dylib -0x00007ff826318000 /System/Library/Frameworks/ImageIO.framework/Versions/A/Resources/libGIF.dylib -0x00007ff825b40000 /System/Library/Frameworks/ImageIO.framework/Versions/A/Resources/libJP2.dylib -0x00007ff825282000 /usr/lib/libexpat.1.dylib -0x00007ff825af5000 /System/Library/PrivateFrameworks/AppleJPEG.framework/Versions/A/AppleJPEG -0x00007ff822417000 /System/Library/PrivateFrameworks/FontServices.framework/libFontParser.dylib -0x00007ff81fac7000 /System/Library/PrivateFrameworks/RunningBoardServices.framework/Versions/A/RunningBoardServices -0x00007ff830a90000 /System/Library/PrivateFrameworks/IOSurfaceAccelerator.framework/Versions/A/IOSurfaceAccelerator -0x00007ff825c1f000 /System/Library/PrivateFrameworks/WatchdogClient.framework/Versions/A/WatchdogClient -0x00007ff81a6d3000 /System/Library/Frameworks/CoreDisplay.framework/Versions/A/CoreDisplay -0x00007ff822316000 /System/Library/Frameworks/CoreMedia.framework/Versions/A/CoreMedia -0x00007ff8221b3000 /System/Library/PrivateFrameworks/IOAccelerator.framework/Versions/A/IOAccelerator -0x00007ff8210c7000 /System/Library/Frameworks/CoreVideo.framework/Versions/A/CoreVideo -0x00007ff82480d000 /System/Library/Frameworks/MetalPerformanceShaders.framework/Versions/A/MetalPerformanceShaders -0x00007ff825c5a000 /System/Library/Frameworks/VideoToolbox.framework/Versions/A/VideoToolbox -0x00007ff81fa0f000 /System/Library/PrivateFrameworks/BaseBoard.framework/Versions/A/BaseBoard -0x00007ff81f7a0000 /System/Library/PrivateFrameworks/ProtocolBuffer.framework/Versions/A/ProtocolBuffer -0x00007ff8262aa000 /System/Library/PrivateFrameworks/GPUWrangler.framework/Versions/A/GPUWrangler -0x00007ff82628d000 /System/Library/PrivateFrameworks/IOPresentment.framework/Versions/A/IOPresentment -0x00007ff8262b4000 /System/Library/PrivateFrameworks/DSExternalDisplay.framework/Versions/A/DSExternalDisplay -0x00007ffa31832000 /System/Library/Frameworks/OpenGL.framework/Versions/A/Libraries/libCoreFSCache.dylib -0x00007ffb1d71a000 /System/Library/PrivateFrameworks/GPUCompiler.framework/Versions/31001/Libraries/libGPUCompilerUtils.dylib -0x00007ff82631d000 /System/Library/PrivateFrameworks/CMCaptureCore.framework/Versions/A/CMCaptureCore -0x00007ff82582e000 /System/Library/PrivateFrameworks/AppServerSupport.framework/Versions/A/AppServerSupport -0x00007ff827a04000 /System/Library/PrivateFrameworks/perfdata.framework/Versions/A/perfdata -0x00007ff81a7ff000 /System/Library/PrivateFrameworks/AudioToolboxCore.framework/Versions/A/AudioToolboxCore -0x00007ff8222f3000 /System/Library/PrivateFrameworks/caulk.framework/Versions/A/caulk -0x00007ff827328000 /usr/lib/libAudioStatistics.dylib -0x00007ff91e5e5000 /System/Library/PrivateFrameworks/SystemPolicy.framework/Versions/A/SystemPolicy -0x00007ff82757d000 /usr/lib/libSMC.dylib -0x00007ff8303c7000 /System/Library/Frameworks/CoreMIDI.framework/Versions/A/CoreMIDI -0x00007ff8261db000 /usr/lib/libAudioToolboxUtility.dylib -0x00007ff91a9f7000 /System/Library/PrivateFrameworks/OSAServicesClient.framework/Versions/A/OSAServicesClient -0x00007ff827a11000 /usr/lib/libperfcheck.dylib -0x00007ff91f077000 /usr/lib/libmis.dylib -0x00007ffa3188e000 /System/Library/Frameworks/OpenGL.framework/Versions/A/Libraries/libGLU.dylib -0x00007ffa31851000 /System/Library/Frameworks/OpenGL.framework/Versions/A/Libraries/libGFXShared.dylib -0x00007ffa31a5f000 /System/Library/Frameworks/OpenGL.framework/Versions/A/Libraries/libGL.dylib -0x00007ffa3185a000 /System/Library/Frameworks/OpenGL.framework/Versions/A/Libraries/libGLImage.dylib -0x00007ffa3184e000 /System/Library/Frameworks/OpenGL.framework/Versions/A/Libraries/libCVMSPluginSupport.dylib -0x00007ffa31839000 /System/Library/Frameworks/OpenGL.framework/Versions/A/Libraries/libCoreVMClient.dylib -0x00007ff822e58000 /System/Library/Frameworks/MetalPerformanceShaders.framework/Versions/A/Frameworks/MPSCore.framework/Versions/A/MPSCore -0x00007ff8241a6000 /System/Library/Frameworks/MetalPerformanceShaders.framework/Versions/A/Frameworks/MPSImage.framework/Versions/A/MPSImage -0x00007ff823c55000 /System/Library/Frameworks/MetalPerformanceShaders.framework/Versions/A/Frameworks/MPSNeuralNetwork.framework/Versions/A/MPSNeuralNetwork -0x00007ff8240ca000 /System/Library/Frameworks/MetalPerformanceShaders.framework/Versions/A/Frameworks/MPSMatrix.framework/Versions/A/MPSMatrix -0x00007ff823e86000 /System/Library/Frameworks/MetalPerformanceShaders.framework/Versions/A/Frameworks/MPSRayIntersector.framework/Versions/A/MPSRayIntersector -0x00007ff824103000 /System/Library/Frameworks/MetalPerformanceShaders.framework/Versions/A/Frameworks/MPSNDArray.framework/Versions/A/MPSNDArray -0x00007ffa36453000 /System/Library/Frameworks/MetalPerformanceShaders.framework/Versions/A/Frameworks/MPSFunctions.framework/Versions/A/MPSFunctions -0x00007ff819bd9000 /System/Library/PrivateFrameworks/MetalTools.framework/Versions/A/MetalTools -0x00007ff822ed2000 /System/Library/PrivateFrameworks/AggregateDictionary.framework/Versions/A/AggregateDictionary -0x00007ff825064000 /usr/lib/libIOReport.dylib -0x00007ffa32e6e000 /System/Library/Frameworks/OpenCL.framework/Versions/A/OpenCL -0x00007ff8253c8000 /System/Library/PrivateFrameworks/FaceCore.framework/Versions/A/FaceCore -0x00007ff8253b9000 /System/Library/PrivateFrameworks/GraphVisualizer.framework/Versions/A/GraphVisualizer -0x00007ffb1d625000 /System/Library/PrivateFrameworks/FontServices.framework/Versions/A/FontServices -0x00007ff8257e5000 /System/Library/PrivateFrameworks/OTSVG.framework/Versions/A/OTSVG -0x00007ff820caf000 /System/Library/Frameworks/ApplicationServices.framework/Versions/A/Frameworks/ATS.framework/Versions/A/Resources/libFontRegistry.dylib -0x00007ff825836000 /System/Library/PrivateFrameworks/FontServices.framework/libhvf.dylib -0x00007ffb1d626000 /System/Library/PrivateFrameworks/FontServices.framework/libXTFontStaticRegistryData.dylib -0x00007ff825232000 /System/Library/PrivateFrameworks/AppleVA.framework/Versions/A/AppleVA -0x00007ff82734c000 /System/Library/Frameworks/ApplicationServices.framework/Versions/A/Frameworks/ATS.framework/Versions/A/ATS -0x00007ff81eeba000 /System/Library/Frameworks/ApplicationServices.framework/Versions/A/Frameworks/HIServices.framework/Versions/A/HIServices -0x00007ff826327000 /System/Library/Frameworks/ApplicationServices.framework/Versions/A/Frameworks/PrintCore.framework/Versions/A/PrintCore -0x00007ff8276d0000 /System/Library/Frameworks/ApplicationServices.framework/Versions/A/Frameworks/QD.framework/Versions/A/QD -0x00007ff8276c8000 /System/Library/Frameworks/ApplicationServices.framework/Versions/A/Frameworks/ColorSyncLegacy.framework/Versions/A/ColorSyncLegacy -0x00007ff82733c000 /System/Library/Frameworks/ApplicationServices.framework/Versions/A/Frameworks/SpeechSynthesis.framework/Versions/A/SpeechSynthesis -0x00007ff8262e8000 /System/Library/Frameworks/ApplicationServices.framework/Versions/A/Frameworks/ATSUI.framework/Versions/A/ATSUI -0x00007ff82765d000 /usr/lib/libcups.2.dylib -0x00007ff827a20000 /System/Library/Frameworks/Kerberos.framework/Versions/A/Kerberos -0x00007ff827a2f000 /System/Library/Frameworks/GSS.framework/Versions/A/GSS -0x00007ff8273b8000 /usr/lib/libresolv.9.dylib -0x00007ff825845000 /System/Library/PrivateFrameworks/Heimdal.framework/Versions/A/Heimdal -0x00007ff82e51d000 /System/Library/Frameworks/Kerberos.framework/Versions/A/Libraries/libHeimdalProxy.dylib -0x00007ff81fb9c000 /System/Library/Frameworks/Network.framework/Versions/A/Network -0x00007ff82529c000 /usr/lib/libheimdal-asn1.dylib -0x00007ff827a7b000 /System/Library/PrivateFrameworks/CommonAuth.framework/Versions/A/CommonAuth -0x00007ff8272b9000 /System/Library/PrivateFrameworks/AudioSession.framework/Versions/A/AudioSession -0x00007ff82532c000 /System/Library/PrivateFrameworks/MediaExperience.framework/Versions/A/MediaExperience -0x00007ff82714d000 /System/Library/PrivateFrameworks/AudioSession.framework/libSessionUtility.dylib -0x00007ff8276dc000 /System/Library/PrivateFrameworks/AudioResourceArbitration.framework/Versions/A/AudioResourceArbitration -0x00007ff82b6ce000 /System/Library/PrivateFrameworks/PowerLog.framework/Versions/A/PowerLog -0x00007ff8252a5000 /System/Library/PrivateFrameworks/IconFoundation.framework/Versions/A/IconFoundation -0x00007ff82a4e8000 /System/Library/PrivateFrameworks/SpeechRecognitionCore.framework/Versions/A/SpeechRecognitionCore -0x000000010edf6000 /Users/sagarrao/Library/Java/JavaVirtualMachines/temurin-1.8.0_322/Contents/Home/jre/lib/server/libjvm.dylib -0x0000000108c2a000 /Users/sagarrao/Library/Java/JavaVirtualMachines/temurin-1.8.0_322/Contents/Home/jre/lib/libverify.dylib -0x0000000108c95000 /Users/sagarrao/Library/Java/JavaVirtualMachines/temurin-1.8.0_322/Contents/Home/jre/lib/libjava.dylib -0x0000000108d36000 /Users/sagarrao/Library/Java/JavaVirtualMachines/temurin-1.8.0_322/Contents/Home/jre/lib/libzip.dylib -0x000000010ecee000 /Users/sagarrao/Library/Java/JavaVirtualMachines/temurin-1.8.0_322/Contents/Home/jre/lib/libnio.dylib -0x000000010ed44000 /Users/sagarrao/Library/Java/JavaVirtualMachines/temurin-1.8.0_322/Contents/Home/jre/lib/libnet.dylib -0x0000000127213000 /Users/sagarrao/Library/Java/JavaVirtualMachines/temurin-1.8.0_322/Contents/Home/jre/lib/libmanagement.dylib -0x000000013243c000 /Users/sagarrao/Library/Java/JavaVirtualMachines/temurin-1.8.0_322/Contents/Home/jre/lib/libsunec.dylib - -VM Arguments: -jvm_args: -Djava.security.manager=worker.org.gradle.process.internal.worker.child.BootstrapSecurityManager -Dorg.gradle.internal.worker.tmpdir=/Users/sagarrao/gitprojects/kafka/connect/runtime/build/tmp/test/work -Dorg.gradle.native=false -Xss4m -XX:+UseParallelGC -Xmx2g -Dfile.encoding=UTF-8 -Duser.country=IN -Duser.language=en -Duser.variant -ea -java_command: worker.org.gradle.process.internal.worker.GradleWorkerMain 'Gradle Test Executor 59' -java_class_path (initial): /Users/sagarrao/.gradle/caches/7.5.1/workerMain/gradle-worker.jar -Launcher Type: SUN_STANDARD - -Environment Variables: -PATH=/opt/homebrew/bin:/opt/homebrew/sbin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/Users/sagarrao/bin -SHELL=/bin/zsh - -Signal Handlers: -SIGSEGV: [libjvm.dylib+0x59c573], sa_mask[0]=11111111111111111111111111111111, sa_flags=SA_ONSTACK|SA_RESTART|SA_SIGINFO -SIGBUS: [libjvm.dylib+0x59c573], sa_mask[0]=11111111111111111111111111111111, sa_flags=SA_RESTART|SA_SIGINFO -SIGFPE: [libjvm.dylib+0x48f207], sa_mask[0]=11111111111111111111111111111111, sa_flags=SA_RESTART|SA_SIGINFO -SIGPIPE: [libjvm.dylib+0x48f207], sa_mask[0]=11111111111111111111111111111111, sa_flags=SA_RESTART|SA_SIGINFO -SIGXFSZ: [libjvm.dylib+0x48f207], sa_mask[0]=11111111111111111111111111111111, sa_flags=SA_RESTART|SA_SIGINFO -SIGILL: [libjvm.dylib+0x48f207], sa_mask[0]=11111111111111111111111111111111, sa_flags=SA_RESTART|SA_SIGINFO -SIGUSR1: SIG_DFL, sa_mask[0]=00000000000000000000000000000000, sa_flags=none -SIGUSR2: [libjvm.dylib+0x48fafe], sa_mask[0]=00100000000000000000000000000000, sa_flags=SA_RESTART|SA_SIGINFO -SIGHUP: [libjvm.dylib+0x48dcf1], sa_mask[0]=11111111111111111111111111111111, sa_flags=SA_RESTART|SA_SIGINFO -SIGINT: [libjvm.dylib+0x48dcf1], sa_mask[0]=11111111111111111111111111111111, sa_flags=SA_RESTART|SA_SIGINFO -SIGTERM: [libjvm.dylib+0x48dcf1], sa_mask[0]=11111111111111111111111111111111, sa_flags=SA_RESTART|SA_SIGINFO -SIGQUIT: [libjvm.dylib+0x48dcf1], sa_mask[0]=11111111111111111111111111111111, sa_flags=SA_RESTART|SA_SIGINFO - - ---------------- S Y S T E M --------------- - -OS:Bsduname:Darwin 21.3.0 Darwin Kernel Version 21.3.0: Wed Jan 5 21:37:58 PST 2022; root:xnu-8019.80.24~20/RELEASE_ARM64_T6000 x86_64 -rlimit: STACK 8176k, CORE 0k, NPROC 2666, NOFILE 10240, AS infinity -load average:22.63 15.85 14.55 - -CPU:total 10 (initial active 10) (1 cores per cpu, 1 threads per core) family 6 model 44 stepping 0, cmov, cx8, fxsr, mmx, sse, sse2, sse3, ssse3, sse4.1, sse4.2, popcnt, aes, clmul, tsc, tscinvbit, tscinv - -Memory: 4k page, physical 16777216k(18992k free) - -/proc/meminfo: - - -vm_info: OpenJDK 64-Bit Server VM (25.322-b06) for bsd-amd64 JRE (1.8.0_322-b06), built on Jan 19 2022 13:09:06 by "jenkins" with gcc 4.2.1 Compatible Apple LLVM 10.0.1 (clang-1001.0.46.4) - -time: Wed Sep 14 22:15:10 2022 -timezone: IST -elapsed time: 54.145735 seconds (0d 0h 0m 54s) - From bce63341166ff956ac70d32a3d8b0d2ba8b0cc0b Mon Sep 17 00:00:00 2001 From: Sagar Rao Date: Thu, 15 Sep 2022 08:12:27 +0530 Subject: [PATCH 15/26] Increasing exp backoff multiplier to 40 --- .../runtime/distributed/IncrementalCooperativeAssignor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java index d506d565ca34d..fd815a2e42464 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java @@ -92,7 +92,7 @@ public IncrementalCooperativeAssignor(LogContext logContext, Time time, int maxD this.numSuccessiveRevokingRebalances = 0; // By default, initial interval is 1. The only corner case is when the user has set maxDelay to 0 // in which case, the exponential backoff delay should be 0 which would return the backoff delay to be 0 always - this.consecutiveRevokingRebalancesBackoff = new ExponentialBackoff(maxDelay == 0 ? 0 : 1, 30, maxDelay, 0); + this.consecutiveRevokingRebalancesBackoff = new ExponentialBackoff(maxDelay == 0 ? 0 : 1, 40, maxDelay, 0); } @Override From fb879f1811c956a3aef6dd93535d26019c961466 Mon Sep 17 00:00:00 2001 From: Sagar Rao Date: Sat, 17 Sep 2022 13:36:40 +0530 Subject: [PATCH 16/26] Removing resetting numSuccessiveRevokingRebalances to 0 --- .../IncrementalCooperativeAssignor.java | 3 +-- .../IncrementalCooperativeAssignorTest.java | 26 +++++++++---------- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java index fd815a2e42464..7726889a996ab 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java @@ -302,7 +302,7 @@ ClusterAssignment performTaskAssignment( // the next round when revoking rebalance would be allowed. Note that delay could be 0, in which // case we would always revoke. if (revokedInPrevious && !toExplicitlyRevoke.isEmpty()) { - numSuccessiveRevokingRebalances++; + numSuccessiveRevokingRebalances++; // Should we consider overflow for this? log.debug("Consecutive revoking rebalances observed. Computing delay and next scheduled rebalance."); delay = (int) consecutiveRevokingRebalancesBackoff.backoff(numSuccessiveRevokingRebalances); if (delay != 0) { @@ -323,7 +323,6 @@ ClusterAssignment performTaskAssignment( // have converged to a balanced load. We can reset the rebalance clock log.debug("Previous round had revocations but this round didn't. Probably, the cluster has reached a " + "balanced load. Resetting the exponential backoff clock"); - numSuccessiveRevokingRebalances = 0; revokedInPrevious = false; } else { // no-op diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java index 7d6959356772b..9087f28849795 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java @@ -150,32 +150,32 @@ public void testAssignmentsWhenWorkersJoinAfterRevocations() { assertTaskAllocations(3, 3, 6); // Fourth assignment and a fourth worker joining - // Since the worker is joining immediately and within the rebalance delay - // there should not be any revoking rebalance + // after first revoking rebalance is expired. We should revoke. + time.sleep(assignor.delay); addNewEmptyWorkers("worker4"); performStandardRebalance(); assertWorkers("worker1", "worker2", "worker3", "worker4"); - assertConnectorAllocations(0, 0, 1, 2); - assertTaskAllocations(0, 3, 3, 6); + assertConnectorAllocations(0, 0, 1, 1); + assertTaskAllocations(0, 3, 3, 3); - // Add new worker immediately. Since a scheduled rebalance is in progress, - // There should still not be be any revocations + // Fifth assignment and a fifth worker joining after a revoking rebalance. + // We shouldn't revoke and set a delay > initial interval addNewEmptyWorkers("worker5"); performStandardRebalance(); - assertTrue(assignor.delay > 0); + assertTrue(assignor.delay > 40); assertWorkers("worker1", "worker2", "worker3", "worker4", "worker5"); - assertConnectorAllocations(0, 0, 0, 1, 2); - assertTaskAllocations(0, 0, 3, 3, 6); + assertConnectorAllocations(0, 0, 1, 1, 1); + assertTaskAllocations(1, 2, 3, 3, 3); - // Add new worker but this time after crossing the delay. - // There would be revocations allowed + // Sixth assignment with sixth worker joining after the expiry. + // Should revoke time.sleep(assignor.delay); addNewEmptyWorkers("worker6"); performStandardRebalance(); assertDelay(0); assertWorkers("worker1", "worker2", "worker3", "worker4", "worker5", "worker6"); - assertConnectorAllocations(0, 0, 0, 0, 1, 1); - assertTaskAllocations(0, 0, 0, 2, 2, 2); + assertConnectorAllocations(0, 0, 0, 1, 1, 1); + assertTaskAllocations(0, 1, 2, 2, 2, 2); // Follow up rebalance since there were revocations performStandardRebalance(); From 0e8f84e6cb9bfd5459388e0ce70b553052be4ec1 Mon Sep 17 00:00:00 2001 From: Sagar Rao Date: Thu, 13 Oct 2022 21:05:17 +0530 Subject: [PATCH 17/26] Resetting revokedInPrevious to false when there's no rebalance + few tests moved around/fixed --- .../IncrementalCooperativeAssignor.java | 4 ++- .../IncrementalCooperativeAssignorTest.java | 4 +++ .../WorkerCoordinatorIncrementalTest.java | 35 ++++++++++++++++++- 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java index 7726889a996ab..0264b26962b1a 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java @@ -324,12 +324,14 @@ ClusterAssignment performTaskAssignment( log.debug("Previous round had revocations but this round didn't. Probably, the cluster has reached a " + "balanced load. Resetting the exponential backoff clock"); revokedInPrevious = false; + numSuccessiveRevokingRebalances = 0; } else { // no-op log.debug("No revocations in previous and current round."); } } else { log.debug("Delayed rebalance is active. Delaying {}ms before revoking connectors and tasks: {}", delay, toRevoke); + revokedInPrevious = false; } assignConnectors(completeWorkerAssignment, newSubmissions.connectors()); @@ -494,7 +496,7 @@ protected void handleLostAssignments(ConnectorsAndTasks lostAssignments, ConnectorsAndTasks newSubmissions, List completeWorkerAssignment) { // There are no lost assignments and there have been no successive revoking rebalances - if (lostAssignments.isEmpty() && numSuccessiveRevokingRebalances == 0) { + if (lostAssignments.isEmpty() && !revokedInPrevious) { resetDelay(); return; } diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java index 9087f28849795..75e9310dfc1c8 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java @@ -89,6 +89,9 @@ public void initAssignor() { @Test public void testTaskAssignmentWhenWorkerJoins() { + // Customize assignor for this test case + time = new MockTime(); + initAssignor(); // First assignment with 1 worker and 2 connectors configured but not yet assigned performStandardRebalance(); assertDelay(0); @@ -190,6 +193,7 @@ public void testImmediateRevocationsWhenMaxDelayIs0() { // Customize assignor for this test case rebalanceDelay = 0; + time = new MockTime(); initAssignor(); addNewConnector("connector3", 4); diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinatorIncrementalTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinatorIncrementalTest.java index d8fb647201873..160011e7cf2e3 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinatorIncrementalTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinatorIncrementalTest.java @@ -456,6 +456,14 @@ public void testTaskAssignmentWhenWorkerBounces() { Collections.emptyList(), 0, anotherMemberAssignment); + ExtendedAssignment leaderAssignment1 = leaderAssignment; + ExtendedAssignment memberAssignment1 = memberAssignment; + ExtendedAssignment anotherMemberAssignment1 = anotherMemberAssignment; + + System.out.println(leaderAssignment); + System.out.println(memberAssignment); + System.out.println(anotherMemberAssignment); + // Second rebalance detects a worker is missing coordinator.metadata(); ++configStorageCalls; @@ -513,7 +521,7 @@ public void testTaskAssignmentWhenWorkerBounces() { result = coordinator.onLeaderElected(leaderId, compatibility.protocol(), responseMembers, false); - // A rebalance after the delay expires re-assigns the lost tasks to the returning member + // A rebalance after the delay expires leads to a revoking rebalance leaderAssignment = deserializeAssignment(result, leaderId); assertAssignment(leaderId, offset, Collections.emptyList(), 0, @@ -532,6 +540,31 @@ public void testTaskAssignmentWhenWorkerBounces() { Collections.emptyList(), 0, anotherMemberAssignment); + responseMembers.clear(); + addJoinGroupResponseMember(responseMembers, leaderId, offset, leaderAssignment1); + addJoinGroupResponseMember(responseMembers, memberId, offset, memberAssignment1); + addJoinGroupResponseMember(responseMembers, anotherMemberId, offset, anotherMemberAssignment1); + + result = coordinator.onLeaderElected(leaderId, compatibility.protocol(), responseMembers, false); + + leaderAssignment = deserializeAssignment(result, leaderId); + assertAssignment(leaderId, offset, + Collections.emptyList(), 0, + Collections.emptyList(), 0, + leaderAssignment); + + memberAssignment = deserializeAssignment(result, memberId); + assertAssignment(leaderId, offset, + Collections.emptyList(), 0, + Collections.emptyList(), 0, + memberAssignment); + + anotherMemberAssignment = deserializeAssignment(result, anotherMemberId); + assertAssignment(leaderId, offset, + Collections.emptyList(), 0, + Collections.emptyList(), 0, + anotherMemberAssignment); + verify(configStorage, times(configStorageCalls)).snapshot(); } From 8d63bf53ad05094e53924cb71c6a6af12b2ae8d4 Mon Sep 17 00:00:00 2001 From: Sagar Rao Date: Thu, 13 Oct 2022 21:30:30 +0530 Subject: [PATCH 18/26] Removing unwanted sysout --- .../runtime/distributed/WorkerCoordinatorIncrementalTest.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinatorIncrementalTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinatorIncrementalTest.java index 160011e7cf2e3..ead229339d81f 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinatorIncrementalTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinatorIncrementalTest.java @@ -460,9 +460,6 @@ public void testTaskAssignmentWhenWorkerBounces() { ExtendedAssignment memberAssignment1 = memberAssignment; ExtendedAssignment anotherMemberAssignment1 = anotherMemberAssignment; - System.out.println(leaderAssignment); - System.out.println(memberAssignment); - System.out.println(anotherMemberAssignment); // Second rebalance detects a worker is missing coordinator.metadata(); @@ -541,6 +538,7 @@ public void testTaskAssignmentWhenWorkerBounces() { anotherMemberAssignment); responseMembers.clear(); + addJoinGroupResponseMember(responseMembers, leaderId, offset, leaderAssignment1); addJoinGroupResponseMember(responseMembers, memberId, offset, memberAssignment1); addJoinGroupResponseMember(responseMembers, anotherMemberId, offset, anotherMemberAssignment1); From f209a1a8882e23401e67e3c1520c8c84b175f572 Mon Sep 17 00:00:00 2001 From: Sagar Rao Date: Fri, 14 Oct 2022 17:25:21 +0530 Subject: [PATCH 19/26] Reverting test --- .../WorkerCoordinatorIncrementalTest.java | 33 +------------------ 1 file changed, 1 insertion(+), 32 deletions(-) diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinatorIncrementalTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinatorIncrementalTest.java index ead229339d81f..d8fb647201873 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinatorIncrementalTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinatorIncrementalTest.java @@ -456,11 +456,6 @@ public void testTaskAssignmentWhenWorkerBounces() { Collections.emptyList(), 0, anotherMemberAssignment); - ExtendedAssignment leaderAssignment1 = leaderAssignment; - ExtendedAssignment memberAssignment1 = memberAssignment; - ExtendedAssignment anotherMemberAssignment1 = anotherMemberAssignment; - - // Second rebalance detects a worker is missing coordinator.metadata(); ++configStorageCalls; @@ -518,7 +513,7 @@ public void testTaskAssignmentWhenWorkerBounces() { result = coordinator.onLeaderElected(leaderId, compatibility.protocol(), responseMembers, false); - // A rebalance after the delay expires leads to a revoking rebalance + // A rebalance after the delay expires re-assigns the lost tasks to the returning member leaderAssignment = deserializeAssignment(result, leaderId); assertAssignment(leaderId, offset, Collections.emptyList(), 0, @@ -537,32 +532,6 @@ public void testTaskAssignmentWhenWorkerBounces() { Collections.emptyList(), 0, anotherMemberAssignment); - responseMembers.clear(); - - addJoinGroupResponseMember(responseMembers, leaderId, offset, leaderAssignment1); - addJoinGroupResponseMember(responseMembers, memberId, offset, memberAssignment1); - addJoinGroupResponseMember(responseMembers, anotherMemberId, offset, anotherMemberAssignment1); - - result = coordinator.onLeaderElected(leaderId, compatibility.protocol(), responseMembers, false); - - leaderAssignment = deserializeAssignment(result, leaderId); - assertAssignment(leaderId, offset, - Collections.emptyList(), 0, - Collections.emptyList(), 0, - leaderAssignment); - - memberAssignment = deserializeAssignment(result, memberId); - assertAssignment(leaderId, offset, - Collections.emptyList(), 0, - Collections.emptyList(), 0, - memberAssignment); - - anotherMemberAssignment = deserializeAssignment(result, anotherMemberId); - assertAssignment(leaderId, offset, - Collections.emptyList(), 0, - Collections.emptyList(), 0, - anotherMemberAssignment); - verify(configStorage, times(configStorageCalls)).snapshot(); } From 71297c35133a7696549a5bb172afe20e095a8053 Mon Sep 17 00:00:00 2001 From: Sagar Rao Date: Tue, 15 Nov 2022 21:27:34 +0530 Subject: [PATCH 20/26] Reenabling RebalanceSourceConnectorsIntegrationTest#testDeleteConnector test --- .../integration/RebalanceSourceConnectorsIntegrationTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/RebalanceSourceConnectorsIntegrationTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/RebalanceSourceConnectorsIntegrationTest.java index 2aaeae7237cda..0b730cb55bc68 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/RebalanceSourceConnectorsIntegrationTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/RebalanceSourceConnectorsIntegrationTest.java @@ -188,7 +188,6 @@ public void testReconfigConnector() throws Exception { } @Test - @Ignore // TODO: To be re-enabled once we can make it less flaky (KAFKA-8391) public void testDeleteConnector() throws Exception { // create test topic connect.kafka().createTopic(TOPIC_NAME, NUM_TOPIC_PARTITIONS); From 0261a23fc3574dbf9732b30451d718509e744fb8 Mon Sep 17 00:00:00 2001 From: Chris Egerton Date: Sun, 23 Oct 2022 18:56:54 -0400 Subject: [PATCH 21/26] KAFKA-12495: Improve rebalance allocation algorithm --- .../IncrementalCooperativeAssignor.java | 404 +++++++++++------- .../distributed/WorkerCoordinator.java | 29 +- .../IncrementalCooperativeAssignorTest.java | 212 +++++---- .../WorkerCoordinatorIncrementalTest.java | 4 +- 4 files changed, 408 insertions(+), 241 deletions(-) diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java index 0264b26962b1a..409654ee5320c 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java @@ -245,10 +245,9 @@ ClusterAssignment performTaskAssignment( ConnectorsAndTasks deleted = diff(previousAssignment, configured); log.debug("Deleted assignments: {}", deleted); - // Derived set: The set of remaining active connectors-and-tasks is a derived set from the - // set difference of active - deleted - ConnectorsAndTasks remainingActive = diff(activeAssignments, deleted); - log.debug("Remaining (excluding deleted) active assignments: {}", remainingActive); + // The connectors and tasks that are currently running on more than one worker each + ConnectorsAndTasks duplicated = duplicatedAssignments(memberAssignments); + log.trace("Duplicated assignments: {}", duplicated); // Derived set: The set of lost or unaccounted connectors-and-tasks is a derived set from // the set difference of previous - active - deleted @@ -257,51 +256,47 @@ ClusterAssignment performTaskAssignment( // Derived set: The set of new connectors-and-tasks is a derived set from the set // difference of configured - previous - active - ConnectorsAndTasks newSubmissions = diff(configured, previousAssignment, activeAssignments); - log.debug("New assignments: {}", newSubmissions); - - // A collection of the complete assignment - List completeWorkerAssignment = workerAssignment(memberAssignments, ConnectorsAndTasks.EMPTY); - log.debug("Complete (ignoring deletions) worker assignments: {}", completeWorkerAssignment); - - // Per worker connector assignments without removing deleted connectors yet - Map> connectorAssignments = - completeWorkerAssignment.stream().collect(Collectors.toMap(WorkerLoad::worker, WorkerLoad::connectors)); - log.debug("Complete (ignoring deletions) connector assignments: {}", connectorAssignments); - - // Per worker task assignments without removing deleted connectors yet - Map> taskAssignments = - completeWorkerAssignment.stream().collect(Collectors.toMap(WorkerLoad::worker, WorkerLoad::tasks)); - log.debug("Complete (ignoring deletions) task assignments: {}", taskAssignments); + ConnectorsAndTasks created = diff(configured, previousAssignment, activeAssignments); + log.debug("New assignments: {}", created); // A collection of the current assignment excluding the connectors-and-tasks to be deleted List currentWorkerAssignment = workerAssignment(memberAssignments, deleted); - Map toRevoke = computeDeleted(deleted, connectorAssignments, taskAssignments); - log.debug("Connector and task to delete assignments: {}", toRevoke); + Map toRevoke = new HashMap<>(); - // Revoking redundant connectors/tasks if the workers have duplicate assignments - toRevoke.putAll(computeDuplicatedAssignments(memberAssignments, connectorAssignments, taskAssignments)); - log.debug("Connector and task to revoke assignments (include duplicated assignments): {}", toRevoke); - - // Recompute the complete assignment excluding the deleted connectors-and-tasks - completeWorkerAssignment = workerAssignment(memberAssignments, deleted); - connectorAssignments = - completeWorkerAssignment.stream().collect(Collectors.toMap(WorkerLoad::worker, WorkerLoad::connectors)); - taskAssignments = - completeWorkerAssignment.stream().collect(Collectors.toMap(WorkerLoad::worker, WorkerLoad::tasks)); + Map deletedAndRevoked = intersection(deleted, memberAssignments); + log.debug("Deleted connectors and tasks to revoke from each worker: {}", deletedAndRevoked); + addAll(toRevoke, deletedAndRevoked); - handleLostAssignments(lostAssignments, newSubmissions, completeWorkerAssignment); + // Revoking redundant connectors/tasks if the workers have duplicate assignments + Map duplicatedAndRevoked = intersection(duplicated, memberAssignments); + log.debug("Duplicated connectors and tasks to revoke from each worker: {}", duplicatedAndRevoked); + addAll(toRevoke, duplicatedAndRevoked); + + // Compute the assignment that will be applied across the cluster after this round of rebalance + // Later on, new submissions and lost-and-reassigned connectors and tasks will be added to these assignments, + // and load-balancing revocations will be removed from them. + List nextWorkerAssignment = workerLoads(memberAssignments); + removeAll(nextWorkerAssignment, deletedAndRevoked); + removeAll(nextWorkerAssignment, duplicatedAndRevoked); + + // Collect the lost assignments that are ready to be reassigned because the workers that were + // originally responsible for them appear to have left the cluster instead of rejoining within + // the scheduled rebalance delay. These assignments will be re-allocated to the existing workers + // in the cluster later on + ConnectorsAndTasks.Builder lostAssignmentsToReassignBuilder = new ConnectorsAndTasks.Builder(); + handleLostAssignments(lostAssignments, lostAssignmentsToReassignBuilder, nextWorkerAssignment); + ConnectorsAndTasks lostAssignmentsToReassign = lostAssignmentsToReassignBuilder.build(); // Do not revoke resources for re-assignment while a delayed rebalance is active if (delay == 0) { - Map toExplicitlyRevoke = - performTaskRevocation(activeAssignments, currentWorkerAssignment); + Map loadBalancingRevocations = + performLoadBalancingRevocations(configured, nextWorkerAssignment); // If this round and the previous round involved revocation, we will calculate a delay for // the next round when revoking rebalance would be allowed. Note that delay could be 0, in which // case we would always revoke. - if (revokedInPrevious && !toExplicitlyRevoke.isEmpty()) { + if (revokedInPrevious && !loadBalancingRevocations.isEmpty()) { numSuccessiveRevokingRebalances++; // Should we consider overflow for this? log.debug("Consecutive revoking rebalances observed. Computing delay and next scheduled rebalance."); delay = (int) consecutiveRevokingRebalancesBackoff.backoff(numSuccessiveRevokingRebalances); @@ -311,12 +306,18 @@ ClusterAssignment performTaskAssignment( delay, scheduledRebalance); } else { log.debug("Revoking assignments immediately since scheduled.rebalance.max.delay.ms is set to 0"); - revoke(toRevoke, toExplicitlyRevoke); + addAll(toRevoke, loadBalancingRevocations); + // Remove all newly-revoked connectors and tasks from the next assignment, both to + // ensure that they are not included in the assignments during this round, and to produce + // an accurate allocation of all newly-created and lost-and-reassigned connectors and tasks + // that will have to be distributed across the cluster during this round + removeAll(nextWorkerAssignment, loadBalancingRevocations); } - } else if (!toExplicitlyRevoke.isEmpty()) { + } else if (!loadBalancingRevocations.isEmpty()) { // We had a revocation in this round but not in the previous round. Let's store that state. log.debug("Performing allocation-balancing revocation immediately as no revocations took place during the previous rebalance"); - revoke(toRevoke, toExplicitlyRevoke); + addAll(toRevoke, loadBalancingRevocations); + removeAll(nextWorkerAssignment, loadBalancingRevocations); revokedInPrevious = true; } else if (revokedInPrevious) { // No revocations in this round but the previous round had one. Probably the workers @@ -334,49 +335,56 @@ ClusterAssignment performTaskAssignment( revokedInPrevious = false; } - assignConnectors(completeWorkerAssignment, newSubmissions.connectors()); - assignTasks(completeWorkerAssignment, newSubmissions.tasks()); - log.debug("Current complete assignments: {}", currentWorkerAssignment); - log.debug("New complete assignments: {}", completeWorkerAssignment); + // The complete set of connectors and tasks that should be newly-assigned during this round + ConnectorsAndTasks toAssign = new ConnectorsAndTasks.Builder() + .addConnectors(created.connectors()) + .addTasks(created.tasks()) + .addConnectors(lostAssignmentsToReassign.connectors()) + .addTasks(lostAssignmentsToReassign.tasks()) + .build(); + + assignConnectors(nextWorkerAssignment, toAssign.connectors()); + assignTasks(nextWorkerAssignment, toAssign.tasks()); + + Map> nextConnectorAssignments = nextWorkerAssignment.stream() + .collect(Collectors.toMap( + WorkerLoad::worker, + WorkerLoad::connectors + )); + Map> nextTaskAssignments = nextWorkerAssignment.stream() + .collect(Collectors.toMap( + WorkerLoad::worker, + WorkerLoad::tasks + )); Map> currentConnectorAssignments = currentWorkerAssignment.stream().collect(Collectors.toMap(WorkerLoad::worker, WorkerLoad::connectors)); Map> currentTaskAssignments = currentWorkerAssignment.stream().collect(Collectors.toMap(WorkerLoad::worker, WorkerLoad::tasks)); Map> incrementalConnectorAssignments = - diff(connectorAssignments, currentConnectorAssignments); + diff(nextConnectorAssignments, currentConnectorAssignments); Map> incrementalTaskAssignments = - diff(taskAssignments, currentTaskAssignments); + diff(nextTaskAssignments, currentTaskAssignments); - previousAssignment = computePreviousAssignment(toRevoke, connectorAssignments, taskAssignments, lostAssignments); + Map revoked = buildAll(toRevoke); + + previousAssignment = computePreviousAssignment(revoked, nextConnectorAssignments, nextTaskAssignments, lostAssignments); previousGenerationId = currentGenerationId; previousMembers = memberAssignments.keySet(); log.debug("Incremental connector assignments: {}", incrementalConnectorAssignments); log.debug("Incremental task assignments: {}", incrementalTaskAssignments); - Map> revokedConnectors = transformValues(toRevoke, ConnectorsAndTasks::connectors); - Map> revokedTasks = transformValues(toRevoke, ConnectorsAndTasks::tasks); + Map> revokedConnectors = transformValues(revoked, ConnectorsAndTasks::connectors); + Map> revokedTasks = transformValues(revoked, ConnectorsAndTasks::tasks); return new ClusterAssignment( incrementalConnectorAssignments, incrementalTaskAssignments, revokedConnectors, revokedTasks, - diff(connectorAssignments, revokedConnectors), - diff(taskAssignments, revokedTasks) - ); - } - - private void revoke(Map toRevoke, Map toExplicitlyRevoke) { - toExplicitlyRevoke.forEach( - (worker, assignment) -> { - ConnectorsAndTasks existing = toRevoke.computeIfAbsent( - worker, - v -> new ConnectorsAndTasks.Builder().build()); - existing.connectors().addAll(assignment.connectors()); - existing.tasks().addAll(assignment.tasks()); - } + diff(nextConnectorAssignments, revokedConnectors), + diff(nextTaskAssignments, revokedTasks) ); } @@ -493,7 +501,7 @@ private Map computeDuplicatedAssignments(Map completeWorkerAssignment) { // There are no lost assignments and there have been no successive revoking rebalances if (lostAssignments.isEmpty() && !revokedInPrevious) { @@ -513,8 +521,8 @@ protected void handleLostAssignments(ConnectorsAndTasks lostAssignments, + "missing assignments that the leader is detecting are probably due to some " + "workers failing to receive the new assignments in the previous rebalance. " + "Will reassign missing tasks as new tasks"); - newSubmissions.connectors().addAll(lostAssignments.connectors()); - newSubmissions.tasks().addAll(lostAssignments.tasks()); + lostAssignmentsToReassign.addConnectors(lostAssignments.connectors()); + lostAssignmentsToReassign.addTasks(lostAssignments.tasks()); return; } @@ -551,8 +559,8 @@ protected void handleLostAssignments(ConnectorsAndTasks lostAssignments, } } else { log.debug("No single candidate worker was found to assign lost tasks. Treating lost tasks as new tasks"); - newSubmissions.connectors().addAll(lostAssignments.connectors()); - newSubmissions.tasks().addAll(lostAssignments.tasks()); + lostAssignmentsToReassign.addConnectors(lostAssignments.connectors()); + lostAssignmentsToReassign.addTasks(lostAssignments.tasks()); } resetDelay(); // Resetting the flag as now we can permit successive revoking rebalances. @@ -604,89 +612,6 @@ private List pickCandidateWorkerForReassignment(List com .collect(Collectors.toList()); } - /** - * Task revocation is based on a rough estimation of the lower average number of tasks before - * and after new workers join the group. If no new workers join, no revocation takes place. - * Based on this estimation, tasks are revoked until the new floor average is reached for - * each existing worker. The revoked tasks, once assigned to the new workers will maintain - * a balanced load among the group. - * - * @param activeAssignments - * @param completeWorkerAssignment - * @return - */ - private Map performTaskRevocation(ConnectorsAndTasks activeAssignments, - Collection completeWorkerAssignment) { - int totalActiveConnectorsNum = activeAssignments.connectors().size(); - int totalActiveTasksNum = activeAssignments.tasks().size(); - Collection existingWorkers = completeWorkerAssignment.stream() - .filter(wl -> wl.size() > 0) - .collect(Collectors.toList()); - int existingWorkersNum = existingWorkers.size(); - int totalWorkersNum = completeWorkerAssignment.size(); - int newWorkersNum = totalWorkersNum - existingWorkersNum; - - if (log.isDebugEnabled()) { - completeWorkerAssignment.forEach(wl -> log.debug( - "Per worker current load size; worker: {} connectors: {} tasks: {}", - wl.worker(), wl.connectorsSize(), wl.tasksSize())); - } - - Map revoking = new HashMap<>(); - // If there are no new workers, or no existing workers to revoke tasks from return early - // after logging the status - if (!(newWorkersNum > 0 && existingWorkersNum > 0)) { - log.debug("No task revocation required; workers with existing load: {} workers with " - + "no load {} total workers {}", - existingWorkersNum, newWorkersNum, totalWorkersNum); - // This is intentionally empty but mutable, because the map is used to include deleted - // connectors and tasks as well - return revoking; - } - - log.debug("Task revocation is required; workers with existing load: {} workers with " - + "no load {} total workers {}", - existingWorkersNum, newWorkersNum, totalWorkersNum); - - // We have at least one worker assignment (the leader itself) so totalWorkersNum can't be 0 - log.debug("Previous rounded down (floor) average number of connectors per worker {}", totalActiveConnectorsNum / existingWorkersNum); - int floorConnectors = totalActiveConnectorsNum / totalWorkersNum; - int ceilConnectors = floorConnectors + ((totalActiveConnectorsNum % totalWorkersNum == 0) ? 0 : 1); - log.debug("New average number of connectors per worker rounded down (floor) {} and rounded up (ceil) {}", floorConnectors, ceilConnectors); - - - log.debug("Previous rounded down (floor) average number of tasks per worker {}", totalActiveTasksNum / existingWorkersNum); - int floorTasks = totalActiveTasksNum / totalWorkersNum; - int ceilTasks = floorTasks + ((totalActiveTasksNum % totalWorkersNum == 0) ? 0 : 1); - log.debug("New average number of tasks per worker rounded down (floor) {} and rounded up (ceil) {}", floorTasks, ceilTasks); - int numToRevoke; - - for (WorkerLoad existing : existingWorkers) { - Iterator connectors = existing.connectors().iterator(); - numToRevoke = existing.connectorsSize() - ceilConnectors; - for (int i = existing.connectorsSize(); i > floorConnectors && numToRevoke > 0; --i, --numToRevoke) { - ConnectorsAndTasks resources = revoking.computeIfAbsent( - existing.worker(), - w -> new ConnectorsAndTasks.Builder().build()); - resources.connectors().add(connectors.next()); - } - } - - for (WorkerLoad existing : existingWorkers) { - Iterator tasks = existing.tasks().iterator(); - numToRevoke = existing.tasksSize() - ceilTasks; - log.debug("Tasks on worker {} is higher than ceiling, so revoking {} tasks", existing, numToRevoke); - for (int i = existing.tasksSize(); i > floorTasks && numToRevoke > 0; --i, --numToRevoke) { - ConnectorsAndTasks resources = revoking.computeIfAbsent( - existing.worker(), - w -> new ConnectorsAndTasks.Builder().build()); - resources.tasks().add(tasks.next()); - } - } - - return revoking; - } - private Map fillAssignments(Collection members, short error, String leaderId, String leaderUrl, long maxOffset, ClusterAssignment clusterAssignment, @@ -753,6 +678,146 @@ private ConnectorsAndTasks assignment(Map memberAssi ).build(); } + /** + * Revoke connectors and tasks from each worker in the cluster until no worker is running more than it would be if: + *
    + *
  • The allocation of connectors and tasks across the cluster were as balanced as possible (i.e., the difference in allocation size between any two workers is at most one)
    • + *
  • Any workers that left the group within the scheduled rebalance delay permanently left the group
    • + *
  • All currently-configured connectors and tasks were allocated (including instances that may be revoked in this round because they are duplicated across workers)
    • + *
+ * @param configured the set of configured connectors and tasks across the entire cluster + * @param workers the workers in the cluster, whose assignments should not include any deleted or duplicated connectors or tasks + * that are already due to be revoked from the worker in this rebalance + * @return which connectors and tasks should be revoked from which workers; never null, but may be empty + * if no load-balancing revocations are necessary or possible + */ + private Map performLoadBalancingRevocations( + ConnectorsAndTasks configured, + Collection workers + ) { + if (log.isTraceEnabled()) { + workers.forEach(wl -> log.trace( + "Per worker current load size; worker: {} connectors: {} tasks: {}", + wl.worker(), wl.connectorsSize(), wl.tasksSize())); + } + + if (workers.stream().allMatch(WorkerLoad::isEmpty)) { + log.trace("No load-balancing revocations required; all workers are either new " + + "or will have all currently-assigned connectors and tasks revoked during this round" + ); + return Collections.emptyMap(); + } + if (configured.isEmpty()) { + log.trace("No load-balancing revocations required; no connectors are currently configured on this cluster"); + return Collections.emptyMap(); + } + + Map result = new HashMap<>(); + + Map> connectorRevocations = loadBalancingRevocations( + "connector", + configured.connectors().size(), + workers, + WorkerLoad::connectors + ); + Map> taskRevocations = loadBalancingRevocations( + "task", + configured.tasks().size(), + workers, + WorkerLoad::tasks + ); + + connectorRevocations.forEach((worker, revoked) -> + result.computeIfAbsent(worker, w -> new ConnectorsAndTasks.Builder()).addConnectors(revoked) + ); + taskRevocations.forEach((worker, revoked) -> + result.computeIfAbsent(worker, w -> new ConnectorsAndTasks.Builder()).addTasks(revoked) + ); + + return buildAll(result); + } + + private Map> loadBalancingRevocations( + String allocatedResourceName, + int totalToAllocate, + Collection workers, + Function> workerAllocation + ) { + int totalWorkers = workers.size(); + // The minimum instances of this resource that should be assigned to each worker + int minAllocatedPerWorker = totalToAllocate / totalWorkers; + // How many workers are going to have to be allocated exactly one extra instance + // (since the total number to allocate may not be a perfect multiple of the number of workers) + int extrasToAllocate = totalToAllocate % totalWorkers; + // Useful function to determine exactly how many instances of the resource a given worker is currently allocated + Function workerAllocationSize = workerAllocation.andThen(Collection::size); + + long workersAllocatedMinimum = workers.stream() + .map(workerAllocationSize) + .filter(n -> n == minAllocatedPerWorker) + .count(); + long workersAllocatedSingleExtra = workers.stream() + .map(workerAllocationSize) + .filter(n -> n == minAllocatedPerWorker + 1) + .count(); + if (workersAllocatedSingleExtra == extrasToAllocate + && workersAllocatedMinimum + workersAllocatedSingleExtra == totalWorkers) { + log.trace( + "No load-balancing {} revocations required; the current allocations, when combined with any newly-created {}, should be balanced", + allocatedResourceName, + allocatedResourceName + ); + return Collections.emptyMap(); + } + + Map> result = new HashMap<>(); + // How many workers we've allocated a single extra resource instance to + int allocatedExtras = 0; + for (WorkerLoad worker : workers) { + int currentAllocationSizeForWorker = workerAllocationSize.apply(worker); + if (currentAllocationSizeForWorker <= minAllocatedPerWorker) { + // This worker isn't allocated more than the minimum; no need to revoke anything + continue; + } + int maxAllocationForWorker; + if (allocatedExtras < extrasToAllocate) { + // We'll allocate one of the extra resource instances to this worker + allocatedExtras++; + if (currentAllocationSizeForWorker == minAllocatedPerWorker + 1) { + // If the worker's running exactly one more than the minimum, and we're allowed to + // allocate an extra to it, there's no need to revoke anything + continue; + } + maxAllocationForWorker = minAllocatedPerWorker + 1; + } else { + maxAllocationForWorker = minAllocatedPerWorker; + } + + Set revokedFromWorker = new LinkedHashSet<>(); + result.put(worker.worker(), revokedFromWorker); + + Iterator currentWorkerAllocation = workerAllocation.apply(worker).iterator(); + // Revoke resources from the worker until it isn't allocated any more than it should be + for (int numRevoked = 0; currentAllocationSizeForWorker - numRevoked > maxAllocationForWorker; numRevoked++) { + if (!currentWorkerAllocation.hasNext()) { + // Should never happen, but better to log a warning and move on than die and fail the whole rebalance if it does + log.warn( + "Unexpectedly ran out of {}s to revoke from worker {} while performing load-balancing revocations; " + + "worker appears to still be allocated {} instances, which is more than the intended allocation of {}", + allocatedResourceName, + worker.worker(), + workerAllocationSize.apply(worker), + maxAllocationForWorker + ); + break; + } + E revocation = currentWorkerAllocation.next(); + revokedFromWorker.add(revocation); + } + } + return result; + } + private int calculateDelay(long now) { long diff = scheduledRebalance - now; return diff > 0 ? (int) Math.min(diff, maxDelay) : 0; @@ -821,7 +886,7 @@ protected void assignTasks(List workerAssignment, Collection workerAssignment(Map memberAssignments, ConnectorsAndTasks toExclude) { ConnectorsAndTasks ignore = new ConnectorsAndTasks.Builder() - .with(new HashSet<>(toExclude.connectors()), new HashSet<>(toExclude.tasks())) + .with(toExclude.connectors(), toExclude.tasks()) .build(); return memberAssignments.entrySet().stream() @@ -836,6 +901,43 @@ private static List workerAssignment(Map ).collect(Collectors.toList()); } + private static void addAll(Map base, Map toAdd) { + toAdd.forEach((worker, assignment) -> base + .computeIfAbsent(worker, w -> new ConnectorsAndTasks.Builder()) + .addConnectors(assignment.connectors()) + .addTasks(assignment.tasks()) + ); + } + + private static Map buildAll(Map builders) { + return transformValues(builders, ConnectorsAndTasks.Builder::build); + } + + private static List workerLoads(Map memberAssignments) { + return memberAssignments.entrySet().stream() + .map(e -> new WorkerLoad.Builder(e.getKey()).with(e.getValue().connectors(), e.getValue().tasks()).build()) + .collect(Collectors.toList()); + } + + private static void removeAll(List workerLoads, Map toRemove) { + workerLoads.forEach(workerLoad -> { + String worker = workerLoad.worker(); + ConnectorsAndTasks toRemoveFromWorker = toRemove.getOrDefault(worker, ConnectorsAndTasks.EMPTY); + workerLoad.connectors().removeAll(toRemoveFromWorker.connectors()); + workerLoad.tasks().removeAll(toRemoveFromWorker.tasks()); + }); + } + + private static Map intersection(ConnectorsAndTasks connectorsAndTasks, Map assignments) { + return transformValues(assignments, assignment -> { + Collection connectors = new HashSet<>(assignment.connectors()); + connectors.retainAll(connectorsAndTasks.connectors()); + Collection tasks = new HashSet<>(assignment.tasks()); + tasks.retainAll(connectorsAndTasks.tasks()); + return new ConnectorsAndTasks.Builder().with(connectors, tasks).build(); + }); + } + static class ClusterAssignment { private final Map> newlyAssignedConnectors; diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinator.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinator.java index 851a1bb8364c2..a85043404c5ea 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinator.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinator.java @@ -37,9 +37,11 @@ import java.util.Collections; import java.util.Comparator; import java.util.HashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; import static org.apache.kafka.common.message.JoinGroupRequestData.JoinGroupRequestProtocolCollection; import static org.apache.kafka.common.message.JoinGroupResponseData.JoinGroupResponseMember; @@ -455,30 +457,31 @@ private ConnectorsAndTasks(Collection connectors, Collection withConnectors; - private Collection withTasks; + private Set withConnectors = new LinkedHashSet<>(); + private Set withTasks = new LinkedHashSet<>(); public Builder() { } - public ConnectorsAndTasks.Builder withCopies(Collection connectors, - Collection tasks) { - withConnectors = new ArrayList<>(connectors); - withTasks = new ArrayList<>(tasks); + public ConnectorsAndTasks.Builder with(Collection connectors, + Collection tasks) { + withConnectors = new LinkedHashSet<>(connectors); + withTasks = new LinkedHashSet<>(tasks); return this; } - public ConnectorsAndTasks.Builder with(Collection connectors, - Collection tasks) { - withConnectors = new ArrayList<>(connectors); - withTasks = new ArrayList<>(tasks); + public ConnectorsAndTasks.Builder addConnectors(Collection connectors) { + this.withConnectors.addAll(connectors); + return this; + } + + public ConnectorsAndTasks.Builder addTasks(Collection tasks) { + this.withTasks.addAll(tasks); return this; } public ConnectorsAndTasks build() { - return new ConnectorsAndTasks( - withConnectors != null ? withConnectors : new ArrayList<>(), - withTasks != null ? withTasks : new ArrayList<>()); + return new ConnectorsAndTasks(withConnectors, withTasks); } } diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java index 75e9310dfc1c8..9ec5368a8bbdb 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignorTest.java @@ -110,7 +110,8 @@ public void testTaskAssignmentWhenWorkerJoins() { // Third assignment after revocations performStandardRebalance(); - assertTrue(assignor.delay > 0); // Sucessive revocation so delay should be non-zero + assertNoRevocations(); + assertDelay(0); assertConnectorAllocations(1, 1); assertTaskAllocations(4, 4); assertBalancedAndCompleteAllocation(); @@ -124,6 +125,9 @@ public void testTaskAssignmentWhenWorkerJoins() { @Test public void testAssignmentsWhenWorkersJoinAfterRevocations() { + // Customize assignor for this test case + time = new MockTime(); + initAssignor(); addNewConnector("connector3", 4); // First assignment with 1 worker and 3 connectors configured but not yet assigned @@ -190,7 +194,6 @@ public void testAssignmentsWhenWorkersJoinAfterRevocations() { @Test public void testImmediateRevocationsWhenMaxDelayIs0() { - // Customize assignor for this test case rebalanceDelay = 0; time = new MockTime(); @@ -220,11 +223,12 @@ public void testImmediateRevocationsWhenMaxDelayIs0() { assertDelay(0); assertWorkers("worker1", "worker2", "worker3"); assertConnectorAllocations(0, 1, 1); - assertTaskAllocations(2, 3, 3); + assertTaskAllocations(3, 3, 4); // Follow up rebalance post revocations performStandardRebalance(); assertWorkers("worker1", "worker2", "worker3"); + assertNoRevocations(); assertConnectorAllocations(1, 1, 1); assertTaskAllocations(4, 4, 4); assertBalancedAndCompleteAllocation(); @@ -232,7 +236,7 @@ public void testImmediateRevocationsWhenMaxDelayIs0() { @Test public void testSuccessiveRevocationsWhenMaxDelayIsEqualToExpBackOffInitialInterval() { - + // Customize assignor for this test case rebalanceDelay = 1; initAssignor(); @@ -263,6 +267,72 @@ public void testSuccessiveRevocationsWhenMaxDelayIsEqualToExpBackOffInitialInter assertTaskAllocations(3, 3, 6); } + @Test + public void testWorkerJoiningDuringDelayedRebalance() { + // Customize assignor for this test case + time = new MockTime(); + initAssignor(); + + addNewConnector("connector3", 4); + // First assignment with 1 worker and 3 connectors configured but not yet assigned + performStandardRebalance(); + assertDelay(0); + assertWorkers("worker1"); + assertConnectorAllocations(3); + assertTaskAllocations(12); + assertBalancedAndCompleteAllocation(); + + // Second assignment with a second worker joining and all connectors running on previous worker + // We should revoke. + addNewEmptyWorkers("worker2"); + performStandardRebalance(); + assertWorkers("worker1", "worker2"); + assertConnectorAllocations(0, 2); + assertTaskAllocations(0, 6); + + // Third assignment immediately after revocations, and a third worker joining. + // This is a successive revoking rebalance. We should not perform any revocations + // in this round, but can allocate the connectors and tasks revoked previously + addNewEmptyWorkers("worker3"); + performStandardRebalance(); + assertTrue(assignor.delay > 0); + assertWorkers("worker1", "worker2", "worker3"); + assertNoRevocations(); + assertConnectorAllocations(0, 1, 2); + assertTaskAllocations(3, 3, 6); + + // Fourth assignment and a fourth worker joining + // while delayed rebalance is active. We should not revoke + time.sleep(assignor.delay / 2); + addNewEmptyWorkers("worker4"); + performStandardRebalance(); + assertTrue(assignor.delay > 0); + assertWorkers("worker1", "worker2", "worker3", "worker4"); + assertNoRevocations(); + assertConnectorAllocations(0, 0, 1, 2); + assertTaskAllocations(0, 3, 3, 6); + + // Fifth assignment and a fifth worker joining + // after the delay has expired. We should perform load-balancing + // revocations + time.sleep(assignor.delay); + addNewEmptyWorkers("worker5"); + performStandardRebalance(); + assertWorkers("worker1", "worker2", "worker3", "worker4", "worker5"); + assertDelay(0); + assertConnectorAllocations(0, 0, 0, 1, 1); + assertTaskAllocations(0, 0, 2, 3, 3); + + // Sixth and final rebalance, as a follow-up to the revocations in the previous round. + // Should allocate all previously-revoked connectors and tasks evenly across the cluster + performStandardRebalance(); + assertDelay(0); + assertNoRevocations(); + assertConnectorAllocations(0, 0, 1, 1, 1); + assertTaskAllocations(2, 2, 2, 3, 3); + assertBalancedAndCompleteAllocation(); + } + @Test public void testTaskAssignmentWhenWorkerLeavesPermanently() { // Customize assignor for this test case @@ -353,12 +423,7 @@ public void testTaskAssignmentWhenWorkerBounces() { // should be revocations giving back the assignments to the reappearing worker performStandardRebalance(); assertDelay(0); - assertConnectorAllocations(1, 1); - assertTaskAllocations(2, 4); - - // Final follow up rebalance leading to balanced assignments - performStandardRebalance(); - assertDelay(0); + assertNoRevocations(); assertConnectorAllocations(1, 1); assertTaskAllocations(4, 4); assertBalancedAndCompleteAllocation(); @@ -366,10 +431,6 @@ public void testTaskAssignmentWhenWorkerBounces() { @Test public void testTaskAssignmentWhenLeaderLeavesPermanently() { - // Customize assignor for this test case - time = new MockTime(); - initAssignor(); - // First assignment with 3 workers and 2 connectors configured but not yet assigned addNewEmptyWorkers("worker2", "worker3"); performStandardRebalance(); @@ -402,10 +463,6 @@ public void testTaskAssignmentWhenLeaderLeavesPermanently() { @Test public void testTaskAssignmentWhenLeaderBounces() { - // Customize assignor for this test case - time = new MockTime(); - initAssignor(); - // First assignment with 3 workers and 2 connectors configured but not yet assigned addNewEmptyWorkers("worker2", "worker3"); performStandardRebalance(); @@ -422,7 +479,6 @@ public void testTaskAssignmentWhenLeaderBounces() { // The fact that the leader bounces means that the assignor starts from a clean slate initAssignor(); - // Capture needs to be reset to point to the new assignor performStandardRebalance(); assertDelay(0); assertWorkers("worker2", "worker3"); @@ -442,7 +498,8 @@ public void testTaskAssignmentWhenLeaderBounces() { // Fourth assignment after revocations performStandardRebalance(); - assertTrue(assignor.delay > 0); // Successive revoking rebalance. Should introduce a delay + assertDelay(0); + assertNoRevocations(); assertConnectorAllocations(0, 1, 1); assertTaskAllocations(2, 3, 3); assertBalancedAndCompleteAllocation(); @@ -450,10 +507,6 @@ public void testTaskAssignmentWhenLeaderBounces() { @Test public void testTaskAssignmentWhenFirstAssignmentAttemptFails() { - // Customize assignor for this test case - time = new MockTime(); - initAssignor(); - // First assignment with 2 workers and 2 connectors configured but not yet assigned addNewEmptyWorkers("worker2"); performFailedRebalance(); @@ -668,11 +721,9 @@ public void testLostAssignmentHandlingWhenWorkerBounces() { configuredAssignment.put("worker1", workerLoad("worker1", 2, 2, 4, 4)); configuredAssignment.put("worker2", workerLoad("worker2", 4, 2, 8, 4)); - ConnectorsAndTasks newSubmissions = new ConnectorsAndTasks.Builder().build(); - // No lost assignments assignor.handleLostAssignments(new ConnectorsAndTasks.Builder().build(), - newSubmissions, + new ConnectorsAndTasks.Builder(), new ArrayList<>(configuredAssignment.values())); assertEquals("Wrong set of workers for reassignments", @@ -686,10 +737,10 @@ public void testLostAssignmentHandlingWhenWorkerBounces() { String flakyWorker = "worker1"; WorkerLoad lostLoad = configuredAssignment.remove(flakyWorker); ConnectorsAndTasks lostAssignments = new ConnectorsAndTasks.Builder() - .withCopies(lostLoad.connectors(), lostLoad.tasks()).build(); + .with(lostLoad.connectors(), lostLoad.tasks()).build(); // Lost assignments detected - No candidate worker has appeared yet (worker with no assignments) - assignor.handleLostAssignments(lostAssignments, newSubmissions, + assignor.handleLostAssignments(lostAssignments, new ConnectorsAndTasks.Builder(), new ArrayList<>(configuredAssignment.values())); assertEquals("Wrong set of workers for reassignments", @@ -704,7 +755,7 @@ public void testLostAssignmentHandlingWhenWorkerBounces() { // A new worker (probably returning worker) has joined configuredAssignment.put(flakyWorker, new WorkerLoad.Builder(flakyWorker).build()); - assignor.handleLostAssignments(lostAssignments, newSubmissions, + assignor.handleLostAssignments(lostAssignments, new ConnectorsAndTasks.Builder(), new ArrayList<>(configuredAssignment.values())); assertEquals("Wrong set of workers for reassignments", @@ -717,7 +768,7 @@ public void testLostAssignmentHandlingWhenWorkerBounces() { time.sleep(rebalanceDelay); // The new worker has still no assignments - assignor.handleLostAssignments(lostAssignments, newSubmissions, + assignor.handleLostAssignments(lostAssignments, new ConnectorsAndTasks.Builder(), new ArrayList<>(configuredAssignment.values())); assertTrue("Wrong assignment of lost connectors", @@ -750,11 +801,9 @@ public void testLostAssignmentHandlingWhenWorkerLeavesPermanently() { configuredAssignment.put("worker1", workerLoad("worker1", 2, 2, 4, 4)); configuredAssignment.put("worker2", workerLoad("worker2", 4, 2, 8, 4)); - ConnectorsAndTasks newSubmissions = new ConnectorsAndTasks.Builder().build(); - // No lost assignments assignor.handleLostAssignments(new ConnectorsAndTasks.Builder().build(), - newSubmissions, + new ConnectorsAndTasks.Builder(), new ArrayList<>(configuredAssignment.values())); assertEquals("Wrong set of workers for reassignments", @@ -768,10 +817,10 @@ public void testLostAssignmentHandlingWhenWorkerLeavesPermanently() { String removedWorker = "worker1"; WorkerLoad lostLoad = configuredAssignment.remove(removedWorker); ConnectorsAndTasks lostAssignments = new ConnectorsAndTasks.Builder() - .withCopies(lostLoad.connectors(), lostLoad.tasks()).build(); + .with(lostLoad.connectors(), lostLoad.tasks()).build(); // Lost assignments detected - No candidate worker has appeared yet (worker with no assignments) - assignor.handleLostAssignments(lostAssignments, newSubmissions, + assignor.handleLostAssignments(lostAssignments, new ConnectorsAndTasks.Builder(), new ArrayList<>(configuredAssignment.values())); assertEquals("Wrong set of workers for reassignments", @@ -785,7 +834,7 @@ public void testLostAssignmentHandlingWhenWorkerLeavesPermanently() { rebalanceDelay /= 2; // No new worker has joined - assignor.handleLostAssignments(lostAssignments, newSubmissions, + assignor.handleLostAssignments(lostAssignments, new ConnectorsAndTasks.Builder(), new ArrayList<>(configuredAssignment.values())); assertEquals("Wrong set of workers for reassignments", @@ -796,13 +845,14 @@ public void testLostAssignmentHandlingWhenWorkerLeavesPermanently() { time.sleep(rebalanceDelay); - assignor.handleLostAssignments(lostAssignments, newSubmissions, + ConnectorsAndTasks.Builder lostAssignmentsToReassign = new ConnectorsAndTasks.Builder(); + assignor.handleLostAssignments(lostAssignments, lostAssignmentsToReassign, new ArrayList<>(configuredAssignment.values())); assertTrue("Wrong assignment of lost connectors", - newSubmissions.connectors().containsAll(lostAssignments.connectors())); + lostAssignmentsToReassign.build().connectors().containsAll(lostAssignments.connectors())); assertTrue("Wrong assignment of lost tasks", - newSubmissions.tasks().containsAll(lostAssignments.tasks())); + lostAssignmentsToReassign.build().tasks().containsAll(lostAssignments.tasks())); assertEquals("Wrong set of workers for reassignments", Collections.emptySet(), assignor.candidateWorkersForReassignment); @@ -825,11 +875,9 @@ public void testLostAssignmentHandlingWithMoreThanOneCandidates() { configuredAssignment.put("worker1", workerLoad("worker1", 2, 2, 4, 4)); configuredAssignment.put("worker2", workerLoad("worker2", 4, 2, 8, 4)); - ConnectorsAndTasks newSubmissions = new ConnectorsAndTasks.Builder().build(); - // No lost assignments assignor.handleLostAssignments(new ConnectorsAndTasks.Builder().build(), - newSubmissions, + new ConnectorsAndTasks.Builder(), new ArrayList<>(configuredAssignment.values())); assertEquals("Wrong set of workers for reassignments", @@ -843,13 +891,13 @@ public void testLostAssignmentHandlingWithMoreThanOneCandidates() { String flakyWorker = "worker1"; WorkerLoad lostLoad = configuredAssignment.remove(flakyWorker); ConnectorsAndTasks lostAssignments = new ConnectorsAndTasks.Builder() - .withCopies(lostLoad.connectors(), lostLoad.tasks()).build(); + .with(lostLoad.connectors(), lostLoad.tasks()).build(); String newWorker = "worker3"; configuredAssignment.put(newWorker, new WorkerLoad.Builder(newWorker).build()); // Lost assignments detected - A new worker also has joined that is not the returning worker - assignor.handleLostAssignments(lostAssignments, newSubmissions, + assignor.handleLostAssignments(lostAssignments, new ConnectorsAndTasks.Builder(), new ArrayList<>(configuredAssignment.values())); assertEquals("Wrong set of workers for reassignments", @@ -864,7 +912,7 @@ public void testLostAssignmentHandlingWithMoreThanOneCandidates() { // Now two new workers have joined configuredAssignment.put(flakyWorker, new WorkerLoad.Builder(flakyWorker).build()); - assignor.handleLostAssignments(lostAssignments, newSubmissions, + assignor.handleLostAssignments(lostAssignments, new ConnectorsAndTasks.Builder(), new ArrayList<>(configuredAssignment.values())); Set expectedWorkers = new HashSet<>(); @@ -883,7 +931,7 @@ public void testLostAssignmentHandlingWithMoreThanOneCandidates() { configuredAssignment.put(newWorker, workerLoad(newWorker, 8, 2, 12, 4)); // we don't reflect these new assignments in memberConfigs currently because they are not // used in handleLostAssignments method - assignor.handleLostAssignments(lostAssignments, newSubmissions, + assignor.handleLostAssignments(lostAssignments, new ConnectorsAndTasks.Builder(), new ArrayList<>(configuredAssignment.values())); // both the newWorkers would need to be considered for re assignment of connectors and tasks @@ -923,11 +971,9 @@ public void testLostAssignmentHandlingWhenWorkerBouncesBackButFinallyLeaves() { configuredAssignment.put("worker1", workerLoad("worker1", 2, 2, 4, 4)); configuredAssignment.put("worker2", workerLoad("worker2", 4, 2, 8, 4)); - ConnectorsAndTasks newSubmissions = new ConnectorsAndTasks.Builder().build(); - // No lost assignments assignor.handleLostAssignments(new ConnectorsAndTasks.Builder().build(), - newSubmissions, + new ConnectorsAndTasks.Builder(), new ArrayList<>(configuredAssignment.values())); assertEquals("Wrong set of workers for reassignments", @@ -941,10 +987,10 @@ public void testLostAssignmentHandlingWhenWorkerBouncesBackButFinallyLeaves() { String veryFlakyWorker = "worker1"; WorkerLoad lostLoad = configuredAssignment.remove(veryFlakyWorker); ConnectorsAndTasks lostAssignments = new ConnectorsAndTasks.Builder() - .withCopies(lostLoad.connectors(), lostLoad.tasks()).build(); + .with(lostLoad.connectors(), lostLoad.tasks()).build(); // Lost assignments detected - No candidate worker has appeared yet (worker with no assignments) - assignor.handleLostAssignments(lostAssignments, newSubmissions, + assignor.handleLostAssignments(lostAssignments, new ConnectorsAndTasks.Builder(), new ArrayList<>(configuredAssignment.values())); assertEquals("Wrong set of workers for reassignments", @@ -959,7 +1005,7 @@ public void testLostAssignmentHandlingWhenWorkerBouncesBackButFinallyLeaves() { // A new worker (probably returning worker) has joined configuredAssignment.put(veryFlakyWorker, new WorkerLoad.Builder(veryFlakyWorker).build()); - assignor.handleLostAssignments(lostAssignments, newSubmissions, + assignor.handleLostAssignments(lostAssignments, new ConnectorsAndTasks.Builder(), new ArrayList<>(configuredAssignment.values())); assertEquals("Wrong set of workers for reassignments", @@ -973,13 +1019,14 @@ public void testLostAssignmentHandlingWhenWorkerBouncesBackButFinallyLeaves() { // The returning worker leaves permanently after joining briefly during the delay configuredAssignment.remove(veryFlakyWorker); - assignor.handleLostAssignments(lostAssignments, newSubmissions, + ConnectorsAndTasks.Builder lostAssignmentsToReassign = new ConnectorsAndTasks.Builder(); + assignor.handleLostAssignments(lostAssignments, lostAssignmentsToReassign, new ArrayList<>(configuredAssignment.values())); assertTrue("Wrong assignment of lost connectors", - newSubmissions.connectors().containsAll(lostAssignments.connectors())); + lostAssignmentsToReassign.build().connectors().containsAll(lostAssignments.connectors())); assertTrue("Wrong assignment of lost tasks", - newSubmissions.tasks().containsAll(lostAssignments.tasks())); + lostAssignmentsToReassign.build().tasks().containsAll(lostAssignments.tasks())); assertEquals("Wrong set of workers for reassignments", Collections.emptySet(), assignor.candidateWorkersForReassignment); @@ -1008,12 +1055,7 @@ public void testTaskAssignmentWhenTasksDuplicatedInWorkerAssignment() { // Third assignment after revocations performStandardRebalance(); assertDelay(0); - assertConnectorAllocations(1, 1); - assertTaskAllocations(2, 4); - - // fourth rebalance after revocations - performStandardRebalance(); - assertDelay(0); + assertNoRevocations(); assertConnectorAllocations(1, 1); assertTaskAllocations(4, 4); assertBalancedAndCompleteAllocation(); @@ -1026,6 +1068,10 @@ public void testTaskAssignmentWhenTasksDuplicatedInWorkerAssignment() { @Test public void testDuplicatedAssignmentHandleWhenTheDuplicatedAssignmentsDeleted() { + // Customize assignor for this test case + time = new MockTime(); + initAssignor(); + // First assignment with 1 worker and 2 connectors configured but not yet assigned performStandardRebalance(); assertDelay(0); @@ -1043,17 +1089,12 @@ public void testDuplicatedAssignmentHandleWhenTheDuplicatedAssignmentsDeleted() assertDelay(0); assertWorkers("worker1", "worker2"); assertConnectorAllocations(0, 1); - assertTaskAllocations(0, 4); + assertTaskAllocations(0, 2); // Third assignment after revocations performStandardRebalance(); assertDelay(0); - assertConnectorAllocations(0, 1); - assertTaskAllocations(0, 2); - - // fourth rebalance after revocations - performStandardRebalance(); - assertTrue(assignor.delay > 0); + assertNoRevocations(); assertConnectorAllocations(0, 1); assertTaskAllocations(2, 2); assertBalancedAndCompleteAllocation(); @@ -1182,7 +1223,11 @@ private void performRebalance(boolean assignmentFailure, boolean generationMisma generationId++; int lastCompletedGenerationId = generationMismatch ? generationId - 2 : generationId - 1; try { - Map memberAssignmentsCopy = new HashMap<>(memberAssignments); + Map memberAssignmentsCopy = memberAssignments.entrySet().stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + e -> new ConnectorsAndTasks.Builder().with(e.getValue().connectors(), e.getValue().tasks()).build() + )); returnedAssignments = assignor.performTaskAssignment(configState(), lastCompletedGenerationId, generationId, memberAssignmentsCopy); } catch (RuntimeException e) { if (assignmentFailure) { @@ -1204,7 +1249,7 @@ private void addNewEmptyWorkers(String... workers) { } private void addNewWorker(String worker, List connectors, List tasks) { - ConnectorsAndTasks assignment = new ConnectorsAndTasks.Builder().withCopies(connectors, tasks).build(); + ConnectorsAndTasks assignment = new ConnectorsAndTasks.Builder().with(connectors, tasks).build(); assertNull( "Worker " + worker + " already exists", memberAssignments.put(worker, assignment) @@ -1296,14 +1341,14 @@ private void applyAssignments() { assertEquals( "Complete connector assignment for worker " + worker + " does not match expectations " + "based on prior assignment and new revocations and assignments", - workerAssignment.connectors(), - returnedAssignments.allAssignedConnectors().get(worker) + new HashSet<>(workerAssignment.connectors()), + new HashSet<>(returnedAssignments.allAssignedConnectors().get(worker)) ); assertEquals( "Complete task assignment for worker " + worker + " does not match expectations " + "based on prior assignment and new revocations and assignments", - workerAssignment.tasks(), - returnedAssignments.allAssignedTasks().get(worker) + new HashSet<>(workerAssignment.tasks()), + new HashSet<>(returnedAssignments.allAssignedTasks().get(worker)) ); }); } @@ -1380,6 +1425,23 @@ private List allocations(Function + assertEquals( + "Expected no revocations to take place during this round, but connector revocations were issued for worker " + worker, + Collections.emptySet(), + new HashSet<>(revocations) + ) + ); + returnedAssignments.newlyRevokedTasks().forEach((worker, revocations) -> + assertEquals( + "Expected no revocations to take place during this round, but task revocations were issued for worker " + worker, + Collections.emptySet(), + new HashSet<>(revocations) + ) + ); + } + private void assertDelay(int expectedDelay) { assertEquals( "Wrong rebalance delay", diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinatorIncrementalTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinatorIncrementalTest.java index d8fb647201873..f8cf14200ca41 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinatorIncrementalTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/runtime/distributed/WorkerCoordinatorIncrementalTest.java @@ -517,13 +517,13 @@ public void testTaskAssignmentWhenWorkerBounces() { leaderAssignment = deserializeAssignment(result, leaderId); assertAssignment(leaderId, offset, Collections.emptyList(), 0, - Collections.emptyList(), 1, + Collections.emptyList(), 0, leaderAssignment); memberAssignment = deserializeAssignment(result, memberId); assertAssignment(leaderId, offset, Collections.emptyList(), 0, - Collections.emptyList(), 1, + Collections.emptyList(), 0, memberAssignment); anotherMemberAssignment = deserializeAssignment(result, anotherMemberId); From fb34e5cdeb318fef4103ad8ceca94d3fcc010d2f Mon Sep 17 00:00:00 2001 From: Chris Egerton Date: Wed, 26 Oct 2022 17:49:10 +0200 Subject: [PATCH 22/26] Address review comments, remove unused code --- .../IncrementalCooperativeAssignor.java | 83 +++---------------- 1 file changed, 11 insertions(+), 72 deletions(-) diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java index 409654ee5320c..79b19e9325b40 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java @@ -264,21 +264,21 @@ ClusterAssignment performTaskAssignment( Map toRevoke = new HashMap<>(); - Map deletedAndRevoked = intersection(deleted, memberAssignments); - log.debug("Deleted connectors and tasks to revoke from each worker: {}", deletedAndRevoked); - addAll(toRevoke, deletedAndRevoked); + Map deletedToRevoke = intersection(deleted, memberAssignments); + log.debug("Deleted connectors and tasks to revoke from each worker: {}", deletedToRevoke); + addAll(toRevoke, deletedToRevoke); // Revoking redundant connectors/tasks if the workers have duplicate assignments - Map duplicatedAndRevoked = intersection(duplicated, memberAssignments); - log.debug("Duplicated connectors and tasks to revoke from each worker: {}", duplicatedAndRevoked); - addAll(toRevoke, duplicatedAndRevoked); + Map duplicatedToRevoke = intersection(duplicated, memberAssignments); + log.debug("Duplicated connectors and tasks to revoke from each worker: {}", duplicatedToRevoke); + addAll(toRevoke, duplicatedToRevoke); // Compute the assignment that will be applied across the cluster after this round of rebalance // Later on, new submissions and lost-and-reassigned connectors and tasks will be added to these assignments, // and load-balancing revocations will be removed from them. List nextWorkerAssignment = workerLoads(memberAssignments); - removeAll(nextWorkerAssignment, deletedAndRevoked); - removeAll(nextWorkerAssignment, duplicatedAndRevoked); + removeAll(nextWorkerAssignment, deletedToRevoke); + removeAll(nextWorkerAssignment, duplicatedToRevoke); // Collect the lost assignments that are ready to be reassigned because the workers that were // originally responsible for them appear to have left the cluster instead of rejoining within @@ -388,31 +388,6 @@ ClusterAssignment performTaskAssignment( ); } - private Map computeDeleted(ConnectorsAndTasks deleted, - Map> connectorAssignments, - Map> taskAssignments) { - // Connector to worker reverse lookup map - Map connectorOwners = WorkerCoordinator.invertAssignment(connectorAssignments); - // Task to worker reverse lookup map - Map taskOwners = WorkerCoordinator.invertAssignment(taskAssignments); - - Map toRevoke = new HashMap<>(); - // Add the connectors that have been deleted to the revoked set - deleted.connectors().forEach(c -> - toRevoke.computeIfAbsent( - connectorOwners.get(c), - v -> new ConnectorsAndTasks.Builder().build() - ).connectors().add(c)); - // Add the tasks that have been deleted to the revoked set - deleted.tasks().forEach(t -> - toRevoke.computeIfAbsent( - taskOwners.get(t), - v -> new ConnectorsAndTasks.Builder().build() - ).tasks().add(t)); - log.debug("Connectors and tasks to delete assignments: {}", toRevoke); - return toRevoke; - } - private ConnectorsAndTasks computePreviousAssignment(Map toRevoke, Map> connectorAssignments, Map> taskAssignments, @@ -463,42 +438,6 @@ private ConnectorsAndTasks duplicatedAssignments(Map return new ConnectorsAndTasks.Builder().with(duplicatedConnectors, duplicatedTasks).build(); } - private Map computeDuplicatedAssignments(Map memberAssignments, - Map> connectorAssignments, - Map> taskAssignment) { - ConnectorsAndTasks duplicatedAssignments = duplicatedAssignments(memberAssignments); - log.debug("Duplicated assignments: {}", duplicatedAssignments); - - Map toRevoke = new HashMap<>(); - if (!duplicatedAssignments.connectors().isEmpty()) { - connectorAssignments.entrySet().stream() - .forEach(entry -> { - Set duplicatedConnectors = new HashSet<>(duplicatedAssignments.connectors()); - duplicatedConnectors.retainAll(entry.getValue()); - if (!duplicatedConnectors.isEmpty()) { - toRevoke.computeIfAbsent( - entry.getKey(), - v -> new ConnectorsAndTasks.Builder().build() - ).connectors().addAll(duplicatedConnectors); - } - }); - } - if (!duplicatedAssignments.tasks().isEmpty()) { - taskAssignment.entrySet().stream() - .forEach(entry -> { - Set duplicatedTasks = new HashSet<>(duplicatedAssignments.tasks()); - duplicatedTasks.retainAll(entry.getValue()); - if (!duplicatedTasks.isEmpty()) { - toRevoke.computeIfAbsent( - entry.getKey(), - v -> new ConnectorsAndTasks.Builder().build() - ).tasks().addAll(duplicatedTasks); - } - }); - } - return toRevoke; - } - // visible for testing protected void handleLostAssignments(ConnectorsAndTasks lostAssignments, ConnectorsAndTasks.Builder lostAssignmentsToReassign, @@ -748,7 +687,7 @@ private Map> loadBalancingRevocations( int minAllocatedPerWorker = totalToAllocate / totalWorkers; // How many workers are going to have to be allocated exactly one extra instance // (since the total number to allocate may not be a perfect multiple of the number of workers) - int extrasToAllocate = totalToAllocate % totalWorkers; + int workersToAllocateExtra = totalToAllocate % totalWorkers; // Useful function to determine exactly how many instances of the resource a given worker is currently allocated Function workerAllocationSize = workerAllocation.andThen(Collection::size); @@ -760,7 +699,7 @@ private Map> loadBalancingRevocations( .map(workerAllocationSize) .filter(n -> n == minAllocatedPerWorker + 1) .count(); - if (workersAllocatedSingleExtra == extrasToAllocate + if (workersAllocatedSingleExtra == workersToAllocateExtra && workersAllocatedMinimum + workersAllocatedSingleExtra == totalWorkers) { log.trace( "No load-balancing {} revocations required; the current allocations, when combined with any newly-created {}, should be balanced", @@ -780,7 +719,7 @@ private Map> loadBalancingRevocations( continue; } int maxAllocationForWorker; - if (allocatedExtras < extrasToAllocate) { + if (allocatedExtras < workersToAllocateExtra) { // We'll allocate one of the extra resource instances to this worker allocatedExtras++; if (currentAllocationSizeForWorker == minAllocatedPerWorker + 1) { From 3eb5e136c4311b05cc6738f74aba90e86ad40a55 Mon Sep 17 00:00:00 2001 From: Chris Egerton Date: Mon, 14 Nov 2022 10:15:45 -0500 Subject: [PATCH 23/26] Address review comments --- .../distributed/IncrementalCooperativeAssignor.java | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java index 79b19e9325b40..d48589423dc78 100644 --- a/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java +++ b/connect/runtime/src/main/java/org/apache/kafka/connect/runtime/distributed/IncrementalCooperativeAssignor.java @@ -257,7 +257,7 @@ ClusterAssignment performTaskAssignment( // Derived set: The set of new connectors-and-tasks is a derived set from the set // difference of configured - previous - active ConnectorsAndTasks created = diff(configured, previousAssignment, activeAssignments); - log.debug("New assignments: {}", created); + log.debug("Created: {}", created); // A collection of the current assignment excluding the connectors-and-tasks to be deleted List currentWorkerAssignment = workerAssignment(memberAssignments, deleted); @@ -618,12 +618,8 @@ private ConnectorsAndTasks assignment(Map memberAssi } /** - * Revoke connectors and tasks from each worker in the cluster until no worker is running more than it would be if: - *
    - *
  • The allocation of connectors and tasks across the cluster were as balanced as possible (i.e., the difference in allocation size between any two workers is at most one)
    • - *
  • Any workers that left the group within the scheduled rebalance delay permanently left the group
    • - *
  • All currently-configured connectors and tasks were allocated (including instances that may be revoked in this round because they are duplicated across workers)
    • - *
+ * Revoke connectors and tasks from each worker in the cluster until no worker is running more than it + * would be with a perfectly-balanced assignment. * @param configured the set of configured connectors and tasks across the entire cluster * @param workers the workers in the cluster, whose assignments should not include any deleted or duplicated connectors or tasks * that are already due to be revoked from the worker in this rebalance @@ -702,7 +698,7 @@ private Map> loadBalancingRevocations( if (workersAllocatedSingleExtra == workersToAllocateExtra && workersAllocatedMinimum + workersAllocatedSingleExtra == totalWorkers) { log.trace( - "No load-balancing {} revocations required; the current allocations, when combined with any newly-created {}, should be balanced", + "No load-balancing {} revocations required; the current allocations, when combined with any newly-created {}s, should be balanced", allocatedResourceName, allocatedResourceName ); @@ -712,6 +708,7 @@ private Map> loadBalancingRevocations( Map> result = new HashMap<>(); // How many workers we've allocated a single extra resource instance to int allocatedExtras = 0; + // Calculate how many (and which) connectors/tasks to revoke from each worker here for (WorkerLoad worker : workers) { int currentAllocationSizeForWorker = workerAllocationSize.apply(worker); if (currentAllocationSizeForWorker <= minAllocatedPerWorker) { From 79ff0ea90fd44a97931571bba82dac54417666df Mon Sep 17 00:00:00 2001 From: Sagar Rao Date: Tue, 15 Nov 2022 22:27:14 +0530 Subject: [PATCH 24/26] Fixing checkstyle --- .../integration/RebalanceSourceConnectorsIntegrationTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/RebalanceSourceConnectorsIntegrationTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/RebalanceSourceConnectorsIntegrationTest.java index 0b730cb55bc68..ebc53edee2dfd 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/RebalanceSourceConnectorsIntegrationTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/RebalanceSourceConnectorsIntegrationTest.java @@ -22,7 +22,6 @@ import org.apache.kafka.test.IntegrationTest; import org.junit.After; import org.junit.Before; -import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; From f17bd149502ceb3abda6b5c94b6dd1e2cbfc30d0 Mon Sep 17 00:00:00 2001 From: Sagar Rao Date: Tue, 15 Nov 2022 22:29:10 +0530 Subject: [PATCH 25/26] Revert "Fixing checkstyle" This reverts commit 2eb1f3fbedb793353d58f61a6a0074b9bbfb6063. --- .../integration/RebalanceSourceConnectorsIntegrationTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/RebalanceSourceConnectorsIntegrationTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/RebalanceSourceConnectorsIntegrationTest.java index ebc53edee2dfd..0b730cb55bc68 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/RebalanceSourceConnectorsIntegrationTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/RebalanceSourceConnectorsIntegrationTest.java @@ -22,6 +22,7 @@ import org.apache.kafka.test.IntegrationTest; import org.junit.After; import org.junit.Before; +import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; From 52d2381820780dfa9e7ed55e534f23303222e0b8 Mon Sep 17 00:00:00 2001 From: Sagar Rao Date: Tue, 15 Nov 2022 22:44:05 +0530 Subject: [PATCH 26/26] fixing checkstyle again --- .../integration/RebalanceSourceConnectorsIntegrationTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/RebalanceSourceConnectorsIntegrationTest.java b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/RebalanceSourceConnectorsIntegrationTest.java index 0b730cb55bc68..ebc53edee2dfd 100644 --- a/connect/runtime/src/test/java/org/apache/kafka/connect/integration/RebalanceSourceConnectorsIntegrationTest.java +++ b/connect/runtime/src/test/java/org/apache/kafka/connect/integration/RebalanceSourceConnectorsIntegrationTest.java @@ -22,7 +22,6 @@ import org.apache.kafka.test.IntegrationTest; import org.junit.After; import org.junit.Before; -import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category;