Skip to content
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -669,6 +669,8 @@ abstract class ShardCoordinator(
var unAckedHostShards = Map.empty[ShardId, Cancellable]
// regions that have requested handoff, for graceful shutdown
var gracefulShutdownInProgress = Set.empty[ActorRef]
var localRegionTerminationInProgress = false
var waitingForLocalRegionToTerminate = false
var aliveRegions = Set.empty[ActorRef]
var regionTerminationInProgress = Set.empty[ActorRef]

Expand Down Expand Up @@ -847,26 +849,26 @@ abstract class ShardCoordinator(
}

case GracefulShutdownReq(region) =>
if (!gracefulShutdownInProgress(region))
if (!gracefulShutdownInProgress(region)) {
state.regions.get(region) match {
case Some(shards) =>
val isLocal = region.path.address.hasLocalScope
if (log.isDebugEnabled) {
if (verboseDebug)
log.debug(
"{}: Graceful shutdown of region [{}] with [{}] shards [{}]",
typeName,
region,
shards.size,
shards.mkString(", "))
"{}: Graceful shutdown of {} region [{}] with [{}] shards [{}]",
Array(typeName, if (isLocal) "local" else "", region, shards.size, shards.mkString(", ")))
else
log.debug("{}: Graceful shutdown of region [{}] with [{}] shards", typeName, region, shards.size)
}
gracefulShutdownInProgress += region
if (isLocal) localRegionTerminationInProgress = true
shutdownShards(region, shards.toSet)

case None =>
log.debug("{}: Unknown region requested graceful shutdown [{}]", typeName, region)
}
}

case ShardRegion.GetClusterShardingStats(waitMax) =>
import akka.pattern.ask
Expand Down Expand Up @@ -917,24 +919,33 @@ abstract class ShardCoordinator(
sender() ! reply

case ShardCoordinator.Internal.Terminate =>
if (rebalanceInProgress.isEmpty)
log.debug("{}: Received termination message.", typeName)
else if (log.isDebugEnabled) {
if (verboseDebug)
log.debug(
"{}: Received termination message. Rebalance in progress of [{}] shards [{}].",
typeName,
rebalanceInProgress.size,
rebalanceInProgress.keySet.mkString(", "))
else
log.debug(
"{}: Received termination message. Rebalance in progress of [{}] shards.",
typeName,
rebalanceInProgress.size)
}
context.stop(self)
terminate()
}: Receive).orElse[Any, Unit](receiveTerminated)

private def terminate(): Unit = {
if (localRegionTerminationInProgress) {
Comment thread
johanandren marked this conversation as resolved.
Resolved
log.debug("{}: Deferring coordinator termination until local region has terminated", typeName)
waitingForLocalRegionToTerminate = true
} else {
if (rebalanceInProgress.isEmpty)
log.debug("{}: Received termination message.", typeName)
else if (log.isDebugEnabled) {
if (verboseDebug)
log.debug(
"{}: Received termination message. Rebalance in progress of [{}] shards [{}].",
typeName,
rebalanceInProgress.size,
rebalanceInProgress.keySet.mkString(", "))
else
log.debug(
"{}: Received termination message. Rebalance in progress of [{}] shards.",
typeName,
rebalanceInProgress.size)
}
context.stop(self)
}
}

private def clearRebalanceInProgress(shard: String): Unit = {
rebalanceInProgress.get(shard) match {
case Some(pendingGetShardHome) =>
Expand Down Expand Up @@ -1070,6 +1081,12 @@ abstract class ShardCoordinator(
regionTerminationInProgress -= ref
aliveRegions -= ref
allocateShardHomesForRememberEntities()
if (ref.path.address.hasLocalScope && waitingForLocalRegionToTerminate) {
Comment thread
johanandren marked this conversation as resolved.
Resolved
// handoff optimization: singleton told coordinator to stop but we deferred stop until the local region
// had completed the handoff
log.debug("{}: Local region stopped", typeName)
terminate()
}
}
}
}
Expand Down