From b665d27f528156030453efed97ec54e5620d3479 Mon Sep 17 00:00:00 2001 From: Viraj Jasani Date: Tue, 19 Oct 2021 21:30:54 +0530 Subject: [PATCH 1/2] HBASE-26371 Prioritize meta region move over other region moves in region_mover --- .../apache/hadoop/hbase/util/RegionMover.java | 84 +++++++++++++------ 1 file changed, 60 insertions(+), 24 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java index 778d66da63d8..d3fbd7acd959 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java @@ -38,7 +38,9 @@ import java.util.Iterator; import java.util.List; import java.util.Locale; +import java.util.Optional; import java.util.Set; +import java.util.concurrent.Callable; import java.util.concurrent.CancellationException; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -284,21 +286,43 @@ public RegionMover build() throws IOException { */ public boolean load() throws ExecutionException, InterruptedException, TimeoutException { ExecutorService loadPool = Executors.newFixedThreadPool(1); - Future loadTask = loadPool.submit(() -> { + Future loadTask = loadPool.submit(getRegionsMovePlan(true)); + boolean isMetaMoved = waitTaskToFinish(loadPool, loadTask, "loading"); + if (!isMetaMoved) { + return false; + } + loadPool = Executors.newFixedThreadPool(1); + loadTask = loadPool.submit(getRegionsMovePlan(false)); + return waitTaskToFinish(loadPool, loadTask, "loading"); + } + + private Callable getRegionsMovePlan(boolean moveMetaRegion) { + return () -> { try { List regionsToMove = readRegionsFromFile(filename); if (regionsToMove.isEmpty()) { LOG.info("No regions to load.Exiting"); return true; } - loadRegions(regionsToMove); + Optional metaRegion = getMetaRegionInfoIfToBeMoved(regionsToMove); + if (moveMetaRegion) { + if (metaRegion.isPresent()) { + loadRegions(Collections.singletonList(metaRegion.get())); + } + } else { + metaRegion.ifPresent(regionsToMove::remove); + loadRegions(regionsToMove); + } } catch (Exception e) { LOG.error("Error while loading regions to " + hostname, e); return false; } return true; - }); - return waitTaskToFinish(loadPool, loadTask, "loading"); + }; + } + + private Optional getMetaRegionInfoIfToBeMoved(List regionsToMove) { + return regionsToMove.stream().filter(RegionInfo::isMetaRegion).findFirst(); } private void loadRegions(List regionsToMove) @@ -472,30 +496,42 @@ private void unloadRegions(ServerName server, List regionServers, } LOG.info("Moving {} regions from {} to {} servers using {} threads .Ack Mode: {}", regionsToMove.size(), this.hostname, regionServers.size(), this.maxthreads, ack); - final ExecutorService moveRegionsPool = Executors.newFixedThreadPool(this.maxthreads); - List> taskList = new ArrayList<>(); - int serverIndex = 0; - for (RegionInfo regionToMove : regionsToMove) { - if (ack) { - Future task = moveRegionsPool.submit( - new MoveWithAck(conn, regionToMove, server, regionServers.get(serverIndex), - movedRegions)); - taskList.add(task); - } else { - Future task = moveRegionsPool.submit( - new MoveWithoutAck(admin, regionToMove, server, regionServers.get(serverIndex), - movedRegions)); - taskList.add(task); - } - serverIndex = (serverIndex + 1) % regionServers.size(); + + Optional metaRegion = getMetaRegionInfoIfToBeMoved(regionsToMove); + if (metaRegion.isPresent()) { + RegionInfo meta = metaRegion.get(); + submitRegionMovesWhileUnloading(server, regionServers, movedRegions, + Collections.singletonList(meta)); + regionsToMove.remove(meta); } - moveRegionsPool.shutdown(); - long timeoutInSeconds = regionsToMove.size() * admin.getConfiguration() - .getLong(MOVE_WAIT_MAX_KEY, DEFAULT_MOVE_WAIT_MAX); - waitMoveTasksToFinish(moveRegionsPool, taskList, timeoutInSeconds); + submitRegionMovesWhileUnloading(server, regionServers, movedRegions, regionsToMove); } } + private void submitRegionMovesWhileUnloading(ServerName server, List regionServers, + List movedRegions, List regionsToMove) throws Exception { + final ExecutorService moveRegionsPool = Executors.newFixedThreadPool(this.maxthreads); + List> taskList = new ArrayList<>(); + int serverIndex = 0; + for (RegionInfo regionToMove : regionsToMove) { + if (ack) { + Future task = moveRegionsPool.submit( + new MoveWithAck(conn, regionToMove, server, regionServers.get(serverIndex), movedRegions)); + taskList.add(task); + } else { + Future task = moveRegionsPool.submit( + new MoveWithoutAck(admin, regionToMove, server, regionServers.get(serverIndex), + movedRegions)); + taskList.add(task); + } + serverIndex = (serverIndex + 1) % regionServers.size(); + } + moveRegionsPool.shutdown(); + long timeoutInSeconds = regionsToMove.size() * admin.getConfiguration() + .getLong(MOVE_WAIT_MAX_KEY, DEFAULT_MOVE_WAIT_MAX); + waitMoveTasksToFinish(moveRegionsPool, taskList, timeoutInSeconds); + } + private boolean waitTaskToFinish(ExecutorService pool, Future task, String operation) throws TimeoutException, InterruptedException, ExecutionException { pool.shutdown(); From 94ec57d9555a9732941543fa1ea463ea8d739b12 Mon Sep 17 00:00:00 2001 From: Viraj Jasani Date: Wed, 20 Oct 2021 15:15:49 +0530 Subject: [PATCH 2/2] addendum --- .../apache/hadoop/hbase/util/RegionMover.java | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java index d3fbd7acd959..286caf8ed3b0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionMover.java @@ -286,16 +286,24 @@ public RegionMover build() throws IOException { */ public boolean load() throws ExecutionException, InterruptedException, TimeoutException { ExecutorService loadPool = Executors.newFixedThreadPool(1); - Future loadTask = loadPool.submit(getRegionsMovePlan(true)); + Future loadTask = loadPool.submit(getMetaRegionMovePlan()); boolean isMetaMoved = waitTaskToFinish(loadPool, loadTask, "loading"); if (!isMetaMoved) { return false; } loadPool = Executors.newFixedThreadPool(1); - loadTask = loadPool.submit(getRegionsMovePlan(false)); + loadTask = loadPool.submit(getNonMetaRegionsMovePlan()); return waitTaskToFinish(loadPool, loadTask, "loading"); } + private Callable getMetaRegionMovePlan() { + return getRegionsMovePlan(true); + } + + private Callable getNonMetaRegionsMovePlan() { + return getRegionsMovePlan(false); + } + private Callable getRegionsMovePlan(boolean moveMetaRegion) { return () -> { try { @@ -516,7 +524,8 @@ private void submitRegionMovesWhileUnloading(ServerName server, List for (RegionInfo regionToMove : regionsToMove) { if (ack) { Future task = moveRegionsPool.submit( - new MoveWithAck(conn, regionToMove, server, regionServers.get(serverIndex), movedRegions)); + new MoveWithAck(conn, regionToMove, server, regionServers.get(serverIndex), + movedRegions)); taskList.add(task); } else { Future task = moveRegionsPool.submit( @@ -528,7 +537,7 @@ private void submitRegionMovesWhileUnloading(ServerName server, List } moveRegionsPool.shutdown(); long timeoutInSeconds = regionsToMove.size() * admin.getConfiguration() - .getLong(MOVE_WAIT_MAX_KEY, DEFAULT_MOVE_WAIT_MAX); + .getLong(MOVE_WAIT_MAX_KEY, DEFAULT_MOVE_WAIT_MAX); waitMoveTasksToFinish(moveRegionsPool, taskList, timeoutInSeconds); }