diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 1d4e7d19f6259..1b83b5832618d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -4527,33 +4527,10 @@ void processExtraRedundancyBlocksOnInService( * based on its liveness. Dead nodes cannot always be safely decommissioned * or in maintenance. */ - boolean isNodeHealthyForDecommissionOrMaintenance(DatanodeDescriptor node) { - if (!node.checkBlockReportReceived()) { - LOG.info("Node {} hasn't sent its first block report.", node); - return false; - } - - if (node.isAlive()) { - return true; - } - + boolean anyLowRedundancyOrPendingReplicationBlocks() { updateState(); - if (pendingReconstructionBlocksCount == 0 && - lowRedundancyBlocksCount == 0) { - LOG.info("Node {} is dead and there are no low redundancy" + - " blocks or blocks pending reconstruction. Safe to decommission or" + - " put in maintenance.", node); - return true; - } - - LOG.warn("Node {} is dead " + - "while in {}. Cannot be safely " + - "decommissioned or be in maintenance since there is risk of reduced " + - "data durability or data loss. Either restart the failed node or " + - "force decommissioning or maintenance by removing, calling " + - "refreshNodes, then re-adding to the excludes or host config files.", - node, node.getAdminState()); - return false; + return pendingReconstructionBlocksCount != 0 || + lowRedundancyBlocksCount != 0; } public int getActiveBlockCount() { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminBackoffMonitor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminBackoffMonitor.java index c04f3daabf70e..b1d5162da6453 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminBackoffMonitor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminBackoffMonitor.java @@ -239,6 +239,7 @@ private void processCancelledNodes() { DatanodeDescriptor dn = cancelledNodes.poll(); outOfServiceNodeBlocks.remove(dn); pendingRep.remove(dn); + dn.setTrackedForDecommissionOrMaintenance(false); } } @@ -301,6 +302,42 @@ private void check() { // Finally move the nodes to their final state if they are ready. processCompletedNodes(toRemove); + + for (final DatanodeDescriptor dn : outOfServiceNodeBlocks.keySet()) { + if (!dn.isAlive()) { + if (blockManager.anyLowRedundancyOrPendingReplicationBlocks()) { + // Dead datanode should remain in decommissioning because there is risk of data loss + LOG.warn("Node {} is dead while in {}. Cannot be safely " + + "decommissioned or be in maintenance since there is risk of reduced " + + "data durability or data loss. Either restart the failed node or " + + "force decommissioning or maintenance by removing, calling " + + "refreshNodes, then re-adding to the excludes or host config files.", dn, + dn.getAdminState()); + // No need to track unhealthy datanodes, when the datanode comes alive again + // and re-registers, it will be re-added to the DatanodeAdminManager + cancelledNodes.add(dn); + } else { + // Dead node can potentially be decommissioned because there is no risk of data loss + LOG.info("Node {} is dead and there are no low redundancy " + + "blocks or blocks pending reconstruction. Safe to decommission or " + + "put in maintenance.", dn); + } + } else if (!dn.checkBlockReportReceived()) { + // Cannot untrack a live node with no first block report because it + // may not be re-added to the DatanodeAdminManager, requeue the node if necessary + LOG.info("Healthy Node {} hasn't sent its first block report.", dn); + } + } + + // Having more nodes decommissioning than can be tracked will impact decommissioning + // performance due to queueing delay + int numDecommissioningNodes = + outOfServiceNodeBlocks.size() + pendingNodes.size() - cancelledNodes.size(); + if (numDecommissioningNodes > maxConcurrentTrackedNodes) { + LOG.warn("There are {} nodes decommissioning but only {} nodes will be tracked at a time. " + + "{} nodes are currently queued waiting to be decommissioned.", + numDecommissioningNodes, maxConcurrentTrackedNodes, getPendingNodes().size()); + } } /** @@ -346,9 +383,8 @@ private void processCompletedNodes(List toRemove) { namesystem.writeLock(); try { for (DatanodeDescriptor dn : toRemove) { - final boolean isHealthy = - blockManager.isNodeHealthyForDecommissionOrMaintenance(dn); - if (isHealthy) { + if (dn.checkBlockReportReceived() && + (dn.isAlive() || !blockManager.anyLowRedundancyOrPendingReplicationBlocks())) { if (dn.isDecommissionInProgress()) { dnAdmin.setDecommissioned(dn); outOfServiceNodeBlocks.remove(dn); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminDefaultMonitor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminDefaultMonitor.java index a217c9978e822..94edade3108ac 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminDefaultMonitor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminDefaultMonitor.java @@ -125,6 +125,7 @@ private boolean exceededNumBlocksPerCheck() { public void stopTrackingNode(DatanodeDescriptor dn) { pendingNodes.remove(dn); outOfServiceNodeBlocks.remove(dn); + dn.setTrackedForDecommissionOrMaintenance(false); } @Override @@ -185,6 +186,7 @@ private void check() { it = new CyclicIteration<>(outOfServiceNodeBlocks, iterkey).iterator(); final List toRemove = new ArrayList<>(); + final List deadToRemove = new ArrayList<>(); while (it.hasNext() && !exceededNumBlocksPerCheck() && namesystem .isRunning()) { @@ -221,7 +223,32 @@ private void check() { LOG.debug("Processing {} node {}", dn.getAdminState(), dn); pruneReliableBlocks(dn, blocks); } - if (blocks.size() == 0) { + + final boolean anyLowRedundancyBlocks = + blockManager.anyLowRedundancyOrPendingReplicationBlocks(); + if (!dn.isAlive() && !anyLowRedundancyBlocks) { + // Dead node can potentially be decommissioned because there is no risk of data loss + LOG.info("Node {} is dead and there are no low redundancy " + + "blocks or blocks pending reconstruction. Safe to decommission or " + + "put in maintenance.", dn); + } + if (!dn.isAlive() && anyLowRedundancyBlocks) { + // Dead datanode should remain in decommissioning because there is risk of data loss + LOG.warn("Node {} is dead while in {}. Cannot be safely " + + "decommissioned or be in maintenance since there is risk of reduced " + + "data durability or data loss. Either restart the failed node or " + + "force decommissioning or maintenance by removing, calling " + + "refreshNodes, then re-adding to the excludes or host config files.", dn, + dn.getAdminState()); + // No need to track unhealthy datanodes, when the datanode comes alive again + // and re-registers, it will be re-added to the DatanodeAdminManager + deadToRemove.add(dn); + } else if (!dn.checkBlockReportReceived()) { + // Cannot untrack a live node with no first block report because it + // may not be re-added to the DatanodeAdminManager + LOG.info("Healthy Node {} hasn't sent its first block report, " + + "cannot be decommissioned yet.", dn); + } else if (blocks.size() == 0) { if (!fullScan) { // If we didn't just do a full scan, need to re-check with the // full block map. @@ -236,9 +263,7 @@ private void check() { } // If the full scan is clean AND the node liveness is okay, // we can finally mark as DECOMMISSIONED or IN_MAINTENANCE. - final boolean isHealthy = - blockManager.isNodeHealthyForDecommissionOrMaintenance(dn); - if (blocks.size() == 0 && isHealthy) { + if (blocks.size() == 0) { if (dn.isDecommissionInProgress()) { dnAdmin.setDecommissioned(dn); toRemove.add(dn); @@ -255,10 +280,10 @@ private void check() { LOG.debug("Node {} is sufficiently replicated and healthy, " + "marked as {}.", dn, dn.getAdminState()); } else { - LOG.info("Node {} {} healthy." + LOG.info("Node {} is healthy." + " It needs to replicate {} more blocks." - + " {} is still in progress.", dn, - isHealthy ? "is": "isn't", blocks.size(), dn.getAdminState()); + + " {} is still in progress.", + dn, blocks.size(), dn.getAdminState()); } } else { LOG.info("Node {} still has {} blocks to replicate " @@ -276,6 +301,17 @@ private void check() { iterkey = dn; } } + + // Having more nodes decommissioning than can be tracked will impact decommissioning + // performance due to queueing delay + int numDecommissioningNodes = + outOfServiceNodeBlocks.size() + getPendingNodes().size() - toRemove.size(); + if (numDecommissioningNodes > maxConcurrentTrackedNodes) { + LOG.warn("There are {} nodes decommissioning but only {} nodes will be tracked at a time. " + + "{} nodes are currently queued waiting to be decommissioned.", + numDecommissioningNodes, maxConcurrentTrackedNodes, getPendingNodes().size()); + } + // Remove the datanodes that are DECOMMISSIONED or in service after // maintenance expiration. for (DatanodeDescriptor dn : toRemove) { @@ -284,6 +320,14 @@ private void check() { dn); outOfServiceNodeBlocks.remove(dn); } + + // Remove dead datanodes which are DECOMMISSION_INPROGRESS + for (DatanodeDescriptor dn : deadToRemove) { + Preconditions.checkState(dn.isDecommissionInProgress() && !dn.isAlive(), + "Removing dead node %s that is not decommission in progress!", dn); + outOfServiceNodeBlocks.remove(dn); + dn.setTrackedForDecommissionOrMaintenance(false); + } } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminManager.java index 42b6ddd8c78fb..9b7b4b4335f5e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminManager.java @@ -186,6 +186,13 @@ public void startDecommission(DatanodeDescriptor node) { node.getLeavingServiceStatus().setStartTime(monotonicNow()); monitor.startTrackingNode(node); } + } else if (!node.isTrackedForDecommissionOrMaintenance() && node.isDecommissionInProgress() + && node.isAlive()) { + for (DatanodeStorageInfo storage : node.getStorageInfos()) { + LOG.info("Resuming decommission of {} {} with {} blocks", node, storage, + storage.numBlocks()); + } + monitor.startTrackingNode(node); } else { LOG.trace("startDecommission: Node {} in {}, nothing to do.", node, node.getAdminState()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminMonitorBase.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminMonitorBase.java index 9eee241edddf8..f187ee9f233e7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminMonitorBase.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminMonitorBase.java @@ -134,6 +134,7 @@ public Configuration getConf() { @Override public void startTrackingNode(DatanodeDescriptor dn) { pendingNodes.add(dn); + dn.setTrackedForDecommissionOrMaintenance(true); } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java index f4ee01d4271a8..1f567da754f20 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java @@ -231,6 +231,10 @@ public Type getType() { // HB processing can use it to tell if it is the first HB since DN restarted private boolean heartbeatedSinceRegistration = false; + // Whether or not the node is tracked within the DatanodeAdminManager for decommissioning + // or maintenance. This is necessary to avoid adding duplicate nodes to the DatanodeAdminManager. + private boolean isTrackedForDecommissionOrMaintenance = false; + /** * DatanodeDescriptor constructor * @param nodeID id of the data node @@ -1088,4 +1092,12 @@ public boolean hasStorageType(StorageType type) { } return false; } + + public void setTrackedForDecommissionOrMaintenance(final boolean isTracked) { + isTrackedForDecommissionOrMaintenance = isTracked; + } + + public boolean isTrackedForDecommissionOrMaintenance() { + return isTrackedForDecommissionOrMaintenance; + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/AdminStatesBaseTest.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/AdminStatesBaseTest.java index 10b18032e13e3..82a983004dfe7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/AdminStatesBaseTest.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/AdminStatesBaseTest.java @@ -449,7 +449,7 @@ protected void refreshNodes(final int nnIndex) throws IOException { refreshNodes(conf); } - static private DatanodeDescriptor getDatanodeDesriptor( + static DatanodeDescriptor getDatanodeDesriptor( final FSNamesystem ns, final String datanodeUuid) { return ns.getBlockManager().getDatanodeManager().getDatanode(datanodeUuid); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java index f7e6dce003311..0eb15030923b1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java @@ -26,6 +26,8 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.PrintStream; +import java.time.Duration; +import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -40,6 +42,8 @@ import java.util.EnumSet; import java.util.function.Supplier; +import java.util.stream.Collectors; + import org.apache.commons.text.TextStringBuilder; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.CommonConfigurationKeys; @@ -1654,4 +1658,180 @@ public Boolean get() { cleanupFile(fileSys, file); } + + /** + * Test DatanodeAdminManager logic related to datanode which dies while DECOMMISSION_INPROGRESS. + * If the datanode never comes alive again, it remains DECOMMISSION_INPROGRESS forever. + */ + @Test(timeout = 120000) + public void testDeadNodesRemainDecommissionInProgress() throws Exception { + final Map deadNodes = + createClusterWithDeadNodesDecommissionInProgress(); + final DatanodeManager datanodeManager = + getCluster().getNamesystem().getBlockManager().getDatanodeManager(); + final DatanodeAdminManager decomManager = datanodeManager.getDatanodeAdminManager(); + + // Do not restart the dead datanodes & validate they remain decommissioning forever + final Duration checkDuration = Duration.ofSeconds(10); + Instant checkUntil = Instant.now().plus(checkDuration); + while (Instant.now().isBefore(checkUntil)) { + BlockManagerTestUtil.recheckDecommissionState(datanodeManager); + assertEquals(0, decomManager.getNumPendingNodes()); + assertEquals(0, decomManager.getNumTrackedNodes()); + for (final DatanodeDescriptor deadNode : deadNodes.keySet()) { + Assert.assertFalse(deadNode.isAlive()); + assertEquals(AdminStates.DECOMMISSION_INPROGRESS, deadNode.getAdminState()); + } + Thread.sleep(500); + } + } + + /** + * Test DatanodeAdminManager logic related to datanode which dies while DECOMMISSION_INPROGRESS. + * If the datanode comes alive again, it can be DECOMMISSIONED. + */ + @Test(timeout = 120000) + public void testRevivedDeadNodeIsDecommissioned() throws Exception { + final Map deadNodes = + createClusterWithDeadNodesDecommissionInProgress(); + final DatanodeManager datanodeManager = + getCluster().getNamesystem().getBlockManager().getDatanodeManager(); + final DatanodeAdminManager decomManager = datanodeManager.getDatanodeAdminManager(); + + // Restart the "dead" datanodes & validate they are still DECOMMISSION_INPROGRESS + for (final MiniDFSCluster.DataNodeProperties deadNode : deadNodes.values()) { + getCluster().restartDataNode(deadNode, true); + } + GenericTestUtils.waitFor(() -> deadNodes.keySet().stream().allMatch( + dn -> dn.isAlive() && dn.getAdminState().equals(AdminStates.DECOMMISSION_INPROGRESS)) + && decomManager.getNumTrackedNodes() == 0 && decomManager.getNumPendingNodes() == 3, 500, + 20000); + + // Validate the revived "dead" datanodes & are eventually decommissioned + GenericTestUtils.waitFor(() -> { + try { + BlockManagerTestUtil.recheckDecommissionState(datanodeManager); + } catch (Exception e) { + LOG.error("Exception running DatanodeAdminManager.Monitor", e); + } + return deadNodes.keySet().stream() + .allMatch(dn -> dn.isAlive() && dn.getAdminState().equals(AdminStates.DECOMMISSIONED)) + && decomManager.getNumTrackedNodes() == 0 && decomManager.getNumPendingNodes() == 0; + }, 500, 20000); + } + + /** + * Test DatanodeAdminManager logic related to datanode which dies while DECOMMISSION_INPROGRESS. + * If the datanode is removed from exclude hosts file while dead,it should transition out of + * DECOMMISSION_INPROGRESS to IN_SERVICE. If the datanode comes alive again, + * it should still have AdminState=IN_SERVICE. + */ + @Test(timeout = 120000) + public void testDeadNodeWithDecommissionInProgressRemovedFromExcludeFile() throws Exception { + final Map deadNodes = + createClusterWithDeadNodesDecommissionInProgress(); + final DatanodeManager datanodeManager = + getCluster().getNamesystem().getBlockManager().getDatanodeManager(); + final DatanodeAdminManager decomManager = datanodeManager.getDatanodeAdminManager(); + + // Remove the dead datanodes from the exclude hosts file & validate they become IN_SERVICE + for (final DatanodeDescriptor deadNode : deadNodes.keySet()) { + putNodeInService(0, deadNode); + } + GenericTestUtils.waitFor(() -> deadNodes.keySet().stream() + .allMatch(dn -> !dn.isAlive() && dn.getAdminState().equals(AdminStates.NORMAL)), 500, + 20000); + + // Restart the dead datanodes & validate they are still IN_SERVICE + for (final MiniDFSCluster.DataNodeProperties deadNode : deadNodes.values()) { + getCluster().restartDataNode(deadNode, true); + } + GenericTestUtils.waitFor(() -> deadNodes.keySet().stream() + .allMatch(dn -> dn.isAlive() && dn.getAdminState().equals(AdminStates.NORMAL)), 500, 20000); + } + + /** + * Create a MiniDFSCluster with 1 live datanode in AdminState=NORMAL and + * 3 dead datanodes in AdminState=DECOMMISSION_INPROGRESS + * + * @return list of 3 dead datanodes in AdminState=DECOMMISSION_INPROGRESS + */ + private Map createClusterWithDeadNodesDecommissionInProgress() + throws Exception { + // Allow 3 datanodes to be decommissioned at a time + getConf().setInt(DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_MAX_CONCURRENT_TRACKED_NODES, 3); + // Disable the normal monitor runs + getConf() + .setInt(MiniDFSCluster.DFS_NAMENODE_DECOMMISSION_INTERVAL_TESTING_KEY, Integer.MAX_VALUE); + + // Start cluster with 4 datanodes + startCluster(1, 4); + final FSNamesystem namesystem = getCluster().getNamesystem(); + final BlockManager blockManager = namesystem.getBlockManager(); + final DatanodeManager datanodeManager = blockManager.getDatanodeManager(); + final DatanodeAdminManager decomManager = datanodeManager.getDatanodeAdminManager(); + assertEquals(4, getCluster().getDataNodes().size()); + getCluster().waitActive(); + + // 1 datanodes will remain "live" + final DatanodeDescriptor liveNode = + getDatanodeDesriptor(namesystem, getCluster().getDataNodes().get(0).getDatanodeUuid()); + assertNotNull(liveNode); + + // 3 datanodes will be "dead" while decommissioning + final List deadNodes = getCluster().getDataNodes().subList(1, 4).stream() + .map(dn -> getDatanodeDesriptor(namesystem, dn.getDatanodeUuid())) + .collect(Collectors.toList()); + assertEquals(3, deadNodes.size()); + + // Create file with block replicas on all 4 nodes + final Path filePath = new Path("/tmp/test"); + writeFile(getCluster().getFileSystem(), filePath, 4, 10); + + // Initially there are no low redundancy blocks + Assert + .assertFalse(BlockManagerTestUtil.anyLowRedundancyOrPendingReplicationBlocks(blockManager)); + + // Cause the 3 "dead" nodes to be lost while in state decommissioning + // and fill the tracked nodes set with those 3 "dead" nodes + final Map deadNodeProps = + new HashMap<>(); + ArrayList decommissionedNodes = Lists.newArrayList(); + for (final DatanodeDescriptor deadNode : deadNodes) { + // Start decommissioning the node + takeNodeOutofService(0, deadNode.getDatanodeUuid(), 0, decommissionedNodes, + AdminStates.DECOMMISSION_INPROGRESS); + decommissionedNodes.add(deadNode); + + // Stop the datanode so that it is lost while decommissioning + MiniDFSCluster.DataNodeProperties dn = getCluster().stopDataNode(deadNode.getXferAddr()); + deadNodeProps.put(deadNode, dn); + deadNode.setLastUpdate(213); // Set last heartbeat to be in the past + } + assertEquals(3, deadNodeProps.size()); + // Wait for the decommissioning nodes to become dead & to be added to "pendingNodes" + GenericTestUtils.waitFor(() -> deadNodes.stream().noneMatch(DatanodeDescriptor::isAlive) + && decomManager.getNumTrackedNodes() == 0 && decomManager.getNumPendingNodes() == 3, 500, + 20000); + + // Run DatanodeAdminManager.Monitor & validate the dead nodes are removed + if (this instanceof TestDecommissionWithBackoffMonitor) { + // For TestDecommissionWithBackoffMonitor an additional tick/execution of the + // DatanodeAdminBackoffMonitor is needed because of how node cancellation is implemented + // - 1st tick, node is de-queued by "processPendingNodes" + // - 2nd tick, node is removed from the DatanodeAdminManager by "processCancelledNodes" + BlockManagerTestUtil.recheckDecommissionState(datanodeManager); + } + BlockManagerTestUtil.recheckDecommissionState(datanodeManager); + assertEquals(0, decomManager.getNumPendingNodes()); + assertEquals(0, decomManager.getNumTrackedNodes()); + assertTrue(BlockManagerTestUtil.anyLowRedundancyOrPendingReplicationBlocks(blockManager)); + + // Delete the file such that its no longer a factor blocking decommissioning + getCluster().getFileSystem().delete(filePath, true); + Assert + .assertFalse(BlockManagerTestUtil.anyLowRedundancyOrPendingReplicationBlocks(blockManager)); + + return deadNodeProps; + } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java index 97c65556e7d15..853b62ccbc765 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java @@ -25,9 +25,11 @@ import java.util.Set; import java.util.concurrent.ExecutionException; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.protocol.Block; +import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerSafeMode.BMSafeModeStatus; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; import org.apache.hadoop.hdfs.server.namenode.NameNode; @@ -39,6 +41,11 @@ import org.apache.hadoop.util.Preconditions; public class BlockManagerTestUtil { + + public static boolean anyLowRedundancyOrPendingReplicationBlocks(final BlockManager blockManager) { + return blockManager.anyLowRedundancyOrPendingReplicationBlocks(); + } + public static void setNodeReplicationLimit(final BlockManager blockManager, final int limit) { blockManager.maxReplicationStreams = limit;