From a243b760e8acd1d223ba0b9ae144dee4b1993e14 Mon Sep 17 00:00:00 2001 From: cndaimin Date: Thu, 17 Mar 2022 12:13:20 +0800 Subject: [PATCH 1/2] Fix decommission UnsupportedOperationException --- .../DatanodeAdminDefaultMonitor.java | 6 ++-- .../apache/hadoop/hdfs/TestDecommission.java | 32 +++++++++++++++++++ 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminDefaultMonitor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminDefaultMonitor.java index c1d62e35c0375..4149d198c72f8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminDefaultMonitor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeAdminDefaultMonitor.java @@ -390,8 +390,10 @@ private void processBlocksInternal( // Remove the block from the list if it's no longer in the block map, // e.g. the containing file has been deleted if (blockManager.blocksMap.getStoredBlock(block) == null) { - LOG.trace("Removing unknown block {}", block); - it.remove(); + if (pruneReliableBlocks) { + LOG.trace("Removing unknown block {}", block); + it.remove(); + } continue; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java index f3b81acf37437..f3e37af42b46b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java @@ -54,6 +54,7 @@ import org.apache.hadoop.fs.BatchedRemoteIterator.BatchedEntries; import org.apache.hadoop.hdfs.client.HdfsDataInputStream; import org.apache.hadoop.hdfs.client.HdfsDataOutputStream; +import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.DatanodeInfo.AdminStates; @@ -64,6 +65,7 @@ import org.apache.hadoop.hdfs.protocol.OpenFileEntry; import org.apache.hadoop.hdfs.protocol.OpenFilesIterator; import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; @@ -673,6 +675,36 @@ public void testDecommissionWithOpenfile() fdos.close(); } + @Test(timeout = 20000) + public void testDecommissionWithUnknownBlock() throws IOException { + startCluster(1, 3); + + FSNamesystem ns = getCluster().getNamesystem(0); + DatanodeManager datanodeManager = ns.getBlockManager().getDatanodeManager(); + + BlockInfo blk = new BlockInfoContiguous(new Block(1L), (short) 1); + DatanodeDescriptor dn = datanodeManager.getDatanodes().iterator().next(); + dn.getStorageInfos()[0].addBlock(blk, blk); + + datanodeManager.getDatanodeAdminManager().startDecommission(dn); + waitNodeDecommissioned(dn); + } + + private void waitNodeDecommissioned(DatanodeInfo node) { + DatanodeInfo.AdminStates state = DatanodeInfo.AdminStates.DECOMMISSIONED; + boolean done = (state == node.getAdminState()); + while (!done) { + LOG.info("Waiting for node " + node + " to change state to " + + state + " current state: " + node.getAdminState()); + try { + Thread.sleep(500); + } catch (InterruptedException e) { + // nothing + } + done = (state == node.getAdminState()); + } + } + private static String scanIntoString(final ByteArrayOutputStream baos) { final TextStringBuilder sb = new TextStringBuilder(); final Scanner scanner = new Scanner(baos.toString()); From b516ffebac3baa6a41c75bb215a154aeb22dbdc3 Mon Sep 17 00:00:00 2001 From: cndaimin Date: Wed, 13 Apr 2022 17:36:48 +0800 Subject: [PATCH 2/2] Fix tests --- .../apache/hadoop/hdfs/TestDecommission.java | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java index f3e37af42b46b..670ca5fd9a6fe 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java @@ -687,22 +687,7 @@ public void testDecommissionWithUnknownBlock() throws IOException { dn.getStorageInfos()[0].addBlock(blk, blk); datanodeManager.getDatanodeAdminManager().startDecommission(dn); - waitNodeDecommissioned(dn); - } - - private void waitNodeDecommissioned(DatanodeInfo node) { - DatanodeInfo.AdminStates state = DatanodeInfo.AdminStates.DECOMMISSIONED; - boolean done = (state == node.getAdminState()); - while (!done) { - LOG.info("Waiting for node " + node + " to change state to " - + state + " current state: " + node.getAdminState()); - try { - Thread.sleep(500); - } catch (InterruptedException e) { - // nothing - } - done = (state == node.getAdminState()); - } + waitNodeState(dn, DatanodeInfo.AdminStates.DECOMMISSIONED); } private static String scanIntoString(final ByteArrayOutputStream baos) {