From e71e324613b7e0268b927ae05b1df4882ebafd7d Mon Sep 17 00:00:00 2001 From: gf13871 Date: Sat, 16 Oct 2021 22:39:17 +0800 Subject: [PATCH 01/12] improvepolicy --- .../blockmanagement/AvailableSpaceBlockPlacementPolicy.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java index 6926c3e01c4482..f84d86dfc9ecc2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java @@ -182,8 +182,7 @@ private DatanodeDescriptor select(DatanodeDescriptor a, DatanodeDescriptor b, */ protected int compareDataNode(final DatanodeDescriptor a, final DatanodeDescriptor b, boolean isBalanceLocal) { - if (a.equals(b) - || Math.abs(a.getDfsUsedPercent() - b.getDfsUsedPercent()) < 5 || (( + if (a.equals(b) || (( isBalanceLocal && a.getDfsUsedPercent() < 50))) { return 0; } From 0ed2dd8ca759253b2660284696ae140ed1aa94f1 Mon Sep 17 00:00:00 2001 From: gf13871 Date: Sun, 17 Oct 2021 19:43:19 +0800 Subject: [PATCH 02/12] make paramter configured --- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 11 +++++++ .../AvailableSpaceBlockPlacementPolicy.java | 15 +++++++--- ...RackFaultTolerantBlockPlacementPolicy.java | 12 +++++--- .../src/main/resources/hdfs-default.xml | 22 +++++++++++++- .../markdown/HdfsBlockPlacementPolicies.md | 29 +++++++++++++++++++ ...estAvailableSpaceBlockPlacementPolicy.java | 3 ++ 6 files changed, 83 insertions(+), 9 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 942929ac285e93..7347c69a1e8d5a 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -1195,6 +1195,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys { "dfs.namenode.available-space-block-placement-policy.balanced-space-preference-fraction"; public static final float DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_DEFAULT = 0.6f; + public static final String DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERATE_KEY = + "dfs.namenode.available-space-block-placement-policy.balanced-space-tolerate"; + public static final int DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERATE_DEFAULT = + 5; public static final String DFS_NAMENODE_AVAILABLE_SPACE_RACK_FAULT_TOLERANT_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY = "dfs.namenode.available-space-rack-fault-tolerant-block-placement-policy" @@ -1202,6 +1206,13 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final float DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_RACK_FAULT_TOLERANT_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_DEFAULT = 0.6f; + public static final String + DFS_NAMENODE_AVAILABLE_SPACE_RACK_FAULT_TOLERANT_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERATE_KEY = + "dfs.namenode.available-space-rack-fault-tolerant-block-placement-policy" + + ".balanced-space-tolerate"; + public static final int + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_RACK_FAULT_TOLERANT_PLACEMENT_POLICY_BALANCED_SPACE_TOLERATE_DEFAULT = + 5; public static final String DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCE_LOCAL_NODE_KEY = "dfs.namenode.available-space-block-placement-policy.balance-local-node"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java index f84d86dfc9ecc2..036a11daa609e1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java @@ -18,9 +18,6 @@ package org.apache.hadoop.hdfs.server.blockmanagement; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_DEFAULT; - import java.util.Collection; import java.util.EnumMap; import java.util.List; @@ -37,6 +34,8 @@ import org.apache.hadoop.net.NetworkTopology; import org.apache.hadoop.net.Node; +import static org.apache.hadoop.hdfs.DFSConfigKeys.*; + /** * Space balanced block placement policy. */ @@ -47,6 +46,8 @@ public class AvailableSpaceBlockPlacementPolicy extends private static final Random RAND = new Random(); private int balancedPreference = (int) (100 * DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_DEFAULT); + private int balancedTolerate = + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERATE_DEFAULT; private boolean optimizeLocal; @Override @@ -62,6 +63,11 @@ public void initialize(Configuration conf, FSClusterStats stats, + DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY + " = " + balancedPreferencePercent); + balancedTolerate = + conf.getInt( + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERATE_KEY, + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERATE_DEFAULT); + optimizeLocal = conf.getBoolean( DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCE_LOCAL_NODE_KEY, DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCE_LOCAL_NODE_DEFAULT); @@ -182,7 +188,8 @@ private DatanodeDescriptor select(DatanodeDescriptor a, DatanodeDescriptor b, */ protected int compareDataNode(final DatanodeDescriptor a, final DatanodeDescriptor b, boolean isBalanceLocal) { - if (a.equals(b) || (( + if (a.equals(b) + || Math.abs(a.getDfsUsedPercent() - b.getDfsUsedPercent()) < balancedTolerate || (( isBalanceLocal && a.getDfsUsedPercent() < 50))) { return 0; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceRackFaultTolerantBlockPlacementPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceRackFaultTolerantBlockPlacementPolicy.java index e673347de630d6..ff265f2e023e23 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceRackFaultTolerantBlockPlacementPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceRackFaultTolerantBlockPlacementPolicy.java @@ -31,8 +31,7 @@ import java.util.Collection; import java.util.Random; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_RACK_FAULT_TOLERANT_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_DEFAULT; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_RACK_FAULT_TOLERANT_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.*; /** * Space balanced rack fault tolerant block placement policy. @@ -45,7 +44,8 @@ public class AvailableSpaceRackFaultTolerantBlockPlacementPolicy private static final Random RAND = new Random(); private int balancedPreference = (int) (100 * DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_RACK_FAULT_TOLERANT_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_DEFAULT); - + private int balancedTolerate = + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_RACK_FAULT_TOLERANT_PLACEMENT_POLICY_BALANCED_SPACE_TOLERATE_DEFAULT; @Override public void initialize(Configuration conf, FSClusterStats stats, NetworkTopology clusterMap, Host2NodesMap host2datanodeMap) { @@ -54,6 +54,10 @@ public void initialize(Configuration conf, FSClusterStats stats, DFS_NAMENODE_AVAILABLE_SPACE_RACK_FAULT_TOLERANT_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY, DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_RACK_FAULT_TOLERANT_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_DEFAULT); + balancedTolerate = conf.getInt( + DFS_NAMENODE_AVAILABLE_SPACE_RACK_FAULT_TOLERANT_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERATE_KEY, + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_RACK_FAULT_TOLERANT_PLACEMENT_POLICY_BALANCED_SPACE_TOLERATE_DEFAULT); + LOG.info("Available space rack fault tolerant block placement policy " + "initialized: " + DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_RACK_FAULT_TOLERANT_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY @@ -118,7 +122,7 @@ private DatanodeDescriptor select(DatanodeDescriptor a, protected int compareDataNode(final DatanodeDescriptor a, final DatanodeDescriptor b) { if (a.equals(b) - || Math.abs(a.getDfsUsedPercent() - b.getDfsUsedPercent()) < 5) { + || Math.abs(a.getDfsUsedPercent() - b.getDfsUsedPercent()) < balancedTolerate) { return 0; } return a.getDfsUsedPercent() < b.getDfsUsedPercent() ? -1 : 1; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index a49137bf582cc0..bbe20a40013e9a 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -5067,6 +5067,17 @@ + + dfs.namenode.available-space-block-placement-policy.balanced-space-tolerate + 5 + + Only used when the dfs.block.replicator.classname is set to + org.apache.hadoop.hdfs.server.blockmanagement.AvailableSpaceBlockPlacementPolicy. + Special value between 0 and 100, noninclusive. Increases tolerance of + placing blocks on Datanodes with similar disk space used. + + + dfs.namenode.available-space-block-placement-policy.balance-local-node @@ -5092,7 +5103,16 @@ high load increases as the value reaches near 0. - + + dfs.namenode.available-space-rack-fault-tolerant-block-placement-policy.balanced-space-tolerate + 5 + + Only used when the dfs.block.replicator.classname is set to + org.apache.hadoop.hdfs.server.blockmanagement.AvailableSpaceRackFaultTolerantBlockPlacementPolicy. + Special value between 0 and 100, noninclusive. Increases tolerance of + placing blocks on Datanodes with similar disk space used. + + dfs.namenode.backup.dnrpc-address diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsBlockPlacementPolicies.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsBlockPlacementPolicies.md index 4550f0441b62e7..97d98bb421bd93 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsBlockPlacementPolicies.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsBlockPlacementPolicies.md @@ -119,6 +119,24 @@ The AvailableSpaceBlockPlacementPolicy is a space balanced block placement polic + +dfs.namenode.available-space-block-placement-policy.balanced-space-tolerate +5 + + Special value between 0 and 100, noninclusive. Increases tolerance of + placing blocks on Datanodes with similar disk space used. + + + + + dfs.namenode.available-space-block-placement-policy.datanode-usgae-tolerace-fraction + 0.6 + + Special value between 1 and 100, noninclusive. Increases chance of + placing blocks on Datanodes with less disk space used. + + + dfs.namenode.available-space-block-placement-policy.balance-local-node @@ -160,6 +178,17 @@ amongst maximum number of racks possible and at the same time will try to choose high load increases as the value reaches near 0. + + + dfs.namenode.available-space-rack-fault-tolerant-block-placement-policy.balanced-space-tolerate + 5 + + Only used when the dfs.block.replicator.classname is set to + org.apache.hadoop.hdfs.server.blockmanagement.AvailableSpaceRackFaultTolerantBlockPlacementPolicy. + Special value between 0 and 100, noninclusive. Increases tolerance of + placing blocks on Datanodes with similar disk space used. + + ``` For more details check [HDFS-15288](https://issues.apache.org/jira/browse/HDFS-15288) \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceBlockPlacementPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceBlockPlacementPolicy.java index f58961eb9d0726..cddaed3cfca85e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceBlockPlacementPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceBlockPlacementPolicy.java @@ -61,6 +61,9 @@ public static void setupCluster() throws Exception { conf.setFloat( DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY, 0.6f); + conf.setInt( + DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERATE_KEY, + 5); String[] racks = new String[numRacks]; for (int i = 0; i < numRacks; i++) { racks[i] = "/rack" + i; From 6ec0b22306fb2a8bcdf5d75ca086243c38c2261f Mon Sep 17 00:00:00 2001 From: gf13871 Date: Wed, 20 Oct 2021 01:58:47 +0800 Subject: [PATCH 03/12] fixcomment --- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 12 +++--- .../AvailableSpaceBlockPlacementPolicy.java | 31 ++++++++++---- ...RackFaultTolerantBlockPlacementPolicy.java | 32 +++++++++++---- .../src/main/resources/hdfs-default.xml | 12 ++++-- .../markdown/HdfsBlockPlacementPolicies.md | 21 ++++------ ...estAvailableSpaceBlockPlacementPolicy.java | 38 ++++++++++++++++- ...estAvailableSpaceRackFaultTolerantBPP.java | 41 +++++++++++++++++++ 7 files changed, 148 insertions(+), 39 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 7347c69a1e8d5a..985b845bd4707c 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -1195,9 +1195,9 @@ public class DFSConfigKeys extends CommonConfigurationKeys { "dfs.namenode.available-space-block-placement-policy.balanced-space-preference-fraction"; public static final float DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_DEFAULT = 0.6f; - public static final String DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERATE_KEY = - "dfs.namenode.available-space-block-placement-policy.balanced-space-tolerate"; - public static final int DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERATE_DEFAULT = + public static final String DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_KEY = + "dfs.namenode.available-space-block-placement-policy.balanced-space-tolerance"; + public static final int DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT = 5; public static final String DFS_NAMENODE_AVAILABLE_SPACE_RACK_FAULT_TOLERANT_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY = @@ -1207,11 +1207,11 @@ public class DFSConfigKeys extends CommonConfigurationKeys { DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_RACK_FAULT_TOLERANT_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_DEFAULT = 0.6f; public static final String - DFS_NAMENODE_AVAILABLE_SPACE_RACK_FAULT_TOLERANT_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERATE_KEY = + DFS_NAMENODE_AVAILABLE_SPACE_RACK_FAULT_TOLERANT_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_KEY = "dfs.namenode.available-space-rack-fault-tolerant-block-placement-policy" - + ".balanced-space-tolerate"; + + ".balanced-space-tolerance"; public static final int - DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_RACK_FAULT_TOLERANT_PLACEMENT_POLICY_BALANCED_SPACE_TOLERATE_DEFAULT = + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_RACK_FAULT_TOLERANT_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT = 5; public static final String DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCE_LOCAL_NODE_KEY = diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java index 036a11daa609e1..2b846c703efa55 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java @@ -18,6 +18,11 @@ package org.apache.hadoop.hdfs.server.blockmanagement; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_KEY; + import java.util.Collection; import java.util.EnumMap; import java.util.List; @@ -34,7 +39,6 @@ import org.apache.hadoop.net.NetworkTopology; import org.apache.hadoop.net.Node; -import static org.apache.hadoop.hdfs.DFSConfigKeys.*; /** * Space balanced block placement policy. @@ -46,8 +50,8 @@ public class AvailableSpaceBlockPlacementPolicy extends private static final Random RAND = new Random(); private int balancedPreference = (int) (100 * DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_DEFAULT); - private int balancedTolerate = - DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERATE_DEFAULT; + private int balancedSpaceTolerance = + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT; private boolean optimizeLocal; @Override @@ -60,13 +64,13 @@ public void initialize(Configuration conf, FSClusterStats stats, DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_DEFAULT); LOG.info("Available space block placement policy initialized: " - + DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY + + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY + " = " + balancedPreferencePercent); - balancedTolerate = + balancedSpaceTolerance = conf.getInt( - DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERATE_KEY, - DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERATE_DEFAULT); + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_KEY, + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT); optimizeLocal = conf.getBoolean( DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCE_LOCAL_NODE_KEY, @@ -83,6 +87,17 @@ public void initialize(Configuration conf, FSClusterStats stats, + " is less than 0.5 so datanodes with more used percent will" + " receive more block allocations."); } + + if (balancedSpaceTolerance >= 20 || balancedSpaceTolerance <= 0 ) { + LOG.warn("The value of " + + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_KEY + + " is invalid, Default value " + + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT + + " will be used instead. Increases tolerance of" + + " placing blocks on Datanodes with similar disk space used "); + balancedSpaceTolerance = + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT;; + } balancedPreference = (int) (100 * balancedPreferencePercent); } @@ -189,7 +204,7 @@ private DatanodeDescriptor select(DatanodeDescriptor a, DatanodeDescriptor b, protected int compareDataNode(final DatanodeDescriptor a, final DatanodeDescriptor b, boolean isBalanceLocal) { if (a.equals(b) - || Math.abs(a.getDfsUsedPercent() - b.getDfsUsedPercent()) < balancedTolerate || (( + || Math.abs(a.getDfsUsedPercent() - b.getDfsUsedPercent()) < balancedSpaceTolerance || (( isBalanceLocal && a.getDfsUsedPercent() < 50))) { return 0; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceRackFaultTolerantBlockPlacementPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceRackFaultTolerantBlockPlacementPolicy.java index ff265f2e023e23..a83d5b1338e58b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceRackFaultTolerantBlockPlacementPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceRackFaultTolerantBlockPlacementPolicy.java @@ -31,7 +31,12 @@ import java.util.Collection; import java.util.Random; -import static org.apache.hadoop.hdfs.DFSConfigKeys.*; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_RACK_FAULT_TOLERANT_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_RACK_FAULT_TOLERANT_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_RACK_FAULT_TOLERANT_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_RACK_FAULT_TOLERANT_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_KEY; /** * Space balanced rack fault tolerant block placement policy. @@ -44,8 +49,8 @@ public class AvailableSpaceRackFaultTolerantBlockPlacementPolicy private static final Random RAND = new Random(); private int balancedPreference = (int) (100 * DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_RACK_FAULT_TOLERANT_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_DEFAULT); - private int balancedTolerate = - DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_RACK_FAULT_TOLERANT_PLACEMENT_POLICY_BALANCED_SPACE_TOLERATE_DEFAULT; + private int balancedSpaceTolerance = + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_RACK_FAULT_TOLERANT_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT; @Override public void initialize(Configuration conf, FSClusterStats stats, NetworkTopology clusterMap, Host2NodesMap host2datanodeMap) { @@ -54,9 +59,9 @@ public void initialize(Configuration conf, FSClusterStats stats, DFS_NAMENODE_AVAILABLE_SPACE_RACK_FAULT_TOLERANT_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY, DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_RACK_FAULT_TOLERANT_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_DEFAULT); - balancedTolerate = conf.getInt( - DFS_NAMENODE_AVAILABLE_SPACE_RACK_FAULT_TOLERANT_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERATE_KEY, - DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_RACK_FAULT_TOLERANT_PLACEMENT_POLICY_BALANCED_SPACE_TOLERATE_DEFAULT); + balancedSpaceTolerance = conf.getInt( + DFS_NAMENODE_AVAILABLE_SPACE_RACK_FAULT_TOLERANT_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_KEY, + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_RACK_FAULT_TOLERANT_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT); LOG.info("Available space rack fault tolerant block placement policy " + "initialized: " @@ -74,6 +79,19 @@ public void initialize(Configuration conf, FSClusterStats stats, + " is less than 0.5 so datanodes with more used percent will" + " receive more block allocations."); } + + + if (balancedSpaceTolerance >= 20 || balancedSpaceTolerance <= 0 ) { + LOG.warn("The value of " + + DFS_NAMENODE_AVAILABLE_SPACE_RACK_FAULT_TOLERANT_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_KEY + + " is invalid, Default value " + + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_RACK_FAULT_TOLERANT_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT + + " will be used instead. Increases tolerance of" + + " placing blocks on Datanodes with similar disk space used "); + balancedSpaceTolerance = + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_RACK_FAULT_TOLERANT_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT;; + } + balancedPreference = (int) (100 * balancedPreferencePercent); } @@ -122,7 +140,7 @@ private DatanodeDescriptor select(DatanodeDescriptor a, protected int compareDataNode(final DatanodeDescriptor a, final DatanodeDescriptor b) { if (a.equals(b) - || Math.abs(a.getDfsUsedPercent() - b.getDfsUsedPercent()) < balancedTolerate) { + || Math.abs(a.getDfsUsedPercent() - b.getDfsUsedPercent()) < balancedSpaceTolerance) { return 0; } return a.getDfsUsedPercent() < b.getDfsUsedPercent() ? -1 : 1; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index bbe20a40013e9a..69fd7e99aaa06a 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -5068,12 +5068,14 @@ - dfs.namenode.available-space-block-placement-policy.balanced-space-tolerate + dfs.namenode.available-space-block-placement-policy.balanced-space-tolerance 5 Only used when the dfs.block.replicator.classname is set to org.apache.hadoop.hdfs.server.blockmanagement.AvailableSpaceBlockPlacementPolicy. - Special value between 0 and 100, noninclusive. Increases tolerance of + Special value between 0 and 20 , noninclusive. if the value is set beyond the scope, + `DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT` + will be set as the default value, Increases tolerance of placing blocks on Datanodes with similar disk space used. @@ -5104,12 +5106,14 @@ - dfs.namenode.available-space-rack-fault-tolerant-block-placement-policy.balanced-space-tolerate + dfs.namenode.available-space-rack-fault-tolerant-block-placement-policy.balanced-space-tolerance 5 Only used when the dfs.block.replicator.classname is set to org.apache.hadoop.hdfs.server.blockmanagement.AvailableSpaceRackFaultTolerantBlockPlacementPolicy. - Special value between 0 and 100, noninclusive. Increases tolerance of + Special value between 0 and 20, noninclusive. if the value is set beyond the scope, + `DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_RACK_FAULT_TOLERANT_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT` + will be set as the default value, Increases tolerance of placing blocks on Datanodes with similar disk space used. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsBlockPlacementPolicies.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsBlockPlacementPolicies.md index 97d98bb421bd93..4ee39899367921 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsBlockPlacementPolicies.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsBlockPlacementPolicies.md @@ -120,23 +120,16 @@ The AvailableSpaceBlockPlacementPolicy is a space balanced block placement polic -dfs.namenode.available-space-block-placement-policy.balanced-space-tolerate +dfs.namenode.available-space-block-placement-policy.balanced-space-tolerance 5 - Special value between 0 and 100, noninclusive. Increases tolerance of + Special value between 0 and 20, noninclusive. if the value is set beyond the scope, + `DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT` + will be set as the default value, Increases tolerance of placing blocks on Datanodes with similar disk space used. - - dfs.namenode.available-space-block-placement-policy.datanode-usgae-tolerace-fraction - 0.6 - - Special value between 1 and 100, noninclusive. Increases chance of - placing blocks on Datanodes with less disk space used. - - - dfs.namenode.available-space-block-placement-policy.balance-local-node @@ -180,12 +173,14 @@ amongst maximum number of racks possible and at the same time will try to choose - dfs.namenode.available-space-rack-fault-tolerant-block-placement-policy.balanced-space-tolerate + dfs.namenode.available-space-rack-fault-tolerant-block-placement-policy.balanced-space-tolerance 5 Only used when the dfs.block.replicator.classname is set to org.apache.hadoop.hdfs.server.blockmanagement.AvailableSpaceRackFaultTolerantBlockPlacementPolicy. - Special value between 0 and 100, noninclusive. Increases tolerance of + Special value between 0 and 20, noninclusive. if the value is set beyond the scope, + `DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_RACK_FAULT_TOLERANT_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT` + will be set as the default value Increases tolerance of placing blocks on Datanodes with similar disk space used. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceBlockPlacementPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceBlockPlacementPolicy.java index cddaed3cfca85e..da6a5f24221564 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceBlockPlacementPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceBlockPlacementPolicy.java @@ -39,6 +39,7 @@ import org.junit.Test; import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; public class TestAvailableSpaceBlockPlacementPolicy { private final static int numRacks = 4; @@ -62,7 +63,7 @@ public static void setupCluster() throws Exception { DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY, 0.6f); conf.setInt( - DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERATE_KEY, + DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_KEY, 5); String[] racks = new String[numRacks]; for (int i = 0; i < numRacks; i++) { @@ -176,6 +177,41 @@ public void testChooseDataNode() { } } + @Test + public void testChooseSimilarDataNode() { + int capacity = 3; + Collection allNodes = new ArrayList<>(capacity); + String[] ownerRackOfNodes = new String[capacity]; + for (int i = 0; i < capacity; i++) { + ownerRackOfNodes[i] = "rack"+i; + } + DatanodeDescriptor tolerateDataNodes[]; + storages = DFSTestUtil.createDatanodeStorageInfos(ownerRackOfNodes); + tolerateDataNodes = DFSTestUtil.toDatanodeDescriptor(storages); + + Collections.addAll(allNodes, tolerateDataNodes); + FSClusterStats statistics = mock(FSClusterStats.class); + AvailableSpaceBlockPlacementPolicy policy = (AvailableSpaceBlockPlacementPolicy) placementPolicy; + placementPolicy.initialize(conf, statistics, null, null); + + updateHeartbeatWithUsage(tolerateDataNodes[0], 20 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * blockSize, + 1 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * blockSize, HdfsServerConstants.MIN_BLOCKS_FOR_WRITE + * blockSize, 0L, 0L, 0L, 0, 0); + + updateHeartbeatWithUsage(tolerateDataNodes[1], 11 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * blockSize, + 1 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * blockSize, HdfsServerConstants.MIN_BLOCKS_FOR_WRITE + * blockSize, 0L, 0L, 0L, 0, 0); + + updateHeartbeatWithUsage(tolerateDataNodes[2], 10 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * blockSize, + 1 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * blockSize, HdfsServerConstants.MIN_BLOCKS_FOR_WRITE + * blockSize, 0L, 0L, 0L, 0, 0); + + assertTrue(policy.compareDataNode(tolerateDataNodes[0],tolerateDataNodes[1],false) == 0); + assertTrue(policy.compareDataNode(tolerateDataNodes[1],tolerateDataNodes[0],false) == 0); + assertTrue(policy.compareDataNode(tolerateDataNodes[0],tolerateDataNodes[2],false) == -1); + assertTrue(policy.compareDataNode(tolerateDataNodes[2],tolerateDataNodes[0],false) == 1); + } + @AfterClass public static void teardownCluster() { if (namenode != null) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceRackFaultTolerantBPP.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceRackFaultTolerantBPP.java index 179c6c6b6931ce..93149efb51a2f2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceRackFaultTolerantBPP.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceRackFaultTolerantBPP.java @@ -41,6 +41,8 @@ import java.util.HashSet; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; /** * Tests AvailableSpaceRackFaultTolerant block placement policy. @@ -66,6 +68,9 @@ public static void setupCluster() throws Exception { conf.setFloat( DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY, 0.6f); + conf.setInt( + DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_RACK_FAULT_TOLERANT_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_KEY, + 5); String[] racks = new String[NUM_RACKS]; for (int i = 0; i < NUM_RACKS; i++) { racks[i] = "/rack" + i; @@ -206,6 +211,42 @@ public void testMaxRackAllocation() { assertEquals(REPLICA, racks.size()); } + @Test + public void testChooseSimilarDataNode() { + int capacity = 3; + Collection allNodes = new ArrayList<>(capacity); + String[] ownerRackOfNodes = new String[capacity]; + for (int i = 0; i < capacity; i++) { + ownerRackOfNodes[i] = "rack"+i; + } + DatanodeDescriptor tolerateDataNodes[]; + storages = DFSTestUtil.createDatanodeStorageInfos(ownerRackOfNodes); + tolerateDataNodes = DFSTestUtil.toDatanodeDescriptor(storages); + + Collections.addAll(allNodes, tolerateDataNodes); + FSClusterStats statistics = mock(FSClusterStats.class); + AvailableSpaceRackFaultTolerantBlockPlacementPolicy policy = + (AvailableSpaceRackFaultTolerantBlockPlacementPolicy) placementPolicy; + placementPolicy.initialize(conf, statistics, null, null); + + updateHeartbeatWithUsage(tolerateDataNodes[0], 20 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE, + 1 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE, HdfsServerConstants.MIN_BLOCKS_FOR_WRITE + * BLOCK_SIZE, 0L, 0L, 0L, 0, 0); + + updateHeartbeatWithUsage(tolerateDataNodes[1], 11 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE, + 1 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE, HdfsServerConstants.MIN_BLOCKS_FOR_WRITE + * BLOCK_SIZE, 0L, 0L, 0L, 0, 0); + + updateHeartbeatWithUsage(tolerateDataNodes[2], 10 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE, + 1 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE, HdfsServerConstants.MIN_BLOCKS_FOR_WRITE + * BLOCK_SIZE, 0L, 0L, 0L, 0, 0); + + assertTrue(policy.compareDataNode(tolerateDataNodes[0],tolerateDataNodes[1]) == 0); + assertTrue(policy.compareDataNode(tolerateDataNodes[1],tolerateDataNodes[0]) == 0); + assertTrue(policy.compareDataNode(tolerateDataNodes[0],tolerateDataNodes[2]) == -1); + assertTrue(policy.compareDataNode(tolerateDataNodes[2],tolerateDataNodes[0]) == 1); + } + @AfterClass public static void teardownCluster() { if (namenode != null) { From e91f9fd64fbf3e66986ea3836d61bddf752cbb06 Mon Sep 17 00:00:00 2001 From: gf13871 Date: Wed, 20 Oct 2021 10:29:06 +0800 Subject: [PATCH 04/12] fixcomment --- .../hadoop-hdfs/src/main/resources/hdfs-default.xml | 6 ++---- .../src/site/markdown/HdfsBlockPlacementPolicies.md | 10 ++++------ 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index 69fd7e99aaa06a..e9e0ae77d2ad5c 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -5074,8 +5074,7 @@ Only used when the dfs.block.replicator.classname is set to org.apache.hadoop.hdfs.server.blockmanagement.AvailableSpaceBlockPlacementPolicy. Special value between 0 and 20 , noninclusive. if the value is set beyond the scope, - `DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT` - will be set as the default value, Increases tolerance of + this value will be set as 5 by default, Increases tolerance of placing blocks on Datanodes with similar disk space used. @@ -5112,8 +5111,7 @@ Only used when the dfs.block.replicator.classname is set to org.apache.hadoop.hdfs.server.blockmanagement.AvailableSpaceRackFaultTolerantBlockPlacementPolicy. Special value between 0 and 20, noninclusive. if the value is set beyond the scope, - `DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_RACK_FAULT_TOLERANT_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT` - will be set as the default value, Increases tolerance of + this value will be set as 5 by default, Increases tolerance of placing blocks on Datanodes with similar disk space used. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsBlockPlacementPolicies.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsBlockPlacementPolicies.md index 4ee39899367921..c87c75345f9607 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsBlockPlacementPolicies.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsBlockPlacementPolicies.md @@ -123,9 +123,8 @@ The AvailableSpaceBlockPlacementPolicy is a space balanced block placement polic dfs.namenode.available-space-block-placement-policy.balanced-space-tolerance 5 - Special value between 0 and 20, noninclusive. if the value is set beyond the scope, - `DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT` - will be set as the default value, Increases tolerance of + Special value between 0 and 20, noninclusive. if the value is set beyond the scope, + this value will be set as 5 by default, Increases tolerance of placing blocks on Datanodes with similar disk space used. @@ -178,9 +177,8 @@ amongst maximum number of racks possible and at the same time will try to choose Only used when the dfs.block.replicator.classname is set to org.apache.hadoop.hdfs.server.blockmanagement.AvailableSpaceRackFaultTolerantBlockPlacementPolicy. - Special value between 0 and 20, noninclusive. if the value is set beyond the scope, - `DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_RACK_FAULT_TOLERANT_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT` - will be set as the default value Increases tolerance of + Special value between 0 and 20, noninclusive. if the value is set beyond the scope, + this value will be set as 5 by default, Increases tolerance of placing blocks on Datanodes with similar disk space used. From 44e478522b7c41581a7c14cfeb0ee37588109538 Mon Sep 17 00:00:00 2001 From: gf13871 Date: Wed, 20 Oct 2021 16:35:45 +0800 Subject: [PATCH 05/12] fixcheckstyle --- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 15 +++++---- .../AvailableSpaceBlockPlacementPolicy.java | 14 ++++---- ...RackFaultTolerantBlockPlacementPolicy.java | 18 +++++----- .../markdown/HdfsBlockPlacementPolicies.md | 2 +- ...estAvailableSpaceBlockPlacementPolicy.java | 33 +++++++++++-------- ...estAvailableSpaceRackFaultTolerantBPP.java | 32 ++++++++++-------- 6 files changed, 63 insertions(+), 51 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 985b845bd4707c..9d36ced882af5f 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -1197,8 +1197,9 @@ public class DFSConfigKeys extends CommonConfigurationKeys { 0.6f; public static final String DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_KEY = "dfs.namenode.available-space-block-placement-policy.balanced-space-tolerance"; - public static final int DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT = - 5; + public static final int + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT = + 5; public static final String DFS_NAMENODE_AVAILABLE_SPACE_RACK_FAULT_TOLERANT_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY = "dfs.namenode.available-space-rack-fault-tolerant-block-placement-policy" @@ -1207,12 +1208,12 @@ public class DFSConfigKeys extends CommonConfigurationKeys { DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_RACK_FAULT_TOLERANT_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_DEFAULT = 0.6f; public static final String - DFS_NAMENODE_AVAILABLE_SPACE_RACK_FAULT_TOLERANT_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_KEY = - "dfs.namenode.available-space-rack-fault-tolerant-block-placement-policy" - + ".balanced-space-tolerance"; + DFS_NAMENODE_AVAILABLE_SPACE_RACK_FAULT_TOLERANT_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_KEY = + "dfs.namenode.available-space-rack-fault-tolerant-block-placement-policy" + + ".balanced-space-tolerance"; public static final int - DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_RACK_FAULT_TOLERANT_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT = - 5; + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_RACK_FAULT_TOLERANT_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT = + 5; public static final String DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCE_LOCAL_NODE_KEY = "dfs.namenode.available-space-block-placement-policy.balance-local-node"; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java index 2b846c703efa55..8e44b1e29b30c6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java @@ -88,15 +88,15 @@ public void initialize(Configuration conf, FSClusterStats stats, + " receive more block allocations."); } - if (balancedSpaceTolerance >= 20 || balancedSpaceTolerance <= 0 ) { + if (balancedSpaceTolerance >= 20 || balancedSpaceTolerance <= 0) { LOG.warn("The value of " - + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_KEY - + " is invalid, Default value " + - DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT - + " will be used instead. Increases tolerance of" - + " placing blocks on Datanodes with similar disk space used "); + + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_KEY + + " is invalid, Default value " + + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT + + " will be used instead. Increases tolerance of" + + " placing blocks on Datanodes with similar disk space used "); balancedSpaceTolerance = - DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT;; + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT; } balancedPreference = (int) (100 * balancedPreferencePercent); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceRackFaultTolerantBlockPlacementPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceRackFaultTolerantBlockPlacementPolicy.java index a83d5b1338e58b..36212285af0c52 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceRackFaultTolerantBlockPlacementPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceRackFaultTolerantBlockPlacementPolicy.java @@ -31,8 +31,6 @@ import java.util.Collection; import java.util.Random; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_RACK_FAULT_TOLERANT_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_RACK_FAULT_TOLERANT_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_RACK_FAULT_TOLERANT_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY; @@ -50,7 +48,7 @@ public class AvailableSpaceRackFaultTolerantBlockPlacementPolicy private int balancedPreference = (int) (100 * DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_RACK_FAULT_TOLERANT_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_DEFAULT); private int balancedSpaceTolerance = - DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_RACK_FAULT_TOLERANT_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT; + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_RACK_FAULT_TOLERANT_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT; @Override public void initialize(Configuration conf, FSClusterStats stats, NetworkTopology clusterMap, Host2NodesMap host2datanodeMap) { @@ -81,15 +79,15 @@ public void initialize(Configuration conf, FSClusterStats stats, } - if (balancedSpaceTolerance >= 20 || balancedSpaceTolerance <= 0 ) { + if (balancedSpaceTolerance >= 20 || balancedSpaceTolerance <= 0) { LOG.warn("The value of " - + DFS_NAMENODE_AVAILABLE_SPACE_RACK_FAULT_TOLERANT_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_KEY - + " is invalid, Default value " + - DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_RACK_FAULT_TOLERANT_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT - + " will be used instead. Increases tolerance of" - + " placing blocks on Datanodes with similar disk space used "); + + DFS_NAMENODE_AVAILABLE_SPACE_RACK_FAULT_TOLERANT_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_KEY + + " is invalid, Default value " + + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_RACK_FAULT_TOLERANT_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT + + " will be used instead. Increases tolerance of" + + " placing blocks on Datanodes with similar disk space used "); balancedSpaceTolerance = - DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_RACK_FAULT_TOLERANT_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT;; + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_RACK_FAULT_TOLERANT_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT; } balancedPreference = (int) (100 * balancedPreferencePercent); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsBlockPlacementPolicies.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsBlockPlacementPolicies.md index c87c75345f9607..0e2f2c46332399 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsBlockPlacementPolicies.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsBlockPlacementPolicies.md @@ -177,7 +177,7 @@ amongst maximum number of racks possible and at the same time will try to choose Only used when the dfs.block.replicator.classname is set to org.apache.hadoop.hdfs.server.blockmanagement.AvailableSpaceRackFaultTolerantBlockPlacementPolicy. - Special value between 0 and 20, noninclusive. if the value is set beyond the scope, + Special value between 0 and 20, noninclusive. if the value is set beyond the scope, this value will be set as 5 by default, Increases tolerance of placing blocks on Datanodes with similar disk space used. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceBlockPlacementPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceBlockPlacementPolicy.java index da6a5f24221564..0b609402ce9a18 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceBlockPlacementPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceBlockPlacementPolicy.java @@ -63,7 +63,7 @@ public static void setupCluster() throws Exception { DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY, 0.6f); conf.setInt( - DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_KEY, + DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_KEY, 5); String[] racks = new String[numRacks]; for (int i = 0; i < numRacks; i++) { @@ -179,37 +179,44 @@ public void testChooseDataNode() { @Test public void testChooseSimilarDataNode() { + DatanodeDescriptor tolerateDataNodes[]; int capacity = 3; Collection allNodes = new ArrayList<>(capacity); String[] ownerRackOfNodes = new String[capacity]; for (int i = 0; i < capacity; i++) { ownerRackOfNodes[i] = "rack"+i; } - DatanodeDescriptor tolerateDataNodes[]; storages = DFSTestUtil.createDatanodeStorageInfos(ownerRackOfNodes); tolerateDataNodes = DFSTestUtil.toDatanodeDescriptor(storages); Collections.addAll(allNodes, tolerateDataNodes); FSClusterStats statistics = mock(FSClusterStats.class); - AvailableSpaceBlockPlacementPolicy policy = (AvailableSpaceBlockPlacementPolicy) placementPolicy; + AvailableSpaceBlockPlacementPolicy policy = + (AvailableSpaceBlockPlacementPolicy) placementPolicy; placementPolicy.initialize(conf, statistics, null, null); - updateHeartbeatWithUsage(tolerateDataNodes[0], 20 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * blockSize, - 1 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * blockSize, HdfsServerConstants.MIN_BLOCKS_FOR_WRITE + updateHeartbeatWithUsage(tolerateDataNodes[0], + 20 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * blockSize, + 1 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * blockSize, + HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * blockSize, 0L, 0L, 0L, 0, 0); - updateHeartbeatWithUsage(tolerateDataNodes[1], 11 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * blockSize, - 1 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * blockSize, HdfsServerConstants.MIN_BLOCKS_FOR_WRITE + updateHeartbeatWithUsage(tolerateDataNodes[1], + 11 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * blockSize, + 1 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * blockSize, + HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * blockSize, 0L, 0L, 0L, 0, 0); - updateHeartbeatWithUsage(tolerateDataNodes[2], 10 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * blockSize, - 1 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * blockSize, HdfsServerConstants.MIN_BLOCKS_FOR_WRITE + updateHeartbeatWithUsage(tolerateDataNodes[2], + 10 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * blockSize, + 1 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * blockSize, + HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * blockSize, 0L, 0L, 0L, 0, 0); - assertTrue(policy.compareDataNode(tolerateDataNodes[0],tolerateDataNodes[1],false) == 0); - assertTrue(policy.compareDataNode(tolerateDataNodes[1],tolerateDataNodes[0],false) == 0); - assertTrue(policy.compareDataNode(tolerateDataNodes[0],tolerateDataNodes[2],false) == -1); - assertTrue(policy.compareDataNode(tolerateDataNodes[2],tolerateDataNodes[0],false) == 1); + assertTrue(policy.compareDataNode(tolerateDataNodes[0], tolerateDataNodes[1], false) == 0); + assertTrue(policy.compareDataNode(tolerateDataNodes[1], tolerateDataNodes[0], false) == 0); + assertTrue(policy.compareDataNode(tolerateDataNodes[0], tolerateDataNodes[2], false) == -1); + assertTrue(policy.compareDataNode(tolerateDataNodes[2], tolerateDataNodes[0], false) == 1); } @AfterClass diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceRackFaultTolerantBPP.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceRackFaultTolerantBPP.java index 93149efb51a2f2..ca0eb696942158 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceRackFaultTolerantBPP.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceRackFaultTolerantBPP.java @@ -69,8 +69,8 @@ public static void setupCluster() throws Exception { DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY, 0.6f); conf.setInt( - DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_RACK_FAULT_TOLERANT_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_KEY, - 5); + DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_RACK_FAULT_TOLERANT_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_KEY, + 5); String[] racks = new String[NUM_RACKS]; for (int i = 0; i < NUM_RACKS; i++) { racks[i] = "/rack" + i; @@ -213,13 +213,13 @@ public void testMaxRackAllocation() { @Test public void testChooseSimilarDataNode() { + DatanodeDescriptor tolerateDataNodes[]; int capacity = 3; Collection allNodes = new ArrayList<>(capacity); String[] ownerRackOfNodes = new String[capacity]; for (int i = 0; i < capacity; i++) { ownerRackOfNodes[i] = "rack"+i; } - DatanodeDescriptor tolerateDataNodes[]; storages = DFSTestUtil.createDatanodeStorageInfos(ownerRackOfNodes); tolerateDataNodes = DFSTestUtil.toDatanodeDescriptor(storages); @@ -229,22 +229,28 @@ public void testChooseSimilarDataNode() { (AvailableSpaceRackFaultTolerantBlockPlacementPolicy) placementPolicy; placementPolicy.initialize(conf, statistics, null, null); - updateHeartbeatWithUsage(tolerateDataNodes[0], 20 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE, - 1 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE, HdfsServerConstants.MIN_BLOCKS_FOR_WRITE + updateHeartbeatWithUsage(tolerateDataNodes[0], + 20 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE, + 1 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE, + HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE, 0L, 0L, 0L, 0, 0); - updateHeartbeatWithUsage(tolerateDataNodes[1], 11 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE, - 1 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE, HdfsServerConstants.MIN_BLOCKS_FOR_WRITE + updateHeartbeatWithUsage(tolerateDataNodes[1], + 11 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE, + 1 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE, + HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE, 0L, 0L, 0L, 0, 0); - updateHeartbeatWithUsage(tolerateDataNodes[2], 10 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE, - 1 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE, HdfsServerConstants.MIN_BLOCKS_FOR_WRITE + updateHeartbeatWithUsage(tolerateDataNodes[2], + 10 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE, + 1 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE, + HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE, 0L, 0L, 0L, 0, 0); - assertTrue(policy.compareDataNode(tolerateDataNodes[0],tolerateDataNodes[1]) == 0); - assertTrue(policy.compareDataNode(tolerateDataNodes[1],tolerateDataNodes[0]) == 0); - assertTrue(policy.compareDataNode(tolerateDataNodes[0],tolerateDataNodes[2]) == -1); - assertTrue(policy.compareDataNode(tolerateDataNodes[2],tolerateDataNodes[0]) == 1); + assertTrue(policy.compareDataNode(tolerateDataNodes[0], tolerateDataNodes[1]) == 0); + assertTrue(policy.compareDataNode(tolerateDataNodes[1], tolerateDataNodes[0]) == 0); + assertTrue(policy.compareDataNode(tolerateDataNodes[0], tolerateDataNodes[2]) == -1); + assertTrue(policy.compareDataNode(tolerateDataNodes[2], tolerateDataNodes[0]) == 1); } @AfterClass From dbfd21bd348c06f1bb777f890b8932a46de3a2df Mon Sep 17 00:00:00 2001 From: gf13871 Date: Wed, 20 Oct 2021 23:28:43 +0800 Subject: [PATCH 06/12] optimizecode --- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 3 +- .../AvailableSpaceBlockPlacementPolicy.java | 6 ++-- ...estAvailableSpaceBlockPlacementPolicy.java | 32 +++++++++---------- ...estAvailableSpaceRackFaultTolerantBPP.java | 30 ++++++++--------- 4 files changed, 36 insertions(+), 35 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 9d36ced882af5f..92f98675e5b1f1 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -1195,7 +1195,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys { "dfs.namenode.available-space-block-placement-policy.balanced-space-preference-fraction"; public static final float DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_DEFAULT = 0.6f; - public static final String DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_KEY = + public static final String + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_KEY = "dfs.namenode.available-space-block-placement-policy.balanced-space-tolerance"; public static final int DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT = diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java index 8e44b1e29b30c6..0da1637bbb0a49 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java @@ -68,9 +68,9 @@ public void initialize(Configuration conf, FSClusterStats stats, + " = " + balancedPreferencePercent); balancedSpaceTolerance = - conf.getInt( - DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_KEY, - DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT); + conf.getInt( + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_KEY, + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT); optimizeLocal = conf.getBoolean( DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCE_LOCAL_NODE_KEY, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceBlockPlacementPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceBlockPlacementPolicy.java index 0b609402ce9a18..6d9c23c658d4ed 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceBlockPlacementPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceBlockPlacementPolicy.java @@ -64,7 +64,7 @@ public static void setupCluster() throws Exception { 0.6f); conf.setInt( DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_KEY, - 5); + 5); String[] racks = new String[numRacks]; for (int i = 0; i < numRacks; i++) { racks[i] = "/rack" + i; @@ -179,21 +179,21 @@ public void testChooseDataNode() { @Test public void testChooseSimilarDataNode() { - DatanodeDescriptor tolerateDataNodes[]; + DatanodeDescriptor[] tolerateDataNodes; + DatanodeStorageInfo[] tolerateStorages; + AvailableSpaceBlockPlacementPolicy toleratePlacementPolicy; int capacity = 3; - Collection allNodes = new ArrayList<>(capacity); - String[] ownerRackOfNodes = new String[capacity]; + Collection allTolerateNodes = new ArrayList<>(capacity); + String[] ownerRackOfTolerateNodes = new String[capacity]; for (int i = 0; i < capacity; i++) { - ownerRackOfNodes[i] = "rack"+i; + ownerRackOfTolerateNodes[i] = "rack"+i; } - storages = DFSTestUtil.createDatanodeStorageInfos(ownerRackOfNodes); - tolerateDataNodes = DFSTestUtil.toDatanodeDescriptor(storages); + tolerateStorages = DFSTestUtil.createDatanodeStorageInfos(ownerRackOfTolerateNodes); + tolerateDataNodes = DFSTestUtil.toDatanodeDescriptor(tolerateStorages); - Collections.addAll(allNodes, tolerateDataNodes); - FSClusterStats statistics = mock(FSClusterStats.class); - AvailableSpaceBlockPlacementPolicy policy = - (AvailableSpaceBlockPlacementPolicy) placementPolicy; - placementPolicy.initialize(conf, statistics, null, null); + Collections.addAll(allTolerateNodes, tolerateDataNodes); + final BlockManager bm = namenode.getNamesystem().getBlockManager(); + toleratePlacementPolicy = (AvailableSpaceBlockPlacementPolicy)bm.getBlockPlacementPolicy(); updateHeartbeatWithUsage(tolerateDataNodes[0], 20 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * blockSize, @@ -213,10 +213,10 @@ public void testChooseSimilarDataNode() { HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * blockSize, 0L, 0L, 0L, 0, 0); - assertTrue(policy.compareDataNode(tolerateDataNodes[0], tolerateDataNodes[1], false) == 0); - assertTrue(policy.compareDataNode(tolerateDataNodes[1], tolerateDataNodes[0], false) == 0); - assertTrue(policy.compareDataNode(tolerateDataNodes[0], tolerateDataNodes[2], false) == -1); - assertTrue(policy.compareDataNode(tolerateDataNodes[2], tolerateDataNodes[0], false) == 1); + assertTrue(toleratePlacementPolicy.compareDataNode(tolerateDataNodes[0], tolerateDataNodes[1], false) == 0); + assertTrue(toleratePlacementPolicy.compareDataNode(tolerateDataNodes[1], tolerateDataNodes[0], false) == 0); + assertTrue(toleratePlacementPolicy.compareDataNode(tolerateDataNodes[0], tolerateDataNodes[2], false) == -1); + assertTrue(toleratePlacementPolicy.compareDataNode(tolerateDataNodes[2], tolerateDataNodes[0], false) == 1); } @AfterClass diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceRackFaultTolerantBPP.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceRackFaultTolerantBPP.java index ca0eb696942158..01c2ef6c84521e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceRackFaultTolerantBPP.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceRackFaultTolerantBPP.java @@ -213,21 +213,21 @@ public void testMaxRackAllocation() { @Test public void testChooseSimilarDataNode() { - DatanodeDescriptor tolerateDataNodes[]; + DatanodeDescriptor[] tolerateDataNodes; + DatanodeStorageInfo[] tolerateStorages; + AvailableSpaceRackFaultTolerantBlockPlacementPolicy toleratePlacementPolicy; int capacity = 3; - Collection allNodes = new ArrayList<>(capacity); - String[] ownerRackOfNodes = new String[capacity]; + Collection allTolerateNodes = new ArrayList<>(capacity); + String[] ownerRackOfTolerateNodes = new String[capacity]; for (int i = 0; i < capacity; i++) { - ownerRackOfNodes[i] = "rack"+i; + ownerRackOfTolerateNodes[i] = "rack"+i; } - storages = DFSTestUtil.createDatanodeStorageInfos(ownerRackOfNodes); - tolerateDataNodes = DFSTestUtil.toDatanodeDescriptor(storages); + tolerateStorages = DFSTestUtil.createDatanodeStorageInfos(ownerRackOfTolerateNodes); + tolerateDataNodes = DFSTestUtil.toDatanodeDescriptor(tolerateStorages); - Collections.addAll(allNodes, tolerateDataNodes); - FSClusterStats statistics = mock(FSClusterStats.class); - AvailableSpaceRackFaultTolerantBlockPlacementPolicy policy = - (AvailableSpaceRackFaultTolerantBlockPlacementPolicy) placementPolicy; - placementPolicy.initialize(conf, statistics, null, null); + Collections.addAll(allTolerateNodes, tolerateDataNodes); + final BlockManager bm = namenode.getNamesystem().getBlockManager(); + toleratePlacementPolicy = (AvailableSpaceRackFaultTolerantBlockPlacementPolicy)bm.getBlockPlacementPolicy(); updateHeartbeatWithUsage(tolerateDataNodes[0], 20 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE, @@ -247,10 +247,10 @@ public void testChooseSimilarDataNode() { HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE, 0L, 0L, 0L, 0, 0); - assertTrue(policy.compareDataNode(tolerateDataNodes[0], tolerateDataNodes[1]) == 0); - assertTrue(policy.compareDataNode(tolerateDataNodes[1], tolerateDataNodes[0]) == 0); - assertTrue(policy.compareDataNode(tolerateDataNodes[0], tolerateDataNodes[2]) == -1); - assertTrue(policy.compareDataNode(tolerateDataNodes[2], tolerateDataNodes[0]) == 1); + assertTrue(toleratePlacementPolicy.compareDataNode(tolerateDataNodes[0], tolerateDataNodes[1]) == 0); + assertTrue(toleratePlacementPolicy.compareDataNode(tolerateDataNodes[1], tolerateDataNodes[0]) == 0); + assertTrue(toleratePlacementPolicy.compareDataNode(tolerateDataNodes[0], tolerateDataNodes[2]) == -1); + assertTrue(toleratePlacementPolicy.compareDataNode(tolerateDataNodes[2], tolerateDataNodes[0]) == 1); } @AfterClass From 3af5420d63a9686a863f12176dbebff165d90c4f Mon Sep 17 00:00:00 2001 From: gf13871 Date: Wed, 20 Oct 2021 23:29:34 +0800 Subject: [PATCH 07/12] remove unused imports --- .../blockmanagement/TestAvailableSpaceBlockPlacementPolicy.java | 1 - .../blockmanagement/TestAvailableSpaceRackFaultTolerantBPP.java | 1 - 2 files changed, 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceBlockPlacementPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceBlockPlacementPolicy.java index 6d9c23c658d4ed..5cfec0e40c08fa 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceBlockPlacementPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceBlockPlacementPolicy.java @@ -39,7 +39,6 @@ import org.junit.Test; import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.mock; public class TestAvailableSpaceBlockPlacementPolicy { private final static int numRacks = 4; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceRackFaultTolerantBPP.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceRackFaultTolerantBPP.java index 01c2ef6c84521e..cf03a7183596e8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceRackFaultTolerantBPP.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceRackFaultTolerantBPP.java @@ -42,7 +42,6 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.mock; /** * Tests AvailableSpaceRackFaultTolerant block placement policy. From 91b8d430f8e7e85062300f4c8e3019b46477ac2c Mon Sep 17 00:00:00 2001 From: gf13871 Date: Thu, 21 Oct 2021 12:39:31 +0800 Subject: [PATCH 08/12] fixcomment --- .../AvailableSpaceBlockPlacementPolicy.java | 6 ++---- ...vailableSpaceRackFaultTolerantBlockPlacementPolicy.java | 5 ++--- .../hadoop-hdfs/src/main/resources/hdfs-default.xml | 4 ++-- .../src/site/markdown/HdfsBlockPlacementPolicies.md | 4 ++-- .../TestAvailableSpaceBlockPlacementPolicy.java | 7 ++----- .../TestAvailableSpaceRackFaultTolerantBPP.java | 7 ++----- 6 files changed, 12 insertions(+), 21 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java index 0da1637bbb0a49..b67a2c6fc3f486 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java @@ -39,7 +39,6 @@ import org.apache.hadoop.net.NetworkTopology; import org.apache.hadoop.net.Node; - /** * Space balanced block placement policy. */ @@ -88,13 +87,12 @@ public void initialize(Configuration conf, FSClusterStats stats, + " receive more block allocations."); } - if (balancedSpaceTolerance >= 20 || balancedSpaceTolerance <= 0) { + if (balancedSpaceTolerance > 20 || balancedSpaceTolerance < 0) { LOG.warn("The value of " + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_KEY + " is invalid, Default value " + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT - + " will be used instead. Increases tolerance of" - + " placing blocks on Datanodes with similar disk space used "); + + " will be used instead."); balancedSpaceTolerance = DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceRackFaultTolerantBlockPlacementPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceRackFaultTolerantBlockPlacementPolicy.java index 36212285af0c52..8bbbe56da54536 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceRackFaultTolerantBlockPlacementPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceRackFaultTolerantBlockPlacementPolicy.java @@ -79,13 +79,12 @@ public void initialize(Configuration conf, FSClusterStats stats, } - if (balancedSpaceTolerance >= 20 || balancedSpaceTolerance <= 0) { + if (balancedSpaceTolerance > 20 || balancedSpaceTolerance < 0) { LOG.warn("The value of " + DFS_NAMENODE_AVAILABLE_SPACE_RACK_FAULT_TOLERANT_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_KEY + " is invalid, Default value " + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_RACK_FAULT_TOLERANT_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT - + " will be used instead. Increases tolerance of" - + " placing blocks on Datanodes with similar disk space used "); + + " will be used instead."); balancedSpaceTolerance = DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_RACK_FAULT_TOLERANT_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index e9e0ae77d2ad5c..64009da9290a38 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -5073,7 +5073,7 @@ Only used when the dfs.block.replicator.classname is set to org.apache.hadoop.hdfs.server.blockmanagement.AvailableSpaceBlockPlacementPolicy. - Special value between 0 and 20 , noninclusive. if the value is set beyond the scope, + Special value between 0 and 20, inclusive. if the value is set beyond the scope, this value will be set as 5 by default, Increases tolerance of placing blocks on Datanodes with similar disk space used. @@ -5110,7 +5110,7 @@ Only used when the dfs.block.replicator.classname is set to org.apache.hadoop.hdfs.server.blockmanagement.AvailableSpaceRackFaultTolerantBlockPlacementPolicy. - Special value between 0 and 20, noninclusive. if the value is set beyond the scope, + Special value between 0 and 20, inclusive. if the value is set beyond the scope, this value will be set as 5 by default, Increases tolerance of placing blocks on Datanodes with similar disk space used. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsBlockPlacementPolicies.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsBlockPlacementPolicies.md index 0e2f2c46332399..cc664ec4987a96 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsBlockPlacementPolicies.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsBlockPlacementPolicies.md @@ -123,7 +123,7 @@ The AvailableSpaceBlockPlacementPolicy is a space balanced block placement polic dfs.namenode.available-space-block-placement-policy.balanced-space-tolerance 5 - Special value between 0 and 20, noninclusive. if the value is set beyond the scope, + Special value between 0 and 20, inclusive. if the value is set beyond the scope, this value will be set as 5 by default, Increases tolerance of placing blocks on Datanodes with similar disk space used. @@ -177,7 +177,7 @@ amongst maximum number of racks possible and at the same time will try to choose Only used when the dfs.block.replicator.classname is set to org.apache.hadoop.hdfs.server.blockmanagement.AvailableSpaceRackFaultTolerantBlockPlacementPolicy. - Special value between 0 and 20, noninclusive. if the value is set beyond the scope, + Special value between 0 and 20, inclusive. if the value is set beyond the scope, this value will be set as 5 by default, Increases tolerance of placing blocks on Datanodes with similar disk space used. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceBlockPlacementPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceBlockPlacementPolicy.java index 5cfec0e40c08fa..e1d99cf5927c81 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceBlockPlacementPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceBlockPlacementPolicy.java @@ -61,9 +61,6 @@ public static void setupCluster() throws Exception { conf.setFloat( DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY, 0.6f); - conf.setInt( - DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_KEY, - 5); String[] racks = new String[numRacks]; for (int i = 0; i < numRacks; i++) { racks[i] = "/rack" + i; @@ -180,7 +177,6 @@ public void testChooseDataNode() { public void testChooseSimilarDataNode() { DatanodeDescriptor[] tolerateDataNodes; DatanodeStorageInfo[] tolerateStorages; - AvailableSpaceBlockPlacementPolicy toleratePlacementPolicy; int capacity = 3; Collection allTolerateNodes = new ArrayList<>(capacity); String[] ownerRackOfTolerateNodes = new String[capacity]; @@ -192,7 +188,8 @@ public void testChooseSimilarDataNode() { Collections.addAll(allTolerateNodes, tolerateDataNodes); final BlockManager bm = namenode.getNamesystem().getBlockManager(); - toleratePlacementPolicy = (AvailableSpaceBlockPlacementPolicy)bm.getBlockPlacementPolicy(); + AvailableSpaceBlockPlacementPolicy toleratePlacementPolicy = + (AvailableSpaceBlockPlacementPolicy)bm.getBlockPlacementPolicy(); updateHeartbeatWithUsage(tolerateDataNodes[0], 20 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * blockSize, diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceRackFaultTolerantBPP.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceRackFaultTolerantBPP.java index cf03a7183596e8..d6149595594f85 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceRackFaultTolerantBPP.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceRackFaultTolerantBPP.java @@ -67,9 +67,6 @@ public static void setupCluster() throws Exception { conf.setFloat( DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_PREFERENCE_FRACTION_KEY, 0.6f); - conf.setInt( - DFSConfigKeys.DFS_NAMENODE_AVAILABLE_SPACE_RACK_FAULT_TOLERANT_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_KEY, - 5); String[] racks = new String[NUM_RACKS]; for (int i = 0; i < NUM_RACKS; i++) { racks[i] = "/rack" + i; @@ -214,7 +211,6 @@ public void testMaxRackAllocation() { public void testChooseSimilarDataNode() { DatanodeDescriptor[] tolerateDataNodes; DatanodeStorageInfo[] tolerateStorages; - AvailableSpaceRackFaultTolerantBlockPlacementPolicy toleratePlacementPolicy; int capacity = 3; Collection allTolerateNodes = new ArrayList<>(capacity); String[] ownerRackOfTolerateNodes = new String[capacity]; @@ -226,7 +222,8 @@ public void testChooseSimilarDataNode() { Collections.addAll(allTolerateNodes, tolerateDataNodes); final BlockManager bm = namenode.getNamesystem().getBlockManager(); - toleratePlacementPolicy = (AvailableSpaceRackFaultTolerantBlockPlacementPolicy)bm.getBlockPlacementPolicy(); + AvailableSpaceRackFaultTolerantBlockPlacementPolicy toleratePlacementPolicy = + (AvailableSpaceRackFaultTolerantBlockPlacementPolicy)bm.getBlockPlacementPolicy(); updateHeartbeatWithUsage(tolerateDataNodes[0], 20 * HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE, From b9bcef2f3d180d488c61e7652bbd2faf6fc0f6a3 Mon Sep 17 00:00:00 2001 From: gf13871 Date: Thu, 21 Oct 2021 19:53:24 +0800 Subject: [PATCH 09/12] fixcheckstyle --- .../TestAvailableSpaceBlockPlacementPolicy.java | 12 ++++++++---- .../TestAvailableSpaceRackFaultTolerantBPP.java | 12 ++++++++---- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceBlockPlacementPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceBlockPlacementPolicy.java index e1d99cf5927c81..af7690d681e49a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceBlockPlacementPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceBlockPlacementPolicy.java @@ -209,10 +209,14 @@ public void testChooseSimilarDataNode() { HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * blockSize, 0L, 0L, 0L, 0, 0); - assertTrue(toleratePlacementPolicy.compareDataNode(tolerateDataNodes[0], tolerateDataNodes[1], false) == 0); - assertTrue(toleratePlacementPolicy.compareDataNode(tolerateDataNodes[1], tolerateDataNodes[0], false) == 0); - assertTrue(toleratePlacementPolicy.compareDataNode(tolerateDataNodes[0], tolerateDataNodes[2], false) == -1); - assertTrue(toleratePlacementPolicy.compareDataNode(tolerateDataNodes[2], tolerateDataNodes[0], false) == 1); + assertTrue(toleratePlacementPolicy.compareDataNode(tolerateDataNodes[0], + tolerateDataNodes[1], false) == 0); + assertTrue(toleratePlacementPolicy.compareDataNode(tolerateDataNodes[1], + tolerateDataNodes[0], false) == 0); + assertTrue(toleratePlacementPolicy.compareDataNode(tolerateDataNodes[0], + tolerateDataNodes[2], false) == -1); + assertTrue(toleratePlacementPolicy.compareDataNode(tolerateDataNodes[2], + tolerateDataNodes[0], false) == 1); } @AfterClass diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceRackFaultTolerantBPP.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceRackFaultTolerantBPP.java index d6149595594f85..4d249c8ef67cd2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceRackFaultTolerantBPP.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestAvailableSpaceRackFaultTolerantBPP.java @@ -243,10 +243,14 @@ public void testChooseSimilarDataNode() { HdfsServerConstants.MIN_BLOCKS_FOR_WRITE * BLOCK_SIZE, 0L, 0L, 0L, 0, 0); - assertTrue(toleratePlacementPolicy.compareDataNode(tolerateDataNodes[0], tolerateDataNodes[1]) == 0); - assertTrue(toleratePlacementPolicy.compareDataNode(tolerateDataNodes[1], tolerateDataNodes[0]) == 0); - assertTrue(toleratePlacementPolicy.compareDataNode(tolerateDataNodes[0], tolerateDataNodes[2]) == -1); - assertTrue(toleratePlacementPolicy.compareDataNode(tolerateDataNodes[2], tolerateDataNodes[0]) == 1); + assertTrue(toleratePlacementPolicy.compareDataNode(tolerateDataNodes[0], + tolerateDataNodes[1]) == 0); + assertTrue(toleratePlacementPolicy.compareDataNode(tolerateDataNodes[1], + tolerateDataNodes[0]) == 0); + assertTrue(toleratePlacementPolicy.compareDataNode(tolerateDataNodes[0], + tolerateDataNodes[2]) == -1); + assertTrue(toleratePlacementPolicy.compareDataNode(tolerateDataNodes[2], + tolerateDataNodes[0]) == 1); } @AfterClass From 77ab40d95cf6a4479d03032e097ae7cdf6c48573 Mon Sep 17 00:00:00 2001 From: gf13871 Date: Thu, 21 Oct 2021 20:23:50 +0800 Subject: [PATCH 10/12] triggerrebuild From e47e0cf0816674862e51396b8e2a2dd211c7b0ed Mon Sep 17 00:00:00 2001 From: gf13871 Date: Thu, 21 Oct 2021 21:10:08 +0800 Subject: [PATCH 11/12] triggerrebuild From 7d5be4e32e772a732e222d058552b79f42bfafbe Mon Sep 17 00:00:00 2001 From: gf13871 Date: Thu, 21 Oct 2021 21:33:15 +0800 Subject: [PATCH 12/12] fixcomment --- .../blockmanagement/AvailableSpaceBlockPlacementPolicy.java | 2 +- .../AvailableSpaceRackFaultTolerantBlockPlacementPolicy.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java index b67a2c6fc3f486..f4bd87542b969d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceBlockPlacementPolicy.java @@ -90,7 +90,7 @@ public void initialize(Configuration conf, FSClusterStats stats, if (balancedSpaceTolerance > 20 || balancedSpaceTolerance < 0) { LOG.warn("The value of " + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_KEY - + " is invalid, Default value " + + + " is invalid, Current value is " + balancedSpaceTolerance + ", Default value " + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT + " will be used instead."); balancedSpaceTolerance = diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceRackFaultTolerantBlockPlacementPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceRackFaultTolerantBlockPlacementPolicy.java index 8bbbe56da54536..365990a7ce9422 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceRackFaultTolerantBlockPlacementPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/AvailableSpaceRackFaultTolerantBlockPlacementPolicy.java @@ -82,7 +82,7 @@ public void initialize(Configuration conf, FSClusterStats stats, if (balancedSpaceTolerance > 20 || balancedSpaceTolerance < 0) { LOG.warn("The value of " + DFS_NAMENODE_AVAILABLE_SPACE_RACK_FAULT_TOLERANT_BLOCK_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_KEY - + " is invalid, Default value " + + + " is invalid, Current value is " + balancedSpaceTolerance + ", Default value " + DFS_NAMENODE_AVAILABLE_SPACE_BLOCK_RACK_FAULT_TOLERANT_PLACEMENT_POLICY_BALANCED_SPACE_TOLERANCE_DEFAULT + " will be used instead."); balancedSpaceTolerance =