diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JvmPauseMonitor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JvmPauseMonitor.java
index 382266a99401f..371c8dda83540 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JvmPauseMonitor.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JvmPauseMonitor.java
@@ -195,11 +195,11 @@ public void run() {
if (extraSleepTime > warnThresholdMs) {
++numGcWarnThresholdExceeded;
- LOG.warn(formatMessage(
+ LOG.debug(formatMessage(
extraSleepTime, gcTimesAfterSleep, gcTimesBeforeSleep));
} else if (extraSleepTime > infoThresholdMs) {
++numGcInfoThresholdExceeded;
- LOG.info(formatMessage(
+ LOG.debug(formatMessage(
extraSleepTime, gcTimesAfterSleep, gcTimesBeforeSleep));
}
totalGcExtraSleepTime += extraSleepTime;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java
index 659f218437539..8530bd0828d55 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java
@@ -259,7 +259,7 @@ DatanodeStorageInfo findStorageInfo(DatanodeDescriptor dn) {
*/
int findStorageInfo(DatanodeStorageInfo storageInfo) {
int len = getCapacity();
- for(int idx = 0; idx < len; idx++) {
+ for (int idx = 0; idx < len; idx++) {
DatanodeStorageInfo cur = getStorageInfo(idx);
if (cur == storageInfo) {
return idx;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java
index 4b8d092935a00..122da667c165e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -128,8 +128,25 @@ boolean addStorage(DatanodeStorageInfo storage, Block reportedBlock) {
DatanodeStorageInfo old = getStorageInfo(index);
if (old != null && !old.equals(storage)) { // over replicated
// check if the storage has been stored
+ boolean blockIdNotEquals = false;
+ long blockGroupId = BlockIdManager.convertToStripedID(reportedBlock.getBlockId() - blockIndex);
+ Iterator blockIterator = old.getBlockIterator();
+ while (blockIterator.hasNext()) {
+ BlockInfo blockInfo = blockIterator.next();
+ if (!blockInfo.isStriped()) {
+ continue;
+ } else {
+ if (BlockIdManager.convertToStripedID(blockInfo.getBlockId()) == blockGroupId) {
+ Block blockOnOldStorage = ((BlockInfoStriped) blockInfo).getBlockOnStorage(old);
+ if (blockOnOldStorage.getBlockId() != reportedBlock.getBlockId()) {
+ blockIdNotEquals = true;
+ break;
+ }
+ }
+ }
+ }
int i = findStorageInfo(storage);
- if (i == -1) {
+ if (i == -1 || blockIdNotEquals) {
index = findSlot();
} else {
return true;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 9133515afbc02..2b3410d656d8c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -2550,16 +2550,18 @@ DatanodeDescriptor[] chooseSourceDatanodes(BlockInfo block,
BitSet liveBitSet = null;
BitSet decommissioningBitSet = null;
+ HashSet alreadyCorruptedSet = null;
if (isStriped) {
int blockNum = ((BlockInfoStriped) block).getTotalBlockNum();
liveBitSet = new BitSet(blockNum);
decommissioningBitSet = new BitSet(blockNum);
+ alreadyCorruptedSet = new HashSet<>();
}
for (DatanodeStorageInfo storage : blocksMap.getStorages(block)) {
final DatanodeDescriptor node = getDatanodeDescriptorFromStorage(storage);
final StoredReplicaState state = checkReplicaOnStorage(numReplicas, block,
- storage, corruptReplicas.getNodes(block), false);
+ storage, corruptReplicas.getNodes(block), false, alreadyCorruptedSet);
if (state == StoredReplicaState.LIVE) {
if (storage.getStorageType() == StorageType.PROVIDED) {
storage = new DatanodeStorageInfo(node, storage.getStorageID(),
@@ -4543,13 +4545,15 @@ public NumberReplicas countNodes(BlockInfo b) {
NumberReplicas countNodes(BlockInfo b, boolean inStartupSafeMode) {
NumberReplicas numberReplicas = new NumberReplicas();
Collection nodesCorrupt = corruptReplicas.getNodes(b);
+ HashSet alreadyCorruptSet = null;
if (b.isStriped()) {
+ alreadyCorruptSet = new HashSet<>();
countReplicasForStripedBlock(numberReplicas, (BlockInfoStriped) b,
- nodesCorrupt, inStartupSafeMode);
+ nodesCorrupt, inStartupSafeMode, alreadyCorruptSet);
} else {
for (DatanodeStorageInfo storage : blocksMap.getStorages(b)) {
checkReplicaOnStorage(numberReplicas, b, storage, nodesCorrupt,
- inStartupSafeMode);
+ inStartupSafeMode, alreadyCorruptSet);
}
}
return numberReplicas;
@@ -4557,11 +4561,16 @@ NumberReplicas countNodes(BlockInfo b, boolean inStartupSafeMode) {
private StoredReplicaState checkReplicaOnStorage(NumberReplicas counters,
BlockInfo b, DatanodeStorageInfo storage,
- Collection nodesCorrupt, boolean inStartupSafeMode) {
+ Collection nodesCorrupt, boolean inStartupSafeMode,
+ HashSet alreadyCorrupt) {
final StoredReplicaState s;
if (storage.getState() == State.NORMAL) {
final DatanodeDescriptor node = storage.getDatanodeDescriptor();
- if (nodesCorrupt != null && nodesCorrupt.contains(node)) {
+ if (nodesCorrupt != null && nodesCorrupt.contains(node) &&
+ (alreadyCorrupt == null || !alreadyCorrupt.contains(node))) {
+ if (alreadyCorrupt != null) {
+ alreadyCorrupt.add(node);
+ }
s = StoredReplicaState.CORRUPT;
} else if (inStartupSafeMode) {
s = StoredReplicaState.LIVE;
@@ -4607,12 +4616,12 @@ private StoredReplicaState checkReplicaOnStorage(NumberReplicas counters,
*/
private void countReplicasForStripedBlock(NumberReplicas counters,
BlockInfoStriped block, Collection nodesCorrupt,
- boolean inStartupSafeMode) {
+ boolean inStartupSafeMode, HashSet alreadyCorrupt) {
BitSet liveBitSet = new BitSet(block.getTotalBlockNum());
BitSet decommissioningBitSet = new BitSet(block.getTotalBlockNum());
for (StorageAndBlockIndex si : block.getStorageAndIndexInfos()) {
StoredReplicaState state = checkReplicaOnStorage(counters, block,
- si.getStorage(), nodesCorrupt, inStartupSafeMode);
+ si.getStorage(), nodesCorrupt, inStartupSafeMode, alreadyCorrupt);
countLiveAndDecommissioningReplicas(counters, state, liveBitSet,
decommissioningBitSet, si.getBlockIndex());
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
index fc6b537ef2ea8..ba20460496342 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
@@ -262,9 +262,19 @@ public AddBlockResult addBlock(BlockInfo b, Block reportedBlock) {
if (otherStorage != null) {
if (otherStorage != this) {
- // The block belongs to a different storage. Remove it first.
- otherStorage.removeBlock(b);
- result = AddBlockResult.REPLACED;
+ if (!b.isStriped()) {
+ // The block belongs to a different storage. Remove it first.
+ otherStorage.removeBlock(b);
+ result = AddBlockResult.REPLACED;
+ } else {
+ long reportBlockId = reportedBlock.getBlockId();
+ Block blockOnOtherStorage = ((BlockInfoStriped) b).getBlockOnStorage(otherStorage);
+ if (reportBlockId == blockOnOtherStorage.getBlockId()) {
+ // The block belongs to a different storage. Remove it first.
+ otherStorage.removeBlock(b);
+ result = AddBlockResult.REPLACED;
+ }
+ }
} else {
// The block is already associated with this storage.
return AddBlockResult.ALREADY_EXIST;