Skip to content

HDFS-16457.Make fs.getspaceused.classname reconfigurable #4069

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
tasanuma merged 15 commits into from
Apr 8, 2022
Merged

HDFS-16457.Make fs.getspaceused.classname reconfigurable #4069

Show file tree
Hide file tree
Changes from 11 commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
b8763c3
HDFS-16457
singer-bin Mar 14, 2022
bbac204
Merge branch 'trunk' of https://github.com/apache/hadoop into HDFS-16457
singer-bin Mar 15, 2022
6e8448f
HDFS-16457
singer-bin Mar 15, 2022
30c4323
reslove conficts
singer-bin Mar 16, 2022
25d5a67
the ut of the pr
singer-bin Mar 17, 2022
d0f7902
about ut
singer-bin Mar 18, 2022
49afa24
fix ut
singer-bin Mar 18, 2022
68b4a87
fix ut
singer-bin Mar 22, 2022
36a6695
merge PR 3863
singer-bin Mar 31, 2022
c55da7b
Extension PR 3863
singer-bin Apr 5, 2022
63182b6
fix ut
singer-bin Apr 6, 2022
e5052de
Add windows os judgment
singer-bin Apr 6, 2022
4f3341c
Remove unused methods
singer-bin Apr 6, 2022
7f45dbe
Sort import sentences
singer-bin Apr 7, 2022
b560680
Sort import sentences
singer-bin Apr 7, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 21 additions & 4 deletions ...fs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,9 @@
package org.apache.hadoop.hdfs.server.datanode;


import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_GETSPACEUSED_CLASSNAME;
import org.apache.hadoop.fs.DU;
import org.apache.hadoop.fs.GetSpaceUsed;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCKREPORT_INITIAL_DELAY_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCKREPORT_INITIAL_DELAY_KEY;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DU_INTERVAL_DEFAULT;
Expand Down Expand Up @@ -349,7 +352,8 @@ public class DataNode extends ReconfigurableBase
DFS_DATANODE_MIN_OUTLIER_DETECTION_DISKS_KEY,
DFS_DATANODE_SLOWDISK_LOW_THRESHOLD_MS_KEY,
FS_DU_INTERVAL_KEY,
FS_GETSPACEUSED_JITTER_KEY));
FS_GETSPACEUSED_JITTER_KEY,
FS_GETSPACEUSED_CLASSNAME));

public static final Log METRICS_LOG = LogFactory.getLog("DataNodeMetricsLog");

Expand Down Expand Up @@ -683,6 +687,7 @@ public String reconfigurePropertyImpl(String property, String newVal)
return reconfSlowDiskParameters(property, newVal);
case FS_DU_INTERVAL_KEY:
case FS_GETSPACEUSED_JITTER_KEY:
case FS_GETSPACEUSED_CLASSNAME:
return reconfDfsUsageParameters(property, newVal);
default:
break;
Expand Down Expand Up @@ -879,7 +884,7 @@ private String reconfDfsUsageParameters(String property, String newVal)
for (FsVolumeImpl fsVolume : volumeList) {
Map<String, BlockPoolSlice> blockPoolSlices = fsVolume.getBlockPoolSlices();
for (BlockPoolSlice value : blockPoolSlices.values()) {
value.updateDfsUsageConfig(interval, null);
value.updateDfsUsageConfig(interval, null, null);
}
}
} else if (property.equals(FS_GETSPACEUSED_JITTER_KEY)) {
Expand All @@ -891,13 +896,25 @@ private String reconfDfsUsageParameters(String property, String newVal)
for (FsVolumeImpl fsVolume : volumeList) {
Map<String, BlockPoolSlice> blockPoolSlices = fsVolume.getBlockPoolSlices();
for (BlockPoolSlice value : blockPoolSlices.values()) {
value.updateDfsUsageConfig(null, jitter);
value.updateDfsUsageConfig(null, jitter, null);
}
}
} else if (property.equals(FS_GETSPACEUSED_CLASSNAME)) {
Preconditions.checkNotNull(data, "FsDatasetSpi has not been initialized.");
Class<? extends GetSpaceUsed> klass = (newVal == null ? DU.class :
Class.forName(newVal).asSubclass(GetSpaceUsed.class));
Copy link
Member

@tasanuma tasanuma Apr 6, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@singer-bin Thanks for updating the PR.
We may need to use WindowsGetSpaceUsed.class if it is windows os.

hadoop/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/GetSpaceUsed.java

Lines 79 to 83 in 4ef1d3e

if (Shell.WINDOWS) {
result = WindowsGetSpaceUsed.class;
} else {
result = DU.class;
}

Copy link
Contributor Author

@singer-bin singer-bin Apr 6, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thank you very much for your review, I will revise it right away. @tasanuma

result = klass.getName();
List<FsVolumeImpl> volumeList = data.getVolumeList();
for (FsVolumeImpl fsVolume : volumeList) {
Map<String, BlockPoolSlice> blockPoolSlices = fsVolume.getBlockPoolSlices();
for (BlockPoolSlice value : blockPoolSlices.values()) {
value.updateDfsUsageConfig(null, null, klass);
}
}
}
LOG.info("RECONFIGURE* changed {} to {}", property, newVal);
return result;
} catch (IllegalArgumentException | IOException e) {
} catch (IllegalArgumentException | IOException | ClassNotFoundException e) {
throw new ReconfigurationException(property, newVal, getConf().get(property), e);
}
}
Expand Down
18 changes: 17 additions & 1 deletion ...s/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.TimeUnit;

import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_GETSPACEUSED_CLASSNAME;
import org.apache.hadoop.hdfs.server.datanode.FSCachingGetSpaceUsed;
import org.apache.hadoop.util.Preconditions;
import org.slf4j.Logger;
Expand Down Expand Up @@ -240,7 +241,8 @@ public void run() {
SHUTDOWN_HOOK_PRIORITY);
}

public void updateDfsUsageConfig(Long interval, Long jitter) throws IOException {
public void updateDfsUsageConfig(Long interval, Long jitter, Class<? extends GetSpaceUsed> klass)
throws IOException {
// Close the old dfsUsage if it is CachingGetSpaceUsed.
if (dfsUsage instanceof CachingGetSpaceUsed) {
((CachingGetSpaceUsed) dfsUsage).close();
Expand All @@ -255,6 +257,10 @@ public void updateDfsUsageConfig(Long interval, Long jitter) throws IOException
FS_GETSPACEUSED_JITTER_KEY + " should be larger than or equal to 0");
config.setLong(FS_GETSPACEUSED_JITTER_KEY, jitter);
}

if (klass != null) {
config.setClass(FS_GETSPACEUSED_CLASSNAME, klass, CachingGetSpaceUsed.class);
}
// Start new dfsUsage.
this.dfsUsage = new FSCachingGetSpaceUsed.Builder().setBpid(bpid)
.setVolume(volume)
Expand All @@ -269,6 +275,16 @@ public GetSpaceUsed getDfsUsage() {
return dfsUsage;
}

public void refreshSpaceUsedKlass(Configuration conf) throws IOException {
((CachingGetSpaceUsed) dfsUsage).close();
this.dfsUsage = new FSCachingGetSpaceUsed.Builder().setBpid(bpid)
.setVolume(volume)
.setPath(bpDir)
.setConf(conf)
.setInitialUsed(loadDfsUsed())
.build();
}

private synchronized static void initializeAddReplicaPool(Configuration conf,
FsDatasetImpl dataset) {
if (addReplicaThreadPool == null) {
Expand Down
33 changes: 32 additions & 1 deletion ...dfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeReconfiguration.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@

package org.apache.hadoop.hdfs.server.datanode;

import org.apache.hadoop.fs.CachingGetSpaceUsed;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_GETSPACEUSED_CLASSNAME;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCKREPORT_INITIAL_DELAY_KEY;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DU_INTERVAL_DEFAULT;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DU_INTERVAL_KEY;
Expand Down Expand Up @@ -58,7 +60,6 @@

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.ReconfigurationException;
import org.apache.hadoop.fs.CachingGetSpaceUsed;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.GetSpaceUsed;
Expand Down Expand Up @@ -86,6 +87,7 @@ public class TestDataNodeReconfiguration {
private final int NUM_NAME_NODE = 1;
private final int NUM_DATA_NODE = 10;
private MiniDFSCluster cluster;
private static long counter = 0;

@Before
public void Setup() throws IOException {
Expand Down Expand Up @@ -756,4 +758,33 @@ public void testDfsUsageParameters() throws ReconfigurationException {
}
}
}

public static class DummyCachingGetSpaceUsed extends CachingGetSpaceUsed {
public DummyCachingGetSpaceUsed(Builder builder) throws IOException {
super(builder.setInterval(1000).setJitter(0L));
}

@Override
protected void refresh() {
counter++;
}
}

@Test
public void testDfsUsageKlass() throws ReconfigurationException, InterruptedException {

long lastCounter = counter;
Thread.sleep(5000);
assertEquals(lastCounter, counter);

for (int i = 0; i < NUM_DATA_NODE; i++) {
DataNode dn = cluster.getDataNodes().get(i);
dn.reconfigurePropertyImpl(FS_GETSPACEUSED_CLASSNAME,
DummyCachingGetSpaceUsed.class.getName());
}

lastCounter = counter;
Thread.sleep(5000);
assertTrue(counter > lastCounter);
}
}
2 changes: 1 addition & 1 deletion hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSAdmin.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -339,7 +339,7 @@ public void testDataNodeGetReconfigurableProperties() throws IOException {
final List<String> outs = Lists.newArrayList();
final List<String> errs = Lists.newArrayList();
getReconfigurableProperties("datanode", address, outs, errs);
assertEquals(18, outs.size());
assertEquals(19, outs.size());
assertEquals(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, outs.get(1));
}

Expand Down