From 45e867e5f09841dca584708398510021245653ed Mon Sep 17 00:00:00 2001 From: Michael Stack Date: Mon, 23 May 2022 08:45:08 -0700 Subject: [PATCH 1/3] =?UTF-8?q?HDFS-16586.=20Purge=20FsDatasetAsyncDiskSer?= =?UTF-8?q?vice=20threadgroup;=20it=20causes=20BP=E2=80=A6=20(#4338)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * HDFS-16586. Purge FsDatasetAsyncDiskService threadgroup; it causes BPServiceActor IllegalThreadStateException 'fatal exception and exit' Remove the ThreadGroup used by executor factories; they are unused and ThreadGroups auto-destroy when their Thread-member count goes to zero. This behavior is incompatible with the configuration we have on the per-volume executor which is set to let all threads die if no use inside the keepalive time. Signed-off-by: Hexiaoqiao --- .../datanode/fsdataset/impl/FsDatasetAsyncDiskService.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java index aa577a4d2a1b0..9e4ce26a95086 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java @@ -73,7 +73,6 @@ class FsDatasetAsyncDiskService { private final DataNode datanode; private final FsDatasetImpl fsdatasetImpl; - private final ThreadGroup threadGroup; private Map executors = new HashMap(); private Map> deletedBlockIds @@ -91,7 +90,6 @@ class FsDatasetAsyncDiskService { FsDatasetAsyncDiskService(DataNode datanode, FsDatasetImpl fsdatasetImpl) { this.datanode = datanode; this.fsdatasetImpl = fsdatasetImpl; - this.threadGroup = new ThreadGroup(getClass().getSimpleName()); maxNumThreadsPerVolume = datanode.getConf().getInt( DFSConfigKeys.DFS_DATANODE_FSDATASETASYNCDISK_MAX_THREADS_PER_VOLUME_KEY, DFSConfigKeys.DFS_DATANODE_FSDATASETASYNCDISK_MAX_THREADS_PER_VOLUME_DEFAULT); @@ -110,7 +108,7 @@ public Thread newThread(Runnable r) { synchronized (this) { thisIndex = counter++; } - Thread t = new Thread(threadGroup, r); + Thread t = new Thread(r); t.setName("Async disk worker #" + thisIndex + " for volume " + volume); return t; From f3f74013f773e0dbe599a80cee462c73b230946a Mon Sep 17 00:00:00 2001 From: stack Date: Mon, 23 May 2022 21:37:24 -0700 Subject: [PATCH 2/3] Non-change to trigger new test run --- .../datanode/fsdataset/impl/FsDatasetAsyncDiskService.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java index 9e4ce26a95086..e848b05597c32 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java @@ -101,7 +101,6 @@ class FsDatasetAsyncDiskService { private void addExecutorForVolume(final FsVolumeImpl volume) { ThreadFactory threadFactory = new ThreadFactory() { int counter = 0; - @Override public Thread newThread(Runnable r) { int thisIndex; From 9b929f5863b9789ca429ee1cf3d7f44d9b349991 Mon Sep 17 00:00:00 2001 From: stack Date: Tue, 24 May 2022 09:27:24 -0700 Subject: [PATCH 3/3] Another non-change to trigger new build --- .../datanode/fsdataset/impl/FsDatasetAsyncDiskService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java index e848b05597c32..9e4ce26a95086 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java @@ -101,6 +101,7 @@ class FsDatasetAsyncDiskService { private void addExecutorForVolume(final FsVolumeImpl volume) { ThreadFactory threadFactory = new ThreadFactory() { int counter = 0; + @Override public Thread newThread(Runnable r) { int thisIndex;