diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 51fc564a97584..0452ecbd15009 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -396,6 +396,11 @@ public class DFSConfigKeys extends CommonConfigurationKeys { "dfs.namenode.edits.asynclogging.pending.queue.size"; public static final int DFS_NAMENODE_EDITS_ASYNC_LOGGING_PENDING_QUEUE_SIZE_DEFAULT = 4096; + public static final String + DFS_NAMENODE_EDITS_ASYNC_LOGSYNCNOTIFY_EXECUTOR_SIZE = + "dfs.namenode.edits.async.logsyncnotify.executor.size"; + public static final int + DFS_NAMENODE_EDITS_ASYNC_LOGSYNCNOTIFY_EXECUTOR_SIZE_DEFAULT = 10; public static final String DFS_NAMENODE_PROVIDED_ENABLED = "dfs.namenode.provided.enabled"; public static final boolean DFS_NAMENODE_PROVIDED_ENABLED_DEFAULT = false; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogAsync.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogAsync.java index f60b458260b83..3ef50e782c106 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogAsync.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogAsync.java @@ -24,6 +24,8 @@ import java.util.List; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.Semaphore; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -50,6 +52,14 @@ class FSEditLogAsync extends FSEditLog implements Runnable { // requires concurrent access from caller threads and syncing thread. private final BlockingQueue editPendingQ; + // Thread pool for executing logSyncNotify + // It should not be shut down when the FSEditLogAsync is closed, because + // 1) when the state transitions from active to standby and then transitions back, + // it is still using the same FSEditLogAsync object, so the same executor should + // keep working; 2) in all possible scenarios, FSEditLogAsync is a singleton + // in the NameNode process, so it is fine to let this executor run forever. + private final ExecutorService logSyncNotifyExecutor; + // only accessed by syncing thread so no synchronization required. // queue is unbounded because it's effectively limited by the size // of the edit log buffer - ie. a sync will eventually be forced. @@ -67,6 +77,12 @@ class FSEditLogAsync extends FSEditLog implements Runnable { DFS_NAMENODE_EDITS_ASYNC_LOGGING_PENDING_QUEUE_SIZE_DEFAULT); editPendingQ = new ArrayBlockingQueue<>(editPendingQSize); + + int logSyncNotifyExecutorSize = conf.getInt( + DFSConfigKeys.DFS_NAMENODE_EDITS_ASYNC_LOGSYNCNOTIFY_EXECUTOR_SIZE, + DFSConfigKeys. + DFS_NAMENODE_EDITS_ASYNC_LOGSYNCNOTIFY_EXECUTOR_SIZE_DEFAULT); + logSyncNotifyExecutor = Executors.newFixedThreadPool(logSyncNotifyExecutorSize); } private boolean isSyncThreadAlive() { @@ -263,7 +279,9 @@ public void run() { syncEx = ex; } while ((edit = syncWaitQ.poll()) != null) { - edit.logSyncNotify(syncEx); + final Edit notifyEdit = edit; + final RuntimeException ex = syncEx; + logSyncNotifyExecutor.submit(() -> notifyEdit.logSyncNotify(ex)); } } } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index a30889bdd0042..86b7dd26a78d5 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -5050,6 +5050,18 @@ + + dfs.namenode.edits.async.logsyncnotify.executor.size + 10 + + The size of the thread pool executing logSyncNotify in FSEditLogAsync. + This thread pool decouples the RPC response network sending from + the FSEditLogAsync thread so that FSEditLogAsync would not be affected + by the potential network hanging issue (HDFS-15869). + This property defaults to 10. + + + dfs.namenode.edits.dir.minimum 1