From 49bcf163d684ba4b3b95fff0c078c3fb3c17b051 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elek=20M=C3=A1rton?= Date: Fri, 21 Feb 2020 13:49:11 +0100 Subject: [PATCH 1/6] HDDS-3053. Decrease the number of the chunk writer threads --- .../main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 2 +- hadoop-hdds/common/src/main/resources/ozone-default.xml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java index eb2a9e53c99e..dfe7bd2ebf2c 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java @@ -47,7 +47,7 @@ public final class ScmConfigKeys { public static final String DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_KEY = "dfs.container.ratis.num.write.chunk.threads"; public static final int DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_DEFAULT - = 60; + = 10; public static final String DFS_CONTAINER_RATIS_REPLICATION_LEVEL_KEY = "dfs.container.ratis.replication.level"; public static final ReplicationLevel diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index 9c16045aa86a..26c6f02ccde2 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -197,10 +197,10 @@ dfs.container.ratis.num.write.chunk.threads - 60 + 10 OZONE, RATIS, PERFORMANCE Maximum number of threads in the thread pool that Ratis - will use for writing chunks (60 by default). + will use for writing chunks (10 by default). From 9395f98bb7ac6abd2061b5f01886b6ffac4f6c13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elek=20M=C3=A1rton?= Date: Fri, 21 Feb 2020 13:49:11 +0100 Subject: [PATCH 2/6] HDDS-3053. Decrease the number of the chunk writer threads --- .../main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 2 +- hadoop-hdds/common/src/main/resources/ozone-default.xml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java index 5d1b9618bbd6..3a8f8e8d883b 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java @@ -47,7 +47,7 @@ public final class ScmConfigKeys { public static final String DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_KEY = "dfs.container.ratis.num.write.chunk.threads"; public static final int DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_DEFAULT - = 60; + = 10; public static final String DFS_CONTAINER_RATIS_REPLICATION_LEVEL_KEY = "dfs.container.ratis.replication.level"; public static final ReplicationLevel diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index d739acca6700..4976d9370657 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -197,10 +197,10 @@ dfs.container.ratis.num.write.chunk.threads - 60 + 10 OZONE, RATIS, PERFORMANCE Maximum number of threads in the thread pool that Ratis - will use for writing chunks (60 by default). + will use for writing chunks (10 by default). From bf7f40a210b29f0556a0f41dc951e844e715248a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elek=20M=C3=A1rton?= Date: Wed, 8 Apr 2020 12:28:03 +0200 Subject: [PATCH 3/6] multiple the number of threads with the number of disks --- .../src/main/resources/ozone-default.xml | 9 +- .../server/ratis/XceiverServerRatis.java | 89 +++++++++++-------- .../common/volume/MutableVolumeSet.java | 23 +++-- 3 files changed, 72 insertions(+), 49 deletions(-) diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index 03e7a8e09b0f..ef2a83b3fafc 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -199,8 +199,10 @@ dfs.container.ratis.num.write.chunk.threads 10 OZONE, RATIS, PERFORMANCE - Maximum number of threads in the thread pool that Ratis - will use for writing chunks (10 by default). + Maximum number of threads in the thread pool that Datanode + will use for writing replicated chunks. + This is a per configured locations! + (10 thread per disk by default). @@ -390,7 +392,8 @@ ozone.client.stream.buffer.size 4MB OZONE, CLIENT - The size of chunks the client will send to the server. + The size of chunks the client will send to the server. + ozone.client.stream.buffer.flush.size diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java index a3dc7501247f..7562d14a702c 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java @@ -18,78 +18,88 @@ package org.apache.hadoop.ozone.container.common.transport.server.ratis; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ImmutableList; -import com.google.common.util.concurrent.ThreadFactoryBuilder; +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.LinkedBlockingDeque; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.StorageUnit; +import org.apache.hadoop.hdds.conf.DatanodeRatisServerConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReport; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ClosePipelineInfo; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineAction; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.PipelineReport; import org.apache.hadoop.hdds.ratis.ContainerCommandRequestMessage; -import org.apache.hadoop.hdds.utils.HddsServerUtil; +import org.apache.hadoop.hdds.ratis.RatisHelper; import org.apache.hadoop.hdds.scm.pipeline.PipelineID; import org.apache.hadoop.hdds.security.x509.SecurityConfig; import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient; import org.apache.hadoop.hdds.tracing.TracingUtil; +import org.apache.hadoop.hdds.utils.HddsServerUtil; import org.apache.hadoop.ozone.OzoneConfigKeys; - import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.hdds.conf.DatanodeRatisServerConfig; import org.apache.hadoop.ozone.container.common.impl.ContainerData; import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher; import org.apache.hadoop.ozone.container.common.statemachine.StateContext; - -import io.opentracing.Scope; import org.apache.hadoop.ozone.container.common.transport.server.XceiverServerSpi; +import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet; import org.apache.hadoop.ozone.container.ozoneimpl.ContainerController; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import io.opentracing.Scope; import org.apache.ratis.RaftConfigKeys; -import org.apache.hadoop.hdds.ratis.RatisHelper; import org.apache.ratis.conf.RaftProperties; import org.apache.ratis.grpc.GrpcConfigKeys; import org.apache.ratis.grpc.GrpcFactory; import org.apache.ratis.grpc.GrpcTlsConfig; import org.apache.ratis.netty.NettyConfigKeys; -import org.apache.ratis.protocol.*; +import org.apache.ratis.proto.RaftProtos; +import org.apache.ratis.proto.RaftProtos.RoleInfoProto; +import org.apache.ratis.protocol.ClientId; +import org.apache.ratis.protocol.GroupInfoReply; +import org.apache.ratis.protocol.GroupInfoRequest; +import org.apache.ratis.protocol.GroupManagementRequest; +import org.apache.ratis.protocol.NotLeaderException; +import org.apache.ratis.protocol.RaftClientReply; +import org.apache.ratis.protocol.RaftClientRequest; +import org.apache.ratis.protocol.RaftGroup; +import org.apache.ratis.protocol.RaftGroupId; +import org.apache.ratis.protocol.RaftGroupMemberId; +import org.apache.ratis.protocol.RaftPeerId; +import org.apache.ratis.protocol.StateMachineException; import org.apache.ratis.rpc.RpcType; import org.apache.ratis.rpc.SupportedRpcType; import org.apache.ratis.server.RaftServer; import org.apache.ratis.server.RaftServerConfigKeys; -import org.apache.ratis.proto.RaftProtos; -import org.apache.ratis.proto.RaftProtos.RoleInfoProto; -import org.apache.ratis.server.protocol.TermIndex; import org.apache.ratis.server.impl.RaftServerProxy; +import org.apache.ratis.server.protocol.TermIndex; import org.apache.ratis.util.SizeInBytes; import org.apache.ratis.util.TimeDuration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; -import java.io.IOException; -import java.util.Collection; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Collections; -import java.util.Set; -import java.util.UUID; -import java.util.ArrayList; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.LinkedBlockingDeque; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; - /** * Creates a ratis server endpoint that acts as the communication layer for * Ozone containers. @@ -756,10 +766,15 @@ private void sendPipelineReport() { private static List createChunkExecutors( Configuration conf) { // TODO create single pool with N threads if using non-incremental chunks - final int threadCount = conf.getInt( + final int threadCountPerDisk = conf.getInt( OzoneConfigKeys.DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_KEY, OzoneConfigKeys.DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_DEFAULT); - ThreadPoolExecutor[] executors = new ThreadPoolExecutor[threadCount]; + + final int numberOfDisks = + MutableVolumeSet.getDatanodeStorageDirs(conf).size(); + + ThreadPoolExecutor[] executors = + new ThreadPoolExecutor[threadCountPerDisk * numberOfDisks]; for (int i = 0; i < executors.length; i++) { ThreadFactory threadFactory = new ThreadFactoryBuilder() .setDaemon(true) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/MutableVolumeSet.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/MutableVolumeSet.java index 101e6804e54b..593018ef95dc 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/MutableVolumeSet.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/MutableVolumeSet.java @@ -162,15 +162,7 @@ private void initializeVolumeSet() throws IOException { failedVolumeMap = new ConcurrentHashMap<>(); volumeStateMap = new EnumMap<>(StorageType.class); - Collection rawLocations = conf.getTrimmedStringCollection( - HDDS_DATANODE_DIR_KEY); - if (rawLocations.isEmpty()) { - rawLocations = conf.getTrimmedStringCollection(DFS_DATANODE_DATA_DIR_KEY); - } - if (rawLocations.isEmpty()) { - throw new IllegalArgumentException("No location configured in either " - + HDDS_DATANODE_DIR_KEY + " or " + DFS_DATANODE_DATA_DIR_KEY); - } + Collection rawLocations = getDatanodeStorageDirs(conf); for (StorageType storageType : StorageType.values()) { volumeStateMap.put(storageType, new ArrayList<>()); @@ -219,6 +211,19 @@ private void initializeVolumeSet() throws IOException { SHUTDOWN_HOOK_PRIORITY); } + public static Collection getDatanodeStorageDirs(Configuration conf) { + Collection rawLocations = conf.getTrimmedStringCollection( + HDDS_DATANODE_DIR_KEY); + if (rawLocations.isEmpty()) { + rawLocations = conf.getTrimmedStringCollection(DFS_DATANODE_DATA_DIR_KEY); + } + if (rawLocations.isEmpty()) { + throw new IllegalArgumentException("No location configured in either " + + HDDS_DATANODE_DIR_KEY + " or " + DFS_DATANODE_DATA_DIR_KEY); + } + return rawLocations; + } + /** * Run a synchronous parallel check of all HDDS volumes, removing * failed volumes. From 19f7fcaf7d3b366f4f704c2b8e420e0f4cdc575d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elek=20M=C3=A1rton?= Date: Thu, 16 Apr 2020 17:07:25 +0200 Subject: [PATCH 4/6] retrigger build with empty commit From 5e14197ff2d57dfa3a5d8dd0e159c9e3baf6c886 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Elek=20M=C3=A1rton?= Date: Tue, 21 Apr 2020 13:42:03 +0200 Subject: [PATCH 5/6] fixing merge problem --- .../hadoop/ozone/container/common/volume/MutableVolumeSet.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/MutableVolumeSet.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/MutableVolumeSet.java index 81c1afcfaf22..bc61811c868b 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/MutableVolumeSet.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/MutableVolumeSet.java @@ -212,7 +212,8 @@ private void initializeVolumeSet() throws IOException { SHUTDOWN_HOOK_PRIORITY); } - public static Collection getDatanodeStorageDirs(Configuration conf) { + public static Collection getDatanodeStorageDirs( + ConfigurationSource conf) { Collection rawLocations = conf.getTrimmedStringCollection( HDDS_DATANODE_DIR_KEY); if (rawLocations.isEmpty()) { From cc64320569c4a4e5a8ead9df3587eec625566e18 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Wed, 22 Apr 2020 19:45:48 +0200 Subject: [PATCH 6/6] trigger new CI check