From 2ae39689c9f13d9ba6475922ee0dc27cdda13703 Mon Sep 17 00:00:00 2001 From: Sadanand Shenoy Date: Thu, 3 Jun 2021 12:48:31 +0530 Subject: [PATCH 01/17] HDDS-5283.getStorageSize cast to int can cause issue --- .../server/ratis/ContainerStateMachine.java | 6 +++--- .../transport/server/ratis/XceiverServerRatis.java | 14 ++++++++------ .../ozone/om/ratis/OzoneManagerRatisServer.java | 4 ++-- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java index b0751de7c0d9..293418b602ad 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java @@ -169,13 +169,13 @@ public ContainerStateMachine(RaftGroupId gid, ContainerDispatcher dispatcher, int numPendingRequests = conf .getObject(DatanodeRatisServerConfig.class) .getLeaderNumPendingRequests(); - int pendingRequestsByteLimit = (int) conf.getStorageSize( + int pendingRequestsMegaBytesLimit = (int) conf.getStorageSize( OzoneConfigKeys.DFS_CONTAINER_RATIS_LEADER_PENDING_BYTES_LIMIT, OzoneConfigKeys.DFS_CONTAINER_RATIS_LEADER_PENDING_BYTES_LIMIT_DEFAULT, - StorageUnit.BYTES); + StorageUnit.MB); stateMachineDataCache = new ResourceLimitCache<>(new ConcurrentHashMap<>(), (index, data) -> new int[] {1, data.size()}, numPendingRequests, - pendingRequestsByteLimit); + pendingRequestsMegaBytesLimit); this.chunkExecutors = chunkExecutors; diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java index cca1c0891ccf..b7ad5b14b28a 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java @@ -102,6 +102,7 @@ import org.apache.ratis.server.protocol.TermIndex; import org.apache.ratis.util.SizeInBytes; import org.apache.ratis.util.TimeDuration; +import org.apache.ratis.util.TraditionalBinaryPrefix; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -219,7 +220,7 @@ private RaftProperties newRaftProperties() { setRaftSegmentAndWriteBufferSize(properties); // set raft segment pre-allocated size - final int raftSegmentPreallocatedSize = + final long raftSegmentPreallocatedSize = setRaftSegmentPreallocatedSize(properties); TimeUnit timeUnit; @@ -380,8 +381,8 @@ private void setTimeoutForRetryCache(RaftProperties properties) { .setExpiryTime(properties, retryCacheTimeout); } - private int setRaftSegmentPreallocatedSize(RaftProperties properties) { - final int raftSegmentPreallocatedSize = (int) conf.getStorageSize( + private long setRaftSegmentPreallocatedSize(RaftProperties properties) { + final long raftSegmentPreallocatedSize = (long) conf.getStorageSize( OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY, OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT, StorageUnit.BYTES); @@ -425,12 +426,13 @@ private RpcType setRpcType(RaftProperties properties) { private void setPendingRequestsLimits(RaftProperties properties) { - final int pendingRequestsByteLimit = (int)conf.getStorageSize( + final int pendingRequestsMegaBytesLimit = (int)conf.getStorageSize( OzoneConfigKeys.DFS_CONTAINER_RATIS_LEADER_PENDING_BYTES_LIMIT, OzoneConfigKeys.DFS_CONTAINER_RATIS_LEADER_PENDING_BYTES_LIMIT_DEFAULT, - StorageUnit.BYTES); + StorageUnit.MB); RaftServerConfigKeys.Write.setByteLimit(properties, - SizeInBytes.valueOf(pendingRequestsByteLimit)); + SizeInBytes.valueOf(pendingRequestsMegaBytesLimit, + TraditionalBinaryPrefix.MEGA)); } public static XceiverServerRatis newXceiverServerRatis( diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServer.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServer.java index c187d6d5358d..0c2eb0baf201 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServer.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerRatisServer.java @@ -553,7 +553,7 @@ private RaftProperties newRaftProperties(ConfigurationSource conf) { RaftServerConfigKeys.LeaderElection.setPreVote(properties, false); // Set RAFT segment size - final int raftSegmentSize = (int) conf.getStorageSize( + final long raftSegmentSize = (long) conf.getStorageSize( OMConfigKeys.OZONE_OM_RATIS_SEGMENT_SIZE_KEY, OMConfigKeys.OZONE_OM_RATIS_SEGMENT_SIZE_DEFAULT, StorageUnit.BYTES); @@ -562,7 +562,7 @@ private RaftProperties newRaftProperties(ConfigurationSource conf) { RaftServerConfigKeys.Log.setPurgeUptoSnapshotIndex(properties, true); // Set RAFT segment pre-allocated size - final int raftSegmentPreallocatedSize = (int) conf.getStorageSize( + final long raftSegmentPreallocatedSize = (long) conf.getStorageSize( OMConfigKeys.OZONE_OM_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY, OMConfigKeys.OZONE_OM_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT, StorageUnit.BYTES); From 144b52469706a59604c6c29dd2aeab812053b399 Mon Sep 17 00:00:00 2001 From: Sadanand Shenoy Date: Thu, 3 Jun 2021 12:52:20 +0530 Subject: [PATCH 02/17] modified another variable --- .../common/transport/server/ratis/XceiverServerRatis.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java index b7ad5b14b28a..69a68bcefe63 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java @@ -405,7 +405,7 @@ private long setRaftSegmentPreallocatedSize(RaftProperties properties) { } private void setRaftSegmentAndWriteBufferSize(RaftProperties properties) { - final int raftSegmentSize = (int)conf.getStorageSize( + final long raftSegmentSize = (long) conf.getStorageSize( OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_SIZE_KEY, OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT, StorageUnit.BYTES); From bb93d1b2fbed255e4f1e39792f89eacdafda6575 Mon Sep 17 00:00:00 2001 From: Sadanand Shenoy Date: Thu, 3 Jun 2021 22:13:58 +0530 Subject: [PATCH 03/17] trigger new CI check From 8e63aca360ec0468f5f9ddc0ff0fdf0380e47e82 Mon Sep 17 00:00:00 2001 From: Sadanand Shenoy Date: Fri, 4 Jun 2021 11:32:40 +0530 Subject: [PATCH 04/17] trigger new CI check From ed61777a2323bf577484905836146aaceaa545ca Mon Sep 17 00:00:00 2001 From: Sadanand Shenoy Date: Fri, 4 Jun 2021 16:14:15 +0530 Subject: [PATCH 05/17] address comments --- .../common/transport/server/ratis/ContainerStateMachine.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java index 293418b602ad..f5f1e5686e65 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java @@ -174,8 +174,8 @@ public ContainerStateMachine(RaftGroupId gid, ContainerDispatcher dispatcher, OzoneConfigKeys.DFS_CONTAINER_RATIS_LEADER_PENDING_BYTES_LIMIT_DEFAULT, StorageUnit.MB); stateMachineDataCache = new ResourceLimitCache<>(new ConcurrentHashMap<>(), - (index, data) -> new int[] {1, data.size()}, numPendingRequests, - pendingRequestsMegaBytesLimit); + (index, data) -> new int[] {1, (int) StorageUnit.MB.fromBytes(data.size())}, + numPendingRequests, pendingRequestsMegaBytesLimit); this.chunkExecutors = chunkExecutors; From cc1849070452035941af367d6e494035e21f05c8 Mon Sep 17 00:00:00 2001 From: Sadanand Shenoy Date: Fri, 4 Jun 2021 17:15:18 +0530 Subject: [PATCH 06/17] fix checkstyle --- .../common/transport/server/ratis/ContainerStateMachine.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java index f5f1e5686e65..baff4d9320d1 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java @@ -174,8 +174,9 @@ public ContainerStateMachine(RaftGroupId gid, ContainerDispatcher dispatcher, OzoneConfigKeys.DFS_CONTAINER_RATIS_LEADER_PENDING_BYTES_LIMIT_DEFAULT, StorageUnit.MB); stateMachineDataCache = new ResourceLimitCache<>(new ConcurrentHashMap<>(), - (index, data) -> new int[] {1, (int) StorageUnit.MB.fromBytes(data.size())}, - numPendingRequests, pendingRequestsMegaBytesLimit); + (index, data) -> new int[] { 1, + (int) StorageUnit.MB.fromBytes(data.size()) }, numPendingRequests, + pendingRequestsMegaBytesLimit); this.chunkExecutors = chunkExecutors; From 1e739378d336e6693fd0a61d4d6d5084f1718207 Mon Sep 17 00:00:00 2001 From: Sadanand Shenoy Date: Fri, 4 Jun 2021 21:58:30 +0530 Subject: [PATCH 07/17] fix checkstyle --- .../common/transport/server/ratis/ContainerStateMachine.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java index baff4d9320d1..bb6083399e34 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java @@ -174,7 +174,7 @@ public ContainerStateMachine(RaftGroupId gid, ContainerDispatcher dispatcher, OzoneConfigKeys.DFS_CONTAINER_RATIS_LEADER_PENDING_BYTES_LIMIT_DEFAULT, StorageUnit.MB); stateMachineDataCache = new ResourceLimitCache<>(new ConcurrentHashMap<>(), - (index, data) -> new int[] { 1, + (index, data) -> new int[] {1, (int) StorageUnit.MB.fromBytes(data.size()) }, numPendingRequests, pendingRequestsMegaBytesLimit); From 71af3c408910f5143f755014c9ebb8e67deacaab Mon Sep 17 00:00:00 2001 From: Sadanand Shenoy Date: Thu, 24 Jun 2021 16:33:45 +0530 Subject: [PATCH 08/17] data size change --- .../transport/server/ratis/ContainerStateMachine.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java index bb6083399e34..3f83fea183dc 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java @@ -174,9 +174,13 @@ public ContainerStateMachine(RaftGroupId gid, ContainerDispatcher dispatcher, OzoneConfigKeys.DFS_CONTAINER_RATIS_LEADER_PENDING_BYTES_LIMIT_DEFAULT, StorageUnit.MB); stateMachineDataCache = new ResourceLimitCache<>(new ConcurrentHashMap<>(), - (index, data) -> new int[] {1, - (int) StorageUnit.MB.fromBytes(data.size()) }, numPendingRequests, - pendingRequestsMegaBytesLimit); + (index, data) -> new int[] { 1, + // if data.size() is non zero but when conversion to MB becomes 0, + // pass 1 MB as the request size. + ((int) StorageUnit.MB.fromBytes(data.size()) == 0 + && data.size() != 0) ? 1 : + (int) StorageUnit.MB.fromBytes(data.size()) }, + numPendingRequests, pendingRequestsMegaBytesLimit); this.chunkExecutors = chunkExecutors; From c1389fc2ab6d57819e0e81fa21264531b5157a3e Mon Sep 17 00:00:00 2001 From: Sadanand Shenoy Date: Mon, 12 Jul 2021 11:43:51 +0530 Subject: [PATCH 09/17] add method to round up mb --- .../main/java/org/apache/hadoop/hdds/HddsUtils.java | 10 ++++++++++ .../server/ratis/ContainerStateMachine.java | 13 +++++-------- .../transport/server/ratis/XceiverServerRatis.java | 10 ++++++---- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java index a62fd6dd747e..871ca7f93133 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java @@ -65,6 +65,7 @@ import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_PORT_KEY; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_NAMES; +import org.apache.ratis.util.SizeInBytes; import org.apache.hadoop.ozone.conf.OzoneServiceConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -90,6 +91,8 @@ public final class HddsUtils { ScmConfigKeys.OZONE_SCM_NAMES + " must contain a single hostname." + " Multiple SCM hosts are currently unsupported"; + private static final int ONE_MB = SizeInBytes.valueOf("1m").getSizeInt(); + private static final int NO_PORT = -1; private HddsUtils() { @@ -612,4 +615,11 @@ public static String format(List nodes) { public static long getShutDownTimeOut(ConfigurationSource conf) { return conf.getObject(OzoneServiceConfig.class).getServiceShutdownTimeout(); } + + /** + * Utility method to round up bytes into the nearest MB. + */ + public static int roundupMb(long bytes) { + return Math.toIntExact((bytes - 1) / ONE_MB + 1); + } } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java index 3f83fea183dc..325db48ac0d1 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java @@ -169,18 +169,15 @@ public ContainerStateMachine(RaftGroupId gid, ContainerDispatcher dispatcher, int numPendingRequests = conf .getObject(DatanodeRatisServerConfig.class) .getLeaderNumPendingRequests(); - int pendingRequestsMegaBytesLimit = (int) conf.getStorageSize( + long pendingRequestsBytesLimit = (long)conf.getStorageSize( OzoneConfigKeys.DFS_CONTAINER_RATIS_LEADER_PENDING_BYTES_LIMIT, OzoneConfigKeys.DFS_CONTAINER_RATIS_LEADER_PENDING_BYTES_LIMIT_DEFAULT, StorageUnit.MB); + int pendingRequestsMegaBytesLimit = + HddsUtils.roundupMb(pendingRequestsBytesLimit); stateMachineDataCache = new ResourceLimitCache<>(new ConcurrentHashMap<>(), - (index, data) -> new int[] { 1, - // if data.size() is non zero but when conversion to MB becomes 0, - // pass 1 MB as the request size. - ((int) StorageUnit.MB.fromBytes(data.size()) == 0 - && data.size() != 0) ? 1 : - (int) StorageUnit.MB.fromBytes(data.size()) }, - numPendingRequests, pendingRequestsMegaBytesLimit); + (index, data) -> new int[] {1, data.size()}, numPendingRequests, + pendingRequestsMegaBytesLimit); this.chunkExecutors = chunkExecutors; diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java index 69a68bcefe63..66670d70e381 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java @@ -41,6 +41,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; +import org.apache.hadoop.hdds.HddsUtils; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.conf.DatanodeRatisServerConfig; import org.apache.hadoop.hdds.conf.StorageUnit; @@ -426,13 +427,14 @@ private RpcType setRpcType(RaftProperties properties) { private void setPendingRequestsLimits(RaftProperties properties) { - final int pendingRequestsMegaBytesLimit = (int)conf.getStorageSize( + long pendingRequestsBytesLimit = (long) conf.getStorageSize( OzoneConfigKeys.DFS_CONTAINER_RATIS_LEADER_PENDING_BYTES_LIMIT, OzoneConfigKeys.DFS_CONTAINER_RATIS_LEADER_PENDING_BYTES_LIMIT_DEFAULT, StorageUnit.MB); - RaftServerConfigKeys.Write.setByteLimit(properties, - SizeInBytes.valueOf(pendingRequestsMegaBytesLimit, - TraditionalBinaryPrefix.MEGA)); + final int pendingRequestsMegaBytesLimit = + HddsUtils.roundupMb(pendingRequestsBytesLimit); + RaftServerConfigKeys.Write.setByteLimit(properties, SizeInBytes + .valueOf(pendingRequestsMegaBytesLimit, TraditionalBinaryPrefix.MEGA)); } public static XceiverServerRatis newXceiverServerRatis( From c678612fcc8db4a7be9d65acc1b415c0ad958e39 Mon Sep 17 00:00:00 2001 From: Sadanand Shenoy Date: Mon, 12 Jul 2021 15:03:24 +0530 Subject: [PATCH 10/17] fix acceptance test failure --- .../common/transport/server/ratis/ContainerStateMachine.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java index 325db48ac0d1..8d2d2e57254a 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java @@ -176,7 +176,7 @@ public ContainerStateMachine(RaftGroupId gid, ContainerDispatcher dispatcher, int pendingRequestsMegaBytesLimit = HddsUtils.roundupMb(pendingRequestsBytesLimit); stateMachineDataCache = new ResourceLimitCache<>(new ConcurrentHashMap<>(), - (index, data) -> new int[] {1, data.size()}, numPendingRequests, + (index, data) -> new int[] {1, HddsUtils.roundupMb(data.size())}, numPendingRequests, pendingRequestsMegaBytesLimit); this.chunkExecutors = chunkExecutors; From 2f77c59b2634f6de557d404713f76eb3cb58dff0 Mon Sep 17 00:00:00 2001 From: Sadanand Shenoy Date: Mon, 12 Jul 2021 15:04:10 +0530 Subject: [PATCH 11/17] fix checkstyle --- .../common/transport/server/ratis/ContainerStateMachine.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java index 8d2d2e57254a..8a5b38d948b4 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java @@ -176,8 +176,8 @@ public ContainerStateMachine(RaftGroupId gid, ContainerDispatcher dispatcher, int pendingRequestsMegaBytesLimit = HddsUtils.roundupMb(pendingRequestsBytesLimit); stateMachineDataCache = new ResourceLimitCache<>(new ConcurrentHashMap<>(), - (index, data) -> new int[] {1, HddsUtils.roundupMb(data.size())}, numPendingRequests, - pendingRequestsMegaBytesLimit); + (index, data) -> new int[] { 1, HddsUtils.roundupMb(data.size()) }, + numPendingRequests, pendingRequestsMegaBytesLimit); this.chunkExecutors = chunkExecutors; From cc4358765e9cf2d12588919c57334812d9ed0087 Mon Sep 17 00:00:00 2001 From: Sadanand Shenoy Date: Mon, 12 Jul 2021 15:49:56 +0530 Subject: [PATCH 12/17] fix checkstyle --- .../common/transport/server/ratis/ContainerStateMachine.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java index 8a5b38d948b4..7f94eb0585b1 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java @@ -176,7 +176,7 @@ public ContainerStateMachine(RaftGroupId gid, ContainerDispatcher dispatcher, int pendingRequestsMegaBytesLimit = HddsUtils.roundupMb(pendingRequestsBytesLimit); stateMachineDataCache = new ResourceLimitCache<>(new ConcurrentHashMap<>(), - (index, data) -> new int[] { 1, HddsUtils.roundupMb(data.size()) }, + (index, data) -> new int[] {1, HddsUtils.roundupMb(data.size())}, numPendingRequests, pendingRequestsMegaBytesLimit); this.chunkExecutors = chunkExecutors; From 720c9e5425e727e9cfbd3b6df40e7a64da147ef2 Mon Sep 17 00:00:00 2001 From: Sadanand Shenoy Date: Mon, 12 Jul 2021 17:24:10 +0530 Subject: [PATCH 13/17] trigger new CI check From 350ef4a5e2741e36f1416d13ddcbdfff8a17f2ee Mon Sep 17 00:00:00 2001 From: Sadanand Shenoy Date: Thu, 15 Jul 2021 10:04:33 +0530 Subject: [PATCH 14/17] fix acceptance --- .../common/transport/server/ratis/ContainerStateMachine.java | 2 +- .../common/transport/server/ratis/XceiverServerRatis.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java index 7f94eb0585b1..c9cf3fe8932e 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java @@ -172,7 +172,7 @@ public ContainerStateMachine(RaftGroupId gid, ContainerDispatcher dispatcher, long pendingRequestsBytesLimit = (long)conf.getStorageSize( OzoneConfigKeys.DFS_CONTAINER_RATIS_LEADER_PENDING_BYTES_LIMIT, OzoneConfigKeys.DFS_CONTAINER_RATIS_LEADER_PENDING_BYTES_LIMIT_DEFAULT, - StorageUnit.MB); + StorageUnit.BYTES); int pendingRequestsMegaBytesLimit = HddsUtils.roundupMb(pendingRequestsBytesLimit); stateMachineDataCache = new ResourceLimitCache<>(new ConcurrentHashMap<>(), diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java index 66670d70e381..f2376299c88d 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java @@ -430,7 +430,7 @@ private void setPendingRequestsLimits(RaftProperties properties) { long pendingRequestsBytesLimit = (long) conf.getStorageSize( OzoneConfigKeys.DFS_CONTAINER_RATIS_LEADER_PENDING_BYTES_LIMIT, OzoneConfigKeys.DFS_CONTAINER_RATIS_LEADER_PENDING_BYTES_LIMIT_DEFAULT, - StorageUnit.MB); + StorageUnit.BYTES); final int pendingRequestsMegaBytesLimit = HddsUtils.roundupMb(pendingRequestsBytesLimit); RaftServerConfigKeys.Write.setByteLimit(properties, SizeInBytes From 0f237e432778a250e7c9d510635171cd38606fa2 Mon Sep 17 00:00:00 2001 From: Sadanand Shenoy Date: Wed, 4 Aug 2021 15:06:15 +0530 Subject: [PATCH 15/17] Update ratis version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index fffdd3708f15..a2e6b4b6544e 100644 --- a/pom.xml +++ b/pom.xml @@ -72,7 +72,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs ${ozone.version} - 2.1.0 + 2.1.0-03f3b68-SNAPSHOT 0.7.0 From 2512a9d2dc10e0469db0e8ed18e602266b45ba63 Mon Sep 17 00:00:00 2001 From: Sadanand Shenoy Date: Thu, 5 Aug 2021 21:24:15 +0530 Subject: [PATCH 16/17] Us Math.ceil instead --- .../common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java index 871ca7f93133..af5d27a41cd3 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java @@ -620,6 +620,6 @@ public static long getShutDownTimeOut(ConfigurationSource conf) { * Utility method to round up bytes into the nearest MB. */ public static int roundupMb(long bytes) { - return Math.toIntExact((bytes - 1) / ONE_MB + 1); + return (int)Math.ceil((double) bytes/(double) ONE_MB); } } From b553e7800c11af629781eb7f70713c2fd98be179 Mon Sep 17 00:00:00 2001 From: Sadanand Shenoy Date: Wed, 11 Aug 2021 09:21:50 +0530 Subject: [PATCH 17/17] trigger new CI check