From 284f3ec4e98ba422959b16ed5ce18cdcb84367a1 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Sun, 4 Feb 2024 16:35:39 +0100 Subject: [PATCH 1/7] HDDS-10292. Extract configurator for size settings from MiniOzoneCluster --- .../org/apache/hadoop/fs/ozone/TestHSync.java | 16 +- .../hadoop/fs/ozone/TestLeaseRecovery.java | 22 ++- .../fs/ozone/TestOzoneFSInputStream.java | 9 +- .../fs/ozone/TestOzoneFileSystemMetrics.java | 9 +- .../TestOzoneFileSystemWithStreaming.java | 17 +- .../hdds/scm/storage/TestCommitWatcher.java | 16 +- .../hadoop/ozone/ClientConfigBuilder.java | 155 ++++++++++++++++++ .../rpc/AbstractTestECKeyOutputStream.java | 17 +- .../client/rpc/Test2WayCommitInRatis.java | 12 +- .../client/rpc/TestBlockDataStreamOutput.java | 12 +- .../client/rpc/TestBlockOutputStream.java | 12 +- .../TestContainerStateMachineFlushDelay.java | 17 +- .../rpc/TestContainerStateMachineStream.java | 18 +- ...TestFailureHandlingByClientFlushDelay.java | 18 +- .../client/rpc/TestOzoneAtRestEncryption.java | 13 +- ...oneClientRetriesOnExceptionFlushDelay.java | 13 +- .../TestOzoneClientRetriesOnExceptions.java | 12 +- .../rpc/TestOzoneRpcClientAbstract.java | 7 +- .../ozone/client/rpc/TestWatchForCommit.java | 11 +- .../client/rpc/read/TestInputStreamBase.java | 12 +- .../container/TestECContainerRecovery.java | 18 +- .../src/test/resources/ozone-site.xml | 34 ++++ 22 files changed, 378 insertions(+), 92 deletions(-) create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ClientConfigBuilder.java diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSync.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSync.java index 559b8da4982e..ae3831c44a35 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSync.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSync.java @@ -27,9 +27,6 @@ import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.atomic.AtomicReference; -import org.apache.hadoop.fs.FileAlreadyExistsException; -import org.apache.hadoop.hdds.utils.IOUtils; -import org.apache.hadoop.conf.StorageUnit; import org.apache.hadoop.crypto.CipherSuite; import org.apache.hadoop.crypto.CryptoCodec; import org.apache.hadoop.crypto.CryptoOutputStream; @@ -37,15 +34,19 @@ import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileAlreadyExistsException; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.StreamCapabilities; import org.apache.hadoop.hdds.client.DefaultReplicationConfig; import org.apache.hadoop.hdds.client.ECReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.protocol.StorageType; +import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.TableIterator; +import org.apache.hadoop.ozone.ClientConfigBuilder; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.OzoneConsts; @@ -116,9 +117,7 @@ public static void init() throws Exception { CONF.setBoolean(OZONE_OM_RATIS_ENABLE_KEY, false); CONF.set(OZONE_DEFAULT_BUCKET_LAYOUT, layout.name()); CONF.setBoolean(OzoneConfigKeys.OZONE_FS_HSYNC_ENABLED, true); - cluster = MiniOzoneCluster.newBuilder(CONF) - .setNumDatanodes(5) - .setTotalPipelineNumLimit(10) + ClientConfigBuilder.newBuilder(CONF) .setBlockSize(blockSize) .setChunkSize(chunkSize) .setStreamBufferFlushSize(flushSize) @@ -127,6 +126,11 @@ public static void init() throws Exception { .setStreamBufferSizeUnit(StorageUnit.BYTES) .setDataStreamMinPacketSize(chunkSize) .setDataStreamStreamWindowSize(5 * chunkSize) + .setOn(CONF); + + cluster = MiniOzoneCluster.newBuilder(CONF) + .setNumDatanodes(5) + .setTotalPipelineNumLimit(10) .build(); cluster.waitForClusterToBeReady(); client = cluster.newClient(); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestLeaseRecovery.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestLeaseRecovery.java index 1e61c9026b15..5a5ec4008a1b 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestLeaseRecovery.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestLeaseRecovery.java @@ -17,14 +17,15 @@ */ package org.apache.hadoop.fs.ozone; -import org.apache.hadoop.conf.StorageUnit; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.utils.IOUtils; +import org.apache.hadoop.ozone.ClientConfigBuilder; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.TestDataUtil; @@ -91,17 +92,20 @@ public void init() throws IOException, InterruptedException, conf.setBoolean(OZONE_OM_RATIS_ENABLE_KEY, false); conf.setBoolean(OzoneConfigKeys.OZONE_FS_HSYNC_ENABLED, true); conf.set(OZONE_DEFAULT_BUCKET_LAYOUT, layout.name()); + ClientConfigBuilder.newBuilder(conf) + .setBlockSize(blockSize) + .setChunkSize(chunkSize) + .setStreamBufferFlushSize(flushSize) + .setStreamBufferMaxSize(maxFlushSize) + .setDataStreamBufferFlushize(maxFlushSize) + .setStreamBufferSizeUnit(StorageUnit.BYTES) + .setDataStreamMinPacketSize(chunkSize) + .setDataStreamStreamWindowSize(5 * chunkSize) + .setOn(conf); + cluster = MiniOzoneCluster.newBuilder(conf) .setNumDatanodes(5) .setTotalPipelineNumLimit(10) - .setBlockSize(blockSize) - .setChunkSize(chunkSize) - .setStreamBufferFlushSize(flushSize) - .setStreamBufferMaxSize(maxFlushSize) - .setDataStreamBufferFlushize(maxFlushSize) - .setStreamBufferSizeUnit(StorageUnit.BYTES) - .setDataStreamMinPacketSize(chunkSize) - .setDataStreamStreamWindowSize(5 * chunkSize) .build(); cluster.waitForClusterToBeReady(); client = cluster.newClient(); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java index 2e00e8c2c622..4051e3aa08e1 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java @@ -38,6 +38,7 @@ import org.apache.hadoop.hdds.protocol.StorageType; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.SequenceFile; +import org.apache.hadoop.ozone.ClientConfigBuilder; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.TestDataUtil; @@ -83,12 +84,16 @@ public static void init() throws Exception { conf = new OzoneConfiguration(); conf.set(OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT, BucketLayout.LEGACY.name()); - cluster = MiniOzoneCluster.newBuilder(conf) - .setNumDatanodes(5) + + ClientConfigBuilder.newBuilder(conf) .setChunkSize(2) // MB .setBlockSize(8) // MB .setStreamBufferFlushSize(2) // MB .setStreamBufferMaxSize(4) // MB + .setOn(conf); + + cluster = MiniOzoneCluster.newBuilder(conf) + .setNumDatanodes(5) .build(); cluster.waitForClusterToBeReady(); client = cluster.newClient(); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemMetrics.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemMetrics.java index d11ea5376d51..4e4f4265dc20 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemMetrics.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemMetrics.java @@ -25,6 +25,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.ozone.ClientConfigBuilder; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.TestDataUtil; @@ -72,12 +73,16 @@ public static void init() throws Exception { conf.set(OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT, BucketLayout.LEGACY.name()); conf.setBoolean(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS, true); - cluster = MiniOzoneCluster.newBuilder(conf) - .setNumDatanodes(3) + + ClientConfigBuilder.newBuilder(conf) .setChunkSize(2) // MB .setBlockSize(8) // MB .setStreamBufferFlushSize(2) // MB .setStreamBufferMaxSize(4) // MB + .setOn(conf); + + cluster = MiniOzoneCluster.newBuilder(conf) + .setNumDatanodes(3) .build(); cluster.waitForClusterToBeReady(); client = cluster.newClient(); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemWithStreaming.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemWithStreaming.java index 9d5009eeb30c..d0ee48c2dffd 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemWithStreaming.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemWithStreaming.java @@ -22,14 +22,15 @@ import java.io.OutputStream; import java.util.concurrent.ThreadLocalRandom; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.hdds.utils.IOUtils; -import org.apache.hadoop.conf.StorageUnit; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.conf.StorageUnit; +import org.apache.hadoop.hdds.utils.IOUtils; +import org.apache.hadoop.ozone.ClientConfigBuilder; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.OzoneBucket; @@ -86,9 +87,8 @@ public static void init() throws Exception { CONF.set(OZONE_FS_DATASTREAM_AUTO_THRESHOLD, AUTO_THRESHOLD + "B"); CONF.setBoolean(OZONE_OM_RATIS_ENABLE_KEY, true); CONF.set(OZONE_DEFAULT_BUCKET_LAYOUT, layout.name()); - cluster = MiniOzoneCluster.newBuilder(CONF) - .setNumDatanodes(5) - .setTotalPipelineNumLimit(10) + + ClientConfigBuilder.newBuilder(CONF) .setBlockSize(blockSize) .setChunkSize(chunkSize) .setStreamBufferFlushSize(flushSize) @@ -97,6 +97,11 @@ public static void init() throws Exception { .setStreamBufferSizeUnit(StorageUnit.BYTES) .setDataStreamMinPacketSize(chunkSize) .setDataStreamStreamWindowSize(5 * chunkSize) + .setOn(CONF); + + cluster = MiniOzoneCluster.newBuilder(CONF) + .setNumDatanodes(5) + .setTotalPipelineNumLimit(10) .build(); cluster.waitForClusterToBeReady(); client = cluster.newClient(); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestCommitWatcher.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestCommitWatcher.java index d4581d1ad4b2..a4f2858eaee0 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestCommitWatcher.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestCommitWatcher.java @@ -25,10 +25,10 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; -import org.apache.hadoop.conf.StorageUnit; import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.hdds.conf.DatanodeRatisServerConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChecksumType; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandResponseProto; @@ -45,6 +45,7 @@ import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB; import org.apache.hadoop.hdds.utils.IOUtils; +import org.apache.hadoop.ozone.ClientConfigBuilder; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.OzoneConsts; @@ -131,17 +132,20 @@ public void init() throws Exception { clientConfig.setChecksumType(ChecksumType.NONE); conf.setFromObject(clientConfig); + ClientConfigBuilder.newBuilder(conf) + .setBlockSize(blockSize) + .setChunkSize(chunkSize) + .setStreamBufferFlushSize(flushSize) + .setStreamBufferMaxSize(maxFlushSize) + .setStreamBufferSizeUnit(StorageUnit.BYTES) + .setOn(conf); + conf.setQuietMode(false); conf.setStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE, 4, StorageUnit.MB); cluster = MiniOzoneCluster.newBuilder(conf) .setNumDatanodes(5) .setTotalPipelineNumLimit(3) - .setBlockSize(blockSize) - .setChunkSize(chunkSize) - .setStreamBufferFlushSize(flushSize) - .setStreamBufferMaxSize(maxFlushSize) - .setStreamBufferSizeUnit(StorageUnit.BYTES) .build(); cluster.waitForClusterToBeReady(); //the easiest way to create an open container is creating a key diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ClientConfigBuilder.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ClientConfigBuilder.java new file mode 100644 index 000000000000..81a523d066d7 --- /dev/null +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ClientConfigBuilder.java @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone; + +import org.apache.hadoop.hdds.conf.ConfigurationSource; +import org.apache.hadoop.hdds.conf.ConfigurationTarget; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.conf.StorageUnit; +import org.apache.hadoop.hdds.scm.OzoneClientConfig; + +import java.util.Objects; +import java.util.OptionalInt; +import java.util.OptionalLong; + +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE; + +/** + * Helper for tests that want to set client stream properties. + */ +public class ClientConfigBuilder { + + private final ConfigurationSource conf; + + private int chunkSize = 1; + private OptionalLong blockSize = OptionalLong.empty(); + private OptionalInt streamBufferSize = OptionalInt.empty(); + private OptionalLong streamBufferFlushSize = OptionalLong.empty(); + private OptionalLong dataStreamBufferFlushSize = OptionalLong.empty(); + private OptionalLong dataStreamWindowSize = OptionalLong.empty(); + private OptionalLong streamBufferMaxSize = OptionalLong.empty(); + private OptionalInt dataStreamMinPacketSize = OptionalInt.empty(); + private StorageUnit unit = StorageUnit.MB; + + public static ClientConfigBuilder newBuilder() { + return new ClientConfigBuilder(); + } + + public static ClientConfigBuilder newBuilder(ConfigurationSource conf) { + return new ClientConfigBuilder(conf); + } + + private ClientConfigBuilder() { + this(new OzoneConfiguration()); + } + + private ClientConfigBuilder(ConfigurationSource conf) { + this.conf = conf; + } + + public ClientConfigBuilder setChunkSize(int size) { + chunkSize = size; + return this; + } + + public ClientConfigBuilder setBlockSize(long size) { + blockSize = OptionalLong.of(size); + return this; + } + + public ClientConfigBuilder setStreamBufferSize(int size) { + streamBufferSize = OptionalInt.of(size); + return this; + } + + public ClientConfigBuilder setStreamBufferFlushSize(long size) { + streamBufferFlushSize = OptionalLong.of(size); + return this; + } + + public ClientConfigBuilder setStreamBufferMaxSize(long size) { + streamBufferMaxSize = OptionalLong.of(size); + return this; + } + + public ClientConfigBuilder setDataStreamMinPacketSize(int size) { + dataStreamMinPacketSize = OptionalInt.of(size); + return this; + } + + // TODO fix typo + public ClientConfigBuilder setDataStreamBufferFlushize(long size) { + dataStreamBufferFlushSize = OptionalLong.of(size); + return this; + } + + // TODO fix typo in name + public ClientConfigBuilder setDataStreamStreamWindowSize(long size) { + dataStreamWindowSize = OptionalLong.of(size); + return this; + } + + // TODO rename + // TODO merge with setChunkSize + public ClientConfigBuilder setStreamBufferSizeUnit(StorageUnit unit) { + this.unit = Objects.requireNonNull(unit); + return this; + } + + public void setOn(ConfigurationTarget target) { + if (!streamBufferSize.isPresent()) { + streamBufferSize = OptionalInt.of(chunkSize); + } + if (!streamBufferFlushSize.isPresent()) { + streamBufferFlushSize = OptionalLong.of(chunkSize); + } + if (!streamBufferMaxSize.isPresent()) { + streamBufferMaxSize = OptionalLong.of(2 * streamBufferFlushSize.getAsLong()); + } + if (!dataStreamBufferFlushSize.isPresent()) { + dataStreamBufferFlushSize = OptionalLong.of(4L * chunkSize); + } + if (!dataStreamMinPacketSize.isPresent()) { + dataStreamMinPacketSize = OptionalInt.of(chunkSize / 4); + } + if (!dataStreamWindowSize.isPresent()) { + dataStreamWindowSize = OptionalLong.of(8L * chunkSize); + } + if (!blockSize.isPresent()) { + blockSize = OptionalLong.of(2 * streamBufferMaxSize.getAsLong()); + } + + OzoneClientConfig clientConfig = conf.getObject(OzoneClientConfig.class); + clientConfig.setStreamBufferSize((int) toBytes(streamBufferSize.getAsInt())); + clientConfig.setStreamBufferMaxSize(toBytes(streamBufferMaxSize.getAsLong())); + clientConfig.setStreamBufferFlushSize(toBytes(streamBufferFlushSize.getAsLong())); + clientConfig.setDataStreamBufferFlushSize(toBytes(dataStreamBufferFlushSize.getAsLong())); + clientConfig.setDataStreamMinPacketSize((int) toBytes(dataStreamMinPacketSize.getAsInt())); + clientConfig.setStreamWindowSize(toBytes(dataStreamWindowSize.getAsLong())); + + target.setFromObject(clientConfig); + target.setStorageSize(OZONE_SCM_CHUNK_SIZE_KEY, chunkSize, unit); + target.setStorageSize(OZONE_SCM_BLOCK_SIZE, blockSize.getAsLong(), unit); + } + + private long toBytes(long value) { + return Math.round(unit.toBytes(value)); + } + +} diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/AbstractTestECKeyOutputStream.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/AbstractTestECKeyOutputStream.java index 3478489edd66..c7c01a5dfb27 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/AbstractTestECKeyOutputStream.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/AbstractTestECKeyOutputStream.java @@ -16,13 +16,13 @@ */ package org.apache.hadoop.ozone.client.rpc; -import org.apache.hadoop.conf.StorageUnit; import org.apache.hadoop.hdds.HddsConfigKeys; import org.apache.hadoop.hdds.client.DefaultReplicationConfig; import org.apache.hadoop.hdds.client.ECReplicationConfig; import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; @@ -31,6 +31,7 @@ import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.pipeline.PipelineManager; import org.apache.hadoop.hdds.utils.IOUtils; +import org.apache.hadoop.ozone.ClientConfigBuilder; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.client.BucketArgs; @@ -120,11 +121,17 @@ protected static void init(boolean zeroCopyEnabled) throws Exception { TimeUnit.SECONDS); conf.setBoolean(OzoneConfigKeys.OZONE_EC_GRPC_ZERO_COPY_ENABLED, zeroCopyEnabled); - cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(10) - .setTotalPipelineNumLimit(10).setBlockSize(blockSize) - .setChunkSize(chunkSize).setStreamBufferFlushSize(flushSize) + + ClientConfigBuilder.newBuilder(conf) + .setBlockSize(blockSize) + .setChunkSize(chunkSize) + .setStreamBufferFlushSize(flushSize) .setStreamBufferMaxSize(maxFlushSize) - .setStreamBufferSizeUnit(StorageUnit.BYTES).build(); + .setStreamBufferSizeUnit(StorageUnit.BYTES) + .setOn(conf); + + cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(10) + .setTotalPipelineNumLimit(10).build(); cluster.waitForClusterToBeReady(); client = OzoneClientFactory.getRpcClient(conf); objectStore = client.getObjectStore(); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/Test2WayCommitInRatis.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/Test2WayCommitInRatis.java index ce89e679df47..37be406677b1 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/Test2WayCommitInRatis.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/Test2WayCommitInRatis.java @@ -17,10 +17,10 @@ package org.apache.hadoop.ozone.client.rpc; -import org.apache.hadoop.conf.StorageUnit; import org.apache.hadoop.hdds.HddsConfigKeys; import org.apache.hadoop.hdds.conf.DatanodeRatisServerConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.ratis.conf.RatisClientConfig; import org.apache.hadoop.hdds.scm.XceiverClientManager; @@ -32,6 +32,7 @@ import org.apache.hadoop.hdds.scm.protocolPB. StorageContainerLocationProtocolClientSideTranslatorPB; import org.apache.hadoop.hdds.utils.IOUtils; +import org.apache.hadoop.ozone.ClientConfigBuilder; import org.apache.hadoop.ozone.HddsDatanodeService; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConsts; @@ -100,14 +101,17 @@ private void startCluster(OzoneConfiguration conf) throws Exception { raftClientConfig.setRpcWatchRequestTimeout(Duration.ofSeconds(10)); conf.setFromObject(raftClientConfig); - conf.setQuietMode(false); - cluster = MiniOzoneCluster.newBuilder(conf) - .setNumDatanodes(3) + ClientConfigBuilder.newBuilder(conf) .setBlockSize(blockSize) .setChunkSize(chunkSize) .setStreamBufferFlushSize(flushSize) .setStreamBufferMaxSize(maxFlushSize) .setStreamBufferSizeUnit(StorageUnit.BYTES) + .setOn(conf); + + conf.setQuietMode(false); + cluster = MiniOzoneCluster.newBuilder(conf) + .setNumDatanodes(3) .build(); cluster.waitForClusterToBeReady(); // the easiest way to create an open container is creating a key diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java index d3caf6238737..0190c203246a 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java @@ -17,9 +17,9 @@ package org.apache.hadoop.ozone.client.rpc; -import org.apache.hadoop.conf.StorageUnit; import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.hdds.scm.OzoneClientConfig; import org.apache.hadoop.hdds.scm.XceiverClientManager; @@ -27,6 +27,7 @@ import org.apache.hadoop.hdds.scm.storage.BlockDataStreamOutput; import org.apache.hadoop.hdds.scm.storage.ByteBufferStreamOutput; import org.apache.hadoop.hdds.utils.IOUtils; +import org.apache.hadoop.ozone.ClientConfigBuilder; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.client.ObjectStore; @@ -92,9 +93,7 @@ public static void init() throws Exception { conf.setStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE, 4, StorageUnit.MB); - cluster = MiniOzoneCluster.newBuilder(conf) - .setNumDatanodes(5) - .setTotalPipelineNumLimit(3) + ClientConfigBuilder.newBuilder(conf) .setBlockSize(blockSize) .setChunkSize(chunkSize) .setStreamBufferFlushSize(flushSize) @@ -103,6 +102,11 @@ public static void init() throws Exception { .setStreamBufferSizeUnit(StorageUnit.BYTES) .setDataStreamMinPacketSize(chunkSize) .setDataStreamStreamWindowSize(5 * chunkSize) + .setOn(conf); + + cluster = MiniOzoneCluster.newBuilder(conf) + .setNumDatanodes(5) + .setTotalPipelineNumLimit(3) .build(); cluster.waitForClusterToBeReady(); //the easiest way to create an open container is creating a key diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockOutputStream.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockOutputStream.java index 74686d363c82..8e6e5d932af3 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockOutputStream.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockOutputStream.java @@ -24,12 +24,12 @@ import java.util.concurrent.TimeoutException; import org.apache.commons.lang3.RandomUtils; -import org.apache.hadoop.conf.StorageUnit; import org.apache.hadoop.hdds.client.ReplicationFactor; import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.conf.DatanodeRatisServerConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChecksumType; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.ratis.conf.RatisClientConfig; @@ -38,6 +38,7 @@ import org.apache.hadoop.hdds.scm.XceiverClientMetrics; import org.apache.hadoop.hdds.scm.storage.BufferPool; import org.apache.hadoop.hdds.scm.storage.RatisBlockOutputStream; +import org.apache.hadoop.ozone.ClientConfigBuilder; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneClient; @@ -105,14 +106,17 @@ static MiniOzoneCluster createCluster() throws IOException, ratisClientConfig.setWatchRequestTimeout(Duration.ofSeconds(30)); conf.setFromObject(ratisClientConfig); - MiniOzoneCluster cluster = MiniOzoneCluster.newBuilder(conf) - .setNumDatanodes(5) - .setTotalPipelineNumLimit(3) + ClientConfigBuilder.newBuilder(conf) .setBlockSize(BLOCK_SIZE) .setChunkSize(CHUNK_SIZE) .setStreamBufferFlushSize(FLUSH_SIZE) .setStreamBufferMaxSize(MAX_FLUSH_SIZE) .setStreamBufferSizeUnit(StorageUnit.BYTES) + .setOn(conf); + + MiniOzoneCluster cluster = MiniOzoneCluster.newBuilder(conf) + .setNumDatanodes(5) + .setTotalPipelineNumLimit(3) .build(); cluster.waitForClusterToBeReady(); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFlushDelay.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFlushDelay.java index 32fc9ba5c939..0daaaaf6bc3a 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFlushDelay.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFlushDelay.java @@ -17,16 +17,17 @@ package org.apache.hadoop.ozone.client.rpc; -import org.apache.hadoop.conf.StorageUnit; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationFactor; import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClientTestImpl; import org.apache.hadoop.hdds.utils.IOUtils; +import org.apache.hadoop.ozone.ClientConfigBuilder; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.client.ObjectStore; @@ -108,13 +109,17 @@ public void setup() throws Exception { conf.set(OzoneConfigKeys.OZONE_SCM_CLOSE_CONTAINER_WAIT_DURATION, "2s"); conf.set(ScmConfigKeys.OZONE_SCM_PIPELINE_SCRUB_INTERVAL, "2s"); conf.set(ScmConfigKeys.OZONE_SCM_PIPELINE_DESTROY_TIMEOUT, "5s"); + + ClientConfigBuilder.newBuilder(conf) + .setBlockSize(blockSize) + .setChunkSize(chunkSize) + .setStreamBufferFlushSize(flushSize) + .setStreamBufferMaxSize(maxFlushSize) + .setStreamBufferSizeUnit(StorageUnit.BYTES) + .setOn(conf); + cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(1) - .setBlockSize(blockSize) - .setChunkSize(chunkSize) - .setStreamBufferFlushSize(flushSize) - .setStreamBufferMaxSize(maxFlushSize) - .setStreamBufferSizeUnit(StorageUnit.BYTES) .setHbInterval(200) .setCertificateClient(new CertificateClientTestImpl(conf)) .setSecretKeyClient(new SecretKeyTestClient()) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineStream.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineStream.java index ab2fbeadb610..0dea62833432 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineStream.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineStream.java @@ -17,14 +17,15 @@ package org.apache.hadoop.ozone.client.rpc; -import org.apache.hadoop.conf.StorageUnit; import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.DatanodeRatisServerConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.ratis.conf.RatisClientConfig; import org.apache.hadoop.hdds.scm.OzoneClientConfig; import org.apache.hadoop.hdds.utils.IOUtils; +import org.apache.hadoop.ozone.ClientConfigBuilder; import org.apache.hadoop.ozone.HddsDatanodeService; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfigKeys; @@ -112,18 +113,21 @@ public void setup() throws Exception { raftClientConfig.setRpcWatchRequestTimeout(Duration.ofSeconds(10)); conf.setFromObject(raftClientConfig); + ClientConfigBuilder.newBuilder(conf) + .setDataStreamMinPacketSize(1024) + .setBlockSize(BLOCK_SIZE) + .setChunkSize(CHUNK_SIZE) + .setStreamBufferFlushSize(FLUSH_SIZE) + .setStreamBufferMaxSize(MAX_FLUSH_SIZE) + .setStreamBufferSizeUnit(StorageUnit.BYTES) + .setOn(conf); + conf.setLong(OzoneConfigKeys.DFS_RATIS_SNAPSHOT_THRESHOLD_KEY, 1); conf.setQuietMode(false); cluster = MiniOzoneCluster.newBuilder(conf) .setNumDatanodes(3) .setHbInterval(200) - .setDataStreamMinPacketSize(1024) - .setBlockSize(BLOCK_SIZE) - .setChunkSize(CHUNK_SIZE) - .setStreamBufferFlushSize(FLUSH_SIZE) - .setStreamBufferMaxSize(MAX_FLUSH_SIZE) - .setStreamBufferSizeUnit(StorageUnit.BYTES) .build(); cluster.waitForClusterToBeReady(); cluster.waitForPipelineTobeReady(HddsProtos.ReplicationFactor.ONE, 60000); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClientFlushDelay.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClientFlushDelay.java index 41438996c279..3b2b19e8d988 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClientFlushDelay.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClientFlushDelay.java @@ -17,13 +17,13 @@ package org.apache.hadoop.ozone.client.rpc; -import org.apache.hadoop.conf.StorageUnit; import org.apache.hadoop.hdds.HddsUtils; import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.DatanodeRatisServerConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.ratis.conf.RatisClientConfig; @@ -35,6 +35,7 @@ import org.apache.hadoop.net.DNSToSwitchMapping; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.net.StaticMapping; +import org.apache.hadoop.ozone.ClientConfigBuilder; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.client.ObjectStore; @@ -129,14 +130,19 @@ private void init() throws Exception { StaticMapping.addNodeToRack(NetUtils.normalizeHostNames( Collections.singleton(HddsUtils.getHostName(conf))).get(0), "/rack1"); - cluster = MiniOzoneCluster.newBuilder(conf) - .setNumDatanodes(10) - .setTotalPipelineNumLimit(15) - .setChunkSize(chunkSize) + + ClientConfigBuilder.newBuilder(conf) .setBlockSize(blockSize) + .setChunkSize(chunkSize) .setStreamBufferFlushSize(flushSize) .setStreamBufferMaxSize(maxFlushSize) - .setStreamBufferSizeUnit(StorageUnit.BYTES).build(); + .setStreamBufferSizeUnit(StorageUnit.BYTES) + .setOn(conf); + + cluster = MiniOzoneCluster.newBuilder(conf) + .setNumDatanodes(10) + .setTotalPipelineNumLimit(15) + .build(); cluster.waitForClusterToBeReady(); //the easiest way to create an open container is creating a key client = OzoneClientFactory.getRpcClient(conf); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneAtRestEncryption.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneAtRestEncryption.java index 08df9c941144..ac833aaeb06d 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneAtRestEncryption.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneAtRestEncryption.java @@ -34,7 +34,6 @@ import java.util.UUID; import com.google.common.cache.Cache; -import org.apache.hadoop.conf.StorageUnit; import org.apache.hadoop.crypto.key.KeyProvider; import org.apache.hadoop.crypto.key.kms.KMSClientProvider; import org.apache.hadoop.crypto.key.kms.server.MiniKMS; @@ -44,11 +43,13 @@ import org.apache.hadoop.hdds.client.ReplicationFactor; import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.container.ContainerInfo; import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB; import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClientTestImpl; import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.ozone.ClientConfigBuilder; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.client.BucketArgs; @@ -142,13 +143,17 @@ static void init() throws Exception { conf.set(OZONE_METADATA_DIRS, testDir.getAbsolutePath()); CertificateClientTestImpl certificateClientTest = new CertificateClientTestImpl(conf); + + ClientConfigBuilder.newBuilder(conf) + .setBlockSize(BLOCK_SIZE) + .setChunkSize(CHUNK_SIZE) + .setStreamBufferSizeUnit(StorageUnit.BYTES) + .setOn(conf); + cluster = MiniOzoneCluster.newBuilder(conf) .setNumDatanodes(10) .setScmId(SCM_ID) .setClusterId(CLUSTER_ID) - .setBlockSize(BLOCK_SIZE) - .setChunkSize(CHUNK_SIZE) - .setStreamBufferSizeUnit(StorageUnit.BYTES) .setCertificateClient(certificateClientTest) .setSecretKeyClient(new SecretKeyTestClient()) .build(); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptionFlushDelay.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptionFlushDelay.java index 925cfd9d9543..9f66a4c1bab0 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptionFlushDelay.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptionFlushDelay.java @@ -20,10 +20,10 @@ import java.io.OutputStream; import java.util.UUID; -import org.apache.hadoop.conf.StorageUnit; import org.apache.hadoop.hdds.client.ReplicationFactor; import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChecksumType; import org.apache.hadoop.hdds.scm.OzoneClientConfig; import org.apache.hadoop.hdds.scm.ScmConfigKeys; @@ -35,6 +35,7 @@ import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.storage.BlockOutputStream; import org.apache.hadoop.hdds.utils.IOUtils; +import org.apache.hadoop.ozone.ClientConfigBuilder; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.client.ObjectStore; @@ -98,14 +99,18 @@ public void init() throws Exception { conf.set(ScmConfigKeys.OZONE_SCM_PIPELINE_SCRUB_INTERVAL, "2s"); conf.set(ScmConfigKeys.OZONE_SCM_PIPELINE_DESTROY_TIMEOUT, "5s"); conf.setQuietMode(false); - cluster = MiniOzoneCluster.newBuilder(conf) - .setNumDatanodes(5) - .setTotalPipelineNumLimit(3) + + ClientConfigBuilder.newBuilder(conf) .setBlockSize(blockSize) .setChunkSize(chunkSize) .setStreamBufferFlushSize(flushSize) .setStreamBufferMaxSize(maxFlushSize) .setStreamBufferSizeUnit(StorageUnit.BYTES) + .setOn(conf); + + cluster = MiniOzoneCluster.newBuilder(conf) + .setNumDatanodes(5) + .setTotalPipelineNumLimit(3) .build(); cluster.waitForClusterToBeReady(); //the easiest way to create an open container is creating a key diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptions.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptions.java index 2c83cf7854d6..06c1ab918787 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptions.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptions.java @@ -22,10 +22,10 @@ import java.util.List; import java.util.UUID; -import org.apache.hadoop.conf.StorageUnit; import org.apache.hadoop.hdds.client.ReplicationFactor; import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChecksumType; import org.apache.hadoop.hdds.scm.OzoneClientConfig; import org.apache.hadoop.hdds.scm.ScmConfigKeys; @@ -38,6 +38,7 @@ import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.storage.BlockOutputStream; import org.apache.hadoop.hdds.utils.IOUtils; +import org.apache.hadoop.ozone.ClientConfigBuilder; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.client.ObjectStore; @@ -109,14 +110,17 @@ public void init() throws Exception { conf.set(ScmConfigKeys.OZONE_SCM_PIPELINE_DESTROY_TIMEOUT, "5s"); conf.setQuietMode(false); - cluster = MiniOzoneCluster.newBuilder(conf) - .setNumDatanodes(5) - .setTotalPipelineNumLimit(3) + ClientConfigBuilder.newBuilder(conf) .setBlockSize(blockSize) .setChunkSize(chunkSize) .setStreamBufferFlushSize(flushSize) .setStreamBufferMaxSize(maxFlushSize) .setStreamBufferSizeUnit(StorageUnit.BYTES) + .setOn(conf); + + cluster = MiniOzoneCluster.newBuilder(conf) + .setNumDatanodes(5) + .setTotalPipelineNumLimit(3) .build(); cluster.waitForClusterToBeReady(); //the easiest way to create an open container is creating a key diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java index c2f1eb069cfc..19c7ba5380f5 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java @@ -62,6 +62,7 @@ import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.ozone.ClientConfigBuilder; import org.apache.hadoop.ozone.HddsDatanodeService; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OmUtils; @@ -208,12 +209,16 @@ static void startCluster(OzoneConfiguration conf) throws Exception { // for testZReadKeyWithUnhealthyContainerReplica. clusterId = UUID.randomUUID().toString(); conf.set("ozone.scm.stale.node.interval", "10s"); + + ClientConfigBuilder.newBuilder(conf) + .setDataStreamMinPacketSize(1) // 1MB + .setOn(conf); + cluster = MiniOzoneCluster.newBuilder(conf) .setNumDatanodes(14) .setTotalPipelineNumLimit(10) .setScmId(scmId) .setClusterId(clusterId) - .setDataStreamMinPacketSize(1) // 1MB .build(); cluster.waitForClusterToBeReady(); ozClient = OzoneClientFactory.getRpcClient(conf); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestWatchForCommit.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestWatchForCommit.java index 69a103201561..0d2a40be585d 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestWatchForCommit.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestWatchForCommit.java @@ -28,10 +28,10 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import org.apache.hadoop.conf.StorageUnit; import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.DatanodeRatisServerConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.ratis.conf.RatisClientConfig; @@ -48,6 +48,7 @@ import org.apache.hadoop.hdds.scm.storage.BlockOutputStream; import org.apache.hadoop.hdds.scm.storage.RatisBlockOutputStream; import org.apache.hadoop.hdds.utils.IOUtils; +import org.apache.hadoop.ozone.ClientConfigBuilder; import org.apache.hadoop.ozone.HddsDatanodeService; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfigKeys; @@ -140,13 +141,17 @@ public void init() throws Exception { conf.setFromObject(raftClientConfig); conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 30, TimeUnit.SECONDS); - cluster = MiniOzoneCluster.newBuilder(conf) - .setNumDatanodes(9) + + ClientConfigBuilder.newBuilder(conf) .setBlockSize(blockSize) .setChunkSize(chunkSize) .setStreamBufferFlushSize(flushSize) .setStreamBufferMaxSize(maxFlushSize) .setStreamBufferSizeUnit(StorageUnit.BYTES) + .setOn(conf); + + cluster = MiniOzoneCluster.newBuilder(conf) + .setNumDatanodes(9) .build(); cluster.waitForClusterToBeReady(); cluster.waitForPipelineTobeReady(HddsProtos.ReplicationFactor.THREE, 60000); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestInputStreamBase.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestInputStreamBase.java index 22ad4f036cfd..47e066a20752 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestInputStreamBase.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestInputStreamBase.java @@ -20,11 +20,12 @@ import java.util.UUID; import java.util.concurrent.TimeUnit; -import org.apache.hadoop.conf.StorageUnit; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.scm.OzoneClientConfig; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.container.replication.ReplicationManager.ReplicationManagerConfiguration; +import org.apache.hadoop.ozone.ClientConfigBuilder; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.container.common.impl.ContainerLayoutVersion; @@ -63,14 +64,17 @@ protected static MiniOzoneCluster newCluster( repConf.setInterval(Duration.ofSeconds(1)); conf.setFromObject(repConf); - return MiniOzoneCluster.newBuilder(conf) - .setNumDatanodes(5) - .setTotalPipelineNumLimit(5) + ClientConfigBuilder.newBuilder(conf) .setBlockSize(BLOCK_SIZE) .setChunkSize(CHUNK_SIZE) .setStreamBufferFlushSize(FLUSH_SIZE) .setStreamBufferMaxSize(MAX_FLUSH_SIZE) .setStreamBufferSizeUnit(StorageUnit.BYTES) + .setOn(conf); + + return MiniOzoneCluster.newBuilder(conf) + .setNumDatanodes(5) + .setTotalPipelineNumLimit(5) .build(); } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestECContainerRecovery.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestECContainerRecovery.java index 5a7e404f507b..a5223707b6fb 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestECContainerRecovery.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestECContainerRecovery.java @@ -16,11 +16,11 @@ */ package org.apache.hadoop.ozone.container; -import org.apache.hadoop.conf.StorageUnit; import org.apache.hadoop.hdds.HddsConfigKeys; import org.apache.hadoop.hdds.client.DefaultReplicationConfig; import org.apache.hadoop.hdds.client.ECReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos; @@ -34,6 +34,7 @@ import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; import org.apache.hadoop.hdds.utils.IOUtils; +import org.apache.hadoop.ozone.ClientConfigBuilder; import org.apache.hadoop.ozone.HddsDatanodeService; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfigKeys; @@ -139,11 +140,18 @@ public static void init() throws Exception { TimeUnit.MILLISECONDS); conf.setTimeDuration(HddsConfigKeys.HDDS_CONTAINER_REPORT_INTERVAL, 1, TimeUnit.SECONDS); - cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(10) - .setTotalPipelineNumLimit(10).setBlockSize(blockSize) - .setChunkSize(chunkSize).setStreamBufferFlushSize(flushSize) + + ClientConfigBuilder.newBuilder(conf) + .setBlockSize(blockSize) + .setChunkSize(chunkSize) + .setStreamBufferFlushSize(flushSize) .setStreamBufferMaxSize(maxFlushSize) - .setStreamBufferSizeUnit(StorageUnit.BYTES).build(); + .setStreamBufferSizeUnit(StorageUnit.BYTES) + .setOn(conf); + + cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(10) + .setTotalPipelineNumLimit(10) + .build(); cluster.waitForClusterToBeReady(); client = OzoneClientFactory.getRpcClient(conf); objectStore = client.getObjectStore(); diff --git a/hadoop-ozone/integration-test/src/test/resources/ozone-site.xml b/hadoop-ozone/integration-test/src/test/resources/ozone-site.xml index 4c415c0c7e0d..211d7529216b 100644 --- a/hadoop-ozone/integration-test/src/test/resources/ozone-site.xml +++ b/hadoop-ozone/integration-test/src/test/resources/ozone-site.xml @@ -63,4 +63,38 @@ ozone.scm.ha.ratis.log.appender.queue.byte-limit 4MB + + + ozone.scm.chunk.size + 1MB + + + ozone.scm.block.size + 4MB + + + ozone.client.stream.buffer.flush.size + 1MB + + + ozone.client.stream.buffer.max.size + 2MB + + + ozone.client.stream.buffer.size + 1MB + + + ozone.client.datastream.buffer.flush.size + 4MB + + + ozone.client.datastream.min.packet.size + 256KB + + + ozone.client.datastream.window.size + 8MB + + From b62ac1ad5a1ae88e2b8afcd631cf236ae84bb35d Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Sun, 4 Feb 2024 22:33:41 +0100 Subject: [PATCH 2/7] Make unit mandatory --- .../org/apache/hadoop/fs/ozone/TestHSync.java | 3 +- .../hadoop/fs/ozone/TestLeaseRecovery.java | 3 +- .../fs/ozone/TestOzoneFSInputStream.java | 11 +-- .../fs/ozone/TestOzoneFileSystemMetrics.java | 11 +-- .../TestOzoneFileSystemWithStreaming.java | 3 +- .../hdds/scm/storage/TestCommitWatcher.java | 3 +- .../hadoop/ozone/ClientConfigBuilder.java | 63 ++++++--------- .../apache/hadoop/ozone/MiniOzoneCluster.java | 76 ------------------- .../hadoop/ozone/MiniOzoneClusterImpl.java | 54 ------------- .../rpc/AbstractTestECKeyOutputStream.java | 3 +- .../client/rpc/Test2WayCommitInRatis.java | 3 +- .../client/rpc/TestBlockDataStreamOutput.java | 3 +- .../client/rpc/TestBlockOutputStream.java | 3 +- .../TestContainerStateMachineFlushDelay.java | 3 +- .../rpc/TestContainerStateMachineStream.java | 3 +- ...TestFailureHandlingByClientFlushDelay.java | 3 +- .../client/rpc/TestOzoneAtRestEncryption.java | 3 +- ...oneClientRetriesOnExceptionFlushDelay.java | 3 +- .../TestOzoneClientRetriesOnExceptions.java | 3 +- .../rpc/TestOzoneRpcClientAbstract.java | 6 +- .../ozone/client/rpc/TestWatchForCommit.java | 3 +- .../client/rpc/read/TestInputStreamBase.java | 3 +- .../container/TestECContainerRecovery.java | 3 +- 23 files changed, 56 insertions(+), 216 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSync.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSync.java index ae3831c44a35..d742611b3f2b 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSync.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSync.java @@ -117,13 +117,12 @@ public static void init() throws Exception { CONF.setBoolean(OZONE_OM_RATIS_ENABLE_KEY, false); CONF.set(OZONE_DEFAULT_BUCKET_LAYOUT, layout.name()); CONF.setBoolean(OzoneConfigKeys.OZONE_FS_HSYNC_ENABLED, true); - ClientConfigBuilder.newBuilder(CONF) + ClientConfigBuilder.newBuilder(CONF, StorageUnit.BYTES) .setBlockSize(blockSize) .setChunkSize(chunkSize) .setStreamBufferFlushSize(flushSize) .setStreamBufferMaxSize(maxFlushSize) .setDataStreamBufferFlushize(maxFlushSize) - .setStreamBufferSizeUnit(StorageUnit.BYTES) .setDataStreamMinPacketSize(chunkSize) .setDataStreamStreamWindowSize(5 * chunkSize) .setOn(CONF); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestLeaseRecovery.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestLeaseRecovery.java index 5a5ec4008a1b..6e216477543d 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestLeaseRecovery.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestLeaseRecovery.java @@ -92,13 +92,12 @@ public void init() throws IOException, InterruptedException, conf.setBoolean(OZONE_OM_RATIS_ENABLE_KEY, false); conf.setBoolean(OzoneConfigKeys.OZONE_FS_HSYNC_ENABLED, true); conf.set(OZONE_DEFAULT_BUCKET_LAYOUT, layout.name()); - ClientConfigBuilder.newBuilder(conf) + ClientConfigBuilder.newBuilder(conf, StorageUnit.BYTES) .setBlockSize(blockSize) .setChunkSize(chunkSize) .setStreamBufferFlushSize(flushSize) .setStreamBufferMaxSize(maxFlushSize) .setDataStreamBufferFlushize(maxFlushSize) - .setStreamBufferSizeUnit(StorageUnit.BYTES) .setDataStreamMinPacketSize(chunkSize) .setDataStreamStreamWindowSize(5 * chunkSize) .setOn(conf); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java index 4051e3aa08e1..1e84c0c11781 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java @@ -35,6 +35,7 @@ import org.apache.hadoop.hdds.client.DefaultReplicationConfig; import org.apache.hadoop.hdds.client.ECReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.protocol.StorageType; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.SequenceFile; @@ -85,11 +86,11 @@ public static void init() throws Exception { conf.set(OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT, BucketLayout.LEGACY.name()); - ClientConfigBuilder.newBuilder(conf) - .setChunkSize(2) // MB - .setBlockSize(8) // MB - .setStreamBufferFlushSize(2) // MB - .setStreamBufferMaxSize(4) // MB + ClientConfigBuilder.newBuilder(conf, StorageUnit.MB) + .setChunkSize(2) + .setBlockSize(8) + .setStreamBufferFlushSize(2) + .setStreamBufferMaxSize(4) .setOn(conf); cluster = MiniOzoneCluster.newBuilder(conf) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemMetrics.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemMetrics.java index 4e4f4265dc20..72c29cfd6e56 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemMetrics.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemMetrics.java @@ -18,6 +18,7 @@ package org.apache.hadoop.fs.ozone; +import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.commons.lang3.RandomStringUtils; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; @@ -74,11 +75,11 @@ public static void init() throws Exception { BucketLayout.LEGACY.name()); conf.setBoolean(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS, true); - ClientConfigBuilder.newBuilder(conf) - .setChunkSize(2) // MB - .setBlockSize(8) // MB - .setStreamBufferFlushSize(2) // MB - .setStreamBufferMaxSize(4) // MB + ClientConfigBuilder.newBuilder(conf, StorageUnit.MB) + .setChunkSize(2) + .setBlockSize(8) + .setStreamBufferFlushSize(2) + .setStreamBufferMaxSize(4) .setOn(conf); cluster = MiniOzoneCluster.newBuilder(conf) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemWithStreaming.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemWithStreaming.java index d0ee48c2dffd..41e053ea6100 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemWithStreaming.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemWithStreaming.java @@ -88,13 +88,12 @@ public static void init() throws Exception { CONF.setBoolean(OZONE_OM_RATIS_ENABLE_KEY, true); CONF.set(OZONE_DEFAULT_BUCKET_LAYOUT, layout.name()); - ClientConfigBuilder.newBuilder(CONF) + ClientConfigBuilder.newBuilder(CONF, StorageUnit.BYTES) .setBlockSize(blockSize) .setChunkSize(chunkSize) .setStreamBufferFlushSize(flushSize) .setStreamBufferMaxSize(maxFlushSize) .setDataStreamBufferFlushize(maxFlushSize) - .setStreamBufferSizeUnit(StorageUnit.BYTES) .setDataStreamMinPacketSize(chunkSize) .setDataStreamStreamWindowSize(5 * chunkSize) .setOn(CONF); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestCommitWatcher.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestCommitWatcher.java index a4f2858eaee0..5c8fd3a9457c 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestCommitWatcher.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestCommitWatcher.java @@ -132,12 +132,11 @@ public void init() throws Exception { clientConfig.setChecksumType(ChecksumType.NONE); conf.setFromObject(clientConfig); - ClientConfigBuilder.newBuilder(conf) + ClientConfigBuilder.newBuilder(conf, StorageUnit.BYTES) .setBlockSize(blockSize) .setChunkSize(chunkSize) .setStreamBufferFlushSize(flushSize) .setStreamBufferMaxSize(maxFlushSize) - .setStreamBufferSizeUnit(StorageUnit.BYTES) .setOn(conf); conf.setQuietMode(false); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ClientConfigBuilder.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ClientConfigBuilder.java index 81a523d066d7..0c9f280e7908 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ClientConfigBuilder.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ClientConfigBuilder.java @@ -19,11 +19,9 @@ import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.conf.ConfigurationTarget; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.scm.OzoneClientConfig; -import java.util.Objects; import java.util.OptionalInt; import java.util.OptionalLong; @@ -33,11 +31,11 @@ /** * Helper for tests that want to set client stream properties. */ -public class ClientConfigBuilder { +public final class ClientConfigBuilder { private final ConfigurationSource conf; - private int chunkSize = 1; + private int chunkSize = 1024 * 1024; private OptionalLong blockSize = OptionalLong.empty(); private OptionalInt streamBufferSize = OptionalInt.empty(); private OptionalLong streamBufferFlushSize = OptionalLong.empty(); @@ -45,70 +43,57 @@ public class ClientConfigBuilder { private OptionalLong dataStreamWindowSize = OptionalLong.empty(); private OptionalLong streamBufferMaxSize = OptionalLong.empty(); private OptionalInt dataStreamMinPacketSize = OptionalInt.empty(); - private StorageUnit unit = StorageUnit.MB; + private final StorageUnit unit; - public static ClientConfigBuilder newBuilder() { - return new ClientConfigBuilder(); + public static ClientConfigBuilder newBuilder(ConfigurationSource conf, StorageUnit unit) { + return new ClientConfigBuilder(conf, unit); } - public static ClientConfigBuilder newBuilder(ConfigurationSource conf) { - return new ClientConfigBuilder(conf); - } - - private ClientConfigBuilder() { - this(new OzoneConfiguration()); - } - - private ClientConfigBuilder(ConfigurationSource conf) { + private ClientConfigBuilder(ConfigurationSource conf, StorageUnit unit) { this.conf = conf; + this.unit = unit; } + // FIXME store everything in bytes public ClientConfigBuilder setChunkSize(int size) { - chunkSize = size; + chunkSize = (int) toBytes(size); return this; } public ClientConfigBuilder setBlockSize(long size) { - blockSize = OptionalLong.of(size); + blockSize = OptionalLong.of(toBytes(size)); return this; } public ClientConfigBuilder setStreamBufferSize(int size) { - streamBufferSize = OptionalInt.of(size); + streamBufferSize = OptionalInt.of((int) toBytes(size)); return this; } public ClientConfigBuilder setStreamBufferFlushSize(long size) { - streamBufferFlushSize = OptionalLong.of(size); + streamBufferFlushSize = OptionalLong.of(toBytes(size)); return this; } public ClientConfigBuilder setStreamBufferMaxSize(long size) { - streamBufferMaxSize = OptionalLong.of(size); + streamBufferMaxSize = OptionalLong.of(toBytes(size)); return this; } public ClientConfigBuilder setDataStreamMinPacketSize(int size) { - dataStreamMinPacketSize = OptionalInt.of(size); + dataStreamMinPacketSize = OptionalInt.of((int) toBytes(size)); return this; } // TODO fix typo public ClientConfigBuilder setDataStreamBufferFlushize(long size) { - dataStreamBufferFlushSize = OptionalLong.of(size); + dataStreamBufferFlushSize = OptionalLong.of(toBytes(size)); return this; } // TODO fix typo in name public ClientConfigBuilder setDataStreamStreamWindowSize(long size) { - dataStreamWindowSize = OptionalLong.of(size); - return this; - } - - // TODO rename - // TODO merge with setChunkSize - public ClientConfigBuilder setStreamBufferSizeUnit(StorageUnit unit) { - this.unit = Objects.requireNonNull(unit); + dataStreamWindowSize = OptionalLong.of(toBytes(size)); return this; } @@ -136,16 +121,16 @@ public void setOn(ConfigurationTarget target) { } OzoneClientConfig clientConfig = conf.getObject(OzoneClientConfig.class); - clientConfig.setStreamBufferSize((int) toBytes(streamBufferSize.getAsInt())); - clientConfig.setStreamBufferMaxSize(toBytes(streamBufferMaxSize.getAsLong())); - clientConfig.setStreamBufferFlushSize(toBytes(streamBufferFlushSize.getAsLong())); - clientConfig.setDataStreamBufferFlushSize(toBytes(dataStreamBufferFlushSize.getAsLong())); - clientConfig.setDataStreamMinPacketSize((int) toBytes(dataStreamMinPacketSize.getAsInt())); - clientConfig.setStreamWindowSize(toBytes(dataStreamWindowSize.getAsLong())); + clientConfig.setStreamBufferSize(streamBufferSize.getAsInt()); + clientConfig.setStreamBufferMaxSize(streamBufferMaxSize.getAsLong()); + clientConfig.setStreamBufferFlushSize(streamBufferFlushSize.getAsLong()); + clientConfig.setDataStreamBufferFlushSize(dataStreamBufferFlushSize.getAsLong()); + clientConfig.setDataStreamMinPacketSize(dataStreamMinPacketSize.getAsInt()); + clientConfig.setStreamWindowSize(dataStreamWindowSize.getAsLong()); target.setFromObject(clientConfig); - target.setStorageSize(OZONE_SCM_CHUNK_SIZE_KEY, chunkSize, unit); - target.setStorageSize(OZONE_SCM_BLOCK_SIZE, blockSize.getAsLong(), unit); + target.setStorageSize(OZONE_SCM_CHUNK_SIZE_KEY, chunkSize, StorageUnit.BYTES); + target.setStorageSize(OZONE_SCM_BLOCK_SIZE, blockSize.getAsLong(), StorageUnit.BYTES); } private long toBytes(long value) { diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java index 458a0ca891fd..5dd9db3881ff 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java @@ -20,11 +20,9 @@ import java.io.IOException; import java.util.List; import java.util.Optional; -import java.util.OptionalInt; import java.util.UUID; import java.util.concurrent.TimeoutException; -import org.apache.hadoop.conf.StorageUnit; import org.apache.hadoop.hdds.HddsConfigKeys; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; @@ -327,15 +325,6 @@ abstract class Builder { protected Boolean enableContainerDatastream = true; protected Optional datanodeReservedSpace = Optional.empty(); - protected Optional chunkSize = Optional.empty(); - protected OptionalInt streamBufferSize = OptionalInt.empty(); - protected Optional streamBufferFlushSize = Optional.empty(); - protected Optional dataStreamBufferFlushSize = Optional.empty(); - protected Optional datastreamWindowSize = Optional.empty(); - protected Optional streamBufferMaxSize = Optional.empty(); - protected OptionalInt dataStreamMinPacketSize = OptionalInt.empty(); - protected Optional blockSize = Optional.empty(); - protected Optional streamBufferSizeUnit = Optional.empty(); protected boolean includeRecon = false; @@ -533,66 +522,6 @@ public Builder setDatanodeReservedSpace(String reservedSpace) { return this; } - /** - * Sets the chunk size. - * - * @return MiniOzoneCluster.Builder - */ - public Builder setChunkSize(int size) { - chunkSize = Optional.of(size); - return this; - } - - public Builder setStreamBufferSize(int size) { - streamBufferSize = OptionalInt.of(size); - return this; - } - - /** - * Sets the flush size for stream buffer. - * - * @return MiniOzoneCluster.Builder - */ - public Builder setStreamBufferFlushSize(long size) { - streamBufferFlushSize = Optional.of(size); - return this; - } - - /** - * Sets the max size for stream buffer. - * - * @return MiniOzoneCluster.Builder - */ - public Builder setStreamBufferMaxSize(long size) { - streamBufferMaxSize = Optional.of(size); - return this; - } - - public Builder setDataStreamBufferFlushize(long size) { - dataStreamBufferFlushSize = Optional.of(size); - return this; - } - - public Builder setDataStreamMinPacketSize(int size) { - dataStreamMinPacketSize = OptionalInt.of(size); - return this; - } - - public Builder setDataStreamStreamWindowSize(long size) { - datastreamWindowSize = Optional.of(size); - return this; - } - - /** - * Sets the block size for stream buffer. - * - * @return MiniOzoneCluster.Builder - */ - public Builder setBlockSize(long size) { - blockSize = Optional.of(size); - return this; - } - public Builder setNumOfOzoneManagers(int numOMs) { this.numOfOMs = numOMs; return this; @@ -603,11 +532,6 @@ public Builder setNumOfActiveOMs(int numActiveOMs) { return this; } - public Builder setStreamBufferSizeUnit(StorageUnit unit) { - this.streamBufferSizeUnit = Optional.of(unit); - return this; - } - public Builder setOMServiceId(String serviceId) { this.omServiceId = serviceId; return this; diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java index 24a3ff84f1cf..7c32154d95cf 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java @@ -27,7 +27,6 @@ import java.util.Collections; import java.util.List; import java.util.Optional; -import java.util.OptionalInt; import java.util.Set; import java.util.UUID; import java.util.Iterator; @@ -36,7 +35,6 @@ import java.util.concurrent.TimeoutException; import org.apache.commons.lang3.StringUtils; -import org.apache.hadoop.conf.StorageUnit; import org.apache.hadoop.hdds.DFSConfigKeysLegacy; import org.apache.hadoop.hdds.HddsConfigKeys; import org.apache.hadoop.hdds.annotation.InterfaceAudience; @@ -45,7 +43,6 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.hdds.scm.OzoneClientConfig; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.HddsTestUtils; import org.apache.hadoop.hdds.scm.ha.SCMHANodeDetails; @@ -656,58 +653,7 @@ protected void initializeConfiguration() throws IOException { Files.createDirectories(metaDir); conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, metaDir.toString()); // conf.setBoolean(ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY, true); - if (!chunkSize.isPresent()) { - //set it to 1MB by default in tests - chunkSize = Optional.of(1); - } - if (!streamBufferSize.isPresent()) { - streamBufferSize = OptionalInt.of(chunkSize.get()); - } - if (!streamBufferFlushSize.isPresent()) { - streamBufferFlushSize = Optional.of((long) chunkSize.get()); - } - if (!streamBufferMaxSize.isPresent()) { - streamBufferMaxSize = Optional.of(2 * streamBufferFlushSize.get()); - } - if (!dataStreamBufferFlushSize.isPresent()) { - dataStreamBufferFlushSize = Optional.of((long) 4 * chunkSize.get()); - } - if (!dataStreamMinPacketSize.isPresent()) { - dataStreamMinPacketSize = OptionalInt.of(chunkSize.get() / 4); - } - if (!datastreamWindowSize.isPresent()) { - datastreamWindowSize = Optional.of((long) 8 * chunkSize.get()); - } - if (!blockSize.isPresent()) { - blockSize = Optional.of(2 * streamBufferMaxSize.get()); - } - - if (!streamBufferSizeUnit.isPresent()) { - streamBufferSizeUnit = Optional.of(StorageUnit.MB); - } - OzoneClientConfig clientConfig = conf.getObject(OzoneClientConfig.class); - clientConfig.setStreamBufferSize( - (int) Math.round( - streamBufferSizeUnit.get().toBytes(streamBufferSize.getAsInt()))); - clientConfig.setStreamBufferMaxSize(Math.round( - streamBufferSizeUnit.get().toBytes(streamBufferMaxSize.get()))); - clientConfig.setStreamBufferFlushSize(Math.round( - streamBufferSizeUnit.get().toBytes(streamBufferFlushSize.get()))); - clientConfig.setDataStreamBufferFlushSize(Math.round( - streamBufferSizeUnit.get().toBytes(dataStreamBufferFlushSize.get()))); - clientConfig.setDataStreamMinPacketSize((int) Math.round( - streamBufferSizeUnit.get() - .toBytes(dataStreamMinPacketSize.getAsInt()))); - clientConfig.setStreamWindowSize(Math.round( - streamBufferSizeUnit.get().toBytes(datastreamWindowSize.get()))); - conf.setFromObject(clientConfig); - - conf.setStorageSize(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY, - chunkSize.get(), streamBufferSizeUnit.get()); - - conf.setStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE, blockSize.get(), - streamBufferSizeUnit.get()); // MiniOzoneCluster should have global pipeline upper limit. conf.setInt(ScmConfigKeys.OZONE_SCM_RATIS_PIPELINE_LIMIT, pipelineNumLimit >= DEFAULT_PIPELINE_LIMIT ? diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/AbstractTestECKeyOutputStream.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/AbstractTestECKeyOutputStream.java index c7c01a5dfb27..a661002f1973 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/AbstractTestECKeyOutputStream.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/AbstractTestECKeyOutputStream.java @@ -122,12 +122,11 @@ protected static void init(boolean zeroCopyEnabled) throws Exception { conf.setBoolean(OzoneConfigKeys.OZONE_EC_GRPC_ZERO_COPY_ENABLED, zeroCopyEnabled); - ClientConfigBuilder.newBuilder(conf) + ClientConfigBuilder.newBuilder(conf, StorageUnit.BYTES) .setBlockSize(blockSize) .setChunkSize(chunkSize) .setStreamBufferFlushSize(flushSize) .setStreamBufferMaxSize(maxFlushSize) - .setStreamBufferSizeUnit(StorageUnit.BYTES) .setOn(conf); cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(10) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/Test2WayCommitInRatis.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/Test2WayCommitInRatis.java index 37be406677b1..1410f3a5eb91 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/Test2WayCommitInRatis.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/Test2WayCommitInRatis.java @@ -101,12 +101,11 @@ private void startCluster(OzoneConfiguration conf) throws Exception { raftClientConfig.setRpcWatchRequestTimeout(Duration.ofSeconds(10)); conf.setFromObject(raftClientConfig); - ClientConfigBuilder.newBuilder(conf) + ClientConfigBuilder.newBuilder(conf, StorageUnit.BYTES) .setBlockSize(blockSize) .setChunkSize(chunkSize) .setStreamBufferFlushSize(flushSize) .setStreamBufferMaxSize(maxFlushSize) - .setStreamBufferSizeUnit(StorageUnit.BYTES) .setOn(conf); conf.setQuietMode(false); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java index 0190c203246a..dfb0cb8a6da2 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java @@ -93,13 +93,12 @@ public static void init() throws Exception { conf.setStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE, 4, StorageUnit.MB); - ClientConfigBuilder.newBuilder(conf) + ClientConfigBuilder.newBuilder(conf, StorageUnit.BYTES) .setBlockSize(blockSize) .setChunkSize(chunkSize) .setStreamBufferFlushSize(flushSize) .setStreamBufferMaxSize(maxFlushSize) .setDataStreamBufferFlushize(maxFlushSize) - .setStreamBufferSizeUnit(StorageUnit.BYTES) .setDataStreamMinPacketSize(chunkSize) .setDataStreamStreamWindowSize(5 * chunkSize) .setOn(conf); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockOutputStream.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockOutputStream.java index 8e6e5d932af3..1df925b5ba41 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockOutputStream.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockOutputStream.java @@ -106,12 +106,11 @@ static MiniOzoneCluster createCluster() throws IOException, ratisClientConfig.setWatchRequestTimeout(Duration.ofSeconds(30)); conf.setFromObject(ratisClientConfig); - ClientConfigBuilder.newBuilder(conf) + ClientConfigBuilder.newBuilder(conf, StorageUnit.BYTES) .setBlockSize(BLOCK_SIZE) .setChunkSize(CHUNK_SIZE) .setStreamBufferFlushSize(FLUSH_SIZE) .setStreamBufferMaxSize(MAX_FLUSH_SIZE) - .setStreamBufferSizeUnit(StorageUnit.BYTES) .setOn(conf); MiniOzoneCluster cluster = MiniOzoneCluster.newBuilder(conf) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFlushDelay.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFlushDelay.java index 0daaaaf6bc3a..38bc89f0d7dc 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFlushDelay.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFlushDelay.java @@ -110,12 +110,11 @@ public void setup() throws Exception { conf.set(ScmConfigKeys.OZONE_SCM_PIPELINE_SCRUB_INTERVAL, "2s"); conf.set(ScmConfigKeys.OZONE_SCM_PIPELINE_DESTROY_TIMEOUT, "5s"); - ClientConfigBuilder.newBuilder(conf) + ClientConfigBuilder.newBuilder(conf, StorageUnit.BYTES) .setBlockSize(blockSize) .setChunkSize(chunkSize) .setStreamBufferFlushSize(flushSize) .setStreamBufferMaxSize(maxFlushSize) - .setStreamBufferSizeUnit(StorageUnit.BYTES) .setOn(conf); cluster = diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineStream.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineStream.java index 0dea62833432..1d345f9a18a4 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineStream.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineStream.java @@ -113,13 +113,12 @@ public void setup() throws Exception { raftClientConfig.setRpcWatchRequestTimeout(Duration.ofSeconds(10)); conf.setFromObject(raftClientConfig); - ClientConfigBuilder.newBuilder(conf) + ClientConfigBuilder.newBuilder(conf, StorageUnit.BYTES) .setDataStreamMinPacketSize(1024) .setBlockSize(BLOCK_SIZE) .setChunkSize(CHUNK_SIZE) .setStreamBufferFlushSize(FLUSH_SIZE) .setStreamBufferMaxSize(MAX_FLUSH_SIZE) - .setStreamBufferSizeUnit(StorageUnit.BYTES) .setOn(conf); conf.setLong(OzoneConfigKeys.DFS_RATIS_SNAPSHOT_THRESHOLD_KEY, 1); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClientFlushDelay.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClientFlushDelay.java index 3b2b19e8d988..1a9d26758d98 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClientFlushDelay.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClientFlushDelay.java @@ -131,12 +131,11 @@ private void init() throws Exception { Collections.singleton(HddsUtils.getHostName(conf))).get(0), "/rack1"); - ClientConfigBuilder.newBuilder(conf) + ClientConfigBuilder.newBuilder(conf, StorageUnit.BYTES) .setBlockSize(blockSize) .setChunkSize(chunkSize) .setStreamBufferFlushSize(flushSize) .setStreamBufferMaxSize(maxFlushSize) - .setStreamBufferSizeUnit(StorageUnit.BYTES) .setOn(conf); cluster = MiniOzoneCluster.newBuilder(conf) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneAtRestEncryption.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneAtRestEncryption.java index ac833aaeb06d..9a00e48507ea 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneAtRestEncryption.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneAtRestEncryption.java @@ -144,10 +144,9 @@ static void init() throws Exception { CertificateClientTestImpl certificateClientTest = new CertificateClientTestImpl(conf); - ClientConfigBuilder.newBuilder(conf) + ClientConfigBuilder.newBuilder(conf, StorageUnit.BYTES) .setBlockSize(BLOCK_SIZE) .setChunkSize(CHUNK_SIZE) - .setStreamBufferSizeUnit(StorageUnit.BYTES) .setOn(conf); cluster = MiniOzoneCluster.newBuilder(conf) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptionFlushDelay.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptionFlushDelay.java index 9f66a4c1bab0..1f3b71385dff 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptionFlushDelay.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptionFlushDelay.java @@ -100,12 +100,11 @@ public void init() throws Exception { conf.set(ScmConfigKeys.OZONE_SCM_PIPELINE_DESTROY_TIMEOUT, "5s"); conf.setQuietMode(false); - ClientConfigBuilder.newBuilder(conf) + ClientConfigBuilder.newBuilder(conf, StorageUnit.BYTES) .setBlockSize(blockSize) .setChunkSize(chunkSize) .setStreamBufferFlushSize(flushSize) .setStreamBufferMaxSize(maxFlushSize) - .setStreamBufferSizeUnit(StorageUnit.BYTES) .setOn(conf); cluster = MiniOzoneCluster.newBuilder(conf) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptions.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptions.java index 06c1ab918787..eb2b4a1dc0e3 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptions.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptions.java @@ -110,12 +110,11 @@ public void init() throws Exception { conf.set(ScmConfigKeys.OZONE_SCM_PIPELINE_DESTROY_TIMEOUT, "5s"); conf.setQuietMode(false); - ClientConfigBuilder.newBuilder(conf) + ClientConfigBuilder.newBuilder(conf, StorageUnit.BYTES) .setBlockSize(blockSize) .setChunkSize(chunkSize) .setStreamBufferFlushSize(flushSize) .setStreamBufferMaxSize(maxFlushSize) - .setStreamBufferSizeUnit(StorageUnit.BYTES) .setOn(conf); cluster = MiniOzoneCluster.newBuilder(conf) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java index 19c7ba5380f5..a5c9d0cf341c 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java @@ -38,7 +38,6 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Stream; -import org.apache.hadoop.conf.StorageUnit; import org.apache.hadoop.hdds.client.DefaultReplicationConfig; import org.apache.hadoop.hdds.client.ECReplicationConfig; import org.apache.hadoop.hdds.client.ECReplicationConfig.EcCodec; @@ -49,6 +48,7 @@ import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.client.StandaloneReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.StorageType; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; @@ -210,8 +210,8 @@ static void startCluster(OzoneConfiguration conf) throws Exception { clusterId = UUID.randomUUID().toString(); conf.set("ozone.scm.stale.node.interval", "10s"); - ClientConfigBuilder.newBuilder(conf) - .setDataStreamMinPacketSize(1) // 1MB + ClientConfigBuilder.newBuilder(conf, StorageUnit.MB) + .setDataStreamMinPacketSize(1) .setOn(conf); cluster = MiniOzoneCluster.newBuilder(conf) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestWatchForCommit.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestWatchForCommit.java index 0d2a40be585d..88fcdf9cab41 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestWatchForCommit.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestWatchForCommit.java @@ -142,12 +142,11 @@ public void init() throws Exception { conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 30, TimeUnit.SECONDS); - ClientConfigBuilder.newBuilder(conf) + ClientConfigBuilder.newBuilder(conf, StorageUnit.BYTES) .setBlockSize(blockSize) .setChunkSize(chunkSize) .setStreamBufferFlushSize(flushSize) .setStreamBufferMaxSize(maxFlushSize) - .setStreamBufferSizeUnit(StorageUnit.BYTES) .setOn(conf); cluster = MiniOzoneCluster.newBuilder(conf) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestInputStreamBase.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestInputStreamBase.java index 47e066a20752..243694138818 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestInputStreamBase.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestInputStreamBase.java @@ -64,12 +64,11 @@ protected static MiniOzoneCluster newCluster( repConf.setInterval(Duration.ofSeconds(1)); conf.setFromObject(repConf); - ClientConfigBuilder.newBuilder(conf) + ClientConfigBuilder.newBuilder(conf, StorageUnit.BYTES) .setBlockSize(BLOCK_SIZE) .setChunkSize(CHUNK_SIZE) .setStreamBufferFlushSize(FLUSH_SIZE) .setStreamBufferMaxSize(MAX_FLUSH_SIZE) - .setStreamBufferSizeUnit(StorageUnit.BYTES) .setOn(conf); return MiniOzoneCluster.newBuilder(conf) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestECContainerRecovery.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestECContainerRecovery.java index a5223707b6fb..e7c46f92aafe 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestECContainerRecovery.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestECContainerRecovery.java @@ -141,12 +141,11 @@ public static void init() throws Exception { conf.setTimeDuration(HddsConfigKeys.HDDS_CONTAINER_REPORT_INTERVAL, 1, TimeUnit.SECONDS); - ClientConfigBuilder.newBuilder(conf) + ClientConfigBuilder.newBuilder(conf, StorageUnit.BYTES) .setBlockSize(blockSize) .setChunkSize(chunkSize) .setStreamBufferFlushSize(flushSize) .setStreamBufferMaxSize(maxFlushSize) - .setStreamBufferSizeUnit(StorageUnit.BYTES) .setOn(conf); cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(10) From 445db63be251e12a8702c644e8301947d0f34939 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Mon, 5 Feb 2024 02:38:34 +0100 Subject: [PATCH 3/7] apply fix from HDDS-10294 to integration tests --- .../java/org/apache/hadoop/ozone/MiniOzoneChaosCluster.java | 2 +- .../org/apache/hadoop/hdds/scm/storage/TestCommitWatcher.java | 2 +- .../ozone/client/rpc/TestCloseContainerHandlingByClient.java | 2 +- .../hadoop/ozone/client/rpc/TestDiscardPreallocatedBlocks.java | 2 +- .../client/rpc/TestOzoneClientRetriesOnExceptionFlushDelay.java | 2 +- .../hadoop/ozone/client/rpc/read/TestInputStreamBase.java | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/MiniOzoneChaosCluster.java b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/MiniOzoneChaosCluster.java index 1ffed5323aab..1ff1752f4135 100644 --- a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/MiniOzoneChaosCluster.java +++ b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/MiniOzoneChaosCluster.java @@ -232,7 +232,7 @@ public Builder addFailures(Class clazz) { protected void initializeConfiguration() throws IOException { super.initializeConfiguration(); - OzoneClientConfig clientConfig = new OzoneClientConfig(); + OzoneClientConfig clientConfig = conf.getObject(OzoneClientConfig.class); clientConfig.setStreamBufferFlushSize(8 * 1024 * 1024); clientConfig.setStreamBufferMaxSize(16 * 1024 * 1024); clientConfig.setStreamBufferSize(4 * 1024); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestCommitWatcher.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestCommitWatcher.java index 5c8fd3a9457c..fd4293d5773f 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestCommitWatcher.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestCommitWatcher.java @@ -128,7 +128,7 @@ public void init() throws Exception { ratisClientConfig.setWatchRequestTimeout(Duration.ofSeconds(10)); conf.setFromObject(ratisClientConfig); - OzoneClientConfig clientConfig = new OzoneClientConfig(); + OzoneClientConfig clientConfig = conf.getObject(OzoneClientConfig.class); clientConfig.setChecksumType(ChecksumType.NONE); conf.setFromObject(clientConfig); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java index f2a5748bffd1..9609dea04819 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestCloseContainerHandlingByClient.java @@ -89,7 +89,7 @@ public static void init() throws Exception { chunkSize = (int) OzoneConsts.MB; blockSize = 4 * chunkSize; - OzoneClientConfig config = new OzoneClientConfig(); + OzoneClientConfig config = conf.getObject(OzoneClientConfig.class); config.setChecksumType(ChecksumType.NONE); conf.setFromObject(config); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestDiscardPreallocatedBlocks.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestDiscardPreallocatedBlocks.java index c689a692ae79..478915868271 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestDiscardPreallocatedBlocks.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestDiscardPreallocatedBlocks.java @@ -85,7 +85,7 @@ public static void init() throws Exception { chunkSize = (int) OzoneConsts.MB; blockSize = 4 * chunkSize; - OzoneClientConfig config = new OzoneClientConfig(); + OzoneClientConfig config = conf.getObject(OzoneClientConfig.class); config.setChecksumType(ChecksumType.NONE); conf.setFromObject(config); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptionFlushDelay.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptionFlushDelay.java index 1f3b71385dff..2aca570f63ef 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptionFlushDelay.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptionFlushDelay.java @@ -89,7 +89,7 @@ public void init() throws Exception { maxFlushSize = 2 * flushSize; blockSize = 2 * maxFlushSize; - OzoneClientConfig config = new OzoneClientConfig(); + OzoneClientConfig config = conf.getObject(OzoneClientConfig.class); config.setChecksumType(ChecksumType.NONE); config.setMaxRetryCount(3); conf.setFromObject(config); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestInputStreamBase.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestInputStreamBase.java index 243694138818..76fa05a2627b 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestInputStreamBase.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestInputStreamBase.java @@ -46,7 +46,7 @@ protected static MiniOzoneCluster newCluster( ContainerLayoutVersion containerLayout) throws Exception { OzoneConfiguration conf = new OzoneConfiguration(); - OzoneClientConfig config = new OzoneClientConfig(); + OzoneClientConfig config = conf.getObject(OzoneClientConfig.class); config.setBytesPerChecksum(BYTES_PER_CHECKSUM); conf.setFromObject(config); From 906a25d150ebb5aedbe1ee8b57b9a2a5b866bd36 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Mon, 5 Feb 2024 10:38:27 +0100 Subject: [PATCH 4/7] fix typos (kept in previous commits to reduce diff) --- .../test/java/org/apache/hadoop/fs/ozone/TestHSync.java | 4 ++-- .../java/org/apache/hadoop/fs/ozone/TestLeaseRecovery.java | 4 ++-- .../hadoop/fs/ozone/TestOzoneFileSystemWithStreaming.java | 4 ++-- .../java/org/apache/hadoop/ozone/ClientConfigBuilder.java | 7 ++----- .../hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java | 4 ++-- 5 files changed, 10 insertions(+), 13 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSync.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSync.java index d742611b3f2b..6afcc8b0766e 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSync.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSync.java @@ -122,9 +122,9 @@ public static void init() throws Exception { .setChunkSize(chunkSize) .setStreamBufferFlushSize(flushSize) .setStreamBufferMaxSize(maxFlushSize) - .setDataStreamBufferFlushize(maxFlushSize) + .setDataStreamBufferFlushSize(maxFlushSize) .setDataStreamMinPacketSize(chunkSize) - .setDataStreamStreamWindowSize(5 * chunkSize) + .setDataStreamWindowSize(5 * chunkSize) .setOn(CONF); cluster = MiniOzoneCluster.newBuilder(CONF) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestLeaseRecovery.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestLeaseRecovery.java index 6e216477543d..c56726b834ab 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestLeaseRecovery.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestLeaseRecovery.java @@ -97,9 +97,9 @@ public void init() throws IOException, InterruptedException, .setChunkSize(chunkSize) .setStreamBufferFlushSize(flushSize) .setStreamBufferMaxSize(maxFlushSize) - .setDataStreamBufferFlushize(maxFlushSize) + .setDataStreamBufferFlushSize(maxFlushSize) .setDataStreamMinPacketSize(chunkSize) - .setDataStreamStreamWindowSize(5 * chunkSize) + .setDataStreamWindowSize(5 * chunkSize) .setOn(conf); cluster = MiniOzoneCluster.newBuilder(conf) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemWithStreaming.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemWithStreaming.java index 41e053ea6100..c4c70b16cca8 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemWithStreaming.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemWithStreaming.java @@ -93,9 +93,9 @@ public static void init() throws Exception { .setChunkSize(chunkSize) .setStreamBufferFlushSize(flushSize) .setStreamBufferMaxSize(maxFlushSize) - .setDataStreamBufferFlushize(maxFlushSize) + .setDataStreamBufferFlushSize(maxFlushSize) .setDataStreamMinPacketSize(chunkSize) - .setDataStreamStreamWindowSize(5 * chunkSize) + .setDataStreamWindowSize(5 * chunkSize) .setOn(CONF); cluster = MiniOzoneCluster.newBuilder(CONF) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ClientConfigBuilder.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ClientConfigBuilder.java index 0c9f280e7908..4fa3945bb9d3 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ClientConfigBuilder.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ClientConfigBuilder.java @@ -54,7 +54,6 @@ private ClientConfigBuilder(ConfigurationSource conf, StorageUnit unit) { this.unit = unit; } - // FIXME store everything in bytes public ClientConfigBuilder setChunkSize(int size) { chunkSize = (int) toBytes(size); return this; @@ -85,14 +84,12 @@ public ClientConfigBuilder setDataStreamMinPacketSize(int size) { return this; } - // TODO fix typo - public ClientConfigBuilder setDataStreamBufferFlushize(long size) { + public ClientConfigBuilder setDataStreamBufferFlushSize(long size) { dataStreamBufferFlushSize = OptionalLong.of(toBytes(size)); return this; } - // TODO fix typo in name - public ClientConfigBuilder setDataStreamStreamWindowSize(long size) { + public ClientConfigBuilder setDataStreamWindowSize(long size) { dataStreamWindowSize = OptionalLong.of(toBytes(size)); return this; } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java index dfb0cb8a6da2..e78ebc3d943b 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java @@ -98,9 +98,9 @@ public static void init() throws Exception { .setChunkSize(chunkSize) .setStreamBufferFlushSize(flushSize) .setStreamBufferMaxSize(maxFlushSize) - .setDataStreamBufferFlushize(maxFlushSize) + .setDataStreamBufferFlushSize(maxFlushSize) .setDataStreamMinPacketSize(chunkSize) - .setDataStreamStreamWindowSize(5 * chunkSize) + .setDataStreamWindowSize(5 * chunkSize) .setOn(conf); cluster = MiniOzoneCluster.newBuilder(conf) From 87fa11f840c784a2a3c1fd614c287a330f7a0656 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Mon, 5 Feb 2024 10:49:32 +0100 Subject: [PATCH 5/7] remove unused import --- .../test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java index 11c0f4497a30..400ae3ee2cc8 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java @@ -26,7 +26,6 @@ import java.util.Collection; import java.util.Collections; import java.util.List; -import java.util.Optional; import java.util.Set; import java.util.UUID; import java.util.Iterator; From 26c153bb90e61105a468d9c258b1dfe141b51e20 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Mon, 5 Feb 2024 10:58:59 +0100 Subject: [PATCH 6/7] rename setOn() to avoid confusion with set... builder methods; document unit parameter; remove separation of config source/target --- .../org/apache/hadoop/fs/ozone/TestHSync.java | 4 +-- .../hadoop/fs/ozone/TestLeaseRecovery.java | 4 +-- .../fs/ozone/TestOzoneFSInputStream.java | 4 +-- .../fs/ozone/TestOzoneFileSystemMetrics.java | 4 +-- .../TestOzoneFileSystemWithStreaming.java | 4 +-- .../hdds/scm/storage/TestCommitWatcher.java | 4 +-- .../hadoop/ozone/ClientConfigBuilder.java | 25 ++++++++++--------- .../rpc/AbstractTestECKeyOutputStream.java | 4 +-- .../client/rpc/Test2WayCommitInRatis.java | 4 +-- .../client/rpc/TestBlockDataStreamOutput.java | 4 +-- .../client/rpc/TestBlockOutputStream.java | 4 +-- .../TestContainerStateMachineFlushDelay.java | 4 +-- .../rpc/TestContainerStateMachineStream.java | 4 +-- ...TestFailureHandlingByClientFlushDelay.java | 4 +-- .../client/rpc/TestOzoneAtRestEncryption.java | 4 +-- ...oneClientRetriesOnExceptionFlushDelay.java | 4 +-- .../TestOzoneClientRetriesOnExceptions.java | 4 +-- .../rpc/TestOzoneRpcClientAbstract.java | 4 +-- .../ozone/client/rpc/TestWatchForCommit.java | 4 +-- .../client/rpc/read/TestInputStreamBase.java | 4 +-- .../container/TestECContainerRecovery.java | 4 +-- 21 files changed, 53 insertions(+), 52 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSync.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSync.java index 6afcc8b0766e..d7436aabebaa 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSync.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSync.java @@ -117,7 +117,7 @@ public static void init() throws Exception { CONF.setBoolean(OZONE_OM_RATIS_ENABLE_KEY, false); CONF.set(OZONE_DEFAULT_BUCKET_LAYOUT, layout.name()); CONF.setBoolean(OzoneConfigKeys.OZONE_FS_HSYNC_ENABLED, true); - ClientConfigBuilder.newBuilder(CONF, StorageUnit.BYTES) + ClientConfigBuilder.newBuilder(StorageUnit.BYTES) .setBlockSize(blockSize) .setChunkSize(chunkSize) .setStreamBufferFlushSize(flushSize) @@ -125,7 +125,7 @@ public static void init() throws Exception { .setDataStreamBufferFlushSize(maxFlushSize) .setDataStreamMinPacketSize(chunkSize) .setDataStreamWindowSize(5 * chunkSize) - .setOn(CONF); + .applyTo(CONF); cluster = MiniOzoneCluster.newBuilder(CONF) .setNumDatanodes(5) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestLeaseRecovery.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestLeaseRecovery.java index c56726b834ab..ec4dae8b42b3 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestLeaseRecovery.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestLeaseRecovery.java @@ -92,7 +92,7 @@ public void init() throws IOException, InterruptedException, conf.setBoolean(OZONE_OM_RATIS_ENABLE_KEY, false); conf.setBoolean(OzoneConfigKeys.OZONE_FS_HSYNC_ENABLED, true); conf.set(OZONE_DEFAULT_BUCKET_LAYOUT, layout.name()); - ClientConfigBuilder.newBuilder(conf, StorageUnit.BYTES) + ClientConfigBuilder.newBuilder(StorageUnit.BYTES) .setBlockSize(blockSize) .setChunkSize(chunkSize) .setStreamBufferFlushSize(flushSize) @@ -100,7 +100,7 @@ public void init() throws IOException, InterruptedException, .setDataStreamBufferFlushSize(maxFlushSize) .setDataStreamMinPacketSize(chunkSize) .setDataStreamWindowSize(5 * chunkSize) - .setOn(conf); + .applyTo(conf); cluster = MiniOzoneCluster.newBuilder(conf) .setNumDatanodes(5) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java index 1e84c0c11781..88a7b4e8741b 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java @@ -86,12 +86,12 @@ public static void init() throws Exception { conf.set(OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT, BucketLayout.LEGACY.name()); - ClientConfigBuilder.newBuilder(conf, StorageUnit.MB) + ClientConfigBuilder.newBuilder(StorageUnit.MB) .setChunkSize(2) .setBlockSize(8) .setStreamBufferFlushSize(2) .setStreamBufferMaxSize(4) - .setOn(conf); + .applyTo(conf); cluster = MiniOzoneCluster.newBuilder(conf) .setNumDatanodes(5) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemMetrics.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemMetrics.java index 72c29cfd6e56..82cbd3288fb1 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemMetrics.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemMetrics.java @@ -75,12 +75,12 @@ public static void init() throws Exception { BucketLayout.LEGACY.name()); conf.setBoolean(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS, true); - ClientConfigBuilder.newBuilder(conf, StorageUnit.MB) + ClientConfigBuilder.newBuilder(StorageUnit.MB) .setChunkSize(2) .setBlockSize(8) .setStreamBufferFlushSize(2) .setStreamBufferMaxSize(4) - .setOn(conf); + .applyTo(conf); cluster = MiniOzoneCluster.newBuilder(conf) .setNumDatanodes(3) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemWithStreaming.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemWithStreaming.java index c4c70b16cca8..62387bfaa7d3 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemWithStreaming.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemWithStreaming.java @@ -88,7 +88,7 @@ public static void init() throws Exception { CONF.setBoolean(OZONE_OM_RATIS_ENABLE_KEY, true); CONF.set(OZONE_DEFAULT_BUCKET_LAYOUT, layout.name()); - ClientConfigBuilder.newBuilder(CONF, StorageUnit.BYTES) + ClientConfigBuilder.newBuilder(StorageUnit.BYTES) .setBlockSize(blockSize) .setChunkSize(chunkSize) .setStreamBufferFlushSize(flushSize) @@ -96,7 +96,7 @@ public static void init() throws Exception { .setDataStreamBufferFlushSize(maxFlushSize) .setDataStreamMinPacketSize(chunkSize) .setDataStreamWindowSize(5 * chunkSize) - .setOn(CONF); + .applyTo(CONF); cluster = MiniOzoneCluster.newBuilder(CONF) .setNumDatanodes(5) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestCommitWatcher.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestCommitWatcher.java index fd4293d5773f..7abe1a3ec44c 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestCommitWatcher.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestCommitWatcher.java @@ -132,12 +132,12 @@ public void init() throws Exception { clientConfig.setChecksumType(ChecksumType.NONE); conf.setFromObject(clientConfig); - ClientConfigBuilder.newBuilder(conf, StorageUnit.BYTES) + ClientConfigBuilder.newBuilder(StorageUnit.BYTES) .setBlockSize(blockSize) .setChunkSize(chunkSize) .setStreamBufferFlushSize(flushSize) .setStreamBufferMaxSize(maxFlushSize) - .setOn(conf); + .applyTo(conf); conf.setQuietMode(false); conf.setStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE, 4, diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ClientConfigBuilder.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ClientConfigBuilder.java index 4fa3945bb9d3..b8552372d3ca 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ClientConfigBuilder.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ClientConfigBuilder.java @@ -17,8 +17,7 @@ */ package org.apache.hadoop.ozone; -import org.apache.hadoop.hdds.conf.ConfigurationSource; -import org.apache.hadoop.hdds.conf.ConfigurationTarget; +import org.apache.hadoop.hdds.conf.MutableConfigurationSource; import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.scm.OzoneClientConfig; @@ -33,8 +32,6 @@ */ public final class ClientConfigBuilder { - private final ConfigurationSource conf; - private int chunkSize = 1024 * 1024; private OptionalLong blockSize = OptionalLong.empty(); private OptionalInt streamBufferSize = OptionalInt.empty(); @@ -45,12 +42,15 @@ public final class ClientConfigBuilder { private OptionalInt dataStreamMinPacketSize = OptionalInt.empty(); private final StorageUnit unit; - public static ClientConfigBuilder newBuilder(ConfigurationSource conf, StorageUnit unit) { - return new ClientConfigBuilder(conf, unit); + /** + * @param unit Defines the unit in which size properties will be passed to the builder. + * All sizes are stored internally converted to {@link StorageUnit#BYTES}. + */ + public static ClientConfigBuilder newBuilder(StorageUnit unit) { + return new ClientConfigBuilder(unit); } - private ClientConfigBuilder(ConfigurationSource conf, StorageUnit unit) { - this.conf = conf; + private ClientConfigBuilder(StorageUnit unit) { this.unit = unit; } @@ -64,6 +64,7 @@ public ClientConfigBuilder setBlockSize(long size) { return this; } + @SuppressWarnings("unused") // kept for completeness public ClientConfigBuilder setStreamBufferSize(int size) { streamBufferSize = OptionalInt.of((int) toBytes(size)); return this; @@ -94,7 +95,7 @@ public ClientConfigBuilder setDataStreamWindowSize(long size) { return this; } - public void setOn(ConfigurationTarget target) { + public void applyTo(MutableConfigurationSource conf) { if (!streamBufferSize.isPresent()) { streamBufferSize = OptionalInt.of(chunkSize); } @@ -125,9 +126,9 @@ public void setOn(ConfigurationTarget target) { clientConfig.setDataStreamMinPacketSize(dataStreamMinPacketSize.getAsInt()); clientConfig.setStreamWindowSize(dataStreamWindowSize.getAsLong()); - target.setFromObject(clientConfig); - target.setStorageSize(OZONE_SCM_CHUNK_SIZE_KEY, chunkSize, StorageUnit.BYTES); - target.setStorageSize(OZONE_SCM_BLOCK_SIZE, blockSize.getAsLong(), StorageUnit.BYTES); + conf.setFromObject(clientConfig); + conf.setStorageSize(OZONE_SCM_CHUNK_SIZE_KEY, chunkSize, StorageUnit.BYTES); + conf.setStorageSize(OZONE_SCM_BLOCK_SIZE, blockSize.getAsLong(), StorageUnit.BYTES); } private long toBytes(long value) { diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/AbstractTestECKeyOutputStream.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/AbstractTestECKeyOutputStream.java index a661002f1973..8fb79f3b31a6 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/AbstractTestECKeyOutputStream.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/AbstractTestECKeyOutputStream.java @@ -122,12 +122,12 @@ protected static void init(boolean zeroCopyEnabled) throws Exception { conf.setBoolean(OzoneConfigKeys.OZONE_EC_GRPC_ZERO_COPY_ENABLED, zeroCopyEnabled); - ClientConfigBuilder.newBuilder(conf, StorageUnit.BYTES) + ClientConfigBuilder.newBuilder(StorageUnit.BYTES) .setBlockSize(blockSize) .setChunkSize(chunkSize) .setStreamBufferFlushSize(flushSize) .setStreamBufferMaxSize(maxFlushSize) - .setOn(conf); + .applyTo(conf); cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(10) .setTotalPipelineNumLimit(10).build(); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/Test2WayCommitInRatis.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/Test2WayCommitInRatis.java index 1410f3a5eb91..2f5af542b915 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/Test2WayCommitInRatis.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/Test2WayCommitInRatis.java @@ -101,12 +101,12 @@ private void startCluster(OzoneConfiguration conf) throws Exception { raftClientConfig.setRpcWatchRequestTimeout(Duration.ofSeconds(10)); conf.setFromObject(raftClientConfig); - ClientConfigBuilder.newBuilder(conf, StorageUnit.BYTES) + ClientConfigBuilder.newBuilder(StorageUnit.BYTES) .setBlockSize(blockSize) .setChunkSize(chunkSize) .setStreamBufferFlushSize(flushSize) .setStreamBufferMaxSize(maxFlushSize) - .setOn(conf); + .applyTo(conf); conf.setQuietMode(false); cluster = MiniOzoneCluster.newBuilder(conf) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java index e78ebc3d943b..fe94fa23d124 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java @@ -93,7 +93,7 @@ public static void init() throws Exception { conf.setStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE, 4, StorageUnit.MB); - ClientConfigBuilder.newBuilder(conf, StorageUnit.BYTES) + ClientConfigBuilder.newBuilder(StorageUnit.BYTES) .setBlockSize(blockSize) .setChunkSize(chunkSize) .setStreamBufferFlushSize(flushSize) @@ -101,7 +101,7 @@ public static void init() throws Exception { .setDataStreamBufferFlushSize(maxFlushSize) .setDataStreamMinPacketSize(chunkSize) .setDataStreamWindowSize(5 * chunkSize) - .setOn(conf); + .applyTo(conf); cluster = MiniOzoneCluster.newBuilder(conf) .setNumDatanodes(5) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockOutputStream.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockOutputStream.java index 1df925b5ba41..9cf307c56938 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockOutputStream.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockOutputStream.java @@ -106,12 +106,12 @@ static MiniOzoneCluster createCluster() throws IOException, ratisClientConfig.setWatchRequestTimeout(Duration.ofSeconds(30)); conf.setFromObject(ratisClientConfig); - ClientConfigBuilder.newBuilder(conf, StorageUnit.BYTES) + ClientConfigBuilder.newBuilder(StorageUnit.BYTES) .setBlockSize(BLOCK_SIZE) .setChunkSize(CHUNK_SIZE) .setStreamBufferFlushSize(FLUSH_SIZE) .setStreamBufferMaxSize(MAX_FLUSH_SIZE) - .setOn(conf); + .applyTo(conf); MiniOzoneCluster cluster = MiniOzoneCluster.newBuilder(conf) .setNumDatanodes(5) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFlushDelay.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFlushDelay.java index 38bc89f0d7dc..60c929abb01f 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFlushDelay.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFlushDelay.java @@ -110,12 +110,12 @@ public void setup() throws Exception { conf.set(ScmConfigKeys.OZONE_SCM_PIPELINE_SCRUB_INTERVAL, "2s"); conf.set(ScmConfigKeys.OZONE_SCM_PIPELINE_DESTROY_TIMEOUT, "5s"); - ClientConfigBuilder.newBuilder(conf, StorageUnit.BYTES) + ClientConfigBuilder.newBuilder(StorageUnit.BYTES) .setBlockSize(blockSize) .setChunkSize(chunkSize) .setStreamBufferFlushSize(flushSize) .setStreamBufferMaxSize(maxFlushSize) - .setOn(conf); + .applyTo(conf); cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(1) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineStream.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineStream.java index 1d345f9a18a4..6aac96dfaf58 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineStream.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineStream.java @@ -113,13 +113,13 @@ public void setup() throws Exception { raftClientConfig.setRpcWatchRequestTimeout(Duration.ofSeconds(10)); conf.setFromObject(raftClientConfig); - ClientConfigBuilder.newBuilder(conf, StorageUnit.BYTES) + ClientConfigBuilder.newBuilder(StorageUnit.BYTES) .setDataStreamMinPacketSize(1024) .setBlockSize(BLOCK_SIZE) .setChunkSize(CHUNK_SIZE) .setStreamBufferFlushSize(FLUSH_SIZE) .setStreamBufferMaxSize(MAX_FLUSH_SIZE) - .setOn(conf); + .applyTo(conf); conf.setLong(OzoneConfigKeys.DFS_RATIS_SNAPSHOT_THRESHOLD_KEY, 1); conf.setQuietMode(false); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClientFlushDelay.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClientFlushDelay.java index 1a9d26758d98..7c8e45c8f8a1 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClientFlushDelay.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClientFlushDelay.java @@ -131,12 +131,12 @@ private void init() throws Exception { Collections.singleton(HddsUtils.getHostName(conf))).get(0), "/rack1"); - ClientConfigBuilder.newBuilder(conf, StorageUnit.BYTES) + ClientConfigBuilder.newBuilder(StorageUnit.BYTES) .setBlockSize(blockSize) .setChunkSize(chunkSize) .setStreamBufferFlushSize(flushSize) .setStreamBufferMaxSize(maxFlushSize) - .setOn(conf); + .applyTo(conf); cluster = MiniOzoneCluster.newBuilder(conf) .setNumDatanodes(10) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneAtRestEncryption.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneAtRestEncryption.java index 84795512798a..63527d73dbb3 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneAtRestEncryption.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneAtRestEncryption.java @@ -142,10 +142,10 @@ static void init() throws Exception { CertificateClientTestImpl certificateClientTest = new CertificateClientTestImpl(conf); - ClientConfigBuilder.newBuilder(conf, StorageUnit.BYTES) + ClientConfigBuilder.newBuilder(StorageUnit.BYTES) .setBlockSize(BLOCK_SIZE) .setChunkSize(CHUNK_SIZE) - .setOn(conf); + .applyTo(conf); cluster = MiniOzoneCluster.newBuilder(conf) .setNumDatanodes(10) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptionFlushDelay.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptionFlushDelay.java index 2aca570f63ef..f5634ebac44e 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptionFlushDelay.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptionFlushDelay.java @@ -100,12 +100,12 @@ public void init() throws Exception { conf.set(ScmConfigKeys.OZONE_SCM_PIPELINE_DESTROY_TIMEOUT, "5s"); conf.setQuietMode(false); - ClientConfigBuilder.newBuilder(conf, StorageUnit.BYTES) + ClientConfigBuilder.newBuilder(StorageUnit.BYTES) .setBlockSize(blockSize) .setChunkSize(chunkSize) .setStreamBufferFlushSize(flushSize) .setStreamBufferMaxSize(maxFlushSize) - .setOn(conf); + .applyTo(conf); cluster = MiniOzoneCluster.newBuilder(conf) .setNumDatanodes(5) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptions.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptions.java index eb2b4a1dc0e3..043a3a5d78d3 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptions.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptions.java @@ -110,12 +110,12 @@ public void init() throws Exception { conf.set(ScmConfigKeys.OZONE_SCM_PIPELINE_DESTROY_TIMEOUT, "5s"); conf.setQuietMode(false); - ClientConfigBuilder.newBuilder(conf, StorageUnit.BYTES) + ClientConfigBuilder.newBuilder(StorageUnit.BYTES) .setBlockSize(blockSize) .setChunkSize(chunkSize) .setStreamBufferFlushSize(flushSize) .setStreamBufferMaxSize(maxFlushSize) - .setOn(conf); + .applyTo(conf); cluster = MiniOzoneCluster.newBuilder(conf) .setNumDatanodes(5) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java index c4b5cfa480df..af594e5d098a 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java @@ -205,9 +205,9 @@ static void startCluster(OzoneConfiguration conf) throws Exception { // for testZReadKeyWithUnhealthyContainerReplica. conf.set("ozone.scm.stale.node.interval", "10s"); - ClientConfigBuilder.newBuilder(conf, StorageUnit.MB) + ClientConfigBuilder.newBuilder(StorageUnit.MB) .setDataStreamMinPacketSize(1) - .setOn(conf); + .applyTo(conf); cluster = MiniOzoneCluster.newBuilder(conf) .setNumDatanodes(14) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestWatchForCommit.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestWatchForCommit.java index 88fcdf9cab41..b1af0e634844 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestWatchForCommit.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestWatchForCommit.java @@ -142,12 +142,12 @@ public void init() throws Exception { conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 30, TimeUnit.SECONDS); - ClientConfigBuilder.newBuilder(conf, StorageUnit.BYTES) + ClientConfigBuilder.newBuilder(StorageUnit.BYTES) .setBlockSize(blockSize) .setChunkSize(chunkSize) .setStreamBufferFlushSize(flushSize) .setStreamBufferMaxSize(maxFlushSize) - .setOn(conf); + .applyTo(conf); cluster = MiniOzoneCluster.newBuilder(conf) .setNumDatanodes(9) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestInputStreamBase.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestInputStreamBase.java index 76fa05a2627b..0378cd30a565 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestInputStreamBase.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestInputStreamBase.java @@ -64,12 +64,12 @@ protected static MiniOzoneCluster newCluster( repConf.setInterval(Duration.ofSeconds(1)); conf.setFromObject(repConf); - ClientConfigBuilder.newBuilder(conf, StorageUnit.BYTES) + ClientConfigBuilder.newBuilder(StorageUnit.BYTES) .setBlockSize(BLOCK_SIZE) .setChunkSize(CHUNK_SIZE) .setStreamBufferFlushSize(FLUSH_SIZE) .setStreamBufferMaxSize(MAX_FLUSH_SIZE) - .setOn(conf); + .applyTo(conf); return MiniOzoneCluster.newBuilder(conf) .setNumDatanodes(5) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestECContainerRecovery.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestECContainerRecovery.java index e7c46f92aafe..b15817bdf718 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestECContainerRecovery.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestECContainerRecovery.java @@ -141,12 +141,12 @@ public static void init() throws Exception { conf.setTimeDuration(HddsConfigKeys.HDDS_CONTAINER_REPORT_INTERVAL, 1, TimeUnit.SECONDS); - ClientConfigBuilder.newBuilder(conf, StorageUnit.BYTES) + ClientConfigBuilder.newBuilder(StorageUnit.BYTES) .setBlockSize(blockSize) .setChunkSize(chunkSize) .setStreamBufferFlushSize(flushSize) .setStreamBufferMaxSize(maxFlushSize) - .setOn(conf); + .applyTo(conf); cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(10) .setTotalPipelineNumLimit(10) From e59fd0f2b60a81d7b50b9aaa75f581ef918678ea Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Tue, 6 Feb 2024 20:04:26 +0100 Subject: [PATCH 7/7] replace Optional(Int|Long); rename to ClientConfigForTesting --- .../org/apache/hadoop/fs/ozone/TestHSync.java | 4 +- .../hadoop/fs/ozone/TestLeaseRecovery.java | 4 +- .../fs/ozone/TestOzoneFSInputStream.java | 4 +- .../fs/ozone/TestOzoneFileSystemMetrics.java | 4 +- .../TestOzoneFileSystemWithStreaming.java | 4 +- .../hdds/scm/storage/TestCommitWatcher.java | 4 +- .../hadoop/ozone/ClientConfigBuilder.java | 138 ------------------ .../hadoop/ozone/ClientConfigForTesting.java | 135 +++++++++++++++++ .../rpc/AbstractTestECKeyOutputStream.java | 4 +- .../client/rpc/Test2WayCommitInRatis.java | 4 +- .../client/rpc/TestBlockDataStreamOutput.java | 4 +- .../client/rpc/TestBlockOutputStream.java | 4 +- .../TestContainerStateMachineFlushDelay.java | 4 +- .../rpc/TestContainerStateMachineStream.java | 4 +- ...TestFailureHandlingByClientFlushDelay.java | 4 +- .../client/rpc/TestOzoneAtRestEncryption.java | 4 +- ...oneClientRetriesOnExceptionFlushDelay.java | 4 +- .../TestOzoneClientRetriesOnExceptions.java | 4 +- .../rpc/TestOzoneRpcClientAbstract.java | 4 +- .../ozone/client/rpc/TestWatchForCommit.java | 4 +- .../client/rpc/read/TestInputStreamBase.java | 4 +- .../container/TestECContainerRecovery.java | 4 +- 22 files changed, 175 insertions(+), 178 deletions(-) delete mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ClientConfigBuilder.java create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ClientConfigForTesting.java diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSync.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSync.java index d7436aabebaa..05d297d38ed8 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSync.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSync.java @@ -46,7 +46,7 @@ import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.TableIterator; -import org.apache.hadoop.ozone.ClientConfigBuilder; +import org.apache.hadoop.ozone.ClientConfigForTesting; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.OzoneConsts; @@ -117,7 +117,7 @@ public static void init() throws Exception { CONF.setBoolean(OZONE_OM_RATIS_ENABLE_KEY, false); CONF.set(OZONE_DEFAULT_BUCKET_LAYOUT, layout.name()); CONF.setBoolean(OzoneConfigKeys.OZONE_FS_HSYNC_ENABLED, true); - ClientConfigBuilder.newBuilder(StorageUnit.BYTES) + ClientConfigForTesting.newBuilder(StorageUnit.BYTES) .setBlockSize(blockSize) .setChunkSize(chunkSize) .setStreamBufferFlushSize(flushSize) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestLeaseRecovery.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestLeaseRecovery.java index ec4dae8b42b3..ca68aad45515 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestLeaseRecovery.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestLeaseRecovery.java @@ -25,7 +25,7 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.utils.IOUtils; -import org.apache.hadoop.ozone.ClientConfigBuilder; +import org.apache.hadoop.ozone.ClientConfigForTesting; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.TestDataUtil; @@ -92,7 +92,7 @@ public void init() throws IOException, InterruptedException, conf.setBoolean(OZONE_OM_RATIS_ENABLE_KEY, false); conf.setBoolean(OzoneConfigKeys.OZONE_FS_HSYNC_ENABLED, true); conf.set(OZONE_DEFAULT_BUCKET_LAYOUT, layout.name()); - ClientConfigBuilder.newBuilder(StorageUnit.BYTES) + ClientConfigForTesting.newBuilder(StorageUnit.BYTES) .setBlockSize(blockSize) .setChunkSize(chunkSize) .setStreamBufferFlushSize(flushSize) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java index 88a7b4e8741b..c6893c57e969 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java @@ -39,7 +39,7 @@ import org.apache.hadoop.hdds.protocol.StorageType; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.SequenceFile; -import org.apache.hadoop.ozone.ClientConfigBuilder; +import org.apache.hadoop.ozone.ClientConfigForTesting; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.TestDataUtil; @@ -86,7 +86,7 @@ public static void init() throws Exception { conf.set(OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT, BucketLayout.LEGACY.name()); - ClientConfigBuilder.newBuilder(StorageUnit.MB) + ClientConfigForTesting.newBuilder(StorageUnit.MB) .setChunkSize(2) .setBlockSize(8) .setStreamBufferFlushSize(2) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemMetrics.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemMetrics.java index 82cbd3288fb1..5aba83bd412e 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemMetrics.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemMetrics.java @@ -26,7 +26,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.ozone.ClientConfigBuilder; +import org.apache.hadoop.ozone.ClientConfigForTesting; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.TestDataUtil; @@ -75,7 +75,7 @@ public static void init() throws Exception { BucketLayout.LEGACY.name()); conf.setBoolean(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS, true); - ClientConfigBuilder.newBuilder(StorageUnit.MB) + ClientConfigForTesting.newBuilder(StorageUnit.MB) .setChunkSize(2) .setBlockSize(8) .setStreamBufferFlushSize(2) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemWithStreaming.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemWithStreaming.java index 62387bfaa7d3..2a6c8c456b9c 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemWithStreaming.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemWithStreaming.java @@ -30,7 +30,7 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.utils.IOUtils; -import org.apache.hadoop.ozone.ClientConfigBuilder; +import org.apache.hadoop.ozone.ClientConfigForTesting; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.OzoneBucket; @@ -88,7 +88,7 @@ public static void init() throws Exception { CONF.setBoolean(OZONE_OM_RATIS_ENABLE_KEY, true); CONF.set(OZONE_DEFAULT_BUCKET_LAYOUT, layout.name()); - ClientConfigBuilder.newBuilder(StorageUnit.BYTES) + ClientConfigForTesting.newBuilder(StorageUnit.BYTES) .setBlockSize(blockSize) .setChunkSize(chunkSize) .setStreamBufferFlushSize(flushSize) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestCommitWatcher.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestCommitWatcher.java index 7abe1a3ec44c..563e0162acc6 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestCommitWatcher.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestCommitWatcher.java @@ -45,7 +45,7 @@ import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB; import org.apache.hadoop.hdds.utils.IOUtils; -import org.apache.hadoop.ozone.ClientConfigBuilder; +import org.apache.hadoop.ozone.ClientConfigForTesting; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.OzoneConsts; @@ -132,7 +132,7 @@ public void init() throws Exception { clientConfig.setChecksumType(ChecksumType.NONE); conf.setFromObject(clientConfig); - ClientConfigBuilder.newBuilder(StorageUnit.BYTES) + ClientConfigForTesting.newBuilder(StorageUnit.BYTES) .setBlockSize(blockSize) .setChunkSize(chunkSize) .setStreamBufferFlushSize(flushSize) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ClientConfigBuilder.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ClientConfigBuilder.java deleted file mode 100644 index b8552372d3ca..000000000000 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ClientConfigBuilder.java +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.ozone; - -import org.apache.hadoop.hdds.conf.MutableConfigurationSource; -import org.apache.hadoop.hdds.conf.StorageUnit; -import org.apache.hadoop.hdds.scm.OzoneClientConfig; - -import java.util.OptionalInt; -import java.util.OptionalLong; - -import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE; - -/** - * Helper for tests that want to set client stream properties. - */ -public final class ClientConfigBuilder { - - private int chunkSize = 1024 * 1024; - private OptionalLong blockSize = OptionalLong.empty(); - private OptionalInt streamBufferSize = OptionalInt.empty(); - private OptionalLong streamBufferFlushSize = OptionalLong.empty(); - private OptionalLong dataStreamBufferFlushSize = OptionalLong.empty(); - private OptionalLong dataStreamWindowSize = OptionalLong.empty(); - private OptionalLong streamBufferMaxSize = OptionalLong.empty(); - private OptionalInt dataStreamMinPacketSize = OptionalInt.empty(); - private final StorageUnit unit; - - /** - * @param unit Defines the unit in which size properties will be passed to the builder. - * All sizes are stored internally converted to {@link StorageUnit#BYTES}. - */ - public static ClientConfigBuilder newBuilder(StorageUnit unit) { - return new ClientConfigBuilder(unit); - } - - private ClientConfigBuilder(StorageUnit unit) { - this.unit = unit; - } - - public ClientConfigBuilder setChunkSize(int size) { - chunkSize = (int) toBytes(size); - return this; - } - - public ClientConfigBuilder setBlockSize(long size) { - blockSize = OptionalLong.of(toBytes(size)); - return this; - } - - @SuppressWarnings("unused") // kept for completeness - public ClientConfigBuilder setStreamBufferSize(int size) { - streamBufferSize = OptionalInt.of((int) toBytes(size)); - return this; - } - - public ClientConfigBuilder setStreamBufferFlushSize(long size) { - streamBufferFlushSize = OptionalLong.of(toBytes(size)); - return this; - } - - public ClientConfigBuilder setStreamBufferMaxSize(long size) { - streamBufferMaxSize = OptionalLong.of(toBytes(size)); - return this; - } - - public ClientConfigBuilder setDataStreamMinPacketSize(int size) { - dataStreamMinPacketSize = OptionalInt.of((int) toBytes(size)); - return this; - } - - public ClientConfigBuilder setDataStreamBufferFlushSize(long size) { - dataStreamBufferFlushSize = OptionalLong.of(toBytes(size)); - return this; - } - - public ClientConfigBuilder setDataStreamWindowSize(long size) { - dataStreamWindowSize = OptionalLong.of(toBytes(size)); - return this; - } - - public void applyTo(MutableConfigurationSource conf) { - if (!streamBufferSize.isPresent()) { - streamBufferSize = OptionalInt.of(chunkSize); - } - if (!streamBufferFlushSize.isPresent()) { - streamBufferFlushSize = OptionalLong.of(chunkSize); - } - if (!streamBufferMaxSize.isPresent()) { - streamBufferMaxSize = OptionalLong.of(2 * streamBufferFlushSize.getAsLong()); - } - if (!dataStreamBufferFlushSize.isPresent()) { - dataStreamBufferFlushSize = OptionalLong.of(4L * chunkSize); - } - if (!dataStreamMinPacketSize.isPresent()) { - dataStreamMinPacketSize = OptionalInt.of(chunkSize / 4); - } - if (!dataStreamWindowSize.isPresent()) { - dataStreamWindowSize = OptionalLong.of(8L * chunkSize); - } - if (!blockSize.isPresent()) { - blockSize = OptionalLong.of(2 * streamBufferMaxSize.getAsLong()); - } - - OzoneClientConfig clientConfig = conf.getObject(OzoneClientConfig.class); - clientConfig.setStreamBufferSize(streamBufferSize.getAsInt()); - clientConfig.setStreamBufferMaxSize(streamBufferMaxSize.getAsLong()); - clientConfig.setStreamBufferFlushSize(streamBufferFlushSize.getAsLong()); - clientConfig.setDataStreamBufferFlushSize(dataStreamBufferFlushSize.getAsLong()); - clientConfig.setDataStreamMinPacketSize(dataStreamMinPacketSize.getAsInt()); - clientConfig.setStreamWindowSize(dataStreamWindowSize.getAsLong()); - - conf.setFromObject(clientConfig); - conf.setStorageSize(OZONE_SCM_CHUNK_SIZE_KEY, chunkSize, StorageUnit.BYTES); - conf.setStorageSize(OZONE_SCM_BLOCK_SIZE, blockSize.getAsLong(), StorageUnit.BYTES); - } - - private long toBytes(long value) { - return Math.round(unit.toBytes(value)); - } - -} diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ClientConfigForTesting.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ClientConfigForTesting.java new file mode 100644 index 000000000000..d436a65dab29 --- /dev/null +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ClientConfigForTesting.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone; + +import org.apache.hadoop.hdds.conf.MutableConfigurationSource; +import org.apache.hadoop.hdds.conf.StorageUnit; +import org.apache.hadoop.hdds.scm.OzoneClientConfig; + +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE; + +/** + * Helper for tests that want to set client stream properties. + */ +public final class ClientConfigForTesting { + + private int chunkSize = 1024 * 1024; + private Long blockSize; + private Integer streamBufferSize; + private Long streamBufferFlushSize; + private Long dataStreamBufferFlushSize; + private Long dataStreamWindowSize; + private Long streamBufferMaxSize; + private Integer dataStreamMinPacketSize; + private final StorageUnit unit; + + /** + * @param unit Defines the unit in which size properties will be passed to the builder. + * All sizes are stored internally converted to {@link StorageUnit#BYTES}. + */ + public static ClientConfigForTesting newBuilder(StorageUnit unit) { + return new ClientConfigForTesting(unit); + } + + private ClientConfigForTesting(StorageUnit unit) { + this.unit = unit; + } + + public ClientConfigForTesting setChunkSize(int size) { + chunkSize = (int) toBytes(size); + return this; + } + + public ClientConfigForTesting setBlockSize(long size) { + blockSize = toBytes(size); + return this; + } + + @SuppressWarnings("unused") // kept for completeness + public ClientConfigForTesting setStreamBufferSize(int size) { + streamBufferSize = (int) toBytes(size); + return this; + } + + public ClientConfigForTesting setStreamBufferFlushSize(long size) { + streamBufferFlushSize = toBytes(size); + return this; + } + + public ClientConfigForTesting setStreamBufferMaxSize(long size) { + streamBufferMaxSize = toBytes(size); + return this; + } + + public ClientConfigForTesting setDataStreamMinPacketSize(int size) { + dataStreamMinPacketSize = (int) toBytes(size); + return this; + } + + public ClientConfigForTesting setDataStreamBufferFlushSize(long size) { + dataStreamBufferFlushSize = toBytes(size); + return this; + } + + public ClientConfigForTesting setDataStreamWindowSize(long size) { + dataStreamWindowSize = toBytes(size); + return this; + } + + public void applyTo(MutableConfigurationSource conf) { + if (streamBufferSize == null) { + streamBufferSize = chunkSize; + } + if (streamBufferFlushSize == null) { + streamBufferFlushSize = (long) chunkSize; + } + if (streamBufferMaxSize == null) { + streamBufferMaxSize = 2 * streamBufferFlushSize; + } + if (dataStreamBufferFlushSize == null) { + dataStreamBufferFlushSize = 4L * chunkSize; + } + if (dataStreamMinPacketSize == null) { + dataStreamMinPacketSize = chunkSize / 4; + } + if (dataStreamWindowSize == null) { + dataStreamWindowSize = 8L * chunkSize; + } + if (blockSize == null) { + blockSize = 2 * streamBufferMaxSize; + } + + OzoneClientConfig clientConfig = conf.getObject(OzoneClientConfig.class); + clientConfig.setStreamBufferSize(streamBufferSize); + clientConfig.setStreamBufferMaxSize(streamBufferMaxSize); + clientConfig.setStreamBufferFlushSize(streamBufferFlushSize); + clientConfig.setDataStreamBufferFlushSize(dataStreamBufferFlushSize); + clientConfig.setDataStreamMinPacketSize(dataStreamMinPacketSize); + clientConfig.setStreamWindowSize(dataStreamWindowSize); + + conf.setFromObject(clientConfig); + conf.setStorageSize(OZONE_SCM_CHUNK_SIZE_KEY, chunkSize, StorageUnit.BYTES); + conf.setStorageSize(OZONE_SCM_BLOCK_SIZE, blockSize, StorageUnit.BYTES); + } + + private long toBytes(long value) { + return Math.round(unit.toBytes(value)); + } + +} diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/AbstractTestECKeyOutputStream.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/AbstractTestECKeyOutputStream.java index 8fb79f3b31a6..0b0b2586c9e2 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/AbstractTestECKeyOutputStream.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/AbstractTestECKeyOutputStream.java @@ -31,7 +31,7 @@ import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.pipeline.PipelineManager; import org.apache.hadoop.hdds.utils.IOUtils; -import org.apache.hadoop.ozone.ClientConfigBuilder; +import org.apache.hadoop.ozone.ClientConfigForTesting; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.client.BucketArgs; @@ -122,7 +122,7 @@ protected static void init(boolean zeroCopyEnabled) throws Exception { conf.setBoolean(OzoneConfigKeys.OZONE_EC_GRPC_ZERO_COPY_ENABLED, zeroCopyEnabled); - ClientConfigBuilder.newBuilder(StorageUnit.BYTES) + ClientConfigForTesting.newBuilder(StorageUnit.BYTES) .setBlockSize(blockSize) .setChunkSize(chunkSize) .setStreamBufferFlushSize(flushSize) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/Test2WayCommitInRatis.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/Test2WayCommitInRatis.java index 2f5af542b915..e7c8be170ca1 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/Test2WayCommitInRatis.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/Test2WayCommitInRatis.java @@ -32,7 +32,7 @@ import org.apache.hadoop.hdds.scm.protocolPB. StorageContainerLocationProtocolClientSideTranslatorPB; import org.apache.hadoop.hdds.utils.IOUtils; -import org.apache.hadoop.ozone.ClientConfigBuilder; +import org.apache.hadoop.ozone.ClientConfigForTesting; import org.apache.hadoop.ozone.HddsDatanodeService; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConsts; @@ -101,7 +101,7 @@ private void startCluster(OzoneConfiguration conf) throws Exception { raftClientConfig.setRpcWatchRequestTimeout(Duration.ofSeconds(10)); conf.setFromObject(raftClientConfig); - ClientConfigBuilder.newBuilder(StorageUnit.BYTES) + ClientConfigForTesting.newBuilder(StorageUnit.BYTES) .setBlockSize(blockSize) .setChunkSize(chunkSize) .setStreamBufferFlushSize(flushSize) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java index fe94fa23d124..8bb791bb103e 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java @@ -27,7 +27,7 @@ import org.apache.hadoop.hdds.scm.storage.BlockDataStreamOutput; import org.apache.hadoop.hdds.scm.storage.ByteBufferStreamOutput; import org.apache.hadoop.hdds.utils.IOUtils; -import org.apache.hadoop.ozone.ClientConfigBuilder; +import org.apache.hadoop.ozone.ClientConfigForTesting; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.client.ObjectStore; @@ -93,7 +93,7 @@ public static void init() throws Exception { conf.setStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE, 4, StorageUnit.MB); - ClientConfigBuilder.newBuilder(StorageUnit.BYTES) + ClientConfigForTesting.newBuilder(StorageUnit.BYTES) .setBlockSize(blockSize) .setChunkSize(chunkSize) .setStreamBufferFlushSize(flushSize) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockOutputStream.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockOutputStream.java index 9cf307c56938..1e9cefbaa481 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockOutputStream.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockOutputStream.java @@ -38,7 +38,7 @@ import org.apache.hadoop.hdds.scm.XceiverClientMetrics; import org.apache.hadoop.hdds.scm.storage.BufferPool; import org.apache.hadoop.hdds.scm.storage.RatisBlockOutputStream; -import org.apache.hadoop.ozone.ClientConfigBuilder; +import org.apache.hadoop.ozone.ClientConfigForTesting; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneClient; @@ -106,7 +106,7 @@ static MiniOzoneCluster createCluster() throws IOException, ratisClientConfig.setWatchRequestTimeout(Duration.ofSeconds(30)); conf.setFromObject(ratisClientConfig); - ClientConfigBuilder.newBuilder(StorageUnit.BYTES) + ClientConfigForTesting.newBuilder(StorageUnit.BYTES) .setBlockSize(BLOCK_SIZE) .setChunkSize(CHUNK_SIZE) .setStreamBufferFlushSize(FLUSH_SIZE) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFlushDelay.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFlushDelay.java index 60c929abb01f..23ab89b80c65 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFlushDelay.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFlushDelay.java @@ -27,7 +27,7 @@ import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClientTestImpl; import org.apache.hadoop.hdds.utils.IOUtils; -import org.apache.hadoop.ozone.ClientConfigBuilder; +import org.apache.hadoop.ozone.ClientConfigForTesting; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.client.ObjectStore; @@ -110,7 +110,7 @@ public void setup() throws Exception { conf.set(ScmConfigKeys.OZONE_SCM_PIPELINE_SCRUB_INTERVAL, "2s"); conf.set(ScmConfigKeys.OZONE_SCM_PIPELINE_DESTROY_TIMEOUT, "5s"); - ClientConfigBuilder.newBuilder(StorageUnit.BYTES) + ClientConfigForTesting.newBuilder(StorageUnit.BYTES) .setBlockSize(blockSize) .setChunkSize(chunkSize) .setStreamBufferFlushSize(flushSize) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineStream.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineStream.java index 6aac96dfaf58..97a3047bfdb0 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineStream.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineStream.java @@ -25,7 +25,7 @@ import org.apache.hadoop.hdds.ratis.conf.RatisClientConfig; import org.apache.hadoop.hdds.scm.OzoneClientConfig; import org.apache.hadoop.hdds.utils.IOUtils; -import org.apache.hadoop.ozone.ClientConfigBuilder; +import org.apache.hadoop.ozone.ClientConfigForTesting; import org.apache.hadoop.ozone.HddsDatanodeService; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfigKeys; @@ -113,7 +113,7 @@ public void setup() throws Exception { raftClientConfig.setRpcWatchRequestTimeout(Duration.ofSeconds(10)); conf.setFromObject(raftClientConfig); - ClientConfigBuilder.newBuilder(StorageUnit.BYTES) + ClientConfigForTesting.newBuilder(StorageUnit.BYTES) .setDataStreamMinPacketSize(1024) .setBlockSize(BLOCK_SIZE) .setChunkSize(CHUNK_SIZE) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClientFlushDelay.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClientFlushDelay.java index 7c8e45c8f8a1..fadc06bd57bd 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClientFlushDelay.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClientFlushDelay.java @@ -35,7 +35,7 @@ import org.apache.hadoop.net.DNSToSwitchMapping; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.net.StaticMapping; -import org.apache.hadoop.ozone.ClientConfigBuilder; +import org.apache.hadoop.ozone.ClientConfigForTesting; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.client.ObjectStore; @@ -131,7 +131,7 @@ private void init() throws Exception { Collections.singleton(HddsUtils.getHostName(conf))).get(0), "/rack1"); - ClientConfigBuilder.newBuilder(StorageUnit.BYTES) + ClientConfigForTesting.newBuilder(StorageUnit.BYTES) .setBlockSize(blockSize) .setChunkSize(chunkSize) .setStreamBufferFlushSize(flushSize) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneAtRestEncryption.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneAtRestEncryption.java index 63527d73dbb3..919654d82a9b 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneAtRestEncryption.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneAtRestEncryption.java @@ -49,7 +49,7 @@ import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB; import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClientTestImpl; import org.apache.hadoop.hdds.utils.db.Table; -import org.apache.hadoop.ozone.ClientConfigBuilder; +import org.apache.hadoop.ozone.ClientConfigForTesting; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.client.BucketArgs; @@ -142,7 +142,7 @@ static void init() throws Exception { CertificateClientTestImpl certificateClientTest = new CertificateClientTestImpl(conf); - ClientConfigBuilder.newBuilder(StorageUnit.BYTES) + ClientConfigForTesting.newBuilder(StorageUnit.BYTES) .setBlockSize(BLOCK_SIZE) .setChunkSize(CHUNK_SIZE) .applyTo(conf); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptionFlushDelay.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptionFlushDelay.java index f5634ebac44e..c3e8a8d461b8 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptionFlushDelay.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptionFlushDelay.java @@ -35,7 +35,7 @@ import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.storage.BlockOutputStream; import org.apache.hadoop.hdds.utils.IOUtils; -import org.apache.hadoop.ozone.ClientConfigBuilder; +import org.apache.hadoop.ozone.ClientConfigForTesting; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.client.ObjectStore; @@ -100,7 +100,7 @@ public void init() throws Exception { conf.set(ScmConfigKeys.OZONE_SCM_PIPELINE_DESTROY_TIMEOUT, "5s"); conf.setQuietMode(false); - ClientConfigBuilder.newBuilder(StorageUnit.BYTES) + ClientConfigForTesting.newBuilder(StorageUnit.BYTES) .setBlockSize(blockSize) .setChunkSize(chunkSize) .setStreamBufferFlushSize(flushSize) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptions.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptions.java index 20fa0b6f8589..cd99382f300b 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptions.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientRetriesOnExceptions.java @@ -38,7 +38,7 @@ import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.storage.BlockOutputStream; import org.apache.hadoop.hdds.utils.IOUtils; -import org.apache.hadoop.ozone.ClientConfigBuilder; +import org.apache.hadoop.ozone.ClientConfigForTesting; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.client.ObjectStore; @@ -109,7 +109,7 @@ public void init() throws Exception { conf.set(ScmConfigKeys.OZONE_SCM_PIPELINE_DESTROY_TIMEOUT, "5s"); conf.setQuietMode(false); - ClientConfigBuilder.newBuilder(StorageUnit.BYTES) + ClientConfigForTesting.newBuilder(StorageUnit.BYTES) .setBlockSize(blockSize) .setChunkSize(chunkSize) .setStreamBufferFlushSize(flushSize) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java index 10926669aefc..a87d05321e27 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java @@ -62,7 +62,7 @@ import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; import org.apache.hadoop.hdds.utils.db.Table; -import org.apache.hadoop.ozone.ClientConfigBuilder; +import org.apache.hadoop.ozone.ClientConfigForTesting; import org.apache.hadoop.ozone.HddsDatanodeService; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OmUtils; @@ -204,7 +204,7 @@ static void startCluster(OzoneConfiguration conf) throws Exception { // for testZReadKeyWithUnhealthyContainerReplica. conf.set("ozone.scm.stale.node.interval", "10s"); - ClientConfigBuilder.newBuilder(StorageUnit.MB) + ClientConfigForTesting.newBuilder(StorageUnit.MB) .setDataStreamMinPacketSize(1) .applyTo(conf); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestWatchForCommit.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestWatchForCommit.java index b1af0e634844..f8e9b552e3ea 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestWatchForCommit.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestWatchForCommit.java @@ -48,7 +48,7 @@ import org.apache.hadoop.hdds.scm.storage.BlockOutputStream; import org.apache.hadoop.hdds.scm.storage.RatisBlockOutputStream; import org.apache.hadoop.hdds.utils.IOUtils; -import org.apache.hadoop.ozone.ClientConfigBuilder; +import org.apache.hadoop.ozone.ClientConfigForTesting; import org.apache.hadoop.ozone.HddsDatanodeService; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfigKeys; @@ -142,7 +142,7 @@ public void init() throws Exception { conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 30, TimeUnit.SECONDS); - ClientConfigBuilder.newBuilder(StorageUnit.BYTES) + ClientConfigForTesting.newBuilder(StorageUnit.BYTES) .setBlockSize(blockSize) .setChunkSize(chunkSize) .setStreamBufferFlushSize(flushSize) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestInputStreamBase.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestInputStreamBase.java index 0378cd30a565..9f5d04c56f94 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestInputStreamBase.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestInputStreamBase.java @@ -25,7 +25,7 @@ import org.apache.hadoop.hdds.scm.OzoneClientConfig; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.container.replication.ReplicationManager.ReplicationManagerConfiguration; -import org.apache.hadoop.ozone.ClientConfigBuilder; +import org.apache.hadoop.ozone.ClientConfigForTesting; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.container.common.impl.ContainerLayoutVersion; @@ -64,7 +64,7 @@ protected static MiniOzoneCluster newCluster( repConf.setInterval(Duration.ofSeconds(1)); conf.setFromObject(repConf); - ClientConfigBuilder.newBuilder(StorageUnit.BYTES) + ClientConfigForTesting.newBuilder(StorageUnit.BYTES) .setBlockSize(BLOCK_SIZE) .setChunkSize(CHUNK_SIZE) .setStreamBufferFlushSize(FLUSH_SIZE) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestECContainerRecovery.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestECContainerRecovery.java index b15817bdf718..e045b48bda96 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestECContainerRecovery.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestECContainerRecovery.java @@ -34,7 +34,7 @@ import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; import org.apache.hadoop.hdds.utils.IOUtils; -import org.apache.hadoop.ozone.ClientConfigBuilder; +import org.apache.hadoop.ozone.ClientConfigForTesting; import org.apache.hadoop.ozone.HddsDatanodeService; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfigKeys; @@ -141,7 +141,7 @@ public static void init() throws Exception { conf.setTimeDuration(HddsConfigKeys.HDDS_CONTAINER_REPORT_INTERVAL, 1, TimeUnit.SECONDS); - ClientConfigBuilder.newBuilder(StorageUnit.BYTES) + ClientConfigForTesting.newBuilder(StorageUnit.BYTES) .setBlockSize(blockSize) .setChunkSize(chunkSize) .setStreamBufferFlushSize(flushSize)