diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/OmBucketTestUtils.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/OmBucketTestUtils.java new file mode 100644 index 000000000000..2ff7effa0bc5 --- /dev/null +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/OmBucketTestUtils.java @@ -0,0 +1,215 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.freon; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.UUID; +import org.apache.hadoop.ozone.client.BucketArgs; +import org.apache.hadoop.ozone.om.lock.OMLockMetrics; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Util Class for OmBucketReadWriteKeyTests. + */ +public final class OmBucketTestUtils { + + private static final Logger LOG = LoggerFactory.getLogger(OmBucketTestUtils.class); + + private OmBucketTestUtils() { + } + + static class ParameterBuilder { + + private static final String BUCKET_NAME = "bucket1"; + + private final String volumeName = "vol-" + UUID.randomUUID(); + private final BucketArgs.Builder bucketArgs = BucketArgs.newBuilder(); + private String prefixFilePath = "/dir1/dir2"; + private int countForRead = 100; + private int countForWrite = 10; + private String dataSize = "256B"; + private int bufferSize = 64; + private int length = 10; + private int totalThreadCount = 100; + private int readThreadPercentage = 90; + private int numOfReadOperations = 50; + private int numOfWriteOperations = 10; + + private String description; + + int getExpectedWriteCount() { + int readThreadCount = (getReadThreadPercentage() * getTotalThreadCount()) / 100; + int writeThreadCount = getTotalThreadCount() - readThreadCount; + return writeThreadCount * getCountForWrite() * getNumOfWriteOperations(); + } + + ParameterBuilder setPrefixFilePath(String newValue) { + prefixFilePath = newValue; + return this; + } + + ParameterBuilder setCountForRead(int newValue) { + countForRead = newValue; + return this; + } + + ParameterBuilder setCountForWrite(int newValue) { + countForWrite = newValue; + return this; + } + + ParameterBuilder setDataSize(String newValue) { + dataSize = newValue; + return this; + } + + ParameterBuilder setBufferSize(int newValue) { + bufferSize = newValue; + return this; + } + + ParameterBuilder setLength(int newValue) { + length = newValue; + return this; + } + + ParameterBuilder setTotalThreadCount(int newValue) { + totalThreadCount = newValue; + return this; + } + + ParameterBuilder setReadThreadPercentage(int newValue) { + readThreadPercentage = newValue; + return this; + } + + ParameterBuilder setNumOfReadOperations(int newValue) { + numOfReadOperations = newValue; + return this; + } + + ParameterBuilder setNumOfWriteOperations(int newValue) { + numOfWriteOperations = newValue; + return this; + } + + ParameterBuilder setDescription(String newValue) { + description = newValue; + return this; + } + + public String getVolumeName() { + return volumeName; + } + + public String getBucketName() { + return BUCKET_NAME; + } + + public String getPrefixFilePath() { + return prefixFilePath; + } + + public BucketArgs.Builder getBucketArgs() { + return bucketArgs; + } + + public int getBufferSize() { + return bufferSize; + } + + public String getDataSize() { + return dataSize; + } + + public int getCountForRead() { + return countForRead; + } + + public int getCountForWrite() { + return countForWrite; + } + + public int getLength() { + return length; + } + + public int getTotalThreadCount() { + return totalThreadCount; + } + + public int getReadThreadPercentage() { + return readThreadPercentage; + } + + public int getNumOfReadOperations() { + return numOfReadOperations; + } + + public int getNumOfWriteOperations() { + return numOfWriteOperations; + } + + public String getDescription() { + return description; + } + + @Override + public String toString() { + return String.format("%s", description); + } + } + + public static void verifyOMLockMetrics(OMLockMetrics omLockMetrics) { + String readLockWaitingTimeMsStat = + omLockMetrics.getReadLockWaitingTimeMsStat(); + LOG.info("Read Lock Waiting Time Stat: " + readLockWaitingTimeMsStat); + LOG.info("Longest Read Lock Waiting Time (ms): " + + omLockMetrics.getLongestReadLockWaitingTimeMs()); + int readWaitingSamples = + Integer.parseInt(readLockWaitingTimeMsStat.split(" ")[2]); + assertThat(readWaitingSamples).isGreaterThan(0); + + String readLockHeldTimeMsStat = omLockMetrics.getReadLockHeldTimeMsStat(); + LOG.info("Read Lock Held Time Stat: " + readLockHeldTimeMsStat); + LOG.info("Longest Read Lock Held Time (ms): " + + omLockMetrics.getLongestReadLockHeldTimeMs()); + int readHeldSamples = + Integer.parseInt(readLockHeldTimeMsStat.split(" ")[2]); + assertThat(readHeldSamples).isGreaterThan(0); + + String writeLockWaitingTimeMsStat = + omLockMetrics.getWriteLockWaitingTimeMsStat(); + LOG.info("Write Lock Waiting Time Stat: " + writeLockWaitingTimeMsStat); + LOG.info("Longest Write Lock Waiting Time (ms): " + + omLockMetrics.getLongestWriteLockWaitingTimeMs()); + int writeWaitingSamples = + Integer.parseInt(writeLockWaitingTimeMsStat.split(" ")[2]); + assertThat(writeWaitingSamples).isGreaterThan(0); + + String writeLockHeldTimeMsStat = omLockMetrics.getWriteLockHeldTimeMsStat(); + LOG.info("Write Lock Held Time Stat: " + writeLockHeldTimeMsStat); + LOG.info("Longest Write Lock Held Time (ms): " + + omLockMetrics.getLongestWriteLockHeldTimeMs()); + int writeHeldSamples = + Integer.parseInt(writeLockHeldTimeMsStat.split(" ")[2]); + assertThat(writeHeldSamples).isGreaterThan(0); + } +} diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOmBucketReadWriteFileOps.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOmBucketReadWriteFileOps.java index 369f577ce4db..38c776644052 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOmBucketReadWriteFileOps.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOmBucketReadWriteFileOps.java @@ -17,22 +17,20 @@ package org.apache.hadoop.ozone.freon; +import static org.apache.hadoop.ozone.freon.OmBucketTestUtils.verifyOMLockMetrics; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY; -import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import java.net.URI; import java.util.Arrays; import java.util.List; -import java.util.UUID; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.TestDataUtil; -import org.apache.hadoop.ozone.client.BucketArgs; import org.apache.hadoop.ozone.client.OzoneClient; -import org.apache.hadoop.ozone.om.lock.OMLockMetrics; +import org.apache.hadoop.ozone.freon.OmBucketTestUtils.ParameterBuilder; import org.apache.ozone.test.NonHATests; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; @@ -54,7 +52,8 @@ static List parameters() { .setNumOfWriteOperations(5) .setReadThreadPercentage(70) .setCountForRead(10) - .setCountForWrite(5), + .setCountForWrite(5) + .setDescription("default"), new ParameterBuilder() .setPrefixFilePath("/dir1/dir2/dir3") .setTotalThreadCount(10) @@ -63,7 +62,8 @@ static List parameters() { .setReadThreadPercentage(80) .setBufferSize(128) .setCountForRead(10) - .setCountForWrite(5), + .setCountForWrite(5) + .setDescription("with increased buffer and read thread %"), new ParameterBuilder() .setPrefixFilePath("/") .setTotalThreadCount(15) @@ -71,7 +71,8 @@ static List parameters() { .setNumOfWriteOperations(3) .setDataSize("128B") .setCountForRead(5) - .setCountForWrite(3), + .setCountForWrite(3) + .setDescription("with 128 byte object"), new ParameterBuilder() .setPrefixFilePath("/dir1/") .setTotalThreadCount(10) @@ -80,13 +81,15 @@ static List parameters() { .setCountForRead(5) .setCountForWrite(3) .setDataSize("64B") - .setBufferSize(16), + .setBufferSize(16) + .setDescription("with 64 byte object"), new ParameterBuilder() .setPrefixFilePath("/dir1/dir2/dir3") .setTotalThreadCount(10) .setNumOfReadOperations(5) .setNumOfWriteOperations(0) - .setCountForRead(5), + .setCountForRead(5) + .setDescription("pure reads"), new ParameterBuilder() .setLength(64) .setPrefixFilePath("/dir1/dir2/dir3/dir4") @@ -95,10 +98,11 @@ static List parameters() { .setNumOfWriteOperations(5) .setCountForRead(0) .setCountForWrite(5) + .setDescription("pure writes") ); } - @ParameterizedTest + @ParameterizedTest(name = "{0}") @MethodSource("parameters") void testOmBucketReadWriteFileOps(ParameterBuilder parameterBuilder) throws Exception { try (OzoneClient client = cluster().newClient()) { @@ -165,166 +169,4 @@ private void verifyFileCreation(int expectedCount, FileStatus[] fileStatuses, } assertEquals(expectedCount, actual, "Mismatch Count!"); } - - private void verifyOMLockMetrics(OMLockMetrics omLockMetrics) { - String readLockWaitingTimeMsStat = - omLockMetrics.getReadLockWaitingTimeMsStat(); - LOG.info("Read Lock Waiting Time Stat: " + readLockWaitingTimeMsStat); - LOG.info("Longest Read Lock Waiting Time (ms): " + - omLockMetrics.getLongestReadLockWaitingTimeMs()); - int readWaitingSamples = - Integer.parseInt(readLockWaitingTimeMsStat.split(" ")[2]); - assertThat(readWaitingSamples).isPositive(); - - String readLockHeldTimeMsStat = omLockMetrics.getReadLockHeldTimeMsStat(); - LOG.info("Read Lock Held Time Stat: " + readLockHeldTimeMsStat); - LOG.info("Longest Read Lock Held Time (ms): " + - omLockMetrics.getLongestReadLockHeldTimeMs()); - int readHeldSamples = - Integer.parseInt(readLockHeldTimeMsStat.split(" ")[2]); - assertThat(readHeldSamples).isPositive(); - - String writeLockWaitingTimeMsStat = - omLockMetrics.getWriteLockWaitingTimeMsStat(); - LOG.info("Write Lock Waiting Time Stat: " + writeLockWaitingTimeMsStat); - LOG.info("Longest Write Lock Waiting Time (ms): " + - omLockMetrics.getLongestWriteLockWaitingTimeMs()); - int writeWaitingSamples = - Integer.parseInt(writeLockWaitingTimeMsStat.split(" ")[2]); - assertThat(writeWaitingSamples).isPositive(); - - String writeLockHeldTimeMsStat = omLockMetrics.getWriteLockHeldTimeMsStat(); - LOG.info("Write Lock Held Time Stat: " + writeLockHeldTimeMsStat); - LOG.info("Longest Write Lock Held Time (ms): " + - omLockMetrics.getLongestWriteLockHeldTimeMs()); - int writeHeldSamples = - Integer.parseInt(writeLockHeldTimeMsStat.split(" ")[2]); - assertThat(writeHeldSamples).isPositive(); - } - - static class ParameterBuilder { - - private static final String BUCKET_NAME = "bucket1"; - - private final String volumeName = "vol-" + UUID.randomUUID(); - private final BucketArgs.Builder bucketArgs = BucketArgs.newBuilder(); - private String prefixFilePath = "/dir1/dir2"; - private int countForRead = 100; - private int countForWrite = 10; - private String dataSize = "256B"; - private int bufferSize = 64; - private int length = 10; - private int totalThreadCount = 100; - private int readThreadPercentage = 90; - private int numOfReadOperations = 50; - private int numOfWriteOperations = 10; - - int getExpectedWriteCount() { - int readThreadCount = (getReadThreadPercentage() * getTotalThreadCount()) / 100; - int writeThreadCount = getTotalThreadCount() - readThreadCount; - return writeThreadCount * getCountForWrite() * getNumOfWriteOperations(); - } - - ParameterBuilder setPrefixFilePath(String newValue) { - prefixFilePath = newValue; - return this; - } - - ParameterBuilder setCountForRead(int newValue) { - countForRead = newValue; - return this; - } - - ParameterBuilder setCountForWrite(int newValue) { - countForWrite = newValue; - return this; - } - - ParameterBuilder setDataSize(String newValue) { - dataSize = newValue; - return this; - } - - ParameterBuilder setBufferSize(int newValue) { - bufferSize = newValue; - return this; - } - - ParameterBuilder setLength(int newValue) { - length = newValue; - return this; - } - - ParameterBuilder setTotalThreadCount(int newValue) { - totalThreadCount = newValue; - return this; - } - - ParameterBuilder setReadThreadPercentage(int newValue) { - readThreadPercentage = newValue; - return this; - } - - ParameterBuilder setNumOfReadOperations(int newValue) { - numOfReadOperations = newValue; - return this; - } - - ParameterBuilder setNumOfWriteOperations(int newValue) { - numOfWriteOperations = newValue; - return this; - } - - public String getVolumeName() { - return volumeName; - } - - public String getBucketName() { - return BUCKET_NAME; - } - - public String getPrefixFilePath() { - return prefixFilePath; - } - - public BucketArgs.Builder getBucketArgs() { - return bucketArgs; - } - - public int getBufferSize() { - return bufferSize; - } - - public String getDataSize() { - return dataSize; - } - - public int getCountForRead() { - return countForRead; - } - - public int getCountForWrite() { - return countForWrite; - } - - public int getLength() { - return length; - } - - public int getTotalThreadCount() { - return totalThreadCount; - } - - public int getReadThreadPercentage() { - return readThreadPercentage; - } - - public int getNumOfReadOperations() { - return numOfReadOperations; - } - - public int getNumOfWriteOperations() { - return numOfWriteOperations; - } - } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOmBucketReadWriteKeyOps.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOmBucketReadWriteKeyOps.java index a837cb785104..dc6c3438d4bc 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOmBucketReadWriteKeyOps.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOmBucketReadWriteKeyOps.java @@ -17,8 +17,8 @@ package org.apache.hadoop.ozone.freon; +import static org.apache.hadoop.ozone.freon.OmBucketTestUtils.verifyOMLockMetrics; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY; -import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import java.io.IOException; @@ -30,9 +30,8 @@ import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneKey; -import org.apache.hadoop.ozone.freon.TestOmBucketReadWriteFileOps.ParameterBuilder; +import org.apache.hadoop.ozone.freon.OmBucketTestUtils.ParameterBuilder; import org.apache.hadoop.ozone.om.helpers.BucketLayout; -import org.apache.hadoop.ozone.om.lock.OMLockMetrics; import org.apache.hadoop.util.Time; import org.apache.ozone.test.NonHATests; import org.junit.jupiter.api.AfterEach; @@ -70,7 +69,8 @@ static List parameters() { .setNumOfWriteOperations(5) .setReadThreadPercentage(80) .setCountForRead(10) - .setCountForWrite(5), + .setCountForWrite(5) + .setDescription("default"), new ParameterBuilder() .setLength(32) .setTotalThreadCount(10) @@ -78,13 +78,15 @@ static List parameters() { .setNumOfWriteOperations(5) .setReadThreadPercentage(70) .setCountForRead(10) - .setCountForWrite(5), + .setCountForWrite(5) + .setDescription("with increased length of 32"), new ParameterBuilder() .setTotalThreadCount(15) .setNumOfReadOperations(5) .setNumOfWriteOperations(3) .setCountForRead(5) - .setCountForWrite(3), + .setCountForWrite(3) + .setDescription("with 0 byte objects"), new ParameterBuilder() .setTotalThreadCount(10) .setNumOfReadOperations(5) @@ -92,22 +94,25 @@ static List parameters() { .setCountForRead(5) .setCountForWrite(3) .setDataSize("64B") - .setBufferSize(16), + .setBufferSize(16) + .setDescription("with 64 byte object"), new ParameterBuilder() .setTotalThreadCount(10) .setNumOfReadOperations(5) .setNumOfWriteOperations(0) - .setCountForRead(5), + .setCountForRead(5) + .setDescription("pure reads"), new ParameterBuilder() .setTotalThreadCount(20) .setNumOfReadOperations(0) .setNumOfWriteOperations(5) .setCountForRead(0) .setCountForWrite(5) + .setDescription("pure writes") ); } - @ParameterizedTest(name = "Filesystem Paths Enabled: {0}") + @ParameterizedTest(name = "{0}") @MethodSource("parameters") void testOmBucketReadWriteKeyOps(ParameterBuilder parameterBuilder) throws Exception { OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(client, @@ -154,41 +159,4 @@ private void verifyKeyCreation(int expectedCount, OzoneBucket bucket, } assertEquals(expectedCount, actual, "Mismatch Count!"); } - - private void verifyOMLockMetrics(OMLockMetrics omLockMetrics) { - String readLockWaitingTimeMsStat = - omLockMetrics.getReadLockWaitingTimeMsStat(); - LOG.info("Read Lock Waiting Time Stat: " + readLockWaitingTimeMsStat); - LOG.info("Longest Read Lock Waiting Time (ms): " + - omLockMetrics.getLongestReadLockWaitingTimeMs()); - int readWaitingSamples = - Integer.parseInt(readLockWaitingTimeMsStat.split(" ")[2]); - assertThat(readWaitingSamples).isGreaterThan(0); - - String readLockHeldTimeMsStat = omLockMetrics.getReadLockHeldTimeMsStat(); - LOG.info("Read Lock Held Time Stat: " + readLockHeldTimeMsStat); - LOG.info("Longest Read Lock Held Time (ms): " + - omLockMetrics.getLongestReadLockHeldTimeMs()); - int readHeldSamples = - Integer.parseInt(readLockHeldTimeMsStat.split(" ")[2]); - assertThat(readHeldSamples).isGreaterThan(0); - - String writeLockWaitingTimeMsStat = - omLockMetrics.getWriteLockWaitingTimeMsStat(); - LOG.info("Write Lock Waiting Time Stat: " + writeLockWaitingTimeMsStat); - LOG.info("Longest Write Lock Waiting Time (ms): " + - omLockMetrics.getLongestWriteLockWaitingTimeMs()); - int writeWaitingSamples = - Integer.parseInt(writeLockWaitingTimeMsStat.split(" ")[2]); - assertThat(writeWaitingSamples).isGreaterThan(0); - - String writeLockHeldTimeMsStat = omLockMetrics.getWriteLockHeldTimeMsStat(); - LOG.info("Write Lock Held Time Stat: " + writeLockHeldTimeMsStat); - LOG.info("Longest Write Lock Held Time (ms): " + - omLockMetrics.getLongestWriteLockHeldTimeMs()); - int writeHeldSamples = - Integer.parseInt(writeLockHeldTimeMsStat.split(" ")[2]); - assertThat(writeHeldSamples).isGreaterThan(0); - } - }