HDDS-7199. Implement new mix workload Read/Write Freon command which meets specific test requirements

hadoop-ozone/dist/src/main/smoketest/freon/read-write-key.robot

@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+Documentation       Test freon read/write key commands
+Resource            ../ozone-lib/freon.robot
+Test Timeout        5 minutes
+
+*** Variables ***
+${PREFIX}    ${EMPTY}
+
+*** Test Cases ***
+Pre-generate 100 keys of size 1 byte each to Ozone
+    ${result} =        Execute          ozone freon ockrw -n 1 -t 10 -r -i 0 -j 99 -g 1 -v voltest -b buckettest -p performanceTest
+
+Read 10 keys from pre-generated keys
+    ${result} =        Execute          ozone freon ockrw -n 10 -t 10 -s 0 -e 9 -v voltest -b buckettest -p performanceTest
+
+Read 10 keys' metadata from pre-generated keys
+    ${result} =        Execute          ozone freon ockrw -n 10 -t 10 -m -s 0 -e 9 -v voltest -b buckettest -p performanceTest
+
+Write 10 keys of size 1 byte each to replace parts of the pre-generated keys
+    ${result} =        Execute          ozone freon ockrw -n 10 -t 10 -i 0 -j 9 -g 1 -v voltest -b buckettest -p performanceTest
+
+Run 90 % of read-key tasks and 10 % of write-key tasks from pre-generated keys
+    ${result} =        Execute          ozone freon ockrw -n 10 -t 10 -x --percentage-read 90 -s 0 -e 9 -i 10 -j 19 -g 1 -v voltest -b buckettest -p performanceTest
+

hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java

@@ -68,6 +68,8 @@
         OmBucketReadWriteFileOps.class,
         OmBucketReadWriteKeyOps.class,
         OmRPCLoadGenerator.class,
+        OzoneClientKeyReadWriteOps.class,
+        RangeKeysGenerator.class
     },
     versionProvider = HddsVersionProvider.class,
     mixinStandardHelpOptions = true)

hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/KeyGeneratorUtil.java

@@ -0,0 +1,68 @@
+package org.apache.hadoop.ozone.freon;
+
+import org.apache.commons.codec.digest.DigestUtils;
+
+import java.util.function.Function;
+
+/**
+ * Utility class to generate key name from a given key index.
+ */
+public class KeyGeneratorUtil {
+  public static final String PURE_INDEX = "pureIndex";
+  public static final String MD5 = "md5";
+  public static final String SIMPLE_HASH = "simpleHash";
+  public static final String FILE_DIR_SEPARATOR = "/";
+  private char[] saltCharArray =
+      {'m', 'D', 'E', 'T', 't', 'q', 'c', 'j', 'X', '8'};
+
+  public String generatePureIndexKeyName(int number) {
+    return String.valueOf(number);
+  }
+  public Function<Integer, String> pureIndexKeyNameFunc() {
+    return number -> String.valueOf(number);
+  }
+
+  public String generateMd5KeyName(int number) {
+    String encodedStr = DigestUtils.md5Hex(String.valueOf(number));
+    return encodedStr.substring(0, 7);
+  }
+
+  public Function<Integer, String> md5KeyNameFunc() {
+    return number -> DigestUtils.md5Hex(String.valueOf(number)).substring(0, 7);
+  }
+
+  public String generateSimpleHashKeyName(int number) {
+    if (number == 0) {
+      return String.valueOf(saltCharArray[number]);
+    }
+
+    StringBuilder keyName = new StringBuilder();
+    while (number > 0) {
+      keyName.append(saltCharArray[number % 10]);
+      number /= 10;
+    }
+    return keyName.toString();
+  }
+
+  public Function<Integer, String> simpleHashKeyNameFunc() {
+    return number -> {
+      if (number == 0) {
+        return String.valueOf(saltCharArray[number]);
+      }
+
+      StringBuilder keyName = new StringBuilder();
+      while (number > 0) {
+        keyName.append(saltCharArray[number % 10]);
+        number /= 10;
+      }
+      return keyName.toString();
+    };
+  }
+
+  public char[] getSaltCharArray() {
+    return saltCharArray;
+  }
+  public void setSaltCharArray(char[] saltCharArray) {
+    this.saltCharArray = saltCharArray;
+  }
+}

hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyReadWriteOps.java

@@ -0,0 +1,258 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership.  The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.freon;
+
+
+import com.codahale.metrics.Timer;
+import org.apache.commons.lang3.RandomUtils;
+import org.apache.hadoop.hdds.cli.HddsVersionProvider;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.client.io.OzoneInputStream;
+import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import picocli.CommandLine;
+
+import java.util.concurrent.Callable;
+import java.util.concurrent.ThreadLocalRandom;
+
+import static org.apache.hadoop.ozone.freon.KeyGeneratorUtil.FILE_DIR_SEPARATOR;
+
+/**
+ * Ozone key generator/reader for performance test.
+ */
+
+@CommandLine.Command(name = "ockrw",
+        aliases = "ozone-client-key-read-write-ops",
+        description = "Read and write keys with the help of the ozone clients.",
+        versionProvider = HddsVersionProvider.class,
+        mixinStandardHelpOptions = true,
+        showDefaultValues = true)
+public class OzoneClientKeyReadWriteOps extends BaseFreonGenerator
+        implements Callable<Void> {
+
+  @CommandLine.Option(names = {"-v", "--volume"},
+          description = "Name of the volume which contains the test data. " +
+                  "Will be created if missing.",
+          defaultValue = "vol1")
+  private String volumeName;
+
+  @CommandLine.Option(names = {"-b", "--bucket"},
+          description = "Name of the bucket which contains the test data.",
+          defaultValue = "bucket1")
+  private String bucketName;
+
+  @CommandLine.Option(names = {"-m", "--read-metadata-only"},
+          description = "If only read key's metadata. " +
+                  "Supported values are Y, F.",
+          defaultValue = "false")
+  private boolean readMetadataOnly;
+
+  @CommandLine.Option(names = {"-r", "--range-client-read"},
+          description = "range of read operation of each client.",
+          defaultValue = "0")
+  private int readRange;
+
+
+  @CommandLine.Option(names = {"-w", "--range-client-write"},
+          description = "range of write operation of each client.",
+          defaultValue = "0")
+  private int writeRange;
+
+//    @CommandLine.Option(names = {"-e", "--end-index-each-client-read"},
+//          description = "end-index of each client's read operation.",
+//          defaultValue = "0")
+//  private int endIndexForRead;
+
+//  @CommandLine.Option(names = {"-j", "--end-index-each-client-write"},
+//          description = "end-index of each client's write operation.",
+//          defaultValue = "0")
+//  private int endIndexForWrite;
+
+  @CommandLine.Option(names = {"-g", "--size"},
+          description = "Generated data size (in bytes) of " +
+                  "each key/file to be " +
+                  "written.",
+          defaultValue = "256")
+  private int writeSizeInBytes;
+
+  @CommandLine.Option(names = {"-k", "--keySorted"},
+          description = "Generated sorted key or not. The key name " +
+                  "will be generated via md5 hash if choose " +
+                  "to use unsorted key.",
+          defaultValue = "false")
+  private boolean keySorted;
+
+  @CommandLine.Option(names = {"-x", "--mix-workload"},
+          description = "Set to True if you would like to " +
+                  "generate mix workload (Read and Write).",
+          defaultValue = "false")
+  private boolean isMixWorkload;
+
+  @CommandLine.Option(names = {"--percentage-read"},
+          description = "Percentage of read tasks in mix workload.",
+          defaultValue = "0")
+  private int percentageRead;
+
+  @CommandLine.Option(names = {"--clients"},
+          description =
+                  "Number of clients, defaults 1.",
+          defaultValue = "1")
+  private int clientsCount = 1;
+
+  @CommandLine.Option(
+          names = "--om-service-id",
+          description = "OM Service ID"
+  )
+  private String omServiceID = null;
+
+  private Timer timer;
+
+  private OzoneClient[] rpcClients;
+
+  private byte[] keyContent;
+
+  private static final Logger LOG =
+          LoggerFactory.getLogger(OzoneClientKeyReadWriteOps.class);
+
+  private final String readTask = "READ_TASK";
+  private final String writeTask = "WRITE_TASK";
+  private KeyGeneratorUtil kg;
+
+  @Override
+  public Void call() throws Exception {
+    init();
+    OzoneConfiguration ozoneConfiguration = createOzoneConfiguration();
+    rpcClients = new OzoneClient[clientsCount];
+    for (int i = 0; i < clientsCount; i++) {
+      rpcClients[i] = createOzoneClient(omServiceID, ozoneConfiguration);
+    }
+
+    ensureVolumeAndBucketExist(rpcClients[0], volumeName, bucketName);
+    timer = getMetrics().timer("key-read-write");
+    if (writeSizeInBytes >= 0) {
+      keyContent = RandomUtils.nextBytes(writeSizeInBytes);
+    }
+    if (kg == null) {
+      kg = new KeyGeneratorUtil();
+    }
+    runTests(this::readWriteKeys);
+
+    for (int i = 0; i < clientsCount; i++) {
+      if (rpcClients[i] != null) {
+        rpcClients[i].close();
+      }
+    }
+    return null;
+  }
+
+  public void readWriteKeys(long counter) throws Exception {
+    int clientIndex = (int)(counter % clientsCount);
+    OzoneClient client = rpcClients[clientIndex];
+    String operationType = decideReadOrWriteTask();
+    String keyName = getKeyName(operationType, clientIndex);
+    timer.time(() -> {
+      try {
+        switch (operationType) {
+        case readTask:
+          processReadTasks(keyName, client);
+          break;
+        case writeTask:
+          processWriteTasks(keyName, client);
+          break;
+        default:
+          break;
+        }
+      } catch (Exception ex) {
+        LOG.error(ex.getMessage());
+      }
+    });
+  }
+
+  public void processReadTasks(String keyName, OzoneClient client)
+          throws Exception {
+    OzoneBucket ozbk = client.getObjectStore().getVolume(volumeName)
+            .getBucket(bucketName);
+
+    if (readMetadataOnly) {
+      ozbk.getKey(keyName);
+    } else {
+      byte[] data = new byte[writeSizeInBytes];
+      try (OzoneInputStream introStream = ozbk.readKey(keyName)) {
+        introStream.read(data);
+      }
+    }
+  }
+  public void processWriteTasks(String keyName, OzoneClient client)
+          throws Exception {
+    OzoneBucket ozbk = client.getObjectStore().getVolume(volumeName)
+            .getBucket(bucketName);
+
+    try (OzoneOutputStream out = ozbk.createKey(keyName, writeSizeInBytes)) {
+      out.write(keyContent);
+      out.flush();
+    }
+  }
+  public String decideReadOrWriteTask() {
+    if (!isMixWorkload) {
+      if (readRange > 0) {
+        return readTask;
+      } else if (writeRange > 0) {
+        return writeTask;
+      }
+    }
+    //mix workload
+    int tmp = ThreadLocalRandom.current().nextInt(100) + 1; // 1 ~ 100
+    if (tmp < percentageRead) {
+      return readTask;
+    } else {
+      return writeTask;
+    }
+  }
+
+  public String getKeyName(String operationType, int counter) {
+    int startIdx, endIdx;
+    switch (operationType) {
+    case readTask:
+      startIdx = counter * readRange;
+      endIdx = startIdx + readRange;
+      break;
+    case writeTask:
+      startIdx = counter * writeRange;
+      endIdx = startIdx + writeRange;
+      break;
+    default:
+      startIdx = 0;
+      endIdx = 0;
+      break;
+    }
+    StringBuilder keyNameSb = new StringBuilder();
+    int randomIdxWithinRange = ThreadLocalRandom.current().
+            nextInt(endIdx + 1 - startIdx) + startIdx;
+    if (keySorted) {
+      keyNameSb.append(getPrefix()).append(FILE_DIR_SEPARATOR).
+              append(randomIdxWithinRange);
+    } else {
+      keyNameSb.append(getPrefix()).append(FILE_DIR_SEPARATOR).
+              append(kg.generateSimpleHashKeyName(randomIdxWithinRange));
+    }
+    return keyNameSb.toString();
+  }
+
+}