diff --git a/hadoop-ozone/dist/src/main/smoketest/freon/read-write-key.robot b/hadoop-ozone/dist/src/main/smoketest/freon/read-write-key.robot new file mode 100644 index 000000000000..0995f757f35d --- /dev/null +++ b/hadoop-ozone/dist/src/main/smoketest/freon/read-write-key.robot @@ -0,0 +1,53 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*** Settings *** +Documentation Test freon read/write key commands +Resource ../ozone-lib/freon.robot +Test Timeout 5 minutes + +*** Variables *** +${PREFIX} ${EMPTY} + + +*** Test Cases *** +Pre-generate 100 keys of size 1 byte each to Ozone + ${result} = Execute ozone freon ork -n 1 -t 10 -r 100 --size 1 -v voltest -b buckettest -p performanceTest + +Read 10 keys from pre-generated keys + ${keysCount} = BuiltIn.Set Variable 10 + ${result} = Execute ozone freon ockrw -n ${keysCount} -t 10 -r 100 -v voltest -b buckettest -p performanceTest + Should contain ${result} Successful executions: ${keysCount} + +Read 10 keys' metadata from pre-generated keys + ${keysCount} = BuiltIn.Set Variable 10 + ${result} = Execute ozone freon ockrw -n ${keysCount} -t 10 -m -r 100 -v voltest -b buckettest -p performanceTest + Should contain ${result} Successful executions: ${keysCount} + +Write 10 keys of size 1 byte each from key index 0 to 99 + ${keysCount} = BuiltIn.Set Variable 10 + ${size} = BuiltIn.Set Variable 1 + ${result} = Execute ozone freon ockrw -n ${keysCount} -t 10 --percentage-read 0 --size ${size} -r 100 -v voltest -b buckettest -p performanceTest + Should contain ${result} Successful executions: ${keysCount} + ${keyName} = Execute echo -n '1' | md5sum | head -c 7 + ${result} = Execute ozone sh key info /voltest/buckettest/performanceTest/${keyName} + Should contain ${result} \"dataSize\" : 1 + + +Run 90 % of read-key tasks and 10 % of write-key tasks for 10 keys from pre-generated keys + ${keysCount} = BuiltIn.Set Variable 10 + ${result} = Execute ozone freon ockrw -n ${keysCount} -t 10 --percentage-read 90 -r 100 -v voltest -b buckettest -p performanceTest + Should contain ${result} Successful executions: ${keysCount} + diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java index b6bb6055eefe..dfb52ca9add6 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/Freon.java @@ -68,6 +68,8 @@ OmBucketReadWriteFileOps.class, OmBucketReadWriteKeyOps.class, OmRPCLoadGenerator.class, + OzoneClientKeyReadWriteOps.class, + RangeKeysGenerator.class }, versionProvider = HddsVersionProvider.class, mixinStandardHelpOptions = true) diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/KeyGeneratorUtil.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/KeyGeneratorUtil.java new file mode 100644 index 000000000000..9773caa44070 --- /dev/null +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/KeyGeneratorUtil.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.ozone.freon; + +import org.apache.commons.codec.digest.DigestUtils; + +import java.util.function.Function; + +/** + * Utility class to generate key name from a given key index. + */ +public class KeyGeneratorUtil { + public static final String PURE_INDEX = "pureIndex"; + public static final String MD5 = "md5"; + public static final String FILE_DIR_SEPARATOR = "/"; + + public String generatePureIndexKeyName(int number) { + return String.valueOf(number); + } + public Function pureIndexKeyNameFunc() { + return number -> String.valueOf(number); + } + + public String generateMd5KeyName(int number) { + String encodedStr = DigestUtils.md5Hex(String.valueOf(number)); + return encodedStr.substring(0, 7); + } + + public Function md5KeyNameFunc() { + return number -> DigestUtils.md5Hex(String.valueOf(number)).substring(0, 7); + } + +} diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyReadWriteOps.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyReadWriteOps.java new file mode 100644 index 000000000000..88aec0763ce6 --- /dev/null +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyReadWriteOps.java @@ -0,0 +1,245 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.ozone.freon; + + +import com.codahale.metrics.Timer; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import org.apache.commons.lang3.RandomUtils; +import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.ozone.client.OzoneClient; +import org.apache.hadoop.ozone.client.OzoneKeyDetails; +import org.apache.hadoop.ozone.client.io.OzoneInputStream; +import org.apache.hadoop.ozone.client.io.OzoneOutputStream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import picocli.CommandLine; + +import java.io.IOException; +import java.util.concurrent.Callable; +import java.util.concurrent.ThreadLocalRandom; +import java.util.HashMap; + +import static org.apache.hadoop.ozone.freon.KeyGeneratorUtil.FILE_DIR_SEPARATOR; + +/** + * Ozone key generator/reader for performance test. + */ + +@CommandLine.Command(name = "ockrw", + aliases = "ozone-client-key-read-write-ops", + description = "Generate keys with a fixed name and ranges that can" + + " be written and read as sub-ranges from multiple clients.", + versionProvider = HddsVersionProvider.class, + mixinStandardHelpOptions = true, + showDefaultValues = true) +public class OzoneClientKeyReadWriteOps extends BaseFreonGenerator + implements Callable { + + @CommandLine.Option(names = {"-v", "--volume"}, + description = "Name of the volume which contains the test data. " + + "Will be created if missing.", + defaultValue = "ockrwvolume") + private String volumeName; + + @CommandLine.Option(names = {"-b", "--bucket"}, + description = "Name of the bucket which contains the test data.", + defaultValue = "ockrwbucket") + private String bucketName; + + @CommandLine.Option(names = {"-m", "--read-metadata-only"}, + description = "If only read key's metadata.", + defaultValue = "false") + private boolean readMetadataOnly; + + @CommandLine.Option(names = {"-s", "--start-index"}, + description = "Start index of keys of read/write operation." + + "This can allow adding keys incrementally or parallel from multiple" + + " clients. Example: Write keys 0-1000000 followed by " + + "keys 1000001-2000000.", + defaultValue = "0") + private int startIndex; + + @CommandLine.Option(names = {"-r", "--range"}, + description = "Range of read/write operations. This in co-ordination" + + " with --start-index can specify the range to read. " + + "Example: Read from --start-index 1000 and read --range 1000 keys.", + defaultValue = "1") + private int range; + + @CommandLine.Option(names = {"--size"}, + description = "Object size (in bytes) " + + "to read/write. If user sets a read size which is larger" + + " than the key size, it only reads bytes up to key size.", + defaultValue = "1") + private int objectSizeInBytes; + + @CommandLine.Option(names = {"--contiguous"}, + description = "By default, the keys are randomized lexically" + + " by calculating the md5 of the key name. If this option is set," + + " the keys are written lexically contiguously.", + defaultValue = "false") + private boolean keySorted; + + @CommandLine.Option(names = {"--percentage-read"}, + description = "Percentage of read tasks in mix workload." + + " The remainder of the percentage will writes to keys." + + " Example --percentage-read 90 will result in 10% writes.", + defaultValue = "100") + private int percentageRead; + + @CommandLine.Option( + names = "--om-service-id", + description = "OM Service ID" + ) + private String omServiceID = null; + + private Timer timer; + + private OzoneClient[] ozoneClients; + + private int clientCount; + + private byte[] keyContent; + + private static final Logger LOG = + LoggerFactory.getLogger(OzoneClientKeyReadWriteOps.class); + + /** + * Task type of read task, or write task. + */ + public enum TaskType { + READ_TASK, + WRITE_TASK + } + private KeyGeneratorUtil kg; + + + @Override + public Void call() throws Exception { + init(); + OzoneConfiguration ozoneConfiguration = createOzoneConfiguration(); + clientCount = getThreadNo(); + ozoneClients = new OzoneClient[clientCount]; + for (int i = 0; i < clientCount; i++) { + ozoneClients[i] = createOzoneClient(omServiceID, ozoneConfiguration); + } + + ensureVolumeAndBucketExist(ozoneClients[0], volumeName, bucketName); + + timer = getMetrics().timer("key-read-write"); + if (objectSizeInBytes >= 0) { + keyContent = RandomUtils.nextBytes(objectSizeInBytes); + } + if (kg == null) { + kg = new KeyGeneratorUtil(); + } + runTests(this::readWriteKeys); + + for (int i = 0; i < clientCount; i++) { + if (ozoneClients[i] != null) { + ozoneClients[i].close(); + } + } + return null; + } + + public void readWriteKeys(long counter) throws RuntimeException, IOException { + int clientIndex = (int)((counter) % clientCount); + TaskType taskType = decideReadOrWriteTask(); + String keyName = getKeyName(); + + timer.time(() -> { + try { + switch (taskType) { + case READ_TASK: + processReadTasks(keyName, ozoneClients[clientIndex]); + break; + case WRITE_TASK: + processWriteTasks(keyName, ozoneClients[clientIndex]); + break; + default: + break; + } + } catch (RuntimeException ex) { + LOG.error(ex.getMessage()); + throw ex; + } catch (IOException ex) { + LOG.error(ex.getMessage()); + throw new RuntimeException(ex.getMessage()); + } + + }); + } + @SuppressFBWarnings + public void processReadTasks(String keyName, OzoneClient client) + throws RuntimeException, IOException { + OzoneKeyDetails keyDetails = client.getProxy(). + getKeyDetails(volumeName, bucketName, keyName); + if (!readMetadataOnly) { + byte[] data = new byte[objectSizeInBytes]; + try (OzoneInputStream introStream = keyDetails.getContent()) { + introStream.read(data); + } catch (Exception ex) { + throw ex; + } + } + } + + public void processWriteTasks(String keyName, OzoneClient ozoneClient) + throws RuntimeException, IOException { + try (OzoneOutputStream out = + ozoneClient.getProxy().createKey(volumeName, bucketName, + keyName, objectSizeInBytes, null, new HashMap())) { + out.write(keyContent); + } catch (Exception ex) { + throw ex; + } + } + + public TaskType decideReadOrWriteTask() { + if (percentageRead == 100) { + return TaskType.READ_TASK; + } else if (percentageRead == 0) { + return TaskType.WRITE_TASK; + } + //mix workload + int tmp = ThreadLocalRandom.current().nextInt(1, 101); + if (tmp <= percentageRead) { + return TaskType.READ_TASK; + } else { + return TaskType.WRITE_TASK; + } + } + + public String getKeyName() { + StringBuilder keyNameSb = new StringBuilder(); + int randomIdxWithinRange = ThreadLocalRandom.current(). + nextInt(startIndex, startIndex + range); + + if (keySorted) { + keyNameSb.append(getPrefix()).append(FILE_DIR_SEPARATOR). + append(randomIdxWithinRange); + } else { + keyNameSb.append(getPrefix()).append(FILE_DIR_SEPARATOR). + append(kg.generateMd5KeyName(randomIdxWithinRange)); + } + return keyNameSb.toString(); + } + +} diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RangeKeysGenerator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RangeKeysGenerator.java new file mode 100644 index 000000000000..e14941028110 --- /dev/null +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RangeKeysGenerator.java @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.ozone.freon; + +import com.codahale.metrics.Timer; +import org.apache.commons.lang3.RandomUtils; +import org.apache.hadoop.hdds.cli.HddsVersionProvider; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.ozone.client.OzoneClient; +import org.apache.hadoop.ozone.client.io.OzoneOutputStream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import picocli.CommandLine; + +import java.util.concurrent.Callable; +import java.util.function.Function; +import java.util.HashMap; + +import static org.apache.hadoop.ozone.freon.KeyGeneratorUtil.PURE_INDEX; +import static org.apache.hadoop.ozone.freon.KeyGeneratorUtil.MD5; +import static org.apache.hadoop.ozone.freon.KeyGeneratorUtil.FILE_DIR_SEPARATOR; + +/** + * Ozone range keys generator for performance test. + */ +@CommandLine.Command(name = "ork", + description = "write range keys with the help of the ozone clients.", + versionProvider = HddsVersionProvider.class, + mixinStandardHelpOptions = true, + showDefaultValues = true) +public class RangeKeysGenerator extends BaseFreonGenerator + implements Callable { + + private static final Logger LOG = + LoggerFactory.getLogger(RangeKeysGenerator.class); + + @CommandLine.Option(names = {"-v", "--volume"}, + description = "Name of the volume which contains the test data. " + + "Will be created if missing.", + defaultValue = "ockrwvolume") + private String volumeName; + + @CommandLine.Option(names = {"-b", "--bucket"}, + description = "Name of the bucket which contains the test data.", + defaultValue = "ockrwbucket") + private String bucketName; + + @CommandLine.Option(names = {"-r", "--range-each-client-write"}, + description = "Write range for each client.", + defaultValue = "0") + private int range; + + @CommandLine.Option(names = {"-s", "--key-start-index"}, + description = "Start index of key.", + defaultValue = "0") + private int startIndex; + + + @CommandLine.Option(names = {"-k", "--key-encode"}, + description = "The algorithm to generate key names. " + + "Options are pureIndex, md5", + defaultValue = "md5") + private String encodeFormat; + + @CommandLine.Option(names = {"-g", "--size"}, + description = "Generated object size (in bytes) " + + "to be written.", + defaultValue = "1") + private int objectSizeInBytes; + + @CommandLine.Option( + names = "--om-service-id", + description = "OM Service ID" + ) + private String omServiceID = null; + private KeyGeneratorUtil kg; + private int clientCount; + private OzoneClient[] ozoneClients; + private byte[] keyContent; + private Timer timer; + + + @Override + public Void call() throws Exception { + init(); + OzoneConfiguration ozoneConfiguration = createOzoneConfiguration(); + clientCount = getThreadNo(); + ozoneClients = new OzoneClient[clientCount]; + for (int i = 0; i < clientCount; i++) { + ozoneClients[i] = createOzoneClient(omServiceID, ozoneConfiguration); + } + + ensureVolumeAndBucketExist(ozoneClients[0], volumeName, bucketName); + if (objectSizeInBytes >= 0) { + keyContent = RandomUtils.nextBytes(objectSizeInBytes); + } + timer = getMetrics().timer("key-read-write"); + + kg = new KeyGeneratorUtil(); + runTests(this::generateRangeKeys); + for (int i = 0; i < clientCount; i++) { + if (ozoneClients[i] != null) { + ozoneClients[i].close(); + } + } + + return null; + } + + public void generateRangeKeys(long count) throws Exception { + int clientIndex = (int)(count % clientCount); + OzoneClient client = ozoneClients[clientIndex]; + int start = startIndex + (int)count * range; + int end = start + range; + + timer.time(() -> { + switch (encodeFormat) { + case PURE_INDEX: + loopRunner(kg.pureIndexKeyNameFunc(), client, start, end); + break; + case MD5: + loopRunner(kg.md5KeyNameFunc(), client, start, end); + break; + default: + loopRunner(kg.md5KeyNameFunc(), client, start, end); + break; + } + return null; + }); + } + + + public void loopRunner(Function keyNameGeneratorfunc, + OzoneClient client, int start, int end) + throws Exception { + String keyName; + for (int i = start; i < end + 1; i++) { + keyName = getPrefix() + FILE_DIR_SEPARATOR + + keyNameGeneratorfunc.apply(i); + try (OzoneOutputStream out = client.getProxy(). + createKey(volumeName, bucketName, keyName, + objectSizeInBytes, null, new HashMap())) { + out.write(keyContent); + } + } + } +}