From 6b705764e7c516b15ec75b505b0e9f7f3a032f4a Mon Sep 17 00:00:00 2001 From: Mohammad Arafat Khan Date: Tue, 8 Feb 2022 12:31:18 +0530 Subject: [PATCH 1/4] HDDS-4126. Freon key generator should support >2g files. --- .../java/org/apache/hadoop/ozone/freon/HadoopFsGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/HadoopFsGenerator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/HadoopFsGenerator.java index 1f0c3e92ab84..755c57ac92f3 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/HadoopFsGenerator.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/HadoopFsGenerator.java @@ -51,7 +51,7 @@ public class HadoopFsGenerator extends BaseFreonGenerator @Option(names = {"-s", "--size"}, description = "Size of the generated files (in bytes)", defaultValue = "10240") - private int fileSize; + private long fileSize; @Option(names = {"--buffer"}, description = "Size of buffer used store the generated key content", From 183c0143d4da38919bd44d68ab9aafffee82e5f4 Mon Sep 17 00:00:00 2001 From: Mohammad Arafat Khan Date: Wed, 9 Feb 2022 16:33:55 +0530 Subject: [PATCH 2/4] HDDS-4126. Freon key generator should support >2g files. --- .../org/apache/hadoop/ozone/freon/HadoopDirTreeGenerator.java | 2 +- .../org/apache/hadoop/ozone/freon/StreamingGenerator.java | 2 +- .../ozone/freon/containergenerator/GeneratorDatanode.java | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/HadoopDirTreeGenerator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/HadoopDirTreeGenerator.java index b0937d0ba61b..53cc34cc51d6 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/HadoopDirTreeGenerator.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/HadoopDirTreeGenerator.java @@ -70,7 +70,7 @@ public class HadoopDirTreeGenerator extends BaseFreonGenerator "written in each directory. Full name --fileSize will be removed " + "in later versions.", defaultValue = "4096") - private int fileSizeInBytes; + private long fileSizeInBytes; @Option(names = {"-b", "--buffer"}, description = "Size of buffer used to generated the file content.", diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/StreamingGenerator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/StreamingGenerator.java index 670a975d36ea..35580fdefab3 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/StreamingGenerator.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/StreamingGenerator.java @@ -63,7 +63,7 @@ public class StreamingGenerator extends BaseFreonGenerator @CommandLine.Option(names = {"--size"}, description = "Size of the generated files.", defaultValue = "104857600") - private int fileSize; + private long fileSize; private static final String SUB_DIR_NAME = "dir1"; diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/containergenerator/GeneratorDatanode.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/containergenerator/GeneratorDatanode.java index 02fa7e6373e7..ea523026b64c 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/containergenerator/GeneratorDatanode.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/containergenerator/GeneratorDatanode.java @@ -256,7 +256,7 @@ public void generateData(long index) throws Exception { //loop to create multiple chunks per blocks while (writtenBytes < getKeySize()) { int currentChunkSize = - Math.min(getKeySize() - writtenBytes, chunkSize); + Math.min(getKeySize() - writtenBytes, chunkSize); String chunkName = "chunk" + chunkIndex++; final byte[] data = new byte[currentChunkSize]; @@ -366,4 +366,4 @@ private void writeChunk( chunkManager.finishWriteChunks(container, new BlockData(blockId)); } -} +} \ No newline at end of file From 5bf8de008c934658cf831a4771a9049ca735ead4 Mon Sep 17 00:00:00 2001 From: Mohammad Arafat Khan Date: Thu, 10 Feb 2022 18:04:39 +0530 Subject: [PATCH 3/4] HDDS-4126. Freon key generator should support >2g files. --- .../ozone/freon/containergenerator/GeneratorDatanode.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/containergenerator/GeneratorDatanode.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/containergenerator/GeneratorDatanode.java index ea523026b64c..86adb31fadc1 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/containergenerator/GeneratorDatanode.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/containergenerator/GeneratorDatanode.java @@ -256,7 +256,7 @@ public void generateData(long index) throws Exception { //loop to create multiple chunks per blocks while (writtenBytes < getKeySize()) { int currentChunkSize = - Math.min(getKeySize() - writtenBytes, chunkSize); + Math.min(getKeySize() - writtenBytes, chunkSize); String chunkName = "chunk" + chunkIndex++; final byte[] data = new byte[currentChunkSize]; From aa2606e5bf79c322705255f00e35c7804dcef625 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Thu, 10 Feb 2022 14:04:26 +0100 Subject: [PATCH 4/4] restore newline at EOF --- .../ozone/freon/containergenerator/GeneratorDatanode.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/containergenerator/GeneratorDatanode.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/containergenerator/GeneratorDatanode.java index 86adb31fadc1..02fa7e6373e7 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/containergenerator/GeneratorDatanode.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/containergenerator/GeneratorDatanode.java @@ -366,4 +366,4 @@ private void writeChunk( chunkManager.finishWriteChunks(container, new BlockData(blockId)); } -} \ No newline at end of file +}