HDDS-12637. Increase max buffer size for tar entry read/write

hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/Archiver.java

@@ -37,10 +37,14 @@
 import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
 import org.apache.commons.io.IOUtils;
 import org.apache.hadoop.hdds.HddsUtils;
+import org.apache.hadoop.ozone.OzoneConsts;
 
 /** Create and extract archives. */
 public final class Archiver {
 
+  static final int MIN_BUFFER_SIZE = 8 * (int) OzoneConsts.KB; // same as IOUtils.DEFAULT_BUFFER_SIZE
+  static final int MAX_BUFFER_SIZE = (int) OzoneConsts.MB;
+
   private Archiver() {
     // no instances (for now)
   }
@@ -68,15 +72,7 @@ public static void extract(File tarFile, Path dir) throws IOException {
   public static byte[] readEntry(InputStream input, final long size)
       throws IOException {
     ByteArrayOutputStream output = new ByteArrayOutputStream();
-    int bufferSize = 1024;
-    byte[] buffer = new byte[bufferSize + 1];
-    long remaining = size;
-    while (remaining > 0) {
-      int len = (int) Math.min(remaining, bufferSize);
-      int read = input.read(buffer, 0, len);
-      remaining -= read;
-      output.write(buffer, 0, read);
-    }
+    IOUtils.copy(input, output, getBufferSize(size));
     return output.toByteArray();
   }
 
@@ -109,7 +105,7 @@ public static long includeFile(File file, String entryName,
     TarArchiveEntry entry = archiveOutput.createArchiveEntry(file, entryName);
     archiveOutput.putArchiveEntry(entry);
     try (InputStream input = Files.newInputStream(file.toPath())) {
-      bytes = IOUtils.copyLarge(input, archiveOutput);
+      bytes = IOUtils.copy(input, archiveOutput, getBufferSize(file.length()));
     }
     archiveOutput.closeArchiveEntry();
     return bytes;
@@ -129,19 +125,7 @@ public static void extractEntry(ArchiveEntry entry, InputStream input, long size
 
       try (OutputStream fileOutput = Files.newOutputStream(path);
            OutputStream output = new BufferedOutputStream(fileOutput)) {
-        int bufferSize = 1024;
-        byte[] buffer = new byte[bufferSize + 1];
-        long remaining = size;
-        while (remaining > 0) {
-          int len = (int) Math.min(remaining, bufferSize);
-          int read = input.read(buffer, 0, len);
-          if (read >= 0) {
-            remaining -= read;
-            output.write(buffer, 0, read);
-          } else {
-            remaining = 0;
-          }
-        }
+        IOUtils.copy(input, output, getBufferSize(size));
       }
     }
   }
@@ -157,4 +141,8 @@ public static ArchiveOutputStream<TarArchiveEntry> tar(OutputStream output) {
     return os;
   }
 
+  static int getBufferSize(long fileSize) {
+    return Math.toIntExact(Math.min(MAX_BUFFER_SIZE, Math.max(fileSize, MIN_BUFFER_SIZE)));
+  }
+
 }

hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/TestArchiver.java

@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.utils;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+/** Test {@link Archiver}. */
+class TestArchiver {
+
+  @ParameterizedTest
+  @ValueSource(longs = { 0, 1, Archiver.MIN_BUFFER_SIZE - 1 })
+  void bufferSizeBelowMinimum(long fileSize) {
+    assertThat(Archiver.getBufferSize(fileSize))
+        .isEqualTo(Archiver.MIN_BUFFER_SIZE);
+  }
+
+  @ParameterizedTest
+  @ValueSource(longs = { Archiver.MIN_BUFFER_SIZE, Archiver.MAX_BUFFER_SIZE })
+  void bufferSizeBetweenLimits(long fileSize) {
+    assertThat(Archiver.getBufferSize(fileSize))
+        .isEqualTo(fileSize);
+  }
+
+  @ParameterizedTest
+  @ValueSource(longs = { Archiver.MAX_BUFFER_SIZE + 1, Integer.MAX_VALUE, Long.MAX_VALUE })
+  void bufferSizeAboveMaximum(long fileSize) {
+    assertThat(Archiver.getBufferSize(fileSize))
+        .isEqualTo(Archiver.MAX_BUFFER_SIZE);
+  }
+
+}