From f066098119ac12d06687aad22318c41020e275c5 Mon Sep 17 00:00:00 2001 From: George Jahad Date: Mon, 17 Apr 2023 10:06:23 -0700 Subject: [PATCH 1/2] Set tar big number mode --- .../java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java index 023f2cb3e1e6..b4639ea0e909 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java @@ -129,6 +129,8 @@ public void writeDbDataToStream(DBCheckpoint checkpoint, new TarArchiveOutputStream(destination)) { archiveOutputStream .setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX); + archiveOutputStream + .setBigNumberMode(TarArchiveOutputStream.BIGNUMBER_POSIX); writeFilesToArchive(copyFiles, hardLinkFiles, archiveOutputStream); } } From 9180c7b0e9d9494985bd598d18fcc5f225d2b78d Mon Sep 17 00:00:00 2001 From: George Jahad Date: Tue, 18 Apr 2023 10:40:46 -0700 Subject: [PATCH 2/2] added setBigNumberMode() to all TarArchiveOutputStream's --- .../hadoop/ozone/container/keyvalue/TarContainerPacker.java | 4 +++- .../ozone/container/keyvalue/TestTarContainerPacker.java | 5 +++-- .../java/org/apache/hadoop/hdds/utils/HddsServerUtil.java | 4 +++- .../main/java/org/apache/hadoop/ozone/recon/ReconUtils.java | 1 + 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/TarContainerPacker.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/TarContainerPacker.java index 77eb825b39fc..54fa6ae09dbb 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/TarContainerPacker.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/TarContainerPacker.java @@ -272,7 +272,9 @@ private static ArchiveInputStream untar(InputStream input) { } private static ArchiveOutputStream tar(OutputStream output) { - return new TarArchiveOutputStream(output); + TarArchiveOutputStream os = new TarArchiveOutputStream(output); + os.setBigNumberMode(TarArchiveOutputStream.BIGNUMBER_POSIX); + return os; } @VisibleForTesting diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestTarContainerPacker.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestTarContainerPacker.java index 0b1757155f69..ac2554a5621c 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestTarContainerPacker.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestTarContainerPacker.java @@ -35,7 +35,6 @@ import java.util.UUID; import java.util.concurrent.atomic.AtomicInteger; -import org.apache.commons.compress.archivers.ArchiveOutputStream; import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.container.common.impl.ContainerLayoutVersion; @@ -381,7 +380,9 @@ private File packContainerWithSingleFile(File file, String entryName) File targetFile = TEMP_DIR.resolve("container.tar").toFile(); try (FileOutputStream output = new FileOutputStream(targetFile); OutputStream compressed = packer.compress(output); - ArchiveOutputStream archive = new TarArchiveOutputStream(compressed)) { + TarArchiveOutputStream archive = + new TarArchiveOutputStream(compressed)) { + archive.setBigNumberMode(TarArchiveOutputStream.BIGNUMBER_POSIX); TarContainerPacker.includeFile(file, entryName, archive); } return targetFile; diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HddsServerUtil.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HddsServerUtil.java index f31ee3174d9c..201a70e42b33 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HddsServerUtil.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HddsServerUtil.java @@ -532,10 +532,12 @@ public static MetricsSystem initializeMetrics( public static void writeDBCheckpointToStream(DBCheckpoint checkpoint, OutputStream destination) throws IOException { - try (ArchiveOutputStream archiveOutputStream = + try (TarArchiveOutputStream archiveOutputStream = new TarArchiveOutputStream(destination); Stream files = Files.list(checkpoint.getCheckpointLocation())) { + archiveOutputStream.setBigNumberMode( + TarArchiveOutputStream.BIGNUMBER_POSIX); for (Path path : files.collect(Collectors.toList())) { if (path != null) { Path fileName = path.getFileName(); diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java index 7adc78657ea5..5f7e78a8b727 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java @@ -109,6 +109,7 @@ public static File createTarFile(Path sourcePath) throws IOException { String fileName = sourceDir.concat(".tar"); fileOutputStream = new FileOutputStream(fileName); tarOs = new TarArchiveOutputStream(fileOutputStream); + tarOs.setBigNumberMode(TarArchiveOutputStream.BIGNUMBER_POSIX); File folder = new File(sourceDir); File[] filesInDir = folder.listFiles(); if (filesInDir != null) {