HDDS-12984. Use InodeID to identify the SST files inside the tarball.

hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/Archiver.java

@@ -38,12 +38,15 @@
 import org.apache.commons.io.IOUtils;
 import org.apache.hadoop.hdds.HddsUtils;
 import org.apache.hadoop.ozone.OzoneConsts;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /** Create and extract archives. */
 public final class Archiver {
 
   static final int MIN_BUFFER_SIZE = 8 * (int) OzoneConsts.KB; // same as IOUtils.DEFAULT_BUFFER_SIZE
   static final int MAX_BUFFER_SIZE = (int) OzoneConsts.MB;
+  private static final Logger LOG = LoggerFactory.getLogger(Archiver.class);
 
   private Archiver() {
     // no instances (for now)
@@ -111,6 +114,46 @@ public static long includeFile(File file, String entryName,
     return bytes;
   }
 
+  /**
+   * Creates a hard link to the specified file in the provided temporary directory,
+   * adds the linked file as an entry to the archive with the given entry name, writes
+   * its contents to the archive output, and then deletes the temporary hard link.
+   * <p>
+   * This approach avoids altering the original file and works around limitations
+   * of certain archiving libraries that may require the source file to be present
+   * in a specific location or have a specific name. Any errors during the hardlink
+   * creation or archiving process are logged.
+   * </p>
+   *
+   * @param file         the file to be included in the archive
+   * @param entryName    the name/path under which the file should appear in the archive
+   * @param archiveOutput the output stream for the archive (e.g., tar)
+   * @param tmpDir       the temporary directory in which to create the hard link
+   * @return number of bytes copied to the archive for this file
+   * @throws IOException if an I/O error occurs other than hardlink creation failure
+   */
+  public static long linkAndIncludeFile(File file, String entryName,
+      ArchiveOutputStream<TarArchiveEntry> archiveOutput, Path tmpDir) throws IOException {
+    File link = tmpDir.resolve(entryName).toFile();
+    long bytes = 0;
+    try {
+      Files.createLink(link.toPath(), file.toPath());
+      TarArchiveEntry entry = archiveOutput.createArchiveEntry(link, entryName);
+      archiveOutput.putArchiveEntry(entry);
+      try (InputStream input = Files.newInputStream(link.toPath())) {
+        bytes = IOUtils.copyLarge(input, archiveOutput);
+      }
+      archiveOutput.closeArchiveEntry();
+    } catch (IOException ioe) {
+      LOG.error("Couldn't create hardlink for file {} while including it in tarball.",
+          file.getAbsolutePath(), ioe);
+      throw ioe;
+    } finally {
+      Files.deleteIfExists(link.toPath());
+    }
+    return bytes;
+  }
+
   public static void extractEntry(ArchiveEntry entry, InputStream input, long size,
       Path ancestor, Path path) throws IOException {
     HddsUtils.validatePath(path, ancestor);

hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java

@@ -22,6 +22,7 @@
 import static org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_REQUEST_TO_EXCLUDE_SST;
 import static org.apache.hadoop.ozone.OzoneConsts.ROCKSDB_SST_SUFFIX;
 
+import com.google.common.annotations.VisibleForTesting;
 import java.io.File;
 import java.io.IOException;
 import java.io.OutputStream;
@@ -122,6 +123,10 @@ public void initialize(DBStore store, DBCheckpointMetrics metrics,
     }
   }
 
+  public File getBootstrapTempData() {
+    return bootstrapTempData;
+  }
+
   private boolean hasPermission(UserGroupInformation user) {
     // Check ACL for dbCheckpoint only when global Ozone ACL and SPNEGO is
     // enabled
@@ -132,7 +137,7 @@ private boolean hasPermission(UserGroupInformation user) {
     }
   }
 
-  private static void logSstFileList(Collection<String> sstList, String msg, int sampleSize) {
+  protected static void logSstFileList(Collection<String> sstList, String msg, int sampleSize) {
     int count = sstList.size();
     if (LOG.isDebugEnabled()) {
       LOG.debug(msg, count, "", sstList);
@@ -199,7 +204,8 @@ private void generateSnapshotCheckpoint(HttpServletRequest request,
     processMetadataSnapshotRequest(request, response, isFormData, flush);
   }
 
-  private void processMetadataSnapshotRequest(HttpServletRequest request, HttpServletResponse response,
+  @VisibleForTesting
+  public void processMetadataSnapshotRequest(HttpServletRequest request, HttpServletResponse response,
       boolean isFormData, boolean flush) {
     List<String> excludedSstList = new ArrayList<>();
     String[] sstParam = isFormData ?
@@ -292,7 +298,7 @@ public DBCheckpoint getCheckpoint(Path ignoredTmpdir, boolean flush)
    * @param request the HTTP servlet request
    * @return array of parsed sst form data parameters for exclusion
    */
-  private static String[] parseFormDataParameters(HttpServletRequest request) {
+  protected static String[] parseFormDataParameters(HttpServletRequest request) {
     ServletFileUpload upload = new ServletFileUpload();
     List<String> sstParam = new ArrayList<>();
 

hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/TestArchiver.java

@@ -18,9 +18,31 @@
 package org.apache.hadoop.hdds.utils;
 
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.CALLS_REAL_METHODS;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.mockStatic;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
 
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
+import org.junit.jupiter.api.Test;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.ValueSource;
+import org.mockito.MockedStatic;
 
 /** Test {@link Archiver}. */
 class TestArchiver {
@@ -46,4 +68,71 @@ void bufferSizeAboveMaximum(long fileSize) {
         .isEqualTo(Archiver.MAX_BUFFER_SIZE);
   }
 
+  @Test
+  void testLinkAndIncludeFileSuccessfulHardLink() throws IOException {
+    Path tmpDir = Files.createTempDirectory("archiver-test");
+    File tempFile = File.createTempFile("test-file", ".txt");
+    String entryName = "test-entry";
+    Files.write(tempFile.toPath(), "Test Content".getBytes(StandardCharsets.UTF_8));
+
+    TarArchiveOutputStream mockArchiveOutput = mock(TarArchiveOutputStream.class);
+    TarArchiveEntry mockEntry = new TarArchiveEntry(entryName);
+    AtomicBoolean isHardLinkCreated = new AtomicBoolean(false);
+    when(mockArchiveOutput.createArchiveEntry(any(File.class), eq(entryName)))
+        .thenAnswer(invocation -> {
+          File linkFile = invocation.getArgument(0);
+          isHardLinkCreated.set(Files.isSameFile(tempFile.toPath(), linkFile.toPath()));
+          return mockEntry;
+        });
+
+    // Call method under test
+    long bytesCopied = Archiver.linkAndIncludeFile(tempFile, entryName, mockArchiveOutput, tmpDir);
+    assertEquals(Files.size(tempFile.toPath()), bytesCopied);
+    // Verify archive interactions
+    verify(mockArchiveOutput, times(1)).putArchiveEntry(mockEntry);
+    verify(mockArchiveOutput, times(1)).closeArchiveEntry();
+    assertTrue(isHardLinkCreated.get());
+    assertFalse(Files.exists(tmpDir.resolve(entryName)));
+    // Cleanup
+    assertTrue(tempFile.delete());
+    Files.deleteIfExists(tmpDir);
+  }
+
+  @Test
+  void testLinkAndIncludeFileFailedHardLink() throws IOException {
+    Path tmpDir = Files.createTempDirectory("archiver-test");
+    File tempFile = File.createTempFile("test-file", ".txt");
+    String entryName = "test-entry";
+    Files.write(tempFile.toPath(), "Test Content".getBytes(StandardCharsets.UTF_8));
+
+    TarArchiveOutputStream mockArchiveOutput =
+        mock(TarArchiveOutputStream.class);
+    TarArchiveEntry mockEntry = new TarArchiveEntry("test-entry");
+    AtomicBoolean isHardLinkCreated = new AtomicBoolean(false);
+    when(mockArchiveOutput.createArchiveEntry(any(File.class), eq(entryName)))
+        .thenAnswer(invocation -> {
+          File linkFile = invocation.getArgument(0);
+          isHardLinkCreated.set(Files.isSameFile(tempFile.toPath(), linkFile.toPath()));
+          return mockEntry;
+        });
+
+    // Mock static Files.createLink to throw IOException
+    try (MockedStatic<Files> mockedFiles = mockStatic(Files.class, CALLS_REAL_METHODS)) {
+      Path linkPath = tmpDir.resolve(entryName);
+      String errorMessage = "Failed to create hardlink";
+      mockedFiles.when(() -> Files.createLink(linkPath, tempFile.toPath()))
+          .thenThrow(new IOException(errorMessage));
+
+      IOException thrown = assertThrows(IOException.class, () ->
+          Archiver.linkAndIncludeFile(tempFile, entryName, mockArchiveOutput, tmpDir)
+      );
+
+      assertTrue(thrown.getMessage().contains(errorMessage));
+    }
+    assertFalse(isHardLinkCreated.get());
+
+    assertTrue(tempFile.delete());
+    Files.deleteIfExists(tmpDir);
+  }
+
 }

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMDbCheckpointServlet.java

@@ -122,6 +122,8 @@ public void init() throws Exception {
         responseMock);
     doCallRealMethod().when(scmDbCheckpointServletMock).getCheckpoint(any(),
         anyBoolean());
+    doCallRealMethod().when(scmDbCheckpointServletMock)
+        .processMetadataSnapshotRequest(any(), any(), anyBoolean(), anyBoolean());
 
     servletContextMock = mock(ServletContext.class);
     when(scmDbCheckpointServletMock.getServletContext())

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java

@@ -229,6 +229,8 @@ public void write(int b) throws IOException {
 
     doCallRealMethod().when(omDbCheckpointServletMock).getCheckpoint(any(),
         anyBoolean());
+    doCallRealMethod().when(omDbCheckpointServletMock)
+        .processMetadataSnapshotRequest(any(), any(), anyBoolean(), anyBoolean());
   }
 
   @Test