HDDS-10300. InstallSnapshot may fail if OM metadata dir and OM DB dir are in different local storage partitions.

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java

@@ -248,7 +248,7 @@ public DBCheckpoint getCheckpoint(Path tmpdir, boolean flush)
       differ.incrementTarballRequestCount();
       FileUtils.copyDirectory(compactionLogDir.getOriginalDir(),
           compactionLogDir.getTmpDir());
-      OmSnapshotUtils.linkFiles(sstBackupDir.getOriginalDir(),
+      OmSnapshotUtils.linkFilesOrCopy(sstBackupDir.getOriginalDir(),
           sstBackupDir.getTmpDir());
       checkpoint = getDbStore().getCheckpoint(flush);
     } finally {

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java

@@ -3711,7 +3711,7 @@ public synchronized TermIndex installSnapshotFromLeader(String leaderId) {
     TermIndex termIndex = null;
     try {
       // Install hard links.
-      OmSnapshotUtils.createHardLinks(omDBCheckpoint.getCheckpointLocation());
+      OmSnapshotUtils.createHardLinksOrCopyFromHardlinkFile(omDBCheckpoint.getCheckpointLocation());
       termIndex = installCheckpoint(leaderId, omDBCheckpoint);
     } catch (Exception ex) {
       LOG.error("Failed to install snapshot from Leader OM.", ex);
@@ -3946,7 +3946,7 @@ private void moveCheckpointFiles(File oldDB, Path checkpointPath, File dbDir,
       // Link each of the candidate DB files to real DB directory.  This
       // preserves the links that already exist between files in the
       // candidate db.
-      OmSnapshotUtils.linkFiles(checkpointPath.toFile(),
+      OmSnapshotUtils.linkFilesOrCopy(checkpointPath.toFile(),
           oldDB);
       moveOmSnapshotData(oldDB.toPath(), dbSnapshotsDir.toPath());
       Files.deleteIfExists(markerFile);

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotUtils.java

@@ -24,9 +24,11 @@
 import java.io.File;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.FileStore;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
 import java.nio.file.attribute.BasicFileAttributes;
 import java.util.List;
 import java.util.Map;
@@ -106,7 +108,7 @@ public static Path createHardLinkList(int truncateLength,
    *
    * @param dbPath Path to db to have links created.
    */
-  public static void createHardLinks(Path dbPath) throws IOException {
+  public static void createHardLinksOrCopyFromHardlinkFile(Path dbPath) throws IOException {
     File hardLinkFile =
         new File(dbPath.toString(), OmSnapshotManager.OM_HARDLINK_FILE);
     if (hardLinkFile.exists()) {
@@ -128,7 +130,7 @@ public static void createHardLinks(Path dbPath) throws IOException {
                   "Failed to create directory: " + parent.toString());
             }
           }
-          Files.createLink(fullToPath, fullFromPath);
+          createHardLinksOrCopy(fullFromPath, fullToPath);
         }
         if (!hardLinkFile.delete()) {
           throw new IOException("Failed to delete: " + hardLinkFile);
@@ -137,13 +139,31 @@ public static void createHardLinks(Path dbPath) throws IOException {
     }
   }
 
+  private static void createHardLinksOrCopy(Path fullFromPath, Path fullToPath) throws IOException {
+    if (isSamePartition(fullFromPath.getParent(), fullToPath.getParent())) {
+      Files.createLink(fullToPath, fullFromPath);
+    } else {
+      Files.copy(fullFromPath, fullToPath, StandardCopyOption.REPLACE_EXISTING);
+    }
+  }
+
+  private static boolean isSamePartition(Path fromPathParent, Path toPathParent) throws IOException {
+    if (fromPathParent == null || toPathParent == null) {
+      throw new IOException("From path: " + fromPathParent + " or To path: " + toPathParent + " is null");
+    }
+    FileStore fromPathStore = Files.getFileStore(fromPathParent);
+    FileStore toPathStore = Files.getFileStore(toPathParent);
+    return fromPathStore.equals(toPathStore);
+  }
+
   /**
-   * Link each of the files in oldDir to newDir.
+   * Link each of the files in oldDir to newDir if they are in the same partition
+   * else copy the files in oldDir to newDir.
    *
    * @param oldDir The dir to create links from.
    * @param newDir The dir to create links to.
    */
-  public static void linkFiles(File oldDir, File newDir) throws IOException {
+  public static void linkFilesOrCopy(File oldDir, File newDir) throws IOException {
     int truncateLength = oldDir.toString().length() + 1;
     List<String> oldDirList;
     try (Stream<Path> files = Files.walk(oldDir.toPath())) {
@@ -169,7 +189,7 @@ public static void linkFiles(File oldDir, File newDir) throws IOException {
           throw new IOException("Directory create fails: " + newFile);
         }
       } else {
-        Files.createLink(newFile.toPath(), oldFile.toPath());
+        createHardLinksOrCopy(oldFile.toPath(), newFile.toPath());
       }
     }
   }

hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java

@@ -297,7 +297,7 @@ public void testHardLinkCreation() throws IOException {
     File s1FileLink = new File(followerSnapDir2, "s1.sst");
 
     // Create links on the follower from list.
-    OmSnapshotUtils.createHardLinks(candidateDir.toPath());
+    OmSnapshotUtils.createHardLinksOrCopyFromHardlinkFile(candidateDir.toPath());
 
     // Confirm expected follower links.
     assertTrue(s1FileLink.exists());

hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotUtils.java

@@ -66,7 +66,7 @@ public void testLinkFiles(@TempDir File tempDir) throws Exception {
     assertFalse(tree2.exists());
     assertFalse(f1Link.exists());
 
-    OmSnapshotUtils.linkFiles(tree1, tree2);
+    OmSnapshotUtils.linkFilesOrCopy(tree1, tree2);
 
     // Expected files/links should exist now.
     assertTrue(tree2.exists());