diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java index 118a17fbb5df..26c93cd17921 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/DBCheckpointServlet.java @@ -20,7 +20,6 @@ import static org.apache.hadoop.hdds.utils.HddsServerUtil.writeDBCheckpointToStream; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_REQUEST_FLUSH; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_DB_CHECKPOINT_REQUEST_TO_EXCLUDE_SST; -import static org.apache.hadoop.ozone.OzoneConsts.ROCKSDB_SST_SUFFIX; import com.google.common.annotations.VisibleForTesting; import java.io.File; @@ -282,7 +281,7 @@ protected static Set extractSstFilesToExclude(String[] sstParam) { Set receivedSstFiles = new HashSet<>(); if (sstParam != null) { receivedSstFiles.addAll( - Arrays.stream(sstParam).filter(s -> s.endsWith(ROCKSDB_SST_SUFFIX)).distinct().collect(Collectors.toList())); + Arrays.stream(sstParam).distinct().collect(Collectors.toList())); logSstFileList(receivedSstFiles, "Received list of {} SST files to be excluded{}: {}", 5); } return receivedSstFiles; diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HAUtils.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HAUtils.java index 8492755d6091..81df3a4cefb3 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HAUtils.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HAUtils.java @@ -23,7 +23,6 @@ import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_INFO_WAIT_DURATION_DEFAULT; import static org.apache.hadoop.hdds.server.ServerUtils.getOzoneMetaDirPath; import static org.apache.hadoop.ozone.OzoneConsts.DB_TRANSIENT_MARKER; -import static org.apache.hadoop.ozone.OzoneConsts.ROCKSDB_SST_SUFFIX; import static org.apache.hadoop.ozone.OzoneConsts.TRANSACTION_INFO_KEY; import com.google.common.annotations.VisibleForTesting; @@ -319,28 +318,24 @@ public static File getMetaDir(DBDefinition definition, } /** - * Scan the DB dir and return the existing SST files, - * including omSnapshot sst files. - * SSTs could be used for avoiding repeated download. + * Scan the DB dir and return the existing files, + * including omSnapshot files. * * @param db the file representing the DB to be scanned - * @return the list of SST file name. If db not exist, will return empty list + * @return the list of file names. If db not exist, will return empty list */ - public static List getExistingSstFiles(File db) throws IOException { + public static List getExistingFiles(File db) throws IOException { List sstList = new ArrayList<>(); if (!db.exists()) { return sstList; } - - int truncateLength = db.toString().length() + 1; // Walk the db dir and get all sst files including omSnapshot files. try (Stream files = Files.walk(db.toPath())) { - sstList = - files.filter(path -> path.toString().endsWith(ROCKSDB_SST_SUFFIX)). - map(p -> p.toString().substring(truncateLength)). + sstList = files.filter(p -> p.toFile().isFile()) + .map(p -> p.getFileName().toString()). collect(Collectors.toList()); if (LOG.isDebugEnabled()) { - LOG.debug("Scanned SST files {} in {}.", sstList, db.getAbsolutePath()); + LOG.debug("Scanned files {} in {}.", sstList, db.getAbsolutePath()); } } return sstList; diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/RDBSnapshotProvider.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/RDBSnapshotProvider.java index 777efcf47ea5..b40a1f84e152 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/RDBSnapshotProvider.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/RDBSnapshotProvider.java @@ -108,14 +108,15 @@ public DBCheckpoint downloadDBSnapshotFromLeader(String leaderNodeID) LOG.info("Prepare to download the snapshot from leader OM {} and " + "reloading state from the snapshot.", leaderNodeID); checkLeaderConsistency(leaderNodeID); + int numParts = 0; while (true) { String snapshotFileName = getSnapshotFileName(leaderNodeID); File targetFile = new File(snapshotDir, snapshotFileName); downloadSnapshot(leaderNodeID, targetFile); - LOG.info( - "Successfully download the latest snapshot {} from leader OM: {}", - targetFile, leaderNodeID); + LOG.info("Successfully download the latest snapshot {} from leader OM: {}, part : {}", + targetFile, leaderNodeID, numParts); + numParts++; numDownloaded.incrementAndGet(); injectPause(); @@ -153,7 +154,7 @@ void checkLeaderConsistency(String currentLeader) throws IOException { return; } - List files = HAUtils.getExistingSstFiles(candidateDir); + List files = HAUtils.getExistingFiles(candidateDir); if (!files.isEmpty()) { LOG.warn("Candidate DB directory {} is not empty when last leader is " + "null.", candidateDir); diff --git a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/TestRDBSnapshotProvider.java b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/TestRDBSnapshotProvider.java index 652c1443b7b3..458a87b6682e 100644 --- a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/TestRDBSnapshotProvider.java +++ b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/TestRDBSnapshotProvider.java @@ -116,9 +116,11 @@ public void downloadSnapshot(String leaderNodeID, File targetFile) concat(String.valueOf(a.length()))) .collect(Collectors.toList())); try (OutputStream outputStream = Files.newOutputStream(targetFile.toPath())) { - writeDBCheckpointToStream(dbCheckpoint, outputStream, - new HashSet<>(HAUtils.getExistingSstFiles( - rdbSnapshotProvider.getCandidateDir()))); + Set existingSstFiles = HAUtils.getExistingFiles(rdbSnapshotProvider.getCandidateDir()) + .stream() + .filter(fName -> fName.endsWith(".sst") && !fName.equals(".sst")) + .collect(Collectors.toSet()); + writeDBCheckpointToStream(dbCheckpoint, outputStream, existingSstFiles); } } }; @@ -141,7 +143,7 @@ public void testDownloadDBSnapshotFromLeader() throws Exception { assertTrue(candidateDir.exists()); DBCheckpoint checkpoint; - int before = HAUtils.getExistingSstFiles( + int before = HAUtils.getExistingFiles( rdbSnapshotProvider.getCandidateDir()).size(); assertEquals(0, before); @@ -149,12 +151,12 @@ public void testDownloadDBSnapshotFromLeader() throws Exception { checkpoint = rdbSnapshotProvider.downloadDBSnapshotFromLeader(LEADER_ID); File checkpointDir = checkpoint.getCheckpointLocation().toFile(); assertEquals(candidateDir, checkpointDir); - int first = HAUtils.getExistingSstFiles( + int first = HAUtils.getExistingFiles( rdbSnapshotProvider.getCandidateDir()).size(); // Get second snapshot checkpoint = rdbSnapshotProvider.downloadDBSnapshotFromLeader(LEADER_ID); - int second = HAUtils.getExistingSstFiles( + int second = HAUtils.getExistingFiles( rdbSnapshotProvider.getCandidateDir()).size(); assertThat(second).withFailMessage("The second snapshot should have more SST files") .isGreaterThan(first); @@ -164,7 +166,7 @@ public void testDownloadDBSnapshotFromLeader() throws Exception { // Get third snapshot checkpoint = rdbSnapshotProvider.downloadDBSnapshotFromLeader(LEADER_ID); - int third = HAUtils.getExistingSstFiles( + int third = HAUtils.getExistingFiles( rdbSnapshotProvider.getCandidateDir()).size(); assertThat(third).withFailMessage("The third snapshot should have more SST files") .isGreaterThan(second); @@ -173,7 +175,7 @@ public void testDownloadDBSnapshotFromLeader() throws Exception { // Test cleanup candidateDB rdbSnapshotProvider.init(); - assertEquals(0, HAUtils.getExistingSstFiles( + assertEquals(0, HAUtils.getExistingFiles( rdbSnapshotProvider.getCandidateDir()).size()); } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServletInodeBasedXfer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServletInodeBasedXfer.java index f9c5ffa878d5..454f7a918111 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServletInodeBasedXfer.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServletInodeBasedXfer.java @@ -227,7 +227,6 @@ void testContentsOfTarballWithSnapshot() throws Exception { .forEachRemaining(snapInfo -> snapshotPaths.add(getSnapshotDBPath(snapInfo.getCheckpointDir()))); Set inodesFromOmDataDir = new HashSet<>(); Set inodesFromTarball = new HashSet<>(); - Set allPathsInTarball = new HashSet<>(); try (Stream filesInTarball = Files.list(newDbDir.toPath())) { List files = filesInTarball.collect(Collectors.toList()); for (Path p : files) { @@ -237,7 +236,6 @@ void testContentsOfTarballWithSnapshot() throws Exception { } String inode = getInode(file.getName()); inodesFromTarball.add(inode); - allPathsInTarball.add(p); } } Map> hardLinkMapFromOmData = new HashMap<>(); @@ -269,10 +267,7 @@ void testContentsOfTarballWithSnapshot() throws Exception { assertTrue(inodesFromTarball.containsAll(inodesFromOmDataDir)); // create hardlinks now - OmSnapshotUtils.createHardLinks(newDbDir.toPath()); - for (Path old : allPathsInTarball) { - assertTrue(old.toFile().delete()); - } + OmSnapshotUtils.createHardLinks(newDbDir.toPath(), true); assertFalse(hardlinkFilePath.toFile().exists()); } @@ -301,7 +296,7 @@ public void testSnapshotDBConsistency() throws Exception { FileUtil.unTar(tempFile, newDbDir); Set allPathsInTarball = getAllPathsInTarball(newDbDir); // create hardlinks now - OmSnapshotUtils.createHardLinks(newDbDir.toPath()); + OmSnapshotUtils.createHardLinks(newDbDir.toPath(), false); for (Path old : allPathsInTarball) { assertTrue(old.toFile().delete()); } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java index fb8d6b30146e..c153c24893c6 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java @@ -245,8 +245,8 @@ public void testInstallSnapshot(@TempDir Path tempDir) throws Exception { // Wait & for follower to update transactions to leader snapshot index. // Timeout error if follower does not load update within 10s GenericTestUtils.waitFor(() -> { - return followerOM.getOmRatisServer().getLastAppliedTermIndex().getIndex() - >= leaderOMSnapshotIndex - 1; + long index = followerOM.getOmRatisServer().getLastAppliedTermIndex().getIndex(); + return index >= leaderOMSnapshotIndex - 1; }, 100, 30_000); long followerOMLastAppliedIndex = @@ -562,7 +562,7 @@ private IncrementData getNextIncrementalTarball( Path increment = Paths.get(tempDir.toString(), "increment" + numKeys); assertTrue(increment.toFile().mkdirs()); unTarLatestTarBall(followerOM, increment); - List sstFiles = HAUtils.getExistingSstFiles(increment.toFile()); + List sstFiles = HAUtils.getExistingFiles(increment.toFile()); Path followerCandidatePath = followerOM.getOmSnapshotProvider(). getCandidateDir().toPath(); @@ -648,7 +648,7 @@ public void testInstallIncrementalSnapshotWithFailure() throws Exception { // Corrupt the mixed checkpoint in the candidate DB dir File followerCandidateDir = followerOM.getOmSnapshotProvider(). getCandidateDir(); - List sstList = HAUtils.getExistingSstFiles(followerCandidateDir); + List sstList = HAUtils.getExistingFiles(followerCandidateDir); assertThat(sstList.size()).isGreaterThan(0); for (int i = 0; i < sstList.size(); i += 2) { File victimSst = new File(followerCandidateDir, sstList.get(i)); @@ -1003,6 +1003,7 @@ public void testInstallCorruptedCheckpointFailure() throws Exception { DBCheckpoint leaderDbCheckpoint = leaderOM.getMetadataManager().getStore() .getCheckpoint(false); Path leaderCheckpointLocation = leaderDbCheckpoint.getCheckpointLocation(); + OmSnapshotUtils.createHardLinks(leaderCheckpointLocation, true); TransactionInfo leaderCheckpointTrxnInfo = OzoneManagerRatisUtils .getTrxnInfoFromCheckpoint(conf, leaderCheckpointLocation); @@ -1157,7 +1158,7 @@ public void pause() throws IOException { // max size config. That way next time through, we get multiple // tarballs. if (count == 1) { - long sstSize = getSizeOfSstFiles(tarball); + long sstSize = getSizeOfFiles(tarball); om.getConfiguration().setLong( OZONE_OM_RATIS_SNAPSHOT_MAX_TOTAL_SST_SIZE_KEY, sstSize / 2); // Now empty the tarball to restart the download @@ -1166,21 +1167,20 @@ public void pause() throws IOException { } else { // Each time we get a new tarball add a set of // its sst file to the list, (i.e. one per tarball.) - sstSetList.add(getSstFilenames(tarball)); + sstSetList.add(getFilenames(tarball)); } } // Get Size of sstfiles in tarball. - private long getSizeOfSstFiles(File tarball) throws IOException { + private long getSizeOfFiles(File tarball) throws IOException { FileUtil.unTar(tarball, tempDir.toFile()); - List sstPaths = Files.walk(tempDir).filter( - path -> path.toString().endsWith(".sst")). + List sstPaths = Files.walk(tempDir). collect(Collectors.toList()); - long sstSize = 0; + long totalFileSize = 0; for (Path sstPath : sstPaths) { - sstSize += Files.size(sstPath); + totalFileSize += Files.size(sstPath); } - return sstSize; + return totalFileSize; } private void createEmptyTarball(File dummyTarFile) @@ -1191,21 +1191,18 @@ private void createEmptyTarball(File dummyTarFile) archiveOutputStream.close(); } - // Return a list of sst files in tarball. - private Set getSstFilenames(File tarball) + // Return a list of files in tarball. + private Set getFilenames(File tarball) throws IOException { - Set sstFilenames = new HashSet<>(); + Set fileNames = new HashSet<>(); try (TarArchiveInputStream tarInput = new TarArchiveInputStream(Files.newInputStream(tarball.toPath()))) { TarArchiveEntry entry; while ((entry = tarInput.getNextTarEntry()) != null) { - String name = entry.getName(); - if (name.toLowerCase().endsWith(".sst")) { - sstFilenames.add(entry.getName()); - } + fileNames.add(entry.getName()); } } - return sstFilenames; + return fileNames; } // Find the tarball in the dir. diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOzoneManagerSnapshotProvider.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOzoneManagerSnapshotProvider.java index 594ca7a9536e..bc4958fdc6be 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOzoneManagerSnapshotProvider.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOzoneManagerSnapshotProvider.java @@ -124,6 +124,8 @@ public void testDownloadCheckpoint() throws Exception { private long getDownloadedSnapshotIndex(DBCheckpoint dbCheckpoint) throws Exception { + OmSnapshotUtils.createHardLinks(dbCheckpoint.getCheckpointLocation(), true); + TransactionInfo trxnInfoFromCheckpoint = OzoneManagerRatisUtils.getTrxnInfoFromCheckpoint(conf, dbCheckpoint.getCheckpointLocation()); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServletInodeBasedXfer.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServletInodeBasedXfer.java index fe2d36e5f966..c0e9d74ec543 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServletInodeBasedXfer.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServletInodeBasedXfer.java @@ -42,13 +42,11 @@ import java.nio.file.StandardOpenOption; import java.time.Duration; import java.time.Instant; -import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashSet; -import java.util.List; import java.util.Map; import java.util.Set; import java.util.UUID; @@ -128,7 +126,6 @@ public void init() throws ServletException { @Override public void processMetadataSnapshotRequest(HttpServletRequest request, HttpServletResponse response, boolean isFormData, boolean flush) { - List excludedSstList = new ArrayList<>(); String[] sstParam = isFormData ? parseFormDataParameters(request) : request.getParameterValues( OZONE_DB_CHECKPOINT_REQUEST_TO_EXCLUDE_SST); @@ -149,7 +146,7 @@ public void processMetadataSnapshotRequest(HttpServletRequest request, HttpServl long duration = Duration.between(start, end).toMillis(); LOG.info("Time taken to write the checkpoint to response output " + "stream: {} milliseconds", duration); - logSstFileList(excludedSstList, + logSstFileList(receivedSstFiles, "Excluded {} SST files from the latest checkpoint{}: {}", 5); } catch (Exception e) { LOG.error( diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index a5df1337dc0d..09853e842afd 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -3980,7 +3980,7 @@ public synchronized TermIndex installSnapshotFromLeader(String leaderId) { TermIndex termIndex = null; try { // Install hard links. - OmSnapshotUtils.createHardLinks(omDBCheckpoint.getCheckpointLocation()); + OmSnapshotUtils.createHardLinks(omDBCheckpoint.getCheckpointLocation(), false); termIndex = installCheckpoint(leaderId, omDBCheckpoint); } catch (Exception ex) { LOG.error("Failed to install snapshot from Leader OM.", ex); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerHttpServer.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerHttpServer.java index 355d6249806f..8836324410b9 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerHttpServer.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerHttpServer.java @@ -36,7 +36,7 @@ public OzoneManagerHttpServer(MutableConfigurationSource conf, addServlet("serviceList", OZONE_OM_SERVICE_LIST_HTTP_ENDPOINT, ServiceListJSONServlet.class); addServlet("dbCheckpoint", OZONE_DB_CHECKPOINT_HTTP_ENDPOINT, - OMDBCheckpointServlet.class); + OMDBCheckpointServletInodeBasedXfer.class); getWebAppContext().setAttribute(OzoneConsts.OM_CONTEXT_ATTRIBUTE, om); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java index a343232c39e3..ef0a46548a4b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis_snapshot/OmRatisSnapshotProvider.java @@ -153,7 +153,7 @@ public void downloadSnapshot(String leaderNodeID, File targetFile) connection.setRequestProperty("Content-Type", contentTypeValue); connection.setDoOutput(true); writeFormData(connection, - HAUtils.getExistingSstFiles(getCandidateDir())); + HAUtils.getExistingFiles(getCandidateDir())); connection.connect(); int errorCode = connection.getResponseCode(); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotUtils.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotUtils.java index 848384ce3e2e..497c7a064b8b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotUtils.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotUtils.java @@ -28,11 +28,14 @@ import java.nio.file.Paths; import java.nio.file.attribute.BasicFileAttributes; import java.nio.file.attribute.FileTime; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.hadoop.ozone.om.OmSnapshotManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Ozone Manager Snapshot Utilities. @@ -41,6 +44,8 @@ public final class OmSnapshotUtils { public static final String DATA_PREFIX = "data"; public static final String DATA_SUFFIX = "txt"; + private static final Logger LOG = + LoggerFactory.getLogger(OmSnapshotUtils.class); private OmSnapshotUtils() { } @@ -124,10 +129,12 @@ public static Path createHardLinkList(int truncateLength, * Create hard links listed in OM_HARDLINK_FILE. * * @param dbPath Path to db to have links created. + * @param deleteSourceFiles - Whether to delete the source files after creating the links. */ - public static void createHardLinks(Path dbPath) throws IOException { + public static void createHardLinks(Path dbPath, boolean deleteSourceFiles) throws IOException { File hardLinkFile = new File(dbPath.toString(), OmSnapshotManager.OM_HARDLINK_FILE); + List filesToDelete = new ArrayList<>(); if (hardLinkFile.exists()) { // Read file. try (Stream s = Files.lines(hardLinkFile.toPath())) { @@ -135,9 +142,15 @@ public static void createHardLinks(Path dbPath) throws IOException { // Create a link for each line. for (String l : lines) { - String from = l.split("\t")[1]; - String to = l.split("\t")[0]; + String[] parts = l.split("\t"); + if (parts.length != 2) { + LOG.warn("Skipping malformed line in hardlink file: {}", l); + continue; + } + String from = parts[1]; + String to = parts[0]; Path fullFromPath = Paths.get(dbPath.toString(), from); + filesToDelete.add(fullFromPath); Path fullToPath = Paths.get(dbPath.toString(), to); // Make parent dir if it doesn't exist. Path parent = fullToPath.getParent(); @@ -154,6 +167,15 @@ public static void createHardLinks(Path dbPath) throws IOException { } } } + if (deleteSourceFiles) { + for (Path fileToDelete : filesToDelete) { + try { + Files.deleteIfExists(fileToDelete); + } catch (IOException e) { + LOG.warn("Couldn't delete source file {} while unpacking the DB", fileToDelete, e); + } + } + } } /** diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java index ff65978728ca..5a6880920230 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java @@ -18,7 +18,7 @@ package org.apache.hadoop.ozone.om; import static org.apache.commons.io.file.PathUtils.copyDirectory; -import static org.apache.hadoop.hdds.utils.HAUtils.getExistingSstFiles; +import static org.apache.hadoop.hdds.utils.HAUtils.getExistingFiles; import static org.apache.hadoop.ozone.OzoneConsts.OM_CHECKPOINT_DIR; import static org.apache.hadoop.ozone.OzoneConsts.OM_DB_NAME; import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; @@ -30,7 +30,6 @@ import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.BUCKET_TABLE; import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.VOLUME_TABLE; import static org.apache.hadoop.ozone.om.snapshot.OmSnapshotUtils.getINode; -import static org.apache.hadoop.ozone.om.snapshot.OmSnapshotUtils.truncateFileName; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -58,7 +57,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.TreeMap; import java.util.UUID; import java.util.concurrent.TimeoutException; import org.apache.hadoop.hdds.HddsConfigKeys; @@ -374,7 +372,7 @@ public void testHardLinkCreation() throws IOException { File s1FileLink = new File(followerSnapDir2, "s1.sst"); // Create links on the follower from list. - OmSnapshotUtils.createHardLinks(candidateDir.toPath()); + OmSnapshotUtils.createHardLinks(candidateDir.toPath(), false); // Confirm expected follower links. assertTrue(s1FileLink.exists()); @@ -417,44 +415,16 @@ public void testGetSnapshotInfo() throws IOException { @Test public void testExcludeUtilities() throws IOException { File noLinkFile = new File(followerSnapDir2, "noLink.sst"); - + File nonSstFile = new File(followerSnapDir2, "nonSstFile"); // Confirm that the list of existing sst files is as expected. - List existingSstList = getExistingSstFiles(candidateDir); + List existingSstList = getExistingFiles(candidateDir); Set existingSstFiles = new HashSet<>(existingSstList); - int truncateLength = candidateDir.toString().length() + 1; - Set expectedSstFiles = new HashSet<>(Arrays.asList( - s1File.toString().substring(truncateLength), - noLinkFile.toString().substring(truncateLength), - f1File.toString().substring(truncateLength))); - assertEquals(expectedSstFiles, existingSstFiles); - - // Confirm that the excluded list is normalized as expected. - // (Normalizing means matches the layout on the leader.) - File leaderSstBackupDir = new File(leaderDir.toString(), "sstBackup"); - assertTrue(leaderSstBackupDir.mkdirs()); - File leaderTmpDir = new File(leaderDir.toString(), "tmp"); - assertTrue(leaderTmpDir.mkdirs()); - OMDBCheckpointServlet.DirectoryData sstBackupDir = - new OMDBCheckpointServlet.DirectoryData(leaderTmpDir.toPath(), - leaderSstBackupDir.toString()); - Path srcSstBackup = Paths.get(sstBackupDir.getTmpDir().toString(), - "backup.sst"); - Path destSstBackup = Paths.get(sstBackupDir.getOriginalDir().toString(), - "backup.sst"); - truncateLength = leaderDir.toString().length() + 1; - existingSstList.add(truncateFileName(truncateLength, destSstBackup)); - Map> normalizedMap = - OMDBCheckpointServlet.normalizeExcludeList(existingSstList, - leaderCheckpointDir.toPath(), sstBackupDir); - Map> expectedMap = new TreeMap<>(); - Path s1 = Paths.get(leaderSnapDir1.toString(), "s1.sst"); - Path noLink = Paths.get(leaderSnapDir2.toString(), "noLink.sst"); - Path f1 = Paths.get(leaderCheckpointDir.toString(), "f1.sst"); - expectedMap.put("s1.sst", ImmutableMap.of(s1, s1)); - expectedMap.put("noLink.sst", ImmutableMap.of(noLink, noLink)); - expectedMap.put("f1.sst", ImmutableMap.of(f1, f1)); - expectedMap.put("backup.sst", ImmutableMap.of(srcSstBackup, destSstBackup)); - assertEquals(expectedMap, new TreeMap<>(normalizedMap)); + Set expectedSstFileNames = new HashSet<>(Arrays.asList( + s1File.getName(), + noLinkFile.getName(), + f1File.getName(), + nonSstFile.getName())); + assertEquals(expectedSstFileNames, existingSstFiles); } /* diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/OzoneManagerServiceProviderImpl.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/OzoneManagerServiceProviderImpl.java index f8b60fdd7faa..e407fe6cd4fe 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/OzoneManagerServiceProviderImpl.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/OzoneManagerServiceProviderImpl.java @@ -83,6 +83,7 @@ import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.helpers.DBUpdates; import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol; +import org.apache.hadoop.ozone.om.snapshot.OmSnapshotUtils; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DBUpdatesRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServicePort.Type; import org.apache.hadoop.ozone.recon.ReconContext; @@ -447,6 +448,7 @@ public DBCheckpoint getOzoneManagerDBSnapshot() { try (InputStream inputStream = reconUtils.makeHttpCall( connectionFactory, getOzoneManagerSnapshotUrl(), isOmSpnegoEnabled()).getInputStream()) { tarExtractor.extractTar(inputStream, untarredDbDir); + OmSnapshotUtils.createHardLinks(untarredDbDir, true); } catch (IOException | InterruptedException e) { reconContext.updateHealthStatus(new AtomicBoolean(false)); reconContext.updateErrors(ReconContext.ErrorCode.GET_OM_DB_SNAPSHOT_FAILED);