HDDS-13628. LocalSnapshotMetadata create should track snapshot from checkpoint instead of active object store

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMDbCheckpointServlet.java

@@ -736,7 +736,7 @@ private String createSnapshot(String vname, String bname)
     String snapshotPath = getSnapshotPath(conf, snapshotInfo)
         + OM_KEY_PREFIX;
     GenericTestUtils.waitFor(() -> new File(snapshotPath).exists(),
-        100, 2000);
+        100, 30000);
     return snapshotPath;
   }
 
@@ -745,7 +745,7 @@ private Set<String> getFiles(Path path, int truncateLength)
     return getFiles(path, truncateLength, new HashSet<>());
   }
 
-  // Get all files below path, recursively, (skipping fabricated files).
+  // Get all files below path, recursively, (skipping fabricated files, archive directory in rocksdb).
   private Set<String> getFiles(Path path, int truncateLength,
       Set<String> fileSet) throws IOException {
     try (Stream<Path> files = Files.list(path)) {
@@ -754,8 +754,11 @@ private Set<String> getFiles(Path path, int truncateLength,
           getFiles(file, truncateLength, fileSet);
         }
         String filename = String.valueOf(file.getFileName());
+        Path parentDir = file.getParent();
+        String parentFileName = parentDir == null ? "null" : parentDir.toFile().getName();
         if (!filename.startsWith("fabricated") &&
-            !filename.startsWith(OZONE_RATIS_SNAPSHOT_COMPLETE_FLAG_NAME)) {
+            !filename.startsWith(OZONE_RATIS_SNAPSHOT_COMPLETE_FLAG_NAME) &&
+            !(filename.equals("archive") && parentFileName.startsWith("om.db"))) {
           fileSet.add(truncateFileName(truncateLength, file));
         }
       }

hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java

@@ -60,7 +60,7 @@
 /**
  * OM metadata manager interface.
  */
-public interface OMMetadataManager extends DBStoreHAManager {
+public interface OMMetadataManager extends DBStoreHAManager, AutoCloseable {
   /**
    * Start metadata manager.
    *

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java

@@ -37,6 +37,7 @@
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Strings;
 import com.google.common.collect.Lists;
+import java.io.Closeable;
 import java.io.File;
 import java.io.IOException;
 import java.io.UncheckedIOException;
@@ -125,7 +126,7 @@
  * Ozone metadata manager interface.
  */
 public class OmMetadataManagerImpl implements OMMetadataManager,
-    S3SecretStore {
+    S3SecretStore, Closeable {
   private static final Logger LOG =
       LoggerFactory.getLogger(OmMetadataManagerImpl.class);
 
@@ -1809,6 +1810,11 @@ public boolean containsIncompleteMPUs(String volume, String bucket)
     return false;
   }
 
+  @Override
+  public void close() throws IOException {
+    stop();
+  }
+
   private final class S3SecretBatcher implements S3Batcher {
     @Override
     public void addWithBatch(AutoCloseable batchOperator, String id, S3SecretValue s3SecretValue)

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java

@@ -517,8 +517,13 @@ public static DBCheckpoint createOmSnapshotCheckpoint(
     }
     OmSnapshotManager omSnapshotManager =
         ((OmMetadataManagerImpl) omMetadataManager).getOzoneManager().getOmSnapshotManager();
-    // Create the snapshot local property file.
-    OmSnapshotManager.createNewOmSnapshotLocalDataFile(omSnapshotManager, omMetadataManager, snapshotInfo, store);
+    OzoneConfiguration configuration = ((OmMetadataManagerImpl) omMetadataManager).getOzoneManager().getConfiguration();
+    try (OmMetadataManagerImpl checkpointMetadataManager =
+             OmMetadataManagerImpl.createCheckpointMetadataManager(configuration, dbCheckpoint)) {
+      // Create the snapshot local property file.
+      OmSnapshotManager.createNewOmSnapshotLocalDataFile(omSnapshotManager,
+          (RDBStore) checkpointMetadataManager.getStore(), snapshotInfo);
+    }
 
     // Clean up active DB's deletedTable right after checkpoint is taken,
     // Snapshot create is processed as a single transaction and
@@ -532,11 +537,11 @@ public static DBCheckpoint createOmSnapshotCheckpoint(
     deleteKeysFromSnapRenamedTableInSnapshotScope(omMetadataManager,
         snapshotInfo.getVolumeName(), snapshotInfo.getBucketName(), batchOperation);
 
-    if (dbCheckpoint != null && snapshotDirExist) {
+    if (snapshotDirExist) {
       LOG.info("Checkpoint: {} for snapshot {} already exists.",
           dbCheckpoint.getCheckpointLocation(), snapshotInfo.getTableKey());
       return dbCheckpoint;
-    } else if (dbCheckpoint != null) {
+    } else {
       LOG.info("Created checkpoint: {} for snapshot {}",
           dbCheckpoint.getCheckpointLocation(),
           snapshotInfo.getTableKey());
@@ -632,16 +637,15 @@ private static List<LiveFileMetaData> getSnapshotSSTFileList(RDBStore store)
 
   /**
    * Creates and writes snapshot local properties to a YAML file with uncompacted SST file list.
-   * @param omMetadataManager the metadata manager
-   * @param snapshotInfo The metadata of snapshot to be created
-   * @param store The store used to get uncompacted SST file list from.
+   * @param snapshotManager snapshot manager instance.
+   * @param snapshotStore snapshot metadata manager.
+   * @param snapshotInfo snapshot info instance corresponding to snapshot.
    */
-  public static void createNewOmSnapshotLocalDataFile(OmSnapshotManager snapshotManager,
-      OMMetadataManager omMetadataManager, SnapshotInfo snapshotInfo, RDBStore store)
-      throws IOException {
-    Path snapshotLocalDataPath = Paths.get(getSnapshotLocalPropertyYamlPath(omMetadataManager, snapshotInfo));
+  public static void createNewOmSnapshotLocalDataFile(OmSnapshotManager snapshotManager, RDBStore snapshotStore,
+      SnapshotInfo snapshotInfo) throws IOException {
+    Path snapshotLocalDataPath = Paths.get(getSnapshotLocalPropertyYamlPath(snapshotStore.getDbLocation().toPath()));
     Files.deleteIfExists(snapshotLocalDataPath);
-    OmSnapshotLocalDataYaml snapshotLocalDataYaml = new OmSnapshotLocalDataYaml(getSnapshotSSTFileList(store),
+    OmSnapshotLocalDataYaml snapshotLocalDataYaml = new OmSnapshotLocalDataYaml(getSnapshotSSTFileList(snapshotStore),
         snapshotInfo.getPathPreviousSnapshotId());
     snapshotLocalDataYaml.writeToYaml(snapshotManager, snapshotLocalDataPath.toFile());
   }

hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java

@@ -28,6 +28,7 @@
 import static org.apache.hadoop.ozone.OzoneConsts.SNAPSHOT_INFO_TABLE;
 import static org.apache.hadoop.ozone.om.OMDBCheckpointServlet.processFile;
 import static org.apache.hadoop.ozone.om.OmSnapshotManager.OM_HARDLINK_FILE;
+import static org.apache.hadoop.ozone.om.OmSnapshotManager.getSnapshotPath;
 import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.BUCKET_TABLE;
 import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.DIRECTORY_TABLE;
 import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.FILE_TABLE;
@@ -311,12 +312,12 @@ public void testCreateNewSnapshotLocalYaml() throws IOException {
 
     Path snapshotYaml = Paths.get(OmSnapshotManager.getSnapshotLocalPropertyYamlPath(
         omMetadataManager, snapshotInfo));
-
+    when(mockedStore.getDbLocation()).thenReturn(getSnapshotPath(omMetadataManager, snapshotInfo).toFile());
     // Create an existing YAML file for the snapshot
     assertTrue(snapshotYaml.toFile().createNewFile());
     assertEquals(0, Files.size(snapshotYaml));
     // Create a new YAML file for the snapshot
-    OmSnapshotManager.createNewOmSnapshotLocalDataFile(omSnapshotManager, omMetadataManager, snapshotInfo, mockedStore);
+    OmSnapshotManager.createNewOmSnapshotLocalDataFile(omSnapshotManager, mockedStore, snapshotInfo);
     // Verify that previous file was overwritten
     assertTrue(Files.exists(snapshotYaml));
     assertTrue(Files.size(snapshotYaml) > 0);

hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotCreateResponse.java

@@ -91,6 +91,7 @@ public void close() {
     };
     OzoneManager ozoneManager = mock(OzoneManager.class);
     OmSnapshotManager omSnapshotManager = mock(OmSnapshotManager.class);
+    when(ozoneManager.getConfiguration()).thenReturn(ozoneConfiguration);
     when(ozoneManager.getOmSnapshotManager()).thenReturn(omSnapshotManager);
     when(omSnapshotManager.getSnapshotLocalYaml()).thenReturn(yamlSupplier);
     omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, ozoneManager);

hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/snapshot/TestOMSnapshotDeleteResponse.java

@@ -83,6 +83,7 @@ public void close() {
     };
     OzoneManager ozoneManager = mock(OzoneManager.class);
     OmSnapshotManager omSnapshotManager = mock(OmSnapshotManager.class);
+    when(ozoneManager.getConfiguration()).thenReturn(ozoneConfiguration);
     when(ozoneManager.getOmSnapshotManager()).thenReturn(omSnapshotManager);
     when(omSnapshotManager.getSnapshotLocalYaml()).thenReturn(yamlSupplier);
     omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, ozoneManager);