HDDS-8577. [Snapshot] Disable compaction log when loading metadata for snapshot

hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java

@@ -99,7 +99,7 @@ public final class DBStoreBuilder {
   private boolean openReadOnly = false;
   private int maxFSSnapshots = 0;
   private final DBProfile defaultCfProfile;
-  private boolean enableCompactionLog;
+  private boolean enableCompactionDag;
   private boolean createCheckpointDirs = true;
   // this is to track the total size of dbUpdates data since sequence
   // number in request to avoid increase in heap memory.
@@ -210,7 +210,7 @@ public DBStore build() throws IOException {
 
       return new RDBStore(dbFile, rocksDBOption, writeOptions, tableConfigs,
           registry.build(), openReadOnly, maxFSSnapshots, dbJmxBeanNameName,
-          enableCompactionLog, maxDbUpdatesSizeThreshold, createCheckpointDirs,
+          enableCompactionDag, maxDbUpdatesSizeThreshold, createCheckpointDirs,
           configuration);
     } finally {
       tableConfigs.forEach(TableConfig::close);
@@ -268,8 +268,8 @@ public DBStoreBuilder setOpenReadOnly(boolean readOnly) {
     return this;
   }
 
-  public DBStoreBuilder setEnableCompactionLog(boolean enableCompactionLog) {
-    this.enableCompactionLog = enableCompactionLog;
+  public DBStoreBuilder setEnableCompactionDag(boolean enableCompactionDag) {
+    this.enableCompactionDag = enableCompactionDag;
     return this;
   }
 

hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java

@@ -78,12 +78,13 @@ public class RDBStore implements DBStore {
   // this is to track the total size of dbUpdates data since sequence
   // number in request to avoid increase in heap memory.
   private long maxDbUpdatesSizeThreshold;
+  private final ManagedDBOptions dbOptions;
 
   @SuppressWarnings("parameternumber")
   public RDBStore(File dbFile, ManagedDBOptions dbOptions,
                   ManagedWriteOptions writeOptions, Set<TableConfig> families,
                   CodecRegistry registry, boolean readOnly, int maxFSSnapshots,
-                  String dbJmxBeanNameName, boolean enableCompactionLog,
+                  String dbJmxBeanNameName, boolean enableCompactionDag,
                   long maxDbUpdatesSizeThreshold,
                   boolean createCheckpointDirs,
                   ConfigurationSource configuration)
@@ -97,11 +98,12 @@ public RDBStore(File dbFile, ManagedDBOptions dbOptions,
     dbLocation = dbFile;
     dbJmxBeanName = dbJmxBeanNameName == null ? dbFile.getName() :
         dbJmxBeanNameName;
+    this.dbOptions = dbOptions;
 
     try {
-      if (enableCompactionLog) {
+      if (enableCompactionDag) {
         rocksDBCheckpointDiffer = RocksDBCheckpointDifferHolder.getInstance(
-            dbLocation.getParent() + OM_KEY_PREFIX + OM_SNAPSHOT_DIFF_DIR,
+            getSnapshotMetadataDir(),
             DB_COMPACTION_SST_BACKUP_DIR, DB_COMPACTION_LOG_DIR,
             dbLocation.toString(), configuration);
         rocksDBCheckpointDiffer.setRocksDBForCompactionTracking(dbOptions);
@@ -140,7 +142,7 @@ public RDBStore(File dbFile, ManagedDBOptions dbOptions,
         Files.createDirectories(snapshotsParentDirPath);
       }
 
-      if (enableCompactionLog) {
+      if (enableCompactionDag) {
         ColumnFamily ssInfoTableCF = db.getColumnFamily(SNAPSHOT_INFO_TABLE);
         Preconditions.checkNotNull(ssInfoTableCF,
             "SnapshotInfoTable column family handle should not be null");
@@ -179,6 +181,10 @@ public RDBStore(File dbFile, ManagedDBOptions dbOptions,
     }
   }
 
+  public String getSnapshotMetadataDir() {
+    return dbLocation.getParent() + OM_KEY_PREFIX + OM_SNAPSHOT_DIFF_DIR;
+  }
+
   public String getSnapshotsParentDir() {
     return snapshotsParentDir;
   }
@@ -187,6 +193,14 @@ public RocksDBCheckpointDiffer getRocksDBCheckpointDiffer() {
     return rocksDBCheckpointDiffer;
   }
 
+
+  /**
+   * Returns the RocksDB's DBOptions.
+   */
+  public ManagedDBOptions getDbOptions() {
+    return dbOptions;
+  }
+
   @Override
   public void compactDB() throws IOException {
     try (ManagedCompactRangeOptions options =

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java

@@ -82,6 +82,8 @@
 import static org.apache.hadoop.ozone.om.helpers.BucketLayout.OBJECT_STORE;
 import static org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.JobStatus.DONE;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
 import static org.junit.Assert.fail;
 import static org.junit.Assert.assertThrows;
 import static java.nio.charset.StandardCharsets.UTF_8;
@@ -916,15 +918,35 @@ public void testUniqueSnapshotId()
   public void testSnapshotOpensWithDisabledAutoCompaction() throws Exception {
     String snapPrefix = createSnapshot(volumeName, bucketName);
     RDBStore snapshotDBStore = (RDBStore)
-            ((OmSnapshot)cluster.getOzoneManager().getOmSnapshotManager()
+        ((OmSnapshot)cluster.getOzoneManager().getOmSnapshotManager()
             .checkForSnapshot(volumeName, bucketName, snapPrefix))
             .getMetadataManager().getStore();
 
     for (String table : snapshotDBStore.getTableNames().values()) {
       Assertions.assertTrue(snapshotDBStore.getDb().getColumnFamily(table)
-              .getHandle().getDescriptor()
-              .getOptions().disableAutoCompactions());
+          .getHandle().getDescriptor()
+          .getOptions().disableAutoCompactions());
     }
   }
 
+  @Test
+  public void testCompactionDagDisableForSnapshotMetadata() throws Exception {
+    String snapshotName = createSnapshot(volumeName, bucketName);
+
+    RDBStore activeDbStore =
+        (RDBStore) cluster.getOzoneManager().getMetadataManager().getStore();
+    // RocksDBCheckpointDiffer should be not null for active DB store.
+    assertNotNull(activeDbStore.getRocksDBCheckpointDiffer());
+    assertEquals(2,  activeDbStore.getDbOptions().listeners().size());
+
+    OmSnapshot omSnapshot = (OmSnapshot) cluster.getOzoneManager()
+        .getOmSnapshotManager()
+        .checkForSnapshot(volumeName, bucketName, snapshotName);
+
+    RDBStore snapshotDbStore =
+        (RDBStore) omSnapshot.getMetadataManager().getStore();
+    // RocksDBCheckpointDiffer should be null for snapshot DB store.
+    assertNull(snapshotDbStore.getRocksDBCheckpointDiffer());
+    assertEquals(0, snapshotDbStore.getDbOptions().listeners().size());
+  }
 }

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java

@@ -388,7 +388,7 @@ private OmMetadataManagerImpl(OzoneConfiguration conf, File dir, String name)
         RDBCheckpointUtils.waitForCheckpointDirectoryExist(checkpoint);
       }
       setStore(loadDB(conf, metaDir, dbName, false,
-          java.util.Optional.of(Boolean.TRUE), false));
+          java.util.Optional.of(Boolean.TRUE), false, false));
       initializeOmTables(false);
     } catch (IOException e) {
       stop();
@@ -530,7 +530,7 @@ public void start(OzoneConfiguration configuration) throws IOException {
   public static DBStore loadDB(OzoneConfiguration configuration, File metaDir)
       throws IOException {
     return loadDB(configuration, metaDir, OM_DB_NAME, false,
-            java.util.Optional.empty(), true);
+            java.util.Optional.empty(), true, true);
   }
 
   public static DBStore loadDB(OzoneConfiguration configuration, File metaDir,
@@ -539,13 +539,14 @@ public static DBStore loadDB(OzoneConfiguration configuration, File metaDir,
                                        disableAutoCompaction)
           throws IOException {
     return loadDB(configuration, metaDir, dbName, readOnly,
-        disableAutoCompaction, true);
+        disableAutoCompaction, true, true);
   }
 
   public static DBStore loadDB(OzoneConfiguration configuration, File metaDir,
                                String dbName, boolean readOnly,
                                java.util.Optional<Boolean>
                                    disableAutoCompaction,
+                               boolean enableCompactionDag,
                                boolean createCheckpointDirs)
       throws IOException {
     final int maxFSSnapshots = configuration.getInt(
@@ -557,7 +558,7 @@ public static DBStore loadDB(OzoneConfiguration configuration, File metaDir,
         .setOpenReadOnly(readOnly)
         .setPath(Paths.get(metaDir.getPath()))
         .setMaxFSSnapshots(maxFSSnapshots)
-        .setEnableCompactionLog(true)
+        .setEnableCompactionDag(enableCompactionDag)
         .setCreateCheckpointDirs(createCheckpointDirs);
     disableAutoCompaction.ifPresent(
             dbStoreBuilder::disableDefaultCFAutoCompaction);

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/SstFilteringService.java

@@ -150,7 +150,7 @@ public BackgroundTaskResult call() throws Exception {
           try (RDBStore rdbStore = (RDBStore) OmMetadataManagerImpl
               .loadDB(ozoneManager.getConfiguration(),
                       new File(snapshotCheckpointDir),
-                      dbName, true, Optional.of(Boolean.TRUE), false)) {
+                      dbName, true, Optional.of(Boolean.TRUE), false, false)) {
             RocksDatabase db = rdbStore.getDb();
             db.deleteFilesNotMatchingPrefix(prefixPairs, filterFunction);
           }

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java

@@ -34,6 +34,7 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
 import java.util.UUID;
@@ -55,6 +56,7 @@
 import org.apache.commons.io.file.PathUtils;
 import org.apache.hadoop.hdds.StringUtils;
 import org.apache.hadoop.hdds.utils.db.CodecRegistry;
+import org.apache.hadoop.hdds.utils.db.RDBStore;
 import org.apache.hadoop.hdds.utils.db.Table;
 import org.apache.hadoop.ozone.OFSPath;
 import org.apache.hadoop.hdds.utils.db.managed.ManagedSSTDumpTool;
@@ -221,7 +223,9 @@ public SnapshotDiffManager(ManagedRocksDB db,
             .build()
     );
 
-    Path path = Paths.get(differ.getMetadataDir(), "snapDiff");
+    RDBStore rdbStore = (RDBStore) ozoneManager.getMetadataManager().getStore();
+    Objects.requireNonNull(rdbStore, "DBStore can't be null.");
+    Path path = Paths.get(rdbStore.getSnapshotMetadataDir(), "snapDiff");
     createEmptySnapDiffDir(path);
     this.sstBackupDirForSnapDiffJobs = path.toString();