HDDS-11914. Snapshot diff should not filter SST Files based by reading SST file reader

hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRocksDB.java

@@ -18,6 +18,7 @@
  */
 package org.apache.hadoop.hdds.utils.db.managed;
 
+import org.apache.commons.io.FilenameUtils;
 import org.rocksdb.ColumnFamilyDescriptor;
 import org.rocksdb.ColumnFamilyHandle;
 import org.rocksdb.DBOptions;
@@ -31,13 +32,21 @@
 import java.io.IOException;
 import java.time.Duration;
 import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
 
 /**
  * Managed {@link RocksDB}.
  */
 public class ManagedRocksDB extends ManagedObject<RocksDB> {
   public static final Class<RocksDB> ORIGINAL_CLASS = RocksDB.class;
   public static final int NOT_FOUND = RocksDB.NOT_FOUND;
+  /**
+   * SST file extension. Must be lower case.
+   * Used to trim the file extension when writing compaction entries to the log
+   * to save space.
+   */
+  public static final String SST_FILE_EXTENSION = ".sst";
 
   static final Logger LOG = LoggerFactory.getLogger(ManagedRocksDB.class);
 
@@ -102,4 +111,15 @@ public void deleteFile(LiveFileMetaData fileToBeDeleted)
     File file = new File(fileToBeDeleted.path(), fileToBeDeleted.fileName());
     ManagedRocksObjectUtils.waitForFileDelete(file, Duration.ofSeconds(60));
   }
+
+  public static Map<String, LiveFileMetaData> getLiveMetadataForSSTFiles(RocksDB db) {
+    return db.getLiveFilesMetaData().stream()
+        .filter(liveFileMetaData -> liveFileMetaData.fileName().endsWith(SST_FILE_EXTENSION)).collect(
+            Collectors.toMap(liveFileMetaData -> FilenameUtils.getBaseName(liveFileMetaData.fileName()),
+                liveFileMetaData -> liveFileMetaData));
+  }
+
+  public Map<String, LiveFileMetaData> getLiveMetadataForSSTFiles() {
+    return getLiveMetadataForSSTFiles(this.get());
+  }
 }

hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/compaction/log/CompactionFileInfo.java

@@ -20,7 +20,9 @@
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdds.StringUtils;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.rocksdb.LiveFileMetaData;
 
 import java.util.Objects;
 
@@ -128,6 +130,16 @@ public Builder setColumnFamily(String columnFamily) {
       return this;
     }
 
+    public Builder setValues(LiveFileMetaData fileMetaData) {
+      if (fileMetaData != null) {
+        String columnFamilyName = StringUtils.bytes2String(fileMetaData.columnFamilyName());
+        String startRangeValue = StringUtils.bytes2String(fileMetaData.smallestKey());
+        String endRangeValue = StringUtils.bytes2String(fileMetaData.largestKey());
+        this.setColumnFamily(columnFamilyName).setStartRange(startRangeValue).setEndRange(endRangeValue);
+      }
+      return this;
+    }
+
     public CompactionFileInfo build() {
       if ((startRange != null || endRange != null || columnFamily != null) &&
           (startRange == null || endRange == null || columnFamily == null)) {

hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/CompactionNode.java

@@ -17,6 +17,8 @@
  */
 package org.apache.ozone.rocksdiff;
 
+import org.apache.ozone.compaction.log.CompactionFileInfo;
+
 /**
  * Node in the compaction DAG that represents an SST file.
  */
@@ -48,6 +50,11 @@ public CompactionNode(String file, long numKeys, long seqNum,
     this.columnFamily = columnFamily;
   }
 
+  public CompactionNode(CompactionFileInfo compactionFileInfo) {
+    this(compactionFileInfo.getFileName(), -1, -1, compactionFileInfo.getStartKey(),
+        compactionFileInfo.getEndKey(), compactionFileInfo.getColumnFamily());
+  }
+
   @Override
   public String toString() {
     return String.format("Node{%s}", fileName);