HDDS-12053. Make print-log-dag command run locally and offline

hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java

@@ -65,10 +65,8 @@
 import org.apache.hadoop.hdds.utils.IOUtils;
 import org.apache.hadoop.hdds.utils.Scheduler;
 import org.apache.hadoop.hdds.utils.db.managed.ManagedDBOptions;
-import org.apache.hadoop.hdds.utils.db.managed.ManagedOptions;
 import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksDB;
 import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator;
-import org.apache.hadoop.hdds.utils.db.managed.ManagedSstFileReader;
 import org.apache.hadoop.ozone.lock.BootstrapStateHandler;
 import org.apache.ozone.compaction.log.CompactionFileInfo;
 import org.apache.ozone.compaction.log.CompactionLogEntry;
@@ -81,7 +79,6 @@
 import org.rocksdb.LiveFileMetaData;
 import org.rocksdb.RocksDB;
 import org.rocksdb.RocksDBException;
-import org.rocksdb.TableProperties;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -322,26 +319,10 @@ public void close() {
   private final MutableGraph<CompactionNode> backwardCompactionDAG =
       GraphBuilder.directed().build();
 
-  public static final Integer DEBUG_DAG_BUILD_UP = 2;
-  public static final Integer DEBUG_DAG_TRAVERSAL = 3;
-  public static final Integer DEBUG_DAG_LIVE_NODES = 4;
-  public static final Integer DEBUG_READ_ALL_DB_KEYS = 5;
-  private static final HashSet<Integer> DEBUG_LEVEL = new HashSet<>();
-
-  static {
-    addDebugLevel(DEBUG_DAG_BUILD_UP);
-    addDebugLevel(DEBUG_DAG_TRAVERSAL);
-    addDebugLevel(DEBUG_DAG_LIVE_NODES);
-  }
-
   static {
     RocksDB.loadLibrary();
   }
 
-  public static void addDebugLevel(Integer level) {
-    DEBUG_LEVEL.add(level);
-  }
-
   public void setRocksDBForCompactionTracking(ManagedDBOptions rocksOptions) {
     List<AbstractEventListener> events = new ArrayList<>();
     events.add(newCompactionBeginListener());
@@ -576,31 +557,6 @@ private void createLink(Path link, Path source) {
     }
   }
 
-  /**
-   * Get number of keys in an SST file.
-   * @param filename SST filename
-   * @return number of keys
-   */
-  private long getSSTFileSummary(String filename)
-      throws RocksDBException, FileNotFoundException {
-
-    if (!filename.endsWith(SST_FILE_EXTENSION)) {
-      filename += SST_FILE_EXTENSION;
-    }
-
-    try (ManagedOptions option = new ManagedOptions();
-         ManagedSstFileReader reader = new ManagedSstFileReader(option)) {
-
-      reader.open(getAbsoluteSstFilePath(filename));
-
-      TableProperties properties = reader.getTableProperties();
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("{} has {} keys", filename, properties.getNumEntries());
-      }
-      return properties.getNumEntries();
-    }
-  }
-
   private String getAbsoluteSstFilePath(String filename)
       throws FileNotFoundException {
     if (!filename.endsWith(SST_FILE_EXTENSION)) {
@@ -1065,7 +1021,7 @@ private CompactionNode addNodeToDAG(String file, long seqNum, String startKey,
                                       String endKey, String columnFamily) {
     long numKeys = 0L;
     try {
-      numKeys = getSSTFileSummary(file);
+      numKeys = RocksDiffUtils.getSSTFileSummary(getAbsoluteSstFilePath(file));
     } catch (RocksDBException e) {
       LOG.warn("Can't get num of keys in SST '{}': {}", file, e.getMessage());
     } catch (FileNotFoundException e) {
@@ -1386,11 +1342,6 @@ public boolean shouldRun() {
     return !suspended.get();
   }
 
-  @VisibleForTesting
-  public boolean debugEnabled(Integer level) {
-    return DEBUG_LEVEL.contains(level);
-  }
-
   @VisibleForTesting
   public static Logger getLog() {
     return LOG;

hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDiffUtils.java

@@ -18,8 +18,10 @@
 package org.apache.ozone.rocksdiff;
 
 import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX;
+import static org.apache.ozone.rocksdiff.RocksDBCheckpointDiffer.SST_FILE_EXTENSION;
 
 import com.google.common.annotations.VisibleForTesting;
+import java.io.FileNotFoundException;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Iterator;
@@ -28,9 +30,13 @@
 import org.apache.commons.collections.MapUtils;
 import org.apache.commons.io.FilenameUtils;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedOptions;
 import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksDB;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedSstFileReader;
 import org.apache.ozone.compaction.log.CompactionFileInfo;
 import org.rocksdb.LiveFileMetaData;
+import org.rocksdb.RocksDBException;
+import org.rocksdb.TableProperties;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -136,4 +142,29 @@ static boolean shouldSkipNode(CompactionNode node,
     return !isKeyWithPrefixPresent(keyPrefix, node.getStartKey(),
         node.getEndKey());
   }
+
+  /**
+   * Get number of keys in an SST file.
+   * @param filename absolute path of SST file
+   * @return number of keys
+   */
+  public static long getSSTFileSummary(String filename)
+      throws RocksDBException, FileNotFoundException {
+
+    if (!filename.endsWith(SST_FILE_EXTENSION)) {
+      filename += SST_FILE_EXTENSION;
+    }
+
+    try (ManagedOptions option = new ManagedOptions();
+         ManagedSstFileReader reader = new ManagedSstFileReader(option)) {
+
+      reader.open(filename);
+
+      TableProperties properties = reader.getTableProperties();
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("{} has {} keys", filename, properties.getNumEntries());
+      }
+      return properties.getNumEntries();
+    }
+  }
 }

hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java

@@ -28,8 +28,6 @@
 import static org.apache.hadoop.util.Time.now;
 import static org.apache.ozone.rocksdiff.RocksDBCheckpointDiffer.COLUMN_FAMILIES_TO_TRACK_IN_DAG;
 import static org.apache.ozone.rocksdiff.RocksDBCheckpointDiffer.COMPACTION_LOG_FILE_NAME_SUFFIX;
-import static org.apache.ozone.rocksdiff.RocksDBCheckpointDiffer.DEBUG_DAG_LIVE_NODES;
-import static org.apache.ozone.rocksdiff.RocksDBCheckpointDiffer.DEBUG_READ_ALL_DB_KEYS;
 import static org.apache.ozone.rocksdiff.RocksDBCheckpointDiffer.SST_FILE_EXTENSION;
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -152,6 +150,18 @@ public class TestRocksDBCheckpointDiffer {
   private ColumnFamilyHandle fileTableCFHandle;
   private ColumnFamilyHandle compactionLogTableCFHandle;
 
+  public static final Integer DEBUG_DAG_BUILD_UP = 2;
+  public static final Integer DEBUG_DAG_TRAVERSAL = 3;
+  public static final Integer DEBUG_DAG_LIVE_NODES = 4;
+  public static final Integer DEBUG_READ_ALL_DB_KEYS = 5;
+  private static final HashSet<Integer> DEBUG_LEVEL = new HashSet<>();
+
+  static {
+    DEBUG_LEVEL.add(DEBUG_DAG_BUILD_UP);
+    DEBUG_LEVEL.add(DEBUG_DAG_TRAVERSAL);
+    DEBUG_LEVEL.add(DEBUG_DAG_LIVE_NODES);
+  }
+
   @BeforeEach
   public void init() throws RocksDBException {
     // Checkpoint differ log level. Set to DEBUG for verbose output
@@ -877,15 +887,15 @@ private void readRocksDBInstance(String dbPathArg,
         LOG.debug("\tLevel: {}", m.level());
         LOG.debug("\tTable: {}", bytes2String(m.columnFamilyName()));
         LOG.debug("\tKey Range: {}", bytes2String(m.smallestKey()) + " <-> " + bytes2String(m.largestKey()));
-        if (differ.debugEnabled(DEBUG_DAG_LIVE_NODES)) {
+        if (debugEnabled(DEBUG_DAG_LIVE_NODES)) {
           printMutableGraphFromAGivenNode(
               differ.getCompactionNodeMap(),
               m.fileName(), m.level(),
               differ.getForwardCompactionDAG());
         }
       }
 
-      if (differ.debugEnabled(DEBUG_READ_ALL_DB_KEYS)) {
+      if (debugEnabled(DEBUG_READ_ALL_DB_KEYS)) {
         try (ManagedRocksIterator iter = new ManagedRocksIterator(rocksDB.get().newIterator())) {
           for (iter.get().seekToFirst(); iter.get().isValid(); iter.get().next()) {
             LOG.debug(
@@ -907,6 +917,10 @@ private void readRocksDBInstance(String dbPathArg,
     }
   }
 
+  public boolean debugEnabled(Integer level) {
+    return DEBUG_LEVEL.contains(level);
+  }
+
   /**
    * Helper that traverses the graphs for testing.
    * @param compactionNodeMap

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java

@@ -1208,6 +1208,7 @@ OzoneSnapshot getSnapshotInfo(String volumeName,
    * @return message which tells the image name, parent dir and OM leader
    * node information.
    */
+  @Deprecated
   String printCompactionLogDag(String fileNamePrefix, String graphType)
       throws IOException;
 

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java

@@ -1027,6 +1027,7 @@ public OzoneSnapshot getSnapshotInfo(String volumeName,
    * @return message which tells the image name, parent dir and OM leader
    * node information.
    */
+  @Deprecated
   @Override
   public String printCompactionLogDag(String fileNamePrefix,
                                       String graphType) throws IOException {

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java

@@ -273,6 +273,7 @@ public static boolean isReadOnly(
     case TransferLeadership:
     case SetSafeMode:
     case PrintCompactionLogDag:
+      // deprecated by HDDS-12053, keeping it here for compatibility
     case GetSnapshotInfo:
     case GetObjectTagging:
     case GetQuotaRepairStatus:

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java

@@ -777,6 +777,7 @@ default SnapshotInfo getSnapshotInfo(String volumeName,
    * @return message which tells the image name, parent dir and OM leader
    * node information.
    */
+  @Deprecated
   default String printCompactionLogDag(String fileNamePrefix, String graphType)
       throws IOException {
     throw new UnsupportedOperationException("OzoneManager does not require " +

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java

@@ -1329,6 +1329,7 @@ public SnapshotInfo getSnapshotInfo(String volumeName, String bucketName,
   /**
    * {@inheritDoc}
    */
+  @Deprecated
   @Override
   public String printCompactionLogDag(String fileNamePrefix, String graphType)
       throws IOException {

hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto

@@ -141,7 +141,7 @@ enum Type {
   ListSnapshotDiffJobs = 122;
   CancelSnapshotDiff = 123;
   SetSafeMode = 124;
-  PrintCompactionLogDag = 125;
+  PrintCompactionLogDag = 125; // [deprecated = true] by HDDS-12053
   ListKeysLight = 126;
   AbortExpiredMultiPartUploads = 127;
   SetSnapshotProperty = 128;
@@ -287,7 +287,7 @@ message OMRequest {
   optional ListSnapshotDiffJobRequest       ListSnapshotDiffJobRequest     = 122;
   optional CancelSnapshotDiffRequest        CancelSnapshotDiffRequest      = 123;
   optional SetSafeModeRequest               SetSafeModeRequest             = 124;
-  optional PrintCompactionLogDagRequest     PrintCompactionLogDagRequest   = 125;
+  optional PrintCompactionLogDagRequest     PrintCompactionLogDagRequest   = 125 [deprecated = true];
 
   optional MultipartUploadsExpiredAbortRequest multipartUploadsExpiredAbortRequest = 126;
   optional SetSnapshotPropertyRequest       SetSnapshotPropertyRequest     = 127;
@@ -420,7 +420,7 @@ message OMResponse {
   optional ListSnapshotDiffJobResponse       ListSnapshotDiffJobResponse   = 122;
   optional CancelSnapshotDiffResponse        cancelSnapshotDiffResponse    = 123;
   optional SetSafeModeResponse               SetSafeModeResponse           = 124;
-  optional PrintCompactionLogDagResponse     PrintCompactionLogDagResponse = 125;
+  optional PrintCompactionLogDagResponse     PrintCompactionLogDagResponse = 125 [deprecated = true];
   optional ListKeysLightResponse             listKeysLightResponse         = 126;
   optional MultipartUploadsExpiredAbortResponse multipartUploadsExpiredAbortResponse = 127;
   optional SetSnapshotPropertyResponse       SetSnapshotPropertyResponse   = 128;

hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java

@@ -723,6 +723,7 @@ public OzoneSnapshot getSnapshotInfo(String volumeName, String bucketName,
   }
 
   @Override
+  @Deprecated
   public String printCompactionLogDag(String fileNamePrefix,
                                       String graphType) throws IOException {
     return null;