HDDS-8961. Write a standalone tool to create a visual graph from compaction log files.

hadoop-hdds/rocksdb-checkpoint-differ/pom.xml

@@ -82,6 +82,14 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
       <groupId>org.apache.ozone</groupId>
       <artifactId>hdds-rocks-native</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.jgrapht</groupId>
+      <artifactId>jgrapht-core</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.jgrapht</groupId>
+      <artifactId>jgrapht-ext</artifactId>
+    </dependency>
   </dependencies>
 
   <build>

hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RelationshipEdge.java → hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/graph/Edge.java

@@ -6,24 +6,25 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- * <p>
+ *
  * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
+ *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.rocksdiff;
+package org.apache.ozone.graph;
 
-// import org.jgrapht.graph.DefaultEdge;
-// Enable this import and extend DefaultEdge if We need to
-// pcitorially represent the DAG constructed.
+import org.jgrapht.graph.DefaultEdge;
 
-//class RelationshipEdge extends DefaultEdge {
-class RelationshipEdge {
-  //@Override
+/**
+ * Overrides the {@link DefaultEdge} so that it doesn't print source and target
+ * vertex names in the image.
+ */
+public class Edge extends DefaultEdge {
+  @Override
   public String toString() {
     return "";
   }

hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/graph/PrintableGraph.java

@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ozone.graph;
+
+import com.google.common.graph.EndpointPair;
+import com.google.common.graph.MutableGraph;
+import com.mxgraph.layout.hierarchical.mxHierarchicalLayout;
+import com.mxgraph.layout.mxIGraphLayout;
+import com.mxgraph.util.mxCellRenderer;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.ozone.rocksdiff.CompactionNode;
+import org.jgrapht.Graph;
+import org.jgrapht.ext.JGraphXAdapter;
+import org.jgrapht.graph.DefaultDirectedGraph;
+
+import javax.imageio.ImageIO;
+import java.awt.Color;
+import java.awt.image.BufferedImage;
+import java.io.File;
+import java.io.IOException;
+
+/**
+ * Wrapped over {@link Graph} to get an image of {@link MutableGraph}.
+ */
+public class PrintableGraph {
+
+  /**
+   * Enum to print different type of node's name in the graph image.
+   */
+  public enum GraphType {
+    /**
+     * To use SST file name as node name.
+     */
+    FILE_NAME,
+
+    /**
+     * To use SST file name and total key in the file as node name.
+     */
+    KEY_SIZE,
+
+    /**
+     * To use SST file name and cumulative key as node name.
+     */
+    CUMULATIVE_SIZE
+  }
+
+  private final Graph<String, Edge> graph;
+
+  public PrintableGraph(MutableGraph<CompactionNode> guavaGraph,
+                        GraphType graphType) {
+    this.graph = getGraph(guavaGraph, graphType);
+  }
+
+  public void generateImage(String fileName) throws IOException {
+    if (CollectionUtils.isEmpty(graph.vertexSet())) {
+      throw new IOException("Graph is empty.");
+    }
+
+    JGraphXAdapter<String, Edge> jGraphXAdapter = new JGraphXAdapter<>(graph);
+    mxIGraphLayout mxIGraphLayout =
+        new mxHierarchicalLayout(jGraphXAdapter);
+    mxIGraphLayout.execute(jGraphXAdapter.getDefaultParent());
+
+    BufferedImage bufferedImage =
+        mxCellRenderer.createBufferedImage(jGraphXAdapter, null, 2,
+            Color.WHITE, true, null);
+
+    File newFIle = new File(fileName);
+    ImageIO.write(bufferedImage, "PNG", newFIle);
+  }
+
+  /**
+   * Convert guava's {@link MutableGraph} to jgrapht's {@link Graph}.
+   */
+  public Graph<String, Edge> getGraph(
+      MutableGraph<CompactionNode> guavaGraph,
+      GraphType graphType
+  ) {
+
+    Graph<String, Edge> jgrapht =
+        new DefaultDirectedGraph<>(Edge.class);
+
+    for (CompactionNode node : guavaGraph.nodes()) {
+      jgrapht.addVertex(getVertex(node, graphType));
+    }
+
+    for (EndpointPair<CompactionNode> edge : guavaGraph.edges()) {
+      jgrapht.addEdge(getVertex(edge.source(), graphType),
+          getVertex(edge.target(), graphType));
+    }
+
+    return jgrapht;
+  }
+
+  private String getVertex(CompactionNode node, GraphType graphType) {
+    switch (graphType) {
+    case KEY_SIZE:
+      return
+          node.getFileName() + "::" + node.getTotalNumberOfKeys();
+    case CUMULATIVE_SIZE:
+      return
+          node.getFileName() + "::" + node.getCumulativeKeysReverseTraversal();
+    case FILE_NAME:
+    default:
+      return node.getFileName();
+    }
+  }
+}

hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/graph/package-info.java

@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ozone.graph;
+
+/**
+ * Generic ozone specific classes.
+ */

hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java

@@ -21,9 +21,11 @@
 import com.google.common.base.Preconditions;
 import com.google.common.graph.GraphBuilder;
 import com.google.common.graph.MutableGraph;
+
 import java.io.FileNotFoundException;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.Objects;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
@@ -37,6 +39,8 @@
 import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator;
 import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksDB;
 import org.apache.ozone.rocksdb.util.RdbUtil;
+import org.apache.ozone.graph.PrintableGraph;
+import org.apache.ozone.graph.PrintableGraph.GraphType;
 import org.rocksdb.AbstractEventListener;
 import org.rocksdb.ColumnFamilyHandle;
 import org.rocksdb.CompactionJobInfo;
@@ -1081,11 +1085,13 @@ void dumpCompactionNodeTable() {
     }
   }
 
-  public MutableGraph<CompactionNode> getForwardCompactionDAG() {
+  @VisibleForTesting
+  MutableGraph<CompactionNode> getForwardCompactionDAG() {
     return forwardCompactionDAG;
   }
 
-  public MutableGraph<CompactionNode> getBackwardCompactionDAG() {
+  @VisibleForTesting
+  MutableGraph<CompactionNode> getBackwardCompactionDAG() {
     return backwardCompactionDAG;
   }
 
@@ -1540,4 +1546,18 @@ public static RocksDBCheckpointDiffer getInstance(
   public BootstrapStateHandler.Lock getBootstrapStateLock() {
     return lock;
   }
+
+  public String pngPrintMutableGraph(String fileName, GraphType graphType)
+      throws IOException {
+    Objects.requireNonNull(fileName, "Image file name is required.");
+    Objects.requireNonNull(graphType, "Graph type is required.");
+
+    PrintableGraph graph;
+    synchronized (this) {
+      graph = new PrintableGraph(forwardCompactionDAG, graphType);
+    }
+
+    graph.generateImage(fileName);
+    return fileName;
+  }
 }

hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/package-info.java

@@ -16,7 +16,7 @@
  * limitations under the License.
  */
 
-package org.apache.hadoop.ozone;
+package org.apache.ozone.rocksdiff;
 
 /**
  * Generic ozone specific classes.

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java

@@ -593,6 +593,18 @@ public Iterator<? extends OzoneSnapshot> listSnapshot(
         volumeName, bucketName, snapshotPrefix, prevSnapshot);
   }
 
+  /**
+   * Create an image of the current compaction log DAG in the OM.
+   * @param fileName     name of the image file.
+   * @param graphType    type of node name to use in the graph image.
+   * @return path of the image file.
+   * @throws IOException
+   */
+  public String printCompactionLogDag(String fileName,
+                                      String graphType) throws IOException {
+    return proxy.printCompactionLogDag(fileName, graphType);
+  }
+
   /**
    * An Iterator to iterate over {@link OzoneSnapshot} list.
    */

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java

@@ -1045,6 +1045,16 @@ String createSnapshot(String volumeName,
   void deleteSnapshot(String volumeName,
       String bucketName, String snapshotName) throws IOException;
 
+  /**
+   * Create an image of the current compaction log DAG in the OM.
+   * @param fileName     name of the image file.
+   * @param graphType    type of node name to use in the graph image.
+   * @return path of the image file.
+   * @throws IOException
+   */
+  String printCompactionLogDag(String fileName, String graphType)
+      throws IOException;
+
   /**
    * List snapshots in a volume/bucket.
    * @param volumeName     volume name

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java

@@ -987,6 +987,19 @@ public void deleteSnapshot(String volumeName,
     ozoneManagerClient.deleteSnapshot(volumeName, bucketName, snapshotName);
   }
 
+  /**
+   * Create an image of the current compaction log DAG in the OM.
+   * @param fileName     name of the image file.
+   * @param graphType    type of node name to use in the graph image.
+   * @return path of the image file.
+   * @throws IOException
+   */
+  @Override
+  public String printCompactionLogDag(String fileName,
+                                      String graphType) throws IOException {
+    return ozoneManagerClient.printCompactionLogDag(fileName, graphType);
+  }
+
   @Override
   public SnapshotDiffResponse snapshotDiff(String volumeName,
                                            String bucketName,

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java

@@ -270,6 +270,7 @@ public static boolean isReadOnly(
     case ListSnapshotDiffJobs:
     case TransferLeadership:
     case SetSafeMode:
+    case PrintCompactionLogDag:
       return true;
     case CreateVolume:
     case SetVolumeProperty:

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java

@@ -691,6 +691,19 @@ default void deleteSnapshot(String volumeName,
         "this to be implemented");
   }
 
+  /**
+   * Create an image of the current compaction log DAG in the OM.
+   * @param fileName     name of the image file.
+   * @param graphType    type of node name to use in the graph image.
+   * @return path of the image file.
+   * @throws IOException
+   */
+  default String printCompactionLogDag(String fileName, String graphType)
+      throws IOException {
+    throw new UnsupportedOperationException("OzoneManager does not require " +
+        "this to be implemented");
+  }
+
   /**
    * List snapshots in a volume/bucket.
    * @param volumeName     volume name

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java

@@ -31,8 +31,7 @@
 import org.apache.hadoop.hdds.client.ECReplicationConfig;
 import org.apache.hadoop.hdds.client.ReplicationConfig;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos.TransferLeadershipRequestProto;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos
-    .UpgradeFinalizationStatus;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos.UpgradeFinalizationStatus;
 import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList;
 import org.apache.hadoop.hdds.tracing.TracingUtil;
 import org.apache.hadoop.io.Text;
@@ -153,6 +152,7 @@
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneAclInfo;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneFileStatusProto;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PrintCompactionLogDagRequest;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RangerBGSyncRequest;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RangerBGSyncResponse;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RecoverLeaseRequest;
@@ -1190,6 +1190,30 @@ public void deleteSnapshot(String volumeName,
     handleError(omResponse);
   }
 
+  /**
+   * {@inheritDoc}
+   */
+  @Override
+  public String printCompactionLogDag(String fileName, String graphType)
+      throws IOException {
+    final PrintCompactionLogDagRequest.Builder request =
+        PrintCompactionLogDagRequest.newBuilder();
+
+    if (fileName != null) {
+      request.setFileName(fileName);
+    }
+    if (graphType != null) {
+      request.setGraphType(graphType);
+    }
+
+    final OMRequest omRequest = createOMRequest(Type.PrintCompactionLogDag)
+        .setPrintCompactionLogDagRequest(request.build())
+        .build();
+    final OMResponse omResponse = submitRequest(omRequest);
+    handleError(omResponse);
+    return omResponse.getPrintCompactionLogDagResponse().getImagePath();
+  }
+
   /**
    * {@inheritDoc}
    */