HDDS-7798. [Snapshot] Add SnapDiff test cases; add config to allow disabling native libs for SnapDiff

hadoop-hdds/common/src/main/resources/ozone-default.xml

@@ -3942,6 +3942,15 @@
     </description>
   </property>
 
+  <property>
+    <name>ozone.om.snapshot.force.non.native.diff</name>
+    <value>false</value>
+    <tag>OZONE, OM</tag>
+    <description>
+      Flag to perform snapshot diff without using native libs(can be slow).
+    </description>
+  </property>
+
   <property>
     <name>ozone.om.snapshot.diff.max.page.size</name>
     <value>1000</value>

hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdb/util/TestManagedSstFileReader.java

@@ -33,8 +33,6 @@
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.ValueSource;
 import org.rocksdb.RocksDBException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 import java.io.File;
 import java.io.IOException;
@@ -58,9 +56,6 @@
  */
 class TestManagedSstFileReader {
 
-  private static final Logger LOG =
-      LoggerFactory.getLogger(TestManagedSstFileReader.class);
-
   // Key prefix containing all characters, to check if all characters can be
   // written & read from rocksdb through SSTDumptool
   private static final String KEY_PREFIX = IntStream.range(0, 256).boxed()
@@ -158,6 +153,5 @@ public void testGetKeyStreamWithTombstone(int numberOfFiles)
     } finally {
       executorService.shutdown();
     }
-
   }
 }

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java

@@ -668,19 +668,23 @@ private List<OzoneSnapshot> getNextListOfSnapshots(String prevSnapshot)
    * @param token to get the index to return diff report from.
    * @param pageSize maximum entries returned to the report.
    * @param forceFullDiff request to force full diff, skipping DAG optimization
+   * @param forceNonNativeDiff request to force diff to perform diffs without
+   *                           native lib
    * @return the difference report between two snapshots
    * @throws IOException in case of any exception while generating snapshot diff
    */
+  @SuppressWarnings("parameternumber")
   public SnapshotDiffResponse snapshotDiff(String volumeName,
                                            String bucketName,
                                            String fromSnapshot,
                                            String toSnapshot,
                                            String token,
                                            int pageSize,
-                                           boolean forceFullDiff)
+                                           boolean forceFullDiff,
+                                           boolean forceNonNativeDiff)
       throws IOException {
     return proxy.snapshotDiff(volumeName, bucketName, fromSnapshot, toSnapshot,
-        token, pageSize, forceFullDiff);
+        token, pageSize, forceFullDiff, forceNonNativeDiff);
   }
 
   /**

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java

@@ -1071,10 +1071,12 @@ List<OzoneSnapshot> listSnapshot(
    * @return the difference report between two snapshots
    * @throws IOException in case of any exception while generating snapshot diff
    */
+  @SuppressWarnings("parameternumber")
   SnapshotDiffResponse snapshotDiff(String volumeName, String bucketName,
                                     String fromSnapshot, String toSnapshot,
                                     String token, int pageSize,
-                                    boolean forceFullDiff)
+                                    boolean forceFullDiff,
+                                    boolean forceNonNativeDiff)
       throws IOException;
 
   /**

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java

@@ -987,14 +987,16 @@ public SnapshotDiffResponse snapshotDiff(String volumeName,
                                            String toSnapshot,
                                            String token,
                                            int pageSize,
-                                           boolean forceFullDiff)
+                                           boolean forceFullDiff,
+                                           boolean forceNonNativeDiff)
       throws IOException {
     Preconditions.checkArgument(StringUtils.isNotBlank(volumeName),
         "volume can't be null or empty.");
     Preconditions.checkArgument(StringUtils.isNotBlank(bucketName),
         "bucket can't be null or empty.");
     return ozoneManagerClient.snapshotDiff(volumeName, bucketName,
-        fromSnapshot, toSnapshot, token, pageSize, forceFullDiff);
+        fromSnapshot, toSnapshot, token, pageSize, forceFullDiff,
+        forceNonNativeDiff);
   }
 
   @Override

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java

@@ -489,6 +489,11 @@ private OMConfigKeys() {
 
   public static final boolean OZONE_OM_SNAPSHOT_FORCE_FULL_DIFF_DEFAULT = false;
 
+  public static final String OZONE_OM_SNAPSHOT_FORCE_NON_NATIVE_DIFF =
+      "ozone.om.snapshot.force.non.native.diff";
+
+  public static final boolean OZONE_OM_SNAPSHOT_FORCE_NON_NATIVE_DIFF_DEFAULT =
+      false;
   public static final String OZONE_OM_SNAPSHOT_DIFF_DB_DIR
       = "ozone.om.snapshot.diff.db.dir";
 

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyLocationInfoGroup.java

@@ -179,7 +179,7 @@ void addAll(long versionToAdd, List<OmKeyLocationInfo> locationInfoList) {
   public String toString() {
     StringBuilder sb = new StringBuilder();
     sb.append("version:").append(version).append(" ");
-    sb.append("isMultipartKey:").append(isMultipartKey);
+    sb.append("isMultipartKey:").append(isMultipartKey).append(" ");
     for (List<OmKeyLocationInfo> kliList : locationVersionMap.values()) {
       for (OmKeyLocationInfo kli: kliList) {
         sb.append(kli.getLocalID()).append(" || ");

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotDiffJob.java

@@ -48,6 +48,7 @@ public static Codec<SnapshotDiffJob> getCodec() {
   private String fromSnapshot;
   private String toSnapshot;
   private boolean forceFullDiff;
+  private boolean forceNonNativeDiff;
   private long totalDiffEntries;
 
   // Reason tells why the job was FAILED. It should be set only if job status
@@ -68,6 +69,7 @@ public SnapshotDiffJob(long creationTime,
                          String fromSnapshot,
                          String toSnapshot,
                          boolean forceFullDiff,
+                         boolean forceNonNativeDiff,
                          long totalDiffEntries) {
     this.creationTime = creationTime;
     this.jobId = jobId;
@@ -77,6 +79,7 @@ public SnapshotDiffJob(long creationTime,
     this.fromSnapshot = fromSnapshot;
     this.toSnapshot = toSnapshot;
     this.forceFullDiff = forceFullDiff;
+    this.forceNonNativeDiff = forceNonNativeDiff;
     this.totalDiffEntries = totalDiffEntries;
     this.reason = StringUtils.EMPTY;
   }
@@ -161,6 +164,14 @@ public void setReason(String reason) {
     this.reason = reason;
   }
 
+  public boolean isForceNonNativeDiff() {
+    return forceNonNativeDiff;
+  }
+
+  public void setForceNonNativeDiff(boolean forceNonNativeDiff) {
+    this.forceNonNativeDiff = forceNonNativeDiff;
+  }
+
   @Override
   public String toString() {
     StringBuilder sb = new StringBuilder("creationTime : ").append(creationTime)
@@ -171,6 +182,7 @@ public String toString() {
         .append(", fromSnapshot: ").append(fromSnapshot)
         .append(", toSnapshot: ").append(toSnapshot)
         .append(", forceFullDiff: ").append(forceFullDiff)
+        .append(", forceNonNativeDiff: ").append(forceNonNativeDiff)
         .append(", totalDiffEntries: ").append(totalDiffEntries);
 
     if (StringUtils.isNotEmpty(reason)) {
@@ -196,15 +208,17 @@ public boolean equals(Object other) {
           Objects.equals(this.toSnapshot, otherJob.toSnapshot) &&
           Objects.equals(this.forceFullDiff, otherJob.forceFullDiff) &&
           Objects.equals(this.totalDiffEntries, otherJob.totalDiffEntries) &&
-          Objects.equals(this.reason, otherJob.reason);
+          Objects.equals(this.forceNonNativeDiff, otherJob.forceNonNativeDiff)
+          && Objects.equals(this.reason, otherJob.reason);
     }
     return false;
   }
 
   @Override
   public int hashCode() {
     return Objects.hash(creationTime, jobId, status, volume, bucket,
-        fromSnapshot, toSnapshot, forceFullDiff, totalDiffEntries, reason);
+        fromSnapshot, toSnapshot, forceFullDiff, forceNonNativeDiff,
+        totalDiffEntries, reason);
   }
 
   public SnapshotDiffJobProto toProtoBuf() {
@@ -217,6 +231,7 @@ public SnapshotDiffJobProto toProtoBuf() {
         .setFromSnapshot(fromSnapshot)
         .setToSnapshot(toSnapshot)
         .setForceFullDiff(forceFullDiff)
+        .setForceNonNativeDiff(forceNonNativeDiff)
         .setTotalDiffEntries(totalDiffEntries)
         .build();
   }
@@ -232,6 +247,7 @@ public static SnapshotDiffJob getFromProtoBuf(
         diffJobProto.getFromSnapshot(),
         diffJobProto.getToSnapshot(),
         diffJobProto.getForceFullDiff(),
+        diffJobProto.getForceNonNativeDiff(),
         diffJobProto.getTotalDiffEntries());
   }
 

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java

@@ -719,13 +719,15 @@ default List<SnapshotInfo> listSnapshot(
    * @return the difference report between two snapshots
    * @throws IOException in case of any exception while generating snapshot diff
    */
+  @SuppressWarnings("parameternumber")
   default SnapshotDiffResponse snapshotDiff(String volumeName,
                                             String bucketName,
                                             String fromSnapshot,
                                             String toSnapshot,
                                             String token,
                                             int pageSize,
-                                            boolean forceFullDiff)
+                                            boolean forceFullDiff,
+                                            boolean forceNonNativeDiff)
       throws IOException {
     throw new UnsupportedOperationException("OzoneManager does not require " +
         "this to be implemented");

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java

@@ -1228,7 +1228,8 @@ public SnapshotDiffResponse snapshotDiff(String volumeName,
                                            String toSnapshot,
                                            String token,
                                            int pageSize,
-                                           boolean forceFullDiff)
+                                           boolean forceFullDiff,
+                                           boolean forceNonNativeDiff)
       throws IOException {
     final OzoneManagerProtocolProtos.SnapshotDiffRequest.Builder
         requestBuilder =
@@ -1238,7 +1239,8 @@ public SnapshotDiffResponse snapshotDiff(String volumeName,
             .setFromSnapshot(fromSnapshot)
             .setToSnapshot(toSnapshot)
             .setPageSize(pageSize)
-            .setForceFullDiff(forceFullDiff);
+            .setForceFullDiff(forceFullDiff)
+            .setForceNonNativeDiff(forceNonNativeDiff);
 
     if (!StringUtils.isBlank(token)) {
       requestBuilder.setToken(token);