[DO NOT MERGE] Backport some fixes from master to ozone-1.4

.github/workflows/populate-cache.yml

@@ -54,7 +54,7 @@ jobs:
 
       - name: Fetch dependencies
         if: steps.restore-cache.outputs.cache-hit != 'true'
-        run: mvn --batch-mode --fail-never --no-transfer-progress --show-version dependency:go-offline
+        run: mvn --batch-mode --fail-never --no-transfer-progress --show-version -Pgo-offline -Pdist clean verify
 
       - name: Delete Ozone jars from repo
         if: steps.restore-cache.outputs.cache-hit != 'true'

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/IOUtils.java

@@ -68,7 +68,7 @@ public static void close(Logger logger, AutoCloseable... closeables) {
    * Close each argument, catching exceptions and logging them as error.
    */
   public static void close(Logger logger,
-      Collection<AutoCloseable> closeables) {
+      Collection<? extends AutoCloseable> closeables) {
     if (closeables == null) {
       return;
     }
@@ -95,7 +95,7 @@ public static void closeQuietly(AutoCloseable... closeables) {
   /**
    * Close each argument, swallowing exceptions.
    */
-  public static void closeQuietly(Collection<AutoCloseable> closeables) {
+  public static void closeQuietly(Collection<? extends AutoCloseable> closeables) {
     close(null, closeables);
   }
 }

hadoop-hdds/common/src/main/resources/ozone-default.xml

@@ -3532,6 +3532,25 @@
     </description>
   </property>
 
+  <property>
+    <name>ozone.snapshot.directory.service.timeout</name>
+    <value>300s</value>
+    <tag>OZONE, PERFORMANCE, OM</tag>
+    <description>
+      Timeout value for SnapshotDirectoryCleaningService.
+    </description>
+  </property>
+
+  <property>
+    <name>ozone.snapshot.directory.service.interval</name>
+    <value>24h</value>
+    <tag>OZONE, PERFORMANCE, OM</tag>
+    <description>
+      The time interval between successive SnapshotDirectoryCleaningService
+      thread run.
+    </description>
+  </property>
+
   <property>
     <name>ozone.scm.event.ContainerReport.thread.pool.size</name>
     <value>10</value>

hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java

@@ -173,7 +173,6 @@ private XceiverServerRatis(DatanodeDetails dd,
     this.streamEnable = conf.getBoolean(
         OzoneConfigKeys.DFS_CONTAINER_RATIS_DATASTREAM_ENABLED,
         OzoneConfigKeys.DFS_CONTAINER_RATIS_DATASTREAM_ENABLED_DEFAULT);
-    RaftProperties serverProperties = newRaftProperties();
     this.context = context;
     this.dispatcher = dispatcher;
     this.containerController = containerController;
@@ -184,6 +183,7 @@ private XceiverServerRatis(DatanodeDetails dd,
     shouldDeleteRatisLogDirectory =
         ratisServerConfig.shouldDeleteRatisLogDirectory();
 
+    RaftProperties serverProperties = newRaftProperties();
     this.server =
         RaftServer.newBuilder().setServerId(raftPeerId)
             .setProperties(serverProperties)

hadoop-hdds/rocks-native/pom.xml

@@ -325,7 +325,8 @@
                                             <arg line="${project.build.directory}/rocksdb/rocksdb-${rocksdb.version}"/>
                                         </exec>
                                         <exec executable="make" dir="${project.build.directory}/rocksdb/rocksdb-${rocksdb.version}" failonerror="true">
-                                            <arg line="DEBUG_LEVEL=0"/>
+                                            <arg line="PORTABLE=1"/>
+                      <arg line="DEBUG_LEVEL=0"/>
                                             <arg line="EXTRA_CXXFLAGS='-fPIC -I${project.build.directory}/snappy/lib -I${project.build.directory}/snappy/snappy-${snappy.version} -I${project.build.directory}/lz4/lz4-${lz4.version}/lib -I${project.build.directory}/zstd/zstd-${zstd.version}/lib -I${project.build.directory}/zstd/zstd-${zstd.version}/lib/dictBuilder -I${project.build.directory}/bzip2/bzip2-${bzip2.version} -I${project.build.directory}/zlib/zlib-${zlib.version}'"/>
                                             <arg line="EXTRA_LDFLAGS='-L${project.build.directory}/snappy/lib -L${project.build.directory}/lz4/lz4-${lz4.version}/lib -L${project.build.directory}/zstd/zstd-${zstd.version}/lib -L${project.build.directory}/bzip2/bzip2-${bzip2.version} -L${project.build.directory}/zlib/zlib-${zlib.version}'"/>
                                             <arg line="-j${system.numCores}"/>

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java

@@ -390,6 +390,18 @@ private OMConfigKeys() {
   public static final String OZONE_DIR_DELETING_SERVICE_INTERVAL_DEFAULT
       = "60s";
 
+  /**
+   * Configuration properties for Snapshot Directory Service.
+   */
+  public static final String OZONE_SNAPSHOT_DIRECTORY_SERVICE_INTERVAL =
+      "ozone.snapshot.directory.service.interval";
+  public static final String OZONE_SNAPSHOT_DIRECTORY_SERVICE_INTERVAL_DEFAULT
+      = "24h";
+  public static final String OZONE_SNAPSHOT_DIRECTORY_SERVICE_TIMEOUT =
+      "ozone.snapshot.directory.service.timeout";
+  public static final String
+      OZONE_SNAPSHOT_DIRECTORY_SERVICE_TIMEOUT_DEFAULT = "300s";
+
   public static final String OZONE_PATH_DELETING_LIMIT_PER_TASK =
       "ozone.path.deleting.limit.per.task";
   // default is 6000 taking account of 32MB buffer size, and assuming

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/S3SecretValue.java

@@ -101,9 +101,7 @@ public S3Secret getProtobuf() {
 
   @Override
   public String toString() {
-    return "awsAccessKey=" + kerberosID + "\nawsSecret=" + awsSecret +
-        "\nisDeleted=" + isDeleted + "\ntransactionLogIndex=" +
-        transactionLogIndex;
+    return "awsAccessKey=" + kerberosID + "\nawsSecret=" + awsSecret;
   }
 
   @Override

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java

@@ -49,8 +49,8 @@
  * This class is used for storing info related to Snapshots.
  *
  * Each snapshot created has an associated SnapshotInfo entry
- * containing the snapshotid, snapshot path,
- * snapshot checkpoint directory, previous snapshotid
+ * containing the snapshotId, snapshot path,
+ * snapshot checkpoint directory, previous snapshotId
  * for the snapshot path & global amongst other necessary fields.
  */
 public final class SnapshotInfo implements Auditable, CopyObject<SnapshotInfo> {
@@ -123,6 +123,7 @@ public static SnapshotStatus valueOf(SnapshotStatusProto status) {
   private long referencedReplicatedSize;
   private long exclusiveSize;
   private long exclusiveReplicatedSize;
+  private boolean deepCleanedDeletedDir;
 
   /**
    * Private constructor, constructed via builder.
@@ -162,7 +163,8 @@ private SnapshotInfo(UUID snapshotId,
                        long referencedSize,
                        long referencedReplicatedSize,
                        long exclusiveSize,
-                       long exclusiveReplicatedSize) {
+                       long exclusiveReplicatedSize,
+                       boolean deepCleanedDeletedDir) {
     this.snapshotId = snapshotId;
     this.name = name;
     this.volumeName = volumeName;
@@ -181,6 +183,7 @@ private SnapshotInfo(UUID snapshotId,
     this.referencedReplicatedSize = referencedReplicatedSize;
     this.exclusiveSize = exclusiveSize;
     this.exclusiveReplicatedSize = exclusiveReplicatedSize;
+    this.deepCleanedDeletedDir = deepCleanedDeletedDir;
   }
 
   public void setName(String name) {
@@ -285,7 +288,7 @@ public void setSstFiltered(boolean sstFiltered) {
   }
 
   public SnapshotInfo.Builder toBuilder() {
-    return new SnapshotInfo.Builder()
+    return new Builder()
         .setSnapshotId(snapshotId)
         .setName(name)
         .setVolumeName(volumeName)
@@ -302,7 +305,8 @@ public SnapshotInfo.Builder toBuilder() {
         .setReferencedSize(referencedSize)
         .setReferencedReplicatedSize(referencedReplicatedSize)
         .setExclusiveSize(exclusiveSize)
-        .setExclusiveReplicatedSize(exclusiveReplicatedSize);
+        .setExclusiveReplicatedSize(exclusiveReplicatedSize)
+        .setDeepCleanedDeletedDir(deepCleanedDeletedDir);
   }
 
   /**
@@ -327,6 +331,7 @@ public static class Builder {
     private long referencedReplicatedSize;
     private long exclusiveSize;
     private long exclusiveReplicatedSize;
+    private boolean deepCleanedDeletedDir;
 
     public Builder() {
       // default values
@@ -423,6 +428,11 @@ public Builder setExclusiveReplicatedSize(long exclusiveReplicatedSize) {
       return this;
     }
 
+    public Builder setDeepCleanedDeletedDir(boolean deepCleanedDeletedDir) {
+      this.deepCleanedDeletedDir = deepCleanedDeletedDir;
+      return this;
+    }
+
     public SnapshotInfo build() {
       Preconditions.checkNotNull(name);
       return new SnapshotInfo(
@@ -443,7 +453,8 @@ public SnapshotInfo build() {
           referencedSize,
           referencedReplicatedSize,
           exclusiveSize,
-          exclusiveReplicatedSize
+          exclusiveReplicatedSize,
+          deepCleanedDeletedDir
       );
     }
   }
@@ -465,7 +476,8 @@ public OzoneManagerProtocolProtos.SnapshotInfo getProtobuf() {
             .setReferencedSize(referencedSize)
             .setReferencedReplicatedSize(referencedReplicatedSize)
             .setExclusiveSize(exclusiveSize)
-            .setExclusiveReplicatedSize(exclusiveReplicatedSize);
+            .setExclusiveReplicatedSize(exclusiveReplicatedSize)
+            .setDeepCleanedDeletedDir(deepCleanedDeletedDir);
 
     if (pathPreviousSnapshotId != null) {
       sib.setPathPreviousSnapshotID(toProtobuf(pathPreviousSnapshotId));
@@ -538,6 +550,11 @@ public static SnapshotInfo getFromProtobuf(
           snapshotInfoProto.getExclusiveReplicatedSize());
     }
 
+    if (snapshotInfoProto.hasDeepCleanedDeletedDir()) {
+      osib.setDeepCleanedDeletedDir(
+          snapshotInfoProto.getDeepCleanedDeletedDir());
+    }
+
     osib.setSnapshotPath(snapshotInfoProto.getSnapshotPath())
         .setCheckpointDir(snapshotInfoProto.getCheckpointDir())
         .setDbTxSequenceNumber(snapshotInfoProto.getDbTxSequenceNumber());
@@ -622,6 +639,14 @@ public long getExclusiveReplicatedSize() {
     return exclusiveReplicatedSize;
   }
 
+  public boolean getDeepCleanedDeletedDir() {
+    return deepCleanedDeletedDir;
+  }
+
+  public void setDeepCleanedDeletedDir(boolean deepCleanedDeletedDir) {
+    this.deepCleanedDeletedDir = deepCleanedDeletedDir;
+  }
+
   /**
    * Generate default name of snapshot, (used if user doesn't provide one).
    */
@@ -655,7 +680,8 @@ public static SnapshotInfo newInstance(String volumeName,
         .setSnapshotPath(volumeName + OM_KEY_PREFIX + bucketName)
         .setVolumeName(volumeName)
         .setBucketName(bucketName)
-        .setDeepClean(true);
+        .setDeepClean(false)
+        .setDeepCleanedDeletedDir(false);
 
     if (snapshotId != null) {
       builder.setCheckpointDir(getCheckpointDirName(snapshotId));
@@ -688,7 +714,8 @@ public boolean equals(Object o) {
         referencedSize == that.referencedSize &&
         referencedReplicatedSize == that.referencedReplicatedSize &&
         exclusiveSize == that.exclusiveSize &&
-        exclusiveReplicatedSize == that.exclusiveReplicatedSize;
+        exclusiveReplicatedSize == that.exclusiveReplicatedSize &&
+        deepCleanedDeletedDir == that.deepCleanedDeletedDir;
   }
 
   @Override
@@ -699,7 +726,7 @@ public int hashCode() {
         globalPreviousSnapshotId, snapshotPath, checkpointDir,
         deepClean, sstFiltered,
         referencedSize, referencedReplicatedSize,
-        exclusiveSize, exclusiveReplicatedSize);
+        exclusiveSize, exclusiveReplicatedSize, deepCleanedDeletedDir);
   }
 
   /**
@@ -726,14 +753,15 @@ public SnapshotInfo copyObject() {
         .setReferencedReplicatedSize(referencedReplicatedSize)
         .setExclusiveSize(exclusiveSize)
         .setExclusiveReplicatedSize(exclusiveReplicatedSize)
+        .setDeepCleanedDeletedDir(deepCleanedDeletedDir)
         .build();
   }
 
   @Override
   public String toString() {
     return "SnapshotInfo{" +
         "snapshotId: '" + snapshotId + '\'' +
-        ", name: '" + name + "'," +
+        ", name: '" + name + '\'' +
         ", volumeName: '" + volumeName + '\'' +
         ", bucketName: '" + bucketName + '\'' +
         ", snapshotStatus: '" + snapshotStatus + '\'' +
@@ -750,6 +778,7 @@ public String toString() {
         ", referencedReplicatedSize: '" + referencedReplicatedSize + '\'' +
         ", exclusiveSize: '" + exclusiveSize + '\'' +
         ", exclusiveReplicatedSize: '" + exclusiveReplicatedSize + '\'' +
+        ", deepCleanedDeletedDir: '" + deepCleanedDeletedDir + '\'' +
         '}';
   }
 }

hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmSnapshotInfo.java

@@ -66,12 +66,13 @@ private SnapshotInfo createSnapshotInfo() {
         .setSnapshotPath(SNAPSHOT_PATH)
         .setCheckpointDir(CHECKPOINT_DIR)
         .setDbTxSequenceNumber(DB_TX_SEQUENCE_NUMBER)
-        .setDeepClean(true)
+        .setDeepClean(false)
         .setSstFiltered(false)
         .setReferencedSize(2000L)
         .setReferencedReplicatedSize(6000L)
         .setExclusiveSize(1000L)
         .setExclusiveReplicatedSize(3000L)
+        .setDeepCleanedDeletedDir(false)
         .build();
   }
 
@@ -89,12 +90,13 @@ private OzoneManagerProtocolProtos.SnapshotInfo createSnapshotInfoProto() {
         .setSnapshotPath(SNAPSHOT_PATH)
         .setCheckpointDir(CHECKPOINT_DIR)
         .setDbTxSequenceNumber(DB_TX_SEQUENCE_NUMBER)
-        .setDeepClean(true)
+        .setDeepClean(false)
         .setSstFiltered(false)
         .setReferencedSize(2000L)
         .setReferencedReplicatedSize(6000L)
         .setExclusiveSize(1000L)
         .setExclusiveReplicatedSize(3000L)
+        .setDeepCleanedDeletedDir(false)
         .build();
   }
 
@@ -140,6 +142,9 @@ public void testSnapshotInfoToProto() {
     Assertions.assertEquals(
         snapshotInfoEntryExpected.getExclusiveReplicatedSize(),
         snapshotInfoEntryActual.getExclusiveReplicatedSize());
+    Assertions.assertEquals(
+        snapshotInfoEntryExpected.getDeepCleanedDeletedDir(),
+        snapshotInfoEntryActual.getDeepCleanedDeletedDir());
 
     Assertions.assertEquals(snapshotInfoEntryExpected, snapshotInfoEntryActual);
   }
@@ -176,6 +181,8 @@ public void testSnapshotInfoProtoToSnapshotInfo() {
         snapshotInfoActual.getExclusiveSize());
     Assertions.assertEquals(snapshotInfoExpected.getExclusiveReplicatedSize(),
         snapshotInfoActual.getExclusiveReplicatedSize());
+    Assertions.assertEquals(snapshotInfoExpected.getDeepCleanedDeletedDir(),
+        snapshotInfoActual.getDeepCleanedDeletedDir());
 
     Assertions.assertEquals(snapshotInfoExpected, snapshotInfoActual);
   }

hadoop-ozone/dist/src/main/smoketest/security/S3-secret.robot

@@ -26,6 +26,8 @@ GetSecret success
     ${output}=             Execute             ozone s3 getsecret -u testuser2
     Should contain         ${output}           awsAccessKey
     Should contain         ${output}           awsSecret
+    Should not contain     ${output}           isDeleted
+    Should not contain     ${output}           transactionLogIndex
 
 GetSecret failure
     ${output2}=            Execute and Ignore Error    ozone s3 getsecret -u testuser2

hadoop-ozone/dist/src/shell/ozone/ozone

@@ -59,6 +59,7 @@ function ozone_usage
   ozone_add_subcommand "dtutil" client "operations related to delegation tokens"
   ozone_add_subcommand "admin" client "Ozone admin tool"
   ozone_add_subcommand "debug" client "Ozone debug tool"
+  ozone_add_subcommand "repair" client "Ozone repair tool"
   ozone_add_subcommand "checknative" client "checks if native libraries are loaded"
 
   ozone_generate_usage "${OZONE_SHELL_EXECNAME}" false
@@ -236,6 +237,11 @@ function ozonecmd_case
       OZONE_DEBUG_OPTS="${OZONE_DEBUG_OPTS} ${OZONE_MODULE_ACCESS_ARGS}"
       OZONE_RUN_ARTIFACT_NAME="ozone-tools"
     ;;
+    repair)
+      OZONE_CLASSNAME=org.apache.hadoop.ozone.repair.OzoneRepair
+      OZONE_DEBUG_OPTS="${OZONE_DEBUG_OPTS} ${OZONE_MODULE_ACCESS_ARGS}"
+      OZONE_RUN_ARTIFACT_NAME="ozone-tools"
+    ;;
     checknative)
         OZONE_CLASSNAME=org.apache.hadoop.ozone.shell.checknative.CheckNative
         OZONE_RUN_ARTIFACT_NAME="ozone-tools"

hadoop-ozone/fault-injection-test/mini-chaos-tests/pom.xml

@@ -43,11 +43,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
       <artifactId>junit-jupiter-migrationsupport</artifactId>
       <scope>test</scope>
     </dependency>
-    <dependency>
-      <groupId>org.junit.jupiter</groupId>
-      <artifactId>junit-jupiter-engine</artifactId>
-      <scope>test</scope>
-    </dependency>
     <dependency>
       <groupId>org.junit.vintage</groupId>
       <artifactId>junit-vintage-engine</artifactId>

hadoop-ozone/integration-test/pom.xml

@@ -139,11 +139,6 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
       <artifactId>junit-jupiter-migrationsupport</artifactId>
       <scope>test</scope>
     </dependency>
-    <dependency>
-      <groupId>org.junit.jupiter</groupId>
-      <artifactId>junit-jupiter-engine</artifactId>
-      <scope>test</scope>
-    </dependency>
     <dependency>
       <groupId>org.junit.vintage</groupId>
       <artifactId>junit-vintage-engine</artifactId>