HDDS-9738. Display pipeline and container counts for the decommissioning Datanode

.github/workflows/ci.yml

@@ -135,13 +135,17 @@ jobs:
       - build-info
       - build
       - basic
-    runs-on: ubuntu-20.04
     timeout-minutes: 30
     if: needs.build-info.outputs.needs-compile == 'true'
     strategy:
       matrix:
         java: [ 11, 17, 21 ]
+        include:
+          - os: ubuntu-20.04
+          - java: 8
+            os: macos-12
       fail-fast: false
+    runs-on: ${{ matrix.os }}
     steps:
       - name: Download Ozone source tarball
         uses: actions/download-artifact@v3

.github/workflows/intermittent-test-check.yml

@@ -92,8 +92,41 @@ jobs:
           printf -v x "%s," "${splits[@]}"
           split_matrix="[${x%,}]"
           echo "matrix=$split_matrix" >> $GITHUB_OUTPUT
+  build:
+    needs:
+      - prepare-job
+    runs-on: ubuntu-20.04
+    timeout-minutes: 60
+    steps:
+      - name: Checkout project
+        uses: actions/checkout@v3
+      - name: Cache for maven dependencies
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.m2/repository
+            !~/.m2/repository/org/apache/ozone
+          key: maven-repo-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            maven-repo-
+      - name: Setup java
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'temurin'
+          java-version: 8
+      - name: Build (most) of Ozone
+        run: hadoop-ozone/dev-support/checks/build.sh -Dskip.npx -Dskip.installnpx -DskipShade
+      - name: Store Maven repo for tests
+        uses: actions/upload-artifact@v3
+        with:
+          name: ozone-repo
+          path: |
+            ~/.m2/repository/org/apache/ozone
+          retention-days: 1
   run-test:
-    needs: prepare-job
+    needs:
+      - prepare-job
+      - build
     name: Run-Split
     runs-on: ubuntu-20.04
     strategy:
@@ -105,21 +138,33 @@ jobs:
         with:
           ref: ${{ github.event.inputs.ref }}
       - name: Cache for maven dependencies
-        uses: actions/cache@v3
+        uses: actions/cache/restore@v3
         with:
-          path: ~/.m2/repository
-          key: maven-repo-${{ hashFiles('**/pom.xml') }}-8-single
+          path: |
+            ~/.m2/repository
+            !~/.m2/repository/org/apache/ozone
+          key: maven-repo-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
-            maven-repo-${{ hashFiles('**/pom.xml') }}-8
-            maven-repo-${{ hashFiles('**/pom.xml') }}
             maven-repo-
+      - name: Download Ozone repo
+        id: download-ozone-repo
+        uses: actions/download-artifact@v3
+        with:
+          name: ozone-repo
+          path: |
+            ~/.m2/repository/org/apache/ozone
+        continue-on-error: true
       - name: Setup java
         uses: actions/setup-java@v3
         with:
           distribution: 'temurin'
           java-version: 8
       - name: Execute tests
         run: |
+          if [[ -e "${{ steps.download-ozone-repo.outputs.download-path }}" ]]; then
+            export OZONE_REPO_CACHED=true
+          fi
+
           test_type=${{ needs.prepare-job.outputs.test_type }}
           args="-DexcludedGroups=unhealthy"
           if [ "$test_type" = "integration" ]; then
@@ -145,12 +190,6 @@ jobs:
         with:
           name: result-${{ env.TEST_CLASS }}-split-${{ matrix.split }}
           path: target/${{ needs.prepare-job.outputs.test_type }}
-      - name: Delete temporary build artifacts before caching
-        run: |
-          #Never cache local artifacts
-          rm -rf ~/.m2/repository/org/apache/ozone/hdds*
-          rm -rf ~/.m2/repository/org/apache/ozone/ozone*
-        if: always()
   count-failures:
     if: ${{ always() }}
     needs: run-test

hadoop-hdds/client/pom.xml

@@ -30,7 +30,6 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
   <packaging>jar</packaging>
 
   <properties>
-    <allow.junit4>false</allow.junit4>
   </properties>
 
   <dependencies>

hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/TestBlockInputStream.java

@@ -19,6 +19,7 @@
 package org.apache.hadoop.hdds.scm.storage;
 
 import com.google.common.primitives.Bytes;
+import org.apache.commons.lang3.RandomUtils;
 import org.apache.hadoop.hdds.client.BlockID;
 import org.apache.hadoop.hdds.client.ContainerBlockID;
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
@@ -50,7 +51,6 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Random;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ThreadLocalRandom;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -186,9 +186,8 @@ public void testSeek() throws Exception {
     assertThrows(EOFException.class, () -> seekAndVerify(finalPos));
 
     // Seek to random positions between 0 and the block size.
-    Random random = new Random();
     for (int i = 0; i < 10; i++) {
-      pos = random.nextInt(blockSize);
+      pos = RandomUtils.nextInt(0, blockSize);
       seekAndVerify(pos);
     }
   }

hadoop-hdds/client/src/test/java/org/apache/hadoop/ozone/client/io/TestECBlockReconstructedStripeInputStream.java

@@ -18,6 +18,7 @@
 package org.apache.hadoop.ozone.client.io;
 
 import com.google.common.collect.ImmutableSet;
+import org.apache.commons.lang3.RandomUtils;
 import org.apache.hadoop.hdds.client.ECReplicationConfig;
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
 import org.apache.hadoop.hdds.protocol.MockDatanodeDetails;
@@ -41,7 +42,6 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Random;
 import java.util.Set;
 import java.util.SplittableRandom;
 import java.util.concurrent.ExecutorService;
@@ -645,7 +645,7 @@ public void testSeekToPartialOffsetFails() {
   }
 
   private Integer getRandomStreamIndex(Set<Integer> set) {
-    return set.stream().skip(new Random().nextInt(set.size()))
+    return set.stream().skip(RandomUtils.nextInt(0, set.size()))
         .findFirst().orElse(null);
   }
 

hadoop-hdds/common/pom.xml

@@ -29,7 +29,6 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
   <packaging>jar</packaging>
 
   <properties>
-    <allow.junit4>false</allow.junit4>
   </properties>
 
   <dependencies>

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfig.java

@@ -73,8 +73,17 @@ public class ScmConfig extends ReconfigurableConfig {
           + "org.apache.hadoop.hdds.scm.PipelineChoosePolicy. "
           + "The class decides which pipeline will be used to find or "
           + "allocate Ratis containers. If not set, "
-          + "org.apache.hadoop.hdds.scm.pipeline.choose.algorithms. "
-          + "RandomPipelineChoosePolicy will be used as default value."
+          + "org.apache.hadoop.hdds.scm.pipeline.choose.algorithms."
+          + "RandomPipelineChoosePolicy will be used as default value. "
+          + "The following values can be used, "
+          + "(1) org.apache.hadoop.hdds.scm.pipeline.choose.algorithms."
+          + "RandomPipelineChoosePolicy : random choose one pipeline. "
+          + "(2) org.apache.hadoop.hdds.scm.pipeline.choose.algorithms."
+          + "HealthyPipelineChoosePolicy : random choose one healthy pipeline. "
+          + "(3) org.apache.hadoop.hdds.scm.pipeline.choose.algorithms."
+          + "CapacityPipelineChoosePolicy : choose the pipeline with lower "
+          + "utilization from the two pipelines. Note that random choose "
+          + "method will be executed twice in this policy."
   )
   private String pipelineChoosePolicyName;
 
@@ -85,11 +94,20 @@ public class ScmConfig extends ReconfigurableConfig {
       tags = { ConfigTag.SCM, ConfigTag.PIPELINE },
       description =
           "The full name of class which implements "
-              + "org.apache.hadoop.hdds.scm.PipelineChoosePolicy. "
-              + "The class decides which pipeline will be used when "
-              + "selecting an EC Pipeline. If not set, "
-              + "org.apache.hadoop.hdds.scm.pipeline.choose.algorithms. "
-              + "RandomPipelineChoosePolicy will be used as default value."
+          + "org.apache.hadoop.hdds.scm.PipelineChoosePolicy. "
+          + "The class decides which pipeline will be used when "
+          + "selecting an EC Pipeline. If not set, "
+          + "org.apache.hadoop.hdds.scm.pipeline.choose.algorithms."
+          + "RandomPipelineChoosePolicy will be used as default value. "
+          + "The following values can be used, "
+          + "(1) org.apache.hadoop.hdds.scm.pipeline.choose.algorithms."
+          + "RandomPipelineChoosePolicy : random choose one pipeline. "
+          + "(2) org.apache.hadoop.hdds.scm.pipeline.choose.algorithms."
+          + "HealthyPipelineChoosePolicy : random choose one healthy pipeline. "
+          + "(3) org.apache.hadoop.hdds.scm.pipeline.choose.algorithms."
+          + "CapacityPipelineChoosePolicy : choose the pipeline with lower "
+          + "utilization from the two pipelines. Note that random choose "
+          + "method will be executed twice in this policy."
   )
   private String ecPipelineChoosePolicyName;
 

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java

@@ -20,10 +20,12 @@
 import org.apache.commons.lang3.tuple.Pair;
 import org.apache.hadoop.hdds.annotation.InterfaceStability;
 import org.apache.hadoop.hdds.client.ReplicationConfig;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos.DeletedBlocksTransactionInfo;
 import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DecommissionScmResponseProto;
 import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.StartContainerBalancerResponseProto;
 import org.apache.hadoop.hdds.scm.DatanodeAdminError;
+import org.apache.hadoop.hdds.scm.container.ContainerID;
 import org.apache.hadoop.hdds.scm.container.ContainerReplicaInfo;
 import org.apache.hadoop.hdds.scm.container.ReplicationManagerReport;
 import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
@@ -39,6 +41,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.UUID;
 
 /**
  * The interface to call into underlying container layer.
@@ -178,6 +181,14 @@ ContainerWithPipeline createContainer(HddsProtos.ReplicationType type,
       HddsProtos.ReplicationFactor replicationFactor,
       String owner) throws IOException;
 
+  /**
+   * Gets the list of underReplicated and unClosed containers on a decommissioning node.
+   *
+   * @param dn - Datanode detail
+   * @return Lists of underReplicated and Unclosed containers
+   */
+  Map<String, List<ContainerID>> getContainersOnDecomNode(DatanodeDetails dn) throws IOException;
+
   /**
    * Returns a set of Nodes that meet a query criteria. Passing null for opState
    * or nodeState acts like a wild card, returning all nodes in that state.
@@ -194,6 +205,14 @@ List<HddsProtos.Node> queryNode(HddsProtos.NodeOperationalState opState,
       HddsProtos.NodeState nodeState, HddsProtos.QueryScope queryScope,
       String poolName) throws IOException;
 
+  /**
+   * Returns a node with the given UUID.
+   * @param uuid - datanode uuid string
+   * @return A nodes that matches the requested UUID.
+   * @throws IOException
+   */
+  HddsProtos.Node queryNode(UUID uuid) throws IOException;
+
   /**
    * Allows a list of hosts to be decommissioned. The hosts are identified
    * by their hostname and optionally port in the format foo.com:port.

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/net/NetworkTopologyImpl.java

@@ -657,7 +657,7 @@ private Node chooseNodeInternal(String scope, int leafIndex,
         ancestorGen);
 
     if (availableNodes <= 0) {
-      LOG.warn("No available node in (scope=\"{}\" excludedScope=\"{}\" " +
+      LOG.info("No available node in (scope=\"{}\" excludedScope=\"{}\" " +
               "excludedNodes=\"{}\"  ancestorGen=\"{}\").",
           scopeNode.getNetworkFullPath(), excludedScopes, excludedNodes,
           ancestorGen);

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java

@@ -19,6 +19,7 @@
 
 import org.apache.commons.lang3.tuple.Pair;
 import org.apache.hadoop.hdds.client.ReplicationConfig;
+import org.apache.hadoop.hdds.protocol.DatanodeDetails;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
 import org.apache.hadoop.hdds.protocol.proto.HddsProtos.DeletedBlocksTransactionInfo;
 import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DecommissionScmResponseProto;
@@ -44,6 +45,7 @@
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
+import java.util.UUID;
 
 /**
  * ContainerLocationProtocol is used by an HDFS node to find the set of nodes
@@ -219,6 +221,14 @@ List<ContainerInfo> listContainer(long startContainerID,
    */
   void deleteContainer(long containerID) throws IOException;
 
+  /**
+   * Gets the list of underReplicated and unClosed containers on a decommissioning node.
+   *
+   * @param dn - Datanode detail
+   * @return Lists of underReplicated and unClosed containers
+   */
+  Map<String, List<ContainerID>> getContainersOnDecomNode(DatanodeDetails dn) throws IOException;
+
   /**
    *  Queries a list of Node Statuses. Passing a null for either opState or
    *  state acts like a wildcard returning all nodes in that state.
@@ -232,6 +242,8 @@ List<HddsProtos.Node> queryNode(HddsProtos.NodeOperationalState opState,
       HddsProtos.NodeState state, HddsProtos.QueryScope queryScope,
       String poolName, int clientVersion) throws IOException;
 
+  HddsProtos.Node queryNode(UUID uuid) throws IOException;
+
   List<DatanodeAdminError> decommissionNodes(List<String> nodes)
       throws IOException;
 

hadoop-hdds/common/src/main/resources/ozone-default.xml

@@ -3600,25 +3600,6 @@
     </description>
   </property>
 
-  <property>
-    <name>ozone.snapshot.directory.service.timeout</name>
-    <value>300s</value>
-    <tag>OZONE, PERFORMANCE, OM</tag>
-    <description>
-      Timeout value for SnapshotDirectoryCleaningService.
-    </description>
-  </property>
-
-  <property>
-    <name>ozone.snapshot.directory.service.interval</name>
-    <value>24h</value>
-    <tag>OZONE, PERFORMANCE, OM</tag>
-    <description>
-      The time interval between successive SnapshotDirectoryCleaningService
-      thread run.
-    </description>
-  </property>
-
   <property>
     <name>ozone.scm.event.ContainerReport.thread.pool.size</name>
     <value>10</value>

hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/common/TestChecksumByteBuffer.java

@@ -22,7 +22,7 @@
 import org.junit.jupiter.api.Test;
 
 import java.nio.charset.StandardCharsets;
-import java.util.Random;
+import org.apache.commons.lang3.RandomUtils;
 import java.util.zip.Checksum;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -59,11 +59,9 @@ void testCorrectness() {
 
       checkBytes("hello world!".getBytes(StandardCharsets.UTF_8));
 
-      final Random random = new Random();
-      final byte[] bytes = new byte[1 << 10];
+      final int len = 1 << 10;
       for (int i = 0; i < 1000; i++) {
-        random.nextBytes(bytes);
-        checkBytes(bytes, random.nextInt(bytes.length));
+        checkBytes(RandomUtils.nextBytes(len), RandomUtils.nextInt(0, len));
       }
     }
 

hadoop-hdds/config/pom.xml

@@ -29,7 +29,6 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
   <packaging>jar</packaging>
 
   <properties>
-    <allow.junit4>false</allow.junit4>
   </properties>
 
   <dependencies>

hadoop-hdds/container-service/pom.xml

@@ -28,7 +28,6 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
   <name>Apache Ozone HDDS Container Service</name>
   <packaging>jar</packaging>
   <properties>
-    <allow.junit4>false</allow.junit4>
   </properties>
   <dependencies>
     <dependency>