HDDS-8188. Support max allowed length in response of ozone admin container list

hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java

@@ -399,8 +399,11 @@ public List<ContainerInfo> listContainer(long startContainerID, int count,
       throws IOException {
     Preconditions.checkState(startContainerID >= 0,
         "Container ID cannot be negative.");
-    Preconditions.checkState(count > 0,
-        "Container count must be greater than 0.");
+    if (count <= 0 && count != -1) {
+      throw new IllegalStateException(
+        "Container count must be either greater than 0 " +
+        "or equal to -1 (list all containers)");
+    }        
     SCMListContainerRequestProto.Builder builder = SCMListContainerRequestProto
         .newBuilder();
     builder.setStartContainerID(startContainerID);

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManager.java

@@ -63,7 +63,7 @@ default List<ContainerInfo> getContainers() {
    *
    * @param startID start containerID, >=0,
    * start searching at the head if 0.
-   * @param count count must be >= 0
+   * @param count count must be >= -1
    *              Usually the count will be replace with a very big
    *              value instead of being unlimited in case the db is very big.
    *
@@ -87,7 +87,7 @@ default List<ContainerInfo> getContainers() {
    *
    * @param startID start containerID, >=0,
    * start searching at the head if 0.
-   * @param count count must be >= 0
+   * @param count count must be >= -1
    *              Usually the count will be replace with a very big
    *              value instead of being unlimited in case the db is very big.
    * @param state container state

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManagerImpl.java

@@ -451,7 +451,8 @@ public SCMHAManager getSCMHAManager() {
   private static List<ContainerID> filterSortAndLimit(
       ContainerID startID, int count, Set<ContainerID> set) {
 
-    if (ContainerID.MIN.equals(startID) && count >= set.size()) {
+    if (ContainerID.MIN.equals(startID) &&
+          (count >= set.size() || count == -1)) {
       List<ContainerID> list = new ArrayList<>(set);
       Collections.sort(list);
       return list;

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMClientProtocolServer.java

@@ -462,27 +462,36 @@ public List<ContainerInfo> listContainer(long startContainerID,
       final ContainerID containerId = ContainerID.valueOf(startContainerID);
       if (state != null) {
         if (factor != null) {
-          return scm.getContainerManager().getContainers(state).stream()
-              .filter(info -> info.containerID().getId() >= startContainerID)
-              //Filtering EC replication type as EC will not have factor.
-              .filter(info -> info
-                  .getReplicationType() != HddsProtos.ReplicationType.EC)
-              .filter(info -> (info.getReplicationFactor() == factor))
-              .sorted().limit(count).collect(Collectors.toList());
+          Stream<ContainerInfo> containerInfoStream =
+              scm.getContainerManager().getContainers(state).stream()
+                .filter(info -> info.containerID().getId() >= startContainerID)
+                //Filtering EC replication type as EC will not have factor.
+                .filter(info -> info
+                    .getReplicationType() != HddsProtos.ReplicationType.EC)
+                .filter(info -> (info.getReplicationFactor() == factor))
+                .sorted();
+          return count == -1 ? containerInfoStream.collect(Collectors.toList()) :
+              containerInfoStream.limit(count).collect(Collectors.toList());
         } else {
-          return scm.getContainerManager().getContainers(state).stream()
-              .filter(info -> info.containerID().getId() >= startContainerID)
-              .sorted().limit(count).collect(Collectors.toList());
+          Stream<ContainerInfo> containerInfoStream =
+              scm.getContainerManager().getContainers(state).stream()
+                  .filter(info -> info.containerID().getId() >= startContainerID)
+                  .sorted();
+          return count == -1 ? containerInfoStream.collect(Collectors.toList()) :
+              containerInfoStream.limit(count).collect(Collectors.toList());
         }
       } else {
         if (factor != null) {
-          return scm.getContainerManager().getContainers().stream()
-              .filter(info -> info.containerID().getId() >= startContainerID)
-              //Filtering EC replication type as EC will not have factor.
-              .filter(info -> info
-                  .getReplicationType() != HddsProtos.ReplicationType.EC)
-              .filter(info -> info.getReplicationFactor() == factor)
-              .sorted().limit(count).collect(Collectors.toList());
+          Stream<ContainerInfo> containerInfoStream =
+              scm.getContainerManager().getContainers().stream()
+                  .filter(info -> info.containerID().getId() >= startContainerID)
+                  //Filtering EC replication type as EC will not have factor.
+                  .filter(info -> info
+                      .getReplicationType() != HddsProtos.ReplicationType.EC)
+                  .filter(info -> info.getReplicationFactor() == factor)
+                  .sorted();
+          return count == -1 ? containerInfoStream.collect(Collectors.toList())
+              : containerInfoStream.limit(count).collect(Collectors.toList());
         } else {
           return scm.getContainerManager().getContainers(containerId, count);
         }
@@ -554,9 +563,9 @@ public List<ContainerInfo> listContainer(long startContainerID,
         containerStream = containerStream
             .filter(info -> info.getReplicationType() == replicationType);
       }
-      return containerStream.sorted()
-          .limit(count)
-          .collect(Collectors.toList());
+      Stream<ContainerInfo> containerInfoStream = containerStream.sorted();
+      return count == -1 ? containerInfoStream.collect(Collectors.toList()) :
+          containerInfoStream.limit(count).collect(Collectors.toList());
     } catch (Exception ex) {
       auditSuccess = false;
       AUDIT.logReadFailure(

hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/TestContainerManagerImpl.java

@@ -147,13 +147,18 @@ void testGetContainers() throws Exception {
         containerManager.getContainers(ContainerID.MIN, 10));
     assertIds(ids.subList(0, 5),
         containerManager.getContainers(ContainerID.MIN, 5));
+    assertIds(ids,
+        containerManager.getContainers(ContainerID.MIN, -1));
 
     assertIds(ids, containerManager.getContainers(ids.get(0), 10));
     assertIds(ids, containerManager.getContainers(ids.get(0), 100));
     assertIds(ids.subList(5, ids.size()),
         containerManager.getContainers(ids.get(5), 100));
     assertIds(emptyList(),
         containerManager.getContainers(ids.get(5), 100, LifeCycleState.CLOSED));
+    assertIds(ids,
+        containerManager.getContainers(ContainerID.MIN, -1,
+            LifeCycleState.OPEN));
 
     containerManager.updateContainerState(ids.get(0),
         HddsProtos.LifeCycleEvent.FINALIZE);

hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMClientProtocolServer.java

@@ -17,12 +17,19 @@
  */
 package org.apache.hadoop.hdds.scm.server;
 
+import org.apache.hadoop.hdds.client.RatisReplicationConfig;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.conf.ReconfigurationHandler;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
 import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DecommissionScmRequestProto;
 import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DecommissionScmResponseProto;
+import org.apache.hadoop.hdds.scm.container.ContainerInfo;
+import org.apache.hadoop.hdds.scm.container.ContainerManagerImpl;
 import org.apache.hadoop.hdds.scm.HddsTestUtils;
 import org.apache.hadoop.hdds.scm.ha.SCMContext;
 import org.apache.hadoop.hdds.scm.ha.SCMHAManagerStub;
+import org.apache.hadoop.hdds.scm.ha.SCMNodeDetails;
+import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
 import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocolServerSideTranslatorPB;
 import org.apache.hadoop.hdds.utils.ProtocolMessageMetrics;
 import org.apache.hadoop.ozone.container.common.SCMTestUtils;
@@ -35,9 +42,13 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.ArrayList;
+import java.util.List;
 
 import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_READONLY_ADMINISTRATORS;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -112,4 +123,51 @@ public void testReadOnlyAdmins() throws IOException {
       UserGroupInformation.reset();
     }
   }
+
+  /**
+   * Tests listContainer of scm.
+   */
+  @Test
+  public void testScmListContainer() throws Exception {
+    SCMClientProtocolServer scmServer =
+        new SCMClientProtocolServer(new OzoneConfiguration(),
+            mockStorageContainerManager(), mock(ReconfigurationHandler.class));
+
+    assertEquals(10, scmServer.listContainer(1, 10,
+        null, HddsProtos.ReplicationType.RATIS, null).size());
+    assertEquals(20, scmServer.listContainer(1, -1,
+        null, HddsProtos.ReplicationType.RATIS, null).size());
+    // Test call from a legacy client, which uses a different method of listContainer
+    assertEquals(10, scmServer.listContainer(1, 10, null,
+        HddsProtos.ReplicationFactor.THREE).size());
+    assertEquals(20, scmServer.listContainer(1, -1, null,
+        HddsProtos.ReplicationFactor.THREE).size());
+  }
+
+  private StorageContainerManager mockStorageContainerManager() {
+    List<ContainerInfo> infos = new ArrayList<>();
+    for (int i = 0; i < 20; i++) {
+      infos.add(newContainerInfoForTest());
+    }
+    ContainerManagerImpl containerManager = mock(ContainerManagerImpl.class);
+    when(containerManager.getContainers()).thenReturn(infos);
+    StorageContainerManager storageContainerManager = mock(StorageContainerManager.class);
+    when(storageContainerManager.getContainerManager()).thenReturn(containerManager);
+
+    SCMNodeDetails scmNodeDetails = mock(SCMNodeDetails.class);
+    when(scmNodeDetails.getClientProtocolServerAddress()).thenReturn(new InetSocketAddress("localhost", 9876));
+    when(scmNodeDetails.getClientProtocolServerAddressKey()).thenReturn("test");
+    when(storageContainerManager.getScmNodeDetails()).thenReturn(scmNodeDetails);
+    return storageContainerManager;
+  }
+
+  private ContainerInfo newContainerInfoForTest() {
+    return new ContainerInfo.Builder()
+        .setContainerID(1)
+        .setPipelineID(PipelineID.randomId())
+        .setReplicationConfig(
+            RatisReplicationConfig
+                .getInstance(HddsProtos.ReplicationFactor.THREE))
+        .build();
+  }
 }

hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ListSubcommand.java

@@ -55,8 +55,9 @@ public class ListSubcommand extends ScmSubcommand {
   private long startId;
 
   @Option(names = {"-c", "--count"},
-      description = "Maximum number of containers to list",
-      defaultValue = "20", showDefaultValue = Visibility.ALWAYS)
+      description = "Maximum number of containers to list." +
+          "Make count=-1 default to list all containers",
+      defaultValue = "-1", showDefaultValue = Visibility.ALWAYS)
   private int count;
 
   @Option(names = {"--state"},