HDDS-8188. Support max allowed length in response of ozone admin container list

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java

@@ -131,6 +131,11 @@ public final class ScmConfigKeys {
       "hdds.ratis.snapshot.threshold";
   public static final long HDDS_RATIS_SNAPSHOT_THRESHOLD_DEFAULT = 100000;
 
+  public static final String HDDS_CONTAINER_LIST_MAX_COUNT =
+      "hdds.container.list.max.count";
+
+  public static final int HDDS_CONTAINER_LIST_MAX_COUNT_DEFAULT = 4096;
+
   // TODO : this is copied from OzoneConsts, may need to move to a better place
   public static final String OZONE_SCM_CHUNK_SIZE_KEY = "ozone.scm.chunk.size";
   // 4 MB by default

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java

@@ -121,10 +121,11 @@ void deleteContainer(long containerId, Pipeline pipeline, boolean force)
    * @param startContainerID start containerID.
    * @param count count must be {@literal >} 0.
    *
-   * @return a list of pipeline.
+   * @return a list of containers capped by max count allowed
+   * in "hdds.container.list.max.count" and total number of containers.
    * @throws IOException
    */
-  List<ContainerInfo> listContainer(long startContainerID,
+  Pair<List<ContainerInfo>, Long> listContainer(long startContainerID,
       int count) throws IOException;
 
   /**
@@ -134,10 +135,11 @@ List<ContainerInfo> listContainer(long startContainerID,
    * @param count count must be {@literal >} 0.
    * @param state Container of this state will be returned.
    * @param replicationConfig container replication Config.
-   * @return a list of pipeline.
+   * @return a list of containers capped by max count allowed
+   * in "hdds.container.list.max.count" and total number of containers.
    * @throws IOException
    */
-  List<ContainerInfo> listContainer(long startContainerID, int count,
+  Pair<List<ContainerInfo>, Long> listContainer(long startContainerID, int count,
       HddsProtos.LifeCycleState state,
       HddsProtos.ReplicationType replicationType,
       ReplicationConfig replicationConfig)

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java

@@ -145,10 +145,11 @@ List<ContainerWithPipeline> getExistContainerWithPipelinesInBatch(
    *              Usually the count will be replace with a very big
    *              value instead of being unlimited in case the db is very big)
    *
-   * @return a list of container.
+   * @return a list of containers capped by max count allowed
+   * in "hdds.container.list.max.count" and total number of containers.
    * @throws IOException
    */
-  List<ContainerInfo> listContainer(long startContainerID,
+  Pair<List<ContainerInfo>, Long> listContainer(long startContainerID,
       int count) throws IOException;
 
   /**
@@ -164,10 +165,11 @@ List<ContainerInfo> listContainer(long startContainerID,
    *              value instead of being unlimited in case the db is very big)
    * @param state Container with this state will be returned.
    *
-   * @return a list of container.
+   * @return a list of containers capped by max count allowed
+   * in "hdds.container.list.max.count" and total number of containers.
    * @throws IOException
    */
-  List<ContainerInfo> listContainer(long startContainerID,
+  Pair<List<ContainerInfo>, Long> listContainer(long startContainerID,
       int count, HddsProtos.LifeCycleState state) throws IOException;
 
   /**
@@ -183,10 +185,11 @@ List<ContainerInfo> listContainer(long startContainerID,
    *              value instead of being unlimited in case the db is very big)
    * @param state Container with this state will be returned.
    * @param factor Container factor
-   * @return a list of container.
+   * @return a list of containers capped by max count allowed
+   * in "hdds.container.list.max.count" and total number of containers.
    * @throws IOException
    */
-  List<ContainerInfo> listContainer(long startContainerID,
+  Pair<List<ContainerInfo>, Long> listContainer(long startContainerID,
       int count, HddsProtos.LifeCycleState state,
       HddsProtos.ReplicationFactor factor) throws IOException;
 
@@ -204,10 +207,11 @@ List<ContainerInfo> listContainer(long startContainerID,
    *              value instead of being unlimited in case the db is very big)
    * @param state Container with this state will be returned.
    * @param replicationConfig Replication config for the containers
-   * @return a list of container.
+   * @return a list of containers capped by max count allowed
+   * in "hdds.container.list.max.count" and total number of containers.
    * @throws IOException
    */
-  List<ContainerInfo> listContainer(long startContainerID,
+  Pair<List<ContainerInfo>, Long> listContainer(long startContainerID,
       int count, HddsProtos.LifeCycleState state,
       HddsProtos.ReplicationType replicationType,
       ReplicationConfig replicationConfig) throws IOException;

hadoop-hdds/common/src/main/resources/ozone-default.xml

@@ -161,6 +161,13 @@
       this not set. Ideally, this should be mapped to a fast disk like an SSD.
     </description>
   </property>
+  <property>
+    <name>hdds.container.list.max.count</name>
+    <value>4096</value>
+    <tag>OZONE, CONTAINER, MANAGEMENT</tag>
+    <description>The max number of containers info could be included in
+      response of ListContainer request.</description>
+  </property>
   <property>
     <name>hdds.datanode.dir</name>
     <value/>

hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java

@@ -380,19 +380,19 @@ public List<ContainerWithPipeline> getExistContainerWithPipelinesInBatch(
    * {@inheritDoc}
    */
   @Override
-  public List<ContainerInfo> listContainer(long startContainerID, int count)
+  public Pair<List<ContainerInfo>, Long> listContainer(long startContainerID, int count)
       throws IOException {
     return listContainer(startContainerID, count, null, null, null);
   }
 
   @Override
-  public List<ContainerInfo> listContainer(long startContainerID, int count,
+  public Pair<List<ContainerInfo>, Long> listContainer(long startContainerID, int count,
       HddsProtos.LifeCycleState state) throws IOException {
     return listContainer(startContainerID, count, state, null, null);
   }
 
   @Override
-  public List<ContainerInfo> listContainer(long startContainerID, int count,
+  public Pair<List<ContainerInfo>, Long> listContainer(long startContainerID, int count,
       HddsProtos.LifeCycleState state,
       HddsProtos.ReplicationType replicationType,
       ReplicationConfig replicationConfig)
@@ -434,12 +434,12 @@ public List<ContainerInfo> listContainer(long startContainerID, int count,
         .getContainersList()) {
       containerList.add(ContainerInfo.fromProtobuf(containerInfoProto));
     }
-    return containerList;
+    return Pair.of(containerList, response.getContainerCount());
   }
 
   @Deprecated
   @Override
-  public List<ContainerInfo> listContainer(long startContainerID, int count,
+  public Pair<List<ContainerInfo>, Long> listContainer(long startContainerID, int count,
       HddsProtos.LifeCycleState state, HddsProtos.ReplicationFactor factor)
       throws IOException {
     throw new UnsupportedOperationException("Should no longer be called from " +
@@ -1171,7 +1171,7 @@ public void close() {
   public List<ContainerInfo> getListOfContainers(
       long startContainerID, int count, HddsProtos.LifeCycleState state)
       throws IOException {
-    return listContainer(startContainerID, count, state);
+    return listContainer(startContainerID, count, state).getLeft();
   }
 
   @Override

hadoop-hdds/interface-admin/src/main/proto/ScmAdminProtocol.proto

@@ -293,6 +293,7 @@ message SCMListContainerRequestProto {
 
 message SCMListContainerResponseProto {
   repeated ContainerInfoProto containers = 1;
+  optional int64 containerCount = 2;
 }
 
 message SCMDeleteContainerRequestProto {

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/ContainerManagerImpl.java

@@ -86,6 +86,8 @@ public class ContainerManagerImpl implements ContainerManager {
   @SuppressWarnings("java:S2245") // no need for secure random
   private final Random random = new Random();
 
+  private int maxCountOfContainerList;
+
   /**
    *
    */
@@ -115,6 +117,10 @@ public ContainerManagerImpl(
         .getInt(ScmConfigKeys.OZONE_SCM_PIPELINE_OWNER_CONTAINER_COUNT,
             ScmConfigKeys.OZONE_SCM_PIPELINE_OWNER_CONTAINER_COUNT_DEFAULT);
 
+    this.maxCountOfContainerList = conf
+        .getInt(ScmConfigKeys.HDDS_CONTAINER_LIST_MAX_COUNT,
+            ScmConfigKeys.HDDS_CONTAINER_LIST_MAX_COUNT_DEFAULT);
+
     this.scmContainerManagerMetrics = SCMContainerManagerMetrics.create();
   }
 

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocolServerSideTranslatorPB.java

@@ -849,21 +849,22 @@ public SCMListContainerResponseProto listContainer(
     } else if (request.hasFactor()) {
       factor = request.getFactor();
     }
-    List<ContainerInfo> containerList;
+    Pair<List<ContainerInfo>, Long> containerListAndTotalCount;
     if (factor != null) {
       // Call from a legacy client
-      containerList =
+      containerListAndTotalCount =
           impl.listContainer(startContainerID, count, state, factor);
     } else {
-      containerList =
+      containerListAndTotalCount =
           impl.listContainer(startContainerID, count, state, replicationType,
               repConfig);
     }
     SCMListContainerResponseProto.Builder builder =
         SCMListContainerResponseProto.newBuilder();
-    for (ContainerInfo container : containerList) {
+    for (ContainerInfo container : containerListAndTotalCount.getLeft()) {
       builder.addContainers(container.getProtobuf());
     }
+    builder.setContainerCount(containerListAndTotalCount.getRight());
     return builder.build();
   }