HDDS-12293. Make ozone.om.server.list.max.size reconfigurable

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OmConfig.java

@@ -21,12 +21,13 @@
 import org.apache.hadoop.hdds.conf.ConfigGroup;
 import org.apache.hadoop.hdds.conf.ConfigTag;
 import org.apache.hadoop.hdds.conf.PostConstruct;
+import org.apache.hadoop.hdds.conf.ReconfigurableConfig;
 
 /**
  * Ozone Manager configuration.
  */
 @ConfigGroup(prefix = "ozone.om")
-public class OmConfig {
+public class OmConfig extends ReconfigurableConfig {
 
   /** This config needs to be enabled, when S3G created objects used via FileSystem API. */
   @Config(
@@ -48,6 +49,7 @@ public class OmConfig {
       key = "server.list.max.size",
       defaultValue = "1000",
       description = "Configuration property to configure the max server side response size for list calls on om.",
+      reconfigurable = true,
       tags = { ConfigTag.OM, ConfigTag.OZONE }
   )
   private long maxListSize;

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/reconfig/TestOmReconfiguration.java

@@ -26,8 +26,10 @@
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
 import com.google.common.collect.ImmutableSet;
+import java.util.Set;
 import org.apache.hadoop.conf.ReconfigurationException;
 import org.apache.hadoop.hdds.conf.ReconfigurationHandler;
+import org.apache.hadoop.ozone.om.OmConfig;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.ValueSource;
@@ -44,10 +46,15 @@ ReconfigurationHandler getSubject() {
 
   @Test
   void reconfigurableProperties() {
-    assertProperties(getSubject(),
-        ImmutableSet.of(OZONE_ADMINISTRATORS, OZONE_READONLY_ADMINISTRATORS,
-            OZONE_OM_VOLUME_LISTALL_ALLOWED,
-            OZONE_KEY_DELETING_LIMIT_PER_TASK));
+    Set<String> expected = ImmutableSet.<String>builder()
+        .add(OZONE_ADMINISTRATORS)
+        .add(OZONE_KEY_DELETING_LIMIT_PER_TASK)
+        .add(OZONE_OM_VOLUME_LISTALL_ALLOWED)
+        .add(OZONE_READONLY_ADMINISTRATORS)
+        .addAll(new OmConfig().reconfigurableProperties())
+        .build();
+
+    assertProperties(getSubject(), expected);
   }
 
   @Test
@@ -73,6 +80,17 @@ void readOnlyAdmins() throws ReconfigurationException {
         cluster().getOzoneManager().getOmReadOnlyAdminUsernames());
   }
 
+  @Test
+  public void maxListSize() throws ReconfigurationException {
+    final long initialValue = cluster().getOzoneManager().getConfig().getMaxListSize();
+
+    getSubject().reconfigurePropertyImpl(OmConfig.Keys.SERVER_LIST_MAX_SIZE,
+        String.valueOf(initialValue + 1));
+
+    assertEquals(initialValue + 1,
+        cluster().getOzoneManager().getConfig().getMaxListSize());
+  }
+
   @Test
   public void keyDeletingLimitPerTask() throws ReconfigurationException {
     int originLimit = cluster().getOzoneManager()

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java

@@ -525,6 +525,7 @@ private OzoneManager(OzoneConfiguration conf, StartupOption startupOption)
     omId = omStorage.getOmId();
     reconfigurationHandler =
         new ReconfigurationHandler("OM", conf, this::checkAdminUserPrivilege)
+            .register(config)
             .register(OZONE_ADMINISTRATORS, this::reconfOzoneAdmins)
             .register(OZONE_READONLY_ADMINISTRATORS,
                 this::reconfOzoneReadOnlyAdmins)

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java

@@ -178,12 +178,10 @@ public class OzoneManagerRequestHandler implements RequestHandler {
       LoggerFactory.getLogger(OzoneManagerRequestHandler.class);
   private final OzoneManager impl;
   private FaultInjector injector;
-  private long maxKeyListSize;
 
 
   public OzoneManagerRequestHandler(OzoneManager om) {
     this.impl = om;
-    this.maxKeyListSize = om.getConfig().getMaxListSize();
   }
 
   //TODO simplify it to make it shorter
@@ -747,7 +745,7 @@ private ListKeysResponse listKeys(ListKeysRequest request, int clientVersion)
         request.getBucketName(),
         request.getStartKey(),
         request.getPrefix(),
-        (int)Math.min(this.maxKeyListSize, request.getCount()));
+        limitListSizeInt(request.getCount()));
     for (OmKeyInfo key : listKeysResult.getKeys()) {
       resp.addKeyInfo(key.getProtobuf(true, clientVersion));
     }
@@ -765,7 +763,7 @@ private ListKeysLightResponse listKeysLight(ListKeysRequest request)
         request.getBucketName(),
         request.getStartKey(),
         request.getPrefix(),
-        (int)Math.min(this.maxKeyListSize, request.getCount()));
+        limitListSizeInt(request.getCount()));
     for (BasicOmKeyInfo key : listKeysLightResult.getKeys()) {
       resp.addBasicKeyInfo(key.getProtobuf());
     }
@@ -923,7 +921,7 @@ private ListOpenFilesResponse listOpenFiles(ListOpenFilesRequest req,
     ListOpenFilesResponse.Builder resp = ListOpenFilesResponse.newBuilder();
 
     ListOpenFilesResult res =
-        impl.listOpenFiles(req.getPath(), req.getCount(), req.getToken());
+        impl.listOpenFiles(req.getPath(), limitListSizeInt(req.getCount()), req.getToken());
     // TODO: Is there a clean way to avoid ser-de for responses:
     //  OM does: ListOpenFilesResult -> ListOpenFilesResponse
     //  Client : ListOpenFilesResponse -> ListOpenFilesResult
@@ -1239,7 +1237,7 @@ private ListStatusResponse listStatus(
         request.hasAllowPartialPrefix() && request.getAllowPartialPrefix();
     List<OzoneFileStatus> statuses =
         impl.listStatus(omKeyArgs, request.getRecursive(),
-            request.getStartKey(), Math.min(this.maxKeyListSize, request.getNumEntries()),
+            request.getStartKey(), limitListSize(request.getNumEntries()),
             allowPartialPrefixes);
     ListStatusResponse.Builder
         listStatusResponseBuilder =
@@ -1265,7 +1263,7 @@ private ListStatusLightResponse listStatusLight(
         request.hasAllowPartialPrefix() && request.getAllowPartialPrefix();
     List<OzoneFileStatusLight> statuses =
         impl.listStatusLight(omKeyArgs, request.getRecursive(),
-            request.getStartKey(), Math.min(this.maxKeyListSize, request.getNumEntries()),
+            request.getStartKey(), limitListSize(request.getNumEntries()),
             allowPartialPrefixes);
     ListStatusLightResponse.Builder
         listStatusLightResponseBuilder =
@@ -1493,7 +1491,7 @@ private OzoneManagerProtocolProtos.ListSnapshotResponse getSnapshots(
       throws IOException {
     ListSnapshotResponse implResponse = impl.listSnapshot(
         request.getVolumeName(), request.getBucketName(), request.getPrefix(),
-        request.getPrevSnapshot(), (int)Math.min(request.getMaxListResult(), maxKeyListSize));
+        request.getPrevSnapshot(), limitListSizeInt(request.getMaxListResult()));
 
     List<OzoneManagerProtocolProtos.SnapshotInfo> snapshotInfoList = implResponse.getSnapshotInfos()
         .stream().map(SnapshotInfo::getProtobuf).collect(Collectors.toList());
@@ -1568,4 +1566,13 @@ private OzoneManagerProtocolProtos.StartQuotaRepairResponse startQuotaRepair(
     impl.startQuotaRepair(req.getBucketsList());
     return OzoneManagerProtocolProtos.StartQuotaRepairResponse.newBuilder().build();
   }
+
+  private int limitListSizeInt(int requestedSize) {
+    return Math.toIntExact(limitListSize(requestedSize));
+  }
+
+  private long limitListSize(long requestedSize) {
+    return Math.min(requestedSize, impl.getConfig().getMaxListSize());
+  }
+
 }