diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OmConfig.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OmConfig.java index 8c516a1caf2c..901a911478b4 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OmConfig.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OmConfig.java @@ -21,12 +21,13 @@ import org.apache.hadoop.hdds.conf.ConfigGroup; import org.apache.hadoop.hdds.conf.ConfigTag; import org.apache.hadoop.hdds.conf.PostConstruct; +import org.apache.hadoop.hdds.conf.ReconfigurableConfig; /** * Ozone Manager configuration. */ @ConfigGroup(prefix = "ozone.om") -public class OmConfig { +public class OmConfig extends ReconfigurableConfig { /** This config needs to be enabled, when S3G created objects used via FileSystem API. */ @Config( @@ -48,6 +49,7 @@ public class OmConfig { key = "server.list.max.size", defaultValue = "1000", description = "Configuration property to configure the max server side response size for list calls on om.", + reconfigurable = true, tags = { ConfigTag.OM, ConfigTag.OZONE } ) private long maxListSize; diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/reconfig/TestOmReconfiguration.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/reconfig/TestOmReconfiguration.java index 0ba35ea835ea..258325ce070c 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/reconfig/TestOmReconfiguration.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/reconfig/TestOmReconfiguration.java @@ -26,8 +26,10 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import com.google.common.collect.ImmutableSet; +import java.util.Set; import org.apache.hadoop.conf.ReconfigurationException; import org.apache.hadoop.hdds.conf.ReconfigurationHandler; +import org.apache.hadoop.ozone.om.OmConfig; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; @@ -44,10 +46,15 @@ ReconfigurationHandler getSubject() { @Test void reconfigurableProperties() { - assertProperties(getSubject(), - ImmutableSet.of(OZONE_ADMINISTRATORS, OZONE_READONLY_ADMINISTRATORS, - OZONE_OM_VOLUME_LISTALL_ALLOWED, - OZONE_KEY_DELETING_LIMIT_PER_TASK)); + Set expected = ImmutableSet.builder() + .add(OZONE_ADMINISTRATORS) + .add(OZONE_KEY_DELETING_LIMIT_PER_TASK) + .add(OZONE_OM_VOLUME_LISTALL_ALLOWED) + .add(OZONE_READONLY_ADMINISTRATORS) + .addAll(new OmConfig().reconfigurableProperties()) + .build(); + + assertProperties(getSubject(), expected); } @Test @@ -73,6 +80,17 @@ void readOnlyAdmins() throws ReconfigurationException { cluster().getOzoneManager().getOmReadOnlyAdminUsernames()); } + @Test + public void maxListSize() throws ReconfigurationException { + final long initialValue = cluster().getOzoneManager().getConfig().getMaxListSize(); + + getSubject().reconfigurePropertyImpl(OmConfig.Keys.SERVER_LIST_MAX_SIZE, + String.valueOf(initialValue + 1)); + + assertEquals(initialValue + 1, + cluster().getOzoneManager().getConfig().getMaxListSize()); + } + @Test public void keyDeletingLimitPerTask() throws ReconfigurationException { int originLimit = cluster().getOzoneManager() diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index fb7e30fd07e8..35d5169da97f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -525,6 +525,7 @@ private OzoneManager(OzoneConfiguration conf, StartupOption startupOption) omId = omStorage.getOmId(); reconfigurationHandler = new ReconfigurationHandler("OM", conf, this::checkAdminUserPrivilege) + .register(config) .register(OZONE_ADMINISTRATORS, this::reconfOzoneAdmins) .register(OZONE_READONLY_ADMINISTRATORS, this::reconfOzoneReadOnlyAdmins) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java index 422da953a4cf..39f20076409a 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java @@ -178,12 +178,10 @@ public class OzoneManagerRequestHandler implements RequestHandler { LoggerFactory.getLogger(OzoneManagerRequestHandler.class); private final OzoneManager impl; private FaultInjector injector; - private long maxKeyListSize; public OzoneManagerRequestHandler(OzoneManager om) { this.impl = om; - this.maxKeyListSize = om.getConfig().getMaxListSize(); } //TODO simplify it to make it shorter @@ -747,7 +745,7 @@ private ListKeysResponse listKeys(ListKeysRequest request, int clientVersion) request.getBucketName(), request.getStartKey(), request.getPrefix(), - (int)Math.min(this.maxKeyListSize, request.getCount())); + limitListSizeInt(request.getCount())); for (OmKeyInfo key : listKeysResult.getKeys()) { resp.addKeyInfo(key.getProtobuf(true, clientVersion)); } @@ -765,7 +763,7 @@ private ListKeysLightResponse listKeysLight(ListKeysRequest request) request.getBucketName(), request.getStartKey(), request.getPrefix(), - (int)Math.min(this.maxKeyListSize, request.getCount())); + limitListSizeInt(request.getCount())); for (BasicOmKeyInfo key : listKeysLightResult.getKeys()) { resp.addBasicKeyInfo(key.getProtobuf()); } @@ -923,7 +921,7 @@ private ListOpenFilesResponse listOpenFiles(ListOpenFilesRequest req, ListOpenFilesResponse.Builder resp = ListOpenFilesResponse.newBuilder(); ListOpenFilesResult res = - impl.listOpenFiles(req.getPath(), req.getCount(), req.getToken()); + impl.listOpenFiles(req.getPath(), limitListSizeInt(req.getCount()), req.getToken()); // TODO: Is there a clean way to avoid ser-de for responses: // OM does: ListOpenFilesResult -> ListOpenFilesResponse // Client : ListOpenFilesResponse -> ListOpenFilesResult @@ -1239,7 +1237,7 @@ private ListStatusResponse listStatus( request.hasAllowPartialPrefix() && request.getAllowPartialPrefix(); List statuses = impl.listStatus(omKeyArgs, request.getRecursive(), - request.getStartKey(), Math.min(this.maxKeyListSize, request.getNumEntries()), + request.getStartKey(), limitListSize(request.getNumEntries()), allowPartialPrefixes); ListStatusResponse.Builder listStatusResponseBuilder = @@ -1265,7 +1263,7 @@ private ListStatusLightResponse listStatusLight( request.hasAllowPartialPrefix() && request.getAllowPartialPrefix(); List statuses = impl.listStatusLight(omKeyArgs, request.getRecursive(), - request.getStartKey(), Math.min(this.maxKeyListSize, request.getNumEntries()), + request.getStartKey(), limitListSize(request.getNumEntries()), allowPartialPrefixes); ListStatusLightResponse.Builder listStatusLightResponseBuilder = @@ -1493,7 +1491,7 @@ private OzoneManagerProtocolProtos.ListSnapshotResponse getSnapshots( throws IOException { ListSnapshotResponse implResponse = impl.listSnapshot( request.getVolumeName(), request.getBucketName(), request.getPrefix(), - request.getPrevSnapshot(), (int)Math.min(request.getMaxListResult(), maxKeyListSize)); + request.getPrevSnapshot(), limitListSizeInt(request.getMaxListResult())); List snapshotInfoList = implResponse.getSnapshotInfos() .stream().map(SnapshotInfo::getProtobuf).collect(Collectors.toList()); @@ -1568,4 +1566,13 @@ private OzoneManagerProtocolProtos.StartQuotaRepairResponse startQuotaRepair( impl.startQuotaRepair(req.getBucketsList()); return OzoneManagerProtocolProtos.StartQuotaRepairResponse.newBuilder().build(); } + + private int limitListSizeInt(int requestedSize) { + return Math.toIntExact(limitListSize(requestedSize)); + } + + private long limitListSize(long requestedSize) { + return Math.min(requestedSize, impl.getConfig().getMaxListSize()); + } + }