Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 0 additions & 9 deletions hadoop-hdds/common/src/main/resources/ozone-default.xml
Original file line number Diff line number Diff line change
Expand Up @@ -4557,13 +4557,4 @@
allowing for better identification and analysis of performance issues.
</description>
</property>

<property>
<name>ozone.om.server.list.max.size</name>
<value>1000</value>
<tag>OZONE, OM</tag>
<description>
Configuration property to configure the max server side response size for list calls on om.
</description>
</property>
</configuration>
Original file line number Diff line number Diff line change
Expand Up @@ -619,9 +619,4 @@ private OMConfigKeys() {
public static final String OZONE_OM_MAX_BUCKET =
"ozone.om.max.buckets";
public static final int OZONE_OM_MAX_BUCKET_DEFAULT = 100000;
/**
* Configuration property to configure the max server side response size for list calls.
*/
public static final String OZONE_OM_SERVER_LIST_MAX_SIZE = "ozone.om.server.list.max.size";
public static final int OZONE_OM_SERVER_LIST_MAX_SIZE_DEFAULT = 1000;
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.hadoop.ozone.om;

import org.apache.hadoop.hdds.conf.Config;
import org.apache.hadoop.hdds.conf.ConfigGroup;
import org.apache.hadoop.hdds.conf.ConfigTag;
import org.apache.hadoop.hdds.conf.PostConstruct;

/**
* Ozone Manager configuration.
*/
@ConfigGroup(prefix = "ozone.om")
public class OmConfig {

@Config(
key = "server.list.max.size",
defaultValue = "1000",
description = "Configuration property to configure the max server side response size for list calls on om.",
tags = { ConfigTag.OM, ConfigTag.OZONE }
)
private long maxListSize;

public long getMaxListSize() {
return maxListSize;
}

public void setMaxListSize(long newValue) {
maxListSize = newValue;
validate();
}

@PostConstruct
public void validate() {
if (maxListSize <= 0) {
maxListSize = Defaults.SERVER_LIST_MAX_SIZE;
}
}

/**
* String keys for tests and grep.
*/
public static final class Keys {
public static final String SERVER_LIST_MAX_SIZE = "ozone.om.server.list.max.size";
}

/**
* Default values for tests.
*/
static final class Defaults {
public static final long SERVER_LIST_MAX_SIZE = 1000;
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.hadoop.ozone.om;

import org.apache.hadoop.hdds.conf.MutableConfigurationSource;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;

import static org.assertj.core.api.Assertions.assertThat;

class TestOmConfig {

@Test
void acceptsValidValues() {
final long validMaxListSize = 42;
MutableConfigurationSource conf = new OzoneConfiguration();
conf.setLong(OmConfig.Keys.SERVER_LIST_MAX_SIZE, validMaxListSize);

OmConfig subject = conf.getObject(OmConfig.class);

assertThat(subject.getMaxListSize())
.isEqualTo(validMaxListSize);
}

@ParameterizedTest
@ValueSource(longs = {-1, 0})
void overridesInvalidListSize(long invalidValue) {
MutableConfigurationSource conf = new OzoneConfiguration();
conf.setLong(OmConfig.Keys.SERVER_LIST_MAX_SIZE, invalidValue);

OmConfig subject = conf.getObject(OmConfig.class);

assertThat(subject.getMaxListSize())
.isEqualTo(OmConfig.Defaults.SERVER_LIST_MAX_SIZE);
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@
import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.ozone.om.OMMetadataManager;
import org.apache.hadoop.ozone.om.OMMetrics;
import org.apache.hadoop.ozone.om.OmConfig;
import org.apache.hadoop.ozone.om.OzonePrefixPathImpl;
import org.apache.hadoop.ozone.om.TrashPolicyOzone;
import org.apache.hadoop.ozone.om.exceptions.OMException;
Expand Down Expand Up @@ -118,7 +119,6 @@
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_ITERATE_BATCH_SIZE;
import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SERVER_LIST_MAX_SIZE;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND;
import static org.apache.hadoop.ozone.om.helpers.BucketLayout.FILE_SYSTEM_OPTIMIZED;
import static org.assertj.core.api.Assertions.assertThat;
Expand Down Expand Up @@ -184,7 +184,7 @@ void init() throws Exception {
conf.setFloat(OMConfigKeys.OZONE_FS_TRASH_INTERVAL_KEY, TRASH_INTERVAL);
conf.setFloat(FS_TRASH_INTERVAL_KEY, TRASH_INTERVAL);
conf.setFloat(FS_TRASH_CHECKPOINT_INTERVAL_KEY, TRASH_INTERVAL / 2);
conf.setInt(OZONE_OM_SERVER_LIST_MAX_SIZE, 2);
conf.setInt(OmConfig.Keys.SERVER_LIST_MAX_SIZE, 2);
conf.setBoolean(OZONE_ACL_ENABLED, true);
conf.setBoolean(OzoneConfigKeys.OZONE_HBASE_ENHANCEMENTS_ALLOWED, true);
conf.setBoolean("ozone.client.hbase.enhancements.allowed", true);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.om.KeyManagerImpl;
import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.ozone.om.OmConfig;
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
import org.apache.hadoop.util.ToolRunner;
Expand All @@ -57,7 +58,6 @@
import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_OFS_URI_SCHEME;
import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_INDICATOR;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SERVER_LIST_MAX_SIZE;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_SST_FILTERING_SERVICE_INTERVAL;
import static org.apache.hadoop.ozone.om.OmSnapshotManager.getSnapshotPath;
import static org.assertj.core.api.Assertions.assertThat;
Expand Down Expand Up @@ -96,7 +96,7 @@ static void initClass() throws Exception {
conf.setBoolean(OMConfigKeys.OZONE_FILESYSTEM_SNAPSHOT_ENABLED_KEY, true);
conf.setTimeDuration(OZONE_SNAPSHOT_DELETING_SERVICE_INTERVAL, 1, TimeUnit.SECONDS);
conf.setInt(OZONE_SNAPSHOT_SST_FILTERING_SERVICE_INTERVAL, KeyManagerImpl.DISABLE_VALUE);
conf.setInt(OZONE_OM_SERVER_LIST_MAX_SIZE, 20);
conf.setInt(OmConfig.Keys.SERVER_LIST_MAX_SIZE, 20);
conf.setInt(OZONE_FS_LISTING_PAGE_SIZE, 30);

// Start the cluster
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,6 @@
import static com.google.common.collect.Lists.newLinkedList;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_LIST_CACHE_SIZE;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_ITERATE_BATCH_SIZE;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SERVER_LIST_MAX_SIZE;
import static org.junit.jupiter.params.provider.Arguments.of;
import static org.junit.jupiter.api.Assertions.assertEquals;

Expand Down Expand Up @@ -81,7 +80,7 @@ public static void init() throws Exception {
// Set the number of keys to be processed during batch operate.
conf.setInt(OZONE_FS_ITERATE_BATCH_SIZE, 3);
conf.setInt(OZONE_CLIENT_LIST_CACHE_SIZE, 3);
conf.setInt(OZONE_OM_SERVER_LIST_MAX_SIZE, 2);
conf.setInt(OmConfig.Keys.SERVER_LIST_MAX_SIZE, 2);
cluster = MiniOzoneCluster.newBuilder(conf).build();
cluster.waitForClusterToBeReady();
client = cluster.newClient();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,6 @@

import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_LIST_CACHE_SIZE;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_ITERATE_BATCH_SIZE;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SERVER_LIST_MAX_SIZE;
import static org.junit.jupiter.api.Assertions.assertEquals;

/**
Expand Down Expand Up @@ -82,7 +81,7 @@ public static void init() throws Exception {
// Set the number of keys to be processed during batch operate.
conf.setInt(OZONE_FS_ITERATE_BATCH_SIZE, 3);
conf.setInt(OZONE_CLIENT_LIST_CACHE_SIZE, 3);
conf.setInt(OZONE_OM_SERVER_LIST_MAX_SIZE, 2);
conf.setInt(OmConfig.Keys.SERVER_LIST_MAX_SIZE, 2);
cluster = MiniOzoneCluster.newBuilder(conf).build();
cluster.waitForClusterToBeReady();
client = cluster.newClient();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -373,6 +373,7 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
private ScmTopologyClient scmTopologyClient;
private final Text omRpcAddressTxt;
private OzoneConfiguration configuration;
private OmConfig config;
private RPC.Server omRpcServer;
private GrpcOzoneManagerServer omS3gGrpcServer;
private final InetSocketAddress omRpcAddress;
Expand Down Expand Up @@ -4136,9 +4137,14 @@ public OzoneConfiguration getConfiguration() {
return configuration;
}

public OmConfig getConfig() {
return config;
}

@VisibleForTesting
public void setConfiguration(OzoneConfiguration conf) {
this.configuration = conf;
config = conf.getObject(OmConfig.class);
}

public OzoneConfiguration reloadConfiguration() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -147,8 +147,6 @@

import com.google.common.collect.Lists;

import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SERVER_LIST_MAX_SIZE;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SERVER_LIST_MAX_SIZE_DEFAULT;
import static org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature.HBASE_SUPPORT;
import static org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature.MULTITENANCY_SCHEMA;
import static org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature.FILESYSTEM_SNAPSHOT;
Expand Down Expand Up @@ -188,11 +186,7 @@ public class OzoneManagerRequestHandler implements RequestHandler {

public OzoneManagerRequestHandler(OzoneManager om) {
this.impl = om;
this.maxKeyListSize = om.getConfiguration().getLong(OZONE_OM_SERVER_LIST_MAX_SIZE,
OZONE_OM_SERVER_LIST_MAX_SIZE_DEFAULT);
if (this.maxKeyListSize <= 0) {
this.maxKeyListSize = OZONE_OM_SERVER_LIST_MAX_SIZE_DEFAULT;
}
this.maxKeyListSize = om.getConfig().getMaxListSize();
}

//TODO simplify it to make it shorter
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,8 @@ public void testMultiTenancyRequestsWhenDisabled() throws IOException {
doCallRealMethod().when(ozoneManager).checkS3MultiTenancyEnabled();
final OzoneConfiguration conf = new OzoneConfiguration();
when(ozoneManager.getConfiguration()).thenReturn(conf);
final OmConfig omConfig = conf.getObject(OmConfig.class);
when(ozoneManager.getConfig()).thenReturn(omConfig);
when(ozoneManager.isS3MultiTenancyEnabled()).thenReturn(false);

final String tenantId = "test-tenant";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import org.apache.hadoop.hdds.utils.db.cache.CacheValue;
import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.ozone.om.OMMetadataManager;
import org.apache.hadoop.ozone.om.OmConfig;
import org.apache.hadoop.ozone.om.OmMetadataManagerImpl;
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.exceptions.OMException;
Expand Down Expand Up @@ -110,6 +111,8 @@ public void testUnknownRequestHandling()
ozoneManager);
when(ozoneManager.getMetadataManager()).thenReturn(omMetadataManager);
when(ozoneManager.getConfiguration()).thenReturn(ozoneConfiguration);
final OmConfig omConfig = ozoneConfiguration.getObject(OmConfig.class);
when(ozoneManager.getConfig()).thenReturn(omConfig);

OzoneManagerRatisServer ratisServer = mock(OzoneManagerRatisServer.class);
ProtocolMessageMetrics<ProtocolMessageEnum> protocolMessageMetrics =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import org.apache.hadoop.hdds.utils.TransactionInfo;
import org.apache.hadoop.hdds.security.SecurityConfig;
import org.apache.hadoop.ozone.om.OMStorage;
import org.apache.hadoop.ozone.om.OmConfig;
import org.apache.hadoop.ozone.security.OMCertificateClient;
import org.apache.hadoop.ozone.OmUtils;
import org.apache.hadoop.ozone.OzoneConsts;
Expand Down Expand Up @@ -122,6 +123,8 @@ public void init(@TempDir Path metaDirPath) throws Exception {
initialTermIndex = TermIndex.valueOf(0, 0);
when(ozoneManager.getTransactionInfo()).thenReturn(TransactionInfo.DEFAULT_VALUE);
when(ozoneManager.getConfiguration()).thenReturn(conf);
final OmConfig omConfig = conf.getObject(OmConfig.class);
when(ozoneManager.getConfig()).thenReturn(omConfig);
secConfig = new SecurityConfig(conf);
HddsProtos.OzoneManagerDetailsProto omInfo =
OzoneManager.getOmDetailsProto(conf, omID);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import org.apache.hadoop.ozone.audit.AuditMessage;
import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.ozone.om.OMMetadataManager;
import org.apache.hadoop.ozone.om.OmConfig;
import org.apache.hadoop.ozone.om.OmMetadataManagerImpl;
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.OzoneManagerPrepareState;
Expand Down Expand Up @@ -100,6 +101,8 @@ public void setup() throws Exception {
when(ozoneManagerRatisServer.getOzoneManager()).thenReturn(ozoneManager);
when(ozoneManager.getTransactionInfo()).thenReturn(mock(TransactionInfo.class));
when(ozoneManager.getConfiguration()).thenReturn(conf);
final OmConfig omConfig = conf.getObject(OmConfig.class);
when(ozoneManager.getConfig()).thenReturn(omConfig);
ozoneManagerStateMachine =
new OzoneManagerStateMachine(ozoneManagerRatisServer, false);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@

import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.ozone.om.OmConfig;
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.helpers.BasicOmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.ListKeysLightResult;
Expand All @@ -40,19 +41,17 @@
import java.util.stream.Collectors;
import java.util.stream.IntStream;

import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SERVER_LIST_MAX_SIZE;

/**
* Test class to test out OzoneManagerRequestHandler.
*/
public class TestOzoneManagerRequestHandler {


private OzoneManagerRequestHandler getRequestHandler(int limitListKeySize) {
OzoneConfiguration conf = new OzoneConfiguration();
conf.setInt(OZONE_OM_SERVER_LIST_MAX_SIZE, limitListKeySize);
OmConfig config = OzoneConfiguration.newInstanceOf(OmConfig.class);
config.setMaxListSize(limitListKeySize);
OzoneManager ozoneManager = Mockito.mock(OzoneManager.class);
Mockito.when(ozoneManager.getConfiguration()).thenReturn(conf);
Mockito.when(ozoneManager.getConfig()).thenReturn(config);
return new OzoneManagerRequestHandler(ozoneManager);
}

Expand Down
Loading