diff --git a/hbase-common/src/main/resources/hbase-default.xml b/hbase-common/src/main/resources/hbase-default.xml index d2ecef6bda3d..61eb4a0059fa 100644 --- a/hbase-common/src/main/resources/hbase-default.xml +++ b/hbase-common/src/main/resources/hbase-default.xml @@ -656,6 +656,12 @@ possible configurations would overwhelm and obscure the important. The minimum size for a region to be considered for a merge, in whole MBs. + + hbase.normalizer.merge.merge_request_max_number_of_regions + 50 + The maximum number of region count in a merge request for merge + normalization. + hbase.table.normalization.enabled false diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java index dfae394b75a6..934b8de1b94c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java @@ -81,6 +81,9 @@ class SimpleRegionNormalizer implements RegionNormalizer, ConfigurationObserver static final int DEFAULT_MERGE_MIN_REGION_AGE_DAYS = 3; static final String MERGE_MIN_REGION_SIZE_MB_KEY = "hbase.normalizer.merge.min_region_size.mb"; static final int DEFAULT_MERGE_MIN_REGION_SIZE_MB = 0; + static final String MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT_KEY = + "hbase.normalizer.merge.merge_request_max_number_of_regions"; + static final long DEFAULT_MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT = 100; private MasterServices masterServices; private NormalizerConfiguration normalizerConfiguration; @@ -138,6 +141,16 @@ private static long parseMergeMinRegionSizeMb(final Configuration conf) { return settledValue; } + private static long parseMergeRequestMaxNumberOfRegionsCount(final Configuration conf) { + final long parsedValue = conf.getLong(MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT_KEY, + DEFAULT_MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT); + final long settledValue = Math.max(2, parsedValue); + if (parsedValue != settledValue) { + warnInvalidValue(MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT_KEY, parsedValue, settledValue); + } + return settledValue; + } + private static void warnInvalidValue(final String key, final T parsedValue, final T settledValue) { LOG.warn("Configured value {}={} is invalid. Setting value to {}.", key, parsedValue, @@ -186,6 +199,10 @@ public long getMergeMinRegionSizeMb() { return normalizerConfiguration.getMergeMinRegionSizeMb(); } + public long getMergeRequestMaxNumberOfRegionsCount() { + return normalizerConfiguration.getMergeRequestMaxNumberOfRegionsCount(); + } + @Override public void setMasterServices(final MasterServices masterServices) { this.masterServices = masterServices; @@ -382,19 +399,21 @@ private List computeMergeNormalizationPlans(final NormalizeCo break; } if ( - rangeMembers.isEmpty() // when there are no range members, seed the range with whatever - // we have. this way we're prepared in case the next region is - // 0-size. - || (rangeMembers.size() == 1 && sumRangeMembersSizeMb == 0) // when there is only one - // region and the size is 0, - // seed the range with - // whatever we have. - || regionSizeMb == 0 // always add an empty region to the current range. - || (regionSizeMb + sumRangeMembersSizeMb <= avgRegionSizeMb) - ) { // add the current region - // to the range when - // there's capacity - // remaining. + // when there are no range members, seed the range with whatever we have. this way we're + // prepared in case the next region is 0-size. + rangeMembers.isEmpty() + // when there is only one region and the size is 0, seed the range with whatever we + // have. + || (rangeMembers.size() == 1 && sumRangeMembersSizeMb == 0) + // add an empty region to the current range only if it doesn't exceed max merge request + // region count + || (regionSizeMb == 0 && rangeMembers.size() < getMergeRequestMaxNumberOfRegionsCount()) + // add region if current range region size is less than avg region size of table + // and current range doesn't exceed max merge request region count + || ((regionSizeMb + sumRangeMembersSizeMb <= avgRegionSizeMb) + && (rangeMembers.size() < getMergeRequestMaxNumberOfRegionsCount())) + ) { + // add the current region to the range when there's capacity remaining. rangeMembers.add(new NormalizationTarget(regionInfo, regionSizeMb)); sumRangeMembersSizeMb += regionSizeMb; continue; @@ -502,6 +521,7 @@ private static final class NormalizerConfiguration { private final int mergeMinRegionCount; private final Period mergeMinRegionAge; private final long mergeMinRegionSizeMb; + private final long mergeRequestMaxNumberOfRegionsCount; private final long cumulativePlansSizeLimitMb; private NormalizerConfiguration() { @@ -511,6 +531,7 @@ private NormalizerConfiguration() { mergeMinRegionCount = DEFAULT_MERGE_MIN_REGION_COUNT; mergeMinRegionAge = Period.ofDays(DEFAULT_MERGE_MIN_REGION_AGE_DAYS); mergeMinRegionSizeMb = DEFAULT_MERGE_MIN_REGION_SIZE_MB; + mergeRequestMaxNumberOfRegionsCount = DEFAULT_MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT; cumulativePlansSizeLimitMb = DEFAULT_CUMULATIVE_SIZE_LIMIT_MB; } @@ -522,6 +543,7 @@ private NormalizerConfiguration(final Configuration conf, mergeMinRegionCount = parseMergeMinRegionCount(conf); mergeMinRegionAge = parseMergeMinRegionAge(conf); mergeMinRegionSizeMb = parseMergeMinRegionSizeMb(conf); + mergeRequestMaxNumberOfRegionsCount = parseMergeRequestMaxNumberOfRegionsCount(conf); cumulativePlansSizeLimitMb = conf.getLong(CUMULATIVE_SIZE_LIMIT_MB_KEY, DEFAULT_CUMULATIVE_SIZE_LIMIT_MB); logConfigurationUpdated(SPLIT_ENABLED_KEY, currentConfiguration.isSplitEnabled(), @@ -534,6 +556,9 @@ private NormalizerConfiguration(final Configuration conf, currentConfiguration.getMergeMinRegionAge(), mergeMinRegionAge); logConfigurationUpdated(MERGE_MIN_REGION_SIZE_MB_KEY, currentConfiguration.getMergeMinRegionSizeMb(), mergeMinRegionSizeMb); + logConfigurationUpdated(MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT_KEY, + currentConfiguration.getMergeRequestMaxNumberOfRegionsCount(), + mergeRequestMaxNumberOfRegionsCount); } public Configuration getConf() { @@ -597,6 +622,10 @@ public long getMergeMinRegionSizeMb(NormalizeContext context) { return mergeMinRegionSizeMb; } + public long getMergeRequestMaxNumberOfRegionsCount() { + return mergeRequestMaxNumberOfRegionsCount; + } + private long getCumulativePlansSizeLimitMb() { return cumulativePlansSizeLimitMb; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java index 5dba036bb705..902205c74636 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java @@ -24,6 +24,7 @@ import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.MERGE_MIN_REGION_AGE_DAYS_KEY; import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.MERGE_MIN_REGION_COUNT_KEY; import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.MERGE_MIN_REGION_SIZE_MB_KEY; +import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT_KEY; import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.MIN_REGION_COUNT_KEY; import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.SPLIT_ENABLED_KEY; import static org.hamcrest.MatcherAssert.assertThat; @@ -503,6 +504,41 @@ public void testHonorsMergeMinRegionSizeInTD() { assertThat(normalizer.computePlansForTable(tableDescriptor), empty()); } + @Test + public void testHonorsMergeRequestMaxNumberOfRegionsCount() { + conf.setBoolean(SPLIT_ENABLED_KEY, false); + conf.setInt(MERGE_MIN_REGION_COUNT_KEY, 1); + conf.setInt(MERGE_MIN_REGION_SIZE_MB_KEY, 0); + conf.setInt(MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT_KEY, 3); + final TableName tableName = name.getTableName(); + final List regionInfos = createRegionInfos(tableName, 5); + final Map regionSizes = createRegionSizesMap(regionInfos, 0, 1, 0, 1, 0); + setupMocksForNormalizer(regionSizes, regionInfos); + assertEquals(3, normalizer.getMergeRequestMaxNumberOfRegionsCount()); + List plans = normalizer.computePlansForTable(tableDescriptor); + assertThat(plans, + contains( + new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(0), 0) + .addTarget(regionInfos.get(1), 1).addTarget(regionInfos.get(2), 0).build(), + new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(3), 1) + .addTarget(regionInfos.get(4), 0).build())); + } + + @Test + public void testHonorsMergeRequestMaxNumberOfRegionsCountDefault() { + conf.setBoolean(SPLIT_ENABLED_KEY, false); + conf.setInt(MERGE_MIN_REGION_COUNT_KEY, 1); + conf.setInt(MERGE_MIN_REGION_SIZE_MB_KEY, 0); + final TableName tableName = name.getTableName(); + final List regionInfos = createRegionInfos(tableName, 3); + final Map regionSizes = createRegionSizesMap(regionInfos, 0, 0, 0); + setupMocksForNormalizer(regionSizes, regionInfos); + assertEquals(50, normalizer.getMergeRequestMaxNumberOfRegionsCount()); + List plans = normalizer.computePlansForTable(tableDescriptor); + assertThat(plans, contains(new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(0), 0) + .addTarget(regionInfos.get(1), 0).addTarget(regionInfos.get(2), 0).build())); + } + @Test public void testMergeEmptyRegions0() { conf.setBoolean(SPLIT_ENABLED_KEY, false);