diff --git a/hbase-common/src/main/resources/hbase-default.xml b/hbase-common/src/main/resources/hbase-default.xml index c84eb8d183de..77a44694c291 100644 --- a/hbase-common/src/main/resources/hbase-default.xml +++ b/hbase-common/src/main/resources/hbase-default.xml @@ -649,6 +649,12 @@ possible configurations would overwhelm and obscure the important. The minimum size for a region to be considered for a merge, in whole MBs. + + hbase.normalizer.merge.merge_request_max_number_of_regions + 50 + The maximum number of region count in a merge request for merge + normalization. + hbase.table.normalization.enabled false diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java index 9efe96ed7660..12bd37f7ff82 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/SimpleRegionNormalizer.java @@ -74,6 +74,9 @@ class SimpleRegionNormalizer implements RegionNormalizer, ConfigurationObserver static final int DEFAULT_MERGE_MIN_REGION_AGE_DAYS = 3; static final String MERGE_MIN_REGION_SIZE_MB_KEY = "hbase.normalizer.merge.min_region_size.mb"; static final int DEFAULT_MERGE_MIN_REGION_SIZE_MB = 1; + static final String MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT_KEY = + "hbase.normalizer.merge.merge_request_max_number_of_regions"; + static final long DEFAULT_MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT = 100; private MasterServices masterServices; private NormalizerConfiguration normalizerConfiguration; @@ -131,6 +134,16 @@ private static long parseMergeMinRegionSizeMb(final Configuration conf) { return settledValue; } + private static long parseMergeRequestMaxNumberOfRegionsCount(final Configuration conf) { + final long parsedValue = conf.getLong(MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT_KEY, + DEFAULT_MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT); + final long settledValue = Math.max(2, parsedValue); + if (parsedValue != settledValue) { + warnInvalidValue(MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT_KEY, parsedValue, settledValue); + } + return settledValue; + } + private static void warnInvalidValue(final String key, final T parsedValue, final T settledValue) { LOG.warn("Configured value {}={} is invalid. Setting value to {}.", key, parsedValue, @@ -179,6 +192,10 @@ public long getMergeMinRegionSizeMb() { return normalizerConfiguration.getMergeMinRegionSizeMb(); } + public long getMergeRequestMaxNumberOfRegionsCount() { + return normalizerConfiguration.getMergeRequestMaxNumberOfRegionsCount(); + } + @Override public void setMasterServices(final MasterServices masterServices) { this.masterServices = masterServices; @@ -367,19 +384,21 @@ private List computeMergeNormalizationPlans(final NormalizeCo break; } if ( - rangeMembers.isEmpty() // when there are no range members, seed the range with whatever - // we have. this way we're prepared in case the next region is - // 0-size. - || (rangeMembers.size() == 1 && sumRangeMembersSizeMb == 0) // when there is only one - // region and the size is 0, - // seed the range with - // whatever we have. - || regionSizeMb == 0 // always add an empty region to the current range. - || (regionSizeMb + sumRangeMembersSizeMb <= avgRegionSizeMb) - ) { // add the current region - // to the range when - // there's capacity - // remaining. + // when there are no range members, seed the range with whatever we have. this way we're + // prepared in case the next region is 0-size. + rangeMembers.isEmpty() + // when there is only one region and the size is 0, seed the range with whatever we + // have. + || (rangeMembers.size() == 1 && sumRangeMembersSizeMb == 0) + // add an empty region to the current range only if it doesn't exceed max merge request + // region count + || (regionSizeMb == 0 && rangeMembers.size() < getMergeRequestMaxNumberOfRegionsCount()) + // add region if current range region size is less than avg region size of table + // and current range doesn't exceed max merge request region count + || ((regionSizeMb + sumRangeMembersSizeMb <= avgRegionSizeMb) + && (rangeMembers.size() < getMergeRequestMaxNumberOfRegionsCount())) + ) { + // add the current region to the range when there's capacity remaining. rangeMembers.add(new NormalizationTarget(regionInfo, regionSizeMb)); sumRangeMembersSizeMb += regionSizeMb; continue; @@ -479,6 +498,7 @@ private static final class NormalizerConfiguration { private final int minRegionCount; private final Period mergeMinRegionAge; private final long mergeMinRegionSizeMb; + private final long mergeRequestMaxNumberOfRegionsCount; private NormalizerConfiguration() { conf = null; @@ -487,6 +507,7 @@ private NormalizerConfiguration() { minRegionCount = DEFAULT_MIN_REGION_COUNT; mergeMinRegionAge = Period.ofDays(DEFAULT_MERGE_MIN_REGION_AGE_DAYS); mergeMinRegionSizeMb = DEFAULT_MERGE_MIN_REGION_SIZE_MB; + mergeRequestMaxNumberOfRegionsCount = DEFAULT_MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT; } private NormalizerConfiguration(final Configuration conf, @@ -497,6 +518,7 @@ private NormalizerConfiguration(final Configuration conf, minRegionCount = parseMinRegionCount(conf); mergeMinRegionAge = parseMergeMinRegionAge(conf); mergeMinRegionSizeMb = parseMergeMinRegionSizeMb(conf); + mergeRequestMaxNumberOfRegionsCount = parseMergeRequestMaxNumberOfRegionsCount(conf); logConfigurationUpdated(SPLIT_ENABLED_KEY, currentConfiguration.isSplitEnabled(), splitEnabled); logConfigurationUpdated(MERGE_ENABLED_KEY, currentConfiguration.isMergeEnabled(), @@ -507,6 +529,9 @@ private NormalizerConfiguration(final Configuration conf, currentConfiguration.getMergeMinRegionAge(), mergeMinRegionAge); logConfigurationUpdated(MERGE_MIN_REGION_SIZE_MB_KEY, currentConfiguration.getMergeMinRegionSizeMb(), mergeMinRegionSizeMb); + logConfigurationUpdated(MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT_KEY, + currentConfiguration.getMergeRequestMaxNumberOfRegionsCount(), + mergeRequestMaxNumberOfRegionsCount); } public Configuration getConf() { @@ -558,6 +583,10 @@ public long getMergeMinRegionSizeMb(NormalizeContext context) { } return mergeMinRegionSizeMb; } + + public long getMergeRequestMaxNumberOfRegionsCount() { + return mergeRequestMaxNumberOfRegionsCount; + } } /** diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java index da3e8db430f6..3dd4d5404a79 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/normalizer/TestSimpleRegionNormalizer.java @@ -22,6 +22,7 @@ import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.MERGE_ENABLED_KEY; import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.MERGE_MIN_REGION_AGE_DAYS_KEY; import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.MERGE_MIN_REGION_SIZE_MB_KEY; +import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT_KEY; import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.MIN_REGION_COUNT_KEY; import static org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer.SPLIT_ENABLED_KEY; import static org.hamcrest.MatcherAssert.assertThat; @@ -468,6 +469,39 @@ public void testHonorsMergeMinRegionSizeInTD() { assertThat(normalizer.computePlansForTable(tableDescriptor), empty()); } + @Test + public void testHonorsMergeRequestMaxNumberOfRegionsCount() { + conf.setBoolean(SPLIT_ENABLED_KEY, false); + conf.setInt(MERGE_MIN_REGION_SIZE_MB_KEY, 0); + conf.setInt(MERGE_REQUEST_MAX_NUMBER_OF_REGIONS_COUNT_KEY, 3); + final TableName tableName = name.getTableName(); + final List regionInfos = createRegionInfos(tableName, 5); + final Map regionSizes = createRegionSizesMap(regionInfos, 0, 1, 0, 1, 0); + setupMocksForNormalizer(regionSizes, regionInfos); + assertEquals(3, normalizer.getMergeRequestMaxNumberOfRegionsCount()); + List plans = normalizer.computePlansForTable(tableDescriptor); + assertThat(plans, + contains( + new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(0), 0) + .addTarget(regionInfos.get(1), 1).addTarget(regionInfos.get(2), 0).build(), + new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(3), 1) + .addTarget(regionInfos.get(4), 0).build())); + } + + @Test + public void testHonorsMergeRequestMaxNumberOfRegionsCountDefault() { + conf.setBoolean(SPLIT_ENABLED_KEY, false); + conf.setInt(MERGE_MIN_REGION_SIZE_MB_KEY, 0); + final TableName tableName = name.getTableName(); + final List regionInfos = createRegionInfos(tableName, 3); + final Map regionSizes = createRegionSizesMap(regionInfos, 0, 0, 0); + setupMocksForNormalizer(regionSizes, regionInfos); + assertEquals(50, normalizer.getMergeRequestMaxNumberOfRegionsCount()); + List plans = normalizer.computePlansForTable(tableDescriptor); + assertThat(plans, contains(new MergeNormalizationPlan.Builder().addTarget(regionInfos.get(0), 0) + .addTarget(regionInfos.get(1), 0).addTarget(regionInfos.get(2), 0).build())); + } + @Test public void testMergeEmptyRegions0() { conf.setBoolean(SPLIT_ENABLED_KEY, false);