From e1b3de6ad03e9ed12d3ad98d92dcb4424297d1ac Mon Sep 17 00:00:00 2001 From: Aman Poonia Date: Thu, 15 Jul 2021 15:07:10 +0530 Subject: [PATCH] HBASE-25986 set default value of normalization enabled from hbase site --- .../apache/hadoop/hbase/HTableDescriptor.java | 3 +- .../hbase/zookeeper/ZooKeeperWatcher.java | 8 ++--- .../src/main/resources/hbase-default.xml | 7 +++++ .../apache/hadoop/hbase/master/HMaster.java | 30 ++++++++++++++----- 4 files changed, 35 insertions(+), 13 deletions(-) diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java index 3bdd7d3eded6..ae54df2d354f 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java @@ -280,7 +280,6 @@ public class HTableDescriptor implements WritableComparable { String.valueOf(DEFAULT_DEFERRED_LOG_FLUSH)); DEFAULT_VALUES.put(DURABILITY, DEFAULT_DURABLITY.name()); //use the enum name DEFAULT_VALUES.put(REGION_REPLICATION, String.valueOf(DEFAULT_REGION_REPLICATION)); - DEFAULT_VALUES.put(NORMALIZATION_ENABLED, String.valueOf(DEFAULT_NORMALIZATION_ENABLED)); DEFAULT_VALUES.put(PRIORITY, String.valueOf(DEFAULT_PRIORITY)); for (String s : DEFAULT_VALUES.keySet()) { RESERVED_KEYWORDS.add(new ImmutableBytesWritable(Bytes.toBytes(s))); @@ -684,7 +683,7 @@ public HTableDescriptor setCompactionEnabled(final boolean isEnable) { * @return true if region normalization is enabled for this table */ public boolean isNormalizationEnabled() { - return isSomething(NORMALIZATION_ENABLED_KEY, DEFAULT_NORMALIZATION_ENABLED); + return isSomething(NORMALIZATION_ENABLED_KEY, false); } /** diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java index b180fb9411d2..e3a662b9af57 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZooKeeperWatcher.java @@ -185,13 +185,13 @@ public class ZooKeeperWatcher implements Watcher, Abortable, Closeable { /** * Instantiate a ZooKeeper connection and watcher. - * @param identifier string that is passed to RecoverableZookeeper to be used as - * identifier for this instance. Use null for default. + * @param identifier string that is passed to RecoverableZookeeper to be used as identifier for + * this instance. Use null for default. * @throws IOException * @throws ZooKeeperConnectionException */ - public ZooKeeperWatcher(Configuration conf, String identifier, - Abortable abortable) throws ZooKeeperConnectionException, IOException { + public ZooKeeperWatcher(Configuration conf, String identifier, Abortable abortable) + throws ZooKeeperConnectionException, IOException { this(conf, identifier, abortable, false); } diff --git a/hbase-common/src/main/resources/hbase-default.xml b/hbase-common/src/main/resources/hbase-default.xml index 2761b4f6d09d..93474a20da73 100644 --- a/hbase-common/src/main/resources/hbase-default.xml +++ b/hbase-common/src/main/resources/hbase-default.xml @@ -642,6 +642,13 @@ possible configurations would overwhelm and obscure the important. The minimum size for a region to be considered for a merge, in whole MBs. + + hbase.table.normalization.enabled + false + This config is used to set default behaviour of normalizer at table level. To override this at table + level one can set NORMALIZATION_ENABLED at table descriptor level and that property will be honored + + hbase.server.thread.wakefrequency 10000 diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index 2d052e32454b..92fb6d1e43d1 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -205,7 +205,8 @@ @SuppressWarnings("deprecation") public class HMaster extends HRegionServer implements MasterServices, Server { private static final Log LOG = LogFactory.getLog(HMaster.class.getName()); - + public static final String HBASE_TABLE_NORMALIZATION_ENABLED = + "hbase.table.normalization.enabled"; /** * Protection against zombie master. Started once Master accepts active responsibility and * starts taking over responsibilities. Allows a finite time window before giving up ownership. @@ -381,6 +382,7 @@ public void run() { private long splitPlanCount; private long mergePlanCount; + private boolean defaultNormalizerTableLevel; /** flag used in test cases in order to simulate RS failures during master initialization */ private volatile boolean initializationBeforeMetaAssignment = false; @@ -529,6 +531,7 @@ public HMaster(final Configuration conf, CoordinatedStateManager csm) activeMasterManager = null; } cachedClusterId = new CachedClusterId(conf); + this.defaultNormalizerTableLevel = extractDefaultNormalizerValue(conf); } // return the actual infoPort, -1 means disable info server. @@ -1019,7 +1022,7 @@ private void unassignExcessMetaReplica(ZooKeeperWatcher zkw, int numMetaReplicas LOG.info("Closing excess replica of meta region " + r.getRegion()); // send a close and wait for a max of 30 seconds ServerManager.closeRegionSilentlyAndWait(getConnection(), r.getServerName(), - r.getRegion(), 30000); + r.getRegion(), 30000); ZKUtil.deleteNode(zkw, zkw.getZNodeForReplica(replicaId)); } } @@ -1117,6 +1120,11 @@ void assignMeta(MonitoredTask status, Set previouslyFailedMetaRSs, i status.setStatus("META assigned."); } + private boolean extractDefaultNormalizerValue(final Configuration configuration) { + String s = configuration.get(HBASE_TABLE_NORMALIZATION_ENABLED); + return Boolean.parseBoolean(s); + } + private void assignMetaZkLess(RegionStates regionStates, RegionState regionState, long timeout, Set previouslyFailedRs) throws IOException, KeeperException { ServerName currentServer = regionState.getServerName(); @@ -1692,12 +1700,20 @@ public boolean normalizeRegions() throws IOException, CoordinatedStateException if (table.isSystemTable()) { continue; } - + boolean normalizationEnabled; HTableDescriptor tableDescriptor = getTableDescriptors().get(table); - if (tableDescriptor != null && !tableDescriptor.isNormalizationEnabled()) { - LOG.debug("Skipping normalization for table: " + table - + ", as it doesn't have auto normalization turned on"); - continue; + if (tableDescriptor != null) { + String defined = tableDescriptor.getValue(HTableDescriptor.NORMALIZATION_ENABLED); + if (defined != null) { + normalizationEnabled = tableDescriptor.isNormalizationEnabled(); + } else { + normalizationEnabled = this.defaultNormalizerTableLevel; + } + if (!normalizationEnabled) { + LOG.debug("Skipping table " + table + " because normalization is disabled in its " + + "table properties and normalization is also disabled at table level by default"); + continue; + } } // make one last check that the cluster isn't shutting down before proceeding. if (skipRegionManagementAction("region normalizer")) {