diff --git a/dev-support/Jenkinsfile_GitHub b/dev-support/Jenkinsfile_GitHub index 95f3d28f12a2..99dbcf3c3da0 100644 --- a/dev-support/Jenkinsfile_GitHub +++ b/dev-support/Jenkinsfile_GitHub @@ -61,13 +61,12 @@ pipeline { stage ('precommit-run') { steps { - withCredentials( - [usernamePassword(credentialsId: 'apache-hbase-at-github.com', - passwordVariable: 'GITHUB_PASSWORD', - usernameVariable: 'GITHUB_USER'), - usernamePassword(credentialsId: 'hbaseqa-at-asf-jira', - passwordVariable: 'JIRA_PASSWORD', - usernameVariable: 'JIRA_USER')]) { + withCredentials([ + usernamePassword( + credentialsId: 'apache-hbase-at-github.com', + passwordVariable: 'GITHUB_PASSWORD', + usernameVariable: 'GITHUB_USER' + )]) { sh '''#!/usr/bin/env bash set -e TESTPATCHBIN="${WORKSPACE}/${YETUS}/precommit/src/main/shell/test-patch.sh" diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java index 94d977b30371..28b2d1ccf664 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java @@ -1390,6 +1390,16 @@ public static enum Modify { "hbase.master.executor.logreplayops.threads"; public static final int MASTER_LOG_REPLAY_OPS_THREADS_DEFAULT = 10; + /** + * Number of rows in a batch operation above which a warning will be logged. + */ + public static final String BATCH_ROWS_THRESHOLD_NAME = "hbase.rpc.rows.warning.threshold"; + + /** + * Default value of {@link #BATCH_ROWS_THRESHOLD_NAME} + */ + public static final int BATCH_ROWS_THRESHOLD_DEFAULT = 5000; + private HConstants() { // Can't be instantiated with this ctor. } diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/HBaseCommonTestingUtility.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/HBaseCommonTestingUtility.java index 19a9ac290b5f..1871d11e1d87 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/HBaseCommonTestingUtility.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/HBaseCommonTestingUtility.java @@ -40,7 +40,7 @@ public class HBaseCommonTestingUtility { protected static final Log LOG = LogFactory.getLog(HBaseCommonTestingUtility.class); - protected Configuration conf; + protected final Configuration conf; public HBaseCommonTestingUtility() { this(null); diff --git a/hbase-hbtop/src/main/java/org/apache/hadoop/hbase/hbtop/field/FieldValue.java b/hbase-hbtop/src/main/java/org/apache/hadoop/hbase/hbtop/field/FieldValue.java index db7d22f3c8ca..bbfe508a7b3b 100644 --- a/hbase-hbtop/src/main/java/org/apache/hadoop/hbase/hbtop/field/FieldValue.java +++ b/hbase-hbtop/src/main/java/org/apache/hadoop/hbase/hbtop/field/FieldValue.java @@ -174,9 +174,12 @@ public String toString() { case INTEGER: case LONG: case FLOAT: - case SIZE: return value.toString(); + case SIZE: + Size size = (Size) value; + return String.format("%.1f", size.get()) + size.getUnit().getSimpleName(); + case PERCENT: return String.format("%.2f", (Float) value) + "%"; diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/Action.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/Action.java index 7a2016e056db..67c74632fc77 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/Action.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/Action.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -39,12 +39,11 @@ import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.util.Bytes; import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * A (possibly mischievous) action that the ChaosMonkey can perform. */ -public class Action { +public abstract class Action { public static final String KILL_MASTER_TIMEOUT_KEY = "hbase.chaosmonkey.action.killmastertimeout"; @@ -65,8 +64,6 @@ public class Action { public static final String START_NAMENODE_TIMEOUT_KEY = "hbase.chaosmonkey.action.startnamenodetimeout"; - private static final Logger LOG = LoggerFactory.getLogger(Action.class); - protected static final long KILL_MASTER_TIMEOUT_DEFAULT = PolicyBasedChaosMonkey.TIMEOUT; protected static final long START_MASTER_TIMEOUT_DEFAULT = PolicyBasedChaosMonkey.TIMEOUT; protected static final long KILL_RS_TIMEOUT_DEFAULT = PolicyBasedChaosMonkey.TIMEOUT; @@ -121,6 +118,11 @@ public void init(ActionContext context) throws IOException { cluster.getConf().getLong(START_NAMENODE_TIMEOUT_KEY, START_NAMENODE_TIMEOUT_DEFAULT); } + /** + * Retrieve the instance's {@link Logger}, for use throughout the class hierarchy. + */ + protected abstract Logger getLogger(); + public void perform() throws Exception { } /** Returns current region servers - active master */ @@ -138,110 +140,110 @@ protected ServerName[] getCurrentServers() throws IOException { ArrayList tmp = new ArrayList<>(count); tmp.addAll(regionServers); tmp.removeAll(masters); - return tmp.toArray(new ServerName[tmp.size()]); + return tmp.toArray(new ServerName[0]); } protected void killMaster(ServerName server) throws IOException { - LOG.info("Killing master:" + server); + getLogger().info("Killing master:" + server); cluster.killMaster(server); cluster.waitForMasterToStop(server, killMasterTimeout); - LOG.info("Killed master server:" + server); + getLogger().info("Killed master server:" + server); } protected void startMaster(ServerName server) throws IOException { - LOG.info("Starting master:" + server.getHostname()); + getLogger().info("Starting master:" + server.getHostname()); cluster.startMaster(server.getHostname(), server.getPort()); cluster.waitForActiveAndReadyMaster(startMasterTimeout); - LOG.info("Started master: " + server); + getLogger().info("Started master: " + server); } protected void stopRs(ServerName server) throws IOException { - LOG.info("Stopping regionserver " + server); + getLogger().info("Stopping regionserver " + server); cluster.stopRegionServer(server); cluster.waitForRegionServerToStop(server, killRsTimeout); - LOG.info(String.format("Stopping regionserver %s. Reported num of rs: %s", server, + getLogger().info(String.format("Stopping regionserver %s. Reported num of rs: %s", server, cluster.getClusterStatus().getLiveServersLoad().size())); } protected void suspendRs(ServerName server) throws IOException { - LOG.info("Suspending regionserver %s" + server); + getLogger().info("Suspending regionserver %s" + server); cluster.suspendRegionServer(server); if(!(cluster instanceof MiniHBaseCluster)){ cluster.waitForRegionServerToStop(server, killRsTimeout); } - LOG.info(String.format("Suspending regionserver %s. Reported num of rs: %s", server, + getLogger().info(String.format("Suspending regionserver %s. Reported num of rs: %s", server, cluster.getClusterStatus().getLiveServersLoad().size())); } protected void resumeRs(ServerName server) throws IOException { - LOG.info("Resuming regionserver " + server); + getLogger().info("Resuming regionserver " + server); cluster.resumeRegionServer(server); if(!(cluster instanceof MiniHBaseCluster)){ cluster.waitForRegionServerToStart(server.getHostname(), server.getPort(), startRsTimeout); } - LOG.info(String.format("Resuming regionserver %s. Reported num of rs: %s", server, + getLogger().info(String.format("Resuming regionserver %s. Reported num of rs: %s", server, cluster.getClusterStatus().getLiveServersLoad().size())); } protected void killRs(ServerName server) throws IOException { - LOG.info("Killing regionserver " + server); + getLogger().info("Killing regionserver " + server); cluster.killRegionServer(server); cluster.waitForRegionServerToStop(server, killRsTimeout); - LOG.info(String.format("Killed regionserver %s. Reported num of rs: %s", server, + getLogger().info(String.format("Killed regionserver %s. Reported num of rs: %s", server, cluster.getClusterStatus().getLiveServersLoad().size())); } protected void startRs(ServerName server) throws IOException { - LOG.info("Starting regionserver " + server.getAddress()); + getLogger().info("Starting regionserver " + server.getAddress()); cluster.startRegionServer(server.getHostname(), server.getPort()); cluster.waitForRegionServerToStart(server.getHostname(), server.getPort(), startRsTimeout); - LOG.info(String.format("Started regionserver %s. Reported num of rs: %s", server.getAddress(), - cluster.getClusterStatus().getLiveServersLoad().size())); + getLogger().info(String.format("Started regionserver %s. Reported num of rs: %s", + server.getAddress(), cluster.getClusterStatus().getLiveServersLoad().size())); } protected void killZKNode(ServerName server) throws IOException { - LOG.info("Killing zookeeper node " + server); + getLogger().info("Killing zookeeper node " + server); cluster.killZkNode(server); cluster.waitForZkNodeToStop(server, killZkNodeTimeout); - LOG.info(String.format("Killed zookeeper node %s. Reported num of rs: %s", server, + getLogger().info(String.format("Killed zookeeper node %s. Reported num of rs: %s", server, cluster.getClusterStatus().getLiveServersLoad().size())); } protected void startZKNode(ServerName server) throws IOException { - LOG.info("Starting zookeeper node " + server.getHostname()); + getLogger().info("Starting zookeeper node " + server.getHostname()); cluster.startZkNode(server.getHostname(), server.getPort()); cluster.waitForZkNodeToStart(server, startZkNodeTimeout); - LOG.info("Started zookeeper node " + server); + getLogger().info("Started zookeeper node " + server); } protected void killDataNode(ServerName server) throws IOException { - LOG.info("Killing datanode " + server); + getLogger().info("Killing datanode " + server); cluster.killDataNode(server); cluster.waitForDataNodeToStop(server, killDataNodeTimeout); - LOG.info(String.format("Killed datanode %s. Reported num of rs: %s", server, + getLogger().info(String.format("Killed datanode %s. Reported num of rs: %s", server, cluster.getClusterStatus().getLiveServersLoad().size())); } protected void startDataNode(ServerName server) throws IOException { - LOG.info("Starting datanode " + server.getHostname()); + getLogger().info("Starting datanode " + server.getHostname()); cluster.startDataNode(server); cluster.waitForDataNodeToStart(server, startDataNodeTimeout); - LOG.info("Started datanode " + server); + getLogger().info("Started datanode " + server); } protected void killNameNode(ServerName server) throws IOException { - LOG.info("Killing namenode : " + server.getHostname()); + getLogger().info("Killing namenode : " + server.getHostname()); cluster.killNameNode(server); cluster.waitForNameNodeToStop(server, killNameNodeTimeout); - LOG.info("Killed namenode: " + server + ". Reported num of rs:" + getLogger().info("Killed namenode: " + server + ". Reported num of rs:" + cluster.getClusterStatus().getServersSize()); } protected void startNameNode(ServerName server) throws IOException { - LOG.info("Starting Namenode : " + server.getHostname()); + getLogger().info("Starting Namenode : " + server.getHostname()); cluster.startNameNode(server); cluster.waitForNameNodeToStart(server, startNameNodeTimeout); - LOG.info("Started namenode: " + server); + getLogger().info("Started namenode: " + server); } protected void unbalanceRegions(ClusterStatus clusterStatus, @@ -253,7 +255,8 @@ protected void unbalanceRegions(ClusterStatus clusterStatus, // Ugh. List regions = new LinkedList(serverLoad.getRegionsLoad().keySet()); int victimRegionCount = (int)Math.ceil(fractionOfRegions * regions.size()); - LOG.debug("Removing " + victimRegionCount + " regions from " + server.getServerName()); + getLogger().debug("Removing " + victimRegionCount + " regions from " + + server.getServerName()); for (int i = 0; i < victimRegionCount; ++i) { int victimIx = RandomUtils.nextInt(regions.size()); String regionId = HRegionInfo.encodeRegionName(regions.remove(victimIx)); @@ -261,7 +264,7 @@ protected void unbalanceRegions(ClusterStatus clusterStatus, } } - LOG.info("Moving " + victimRegions.size() + " regions from " + fromServers.size() + getLogger().info("Moving " + victimRegions.size() + " regions from " + fromServers.size() + " servers to " + toServers.size() + " different servers"); Admin admin = this.context.getHBaseIntegrationTestingUtility().getHBaseAdmin(); for (byte[] victimRegion : victimRegions) { @@ -281,10 +284,10 @@ protected void forceBalancer() throws Exception { try { result = admin.balancer(); } catch (Exception e) { - LOG.warn("Got exception while doing balance ", e); + getLogger().warn("Got exception while doing balance ", e); } if (!result) { - LOG.error("Balancer didn't succeed"); + getLogger().error("Balancer didn't succeed"); } } @@ -293,7 +296,7 @@ protected void setBalancer(boolean onOrOff, boolean synchronous) throws Exceptio try { admin.setBalancerRunning(onOrOff, synchronous); } catch (Exception e) { - LOG.warn("Got exception while switching balance ", e); + getLogger().warn("Got exception while switching balance ", e); } } diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/AddColumnAction.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/AddColumnAction.java index 08eef68727e9..161e8dbbc054 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/AddColumnAction.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/AddColumnAction.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -40,6 +40,10 @@ public AddColumnAction(TableName tableName) { this.tableName = tableName; } + @Override protected Logger getLogger() { + return LOG; + } + @Override public void init(ActionContext context) throws IOException { super.init(context); @@ -61,7 +65,7 @@ public void perform() throws Exception { return; } - LOG.debug("Performing action: Adding " + columnDescriptor + " to " + tableName); + getLogger().debug("Performing action: Adding " + columnDescriptor + " to " + tableName); tableDescriptor.addFamily(columnDescriptor); admin.modifyTable(tableName, tableDescriptor); diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/BatchRestartRsAction.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/BatchRestartRsAction.java index 129721be72fc..ed48031265fb 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/BatchRestartRsAction.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/BatchRestartRsAction.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -21,7 +21,6 @@ import java.util.HashSet; import java.util.List; import java.util.Set; - import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.chaos.monkies.PolicyBasedChaosMonkey; import org.slf4j.Logger; @@ -32,17 +31,20 @@ */ public class BatchRestartRsAction extends RestartActionBaseAction { float ratio; //ratio of regionservers to restart - private static final Logger LOG = - LoggerFactory.getLogger(BatchRestartRsAction.class); + private static final Logger LOG = LoggerFactory.getLogger(BatchRestartRsAction.class); public BatchRestartRsAction(long sleepTime, float ratio) { super(sleepTime); this.ratio = ratio; } + @Override protected Logger getLogger() { + return LOG; + } + @Override public void perform() throws Exception { - LOG.info(String.format("Performing action: Batch restarting %d%% of region servers", + getLogger().info(String.format("Performing action: Batch restarting %d%% of region servers", (int)(ratio * 100))); List selectedServers = PolicyBasedChaosMonkey.selectRandomItems(getCurrentServers(), ratio); @@ -55,7 +57,7 @@ public void perform() throws Exception { if (context.isStopping()) { break; } - LOG.info("Killing region server:" + server); + getLogger().info("Killing region server:" + server); cluster.killRegionServer(server); killedServers.add(server); } @@ -64,13 +66,13 @@ public void perform() throws Exception { cluster.waitForRegionServerToStop(server, PolicyBasedChaosMonkey.TIMEOUT); } - LOG.info("Killed " + killedServers.size() + " region servers. Reported num of rs:" + getLogger().info("Killed " + killedServers.size() + " region servers. Reported num of rs:" + cluster.getClusterStatus().getServersSize()); sleep(sleepTime); for (ServerName server : killedServers) { - LOG.info("Starting region server:" + server.getHostname()); + getLogger().info("Starting region server:" + server.getHostname()); cluster.startRegionServer(server.getHostname(), server.getPort()); } @@ -79,7 +81,7 @@ public void perform() throws Exception { server.getPort(), PolicyBasedChaosMonkey.TIMEOUT); } - LOG.info("Started " + killedServers.size() +" region servers. Reported num of rs:" + getLogger().info("Started " + killedServers.size() +" region servers. Reported num of rs:" + cluster.getClusterStatus().getServersSize()); } } diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/ChangeBloomFilterAction.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/ChangeBloomFilterAction.java index 907c3f963450..57caa197d348 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/ChangeBloomFilterAction.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/ChangeBloomFilterAction.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -47,13 +47,17 @@ public ChangeBloomFilterAction(int sleepTime, TableName tableName) { this.tableName = tableName; } + @Override protected Logger getLogger() { + return LOG; + } + @Override public void perform() throws Exception { Random random = new Random(); HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility(); Admin admin = util.getHBaseAdmin(); - LOG.info("Performing action: Change bloom filter on all columns of table " + getLogger().info("Performing action: Change bloom filter on all columns of table " + tableName); HTableDescriptor tableDescriptor = admin.getTableDescriptor(tableName); HColumnDescriptor[] columnDescriptors = tableDescriptor.getColumnFamilies(); @@ -67,11 +71,11 @@ public void perform() throws Exception { for (HColumnDescriptor descriptor : columnDescriptors) { int bloomFilterIndex = random.nextInt(bloomArraySize); - LOG.debug("Performing action: About to set bloom filter type to " + getLogger().debug("Performing action: About to set bloom filter type to " + bloomArray[bloomFilterIndex] + " on column " + descriptor.getNameAsString() + " of table " + tableName); descriptor.setBloomFilterType(bloomArray[bloomFilterIndex]); - LOG.debug("Performing action: Just set bloom filter type to " + getLogger().debug("Performing action: Just set bloom filter type to " + bloomArray[bloomFilterIndex] + " on column " + descriptor.getNameAsString() + " of table " + tableName); } diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/ChangeCompressionAction.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/ChangeCompressionAction.java index 9a2a5f01e451..45ebcacdbd81 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/ChangeCompressionAction.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/ChangeCompressionAction.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -46,6 +46,10 @@ public ChangeCompressionAction(TableName tableName) { this.random = new Random(); } + @Override protected Logger getLogger() { + return LOG; + } + @Override public void init(ActionContext context) throws IOException { super.init(context); @@ -82,12 +86,12 @@ public void perform() throws Exception { algo.returnCompressor(c); break; } catch (Throwable t) { - LOG.info("Performing action: Changing compression algorithms to " + algo + + getLogger().info("Performing action: Changing compression algorithms to " + algo + " is not supported, pick another one"); } } while (true); - LOG.debug("Performing action: Changing compression algorithms on " + getLogger().debug("Performing action: Changing compression algorithms on " + tableName.getNameAsString() + " to " + algo); for (HColumnDescriptor descriptor : columnDescriptors) { if (random.nextBoolean()) { diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/ChangeEncodingAction.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/ChangeEncodingAction.java index e678afe1d463..0c5df160ab56 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/ChangeEncodingAction.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/ChangeEncodingAction.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -44,6 +44,10 @@ public ChangeEncodingAction(TableName tableName) { this.random = new Random(); } + @Override protected Logger getLogger() { + return LOG; + } + @Override public void init(ActionContext context) throws IOException { super.init(context); @@ -59,13 +63,13 @@ public void perform() throws Exception { return; } - LOG.debug("Performing action: Changing encodings on " + tableName); + getLogger().debug("Performing action: Changing encodings on " + tableName); // possible DataBlockEncoding id's int[] possibleIds = {0, 2, 3, 4, 6}; for (HColumnDescriptor descriptor : columnDescriptors) { short id = (short) possibleIds[random.nextInt(possibleIds.length)]; descriptor.setDataBlockEncoding(DataBlockEncoding.getEncodingById(id)); - LOG.debug("Set encoding of column family " + descriptor.getNameAsString() + getLogger().debug("Set encoding of column family " + descriptor.getNameAsString() + " to: " + descriptor.getDataBlockEncoding()); } diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/ChangeSplitPolicyAction.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/ChangeSplitPolicyAction.java index dc1cfee5c559..95f339b2012d 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/ChangeSplitPolicyAction.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/ChangeSplitPolicyAction.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -45,17 +45,20 @@ public ChangeSplitPolicyAction(TableName tableName) { this.random = new Random(); } + @Override protected Logger getLogger() { + return LOG; + } @Override public void perform() throws Exception { HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility(); Admin admin = util.getHBaseAdmin(); - LOG.info("Performing action: Change split policy of table " + tableName); + getLogger().info("Performing action: Change split policy of table " + tableName); HTableDescriptor tableDescriptor = admin.getTableDescriptor(tableName); String chosenPolicy = possiblePolicies[random.nextInt(possiblePolicies.length)]; tableDescriptor.setRegionSplitPolicyClassName(chosenPolicy); - LOG.info("Changing " + tableName + " split policy to " + chosenPolicy); + getLogger().info("Changing " + tableName + " split policy to " + chosenPolicy); admin.modifyTable(tableName, tableDescriptor); } } diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/ChangeVersionsAction.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/ChangeVersionsAction.java index 7e2332c89677..0c221c75d73b 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/ChangeVersionsAction.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/ChangeVersionsAction.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -34,17 +34,21 @@ * Always keeps at least 1 as the number of versions. */ public class ChangeVersionsAction extends Action { - private final TableName tableName; private static final Logger LOG = LoggerFactory.getLogger(ChangeVersionsAction.class); + private final TableName tableName; + private final Random random; private Admin admin; - private Random random; public ChangeVersionsAction(TableName tableName) { this.tableName = tableName; this.random = new Random(); } + @Override protected Logger getLogger() { + return LOG; + } + @Override public void init(ActionContext context) throws IOException { super.init(context); @@ -68,7 +72,7 @@ public void perform() throws Exception { if (context.isStopping()) { return; } - LOG.debug("Performing action: Changing versions on " + tableName.getNameAsString()); + getLogger().debug("Performing action: Changing versions on " + tableName.getNameAsString()); admin.modifyTable(tableName, tableDescriptor); } } diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/CompactRandomRegionOfTableAction.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/CompactRandomRegionOfTableAction.java index 6003240c48f6..47c5da7203ac 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/CompactRandomRegionOfTableAction.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/CompactRandomRegionOfTableAction.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -33,11 +33,11 @@ * Region that queues a compaction of a random region from the table. */ public class CompactRandomRegionOfTableAction extends Action { + private static final Logger LOG = LoggerFactory.getLogger(CompactRandomRegionOfTableAction.class); + private final int majorRatio; private final long sleepTime; private final TableName tableName; - private static final Logger LOG = - LoggerFactory.getLogger(CompactRandomRegionOfTableAction.class); public CompactRandomRegionOfTableAction( TableName tableName, float majorRatio) { @@ -51,33 +51,37 @@ public CompactRandomRegionOfTableAction( this.tableName = tableName; } + @Override protected Logger getLogger() { + return LOG; + } + @Override public void perform() throws Exception { HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility(); Admin admin = util.getHBaseAdmin(); boolean major = RandomUtils.nextInt(100) < majorRatio; - LOG.info("Performing action: Compact random region of table " + getLogger().info("Performing action: Compact random region of table " + tableName + ", major=" + major); List regions = admin.getTableRegions(tableName); if (regions == null || regions.isEmpty()) { - LOG.info("Table " + tableName + " doesn't have regions to compact"); + getLogger().info("Table " + tableName + " doesn't have regions to compact"); return; } HRegionInfo region = PolicyBasedChaosMonkey.selectRandomItem( - regions.toArray(new HRegionInfo[regions.size()])); + regions.toArray(new HRegionInfo[0])); try { if (major) { - LOG.debug("Major compacting region " + region.getRegionNameAsString()); + getLogger().debug("Major compacting region " + region.getRegionNameAsString()); admin.majorCompactRegion(region.getRegionName()); } else { - LOG.debug("Compacting region " + region.getRegionNameAsString()); + getLogger().debug("Compacting region " + region.getRegionNameAsString()); admin.compactRegion(region.getRegionName()); } } catch (Exception ex) { - LOG.warn("Compaction failed, might be caused by other chaos: " + ex.getMessage()); + getLogger().warn("Compaction failed, might be caused by other chaos: " + ex.getMessage()); } if (sleepTime > 0) { Thread.sleep(sleepTime); diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/CompactTableAction.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/CompactTableAction.java index 2f5436a4217c..b0623fd126ee 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/CompactTableAction.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/CompactTableAction.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -29,10 +29,11 @@ * Action that queues a table compaction. */ public class CompactTableAction extends Action { + private static final Logger LOG = LoggerFactory.getLogger(CompactTableAction.class); + private final TableName tableName; private final int majorRatio; private final long sleepTime; - private static final Logger LOG = LoggerFactory.getLogger(CompactTableAction.class); public CompactTableAction(TableName tableName, float majorRatio) { this(-1, tableName, majorRatio); @@ -45,13 +46,17 @@ public CompactTableAction( this.sleepTime = sleepTime; } + @Override protected Logger getLogger() { + return LOG; + } + @Override public void perform() throws Exception { HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility(); Admin admin = util.getHBaseAdmin(); boolean major = RandomUtils.nextInt(100) < majorRatio; - LOG.info("Performing action: Compact table " + tableName + ", major=" + major); + getLogger().info("Performing action: Compact table " + tableName + ", major=" + major); try { if (major) { admin.majorCompact(tableName); @@ -59,7 +64,7 @@ public void perform() throws Exception { admin.compact(tableName); } } catch (Exception ex) { - LOG.warn("Compaction failed, might be caused by other chaos: " + ex.getMessage()); + getLogger().warn("Compaction failed, might be caused by other chaos: " + ex.getMessage()); } if (sleepTime > 0) { Thread.sleep(sleepTime); diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/DecreaseMaxHFileSizeAction.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/DecreaseMaxHFileSizeAction.java index eaada1693bd8..8a30b9cfc446 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/DecreaseMaxHFileSizeAction.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/DecreaseMaxHFileSizeAction.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -25,10 +25,13 @@ import org.apache.hadoop.hbase.client.Admin; import java.util.Random; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class DecreaseMaxHFileSizeAction extends Action { + private static final Logger LOG = LoggerFactory.getLogger(DecreaseMaxHFileSizeAction.class); - private static final long minFileSize = 1 * 1024 * 1024 * 1024L; + private static final long minFileSize = 1024 * 1024 * 1024L; private final long sleepTime; private final TableName tableName; @@ -40,6 +43,10 @@ public DecreaseMaxHFileSizeAction(long sleepTime, TableName tableName) { this.random = new Random(); } + @Override protected Logger getLogger() { + return LOG; + } + @Override public void perform() throws Exception { HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility(); diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/DumpClusterStatusAction.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/DumpClusterStatusAction.java index 11246ea4a279..f6a1c4251b56 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/DumpClusterStatusAction.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/DumpClusterStatusAction.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -19,7 +19,14 @@ package org.apache.hadoop.hbase.chaos.actions; import java.io.IOException; - +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import org.apache.hadoop.hbase.ClusterStatus; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.net.Address; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -27,17 +34,73 @@ * Action to dump the cluster status. */ public class DumpClusterStatusAction extends Action { - private static final Logger LOG = - LoggerFactory.getLogger(DumpClusterStatusAction.class); + private static final Logger LOG = LoggerFactory.getLogger(DumpClusterStatusAction.class); + + private Set
initialRegionServers; + + @Override + protected Logger getLogger() { + return LOG; + } @Override public void init(ActionContext context) throws IOException { super.init(context); + initialRegionServers = collectKnownRegionServers(initialStatus); } @Override public void perform() throws Exception { - LOG.debug("Performing action: Dump cluster status"); - LOG.info("Cluster status\n" + cluster.getClusterStatus()); + getLogger().debug("Performing action: Dump cluster status"); + final ClusterStatus currentMetrics = cluster.getClusterStatus(); + getLogger().info("Cluster status\n{}", currentMetrics); + reportMissingRegionServers(currentMetrics); + reportNewRegionServers(currentMetrics); + } + + /** + * Build a set of all the host:port pairs of region servers known to this cluster. + */ + private static Set
collectKnownRegionServers(final ClusterStatus clusterStatus) { + final Set
regionServers = new HashSet<>(); + final Set serverNames = clusterStatus.getLiveServersLoad().keySet(); + serverNames.addAll(clusterStatus.getDeadServerNames()); + + for (final ServerName serverName : serverNames) { + regionServers.add(serverName.getAddress()); + } + return Collections.unmodifiableSet(regionServers); + } + + private void reportMissingRegionServers(final ClusterStatus clusterStatus) { + final Set
regionServers = collectKnownRegionServers(clusterStatus); + final Set
missingRegionServers = new HashSet<>(initialRegionServers); + missingRegionServers.removeAll(regionServers); + if (!missingRegionServers.isEmpty()) { + final StringBuilder stringBuilder = new StringBuilder() + .append("region server(s) are missing from this cluster report"); + final List
sortedAddresses = new ArrayList<>(missingRegionServers); + Collections.sort(sortedAddresses); + for (final Address address : sortedAddresses) { + stringBuilder.append("\n ").append(address); + } + getLogger().warn(stringBuilder.toString()); + } + } + + private void reportNewRegionServers(final ClusterStatus clusterStatus) { + final Set
regionServers = collectKnownRegionServers(clusterStatus); + final Set
newRegionServers = new HashSet<>(regionServers); + newRegionServers.removeAll(initialRegionServers); + if (!newRegionServers.isEmpty()) { + final StringBuilder stringBuilder = new StringBuilder() + .append("region server(s) are new for this cluster report"); + final List
sortedAddresses = new ArrayList<>(newRegionServers); + Collections.sort(sortedAddresses); + for (final Address address : sortedAddresses) { + stringBuilder.append("\n ").append(address); + } + getLogger().warn(stringBuilder.toString()); + } } } diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/FlushRandomRegionOfTableAction.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/FlushRandomRegionOfTableAction.java index c4286dbf1f9e..f7cab7f5ce71 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/FlushRandomRegionOfTableAction.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/FlushRandomRegionOfTableAction.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -46,25 +46,29 @@ public FlushRandomRegionOfTableAction(int sleepTime, TableName tableName) { this.tableName = tableName; } + @Override protected Logger getLogger() { + return LOG; + } + @Override public void perform() throws Exception { HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility(); Admin admin = util.getHBaseAdmin(); - LOG.info("Performing action: Flush random region of table " + tableName); + getLogger().info("Performing action: Flush random region of table " + tableName); List regions = admin.getTableRegions(tableName); if (regions == null || regions.isEmpty()) { - LOG.info("Table " + tableName + " doesn't have regions to flush"); + getLogger().info("Table " + tableName + " doesn't have regions to flush"); return; } HRegionInfo region = PolicyBasedChaosMonkey.selectRandomItem( - regions.toArray(new HRegionInfo[regions.size()])); - LOG.debug("Flushing region " + region.getRegionNameAsString()); + regions.toArray(new HRegionInfo[0])); + getLogger().debug("Flushing region " + region.getRegionNameAsString()); try { admin.flushRegion(region.getRegionName()); } catch (Exception ex) { - LOG.warn("Flush failed, might be caused by other chaos: " + ex.getMessage()); + getLogger().warn("Flush failed, might be caused by other chaos: " + ex.getMessage()); } if (sleepTime > 0) { Thread.sleep(sleepTime); diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/FlushTableAction.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/FlushTableAction.java index 994fd057a5f1..5b8a56b8d558 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/FlushTableAction.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/FlushTableAction.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -28,8 +28,7 @@ * Action that tries to flush a table. */ public class FlushTableAction extends Action { - private static final Logger LOG = - LoggerFactory.getLogger(FlushTableAction.class); + private static final Logger LOG = LoggerFactory.getLogger(FlushTableAction.class); private final long sleepTime; private final TableName tableName; @@ -42,6 +41,10 @@ public FlushTableAction(int sleepTime, TableName tableName) { this.tableName = tableName; } + @Override protected Logger getLogger() { + return LOG; + } + @Override public void perform() throws Exception { HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility(); @@ -52,11 +55,11 @@ public void perform() throws Exception { return; } - LOG.info("Performing action: Flush table " + tableName); + getLogger().info("Performing action: Flush table " + tableName); try { admin.flush(tableName); } catch (Exception ex) { - LOG.warn("Flush failed, might be caused by other chaos: " + ex.getMessage()); + getLogger().warn("Flush failed, might be caused by other chaos: " + ex.getMessage()); } if (sleepTime > 0) { Thread.sleep(sleepTime); diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/ForceBalancerAction.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/ForceBalancerAction.java index d75475432a12..00767b026adb 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/ForceBalancerAction.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/ForceBalancerAction.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -25,8 +25,11 @@ * Action that tries to force a balancer run. */ public class ForceBalancerAction extends Action { - private static final Logger LOG = - LoggerFactory.getLogger(ForceBalancerAction.class); + private static final Logger LOG = LoggerFactory.getLogger(ForceBalancerAction.class); + + @Override protected Logger getLogger() { + return LOG; + } @Override public void perform() throws Exception { @@ -34,7 +37,7 @@ public void perform() throws Exception { if (context.isStopping()) { return; } - LOG.info("Balancing regions"); + getLogger().info("Balancing regions"); forceBalancer(); } } diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/MergeRandomAdjacentRegionsOfTableAction.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/MergeRandomAdjacentRegionsOfTableAction.java index eac7d30100a9..84b3475d8b35 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/MergeRandomAdjacentRegionsOfTableAction.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/MergeRandomAdjacentRegionsOfTableAction.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -46,22 +46,26 @@ public MergeRandomAdjacentRegionsOfTableAction(int sleepTime, TableName tableNam this.sleepTime = sleepTime; } + @Override protected Logger getLogger() { + return LOG; + } + @Override public void perform() throws Exception { HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility(); Admin admin = util.getHBaseAdmin(); - LOG.info("Performing action: Merge random adjacent regions of table " + tableName); + getLogger().info("Performing action: Merge random adjacent regions of table " + tableName); List regions = admin.getTableRegions(tableName); if (regions == null || regions.size() < 2) { - LOG.info("Table " + tableName + " doesn't have enough regions to merge"); + getLogger().info("Table " + tableName + " doesn't have enough regions to merge"); return; } int i = RandomUtils.nextInt(regions.size() - 1); HRegionInfo a = regions.get(i++); HRegionInfo b = regions.get(i); - LOG.debug("Merging " + a.getRegionNameAsString() + " and " + b.getRegionNameAsString()); + getLogger().debug("Merging " + a.getRegionNameAsString() + " and " + b.getRegionNameAsString()); // Don't try the merge if we're stopping if (context.isStopping()) { @@ -71,7 +75,7 @@ public void perform() throws Exception { try { admin.mergeRegions(a.getEncodedNameAsBytes(), b.getEncodedNameAsBytes(), false); } catch (Exception ex) { - LOG.warn("Merge failed, might be caused by other chaos: " + ex.getMessage()); + getLogger().warn("Merge failed, might be caused by other chaos: " + ex.getMessage()); } if (sleepTime > 0) { Thread.sleep(sleepTime); diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/MoveRandomRegionOfTableAction.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/MoveRandomRegionOfTableAction.java index 52816f323d41..5e07f45abc61 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/MoveRandomRegionOfTableAction.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/MoveRandomRegionOfTableAction.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -46,6 +46,10 @@ public MoveRandomRegionOfTableAction(long sleepTime, TableName tableName) { this.tableName = tableName; } + @Override protected Logger getLogger() { + return LOG; + } + @Override public void perform() throws Exception { if (sleepTime > 0) { @@ -55,16 +59,16 @@ public void perform() throws Exception { HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility(); Admin admin = util.getHBaseAdmin(); - LOG.info("Performing action: Move random region of table " + tableName); + getLogger().info("Performing action: Move random region of table " + tableName); List regions = admin.getTableRegions(tableName); if (regions == null || regions.isEmpty()) { - LOG.info("Table " + tableName + " doesn't have regions to move"); + getLogger().info("Table " + tableName + " doesn't have regions to move"); return; } HRegionInfo region = PolicyBasedChaosMonkey.selectRandomItem( regions.toArray(new HRegionInfo[regions.size()])); - LOG.debug("Unassigning region " + region.getRegionNameAsString()); + getLogger().debug("Unassigning region " + region.getRegionNameAsString()); admin.unassign(region.getRegionName(), false); if (sleepTime > 0) { Thread.sleep(sleepTime); diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/MoveRegionsOfTableAction.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/MoveRegionsOfTableAction.java index e38309b4d36c..36c8f6a41a1f 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/MoveRegionsOfTableAction.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/MoveRegionsOfTableAction.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -46,6 +46,10 @@ public MoveRegionsOfTableAction(TableName tableName) { this(-1, MonkeyConstants.DEFAULT_MOVE_REGIONS_MAX_TIME, tableName); } + @Override protected Logger getLogger() { + return LOG; + } + public MoveRegionsOfTableAction(long sleepTime, long maxSleepTime, TableName tableName) { this.sleepTime = sleepTime; this.tableName = tableName; @@ -62,10 +66,10 @@ public void perform() throws Exception { Collection serversList = admin.getClusterStatus().getServers(); ServerName[] servers = serversList.toArray(new ServerName[serversList.size()]); - LOG.info("Performing action: Move regions of table " + tableName); + getLogger().info("Performing action: Move regions of table " + tableName); List regions = admin.getTableRegions(tableName); if (regions == null || regions.isEmpty()) { - LOG.info("Table " + tableName + " doesn't have regions to move"); + getLogger().info("Table " + tableName + " doesn't have regions to move"); return; } @@ -82,10 +86,10 @@ public void perform() throws Exception { try { String destServerName = servers[RandomUtils.nextInt(servers.length)].getServerName(); - LOG.debug("Moving " + regionInfo.getRegionNameAsString() + " to " + destServerName); + getLogger().debug("Moving " + regionInfo.getRegionNameAsString() + " to " + destServerName); admin.move(regionInfo.getEncodedNameAsBytes(), Bytes.toBytes(destServerName)); } catch (Exception ex) { - LOG.warn("Move failed, might be caused by other chaos: " + ex.getMessage()); + getLogger().warn("Move failed, might be caused by other chaos: " + ex.getMessage()); } if (sleepTime > 0) { Thread.sleep(sleepTime); diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RemoveColumnAction.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RemoveColumnAction.java index e5ca3e857d2a..08156306e2c6 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RemoveColumnAction.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RemoveColumnAction.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -47,6 +47,10 @@ public RemoveColumnAction(TableName tableName, Set protectedColumns) { random = new Random(); } + @Override protected Logger getLogger() { + return LOG; + } + @Override public void init(ActionContext context) throws IOException { super.init(context); @@ -68,7 +72,7 @@ public void perform() throws Exception { index = random.nextInt(columnDescriptors.length); } byte[] colDescName = columnDescriptors[index].getName(); - LOG.debug("Performing action: Removing " + Bytes.toString(colDescName)+ " from " + getLogger().debug("Performing action: Removing " + Bytes.toString(colDescName)+ " from " + tableName.getNameAsString()); tableDescriptor.removeFamily(colDescName); diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RestartActionBaseAction.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RestartActionBaseAction.java index 54836d24b227..78b11f03ef7a 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RestartActionBaseAction.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RestartActionBaseAction.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -37,8 +37,12 @@ public RestartActionBaseAction(long sleepTime) { this.sleepTime = sleepTime; } + @Override protected Logger getLogger() { + return LOG; + } + void sleep(long sleepTime) { - LOG.info("Sleeping for:" + sleepTime); + getLogger().info("Sleeping for:" + sleepTime); Threads.sleep(sleepTime); } @@ -49,10 +53,10 @@ void restartMaster(ServerName server, long sleepTime) throws IOException { return; } - LOG.info("Killing master: " + server); + getLogger().info("Killing master: " + server); killMaster(server); sleep(sleepTime); - LOG.info("Starting master: " + server); + getLogger().info("Starting master: " + server); startMaster(server); } @@ -68,10 +72,10 @@ void gracefulRestartRs(ServerName server, long sleepTime) throws IOException { if (context.isStopping()) { return; } - LOG.info("Stopping region server: " + server); + getLogger().info("Stopping region server: " + server); stopRs(server); sleep(sleepTime); - LOG.info("Starting region server: " + server); + getLogger().info("Starting region server: " + server); startRs(server); } @@ -81,10 +85,10 @@ void restartRs(ServerName server, long sleepTime) throws IOException { if (context.isStopping()) { return; } - LOG.info("Killing region server: " + server); + getLogger().info("Killing region server: " + server); killRs(server); sleep(sleepTime); - LOG.info("Starting region server: " + server); + getLogger().info("Starting region server: " + server); startRs(server); } @@ -94,10 +98,10 @@ void restartZKNode(ServerName server, long sleepTime) throws IOException { if (context.isStopping()) { return; } - LOG.info("Killing zookeeper node: " + server); + getLogger().info("Killing zookeeper node: " + server); killZKNode(server); sleep(sleepTime); - LOG.info("Starting zookeeper node: " + server); + getLogger().info("Starting zookeeper node: " + server); startZKNode(server); } @@ -107,10 +111,10 @@ void restartDataNode(ServerName server, long sleepTime) throws IOException { if (context.isStopping()) { return; } - LOG.info("Killing data node: " + server); + getLogger().info("Killing data node: " + server); killDataNode(server); sleep(sleepTime); - LOG.info("Starting data node: " + server); + getLogger().info("Starting data node: " + server); startDataNode(server); } @@ -120,10 +124,10 @@ void restartNameNode(ServerName server, long sleepTime) throws IOException { if (context.isStopping()) { return; } - LOG.info("Killing name node: " + server); + getLogger().info("Killing name node: " + server); killNameNode(server); sleep(sleepTime); - LOG.info("Starting name node: " + server); + getLogger().info("Starting name node: " + server); startNameNode(server); } } diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RestartActiveMasterAction.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RestartActiveMasterAction.java index ab7decd3f1c8..6ae5b591a117 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RestartActiveMasterAction.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RestartActiveMasterAction.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -26,14 +26,19 @@ * Action that tries to restart the active master. */ public class RestartActiveMasterAction extends RestartActionBaseAction { - private static final Logger LOG = LoggerFactory.getLogger(RestartActionBaseAction.class); + private static final Logger LOG = LoggerFactory.getLogger(RestartActiveMasterAction.class); public RestartActiveMasterAction(long sleepTime) { super(sleepTime); } + + @Override protected Logger getLogger() { + return LOG; + } + @Override public void perform() throws Exception { - LOG.info("Performing action: Restart active master"); + getLogger().info("Performing action: Restart active master"); ServerName master = cluster.getClusterStatus().getMaster(); restartMaster(master, sleepTime); diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RestartActiveNameNodeAction.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RestartActiveNameNodeAction.java index d9cbfbddc8ae..d03565ee2ae6 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RestartActiveNameNodeAction.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RestartActiveNameNodeAction.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -51,9 +51,13 @@ public RestartActiveNameNodeAction(long sleepTime) { super(sleepTime); } + @Override protected Logger getLogger() { + return LOG; + } + @Override public void perform() throws Exception { - LOG.info("Performing action: Restart active namenode"); + getLogger().info("Performing action: Restart active namenode"); Configuration conf = FSUtils.getRootDir(getConf()).getFileSystem(getConf()).getConf(); String nameServiceID = DFSUtil.getNamenodeNameServiceId(conf); if (!HAUtil.isHAEnabled(conf, nameServiceID)) { @@ -85,9 +89,9 @@ public void perform() throws Exception { if (activeNamenode == null) { throw new Exception("No active Name node found in zookeeper under " + hadoopHAZkNode); } - LOG.info("Found active namenode host:" + activeNamenode); + getLogger().info("Found active namenode host:" + activeNamenode); ServerName activeNNHost = ServerName.valueOf(activeNamenode, -1, -1); - LOG.info("Restarting Active NameNode :" + activeNamenode); + getLogger().info("Restarting Active NameNode :" + activeNamenode); restartNameNode(activeNNHost, sleepTime); } } diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RestartRandomDataNodeAction.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RestartRandomDataNodeAction.java index 09e2990db613..421e458cf993 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RestartRandomDataNodeAction.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RestartRandomDataNodeAction.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -42,9 +42,13 @@ public RestartRandomDataNodeAction(long sleepTime) { super(sleepTime); } + @Override protected Logger getLogger() { + return LOG; + } + @Override public void perform() throws Exception { - LOG.info("Performing action: Restart random data node"); + getLogger().info("Performing action: Restart random data node"); ServerName server = PolicyBasedChaosMonkey.selectRandomItem(getDataNodes()); restartDataNode(server, sleepTime); } @@ -57,6 +61,6 @@ public ServerName[] getDataNodes() throws IOException { for (DatanodeInfo dataNode: dfsClient.datanodeReport(HdfsConstants.DatanodeReportType.LIVE)) { hosts.add(ServerName.valueOf(dataNode.getHostName(), -1, -1)); } - return hosts.toArray(new ServerName[hosts.size()]); + return hosts.toArray(new ServerName[0]); } } diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RestartRandomRsAction.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RestartRandomRsAction.java index 48458b68dcf4..607c80f2faf7 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RestartRandomRsAction.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RestartRandomRsAction.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -33,9 +33,13 @@ public RestartRandomRsAction(long sleepTime) { super(sleepTime); } + @Override protected Logger getLogger() { + return LOG; + } + @Override public void perform() throws Exception { - LOG.info("Performing action: Restart random region server"); + getLogger().info("Performing action: Restart random region server"); ServerName server = PolicyBasedChaosMonkey.selectRandomItem(getCurrentServers()); restartRs(server, sleepTime); diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RestartRandomRsExceptMetaAction.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RestartRandomRsExceptMetaAction.java index b78144a1ca2f..a12002681e76 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RestartRandomRsExceptMetaAction.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RestartRandomRsExceptMetaAction.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -20,12 +20,20 @@ import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.chaos.monkies.PolicyBasedChaosMonkey; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class RestartRandomRsExceptMetaAction extends RestartRandomRsAction { + private static final Logger LOG = LoggerFactory.getLogger(RestartRandomRsExceptMetaAction.class); + public RestartRandomRsExceptMetaAction(long sleepTime) { super(sleepTime); } + @Override protected Logger getLogger() { + return LOG; + } + @Override public void perform() throws Exception { int tries = 10; diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RestartRandomZKNodeAction.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RestartRandomZKNodeAction.java index 7984af7ba4a3..f6f7239a11e5 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RestartRandomZKNodeAction.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RestartRandomZKNodeAction.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -34,9 +34,13 @@ public RestartRandomZKNodeAction(long sleepTime) { super(sleepTime); } + @Override protected Logger getLogger() { + return LOG; + } + @Override public void perform() throws Exception { - LOG.info("Performing action: Restart random zookeeper node"); + getLogger().info("Performing action: Restart random zookeeper node"); ServerName server = PolicyBasedChaosMonkey.selectRandomItem( ZKServerTool.readZKNodes(getConf())); restartZKNode(server, sleepTime); diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RestartRsHoldingMetaAction.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RestartRsHoldingMetaAction.java index 09b3db69c3d5..ba47ce769dfd 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RestartRsHoldingMetaAction.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RestartRsHoldingMetaAction.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -33,12 +33,17 @@ public class RestartRsHoldingMetaAction extends RestartActionBaseAction { public RestartRsHoldingMetaAction(long sleepTime) { super(sleepTime); } + + @Override protected Logger getLogger() { + return LOG; + } + @Override public void perform() throws Exception { - LOG.info("Performing action: Restart region server holding META"); + getLogger().info("Performing action: Restart region server holding META"); ServerName server = cluster.getServerHoldingMeta(); if (server == null) { - LOG.warn("No server is holding hbase:meta right now."); + getLogger().warn("No server is holding hbase:meta right now."); return; } ClusterStatus clusterStatus = cluster.getClusterStatus(); diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RestartRsHoldingTableAction.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RestartRsHoldingTableAction.java index 79e91fe08202..abe83343ec86 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RestartRsHoldingTableAction.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RestartRsHoldingTableAction.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -43,15 +43,19 @@ public RestartRsHoldingTableAction(long sleepTime, String tableName) { this.tableName = tableName; } + @Override protected Logger getLogger() { + return LOG; + } + @Override public void perform() throws Exception { HTable table = null; try { - LOG.info("Performing action: Restart random RS holding table " + this.tableName); + getLogger().info("Performing action: Restart random RS holding table " + this.tableName); Configuration conf = context.getHBaseIntegrationTestingUtility().getConfiguration(); table = new HTable(conf, TableName.valueOf(tableName)); } catch (IOException e) { - LOG.debug("Error creating HTable used to get list of region locations.", e); + getLogger().debug("Error creating HTable used to get list of region locations.", e); return; } diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RollingBatchRestartRsAction.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RollingBatchRestartRsAction.java index 7db9f5ac9a52..d4be6faf94ee 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RollingBatchRestartRsAction.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RollingBatchRestartRsAction.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -27,10 +27,10 @@ import java.util.Queue; import org.apache.commons.lang.math.RandomUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.chaos.monkies.PolicyBasedChaosMonkey; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Restarts a ratio of the regionservers in a rolling fashion. At each step, either kills a @@ -39,7 +39,7 @@ * can be down at the same time during rolling restarts. */ public class RollingBatchRestartRsAction extends BatchRestartRsAction { - private static final Log LOG = LogFactory.getLog(RollingBatchRestartRsAction.class); + private static final Logger LOG = LoggerFactory.getLogger(RollingBatchRestartRsAction.class); protected int maxDeadServers; // number of maximum dead servers at any given time. Defaults to 5 public RollingBatchRestartRsAction(long sleepTime, float ratio) { @@ -56,9 +56,14 @@ enum KillOrStart { START } + @Override protected Logger getLogger() { + return LOG; + } + @Override public void perform() throws Exception { - LOG.info(String.format("Performing action: Rolling batch restarting %d%% of region servers", + getLogger().info( + String.format("Performing action: Rolling batch restarting %d%% of region servers", (int)(ratio * 100))); List selectedServers = selectServers(); @@ -91,7 +96,7 @@ public void perform() throws Exception { } catch (org.apache.hadoop.util.Shell.ExitCodeException e) { // We've seen this in test runs where we timeout but the kill went through. HBASE-9743 // So, add to deadServers even if exception so the start gets called. - LOG.info("Problem killing but presume successful; code=" + e.getExitCode(), e); + getLogger().info("Problem killing but presume successful; code=" + e.getExitCode(), e); } deadServers.add(server); break; @@ -105,7 +110,7 @@ public void perform() throws Exception { // The start may fail but better to just keep going though we may lose server. // Shuffle the dead list to avoid getting stuck on a single stubborn host. Collections.shuffle(deadServers); - LOG.info(String.format( + getLogger().info(String.format( "Problem starting %s, will retry; code=%s", server, e.getExitCode(), e)); } break; @@ -139,7 +144,7 @@ protected ServerName[] getCurrentServers() throws IOException { @Override protected void killRs(ServerName server) throws IOException { - LOG.info("Killed " + server); + getLogger().info("Killed " + server); if (this.invocations++ % 3 == 0) { throw new org.apache.hadoop.util.Shell.ExitCodeException(-1, "Failed"); } @@ -147,7 +152,7 @@ protected void killRs(ServerName server) throws IOException { @Override protected void startRs(ServerName server) throws IOException { - LOG.info("Started " + server); + getLogger().info("Started " + server); if (this.invocations++ % 3 == 0) { throw new org.apache.hadoop.util.Shell.ExitCodeException(-1, "Failed"); } @@ -156,4 +161,4 @@ protected void startRs(ServerName server) throws IOException { action.perform(); } -} \ No newline at end of file +} diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RollingBatchSuspendResumeRsAction.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RollingBatchSuspendResumeRsAction.java index d3c9bce83059..659a4060dc33 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RollingBatchSuspendResumeRsAction.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RollingBatchSuspendResumeRsAction.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -58,9 +58,14 @@ enum SuspendOrResume { SUSPEND, RESUME } + @Override protected Logger getLogger() { + return LOG; + } + @Override public void perform() throws Exception { - LOG.info(String.format("Performing action: Rolling batch restarting %d%% of region servers", + getLogger().info( + String.format("Performing action: Rolling batch restarting %d%% of region servers", (int) (ratio * 100))); List selectedServers = selectServers(); @@ -91,7 +96,8 @@ public void perform() throws Exception { try { suspendRs(server); } catch (Shell.ExitCodeException e) { - LOG.warn("Problem suspending but presume successful; code=" + e.getExitCode(), e); + getLogger().warn("Problem suspending but presume successful; code=" + + e.getExitCode(), e); } suspendedServers.add(server); break; @@ -100,7 +106,7 @@ public void perform() throws Exception { try { resumeRs(server); } catch (Shell.ExitCodeException e) { - LOG.info("Problem resuming, will retry; code= " + e.getExitCode(), e); + getLogger().info("Problem resuming, will retry; code= " + e.getExitCode(), e); } break; default: @@ -108,7 +114,7 @@ public void perform() throws Exception { "Encountered unexpected action type: " + action.name()); } - LOG.info("Sleeping for: " + sleepTime); + getLogger().info("Sleeping for: " + sleepTime); Threads.sleep(sleepTime); } } diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/SnapshotTableAction.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/SnapshotTableAction.java index 7e7dc8da24ce..7603443239c2 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/SnapshotTableAction.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/SnapshotTableAction.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -28,8 +28,7 @@ * Action that tries to take a snapshot of a table. */ public class SnapshotTableAction extends Action { - private static final Logger LOG = - LoggerFactory.getLogger(SnapshotTableAction.class); + private static final Logger LOG = LoggerFactory.getLogger(SnapshotTableAction.class); private final TableName tableName; private final long sleepTime; @@ -42,6 +41,10 @@ public SnapshotTableAction(int sleepTime, TableName tableName) { this.sleepTime = sleepTime; } + @Override protected Logger getLogger() { + return LOG; + } + @Override public void perform() throws Exception { HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility(); @@ -53,7 +56,7 @@ public void perform() throws Exception { return; } - LOG.info("Performing action: Snapshot table " + tableName); + getLogger().info("Performing action: Snapshot table {}", tableName); admin.snapshot(snapshotName, tableName); if (sleepTime > 0) { Thread.sleep(sleepTime); diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/SplitAllRegionOfTableAction.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/SplitAllRegionOfTableAction.java index 5a24af9eeb62..748d598fe704 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/SplitAllRegionOfTableAction.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/SplitAllRegionOfTableAction.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -19,14 +19,12 @@ import java.io.IOException; import java.util.concurrent.ThreadLocalRandom; - import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Admin; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - public class SplitAllRegionOfTableAction extends Action { private static final Logger LOG = LoggerFactory.getLogger(SplitAllRegionOfTableAction.class); @@ -47,6 +45,10 @@ public void init(ActionContext context) throws IOException { this.maxFullTableSplits = getConf().getInt(MAX_SPLIT_KEY, DEFAULT_MAX_SPLITS); } + @Override protected Logger getLogger() { + return LOG; + } + @Override public void perform() throws Exception { HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility(); @@ -61,10 +63,10 @@ public void perform() throws Exception { if (ThreadLocalRandom.current().nextDouble() < (((double) splits) / ((double) maxFullTableSplits)) / ((double) 2)) { splits++; - LOG.info("Performing action: Split all regions of " + tableName); + getLogger().info("Performing action: Split all regions of {}", tableName); admin.split(tableName); } else { - LOG.info("Skipping split of all regions."); + getLogger().info("Skipping split of all regions."); } } } diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/SplitRandomRegionOfTableAction.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/SplitRandomRegionOfTableAction.java index df424ec9e475..47c7035351e5 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/SplitRandomRegionOfTableAction.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/SplitRandomRegionOfTableAction.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -46,15 +46,19 @@ public SplitRandomRegionOfTableAction(int sleepTime, TableName tableName) { this.tableName = tableName; } + @Override protected Logger getLogger() { + return LOG; + } + @Override public void perform() throws Exception { HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility(); Admin admin = util.getHBaseAdmin(); - LOG.info("Performing action: Split random region of table " + tableName); + getLogger().info("Performing action: Split random region of table " + tableName); List regions = admin.getTableRegions(tableName); if (regions == null || regions.isEmpty()) { - LOG.info("Table " + tableName + " doesn't have regions to split"); + getLogger().info("Table " + tableName + " doesn't have regions to split"); return; } // Don't try the split if we're stopping @@ -64,11 +68,11 @@ public void perform() throws Exception { HRegionInfo region = PolicyBasedChaosMonkey.selectRandomItem( regions.toArray(new HRegionInfo[regions.size()])); - LOG.debug("Splitting region " + region.getRegionNameAsString()); + getLogger().debug("Splitting region " + region.getRegionNameAsString()); try { admin.splitRegion(region.getRegionName()); } catch (Exception ex) { - LOG.warn("Split failed, might be caused by other chaos: " + ex.getMessage()); + getLogger().warn("Split failed, might be caused by other chaos: " + ex.getMessage()); } if (sleepTime > 0) { Thread.sleep(sleepTime); diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/TruncateTableAction.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/TruncateTableAction.java index 12bbd094ce67..ad1b23c763a5 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/TruncateTableAction.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/TruncateTableAction.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -19,7 +19,6 @@ package org.apache.hadoop.hbase.chaos.actions; import java.util.Random; - import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.HBaseAdmin; @@ -31,8 +30,7 @@ * Action that tries to truncate of a table. */ public class TruncateTableAction extends Action { - private static final Logger LOG = - LoggerFactory.getLogger(TruncateTableAction.class); + private static final Logger LOG = LoggerFactory.getLogger(TruncateTableAction.class); private final TableName tableName; private final Random random; @@ -41,6 +39,10 @@ public TruncateTableAction(String tableName) { this.random = new Random(); } + @Override protected Logger getLogger() { + return LOG; + } + @Override public void perform() throws Exception { HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility(); @@ -52,8 +54,8 @@ public void perform() throws Exception { } boolean preserveSplits = random.nextBoolean(); - LOG.info("Performing action: Truncate table " + tableName.getNameAsString() + - "preserve splits " + preserveSplits); + getLogger().info("Performing action: Truncate table {} preserve splits {}", + tableName.getNameAsString(), preserveSplits); admin.truncateTable(tableName, preserveSplits); } } diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/UnbalanceKillAndRebalanceAction.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/UnbalanceKillAndRebalanceAction.java index 264a54f191cb..8cb15dd9bd2b 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/UnbalanceKillAndRebalanceAction.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/UnbalanceKillAndRebalanceAction.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -56,6 +56,10 @@ public UnbalanceKillAndRebalanceAction(long waitUnbalance, long waitKill, long w this.killMetaRs = killMetaRs; } + @Override protected Logger getLogger() { + return LOG; + } + @Override public void perform() throws Exception { ClusterStatus status = this.cluster.getClusterStatus(); @@ -86,7 +90,7 @@ public void perform() throws Exception { } if (!killMetaRs && targetServer.equals(metaServer)) { - LOG.info("Not killing server because it holds hbase:meta."); + getLogger().info("Not killing server because it holds hbase:meta."); } else { killRs(targetServer); killedServers.add(targetServer); diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/UnbalanceRegionsAction.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/UnbalanceRegionsAction.java index 54690bf3fff5..a349134a50ef 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/UnbalanceRegionsAction.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/UnbalanceRegionsAction.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -48,9 +48,13 @@ public UnbalanceRegionsAction(double fractionOfRegions, double fractionOfServers this.fractionOfServers = fractionOfServers; } + @Override protected Logger getLogger() { + return LOG; + } + @Override public void perform() throws Exception { - LOG.info("Unbalancing regions"); + getLogger().info("Unbalancing regions"); ClusterStatus status = this.cluster.getClusterStatus(); List victimServers = new LinkedList(status.getServers()); int targetServerCount = (int)Math.ceil(fractionOfServers * victimServers.size()); diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/monkies/PolicyBasedChaosMonkey.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/monkies/PolicyBasedChaosMonkey.java index 277e221d0e73..4c5d701b07a2 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/monkies/PolicyBasedChaosMonkey.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/monkies/PolicyBasedChaosMonkey.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,11 +18,16 @@ package org.apache.hadoop.hbase.chaos.monkies; +import com.google.common.util.concurrent.ThreadFactoryBuilder; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Objects; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import org.apache.commons.lang.math.RandomUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -37,28 +42,44 @@ public class PolicyBasedChaosMonkey extends ChaosMonkey { private static final Log LOG = LogFactory.getLog(PolicyBasedChaosMonkey.class); private static final long ONE_SEC = 1000; - private static final long FIVE_SEC = 5 * ONE_SEC; private static final long ONE_MIN = 60 * ONE_SEC; public static final long TIMEOUT = ONE_MIN; final IntegrationTestingUtility util; + private final Policy[] policies; + private final ExecutorService monkeyThreadPool; + /** * Construct a new ChaosMonkey * @param util the HBaseIntegrationTestingUtility already configured * @param policies custom policies to use */ - public PolicyBasedChaosMonkey(IntegrationTestingUtility util, Policy... policies) { - this.util = util; - this.policies = policies; + public PolicyBasedChaosMonkey(IntegrationTestingUtility util, Collection policies) { + this(util, policies.toArray(new Policy[0])); } - public PolicyBasedChaosMonkey(IntegrationTestingUtility util, Collection policies) { - this.util = util; - this.policies = policies.toArray(new Policy[policies.size()]); + public PolicyBasedChaosMonkey(IntegrationTestingUtility util, Policy... policies) { + this.util = Objects.requireNonNull(util); + this.policies = Objects.requireNonNull(policies); + if (policies.length == 0) { + throw new IllegalArgumentException("policies may not be empty"); + } + this.monkeyThreadPool = buildMonkeyThreadPool(policies.length); } + private static ExecutorService buildMonkeyThreadPool(final int size) { + return Executors.newFixedThreadPool(size, new ThreadFactoryBuilder() + .setDaemon(false) + .setNameFormat("ChaosMonkey-%d") + .setUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() { + @Override public void uncaughtException(Thread t, Throwable e) { + throw new RuntimeException(e); + } + }) + .build()); + } /** Selects a random item from the given items */ public static T selectRandomItem(T[] items) { @@ -99,27 +120,20 @@ public static List selectRandomItems(T[] items, float ratio) { return originalItems.subList(startIndex, startIndex + selectedNumber); } - private Policy[] policies; - private Thread[] monkeyThreads; - @Override public void start() throws Exception { - monkeyThreads = new Thread[policies.length]; - - for (int i=0; i policies; + private final List policies; public CompositeSequentialPolicy(Policy... policies) { this.policies = Arrays.asList(policies); } diff --git a/hbase-rest/src/main/resources/hbase-webapps/rest/rest.jsp b/hbase-rest/src/main/resources/hbase-webapps/rest/rest.jsp index f0b568493787..1085d4b61d6c 100644 --- a/hbase-rest/src/main/resources/hbase-webapps/rest/rest.jsp +++ b/hbase-rest/src/main/resources/hbase-webapps/rest/rest.jsp @@ -20,6 +20,7 @@ <%@ page contentType="text/html;charset=UTF-8" import="org.apache.hadoop.conf.Configuration" import="org.apache.hadoop.hbase.HBaseConfiguration" + import="org.apache.hadoop.hbase.rest.model.VersionModel" import="org.apache.hadoop.hbase.util.VersionInfo" import="java.util.Date"%> <% @@ -85,6 +86,11 @@ String listenPort = conf.get("hbase.rest.port", "8080"); Value Description + + JVM Version + <%= new VersionModel(getServletContext()).getJVMVersion() %> + JVM vendor and version + HBase Version <%= VersionInfo.getVersion() %>, revision=<%= VersionInfo.getRevision() %> diff --git a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java index fa044309694c..6799e69fb9f4 100644 --- a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java +++ b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java @@ -333,7 +333,7 @@ private synchronized void refresh(boolean forceOnline) throws IOException { // so it overwrites the default group loaded // from region group table or zk groupList.add(new RSGroupInfo(RSGroupInfo.DEFAULT_GROUP, - Sets.newHashSet(getDefaultServers()), + Sets.newHashSet(getDefaultServers(groupList)), orphanTables)); // populate the data @@ -479,9 +479,13 @@ private List getOnlineRS() throws IOException { } private List
getDefaultServers() throws IOException { + return getDefaultServers(listRSGroups() /* get from rsGroupMap */); + } + + private List
getDefaultServers(List rsGroupInfoList) throws IOException { // Build a list of servers in other groups than default group, from rsGroupMap Set
serverAddressesInOtherGroups = new HashSet<>(); - for (RSGroupInfo group : listRSGroups() /* get from rsGroupMap */) { + for (RSGroupInfo group : rsGroupInfoList) { if (!RSGroupInfo.DEFAULT_GROUP.equals(group.getName())) { // not default group serverAddressesInOtherGroups.addAll(group.getServers()); } diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon index 1dab104fed5b..0d02f997ecdf 100644 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon +++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/master/MasterStatusTmpl.jamon @@ -263,6 +263,11 @@ AssignmentManager assignmentManager = master.getAssignmentManager(); Value Description + + JVM Version + <% JvmVersion.getVersion() %> + JVM vendor and version + HBase Version <% org.apache.hadoop.hbase.util.VersionInfo.getVersion() %>, revision=<% org.apache.hadoop.hbase.util.VersionInfo.getRevision() %>HBase version and revision diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon index 3afa7318f02e..0f0fce4f711d 100644 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon +++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon @@ -31,6 +31,7 @@ org.apache.hadoop.hbase.ServerName; org.apache.hadoop.hbase.HBaseConfiguration; org.apache.hadoop.hbase.protobuf.ProtobufUtil; org.apache.hadoop.hbase.protobuf.generated.AdminProtos.ServerInfo; +org.apache.hadoop.hbase.util.JvmVersion; org.apache.hadoop.hbase.zookeeper.MasterAddressTracker; <%doc>If json AND bcn is NOT an empty string presume it a block cache view request. @@ -136,6 +137,11 @@ org.apache.hadoop.hbase.zookeeper.MasterAddressTracker; Value Description + + JVM Version + <% JvmVersion.getVersion() %> + JVM vendor and version + HBase Version <% org.apache.hadoop.hbase.util.VersionInfo.getVersion() %>, revision=<% org.apache.hadoop.hbase.util.VersionInfo.getRevision() %> diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/http/InfoServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/http/InfoServer.java index 0f6c3ddb0e47..9dbd766d9a93 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/http/InfoServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/http/InfoServer.java @@ -19,6 +19,8 @@ package org.apache.hadoop.hbase.http; +import com.google.common.net.HostAndPort; + import java.io.IOException; import java.net.URI; @@ -28,6 +30,7 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; + /** * Create a Jetty embedded server to answer http requests. The primary goal * is to serve up status information for the server. @@ -60,8 +63,8 @@ public InfoServer(String name, String bindAddress, int port, boolean findPort, new org.apache.hadoop.hbase.http.HttpServer.Builder(); builder.setName(name).addEndpoint(URI.create(httpConfig.getSchemePrefix() + - bindAddress + ":" + - port)).setAppDir(HBASE_APP_DIR).setFindPort(findPort).setConf(c); + HostAndPort.fromParts(bindAddress, port).toString())) + .setAppDir(HBASE_APP_DIR).setFindPort(findPort).setConf(c); String logDir = System.getProperty("hbase.log.dir"); if (logDir != null) { builder.setLogDir(logDir); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputFormat.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputFormat.java index 443d4b2d97c2..2109e05100d1 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputFormat.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputFormat.java @@ -173,8 +173,8 @@ public RecordWriter getRecordWriter(TaskAttemptContext context) @Override public void checkOutputSpecs(JobContext context) throws IOException, InterruptedException { - - try (Admin admin = ConnectionFactory.createConnection(getConf()).getAdmin()) { + try (Connection conn = ConnectionFactory.createConnection(getConf()); + Admin admin = conn.getAdmin()) { TableName tableName = TableName.valueOf(this.conf.get(OUTPUT_TABLE)); if (!admin.tableExists(tableName)) { throw new TableNotFoundException("Can't write, table does not exist:" + diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java index 7b1cf9ff5b56..ae89d86ebd17 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java @@ -51,6 +51,7 @@ import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster.LocalityType; import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster.MoveRegionAction; import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster.SwapRegionsAction; +import org.apache.hadoop.hbase.regionserver.compactions.OffPeakHours; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.ReflectionUtils; @@ -1166,26 +1167,34 @@ protected double scale(double min, double max, double value) { */ static class MoveCostFunction extends CostFunction { private static final String MOVE_COST_KEY = "hbase.master.balancer.stochastic.moveCost"; + private static final String MOVE_COST_OFFPEAK_KEY = + "hbase.master.balancer.stochastic.moveCost.offpeak"; private static final String MAX_MOVES_PERCENT_KEY = "hbase.master.balancer.stochastic.maxMovePercent"; - private static final float DEFAULT_MOVE_COST = 7; + static final float DEFAULT_MOVE_COST = 7; + static final float DEFAULT_MOVE_COST_OFFPEAK = 3; private static final int DEFAULT_MAX_MOVES = 600; private static final float DEFAULT_MAX_MOVE_PERCENT = 0.25f; private final float maxMovesPercent; + private final Configuration conf; MoveCostFunction(Configuration conf) { super(conf); - - // Move cost multiplier should be the same cost or higher than the rest of the costs to ensure - // that large benefits are need to overcome the cost of a move. - this.setMultiplier(conf.getFloat(MOVE_COST_KEY, DEFAULT_MOVE_COST)); + this.conf = conf; // What percent of the number of regions a single run of the balancer can move. maxMovesPercent = conf.getFloat(MAX_MOVES_PERCENT_KEY, DEFAULT_MAX_MOVE_PERCENT); } @Override protected double cost() { + // Move cost multiplier should be the same cost or higher than the rest of the costs to ensure + // that large benefits are need to overcome the cost of a move. + if (OffPeakHours.getInstance(conf).isOffPeakHour()) { + this.setMultiplier(conf.getFloat(MOVE_COST_OFFPEAK_KEY, DEFAULT_MOVE_COST_OFFPEAK)); + } else { + this.setMultiplier(conf.getFloat(MOVE_COST_KEY, DEFAULT_MOVE_COST)); + } // Try and size the max number of Moves, but always be prepared to move some. int maxMoves = Math.max((int) (cluster.numRegions * maxMovesPercent), DEFAULT_MAX_MOVES); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java index 84ed7b36d4e5..0d44ee9fc843 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java @@ -761,8 +761,14 @@ public HRegion(final HRegionFileSystem fs, final WAL wal, final Configuration co throw new IllegalArgumentException(MEMSTORE_FLUSH_PER_CHANGES + " can not exceed " + MAX_FLUSH_PER_CHANGES); } - this.rowLockWaitDuration = conf.getInt("hbase.rowlock.wait.duration", - DEFAULT_ROWLOCK_WAIT_DURATION); + int tmpRowLockDuration = conf.getInt("hbase.rowlock.wait.duration", + DEFAULT_ROWLOCK_WAIT_DURATION); + if (tmpRowLockDuration <= 0) { + LOG.info("Found hbase.rowlock.wait.duration set to " + tmpRowLockDuration + ". values <= 0 " + + "will cause all row locking to fail. Treating it as 1ms to avoid region failure."); + tmpRowLockDuration = 1; + } + this.rowLockWaitDuration = tmpRowLockDuration; this.maxWaitForSeqId = conf.getInt(MAX_WAIT_FOR_SEQ_ID_KEY, DEFAULT_MAX_WAIT_FOR_SEQ_ID); this.isLoadingCfsOnDemandDefault = conf.getBoolean(LOAD_CFS_ON_DEMAND_CONFIG_KEY, true); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java index 34f322c9ba46..86bbe8943f0a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java @@ -231,15 +231,6 @@ public class RSRpcServices implements HBaseRPCErrorHandler, */ private static final long DEFAULT_REGION_SERVER_RPC_MINIMUM_SCAN_TIME_LIMIT_DELTA = 10; - /** - * Number of rows in a batch operation above which a warning will be logged. - */ - static final String BATCH_ROWS_THRESHOLD_NAME = "hbase.rpc.rows.warning.threshold"; - /** - * Default value of {@link RSRpcServices#BATCH_ROWS_THRESHOLD_NAME} - */ - static final int BATCH_ROWS_THRESHOLD_DEFAULT = 5000; - /* * Whether to reject rows with size > threshold defined by * {@link RSRpcServices#BATCH_ROWS_THRESHOLD_NAME} @@ -1128,7 +1119,8 @@ public RSRpcServices(HRegionServer rs) throws IOException { RSRpcServices(HRegionServer rs, LogDelegate ld) throws IOException { this.ld = ld; regionServer = rs; - rowSizeWarnThreshold = rs.conf.getInt(BATCH_ROWS_THRESHOLD_NAME, BATCH_ROWS_THRESHOLD_DEFAULT); + rowSizeWarnThreshold = rs.conf.getInt( + HConstants.BATCH_ROWS_THRESHOLD_NAME, HConstants.BATCH_ROWS_THRESHOLD_DEFAULT); RpcSchedulerFactory rpcSchedulerFactory; rejectRowsWithSizeOverThreshold = rs.conf .getBoolean(REJECT_BATCH_ROWS_OVER_THRESHOLD, DEFAULT_REJECT_BATCH_ROWS_OVER_THRESHOLD); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java index d6f48b962e8d..03cd86bfd9a4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/Replication.java @@ -245,7 +245,7 @@ public void startReplicationService() throws IOException { } catch (ReplicationException e) { throw new IOException(e); } - this.replicationSink = new ReplicationSink(this.conf, this.server); + this.replicationSink = new ReplicationSink(this.conf); this.scheduleThreadPool.scheduleAtFixedRate( new ReplicationStatisticsThread(this.replicationSink, this.replicationManager), statsThreadPeriod, statsThreadPeriod, TimeUnit.SECONDS); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java index 9143f3dd805f..34cb867f20bd 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSink.java @@ -18,10 +18,13 @@ */ package org.apache.hadoop.hbase.replication.regionserver; +import com.google.common.collect.Lists; + import java.io.IOException; import java.io.InterruptedIOException; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -40,7 +43,6 @@ import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.Stoppable; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.TableNotFoundException; import org.apache.hadoop.hbase.classification.InterfaceAudience; @@ -90,16 +92,21 @@ public class ReplicationSink { private long hfilesReplicated = 0; private SourceFSConfigurationProvider provider; + /** + * Row size threshold for multi requests above which a warning is logged + */ + private final int rowSizeWarnThreshold; + /** * Create a sink for replication - * - * @param conf conf object - * @param stopper boolean to tell this thread to stop + * @param conf conf object * @throws IOException thrown when HDFS goes bad or bad file name */ - public ReplicationSink(Configuration conf, Stoppable stopper) + public ReplicationSink(Configuration conf) throws IOException { this.conf = HBaseConfiguration.create(conf); + rowSizeWarnThreshold = conf.getInt( + HConstants.BATCH_ROWS_THRESHOLD_NAME, HConstants.BATCH_ROWS_THRESHOLD_DEFAULT); decorateConf(); this.metrics = new MetricsSink(); @@ -215,7 +222,7 @@ public void replicateEntries(List entries, final CellScanner cells, if (!rowMap.isEmpty()) { LOG.debug("Started replicating mutations."); for (Entry, List>> entry : rowMap.entrySet()) { - batch(entry.getKey(), entry.getValue().values()); + batch(entry.getKey(), entry.getValue().values(), rowSizeWarnThreshold); } LOG.debug("Finished replicating mutations."); } @@ -380,9 +387,10 @@ public void stopReplicationSinkServices() { * Do the changes and handle the pool * @param tableName table to insert into * @param allRows list of actions - * @throws IOException + * @param batchRowSizeThreshold rowSize threshold for batch mutation */ - protected void batch(TableName tableName, Collection> allRows) throws IOException { + private void batch(TableName tableName, Collection> allRows, int batchRowSizeThreshold) + throws IOException { if (allRows.isEmpty()) { return; } @@ -391,7 +399,15 @@ protected void batch(TableName tableName, Collection> allRows) throws Connection connection = getConnection(); table = connection.getTable(tableName); for (List rows : allRows) { - table.batch(rows); + List> batchRows; + if (rows.size() > batchRowSizeThreshold) { + batchRows = Lists.partition(rows, batchRowSizeThreshold); + } else { + batchRows = Collections.singletonList(rows); + } + for (List rowList : batchRows) { + table.batch(rowList); + } } } catch (RetriesExhaustedWithDetailsException rewde) { for (Throwable ex : rewde.getCauses()) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java index 144f358a354c..69a3a51c5ad8 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSource.java @@ -858,7 +858,7 @@ private void startWALReaderThread(String threadName, Thread.UncaughtExceptionHan new ClusterMarkingEntryFilter(clusterId, peerClusterId, replicationEndpoint)); ChainWALEntryFilter readerFilter = new ChainWALEntryFilter(filters); entryReader = new ReplicationSourceWALReaderThread(manager, replicationQueueInfo, queue, - startPosition, fs, conf, readerFilter, metrics); + startPosition, fs, conf, readerFilter, metrics, ReplicationSource.this); Threads.setDaemonThreadRunning(entryReader, threadName + ".replicationSource.replicationWALReaderThread." + walGroupId + "," + peerClusterZnode, handler); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceWALReaderThread.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceWALReaderThread.java index f795db9404d8..e3c4f87f1f67 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceWALReaderThread.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/ReplicationSourceWALReaderThread.java @@ -80,6 +80,8 @@ public class ReplicationSourceWALReaderThread extends Thread { private AtomicLong totalBufferUsed; private long totalBufferQuota; + private ReplicationSource source; + /** * Creates a reader worker for a given WAL queue. Reads WAL entries off a given queue, batches the * entries, and puts them on a batch queue. @@ -94,8 +96,8 @@ public class ReplicationSourceWALReaderThread extends Thread { */ public ReplicationSourceWALReaderThread(ReplicationSourceManager manager, ReplicationQueueInfo replicationQueueInfo, PriorityBlockingQueue logQueue, - long startPosition, - FileSystem fs, Configuration conf, WALEntryFilter filter, MetricsSource metrics) { + long startPosition, FileSystem fs, Configuration conf, WALEntryFilter filter, + MetricsSource metrics, ReplicationSource source) { this.replicationQueueInfo = replicationQueueInfo; this.logQueue = logQueue; this.lastReadPath = logQueue.peek(); @@ -118,6 +120,7 @@ public ReplicationSourceWALReaderThread(ReplicationSourceManager manager, this.conf.getInt("replication.source.maxretriesmultiplier", 300); // 5 minutes @ 1 sec per this.metrics = metrics; this.entryBatchQueue = new LinkedBlockingQueue<>(batchCount); + this.source = source; LOG.info("peerClusterZnode=" + replicationQueueInfo.getPeerClusterZnode() + ", ReplicationSourceWALReaderThread : " + replicationQueueInfo.getPeerId() + " inited, replicationBatchSizeCapacity=" + replicationBatchSizeCapacity @@ -132,6 +135,10 @@ public void run() { try (WALEntryStream entryStream = new WALEntryStream(logQueue, fs, conf, lastReadPosition, metrics)) { while (isReaderRunning()) { // loop here to keep reusing stream while we can + if (!source.isPeerEnabled()) { + Threads.sleep(sleepForRetries); + continue; + } if (!checkQuota()) { continue; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java index 483433345f76..c76a3a9c1544 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotManifest.java @@ -23,11 +23,15 @@ import com.google.protobuf.InvalidProtocolBufferException; import java.io.FileNotFoundException; import java.io.IOException; +import java.io.InterruptedIOException; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorCompletionService; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import org.apache.commons.logging.Log; @@ -427,43 +431,69 @@ private void convertToV2SingleManifest() throws IOException { workingDir, desc); v2Regions = SnapshotManifestV2.loadRegionManifests(conf, tpool, workingDirFs, workingDir, desc, manifestSizeLimit); - } finally { - tpool.shutdown(); - } - SnapshotDataManifest.Builder dataManifestBuilder = SnapshotDataManifest.newBuilder(); - dataManifestBuilder.setTableSchema(htd.convert()); + SnapshotDataManifest.Builder dataManifestBuilder = SnapshotDataManifest.newBuilder(); + dataManifestBuilder.setTableSchema(htd.convert()); - if (v1Regions != null && v1Regions.size() > 0) { - dataManifestBuilder.addAllRegionManifests(v1Regions); - } - if (v2Regions != null && v2Regions.size() > 0) { - dataManifestBuilder.addAllRegionManifests(v2Regions); - } + if (v1Regions != null && v1Regions.size() > 0) { + dataManifestBuilder.addAllRegionManifests(v1Regions); + } + if (v2Regions != null && v2Regions.size() > 0) { + dataManifestBuilder.addAllRegionManifests(v2Regions); + } - // Write the v2 Data Manifest. - // Once the data-manifest is written, the snapshot can be considered complete. - // Currently snapshots are written in a "temporary" directory and later - // moved to the "complated" snapshot directory. - setStatusMsg("Writing data manifest for " + this.desc.getName()); - SnapshotDataManifest dataManifest = dataManifestBuilder.build(); - writeDataManifest(dataManifest); - this.regionManifests = dataManifest.getRegionManifestsList(); - - // Remove the region manifests. Everything is now in the data-manifest. - // The delete operation is "relaxed", unless we get an exception we keep going. - // The extra files in the snapshot directory will not give any problem, - // since they have the same content as the data manifest, and even by re-reading - // them we will get the same information. - if (v1Regions != null && v1Regions.size() > 0) { - for (SnapshotRegionManifest regionManifest: v1Regions) { - SnapshotManifestV1.deleteRegionManifest(workingDirFs, workingDir, regionManifest); + // Write the v2 Data Manifest. + // Once the data-manifest is written, the snapshot can be considered complete. + // Currently snapshots are written in a "temporary" directory and later + // moved to the "complated" snapshot directory. + setStatusMsg("Writing data manifest for " + this.desc.getName()); + SnapshotDataManifest dataManifest = dataManifestBuilder.build(); + writeDataManifest(dataManifest); + this.regionManifests = dataManifest.getRegionManifestsList(); + + // Remove the region manifests. Everything is now in the data-manifest. + // The delete operation is "relaxed", unless we get an exception we keep going. + // The extra files in the snapshot directory will not give any problem, + // since they have the same content as the data manifest, and even by re-reading + // them we will get the same information. + int totalDeletes = 0; + ExecutorCompletionService completionService = new ExecutorCompletionService<>(tpool); + if (v1Regions != null) { + for (final SnapshotRegionManifest regionManifest: v1Regions) { + ++totalDeletes; + completionService.submit(new Callable() { + @Override + public Void call() throws Exception { + SnapshotManifestV1.deleteRegionManifest(workingDirFs, workingDir, regionManifest); + return null; + } + }); + } } - } - if (v2Regions != null && v2Regions.size() > 0) { - for (SnapshotRegionManifest regionManifest: v2Regions) { - SnapshotManifestV2.deleteRegionManifest(workingDirFs, workingDir, regionManifest); + if (v2Regions != null) { + for (final SnapshotRegionManifest regionManifest: v2Regions) { + ++totalDeletes; + completionService.submit(new Callable() { + @Override + public Void call() throws Exception { + SnapshotManifestV2.deleteRegionManifest(workingDirFs, workingDir, regionManifest); + return null; + } + }); + } } + // Wait for the deletes to finish. + for (int i = 0; i < totalDeletes; i++) { + try { + completionService.take().get(); + } catch (InterruptedException ie) { + throw new InterruptedIOException(ie.getMessage()); + } catch (ExecutionException e) { + throw new IOException("Error deleting region manifests", e.getCause()); + } + } + } finally { + tpool.shutdown(); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/JvmVersion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/JvmVersion.java index b0bca000732b..e62c78661bd9 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/JvmVersion.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/JvmVersion.java @@ -25,9 +25,7 @@ import org.apache.hadoop.hbase.classification.InterfaceStability; /** - * Certain JVM versions are known to be unstable with HBase. This - * class has a utility function to determine whether the current JVM - * is known to be unstable. + * Utility class to get and check the current JVM version. */ @InterfaceAudience.Private @InterfaceStability.Stable @@ -38,10 +36,19 @@ public abstract class JvmVersion { } /** - * Return true if the current JVM is known to be unstable. + * Return true if the current JVM version is known to be unstable with HBase. */ public static boolean isBadJvmVersion() { String version = System.getProperty("java.version"); return version != null && BAD_JVM_VERSIONS.contains(version); } + + /** + * Return the current JVM version information. + */ + public static String getVersion() { + return System.getProperty("java.vm.vendor", "UNKNOWN_VM_VENDOR") + ' ' + + System.getProperty("java.version", "UNKNOWN_JAVA_VERSION") + '-' + + System.getProperty("java.vm.version", "UNKNOWN_VM_VERSION"); + } } diff --git a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp index 3d5e2303bc8b..1f160ea45ec8 100644 --- a/hbase-server/src/main/resources/hbase-webapps/master/table.jsp +++ b/hbase-server/src/main/resources/hbase-webapps/master/table.jsp @@ -27,6 +27,8 @@ import="java.util.List" import="java.util.LinkedHashMap" import="java.util.Map" + import="java.util.Set" + import="java.util.HashSet" import="java.util.Collections" import="java.util.Collection" import="org.apache.commons.lang.StringEscapeUtils" @@ -45,6 +47,7 @@ import="org.apache.hadoop.hbase.zookeeper.MetaTableLocator" import="org.apache.hadoop.hbase.util.Bytes" import="org.apache.hadoop.hbase.util.FSUtils" + import="org.apache.hadoop.hbase.io.ImmutableBytesWritable" import="org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest" import="org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse.CompactionState" import="org.apache.hadoop.hbase.protobuf.generated.ClusterStatusProtos" @@ -417,40 +420,45 @@ if ( fqtn != null ) {

Table Schema

+<% + Collection families = table.getTableDescriptor().getFamilies(); + Set familyKeySet = new HashSet<>(); + for (HColumnDescriptor family: families) { + familyKeySet.addAll(family.getValues().keySet()); + } +%> - - - - <% - Collection families = table.getTableDescriptor().getFamilies(); + + <% for (HColumnDescriptor family: families) { - %> - - - - - <% } %>
Column Family Name
Property \ Column Family Name
<%= StringEscapeUtils.escapeHtml(family.getNameAsString()) %> - - - - - + %> + + <% } %> + <% - Map familyValues = family.getValues(); - for (ImmutableBytesWritable familyKey: familyValues.keySet()) { - final ImmutableBytesWritable familyValue = familyValues.get(familyKey); + for (ImmutableBytesWritable familyKey: familyKeySet) { %> + + <% + for (HColumnDescriptor family: families) { + String familyValueStr = "-"; + Map familyValues = family.getValues(); + if(familyValues.containsKey(familyKey)){ + final ImmutableBytesWritable familyValue = familyValues.get(familyKey); + familyValueStr = Bytes.toString(familyValue.get(), familyValue.getOffset(), familyValue.getLength()); + } + %> + <% } %> <% } %> -
PropertyValue
+ <%= StringEscapeUtils.escapeHtml(family.getNameAsString()) %> +
<%= StringEscapeUtils.escapeHtml(Bytes.toString(familyKey.get(), familyKey.getOffset(), familyKey.getLength())) %> - - <%= StringEscapeUtils.escapeHtml(Bytes.toString(familyValue.get(), familyValue.getOffset(), familyValue.getLength())) %> + <%= StringEscapeUtils.escapeHtml(familyValueStr) %>
-
<% long totalReadReq = 0; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java index 90ed49ccb4fa..3335b493aaa9 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java @@ -46,6 +46,7 @@ import java.util.Set; import java.util.TreeSet; import java.util.UUID; +import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.TimeUnit; import org.apache.commons.io.FileUtils; @@ -190,10 +191,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { * HBaseTestingUtility*/ private Path dataTestDirOnTestFS = null; - /** - * Shared cluster connection. - */ - private volatile Connection connection; + private final AtomicReference connectionRef = new AtomicReference<>(); /** * System property key to get test directory value. @@ -1170,10 +1168,6 @@ public MiniHBaseCluster getMiniHBaseCluster() { */ public void shutdownMiniCluster() throws Exception { LOG.info("Shutting down minicluster"); - if (this.connection != null && !this.connection.isClosed()) { - this.connection.close(); - this.connection = null; - } shutdownMiniHBaseCluster(); if (!this.passedZkCluster){ shutdownMiniZKCluster(); @@ -1203,10 +1197,7 @@ public boolean cleanupTestDir() { * @throws IOException */ public void shutdownMiniHBaseCluster() throws IOException { - if (hbaseAdmin != null) { - hbaseAdmin.close0(); - hbaseAdmin = null; - } + closeConnection(); // unset the configuration for MIN and MAX RS to start conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, -1); @@ -3020,16 +3011,26 @@ public HBaseCluster getHBaseClusterInterface() { } /** - * Get a Connection to the cluster. - * Not thread-safe (This class needs a lot of work to make it thread-safe). + * Get a shared Connection to the cluster. + * this method is threadsafe. * @return A Connection that can be shared. Don't close. Will be closed on shutdown of cluster. * @throws IOException */ public Connection getConnection() throws IOException { - if (this.connection == null) { - this.connection = ConnectionFactory.createConnection(this.conf); + Connection connection = this.connectionRef.get(); + while (connection == null) { + connection = ConnectionFactory.createConnection(this.conf); + if (! this.connectionRef.compareAndSet(null, connection)) { + try { + connection.close(); + } catch (IOException exception) { + LOG.debug("Ignored failure while closing connection on contended connection creation.", + exception); + } + connection = this.connectionRef.get(); + } } - return this.connection; + return connection; } /** @@ -3067,6 +3068,25 @@ private synchronized void close0() throws IOException { } } + public void closeConnection() throws IOException { + if (hbaseAdmin != null) { + try { + hbaseAdmin.close0(); + } catch (IOException exception) { + LOG.debug("Ignored failure while closing admin.", exception); + } + hbaseAdmin = null; + } + Connection connection = this.connectionRef.getAndSet(null); + if (connection != null) { + try { + connection.close(); + } catch (IOException exception) { + LOG.debug("Ignored failure while closing connection.", exception); + } + } + } + /** * Returns a ZooKeeperWatcher instance. * This instance is shared between HBaseTestingUtility instance users. @@ -3240,7 +3260,7 @@ public String explainTableAvailability(TableName tableName) throws IOException { .getRegionAssignments(); final List> metaLocations = MetaTableAccessor - .getTableRegionsAndLocations(getZooKeeperWatcher(), connection, tableName); + .getTableRegionsAndLocations(getZooKeeperWatcher(), getConnection(), tableName); for (Pair metaLocation : metaLocations) { HRegionInfo hri = metaLocation.getFirst(); ServerName sn = metaLocation.getSecond(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java index da11771c99e6..c36de923a10a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java @@ -226,6 +226,27 @@ public void testLocalityCost() throws Exception { } } + @Test + public void testMoveCostMultiplier() throws Exception { + Configuration conf = HBaseConfiguration.create(); + StochasticLoadBalancer.CostFunction + costFunction = new StochasticLoadBalancer.MoveCostFunction(conf); + BaseLoadBalancer.Cluster cluster = mockCluster(clusterStateMocks[0]); + costFunction.init(cluster); + costFunction.cost(); + assertEquals(StochasticLoadBalancer.MoveCostFunction.DEFAULT_MOVE_COST, + costFunction.getMultiplier(), 0.01); + + //In offpeak hours, the multiplier of move cost should be lower + conf.setInt("hbase.offpeak.start.hour",0); + conf.setInt("hbase.offpeak.end.hour",23); + costFunction = new StochasticLoadBalancer.MoveCostFunction(conf); + costFunction.init(cluster); + costFunction.cost(); + assertEquals(StochasticLoadBalancer.MoveCostFunction.DEFAULT_MOVE_COST_OFFPEAK + , costFunction.getMultiplier(), 0.01); + } + @Test public void testMoveCost() throws Exception { Configuration conf = HBaseConfiguration.create(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java index 0d2e35cacd34..bafe420208fa 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java @@ -6355,6 +6355,68 @@ public Void call() throws Exception { CONF.setInt("hbase.rowlock.wait.duration", prevLockTimeout); } + @Test + public void testBatchMutateWithZeroRowLockWait() throws Exception { + final byte[] a = Bytes.toBytes("a"); + final byte[] b = Bytes.toBytes("b"); + final byte[] c = Bytes.toBytes("c"); // exclusive + + Configuration conf = new Configuration(CONF); + conf.setInt("hbase.rowlock.wait.duration", 0); + final HRegionInfo hri = new HRegionInfo(TableName.valueOf(tableName), a, c); + final HTableDescriptor htd = new HTableDescriptor(tableName); + htd.addFamily(new HColumnDescriptor(fam1)); + region = HRegion.createHRegion(hri, TEST_UTIL.getDataTestDir(), conf, htd, TEST_UTIL.createWal(conf, TEST_UTIL.getDataTestDir(), TEST_UTIL.getDataTestDirOnTestFS(method + ".log"), hri)); + + Mutation[] mutations = new Mutation[] { + new Put(a).addImmutable(fam1, null, null), + new Put(b).addImmutable(fam1, null, null), + }; + + OperationStatus[] status = region.batchMutate(mutations); + assertEquals(OperationStatusCode.SUCCESS, status[0].getOperationStatusCode()); + assertEquals(OperationStatusCode.SUCCESS, status[1].getOperationStatusCode()); + + + // test with a row lock held for a long time + final CountDownLatch obtainedRowLock = new CountDownLatch(1); + ExecutorService exec = Executors.newFixedThreadPool(2); + Future f1 = exec.submit(new Callable() { + @Override + public Void call() throws Exception { + LOG.info("Acquiring row lock"); + RowLock rl = region.getRowLock(b); + obtainedRowLock.countDown(); + LOG.info("Waiting for 5 seconds before releasing lock"); + Threads.sleep(5000); + LOG.info("Releasing row lock"); + rl.release(); + return null; + } + }); + obtainedRowLock.await(30, TimeUnit.SECONDS); + + Future f2 = exec.submit(new Callable() { + @Override + public Void call() throws Exception { + Mutation[] mutations = new Mutation[] { + new Put(a).addImmutable(fam1, null, null), + new Put(b).addImmutable(fam1, null, null), + }; + // when handling row b we are going to spin on the failure to get the row lock + // until the lock above is released, but we will still succeed so long as that + // takes less time then the test time out. + OperationStatus[] status = region.batchMutate(mutations); + assertEquals(OperationStatusCode.SUCCESS, status[0].getOperationStatusCode()); + assertEquals(OperationStatusCode.SUCCESS, status[1].getOperationStatusCode()); + return null; + } + }); + + f1.get(); + f2.get(); + } + @Test public void testCheckAndRowMutateTimestampsAreMonotonic() throws IOException { region = initHRegion(tableName, name.getMethodName(), CONF, fam1); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMultiLogThreshold.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMultiLogThreshold.java index af38ea2bb791..c4ca78c62460 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMultiLogThreshold.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMultiLogThreshold.java @@ -24,6 +24,7 @@ import org.junit.Before; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.ipc.HBaseRpcController; import org.apache.hadoop.hbase.testclassification.LargeTests; @@ -75,8 +76,8 @@ public void setupTest() throws Exception { final TableName tableName = TableName.valueOf("tableName"); TEST_UTIL = HBaseTestingUtility.createLocalHTU(); CONF = TEST_UTIL.getConfiguration(); - THRESHOLD = CONF.getInt(RSRpcServices.BATCH_ROWS_THRESHOLD_NAME, - RSRpcServices.BATCH_ROWS_THRESHOLD_DEFAULT); + THRESHOLD = CONF.getInt(HConstants.BATCH_ROWS_THRESHOLD_NAME, + HConstants.BATCH_ROWS_THRESHOLD_DEFAULT); CONF.setBoolean("hbase.rpc.rows.size.threshold.reject", rejectLargeBatchOp); TEST_UTIL.startMiniCluster(); TEST_UTIL.createTable(tableName, TEST_FAM); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestFIFOCompactionPolicy.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestFIFOCompactionPolicy.java index 3f9bc65fa7ac..46b44a14f5f3 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestFIFOCompactionPolicy.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestFIFOCompactionPolicy.java @@ -115,6 +115,11 @@ public static void setEnvironmentEdge() throws Exception { EnvironmentEdgeManager.injectEdge(ee); Configuration conf = TEST_UTIL.getConfiguration(); conf.setInt(HStore.BLOCKING_STOREFILES_KEY, 10000); + // Expired store file deletion during compaction optimization interferes with the FIFO + // compaction policy. The race causes changes to in-flight-compaction files resulting in a + // non-deterministic number of files selected by compaction policy. Disables that optimization + // for this test run. + conf.setBoolean("hbase.store.delete.expired.storefile", false); TEST_UTIL.startMiniCluster(1); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java index ddd319586b49..b0f3f340be08 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationBase.java @@ -145,12 +145,10 @@ public static void setUpBeforeClass() throws Exception { table.addFamily(fam); fam = new HColumnDescriptor(noRepfamName); table.addFamily(fam); - Connection connection1 = ConnectionFactory.createConnection(conf1); - Connection connection2 = ConnectionFactory.createConnection(conf2); - try (Admin admin1 = connection1.getAdmin()) { + Connection connection1 = utility1.getConnection(); + Connection connection2 = utility2.getConnection(); + try (Admin admin1 = connection1.getAdmin(); Admin admin2 = connection2.getAdmin()) { admin1.createTable(table, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE); - } - try (Admin admin2 = connection2.getAdmin()) { admin2.createTable(table, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE); } utility1.waitUntilAllRegionsAssigned(tableName); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java index f94ad5a1b468..0bd339a179d0 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java @@ -518,12 +518,10 @@ public void testVerifyRepJobWithRawOptions() throws Exception { fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL); table.addFamily(fam); - Connection connection1 = ConnectionFactory.createConnection(conf1); - Connection connection2 = ConnectionFactory.createConnection(conf2); - try (Admin admin1 = connection1.getAdmin()) { + try (Admin admin1 = utility1.getConnection().getAdmin()) { admin1.createTable(table, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE); } - try (Admin admin2 = connection2.getAdmin()) { + try (Admin admin2 = utility2.getConnection().getAdmin()) { admin2.createTable(table, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE); } utility1.waitUntilAllRegionsAssigned(tablename); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSink.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSink.java index d475e007a4b7..9a489385f024 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSink.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestReplicationSink.java @@ -19,7 +19,6 @@ package org.apache.hadoop.hbase.replication.regionserver; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; import java.security.SecureRandom; import java.util.ArrayList; @@ -37,6 +36,7 @@ import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException; import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.util.ByteStringer; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -50,7 +50,6 @@ import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.Stoppable; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Connection; @@ -76,7 +75,7 @@ import org.junit.Test; import org.junit.experimental.categories.Category; -@Category(MediumTests.class) +@Category(LargeTests.class) public class TestReplicationSink { private static final Log LOG = LogFactory.getLog(TestReplicationSink.class); private static final int BATCH_SIZE = 10; @@ -123,10 +122,8 @@ public static void setUpBeforeClass() throws Exception { HConstants.REPLICATION_ENABLE_DEFAULT); TEST_UTIL.getConfiguration().set("hbase.replication.source.fs.conf.provider", TestSourceFSConfigurationProvider.class.getCanonicalName()); - TEST_UTIL.startMiniCluster(3); - SINK = - new ReplicationSink(new Configuration(TEST_UTIL.getConfiguration()), STOPPABLE); + SINK = new ReplicationSink(new Configuration(TEST_UTIL.getConfiguration())); table1 = TEST_UTIL.createTable(TABLE_NAME1, FAM_NAME1); table2 = TEST_UTIL.createTable(TABLE_NAME2, FAM_NAME2); Path rootDir = FSUtils.getRootDir(TEST_UTIL.getConfiguration()); @@ -199,6 +196,40 @@ public void testMixedPutDelete() throws Exception { assertEquals(BATCH_SIZE/2, scanRes.next(BATCH_SIZE).length); } + @Test + public void testLargeEditsPutDelete() throws Exception { + List entries = new ArrayList<>(); + List cells = new ArrayList<>(); + for (int i = 0; i < 5510; i++) { + entries.add(createEntry(TABLE_NAME1, i, KeyValue.Type.Put, cells)); + } + SINK.replicateEntries(entries, CellUtil.createCellScanner(cells), replicationClusterId, + baseNamespaceDir, hfileArchiveDir); + + ResultScanner resultScanner = table1.getScanner(new Scan()); + int totalRows = 0; + while (resultScanner.next() != null) { + totalRows++; + } + assertEquals(5510, totalRows); + + entries = new ArrayList<>(); + cells = new ArrayList<>(); + for (int i = 0; i < 11000; i++) { + entries.add( + createEntry(TABLE_NAME1, i, i % 2 != 0 ? KeyValue.Type.Put : KeyValue.Type.DeleteColumn, + cells)); + } + SINK.replicateEntries(entries, CellUtil.createCellScanner(cells), replicationClusterId, + baseNamespaceDir, hfileArchiveDir); + resultScanner = table1.getScanner(new Scan()); + totalRows = 0; + while (resultScanner.next() != null) { + totalRows++; + } + assertEquals(5500, totalRows); + } + /** * Insert to 2 different tables * @throws Exception @@ -217,7 +248,11 @@ public void testMixedPutTables() throws Exception { Scan scan = new Scan(); ResultScanner scanRes = table2.getScanner(scan); for(Result res : scanRes) { - assertTrue(Bytes.toInt(res.getRow()) % 2 == 0); + assertEquals(0, Bytes.toInt(res.getRow()) % 2); + } + scanRes = table1.getScanner(scan); + for(Result res : scanRes) { + assertEquals(1, Bytes.toInt(res.getRow()) % 2); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestWALEntryStream.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestWALEntryStream.java index 7ad7260c3c38..1828ad8f9cf7 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestWALEntryStream.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestWALEntryStream.java @@ -27,6 +27,8 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.timeout; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import java.io.IOException; @@ -38,7 +40,12 @@ import java.util.NavigableMap; import java.util.NoSuchElementException; import java.util.TreeMap; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.PriorityBlockingQueue; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import org.apache.hadoop.conf.Configuration; @@ -81,7 +88,9 @@ import org.junit.rules.TestName; import org.junit.runner.RunWith; import org.mockito.Mockito; +import org.mockito.invocation.InvocationOnMock; import org.mockito.runners.MockitoJUnitRunner; +import org.mockito.stubbing.Answer; @RunWith(MockitoJUnitRunner.class) @Category({ ReplicationTests.class, LargeTests.class }) @@ -359,10 +368,12 @@ public void testReplicationSourceWALReaderThread() throws Exception { // start up a batcher ReplicationSourceManager mockSourceManager = Mockito.mock(ReplicationSourceManager.class); + ReplicationSource source = Mockito.mock(ReplicationSource.class); + when(source.isPeerEnabled()).thenReturn(true); when(mockSourceManager.getTotalBufferUsed()).thenReturn(new AtomicLong(0)); ReplicationSourceWALReaderThread batcher = new ReplicationSourceWALReaderThread(mockSourceManager, getQueueInfo(),walQueue, 0, - fs, conf, getDummyFilter(), new MetricsSource("1")); + fs, conf, getDummyFilter(), new MetricsSource("1"), source); Path walPath = walQueue.peek(); batcher.start(); WALEntryBatch entryBatch = batcher.take(); @@ -398,10 +409,13 @@ fs, conf, new MetricsSource("1"))) { } ReplicationSourceManager mockSourceManager = mock(ReplicationSourceManager.class); + ReplicationSource source = Mockito.mock(ReplicationSource.class); + when(source.isPeerEnabled()).thenReturn(true); when(mockSourceManager.getTotalBufferUsed()).thenReturn(new AtomicLong(0)); ReplicationSourceWALReaderThread reader = new ReplicationSourceWALReaderThread(mockSourceManager, getRecoveredQueueInfo(), - walQueue, 0, fs, conf, getDummyFilter(), new MetricsSource("1")); + walQueue, 0, fs, conf, getDummyFilter(), + new MetricsSource("1"), source); Path walPath = walQueue.toArray(new Path[2])[1]; reader.start(); WALEntryBatch entryBatch = reader.take(); @@ -456,10 +470,12 @@ public void testReplicationSourceWALReaderThreadWithFilter() throws Exception { appendEntriesToLog(2); ReplicationSourceManager mockSourceManager = mock(ReplicationSourceManager.class); + ReplicationSource source = Mockito.mock(ReplicationSource.class); + when(source.isPeerEnabled()).thenReturn(true); when(mockSourceManager.getTotalBufferUsed()).thenReturn(new AtomicLong(0)); final ReplicationSourceWALReaderThread reader = new ReplicationSourceWALReaderThread(mockSourceManager, getQueueInfo(), walQueue, - 0, fs, conf, filter, new MetricsSource("1")); + 0, fs, conf, filter, new MetricsSource("1"), source); reader.start(); WALEntryBatch entryBatch = reader.take(); @@ -490,10 +506,12 @@ public void testReplicationSourceWALReaderThreadWithFilterWhenLogRolled() throws final long eof = getPosition(firstWAL); ReplicationSourceManager mockSourceManager = mock(ReplicationSourceManager.class); + ReplicationSource source = Mockito.mock(ReplicationSource.class); + when(source.isPeerEnabled()).thenReturn(true); when(mockSourceManager.getTotalBufferUsed()).thenReturn(new AtomicLong(0)); final ReplicationSourceWALReaderThread reader = new ReplicationSourceWALReaderThread(mockSourceManager, getQueueInfo(), walQueue, - 0, fs, conf, filter, new MetricsSource("1")); + 0, fs, conf, filter, new MetricsSource("1"), source); reader.start(); // reader won't put any batch, even if EOF reached. @@ -613,4 +631,65 @@ public void preLogRoll(Path oldPath, Path newPath) throws IOException { currentPath = newPath; } } + + @Test + public void testReplicationSourceWALReaderDisabled() + throws IOException, InterruptedException, ExecutionException { + for(int i=0; i<3; i++) { + //append and sync + appendToLog("key" + i); + } + // get ending position + long position; + try (WALEntryStream entryStream = + new WALEntryStream(walQueue, fs, conf, 0, new MetricsSource("1"))) { + entryStream.next(); + entryStream.next(); + entryStream.next(); + position = entryStream.getPosition(); + } + + // start up a reader + Path walPath = walQueue.peek(); + ReplicationSource source = Mockito.mock(ReplicationSource.class); + when(source.getSourceMetrics()).thenReturn(new MetricsSource("1")); + + final AtomicBoolean enabled = new AtomicBoolean(false); + when(source.isPeerEnabled()).thenAnswer(new Answer() { + @Override + public Boolean answer(InvocationOnMock invocationOnMock) throws Throwable { + return enabled.get(); + } + }); + + ReplicationSourceManager mockSourceManager = mock(ReplicationSourceManager.class); + when(mockSourceManager.getTotalBufferUsed()).thenReturn(new AtomicLong(0)); + final ReplicationSourceWALReaderThread reader = + new ReplicationSourceWALReaderThread(mockSourceManager, getQueueInfo(), walQueue, + 0, fs, conf, getDummyFilter(), new MetricsSource("1"), source); + + reader.start(); + Future future = + Executors.newSingleThreadExecutor().submit(new Callable() { + @Override + public WALEntryBatch call() throws Exception { + return reader.take(); + } + }); + + // make sure that the isPeerEnabled has been called several times + verify(source, timeout(30000).atLeast(5)).isPeerEnabled(); + // confirm that we can read nothing if the peer is disabled + assertFalse(future.isDone()); + // then enable the peer, we should get the batch + enabled.set(true); + WALEntryBatch entryBatch = future.get(); + + // should've batched up our entries + assertNotNull(entryBatch); + assertEquals(3, entryBatch.getWalEntries().size()); + assertEquals(position, entryBatch.getLastWalPosition()); + assertEquals(walPath, entryBatch.getLastWalPath()); + assertEquals(3, entryBatch.getNbRowKeys()); + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestNamespaceCommands.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestNamespaceCommands.java index a20ceeb576a2..72cc7e679f06 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestNamespaceCommands.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/access/TestNamespaceCommands.java @@ -303,14 +303,9 @@ public void testListNamespaces() throws Exception { AccessTestAction listAction = new AccessTestAction() { @Override public Object run() throws Exception { - Connection unmanagedConnection = - ConnectionFactory.createConnection(UTIL.getConfiguration()); - Admin admin = unmanagedConnection.getAdmin(); - try { + try (Connection conn = ConnectionFactory.createConnection(UTIL.getConfiguration()); + Admin admin = conn.getAdmin()) { return Arrays.asList(admin.listNamespaceDescriptors()); - } finally { - admin.close(); - unmanagedConnection.close(); } } }; diff --git a/hbase-thrift/src/main/resources/hbase-webapps/thrift/thrift.jsp b/hbase-thrift/src/main/resources/hbase-webapps/thrift/thrift.jsp index 35cab0451b3e..7026b83f1f69 100644 --- a/hbase-thrift/src/main/resources/hbase-webapps/thrift/thrift.jsp +++ b/hbase-thrift/src/main/resources/hbase-webapps/thrift/thrift.jsp @@ -20,6 +20,7 @@ <%@ page contentType="text/html;charset=UTF-8" import="org.apache.hadoop.conf.Configuration" import="org.apache.hadoop.hbase.HBaseConfiguration" + import="org.apache.hadoop.hbase.util.JvmVersion" import="org.apache.hadoop.hbase.util.VersionInfo" import="java.util.Date" %> @@ -92,6 +93,11 @@ String framed = conf.get("hbase.regionserver.thrift.framed", "false"); Value Description + + JVM Version + <%= JvmVersion.getVersion() %> + JVM vendor and version information + HBase Version <%= VersionInfo.getVersion() %>, r<%= VersionInfo.getRevision() %>