From 595fcef440839881bb4bf82f83c597f35d217a89 Mon Sep 17 00:00:00 2001 From: xicm Date: Tue, 16 Nov 2021 16:28:22 +0800 Subject: [PATCH 1/8] HBASE-26167: add method in TestingHBaseCluster to use external zookeeper and dfs cluster. --- .../apache/hadoop/hbase/HBaseTestingUtil.java | 108 +++++++++++++++--- .../hadoop/hbase/TestHBaseTestingUtil.java | 41 +++++++ 2 files changed, 132 insertions(+), 17 deletions(-) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtil.java index 589b1a6452a3..7e127d4ab7fe 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtil.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtil.java @@ -133,9 +133,7 @@ import org.apache.hadoop.hbase.util.Threads; import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.wal.WALFactory; -import org.apache.hadoop.hbase.zookeeper.EmptyWatcher; -import org.apache.hadoop.hbase.zookeeper.ZKConfig; -import org.apache.hadoop.hbase.zookeeper.ZKWatcher; +import org.apache.hadoop.hbase.zookeeper.*; import org.apache.hadoop.hdfs.DFSClient; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; @@ -144,8 +142,10 @@ import org.apache.hadoop.mapred.MiniMRCluster; import org.apache.hadoop.mapred.TaskLog; import org.apache.hadoop.minikdc.MiniKdc; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.yetus.audience.InterfaceAudience; import org.apache.yetus.audience.InterfaceStability; +import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.WatchedEvent; import org.apache.zookeeper.ZooKeeper; import org.apache.zookeeper.ZooKeeper.States; @@ -220,6 +220,11 @@ public class HBaseTestingUtil extends HBaseZKTestingUtil { /** This is for unit tests parameterized with a single boolean. */ public static final List MEMSTORETS_TAGS_PARAMETRIZED = memStoreTSAndTagsCombination(); + // Whether to use external DFS. + private boolean externalDFS = false; + // Whether to use external ZK. + private boolean externalZK = false; + /** * Checks to see if a specific port is available. * @param port the port number to check for availability @@ -341,6 +346,34 @@ public HBaseTestingUtil(@Nullable Configuration conf) { this.conf.getBoolean(LocalHBaseCluster.ASSIGN_RANDOM_PORTS, true)); } + /** + * Create an HBaseTestingUtility with the option to use external DFS or ZK. + * Conf should contain properties about the connection to DFS and ZK. + */ + public HBaseTestingUtil(@Nullable Configuration conf, boolean externalDFS, boolean externalZK) { + super(conf); + + this.externalDFS = externalDFS; + this.externalZK = externalZK; + + // a hbase checksum verification failure will cause unit tests to fail + ChecksumUtil.generateExceptionForChecksumFailureForTest(true); + + // Save this for when setting default file:// breaks things + if (this.conf.get("fs.defaultFS") != null) { + this.conf.set("original.defaultFS", this.conf.get("fs.defaultFS")); + } + if (this.conf.get(HConstants.HBASE_DIR) != null) { + this.conf.set("original.hbase.dir", this.conf.get(HConstants.HBASE_DIR)); + } + + this.conf.setBoolean(CommonFSUtils.UNSAFE_STREAM_CAPABILITY_ENFORCE, false); + // If the value for random ports isn't set set it to true, thus making + // tests opt-out for random port assignment + this.conf.setBoolean(LocalHBaseCluster.ASSIGN_RANDOM_PORTS, + this.conf.getBoolean(LocalHBaseCluster.ASSIGN_RANDOM_PORTS, true)); + } + /** * Close both the region {@code r} and it's underlying WAL. For use in tests. */ @@ -772,21 +805,36 @@ public SingleProcessHBaseCluster startMiniCluster(StartTestingClusterOption opti } miniClusterRunning = true; - setupClusterTestDir(); - System.setProperty(TEST_DIRECTORY_KEY, this.clusterTestDir.getPath()); - - // Bring up mini dfs cluster. This spews a bunch of warnings about missing - // scheme. Complaints are 'Scheme is undefined for build/test/data/dfs/name1'. - if (dfsCluster == null) { - LOG.info("STARTING DFS"); - dfsCluster = startMiniDFSCluster(option.getNumDataNodes(), option.getDataNodeHosts()); + if (!externalDFS) { + setupClusterTestDir(); + System.setProperty(TEST_DIRECTORY_KEY, this.clusterTestDir.getPath()); + // Bring up mini dfs cluster. This spews a bunch of warnings about missing + // scheme. Complaints are 'Scheme is undefined for build/test/data/dfs/name1'. + if (dfsCluster == null) { + LOG.info("STARTING DFS"); + dfsCluster = startMiniDFSCluster(option.getNumDataNodes(), option.getDataNodeHosts()); + } else { + LOG.info("NOT STARTING DFS"); + } } else { - LOG.info("NOT STARTING DFS"); + if (System.getProperty("HADOOP_USER_NAME") == null) { + System.setProperty("HADOOP_USER_NAME", "hdfs"); + } + + // RS is started with a different user, @see #HBaseTestingUtil.getDifferentUser + // this is to ensure the user has permissions to read and write HDFS. + conf.set("fs.permissions.umask-mode", "000"); + LOG.info("USING EXTERNAL DFS: {}, user: {}.", + conf.get("fs.defaultFS"), UserGroupInformation.getCurrentUser().getUserName()); } - // Start up a zk cluster. - if (getZkCluster() == null) { - startMiniZKCluster(option.getNumZkServers()); + if (!externalZK) { + // Start up a zk cluster. + if (getZkCluster() == null) { + startMiniZKCluster(option.getNumZkServers()); + } + }else { + LOG.info("USING EXTERNAL ZK: " + conf.get(HConstants.ZOOKEEPER_QUORUM)); } // Start the MiniHBaseCluster @@ -979,15 +1027,35 @@ public SingleProcessHBaseCluster getMiniHBaseCluster() { /** * Stops mini hbase, zk, and hdfs clusters. + * If zk or hdfs is external, clean the znode or dfs path. * @see #startMiniCluster(int) */ public void shutdownMiniCluster() throws IOException { LOG.info("Shutting down minicluster"); shutdownMiniHBaseCluster(); - shutdownMiniDFSCluster(); - shutdownMiniZKCluster(); + + if (externalDFS) { + FileSystem fs = FileSystem.get(this.conf); + fs.delete(new Path(this.conf.get(HConstants.HBASE_DIR)).getParent(), true); + fs.close(); + } else { + shutdownMiniDFSCluster(); + } + if (externalZK) { + try (ZKWatcher watcher = new ZKWatcher(this.conf, "", null)) { + String znode = this.conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT, HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT); + if (ZKUtil.checkExists(watcher, znode) != -1) { + ZKUtil.deleteNodeRecursively(watcher, znode); + } + } catch(KeeperException e) { + throw new IOException(e.getMessage(), e); + } + } else { + shutdownMiniZKCluster(); + } cleanupTestDir(); + resetUserGroupInformation(); miniClusterRunning = false; LOG.info("Minicluster is down"); } @@ -1034,6 +1102,12 @@ private void cleanup() throws IOException { conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MAXTOSTART, -1); } + // When we use external HDFS, we should use an authorised user. + // If UGI is not reset, setting hadoop user with HADOOP_USER_NAME does not work. + public void resetUserGroupInformation() { + UserGroupInformation.reset(); + } + /** * Returns the path to the default root dir the minicluster uses. If create is true, * a new root directory path is fetched irrespective of whether it has been fetched before or not. diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtil.java index 7076bf936126..b273fb3c5f10 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtil.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtil.java @@ -28,6 +28,7 @@ import java.io.File; import java.util.List; import java.util.Random; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileUtil; @@ -41,6 +42,8 @@ import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster; +import org.apache.hadoop.hbase.zookeeper.ZKUtil; +import org.apache.hadoop.hbase.zookeeper.ZKWatcher; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.security.ssl.SSLFactory; import org.junit.ClassRule; @@ -54,6 +57,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; + /** * Test our testing utility class */ @@ -140,6 +144,42 @@ public void testMultiClusters() throws Exception { } } + @Test + public void testMiniClusterWithExternalDFSAndZK() throws Exception { + HBaseTestingUtil hbt1 = new HBaseTestingUtil(); + + // Let's say this is external Zk and DFS + MiniDFSCluster dfsCluster = hbt1.startMiniDFSCluster(3); + MiniZooKeeperCluster zkCluster = hbt1.startMiniZKCluster(); + + Configuration conf = HBaseConfiguration.create(); + conf.set("fs.defaultFS", new Path(dfsCluster.getFileSystem().getUri()).toString()); + + conf.set(HConstants.ZOOKEEPER_QUORUM, zkCluster.getAddress().getHostName()); + conf.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, zkCluster.getAddress().getPort()); + conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/hbase_test"); + + HBaseTestingUtil hbt2 = new HBaseTestingUtil(conf, true, true); + SingleProcessHBaseCluster cluster = hbt2.startMiniCluster(); + String hbaseRootDir = hbt2.conf.get(HConstants.HBASE_DIR); + String znode = hbt2.conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT); + try { + assertEquals(1, cluster.getLiveRegionServerThreads().size()); + } finally { + hbt2.shutdownMiniCluster(); + } + + // check if we cleaned the dir or znode created by miniHbaseCluster + FileSystem fs = FileSystem.get(conf); + assertFalse(fs.exists(new Path(hbaseRootDir).getParent())); + + ZKWatcher watcher = new ZKWatcher(hbt1.conf, "TestHbase", null); + assertEquals(ZKUtil.checkExists(watcher, znode), -1); + + hbt1.shutdownMiniDFSCluster(); + hbt1.shutdownMiniZKCluster(); + } + @Test public void testMiniCluster() throws Exception { HBaseTestingUtil hbt = new HBaseTestingUtil(); @@ -406,6 +446,7 @@ public void testMiniZooKeeperWithMultipleClientPorts() throws Exception { assertTrue(!fs.exists(testdir)); assertTrue(fs.mkdirs(testdir)); assertTrue(hbt.cleanupTestDir()); + hbt.resetUserGroupInformation(); } @Test public void testResolvePortConflict() throws Exception { From a77b6dd60089030773a6698049df3150e87ad317 Mon Sep 17 00:00:00 2001 From: xicm Date: Fri, 17 Dec 2021 18:43:50 +0800 Subject: [PATCH 2/8] add external dfs and zk option in TestingHBaseClusterOption and enbale TestingHBaseClusterImpl to use enternal dfs or zk --- .../apache/hadoop/hbase/HBaseTestingUtil.java | 141 ++++++++++-------- .../hbase/StartTestingClusterOption.java | 38 ++++- .../hadoop/hbase/TestHBaseTestingUtil.java | 8 +- .../testing/TestingHBaseClusterImpl.java | 4 +- .../testing/TestingHBaseClusterOption.java | 41 ++++- 5 files changed, 156 insertions(+), 76 deletions(-) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtil.java index 7e127d4ab7fe..22105ce00518 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtil.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtil.java @@ -17,10 +17,6 @@ */ package org.apache.hadoop.hbase; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - import edu.umd.cs.findbugs.annotations.Nullable; import java.io.File; import java.io.IOException; @@ -154,6 +150,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; +import static org.junit.Assert.*; + /** * Facility for testing HBase. Replacement for old HBaseTestCase and HBaseClusterTestCase * functionality. Create an instance and keep it around testing HBase. @@ -220,10 +218,6 @@ public class HBaseTestingUtil extends HBaseZKTestingUtil { /** This is for unit tests parameterized with a single boolean. */ public static final List MEMSTORETS_TAGS_PARAMETRIZED = memStoreTSAndTagsCombination(); - // Whether to use external DFS. - private boolean externalDFS = false; - // Whether to use external ZK. - private boolean externalZK = false; /** * Checks to see if a specific port is available. @@ -335,36 +329,11 @@ public HBaseTestingUtil(@Nullable Configuration conf) { } // Every cluster is a local cluster until we start DFS // Note that conf could be null, but this.conf will not be - String dataTestDir = getDataTestDir().toString(); - this.conf.set("fs.defaultFS", "file:///"); - this.conf.set(HConstants.HBASE_DIR, "file://" + dataTestDir); - LOG.debug("Setting {} to {}", HConstants.HBASE_DIR, dataTestDir); - this.conf.setBoolean(CommonFSUtils.UNSAFE_STREAM_CAPABILITY_ENFORCE, false); - // If the value for random ports isn't set set it to true, thus making - // tests opt-out for random port assignment - this.conf.setBoolean(LocalHBaseCluster.ASSIGN_RANDOM_PORTS, - this.conf.getBoolean(LocalHBaseCluster.ASSIGN_RANDOM_PORTS, true)); - } - - /** - * Create an HBaseTestingUtility with the option to use external DFS or ZK. - * Conf should contain properties about the connection to DFS and ZK. - */ - public HBaseTestingUtil(@Nullable Configuration conf, boolean externalDFS, boolean externalZK) { - super(conf); - - this.externalDFS = externalDFS; - this.externalZK = externalZK; - - // a hbase checksum verification failure will cause unit tests to fail - ChecksumUtil.generateExceptionForChecksumFailureForTest(true); - - // Save this for when setting default file:// breaks things - if (this.conf.get("fs.defaultFS") != null) { - this.conf.set("original.defaultFS", this.conf.get("fs.defaultFS")); - } - if (this.conf.get(HConstants.HBASE_DIR) != null) { - this.conf.set("original.hbase.dir", this.conf.get(HConstants.HBASE_DIR)); + if (this.conf.get("fs.defaultFS").isEmpty()) { + String dataTestDir = getDataTestDir().toString(); + this.conf.set("fs.defaultFS", "file:///"); + this.conf.set(HConstants.HBASE_DIR, "file://" + dataTestDir); + LOG.debug("Setting {} to {}", HConstants.HBASE_DIR, dataTestDir); } this.conf.setBoolean(CommonFSUtils.UNSAFE_STREAM_CAPABILITY_ENFORCE, false); @@ -805,7 +774,7 @@ public SingleProcessHBaseCluster startMiniCluster(StartTestingClusterOption opti } miniClusterRunning = true; - if (!externalDFS) { + if (!option.isExternalDFS()) { setupClusterTestDir(); System.setProperty(TEST_DIRECTORY_KEY, this.clusterTestDir.getPath()); // Bring up mini dfs cluster. This spews a bunch of warnings about missing @@ -816,25 +785,13 @@ public SingleProcessHBaseCluster startMiniCluster(StartTestingClusterOption opti } else { LOG.info("NOT STARTING DFS"); } - } else { - if (System.getProperty("HADOOP_USER_NAME") == null) { - System.setProperty("HADOOP_USER_NAME", "hdfs"); - } - - // RS is started with a different user, @see #HBaseTestingUtil.getDifferentUser - // this is to ensure the user has permissions to read and write HDFS. - conf.set("fs.permissions.umask-mode", "000"); - LOG.info("USING EXTERNAL DFS: {}, user: {}.", - conf.get("fs.defaultFS"), UserGroupInformation.getCurrentUser().getUserName()); } - if (!externalZK) { + if (!option.isExternalZK()) { // Start up a zk cluster. if (getZkCluster() == null) { startMiniZKCluster(option.getNumZkServers()); } - }else { - LOG.info("USING EXTERNAL ZK: " + conf.get(HConstants.ZOOKEEPER_QUORUM)); } // Start the MiniHBaseCluster @@ -850,6 +807,24 @@ public SingleProcessHBaseCluster startMiniCluster(StartTestingClusterOption opti */ public SingleProcessHBaseCluster startMiniHBaseCluster(StartTestingClusterOption option) throws IOException, InterruptedException { + + if (option.isExternalDFS()) { + assertNotNull("fs.defaultFS can not be null, if use external DFS", conf.get("fs.defaultFS")); + if (System.getProperty("HADOOP_USER_NAME") == null) { + System.setProperty("HADOOP_USER_NAME", "hdfs"); + } + + // RS is started with a different user, @see #HBaseTestingUtil.getDifferentUser + // this is to ensure the user has permissions to read and write HDFS. + conf.set("fs.permissions.umask-mode", "000"); + LOG.info("USING EXTERNAL DFS: {}, user: {}.", + conf.get("fs.defaultFS"), UserGroupInformation.getCurrentUser().getUserName()); + } + + if (option.isExternalZK()) { + assertNotNull("ZOOKEEPER_QUORUM can not be null, if use external ZK", conf.get(HConstants.ZOOKEEPER_QUORUM)); + } + // Now do the mini hbase cluster. Set the hbase.rootdir in config. createRootDir(option.isCreateRootDir()); if (option.isCreateWALDir()) { @@ -1030,18 +1005,63 @@ public SingleProcessHBaseCluster getMiniHBaseCluster() { * If zk or hdfs is external, clean the znode or dfs path. * @see #startMiniCluster(int) */ + public void shutdownMiniCluster(StartTestingClusterOption option) throws IOException { + LOG.info("Shutting down minicluster"); + shutdownMiniHBaseCluster(option); + + if (!option.isExternalDFS()) { + shutdownMiniDFSCluster(); + } + if (!option.isExternalZK()) { + shutdownMiniZKCluster(); + } + + cleanupTestDir(); + resetUserGroupInformation(); + miniClusterRunning = false; + LOG.info("Minicluster is down"); + } + + /** + * Stops mini hbase, zk, and hdfs clusters. + * @see #startMiniCluster(int) + */ public void shutdownMiniCluster() throws IOException { LOG.info("Shutting down minicluster"); shutdownMiniHBaseCluster(); + shutdownMiniDFSCluster(); + shutdownMiniZKCluster(); + + cleanupTestDir(); + miniClusterRunning = false; + LOG.info("Minicluster is down"); + } + + /** + * Shutdown HBase mini cluster.Does not shutdown zk or dfs if running. + * If use external dfs or zk, clean the directory. + * @throws java.io.IOException in case command is unsuccessful + */ + public void shutdownMiniHBaseCluster(StartTestingClusterOption option) throws IOException { + cleanup(); + if (this.hbaseCluster != null) { + this.hbaseCluster.shutdown(); + // Wait till hbase is down before going on to shutdown zk. + this.hbaseCluster.waitUntilShutDown(); + this.hbaseCluster = null; + } + if (zooKeeperWatcher != null) { + zooKeeperWatcher.close(); + zooKeeperWatcher = null; + } - if (externalDFS) { + // clean external dfs dir and znode + if (option.isExternalDFS()) { FileSystem fs = FileSystem.get(this.conf); fs.delete(new Path(this.conf.get(HConstants.HBASE_DIR)).getParent(), true); fs.close(); - } else { - shutdownMiniDFSCluster(); } - if (externalZK) { + if (option.isExternalZK()) { try (ZKWatcher watcher = new ZKWatcher(this.conf, "", null)) { String znode = this.conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT, HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT); if (ZKUtil.checkExists(watcher, znode) != -1) { @@ -1050,14 +1070,7 @@ public void shutdownMiniCluster() throws IOException { } catch(KeeperException e) { throw new IOException(e.getMessage(), e); } - } else { - shutdownMiniZKCluster(); } - - cleanupTestDir(); - resetUserGroupInformation(); - miniClusterRunning = false; - LOG.info("Minicluster is down"); } /** @@ -1103,7 +1116,7 @@ private void cleanup() throws IOException { } // When we use external HDFS, we should use an authorised user. - // If UGI is not reset, setting hadoop user with HADOOP_USER_NAME does not work. + // If UGI is not reseted, setting hadoop user with HADOOP_USER_NAME does not work. public void resetUserGroupInformation() { UserGroupInformation.reset(); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/StartTestingClusterOption.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/StartTestingClusterOption.java index afd74bb9314f..e5554ad5e113 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/StartTestingClusterOption.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/StartTestingClusterOption.java @@ -106,6 +106,15 @@ public final class StartTestingClusterOption { */ private final boolean createWALDir; + /** + * Whether to use external DFS. + */ + private boolean externalDFS = false; + /** + * Whether to use external ZK. + */ + private boolean externalZK = false; + /** * Private constructor. Use {@link Builder#build()}. */ @@ -113,7 +122,7 @@ private StartTestingClusterOption(int numMasters, int numAlwaysStandByMasters, Class masterClass, int numRegionServers, List rsPorts, Class rsClass, int numDataNodes, String[] dataNodeHosts, int numZkServers, boolean createRootDir, - boolean createWALDir) { + boolean createWALDir, boolean externalDFS, boolean externalZK) { this.numMasters = numMasters; this.numAlwaysStandByMasters = numAlwaysStandByMasters; this.masterClass = masterClass; @@ -125,6 +134,8 @@ private StartTestingClusterOption(int numMasters, int numAlwaysStandByMasters, this.numZkServers = numZkServers; this.createRootDir = createRootDir; this.createWALDir = createWALDir; + this.externalDFS = externalDFS; + this.externalZK = externalZK; } public int getNumMasters() { @@ -171,13 +182,22 @@ public boolean isCreateWALDir() { return createWALDir; } + public boolean isExternalDFS() { + return externalDFS; + } + + public boolean isExternalZK() { + return externalZK; + } + @Override public String toString() { return "StartMiniClusterOption{" + "numMasters=" + numMasters + ", masterClass=" + masterClass + ", numRegionServers=" + numRegionServers + ", rsPorts=" + StringUtils.join(rsPorts) + ", rsClass=" + rsClass + ", numDataNodes=" + numDataNodes + ", dataNodeHosts=" + Arrays.toString(dataNodeHosts) + ", numZkServers=" + numZkServers + ", createRootDir=" + - createRootDir + ", createWALDir=" + createWALDir + '}'; + createRootDir + ", createWALDir=" + createWALDir + ", externalDFS=" + externalDFS + + ", externalZK=" + externalZK +'}'; } /** @@ -205,6 +225,8 @@ public static final class Builder { private int numZkServers = 1; private boolean createRootDir = false; private boolean createWALDir = false; + private boolean externalDFS = false; + private boolean externalZK = false; private Builder() { } @@ -215,7 +237,7 @@ public StartTestingClusterOption build() { } return new StartTestingClusterOption(numMasters, numAlwaysStandByMasters, masterClass, numRegionServers, rsPorts, rsClass, numDataNodes, dataNodeHosts, numZkServers, - createRootDir, createWALDir); + createRootDir, createWALDir, externalDFS, externalZK); } public Builder numMasters(int numMasters) { @@ -273,6 +295,16 @@ public Builder createWALDir(boolean createWALDir) { this.createWALDir = createWALDir; return this; } + + public Builder externalDFS(boolean externalDFS) { + this.externalDFS = externalDFS; + return this; + } + + public Builder externalZK(boolean externalZK) { + this.externalZK = externalZK; + return this; + } } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtil.java index b273fb3c5f10..c61900bdbbcc 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtil.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtil.java @@ -159,14 +159,16 @@ public void testMiniClusterWithExternalDFSAndZK() throws Exception { conf.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, zkCluster.getAddress().getPort()); conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/hbase_test"); - HBaseTestingUtil hbt2 = new HBaseTestingUtil(conf, true, true); - SingleProcessHBaseCluster cluster = hbt2.startMiniCluster(); + HBaseTestingUtil hbt2 = new HBaseTestingUtil(conf); + StartTestingClusterOption option = StartTestingClusterOption.builder() + .externalZK(true).externalDFS(true).build(); + SingleProcessHBaseCluster cluster = hbt2.startMiniCluster(option); String hbaseRootDir = hbt2.conf.get(HConstants.HBASE_DIR); String znode = hbt2.conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT); try { assertEquals(1, cluster.getLiveRegionServerThreads().size()); } finally { - hbt2.shutdownMiniCluster(); + hbt2.shutdownMiniCluster(option); } // check if we cleaned the dir or znode created by miniHbaseCluster diff --git a/hbase-testing-util/src/main/java/org/apache/hadoop/hbase/testing/TestingHBaseClusterImpl.java b/hbase-testing-util/src/main/java/org/apache/hadoop/hbase/testing/TestingHBaseClusterImpl.java index 8cea789ab684..d1bd4545c620 100644 --- a/hbase-testing-util/src/main/java/org/apache/hadoop/hbase/testing/TestingHBaseClusterImpl.java +++ b/hbase-testing-util/src/main/java/org/apache/hadoop/hbase/testing/TestingHBaseClusterImpl.java @@ -122,7 +122,7 @@ public CompletableFuture stopRegionServer(ServerName serverName) throws Ex public void stopHBaseCluster() throws Exception { Preconditions.checkState(miniClusterRunning, "Cluster has already been stopped"); Preconditions.checkState(miniHBaseClusterRunning, "HBase cluster has already been started"); - util.shutdownMiniHBaseCluster(); + util.shutdownMiniHBaseCluster(option); miniHBaseClusterRunning = false; } @@ -145,7 +145,7 @@ public void start() throws Exception { @Override public void stop() throws Exception { Preconditions.checkState(miniClusterRunning, "Cluster has already been stopped"); - util.shutdownMiniCluster(); + util.shutdownMiniCluster(option); miniClusterRunning = false; miniHBaseClusterRunning = false; } diff --git a/hbase-testing-util/src/main/java/org/apache/hadoop/hbase/testing/TestingHBaseClusterOption.java b/hbase-testing-util/src/main/java/org/apache/hadoop/hbase/testing/TestingHBaseClusterOption.java index 87d6e2a07de8..70b555b2aeca 100644 --- a/hbase-testing-util/src/main/java/org/apache/hadoop/hbase/testing/TestingHBaseClusterOption.java +++ b/hbase-testing-util/src/main/java/org/apache/hadoop/hbase/testing/TestingHBaseClusterOption.java @@ -98,12 +98,21 @@ public final class TestingHBaseClusterOption { */ private final boolean createWALDir; + /** + * Whether to use external DFS. + */ + private boolean externalDFS = false; + /** + * Whether to use external ZK. + */ + private boolean externalZK = false; + /** * Private constructor. Use {@link Builder#build()}. */ private TestingHBaseClusterOption(Configuration conf, int numMasters, int numAlwaysStandByMasters, int numRegionServers, List rsPorts, int numDataNodes, String[] dataNodeHosts, - int numZkServers, boolean createRootDir, boolean createWALDir) { + int numZkServers, boolean createRootDir, boolean createWALDir, boolean externalZK, boolean externalDFS) { this.conf = conf; this.numMasters = numMasters; this.numAlwaysStandByMasters = numAlwaysStandByMasters; @@ -114,6 +123,8 @@ private TestingHBaseClusterOption(Configuration conf, int numMasters, int numAlw this.numZkServers = numZkServers; this.createRootDir = createRootDir; this.createWALDir = createWALDir; + this.externalDFS = externalDFS; + this.externalZK = externalZK; } public Configuration conf() { @@ -156,12 +167,21 @@ public boolean isCreateWALDir() { return createWALDir; } + public boolean isExternalDFS() { + return externalDFS; + } + + public boolean isExternalZK() { + return externalZK; + } + @Override public String toString() { return "StartMiniClusterOption{" + "numMasters=" + numMasters + ", numRegionServers=" + numRegionServers + ", rsPorts=" + StringUtils.join(rsPorts) + ", numDataNodes=" + numDataNodes + ", dataNodeHosts=" + Arrays.toString(dataNodeHosts) + ", numZkServers=" + - numZkServers + ", createRootDir=" + createRootDir + ", createWALDir=" + createWALDir + '}'; + numZkServers + ", createRootDir=" + createRootDir + ", createWALDir=" + createWALDir + + ", externalDFS=" + externalDFS + ", externalZK=" + externalZK + '}'; } /** @@ -171,7 +191,8 @@ StartTestingClusterOption convert() { return StartTestingClusterOption.builder().numMasters(numMasters) .numAlwaysStandByMasters(numAlwaysStandByMasters).numRegionServers(numRegionServers) .rsPorts(rsPorts).numDataNodes(numDataNodes).dataNodeHosts(dataNodeHosts) - .numZkServers(numZkServers).createRootDir(createRootDir).createWALDir(createWALDir).build(); + .numZkServers(numZkServers).createRootDir(createRootDir).createWALDir(createWALDir) + .externalDFS(externalDFS).externalZK(externalZK).build(); } /** @@ -197,6 +218,8 @@ public static final class Builder { private int numZkServers = 1; private boolean createRootDir = false; private boolean createWALDir = false; + private boolean externalZK = false; + private boolean externalDFS = false; private Builder() { } @@ -207,7 +230,7 @@ public TestingHBaseClusterOption build() { } return new TestingHBaseClusterOption(conf, numMasters, numAlwaysStandByMasters, numRegionServers, rsPorts, numDataNodes, dataNodeHosts, numZkServers, createRootDir, - createWALDir); + createWALDir, externalZK, externalDFS); } public Builder conf(Configuration conf) { @@ -259,5 +282,15 @@ public Builder createWALDir(boolean createWALDir) { this.createWALDir = createWALDir; return this; } + + public Builder externalDFS(boolean externalDFS) { + this.externalDFS = externalDFS; + return this; + } + + public Builder externalZK(boolean externalZK) { + this.externalZK = externalZK; + return this; + } } } From 72b1228b2965e6b26fa7fdd3e3c2a87a44d5c0a2 Mon Sep 17 00:00:00 2001 From: xicm Date: Sat, 18 Dec 2021 00:56:09 +0800 Subject: [PATCH 3/8] fix bug --- .../src/test/java/org/apache/hadoop/hbase/HBaseTestingUtil.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtil.java index 22105ce00518..a0b8088cff23 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtil.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtil.java @@ -329,7 +329,7 @@ public HBaseTestingUtil(@Nullable Configuration conf) { } // Every cluster is a local cluster until we start DFS // Note that conf could be null, but this.conf will not be - if (this.conf.get("fs.defaultFS").isEmpty()) { + if (this.conf.get("fs.defaultFS") != null && !this.conf.get("fs.defaultFS").startsWith("hdfs://")) { String dataTestDir = getDataTestDir().toString(); this.conf.set("fs.defaultFS", "file:///"); this.conf.set(HConstants.HBASE_DIR, "file://" + dataTestDir); From 740b338c2b275332fcd888253d29e29637f95a2f Mon Sep 17 00:00:00 2001 From: xicm Date: Tue, 21 Dec 2021 20:51:05 +0800 Subject: [PATCH 4/8] provide external dfs or zk by Properties --- .../apache/hadoop/hbase/HBaseTestingUtil.java | 113 ++++++++---------- .../hbase/StartTestingClusterOption.java | 26 ++-- .../hadoop/hbase/TestHBaseTestingUtil.java | 23 ++-- .../testing/TestingHBaseClusterImpl.java | 4 +- .../testing/TestingHBaseClusterOption.java | 28 +++-- 5 files changed, 97 insertions(+), 97 deletions(-) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtil.java index a0b8088cff23..f6b04889195b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtil.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtil.java @@ -17,6 +17,10 @@ */ package org.apache.hadoop.hbase; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + import edu.umd.cs.findbugs.annotations.Nullable; import java.io.File; import java.io.IOException; @@ -129,7 +133,10 @@ import org.apache.hadoop.hbase.util.Threads; import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.wal.WALFactory; -import org.apache.hadoop.hbase.zookeeper.*; +import org.apache.hadoop.hbase.zookeeper.EmptyWatcher; +import org.apache.hadoop.hbase.zookeeper.ZKConfig; +import org.apache.hadoop.hbase.zookeeper.ZKUtil; +import org.apache.hadoop.hbase.zookeeper.ZKWatcher; import org.apache.hadoop.hdfs.DFSClient; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; @@ -150,7 +157,6 @@ import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; -import static org.junit.Assert.*; /** * Facility for testing HBase. Replacement for old HBaseTestCase and HBaseClusterTestCase @@ -218,6 +224,9 @@ public class HBaseTestingUtil extends HBaseZKTestingUtil { /** This is for unit tests parameterized with a single boolean. */ public static final List MEMSTORETS_TAGS_PARAMETRIZED = memStoreTSAndTagsCombination(); + private boolean isExternalDFS = false; + private boolean isExternalZK = false; + /** * Checks to see if a specific port is available. @@ -329,12 +338,14 @@ public HBaseTestingUtil(@Nullable Configuration conf) { } // Every cluster is a local cluster until we start DFS // Note that conf could be null, but this.conf will not be - if (this.conf.get("fs.defaultFS") != null && !this.conf.get("fs.defaultFS").startsWith("hdfs://")) { - String dataTestDir = getDataTestDir().toString(); - this.conf.set("fs.defaultFS", "file:///"); - this.conf.set(HConstants.HBASE_DIR, "file://" + dataTestDir); - LOG.debug("Setting {} to {}", HConstants.HBASE_DIR, dataTestDir); - } + String dataTestDir = getDataTestDir().toString(); + this.conf.set("fs.defaultFS", "file:///"); + this.conf.set(HConstants.HBASE_DIR, "file://" + dataTestDir); + LOG.debug("Setting {} to {}", HConstants.HBASE_DIR, dataTestDir); + + // RS is started with a different user, @see #HBaseTestingUtil.getDifferentUser + // this is to ensure the user has permissions to read and write external HDFS. + this.conf.set("fs.permissions.umask-mode", "000"); this.conf.setBoolean(CommonFSUtils.UNSAFE_STREAM_CAPABILITY_ENFORCE, false); // If the value for random ports isn't set set it to true, thus making @@ -722,6 +733,12 @@ private String createDirAndSetProperty(final String relPath, String property) { return path; } + private void addPropertiesToConf(Properties properties) { + properties.forEach((k, v) -> { + this.conf.set(k.toString(), v.toString()); + }); + } + /** * Shuts down instance created by call to {@link #startMiniDFSCluster(int)} or does nothing. */ @@ -774,7 +791,7 @@ public SingleProcessHBaseCluster startMiniCluster(StartTestingClusterOption opti } miniClusterRunning = true; - if (!option.isExternalDFS()) { + if (option.getExternalDFS() == null) { setupClusterTestDir(); System.setProperty(TEST_DIRECTORY_KEY, this.clusterTestDir.getPath()); // Bring up mini dfs cluster. This spews a bunch of warnings about missing @@ -787,7 +804,7 @@ public SingleProcessHBaseCluster startMiniCluster(StartTestingClusterOption opti } } - if (!option.isExternalZK()) { + if (option.getExternalZK() == null) { // Start up a zk cluster. if (getZkCluster() == null) { startMiniZKCluster(option.getNumZkServers()); @@ -808,21 +825,29 @@ public SingleProcessHBaseCluster startMiniCluster(StartTestingClusterOption opti public SingleProcessHBaseCluster startMiniHBaseCluster(StartTestingClusterOption option) throws IOException, InterruptedException { - if (option.isExternalDFS()) { - assertNotNull("fs.defaultFS can not be null, if use external DFS", conf.get("fs.defaultFS")); + if (option.getExternalZK() != null) { + if(option.getExternalZK().get(HConstants.ZOOKEEPER_QUORUM) == null) { + throw new IllegalArgumentException("ZOOKEEPER_QUORUM can not be null."); + } + + addPropertiesToConf(option.getExternalZK()); + this.isExternalZK = true; + } + + if (option.getExternalDFS() != null) { + if(option.getExternalDFS().get("fs.defaultFS") == null) { + throw new IllegalArgumentException("fs.defaultFS can not be null."); + } if (System.getProperty("HADOOP_USER_NAME") == null) { System.setProperty("HADOOP_USER_NAME", "hdfs"); } - // RS is started with a different user, @see #HBaseTestingUtil.getDifferentUser - // this is to ensure the user has permissions to read and write HDFS. - conf.set("fs.permissions.umask-mode", "000"); + addPropertiesToConf(option.getExternalDFS()); + LOG.info("USING EXTERNAL DFS: {}, user: {}.", conf.get("fs.defaultFS"), UserGroupInformation.getCurrentUser().getUserName()); - } - if (option.isExternalZK()) { - assertNotNull("ZOOKEEPER_QUORUM can not be null, if use external ZK", conf.get(HConstants.ZOOKEEPER_QUORUM)); + this.isExternalDFS = true; } // Now do the mini hbase cluster. Set the hbase.rootdir in config. @@ -1002,36 +1027,18 @@ public SingleProcessHBaseCluster getMiniHBaseCluster() { /** * Stops mini hbase, zk, and hdfs clusters. - * If zk or hdfs is external, clean the znode or dfs path. * @see #startMiniCluster(int) */ - public void shutdownMiniCluster(StartTestingClusterOption option) throws IOException { + public void shutdownMiniCluster() throws IOException { LOG.info("Shutting down minicluster"); - shutdownMiniHBaseCluster(option); - - if (!option.isExternalDFS()) { + shutdownMiniHBaseCluster(); + if (!this.isExternalDFS) { shutdownMiniDFSCluster(); } - if (!option.isExternalZK()) { + if (!this.isExternalZK) { shutdownMiniZKCluster(); } - cleanupTestDir(); - resetUserGroupInformation(); - miniClusterRunning = false; - LOG.info("Minicluster is down"); - } - - /** - * Stops mini hbase, zk, and hdfs clusters. - * @see #startMiniCluster(int) - */ - public void shutdownMiniCluster() throws IOException { - LOG.info("Shutting down minicluster"); - shutdownMiniHBaseCluster(); - shutdownMiniDFSCluster(); - shutdownMiniZKCluster(); - cleanupTestDir(); miniClusterRunning = false; LOG.info("Minicluster is down"); @@ -1039,10 +1046,9 @@ public void shutdownMiniCluster() throws IOException { /** * Shutdown HBase mini cluster.Does not shutdown zk or dfs if running. - * If use external dfs or zk, clean the directory. * @throws java.io.IOException in case command is unsuccessful */ - public void shutdownMiniHBaseCluster(StartTestingClusterOption option) throws IOException { + public void shutdownMiniHBaseCluster() throws IOException { cleanup(); if (this.hbaseCluster != null) { this.hbaseCluster.shutdown(); @@ -1056,12 +1062,12 @@ public void shutdownMiniHBaseCluster(StartTestingClusterOption option) throws IO } // clean external dfs dir and znode - if (option.isExternalDFS()) { + if (this.isExternalDFS) { FileSystem fs = FileSystem.get(this.conf); fs.delete(new Path(this.conf.get(HConstants.HBASE_DIR)).getParent(), true); fs.close(); } - if (option.isExternalZK()) { + if (this.isExternalZK) { try (ZKWatcher watcher = new ZKWatcher(this.conf, "", null)) { String znode = this.conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT, HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT); if (ZKUtil.checkExists(watcher, znode) != -1) { @@ -1073,24 +1079,6 @@ public void shutdownMiniHBaseCluster(StartTestingClusterOption option) throws IO } } - /** - * Shutdown HBase mini cluster.Does not shutdown zk or dfs if running. - * @throws java.io.IOException in case command is unsuccessful - */ - public void shutdownMiniHBaseCluster() throws IOException { - cleanup(); - if (this.hbaseCluster != null) { - this.hbaseCluster.shutdown(); - // Wait till hbase is down before going on to shutdown zk. - this.hbaseCluster.waitUntilShutDown(); - this.hbaseCluster = null; - } - if (zooKeeperWatcher != null) { - zooKeeperWatcher.close(); - zooKeeperWatcher = null; - } - } - /** * Abruptly Shutdown HBase mini cluster. Does not shutdown zk or dfs if running. * @throws java.io.IOException throws in case command is unsuccessful @@ -1162,6 +1150,7 @@ public Path createRootDir(boolean create) throws IOException { CommonFSUtils.setRootDir(this.conf, hbaseRootdir); fs.mkdirs(hbaseRootdir); FSUtils.setVersion(fs, hbaseRootdir); + return hbaseRootdir; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/StartTestingClusterOption.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/StartTestingClusterOption.java index e5554ad5e113..fbfd5cdc42c0 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/StartTestingClusterOption.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/StartTestingClusterOption.java @@ -20,6 +20,8 @@ import java.util.Arrays; import java.util.List; +import java.util.Properties; + import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hbase.master.HMaster; import org.apache.yetus.audience.InterfaceAudience; @@ -107,13 +109,15 @@ public final class StartTestingClusterOption { private final boolean createWALDir; /** - * Whether to use external DFS. + * conf of external DFS. + * Use the Properties rather than Configuration, + * because the Configuration may contain default conf. */ - private boolean externalDFS = false; + private Properties externalDFS; /** - * Whether to use external ZK. + * conf of external ZK. */ - private boolean externalZK = false; + private Properties externalZK; /** * Private constructor. Use {@link Builder#build()}. @@ -122,7 +126,7 @@ private StartTestingClusterOption(int numMasters, int numAlwaysStandByMasters, Class masterClass, int numRegionServers, List rsPorts, Class rsClass, int numDataNodes, String[] dataNodeHosts, int numZkServers, boolean createRootDir, - boolean createWALDir, boolean externalDFS, boolean externalZK) { + boolean createWALDir, Properties externalDFS, Properties externalZK) { this.numMasters = numMasters; this.numAlwaysStandByMasters = numAlwaysStandByMasters; this.masterClass = masterClass; @@ -182,11 +186,11 @@ public boolean isCreateWALDir() { return createWALDir; } - public boolean isExternalDFS() { + public Properties getExternalDFS() { return externalDFS; } - public boolean isExternalZK() { + public Properties getExternalZK() { return externalZK; } @@ -225,8 +229,8 @@ public static final class Builder { private int numZkServers = 1; private boolean createRootDir = false; private boolean createWALDir = false; - private boolean externalDFS = false; - private boolean externalZK = false; + private Properties externalDFS = null; + private Properties externalZK = null; private Builder() { } @@ -296,12 +300,12 @@ public Builder createWALDir(boolean createWALDir) { return this; } - public Builder externalDFS(boolean externalDFS) { + public Builder externalDFS(Properties externalDFS) { this.externalDFS = externalDFS; return this; } - public Builder externalZK(boolean externalZK) { + public Builder externalZK(Properties externalZK) { this.externalZK = externalZK; return this; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtil.java index c61900bdbbcc..77e0ea8d8f54 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtil.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtil.java @@ -27,6 +27,7 @@ import java.io.File; import java.util.List; +import java.util.Properties; import java.util.Random; import org.apache.hadoop.conf.Configuration; @@ -45,6 +46,7 @@ import org.apache.hadoop.hbase.zookeeper.ZKUtil; import org.apache.hadoop.hbase.zookeeper.ZKWatcher; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.ssl.SSLFactory; import org.junit.ClassRule; import org.junit.Rule; @@ -152,30 +154,31 @@ public void testMiniClusterWithExternalDFSAndZK() throws Exception { MiniDFSCluster dfsCluster = hbt1.startMiniDFSCluster(3); MiniZooKeeperCluster zkCluster = hbt1.startMiniZKCluster(); - Configuration conf = HBaseConfiguration.create(); - conf.set("fs.defaultFS", new Path(dfsCluster.getFileSystem().getUri()).toString()); + Properties hdfsConf = new Properties(); + hdfsConf.setProperty("fs.defaultFS", new Path(dfsCluster.getFileSystem().getUri()).toString()); - conf.set(HConstants.ZOOKEEPER_QUORUM, zkCluster.getAddress().getHostName()); - conf.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, zkCluster.getAddress().getPort()); - conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/hbase_test"); + Properties zkConf = new Properties(); + zkConf.setProperty(HConstants.ZOOKEEPER_QUORUM, zkCluster.getAddress().getHostName()); + zkConf.setProperty(HConstants.ZOOKEEPER_CLIENT_PORT, Integer.toString(zkCluster.getAddress().getPort())); + zkConf.setProperty(HConstants.ZOOKEEPER_ZNODE_PARENT, "/hbase_test"); - HBaseTestingUtil hbt2 = new HBaseTestingUtil(conf); + HBaseTestingUtil hbt2 = new HBaseTestingUtil(); StartTestingClusterOption option = StartTestingClusterOption.builder() - .externalZK(true).externalDFS(true).build(); + .externalZK(zkConf).externalDFS(hdfsConf).build(); SingleProcessHBaseCluster cluster = hbt2.startMiniCluster(option); String hbaseRootDir = hbt2.conf.get(HConstants.HBASE_DIR); String znode = hbt2.conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT); try { assertEquals(1, cluster.getLiveRegionServerThreads().size()); } finally { - hbt2.shutdownMiniCluster(option); + hbt2.shutdownMiniCluster(); } // check if we cleaned the dir or znode created by miniHbaseCluster - FileSystem fs = FileSystem.get(conf); + FileSystem fs = FileSystem.get(hbt2.conf); assertFalse(fs.exists(new Path(hbaseRootDir).getParent())); - ZKWatcher watcher = new ZKWatcher(hbt1.conf, "TestHbase", null); + ZKWatcher watcher = new ZKWatcher(hbt2.conf, "TestHbase", null); assertEquals(ZKUtil.checkExists(watcher, znode), -1); hbt1.shutdownMiniDFSCluster(); diff --git a/hbase-testing-util/src/main/java/org/apache/hadoop/hbase/testing/TestingHBaseClusterImpl.java b/hbase-testing-util/src/main/java/org/apache/hadoop/hbase/testing/TestingHBaseClusterImpl.java index d1bd4545c620..8cea789ab684 100644 --- a/hbase-testing-util/src/main/java/org/apache/hadoop/hbase/testing/TestingHBaseClusterImpl.java +++ b/hbase-testing-util/src/main/java/org/apache/hadoop/hbase/testing/TestingHBaseClusterImpl.java @@ -122,7 +122,7 @@ public CompletableFuture stopRegionServer(ServerName serverName) throws Ex public void stopHBaseCluster() throws Exception { Preconditions.checkState(miniClusterRunning, "Cluster has already been stopped"); Preconditions.checkState(miniHBaseClusterRunning, "HBase cluster has already been started"); - util.shutdownMiniHBaseCluster(option); + util.shutdownMiniHBaseCluster(); miniHBaseClusterRunning = false; } @@ -145,7 +145,7 @@ public void start() throws Exception { @Override public void stop() throws Exception { Preconditions.checkState(miniClusterRunning, "Cluster has already been stopped"); - util.shutdownMiniCluster(option); + util.shutdownMiniCluster(); miniClusterRunning = false; miniHBaseClusterRunning = false; } diff --git a/hbase-testing-util/src/main/java/org/apache/hadoop/hbase/testing/TestingHBaseClusterOption.java b/hbase-testing-util/src/main/java/org/apache/hadoop/hbase/testing/TestingHBaseClusterOption.java index 70b555b2aeca..54e6d6180b57 100644 --- a/hbase-testing-util/src/main/java/org/apache/hadoop/hbase/testing/TestingHBaseClusterOption.java +++ b/hbase-testing-util/src/main/java/org/apache/hadoop/hbase/testing/TestingHBaseClusterOption.java @@ -19,6 +19,8 @@ import java.util.Arrays; import java.util.List; +import java.util.Properties; + import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.StartTestingClusterOption; @@ -99,20 +101,22 @@ public final class TestingHBaseClusterOption { private final boolean createWALDir; /** - * Whether to use external DFS. + * conf of external DFS. + * Use the Properties rather than Configuration, + * because the Configuration may contain default conf. */ - private boolean externalDFS = false; + private Properties externalDFS; /** - * Whether to use external ZK. + * conf of external ZK. */ - private boolean externalZK = false; + private Properties externalZK; /** * Private constructor. Use {@link Builder#build()}. */ private TestingHBaseClusterOption(Configuration conf, int numMasters, int numAlwaysStandByMasters, int numRegionServers, List rsPorts, int numDataNodes, String[] dataNodeHosts, - int numZkServers, boolean createRootDir, boolean createWALDir, boolean externalZK, boolean externalDFS) { + int numZkServers, boolean createRootDir, boolean createWALDir, Properties externalZK, Properties externalDFS) { this.conf = conf; this.numMasters = numMasters; this.numAlwaysStandByMasters = numAlwaysStandByMasters; @@ -167,11 +171,11 @@ public boolean isCreateWALDir() { return createWALDir; } - public boolean isExternalDFS() { + public Properties getExternalDFS() { return externalDFS; } - public boolean isExternalZK() { + public Properties getExternalZK() { return externalZK; } @@ -218,8 +222,8 @@ public static final class Builder { private int numZkServers = 1; private boolean createRootDir = false; private boolean createWALDir = false; - private boolean externalZK = false; - private boolean externalDFS = false; + private Properties externalZK = null; + private Properties externalDFS = null; private Builder() { } @@ -283,13 +287,13 @@ public Builder createWALDir(boolean createWALDir) { return this; } - public Builder externalDFS(boolean externalDFS) { + public Builder externalDFS(Properties externalDFS) { this.externalDFS = externalDFS; return this; } - public Builder externalZK(boolean externalZK) { - this.externalZK = externalZK; + public Builder externalZK(Properties extetrnalZK) { + this.externalZK = externalDFS; return this; } } From f90065fc1d9538ad3e2e5ca6546eb64377e482c7 Mon Sep 17 00:00:00 2001 From: xicm Date: Wed, 22 Dec 2021 11:29:45 +0800 Subject: [PATCH 5/8] modify hdfs unmask config --- .../java/org/apache/hadoop/hbase/HBaseTestingUtil.java | 8 ++++---- .../org/apache/hadoop/hbase/TestHBaseTestingUtil.java | 1 - 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtil.java index f6b04889195b..237cbfd9fe1c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtil.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtil.java @@ -343,10 +343,6 @@ public HBaseTestingUtil(@Nullable Configuration conf) { this.conf.set(HConstants.HBASE_DIR, "file://" + dataTestDir); LOG.debug("Setting {} to {}", HConstants.HBASE_DIR, dataTestDir); - // RS is started with a different user, @see #HBaseTestingUtil.getDifferentUser - // this is to ensure the user has permissions to read and write external HDFS. - this.conf.set("fs.permissions.umask-mode", "000"); - this.conf.setBoolean(CommonFSUtils.UNSAFE_STREAM_CAPABILITY_ENFORCE, false); // If the value for random ports isn't set set it to true, thus making // tests opt-out for random port assignment @@ -844,6 +840,9 @@ public SingleProcessHBaseCluster startMiniHBaseCluster(StartTestingClusterOption addPropertiesToConf(option.getExternalDFS()); + // RS is started with a different user, @see #HBaseTestingUtil.getDifferentUser + // this is to ensure the user has permissions to read and write external HDFS. + this.conf.set("fs.permissions.umask-mode", "000"); LOG.info("USING EXTERNAL DFS: {}, user: {}.", conf.get("fs.defaultFS"), UserGroupInformation.getCurrentUser().getUserName()); @@ -1077,6 +1076,7 @@ public void shutdownMiniHBaseCluster() throws IOException { throw new IOException(e.getMessage(), e); } } + resetUserGroupInformation(); } /** diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtil.java index 77e0ea8d8f54..2526a6e583b1 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtil.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtil.java @@ -46,7 +46,6 @@ import org.apache.hadoop.hbase.zookeeper.ZKUtil; import org.apache.hadoop.hbase.zookeeper.ZKWatcher; import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.ssl.SSLFactory; import org.junit.ClassRule; import org.junit.Rule; From 6f91b4907af7f0cffe4eb040cd0f8b010c74447c Mon Sep 17 00:00:00 2001 From: xicm Date: Tue, 4 Jan 2022 13:08:49 +0800 Subject: [PATCH 6/8] delete blank --- .../java/org/apache/hadoop/hbase/HBaseTestingUtil.java | 7 +------ .../org/apache/hadoop/hbase/StartTestingClusterOption.java | 1 - .../java/org/apache/hadoop/hbase/TestHBaseTestingUtil.java | 2 -- .../hadoop/hbase/testing/TestingHBaseClusterOption.java | 1 - 4 files changed, 1 insertion(+), 10 deletions(-) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtil.java index 237cbfd9fe1c..353c9fdda4c1 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtil.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtil.java @@ -157,7 +157,6 @@ import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; - /** * Facility for testing HBase. Replacement for old HBaseTestCase and HBaseClusterTestCase * functionality. Create an instance and keep it around testing HBase. @@ -227,7 +226,6 @@ public class HBaseTestingUtil extends HBaseZKTestingUtil { private boolean isExternalDFS = false; private boolean isExternalZK = false; - /** * Checks to see if a specific port is available. * @param port the port number to check for availability @@ -342,7 +340,6 @@ public HBaseTestingUtil(@Nullable Configuration conf) { this.conf.set("fs.defaultFS", "file:///"); this.conf.set(HConstants.HBASE_DIR, "file://" + dataTestDir); LOG.debug("Setting {} to {}", HConstants.HBASE_DIR, dataTestDir); - this.conf.setBoolean(CommonFSUtils.UNSAFE_STREAM_CAPABILITY_ENFORCE, false); // If the value for random ports isn't set set it to true, thus making // tests opt-out for random port assignment @@ -820,7 +817,6 @@ public SingleProcessHBaseCluster startMiniCluster(StartTestingClusterOption opti */ public SingleProcessHBaseCluster startMiniHBaseCluster(StartTestingClusterOption option) throws IOException, InterruptedException { - if (option.getExternalZK() != null) { if(option.getExternalZK().get(HConstants.ZOOKEEPER_QUORUM) == null) { throw new IllegalArgumentException("ZOOKEEPER_QUORUM can not be null."); @@ -1039,6 +1035,7 @@ public void shutdownMiniCluster() throws IOException { } cleanupTestDir(); + resetUserGroupInformation(); miniClusterRunning = false; LOG.info("Minicluster is down"); } @@ -1076,7 +1073,6 @@ public void shutdownMiniHBaseCluster() throws IOException { throw new IOException(e.getMessage(), e); } } - resetUserGroupInformation(); } /** @@ -1150,7 +1146,6 @@ public Path createRootDir(boolean create) throws IOException { CommonFSUtils.setRootDir(this.conf, hbaseRootdir); fs.mkdirs(hbaseRootdir); FSUtils.setVersion(fs, hbaseRootdir); - return hbaseRootdir; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/StartTestingClusterOption.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/StartTestingClusterOption.java index fbfd5cdc42c0..3a6a7729bcab 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/StartTestingClusterOption.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/StartTestingClusterOption.java @@ -21,7 +21,6 @@ import java.util.Arrays; import java.util.List; import java.util.Properties; - import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hbase.master.HMaster; import org.apache.yetus.audience.InterfaceAudience; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtil.java index 2526a6e583b1..d791b768e431 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtil.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtil.java @@ -29,7 +29,6 @@ import java.util.List; import java.util.Properties; import java.util.Random; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileUtil; @@ -58,7 +57,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; - /** * Test our testing utility class */ diff --git a/hbase-testing-util/src/main/java/org/apache/hadoop/hbase/testing/TestingHBaseClusterOption.java b/hbase-testing-util/src/main/java/org/apache/hadoop/hbase/testing/TestingHBaseClusterOption.java index 54e6d6180b57..c53d1b6759d4 100644 --- a/hbase-testing-util/src/main/java/org/apache/hadoop/hbase/testing/TestingHBaseClusterOption.java +++ b/hbase-testing-util/src/main/java/org/apache/hadoop/hbase/testing/TestingHBaseClusterOption.java @@ -20,7 +20,6 @@ import java.util.Arrays; import java.util.List; import java.util.Properties; - import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.StartTestingClusterOption; From 9d2fd0d921c77b39fff43695071cdb46e0fb709f Mon Sep 17 00:00:00 2001 From: xicm Date: Tue, 4 Jan 2022 17:59:59 +0800 Subject: [PATCH 7/8] delete UGI.reset --- .../src/test/java/org/apache/hadoop/hbase/HBaseTestingUtil.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtil.java index 353c9fdda4c1..9ebeab8b4fd0 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtil.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtil.java @@ -1035,7 +1035,6 @@ public void shutdownMiniCluster() throws IOException { } cleanupTestDir(); - resetUserGroupInformation(); miniClusterRunning = false; LOG.info("Minicluster is down"); } From f6025c6a213d617718d2f637e123fdbbbca47953 Mon Sep 17 00:00:00 2001 From: xicm Date: Fri, 14 Jan 2022 14:36:36 +0800 Subject: [PATCH 8/8] remove set HADOOP_USER_NAME --- .../test/java/org/apache/hadoop/hbase/HBaseTestingUtil.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtil.java index 9ebeab8b4fd0..e92cf959f931 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtil.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtil.java @@ -830,9 +830,6 @@ public SingleProcessHBaseCluster startMiniHBaseCluster(StartTestingClusterOption if(option.getExternalDFS().get("fs.defaultFS") == null) { throw new IllegalArgumentException("fs.defaultFS can not be null."); } - if (System.getProperty("HADOOP_USER_NAME") == null) { - System.setProperty("HADOOP_USER_NAME", "hdfs"); - } addPropertiesToConf(option.getExternalDFS());