HBASE-28432 Refactor tools which are under test packaging to a new module hbase-diagnostics

hbase-assembly/pom.xml

@@ -188,6 +188,10 @@
       <groupId>org.apache.hbase</groupId>
       <artifactId>hbase-compression-zstd</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.apache.hbase</groupId>
+      <artifactId>hbase-diagnostics</artifactId>
+    </dependency>
     <dependency>
       <groupId>jline</groupId>
       <artifactId>jline</artifactId>

hbase-assembly/src/main/assembly/hadoop-three-compat.xml

@@ -46,6 +46,7 @@
         <include>org.apache.hbase:hbase-it</include>
         <include>org.apache.hbase:hbase-logging</include>
         <include>org.apache.hbase:hbase-mapreduce</include>
+        <include>org.apache.hbase:hbase-diagnostics</include>
         <include>org.apache.hbase:hbase-metrics</include>
         <include>org.apache.hbase:hbase-metrics-api</include>
         <include>org.apache.hbase:hbase-procedure</include>

hbase-asyncfs/src/test/java/org/apache/hadoop/hbase/security/HBaseKerberosUtils.java

@@ -18,8 +18,6 @@
 package org.apache.hadoop.hbase.security;
 
 import java.io.File;
-import java.io.IOException;
-import java.net.InetAddress;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeys;
 import org.apache.hadoop.hbase.AuthUtil;
@@ -172,23 +170,6 @@ public static void setSSLConfiguration(HBaseCommonTestingUtil utility, Class<?>
     KeyStoreTestUtil.setupSSLConfig(keystoresDir.getAbsolutePath(), sslConfDir, conf, false);
   }
 
-  public static UserGroupInformation loginAndReturnUGI(Configuration conf, String username)
-    throws IOException {
-    String hostname = InetAddress.getLocalHost().getHostName();
-    String keyTabFileConfKey = "hbase." + username + ".keytab.file";
-    String keyTabFileLocation = conf.get(keyTabFileConfKey);
-    String principalConfKey = "hbase." + username + ".kerberos.principal";
-    String principal = org.apache.hadoop.security.SecurityUtil
-      .getServerPrincipal(conf.get(principalConfKey), hostname);
-    if (keyTabFileLocation == null || principal == null) {
-      LOG.warn(
-        "Principal or key tab file null for : " + principalConfKey + ", " + keyTabFileConfKey);
-    }
-    UserGroupInformation ugi =
-      UserGroupInformation.loginUserFromKeytabAndReturnUGI(principal, keyTabFileLocation);
-    return ugi;
-  }
-
   public static UserGroupInformation loginKerberosPrincipal(String krbKeytab, String krbPrincipal)
     throws Exception {
     Configuration conf = new Configuration();

hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/KeyProviderForTesting.java → hbase-common/src/main/java/org/apache/hadoop/hbase/io/crypto/MockAesKeyProvider.java

@@ -19,11 +19,13 @@
 
 import java.security.Key;
 import javax.crypto.spec.SecretKeySpec;
+import org.apache.yetus.audience.InterfaceAudience;
 
 /**
  * Return a fixed secret key for AES for testing.
  */
-public class KeyProviderForTesting implements KeyProvider {
+@InterfaceAudience.Private
+public class MockAesKeyProvider implements KeyProvider {
 
   @Override
   public void init(String parameters) {

hbase-common/src/test/java/org/apache/hadoop/hbase/util/RandomDistribution.java → hbase-common/src/main/java/org/apache/hadoop/hbase/util/RandomDistribution.java

@@ -21,6 +21,7 @@
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.Random;
+import org.apache.yetus.audience.InterfaceAudience;
 
 /**
  * A class that generates random numbers that follow some distribution.
@@ -29,6 +30,7 @@
  * Remove after tfile is committed and use the tfile version of this class instead.
  * </p>
  */
+@InterfaceAudience.Private
 public class RandomDistribution {
   /**
    * Interface for discrete (integer) random distributions.

hbase-common/src/test/java/org/apache/hadoop/hbase/io/crypto/TestKeyProvider.java

@@ -43,10 +43,10 @@ public class TestKeyProvider {
   @Test
   public void testTestProvider() {
     Configuration conf = HBaseConfiguration.create();
-    conf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, KeyProviderForTesting.class.getName());
+    conf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, MockAesKeyProvider.class.getName());
     KeyProvider provider = Encryption.getKeyProvider(conf);
     assertNotNull("Null returned for provider", provider);
-    assertTrue("Provider is not the expected type", provider instanceof KeyProviderForTesting);
+    assertTrue("Provider is not the expected type", provider instanceof MockAesKeyProvider);
 
     Key key = provider.getKey("foo");
     assertNotNull("Test provider did not return a key as expected", key);

hbase-diagnostics/pom.xml

@@ -0,0 +1,190 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <!--
+  /**
+   * Licensed to the Apache Software Foundation (ASF) under one
+   * or more contributor license agreements.  See the NOTICE file
+   * distributed with this work for additional information
+   * regarding copyright ownership.  The ASF licenses this file
+   * to you under the Apache License, Version 2.0 (the
+   * "License"); you may not use this file except in compliance
+   * with the License.  You may obtain a copy of the License at
+   *
+   *     http://www.apache.org/licenses/LICENSE-2.0
+   *
+   * Unless required by applicable law or agreed to in writing, software
+   * distributed under the License is distributed on an "AS IS" BASIS,
+   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   * See the License for the specific language governing permissions and
+   * limitations under the License.
+   */
+  -->
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache.hbase</groupId>
+    <artifactId>hbase-build-configuration</artifactId>
+    <version>${revision}</version>
+    <relativePath>../hbase-build-configuration</relativePath>
+  </parent>
+  <artifactId>hbase-diagnostics</artifactId>
+  <name>Apache HBase - Diagnostics Tools</name>
+  <description>Diagnostics Tools for HBase</description>
+
+  <dependencies>
+    <!-- Intra-project dependencies -->
+    <dependency>
+      <groupId>org.apache.hbase</groupId>
+      <artifactId>hbase-mapreduce</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hbase</groupId>
+      <artifactId>hbase-logging</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hbase.thirdparty</groupId>
+      <artifactId>hbase-shaded-miscellaneous</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hbase</groupId>
+      <artifactId>hbase-common</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hbase</groupId>
+      <artifactId>hbase-protocol-shaded</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hbase</groupId>
+      <artifactId>hbase-client</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hbase</groupId>
+      <artifactId>hbase-zookeeper</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hbase</groupId>
+      <artifactId>hbase-balancer</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hbase.thirdparty</groupId>
+      <artifactId>hbase-shaded-gson</artifactId>
+    </dependency>
+    <!-- General dependencies -->
+    <dependency>
+      <groupId>io.opentelemetry</groupId>
+      <artifactId>opentelemetry-context</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>io.opentelemetry</groupId>
+      <artifactId>opentelemetry-api</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>io.dropwizard.metrics</groupId>
+      <artifactId>metrics-core</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>commons-io</groupId>
+      <artifactId>commons-io</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-lang3</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-math3</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.zookeeper</groupId>
+      <artifactId>zookeeper</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-common</artifactId>
+      <version>${hadoop-three.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-mapreduce-client-core</artifactId>
+      <version>${hadoop-three.version}</version>
+    </dependency>
+    <!-- Intra-project test dependencies -->
+    <dependency>
+      <groupId>org.apache.hbase</groupId>
+      <artifactId>hbase-common</artifactId>
+      <type>test-jar</type>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hbase</groupId>
+      <artifactId>hbase-server</artifactId>
+      <type>test-jar</type>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hbase</groupId>
+      <artifactId>hbase-annotations</artifactId>
+      <type>test-jar</type>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hbase</groupId>
+      <artifactId>hbase-zookeeper</artifactId>
+      <type>test-jar</type>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hbase</groupId>
+      <artifactId>hbase-hadoop-compat</artifactId>
+      <type>test-jar</type>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-hdfs</artifactId>
+      <version>${hadoop-three.version}</version>
+      <type>test-jar</type>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-minicluster</artifactId>
+      <version>${hadoop-three.version}</version>
+      <scope>test</scope>
+      <exclusions>
+        <exclusion>
+          <groupId>javax.xml.bind</groupId>
+          <artifactId>jaxb-api</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>javax.ws.rs</groupId>
+          <artifactId>jsr311-api</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <!-- General test dependencies -->
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.mockito</groupId>
+      <artifactId>mockito-core</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>jul-to-slf4j</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-1.2-api</artifactId>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+</project>

hbase-server/src/test/java/org/apache/hadoop/hbase/HFilePerformanceEvaluation.java → hbase-diagnostics/src/main/java/org/apache/hadoop/hbase/HFilePerformanceEvaluation.java

@@ -28,7 +28,7 @@
 import org.apache.hadoop.hbase.io.crypto.CryptoCipherProvider;
 import org.apache.hadoop.hbase.io.crypto.DefaultCipherProvider;
 import org.apache.hadoop.hbase.io.crypto.Encryption;
-import org.apache.hadoop.hbase.io.crypto.KeyProviderForTesting;
+import org.apache.hadoop.hbase.io.crypto.MockAesKeyProvider;
 import org.apache.hadoop.hbase.io.crypto.aes.AES;
 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
 import org.apache.hadoop.hbase.io.hfile.HFile;
@@ -121,7 +121,7 @@ private void runBenchmarks() throws Exception {
 
     // Add configuration for AES cipher
     final Configuration aesconf = new Configuration();
-    aesconf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, KeyProviderForTesting.class.getName());
+    aesconf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, MockAesKeyProvider.class.getName());
     aesconf.set(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, "hbase");
     aesconf.setInt("hfile.format.version", 3);
     final FileSystem aesfs = FileSystem.get(aesconf);
@@ -137,7 +137,7 @@ private void runBenchmarks() throws Exception {
 
     // Add configuration for Commons cipher
     final Configuration cryptoconf = new Configuration();
-    cryptoconf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, KeyProviderForTesting.class.getName());
+    cryptoconf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, MockAesKeyProvider.class.getName());
     cryptoconf.set(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, "hbase");
     cryptoconf.setInt("hfile.format.version", 3);
     cryptoconf.set(HConstants.CRYPTO_CIPHERPROVIDER_CONF_KEY, CryptoCipherProvider.class.getName());

hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluationCommons.java → hbase-diagnostics/src/main/java/org/apache/hadoop/hbase/PerformanceEvaluationCommons.java

@@ -21,12 +21,14 @@
 import java.util.ArrayList;
 import java.util.List;
 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.yetus.audience.InterfaceAudience;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 /**
  * Code shared by PE tests.
  */
+@InterfaceAudience.Private
 public class PerformanceEvaluationCommons {
   private static final Logger LOG =
     LoggerFactory.getLogger(PerformanceEvaluationCommons.class.getName());

hbase-balancer/src/test/java/org/apache/hadoop/hbase/master/balancer/LoadBalancerPerformanceEvaluation.java → hbase-diagnostics/src/main/java/org/apache/hadoop/hbase/master/balancer/LoadBalancerPerformanceEvaluation.java

@@ -24,7 +24,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
-import org.apache.hadoop.hbase.HBaseCommonTestingUtil;
+import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HBaseInterfaceAudience;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.ServerName;
@@ -54,8 +54,6 @@ public class LoadBalancerPerformanceEvaluation extends AbstractHBaseTool {
   private static final Logger LOG =
     LoggerFactory.getLogger(LoadBalancerPerformanceEvaluation.class.getName());
 
-  protected static final HBaseCommonTestingUtil UTIL = new HBaseCommonTestingUtil();
-
   private static final int DEFAULT_NUM_REGIONS = 1000000;
   private static Option NUM_REGIONS_OPT = new Option("regions", true,
     "Number of regions to consider by load balancer. Default: " + DEFAULT_NUM_REGIONS);
@@ -177,7 +175,7 @@ protected int doWork() throws Exception {
 
   public static void main(String[] args) throws IOException {
     LoadBalancerPerformanceEvaluation tool = new LoadBalancerPerformanceEvaluation();
-    tool.setConf(UTIL.getConfiguration());
+    tool.setConf(HBaseConfiguration.create());
     tool.run(args);
   }
 }