HBASE-24135 HBase RS Group

bin/hbase

@@ -134,6 +134,21 @@ if [ -f "$HBASE_HOME/conf/hbase-env-$COMMAND.sh" ]; then
   . "$HBASE_HOME/conf/hbase-env-$COMMAND.sh"
 fi
 
+# establish a default value for HBASE_OPTS if it's not already set. For now,
+# all we set is the garbage collector.
+if [ -z "${HBASE_OPTS}" ] ; then
+  major_version_number="$(parse_java_major_version "$(read_java_version)")"
+  case "$major_version_number" in
+  8|9|10)
+    HBASE_OPTS="-XX:+UseConcMarkSweepGC"
+    ;;
+  11|*)
+    HBASE_OPTS="-XX:+UseG1GC"
+    ;;
+  esac
+  export HBASE_OPTS
+fi
+
 add_size_suffix() {
     # add an 'm' suffix if the argument is missing one, otherwise use whats there
     local val="$1"
@@ -678,18 +693,15 @@ if [ "${HBASE_JDK11}" != "" ]; then
   fi
 else
   # Use JDK detection
-  JAVA=$JAVA_HOME/bin/java
-
-  version=$($JAVA -version 2>&1 | awk -F '"' '/version/ {print $2}')
-  # '-' check is for cases such as "13-ea"
-  version_number=$(echo "$version" | cut -d'.' -f1 | cut -d'-' -f1)
+  version="$(read_java_version)"
+  major_version_number="$(parse_java_major_version "$version")"
 
   if [ "${DEBUG}" = "true" ]; then
     echo "HBASE_JDK11 not set hence using JDK detection."
-    echo "Extracted JDK version - ${version}, version_number - ${version_number}"
+    echo "Extracted JDK version - ${version}, major_version_number - ${major_version_number}"
   fi
 
-  if [[ "$version_number" -ge "11" ]]; then
+  if [[ "$major_version_number" -ge "11" ]]; then
     if [ "${DEBUG}" = "true" ]; then
       echo "Version ${version} is greater-than/equal to 11 hence adding JDK11 jars to classpath."
     fi

bin/hbase-config.sh

@@ -168,3 +168,26 @@ if [ -z "$JAVA_HOME" ]; then
 EOF
     exit 1
 fi
+
+function read_java_version() {
+  properties="$("${JAVA_HOME}/bin/java" -XshowSettings:properties -version 2>&1)"
+  echo "${properties}" | "${GREP}" java.runtime.version | head -1 | "${SED}" -e 's/.* = \([^ ]*\)/\1/'
+}
+
+# Inspect the system properties exposed by this JVM to identify the major
+# version number. Normalize on the popular version number, thus consider JDK
+# 1.8 as version "8".
+function parse_java_major_version() {
+  complete_version=$1
+  # split off suffix version info like '-b10' or '+10' or '_10'
+  # careful to not use GNU Sed extensions
+  version="$(echo "$complete_version" | "${SED}" -e 's/+/_/g' -e 's/-/_/g' | cut -d'_' -f1)"
+  case "$version" in
+  1.*)
+    echo "$version" | cut -d'.' -f2
+    ;;
+  *)
+    echo "$version" | cut -d'.' -f1
+    ;;
+  esac
+}

conf/hbase-env.sh

@@ -38,10 +38,10 @@
 # export HBASE_OFFHEAPSIZE=1G
 
 # Extra Java runtime options.
-# Below are what we set by default.  May only work with SUN JVM.
-# For more on why as well as other possible settings,
-# see http://hbase.apache.org/book.html#performance
-export HBASE_OPTS="$HBASE_OPTS -XX:+UseConcMarkSweepGC"
+# Default settings are applied according to the detected JVM version. Override these default
+# settings by specifying a value here. For more details on possible settings,
+# see http://hbase.apache.org/book.html#_jvm_tuning
+# export HBASE_OPTS
 
 # Uncomment one of the below three options to enable java garbage collection logging for the server-side processes.
 
@@ -137,3 +137,7 @@ export HBASE_OPTS="$HBASE_OPTS -XX:+UseConcMarkSweepGC"
 # Tell HBase whether it should include Hadoop's lib when start up,
 # the default value is false,means that includes Hadoop's lib.
 # export HBASE_DISABLE_HADOOP_CLASSPATH_LOOKUP="true"
+
+# Override text processing tools for use by these launch scripts.
+export GREP="${GREP-grep}"
+export SED="${SED-sed}"

dev-support/docker/Dockerfile

@@ -39,7 +39,7 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get -qq update && \
     bash=4.4.18-2ubuntu1.2 \
     build-essential=12.4ubuntu1 \
     diffutils=1:3.6-1 \
-    git=1:2.17.1-1ubuntu0.5 \
+    git=1:2.17.1-1ubuntu0.6 \
     rsync=3.1.2-2.1ubuntu1 \
     tar=1.29b-2ubuntu0.1 \
     wget=1.19.4-1ubuntu2.2 \
@@ -178,7 +178,7 @@ RUN mkdir -p /usr/lib/jvm && \
 # TODO (nd): is this really true? investigate and file a ticket.
 ENV SPOTBUGS_HOME '/opt/spotbugs'
 ENV MAVEN_HOME '/opt/maven'
-ENV MAVEN_OPTS '-Xmx4G'
+ENV MAVEN_OPTS '-Xmx3.6G'
 
 CMD ["/bin/bash"]
 

dev-support/flaky-tests/flaky-reporting.Jenkinsfile

@@ -43,7 +43,7 @@ pipeline {
           flaky_args=("${flaky_args[@]}" --urls "${JENKINS_URL}/job/HBase%20Nightly/job/${BRANCH_NAME}" --is-yetus True --max-builds 10)
           flaky_args=("${flaky_args[@]}" --urls "${JENKINS_URL}/job/HBase-Flaky-Tests/job/${BRANCH_NAME}" --is-yetus False --max-builds 30)
           docker build -t hbase-dev-support dev-support
-          docker run -v "${WORKSPACE}":/hbase --workdir=/hbase hbase-dev-support python dev-support/flaky-tests/report-flakies.py --mvn -v "${flaky_args[@]}"
+          docker run --ulimit nproc=12500 -v "${WORKSPACE}":/hbase --workdir=/hbase hbase-dev-support python dev-support/flaky-tests/report-flakies.py --mvn -v "${flaky_args[@]}"
 '''
       }
     }

dev-support/hbase-personality.sh

@@ -82,7 +82,7 @@ function personality_globals
   # Yetus 0.7.0 enforces limits. Default proclimit is 1000.
   # Up it. See HBASE-19902 for how we arrived at this number.
   #shellcheck disable=SC2034
-  PROCLIMIT=10000
+  PROC_LIMIT=12500
 
   # Set docker container to run with 20g. Default is 4g in yetus.
   # See HBASE-19902 for how we arrived at 20g.
@@ -137,7 +137,10 @@ function personality_modules
 
   clear_personality_queue
 
-  extra="-DHBasePatchProcess"
+  # At a few points, hbase modules can run build, test, etc. in parallel
+  # Let it happen. Means we'll use more CPU but should be for short bursts.
+  # https://cwiki.apache.org/confluence/display/MAVEN/Parallel+builds+in+Maven+3
+  extra="--threads=2 -DHBasePatchProcess"
   if [[ "${PATCH_BRANCH}" = branch-1* ]]; then
     extra="${extra} -Dhttps.protocols=TLSv1.2"
   fi

dev-support/hbase_docker.sh

@@ -159,4 +159,4 @@ done
 echo "Successfully built ${IMAGE_NAME}."
 
 echo "Starting hbase shell..."
-docker run -it ${IMAGE_NAME}
+docker run --ulimit nproc=12500 -it ${IMAGE_NAME}

dev-support/hbase_docker/Dockerfile

@@ -21,7 +21,7 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get -qq update && \
   DEBIAN_FRONTEND=noninteractive apt-get -qq install --no-install-recommends -y \
     ca-certificates=20180409 \
     curl=7.58.0-2ubuntu3.8 \
-    git=1:2.17.1-1ubuntu0.5 \
+    git=1:2.17.1-1ubuntu0.6 \
     locales=2.27-3ubuntu1 \
     && \
     apt-get clean && \

dev-support/hbase_nightly_source-artifact.sh

@@ -172,7 +172,7 @@ fi
 
 cd "${unpack_dir}"
 echo "Follow the ref guide section on making a RC: Step 8 Build the binary tarball."
-if mvn -DskipTests -Prelease --batch-mode -Dmaven.repo.local="${m2_tarbuild}" clean install \
+if mvn --threads=2 -DskipTests -Prelease --batch-mode -Dmaven.repo.local="${m2_tarbuild}" clean install \
     assembly:single >"${working_dir}/srctarball_install.log" 2>&1; then
   for artifact in "${unpack_dir}"/hbase-assembly/target/hbase-*-bin.tar.gz; do
     if [ -f "${artifact}" ]; then

dev-support/hbase_nightly_yetus.sh

@@ -65,8 +65,6 @@ YETUS_ARGS=("--sentinel" "${YETUS_ARGS[@]}")
 YETUS_ARGS=("--branch=${BRANCH_NAME}" "${YETUS_ARGS[@]}")
 YETUS_ARGS=("--tests-filter=${TESTS_FILTER}" "${YETUS_ARGS[@]}")
 YETUS_ARGS=("--ignore-unknown-options=true" "${YETUS_ARGS[@]}")
-# Why are these not being picked up from hbase-personality?
-YETUS_ARGS=("--proclimit=10000" "${YETUS_ARGS[@]}")
 YETUS_ARGS=("--dockermemlimit=20g" "${YETUS_ARGS[@]}")
 
 if [[ -n "${EXCLUDE_TESTS_URL}" ]]; then

dev-support/jenkins_precommit_github_yetus.sh

@@ -102,7 +102,6 @@ YETUS_ARGS+=("--reapermode=kill")
 # set relatively high limits for ASF machines
 # changing these to higher values may cause problems
 # with other jobs on systemd-enabled machines
-YETUS_ARGS+=("--proclimit=10000")
 YETUS_ARGS+=("--dockermemlimit=20g")
 # -1 spotbugs issues that show up prior to the patch being applied
 YETUS_ARGS+=("--spotbugs-strict-precheck")

hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java

@@ -67,6 +67,8 @@
 import org.apache.hadoop.hbase.util.Pair;
 import org.apache.yetus.audience.InterfaceAudience;
 
+import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableList;
+
 /**
  * The administrative API for HBase. Obtain an instance from {@link Connection#getAdmin()} and
  * call {@link #close()} when done.
@@ -1660,7 +1662,28 @@ default Collection<ServerName> getBackupMasters() throws IOException {
    * @throws IOException if a remote or network exception occurs
    */
   default Collection<ServerName> getRegionServers() throws IOException {
-    return getClusterMetrics(EnumSet.of(Option.SERVERS_NAME)).getServersName();
+    return getRegionServers(false);
+  }
+
+  /**
+   * Retrieve all current live region servers including decommissioned
+   * if excludeDecommissionedRS is false, else non-decommissioned ones only
+   *
+   * @param excludeDecommissionedRS should we exclude decommissioned RS nodes
+   * @return all current live region servers including/excluding decommissioned hosts
+   * @throws IOException if a remote or network exception occurs
+   */
+  default Collection<ServerName> getRegionServers(boolean excludeDecommissionedRS)
+      throws IOException {
+    List<ServerName> allServers =
+      getClusterMetrics(EnumSet.of(Option.SERVERS_NAME)).getServersName();
+    if (!excludeDecommissionedRS) {
+      return allServers;
+    }
+    List<ServerName> decommissionedRegionServers = listDecommissionedRegionServers();
+    return allServers.stream()
+      .filter(s -> !decommissionedRegionServers.contains(s))
+      .collect(ImmutableList.toImmutableList());
   }
 
   /**

hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncRegionLocatorHelper.java

@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -19,7 +19,6 @@
 
 import static org.apache.hadoop.hbase.exceptions.ClientExceptionsUtil.findException;
 import static org.apache.hadoop.hbase.exceptions.ClientExceptionsUtil.isMetaClearingException;
-
 import java.util.Arrays;
 import java.util.function.Consumer;
 import java.util.function.Function;
@@ -45,7 +44,13 @@ private AsyncRegionLocatorHelper() {
   static boolean canUpdateOnError(HRegionLocation loc, HRegionLocation oldLoc) {
     // Do not need to update if no such location, or the location is newer, or the location is not
     // the same with us
-    return oldLoc != null && oldLoc.getSeqNum() <= loc.getSeqNum() &&
+    if (loc == null || loc.getServerName() == null) {
+      return false;
+    }
+    if (oldLoc == null || oldLoc.getServerName() == null) {
+      return false;
+    }
+    return oldLoc.getSeqNum() <= loc.getSeqNum() &&
       oldLoc.getServerName().equals(loc.getServerName());
   }
 

hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/CopyKeyDataBlockEncoder.java

@@ -81,35 +81,7 @@ public String toString() {
 
   @Override
   public EncodedSeeker createSeeker(final HFileBlockDecodingContext decodingCtx) {
-    return new BufferedEncodedSeeker<SeekerState>(decodingCtx) {
-      @Override
-      protected void decodeNext() {
-        current.keyLength = currentBuffer.getInt();
-        current.valueLength = currentBuffer.getInt();
-        current.ensureSpaceForKey();
-        currentBuffer.get(current.keyBuffer, 0, current.keyLength);
-        current.valueOffset = currentBuffer.position();
-        currentBuffer.skip(current.valueLength);
-        if (includesTags()) {
-          // Read short as unsigned, high byte first
-          current.tagsLength = ((currentBuffer.get() & 0xff) << 8) ^ (currentBuffer.get() & 0xff);
-          currentBuffer.skip(current.tagsLength);
-        }
-        if (includesMvcc()) {
-          current.memstoreTS = ByteBufferUtils.readVLong(currentBuffer);
-        } else {
-          current.memstoreTS = 0;
-        }
-        current.nextKvOffset = currentBuffer.position();
-      }
-
-      @Override
-      protected void decodeFirst() {
-        currentBuffer.skip(Bytes.SIZEOF_INT);
-        current.lastCommonPrefix = 0;
-        decodeNext();
-      }
-    };
+    return new SeekerStateBufferedEncodedSeeker(decodingCtx);
   }
 
   @Override
@@ -123,4 +95,41 @@ protected ByteBuffer internalDecodeKeyValues(DataInputStream source, int allocat
 
     return buffer;
   }
+
+  private static class SeekerStateBufferedEncodedSeeker
+      extends BufferedEncodedSeeker<SeekerState> {
+
+    private SeekerStateBufferedEncodedSeeker(HFileBlockDecodingContext decodingCtx) {
+      super(decodingCtx);
+    }
+
+    @Override
+    protected void decodeNext() {
+      current.keyLength = currentBuffer.getInt();
+      current.valueLength = currentBuffer.getInt();
+      current.ensureSpaceForKey();
+      currentBuffer.get(current.keyBuffer, 0, current.keyLength);
+      current.valueOffset = currentBuffer.position();
+      currentBuffer.skip(current.valueLength);
+      if (includesTags()) {
+        // Read short as unsigned, high byte first
+        current.tagsLength = ((currentBuffer.get() & 0xff) << 8) ^ (currentBuffer.get() & 0xff);
+        currentBuffer.skip(current.tagsLength);
+      }
+      if (includesMvcc()) {
+        current.memstoreTS = ByteBufferUtils.readVLong(currentBuffer);
+      } else {
+        current.memstoreTS = 0;
+      }
+      current.nextKvOffset = currentBuffer.position();
+    }
+
+    @Override
+    protected void decodeFirst() {
+      currentBuffer.skip(Bytes.SIZEOF_INT);
+      current.lastCommonPrefix = 0;
+      decodeNext();
+    }
+  }
+
 }