HBASE-24408 Introduce a general 'local region' to store data on master

hbase-common/src/main/resources/hbase-default.xml

@@ -125,7 +125,7 @@ possible configurations would overwhelm and obscure the important.
   </property>
   <property>
     <name>hbase.master.logcleaner.plugins</name>
-    <value>org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner,org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner</value>
+    <value>org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner,org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner,org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner</value>
     <description>A comma-separated list of BaseLogCleanerDelegate invoked by
     the LogsCleaner service. These WAL cleaners are called in order,
     so put the cleaner that prunes the most files in front. To
@@ -139,16 +139,9 @@ possible configurations would overwhelm and obscure the important.
     <description>How long a WAL remain in the archive ({hbase.rootdir}/oldWALs) directory,
     after which it will be cleaned by a Master thread. The value is in milliseconds.</description>
   </property>
-  <property>
-    <name>hbase.master.procedurewalcleaner.ttl</name>
-    <value>604800000</value>
-    <description>How long a Procedure WAL will remain in the
-    archive directory, after which it will be cleaned
-    by a Master thread. The value is in milliseconds.</description>
-  </property>
   <property>
     <name>hbase.master.hfilecleaner.plugins</name>
-    <value>org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner</value>
+    <value>org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner,org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner</value>
     <description>A comma-separated list of BaseHFileCleanerDelegate invoked by
     the HFileCleaner service. These HFiles cleaners are called in order,
     so put the cleaner that prunes the most files in front. To
@@ -157,17 +150,6 @@ possible configurations would overwhelm and obscure the important.
     default hfile cleaners in the list as they will be overwritten in
     hbase-site.xml.</description>
   </property>
-  <property>
-    <name>hbase.procedure.store.region.hfilecleaner.plugins</name>
-    <value>org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner</value>
-    <description>A comma-separated list of BaseHFileCleanerDelegate invoked by
-    the RegionProcedureStore HFileCleaner service. These HFiles cleaners are
-    called in order, so put the cleaner that prunes the most files in front. To
-    implement your own BaseHFileCleanerDelegate, just put it in HBase's classpath
-    and add the fully qualified class name here. Always add the above
-    default hfile cleaners in the list as they will be overwritten in
-    hbase-site.xml.</description>
-  </property>
   <property>
     <name>hbase.master.infoserver.redirect</name>
     <value>true</value>

hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/store/ProcedureStore.java

@@ -186,7 +186,12 @@ public interface ProcedureLoader {
 
   /**
    * Acquire the lease for the procedure store.
+   * @deprecated since 2.3.0, will be removed in 4.0.0 along with
+   *             {@link org.apache.hadoop.hbase.procedure2.store.wal.WALProcedureStore}. As now we
+   *             will store the procedure data in a master local region, and master itself will deal
+   *             with the lease recovery of the region.
    */
+  @Deprecated
   void recoverLease() throws IOException;
 
   /**

hbase-server/src/main/java/org/apache/hadoop/hbase/io/FileLink.java

@@ -521,12 +521,13 @@ public static String getBackReferenceFileName(final Path dirPath) {
 
   /**
    * Checks if the specified directory path is a back reference links folder.
-   *
    * @param dirPath Directory path to verify
    * @return True if the specified directory is a link references folder
    */
   public static boolean isBackReferencesDir(final Path dirPath) {
-    if (dirPath == null) return false;
+    if (dirPath == null) {
+      return false;
+    }
     return dirPath.getName().startsWith(BACK_REFERENCES_DIRECTORY_PREFIX);
   }
 

hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java

@@ -209,7 +209,9 @@ public static boolean isHFileLink(final Path path) {
    */
   public static boolean isHFileLink(String fileName) {
     Matcher m = LINK_NAME_PATTERN.matcher(fileName);
-    if (!m.matches()) return false;
+    if (!m.matches()) {
+      return false;
+    }
     return m.groupCount() > 2 && m.group(4) != null && m.group(3) != null && m.group(2) != null;
   }
 

hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java

@@ -144,6 +144,7 @@
 import org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure;
 import org.apache.hadoop.hbase.master.slowlog.SlowLogMasterService;
 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
+import org.apache.hadoop.hbase.master.store.LocalStore;
 import org.apache.hadoop.hbase.master.zksyncer.MasterAddressSyncer;
 import org.apache.hadoop.hbase.master.zksyncer.MetaLocationSyncer;
 import org.apache.hadoop.hbase.mob.MobFileCleanerChore;
@@ -449,6 +450,9 @@ public void run() {
   private ProcedureExecutor<MasterProcedureEnv> procedureExecutor;
   private ProcedureStore procedureStore;
 
+  // the master local storage to store procedure data, etc.
+  private LocalStore localStore;
+
   // handle table states
   private TableStateManager tableStateManager;
 
@@ -913,7 +917,8 @@ private void finishActiveMasterInitialization(MonitoredTask status) throws IOExc
     this.masterActiveTime = System.currentTimeMillis();
     // TODO: Do this using Dependency Injection, using PicoContainer, Guice or Spring.
 
-    // always initialize the MemStoreLAB as we use a region to store procedure now.
+    // always initialize the MemStoreLAB as we use a region to store data in master now, see
+    // localStore.
     initializeMemStoreChunkCreator();
     this.fileSystemManager = new MasterFileSystem(conf);
     this.walManager = new MasterWalManager(this);
@@ -959,6 +964,9 @@ private void finishActiveMasterInitialization(MonitoredTask status) throws IOExc
       DEFAULT_HBASE_SPLIT_COORDINATED_BY_ZK)) {
       this.splitWALManager = new SplitWALManager(this);
     }
+
+    // initialize local store
+    localStore = LocalStore.create(this);
     createProcedureExecutor();
     Map<Class<?>, List<Procedure<MasterProcedureEnv>>> procsByType =
       procedureExecutor.getActiveProceduresNoCopy().stream()
@@ -1447,6 +1455,8 @@ private void startServiceThreads() throws IOException {
     this.executorService.startExecutorService(ExecutorType.MASTER_TABLE_OPERATIONS, 1);
     startProcedureExecutor();
 
+    // Create cleaner thread pool
+    cleanerPool = new DirScanPool(conf);
     // Start log cleaner thread
     int cleanerInterval =
       conf.getInt(HBASE_MASTER_CLEANER_INTERVAL, DEFAULT_HBASE_MASTER_CLEANER_INTERVAL);
@@ -1534,6 +1544,9 @@ protected void stopServiceThreads() {
 
     stopProcedureExecutor();
 
+    if (localStore != null) {
+      localStore.close(isAborted());
+    }
     if (this.walManager != null) {
       this.walManager.stop();
     }
@@ -1550,10 +1563,8 @@ protected void stopServiceThreads() {
 
   private void createProcedureExecutor() throws IOException {
     MasterProcedureEnv procEnv = new MasterProcedureEnv(this);
-    // Create cleaner thread pool
-    cleanerPool = new DirScanPool(conf);
-    procedureStore = new RegionProcedureStore(this, cleanerPool,
-      new MasterProcedureEnv.FsUtilsLeaseRecovery(this));
+    procedureStore =
+      new RegionProcedureStore(this, localStore, new MasterProcedureEnv.FsUtilsLeaseRecovery(this));
     procedureStore.registerListener(new ProcedureStoreListener() {
 
       @Override

hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/BaseTimeToLiveFileCleaner.java

@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.cleaner;
+
+import java.time.Instant;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Base class for time to live file cleaner.
+ */
+@InterfaceAudience.Private
+public abstract class BaseTimeToLiveFileCleaner extends BaseLogCleanerDelegate {
+
+  private static final Logger LOG =
+    LoggerFactory.getLogger(BaseTimeToLiveFileCleaner.class.getName());
+
+  private static final DateTimeFormatter FORMATTER =
+    DateTimeFormatter.ISO_DATE_TIME.withZone(ZoneOffset.systemDefault());
+
+  // Configured time a log can be kept after it was closed
+  private long ttlMs;
+
+  private volatile boolean stopped = false;
+
+  @Override
+  public final void setConf(Configuration conf) {
+    super.setConf(conf);
+    this.ttlMs = getTtlMs(conf);
+  }
+
+  @Override
+  public boolean isFileDeletable(FileStatus status) {
+    // Files are validated for the second time here,
+    // if it causes a bottleneck this logic needs refactored
+    if (!valiateFilename(status.getPath())) {
+      return true;
+    }
+    long currentTime = EnvironmentEdgeManager.currentTime();
+    long time = status.getModificationTime();
+    long life = currentTime - time;
+
+    if (LOG.isTraceEnabled()) {
+      LOG.trace("File life:{}ms, ttl:{}ms, current:{}, from{}", life, ttlMs,
+        FORMATTER.format(Instant.ofEpochMilli(currentTime)),
+        FORMATTER.format(Instant.ofEpochMilli(time)));
+    }
+    if (life < 0) {
+      LOG.warn("Found a file ({}) newer than current time ({} < {}), probably a clock skew",
+        status.getPath(), FORMATTER.format(Instant.ofEpochMilli(currentTime)),
+        FORMATTER.format(Instant.ofEpochMilli(time)));
+      return false;
+    }
+    return life > ttlMs;
+  }
+
+  @Override
+  public void stop(String why) {
+    this.stopped = true;
+  }
+
+  @Override
+  public boolean isStopped() {
+    return this.stopped;
+  }
+
+  protected abstract long getTtlMs(Configuration conf);
+
+  protected abstract boolean valiateFilename(Path file);
+}

hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java

@@ -32,6 +32,7 @@
 import org.apache.hadoop.hbase.Stoppable;
 import org.apache.hadoop.hbase.conf.ConfigurationObserver;
 import org.apache.hadoop.hbase.io.HFileLink;
+import org.apache.hadoop.hbase.master.store.LocalStore;
 import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
 import org.apache.hadoop.hbase.util.StealJobQueue;
 import org.apache.yetus.audience.InterfaceAudience;
@@ -158,10 +159,9 @@ public HFileCleaner(String name, int period, Stoppable stopper, Configuration co
 
   @Override
   protected boolean validate(Path file) {
-    if (HFileLink.isBackReferencesDir(file) || HFileLink.isBackReferencesDir(file.getParent())) {
-      return true;
-    }
-    return StoreFileInfo.validateStoreFileName(file.getName());
+    return HFileLink.isBackReferencesDir(file) || HFileLink.isBackReferencesDir(file.getParent()) ||
+      StoreFileInfo.validateStoreFileName(file.getName()) ||
+      file.getName().endsWith(LocalStore.ARCHIVED_HFILE_SUFFIX);
   }
 
   /**

hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/LogCleaner.java

@@ -35,6 +35,7 @@
 import org.apache.hadoop.hbase.Stoppable;
 import org.apache.hadoop.hbase.conf.ConfigurationObserver;
 import org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil;
+import org.apache.hadoop.hbase.master.store.LocalStore;
 import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
 import org.apache.yetus.audience.InterfaceAudience;
 import org.slf4j.Logger;
@@ -86,8 +87,9 @@ public LogCleaner(final int period, final Stoppable stopper, Configuration conf,
 
   @Override
   protected boolean validate(Path file) {
-    return AbstractFSWALProvider.validateWALFilename(file.getName())
-        || MasterProcedureUtil.validateProcedureWALFilename(file.getName());
+    return AbstractFSWALProvider.validateWALFilename(file.getName()) ||
+      MasterProcedureUtil.validateProcedureWALFilename(file.getName()) ||
+      file.getName().endsWith(LocalStore.ARCHIVED_WAL_SUFFIX);
   }
 
   @Override

hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/TimeToLiveHFileCleaner.java

@@ -17,48 +17,33 @@
  */
 package org.apache.hadoop.hbase.master.cleaner;
 
-import org.apache.yetus.audience.InterfaceAudience;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseInterfaceAudience;
-import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.hadoop.hbase.io.HFileLink;
+import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
+import org.apache.yetus.audience.InterfaceAudience;
 
 /**
  * HFile cleaner that uses the timestamp of the hfile to determine if it should be deleted. By
  * default they are allowed to live for {@value #DEFAULT_TTL}
  */
 @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG)
-public class TimeToLiveHFileCleaner extends BaseHFileCleanerDelegate {
+public class TimeToLiveHFileCleaner extends BaseTimeToLiveFileCleaner {
 
-  private static final Logger LOG = LoggerFactory.getLogger(TimeToLiveHFileCleaner.class.getName());
   public static final String TTL_CONF_KEY = "hbase.master.hfilecleaner.ttl";
+
   // default ttl = 5 minutes
   public static final long DEFAULT_TTL = 60000 * 5;
-  // Configured time a hfile can be kept after it was moved to the archive
-  private long ttl;
 
   @Override
-  public void setConf(Configuration conf) {
-    this.ttl = conf.getLong(TTL_CONF_KEY, DEFAULT_TTL);
-    super.setConf(conf);
+  protected long getTtlMs(Configuration conf) {
+    return conf.getLong(TTL_CONF_KEY, DEFAULT_TTL);
   }
 
   @Override
-  public boolean isFileDeletable(FileStatus fStat) {
-    long currentTime = EnvironmentEdgeManager.currentTime();
-    long time = fStat.getModificationTime();
-    long life = currentTime - time;
-    if (LOG.isTraceEnabled()) {
-      LOG.trace("HFile life:" + life + ", ttl:" + ttl + ", current:" + currentTime + ", from: "
-          + time);
-    }
-    if (life < 0) {
-      LOG.warn("Found a hfile (" + fStat.getPath() + ") newer than current time (" + currentTime
-          + " < " + time + "), probably a clock skew");
-      return false;
-    }
-    return life > ttl;
+  protected boolean valiateFilename(Path file) {
+    return HFileLink.isBackReferencesDir(file) || HFileLink.isBackReferencesDir(file.getParent()) ||
+      StoreFileInfo.validateStoreFileName(file.getName());
   }
 }