HBASE-25393 Support split and merge region with direct insert into CF…

hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/DirectStoreMergeRegionsStrategy.java

@@ -0,0 +1,105 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.assignment;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.io.hfile.CacheConfig;
+import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
+import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
+import org.apache.hadoop.hbase.regionserver.HStoreFile;
+import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
+import org.apache.yetus.audience.InterfaceAudience;
+
+import java.io.IOException;
+import java.util.Collection;
+
+/**
+ * <code></code>MergeRegionStrategy</code> implementation to be used in combination with
+ * <code>PersistedStoreEngine</code> to avoid renames when merging regions.
+ *
+ * To use it, define the following properties under master configuration:
+ * 1) <property>
+ *      <name>hbase.hregion.file.write.strategy</name>
+ *      <value>org.apache.hadoop.hbase.regionserver.DirectStoreFSWriteStrategy</value>
+ *    </property>
+ * 2) <property>
+ *      <name>hbase.hregion.merge.strategy</name>
+ *      <value>org.apache.hadoop.hbase.master.assignment.DirectStoreMergeRegionsStrategy</value>
+ *    </property>
+ *
+ * This will create the resulting merging region directory straight under the table dir, instead of
+ * creating it under the temporary ".merges" dir as done by the default implementation.
+ *
+ *
+ */
+@InterfaceAudience.Private
+public class DirectStoreMergeRegionsStrategy extends MergeRegionsStrategy {
+
+  /**
+   * Inner logic of creating a merging region, this relies on DirectStoreMergeRegionsStrategy to
+   * return the actual paths where to create the new region (avoiding temp dirs and subsequent
+   * renames).
+   * @param env the MasterProcedureEnv wrapping several meta information required.
+   * @param fs the FileSystem instance to write the region directory.
+   * @param regionsToMerge array of RegionInfo representing the regions being merged.
+   * @param tableDir Path instance for the table dir.
+   * @param mergedRegion the resulting merging region.
+   * @return HRegionFileSystem for the resulting merging region.
+   * @throws IOException if any error occurs while creating the region dir.
+   */
+  @Override
+  public HRegionFileSystem innerMergeRegions(MasterProcedureEnv env, FileSystem fs,
+      RegionInfo[] regionsToMerge, Path tableDir, RegionInfo mergedRegion) throws IOException {
+    //creates the resulting merge region dir directly under the table directory, instead of
+    //the temp ".merges" dir
+    HRegionFileSystem mergeRegionFs = HRegionFileSystem.createRegionOnFileSystem(
+      env.getMasterConfiguration(), fs, tableDir, mergedRegion);
+    mergeRegionFs.createMergesDir();
+    for (RegionInfo ri: regionsToMerge) {
+      HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem(
+        env.getMasterConfiguration(), fs, tableDir, ri, false);
+      mergeStoreFiles(env, regionFs, mergeRegionFs, mergedRegion);
+    }
+    return mergeRegionFs;
+  }
+
+  private void mergeStoreFiles(MasterProcedureEnv env, HRegionFileSystem regionFs,
+      HRegionFileSystem mergeRegionFs, RegionInfo mergedRegion) throws IOException {
+    final TableDescriptor htd = env.getMasterServices().getTableDescriptors()
+      .get(mergedRegion.getTable());
+    for (ColumnFamilyDescriptor hcd : htd.getColumnFamilies()) {
+      String family = hcd.getNameAsString();
+      final Collection<StoreFileInfo> storeFiles = regionFs.getStoreFiles(family);
+      if (storeFiles != null && storeFiles.size() > 0) {
+        for (StoreFileInfo storeFileInfo : storeFiles) {
+          // Create reference file(s) to parent region file here in mergedDir.
+          // As this procedure is running on master, use CacheConfig.DISABLED means
+          // don't cache any block.
+          mergeRegionFs.mergeStoreFile(regionFs.getRegionInfo(), family,
+            new HStoreFile(storeFileInfo, hcd.getBloomFilterType(), CacheConfig.DISABLED),
+            mergeRegionFs.getMergesDir());
+        }
+      }
+    }
+  }
+
+}

hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/MergeRegionsStrategy.java

@@ -0,0 +1,99 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.assignment;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.master.MasterFileSystem;
+import org.apache.hadoop.hbase.master.RegionState;
+import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
+import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
+import org.apache.hadoop.hbase.util.CommonFSUtils;
+import org.apache.yetus.audience.InterfaceAudience;
+
+import java.io.IOException;
+
+/**
+ * Region merge directory creation strategy to decouple create dir logic from
+ * <code></code>MergeTableRegionsProcedure</code> and allow for plugable behaviour.
+ */
+@InterfaceAudience.Private
+public abstract class MergeRegionsStrategy {
+
+  /**
+   * Creates the resulting merging region dir and files in the file system, then updates
+   * meta table information for the given region. Specific logic on where in the files system to
+   * create the region structure is delegated to <method>innerMergeRegions</method> and the
+   * actual <code>HRegionFileSystemWriteStrategy</code> implementation.
+   * @param env the MasterProcedureEnv wrapping several meta information required.
+   * @param regionsToMerge array of RegionInfo representing the regions being merged.
+   * @param mergedRegion the resulting merging region.
+   * @throws IOException if any error occurs while creating the region dir.
+   */
+  public void createMergedRegion(MasterProcedureEnv env, RegionInfo[] regionsToMerge,
+    RegionInfo mergedRegion) throws IOException {
+    final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
+    final Path tabledir = CommonFSUtils.getTableDir(mfs.getRootDir(), regionsToMerge[0].getTable());
+    final FileSystem fs = mfs.getFileSystem();
+    HRegionFileSystem mergeRegionFs = innerMergeRegions(env, fs, regionsToMerge,
+      tabledir, mergedRegion);
+    assert mergeRegionFs != null;
+    mergeRegionFs.commitMergedRegion(mergedRegion);
+    // Prepare to create merged regions
+    env.getAssignmentManager().getRegionStates().
+      getOrCreateRegionStateNode(mergedRegion).setState(RegionState.State.MERGING_NEW);
+  }
+
+  /**
+   * Should define specific logic about where in the file system the region structure should be
+   * created.
+   * @param env the MasterProcedureEnv wrapping several meta information required.
+   * @param fs the FileSystem instance to write the region directory.
+   * @param regionsToMerge array of RegionInfo representing the regions being merged.
+   * @param tableDir Path instance for the table dir.
+   * @param mergedRegion the resulting merging region.
+   * @return HRegionFileSystem for the resulting merging region.
+   * @throws IOException if any error occurs while creating the region dir.
+   */
+  abstract protected HRegionFileSystem innerMergeRegions(MasterProcedureEnv env, FileSystem fs,
+    RegionInfo[] regionsToMerge, Path tableDir, RegionInfo mergedRegion) throws IOException;
+
+
+  /**
+   * Clean up a merged region on rollback after failure.
+   * @param env the MasterProcedureEnv wrapping several meta information required.
+   * @param regionsToMerge array of RegionInfo representing the regions being merged.
+   * @param mergedRegion the resulting merging region.
+   * @throws IOException if any error occurs while creating the region dir.
+   */
+  public void cleanupMergedRegion(MasterProcedureEnv env, RegionInfo[] regionsToMerge,
+    RegionInfo mergedRegion) throws IOException {
+    final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
+    TableName tn = regionsToMerge[0].getTable();
+    final Path tableDir = CommonFSUtils.getTableDir(mfs.getRootDir(), tn);
+    final FileSystem fs = mfs.getFileSystem();
+    // See createMergedRegion above where we specify the merge dir as being in the
+    // FIRST merge parent region.
+    HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem(
+      env.getMasterConfiguration(), fs, tableDir, regionsToMerge[0], false);
+    regionFs.cleanupMergedRegion(mergedRegion);
+  }
+
+}