HBASE-27028 Add a shell command for flushing master local region

2005hithlj · 2005hithlj · commit ec200e7fa196 · 2022-05-24T12:06:59.000+08:00
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
@@ -2539,4 +2539,9 @@ default BalanceResponse balanceRSGroup(String groupName) throws IOException {
    */
   List<LogEntry> getLogEntries(Set<ServerName> serverNames, String logType, ServerType serverType,
     int limit, Map<String, Object> filterParams) throws IOException;
+
+  /**
+   * Flush master local region
+   */
+  void flushMasterStore() throws IOException;
 }
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AdminOverAsyncAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AdminOverAsyncAdmin.java
@@ -1078,4 +1078,9 @@ public List<LogEntry> getLogEntries(Set<ServerName> serverNames, String logType,
     ServerType serverType, int limit, Map<String, Object> filterParams) throws IOException {
     return get(admin.getLogEntries(serverNames, logType, serverType, limit, filterParams));
   }
+
+  @Override
+  public void flushMasterStore() throws IOException {
+    get(admin.flushMasterStore());
+  }
 }
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java
@@ -1785,4 +1785,9 @@ default CompletableFuture<BalanceResponse> balanceRSGroup(String groupName) {
    */
   CompletableFuture<List<LogEntry>> getLogEntries(Set<ServerName> serverNames, String logType,
     ServerType serverType, int limit, Map<String, Object> filterParams);
+
+  /**
+   * Flush master local region
+   */
+  CompletableFuture<Void> flushMasterStore();
 }
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java
@@ -949,4 +949,9 @@ public CompletableFuture<List<LogEntry>> getLogEntries(Set<ServerName> serverNam
     String logType, ServerType serverType, int limit, Map<String, Object> filterParams) {
     return wrap(rawAdmin.getLogEntries(serverNames, logType, serverType, limit, filterParams));
   }
+
+  @Override
+  public CompletableFuture<Void> flushMasterStore() {
+    return wrap(rawAdmin.flushMasterStore());
+  }
 }
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java
@@ -178,6 +178,8 @@
 import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.EnableTableResponse;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ExecProcedureRequest;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ExecProcedureResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.FlushMasterStoreRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.FlushMasterStoreResponse;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.GetClusterStatusRequest;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.GetClusterStatusResponse;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.GetCompletedSnapshotsRequest;
@@ -4280,4 +4282,14 @@ public CompletableFuture<List<LogEntry>> getLogEntries(Set<ServerName> serverNam
         return CompletableFuture.completedFuture(Collections.emptyList());
     }
   }
+
+  @Override
+  public CompletableFuture<Void> flushMasterStore() {
+    FlushMasterStoreRequest.Builder request = FlushMasterStoreRequest.newBuilder();
+    return this.<Void> newMasterCaller()
+      .action(((controller, stub) -> this.<FlushMasterStoreRequest, FlushMasterStoreResponse,
+        Void> call(controller, stub, request.build(),
+          (s, c, req, done) -> s.flushMasterStore(c, req, done), resp -> null)))
+      .call();
+  }
 }
diff --git a/hbase-protocol-shaded/src/main/protobuf/server/master/Master.proto b/hbase-protocol-shaded/src/main/protobuf/server/master/Master.proto
@@ -758,6 +758,9 @@ message ModifyColumnStoreFileTrackerResponse {
   optional uint64 proc_id = 1;
 }
 
+message FlushMasterStoreRequest {}
+message FlushMasterStoreResponse {}
+
 service MasterService {
   /** Used by the client to get the number of regions that have received the updated schema */
   rpc GetSchemaAlterStatus(GetSchemaAlterStatusRequest)
@@ -1197,6 +1200,9 @@ service MasterService {
 
   rpc ModifyColumnStoreFileTracker(ModifyColumnStoreFileTrackerRequest)
     returns(ModifyColumnStoreFileTrackerResponse);
+
+  rpc FlushMasterStore(FlushMasterStoreRequest)
+    returns(FlushMasterStoreResponse);
 }
 
 // HBCK Service definitions.
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -4201,6 +4201,11 @@ public List<HRegionLocation> getMetaLocations() {
     return metaRegionLocationCache.getMetaRegionLocations();
   }
 
+  @Override
+  public void flushMasterStore() {
+    masterRegion.rpcRequestFlush();
+  }
+
   public Collection<ServerName> getLiveRegionServers() {
     return regionServerTracker.getRegionServers();
   }
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
@@ -229,6 +229,8 @@
 import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.ExecProcedureResponse;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.FixMetaRequest;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.FixMetaResponse;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.FlushMasterStoreRequest;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.FlushMasterStoreResponse;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.GetClusterStatusRequest;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.GetClusterStatusResponse;
 import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.GetCompletedSnapshotsRequest;
@@ -3468,4 +3470,12 @@ public ReplicateWALEntryResponse replicateToReplica(RpcController controller,
     ReplicateWALEntryRequest request) throws ServiceException {
     throw new ServiceException(new DoNotRetryIOException("Unsupported method on master"));
   }
+
+  @Override
+  public FlushMasterStoreResponse flushMasterStore(RpcController controller,
+    FlushMasterStoreRequest request) throws ServiceException {
+    rpcPreCheck("flushMasterStore");
+    server.flushMasterStore();
+    return FlushMasterStoreResponse.newBuilder().build();
+  }
 }
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
@@ -510,4 +510,9 @@ boolean normalizeRegions(final NormalizeTableFilterParams ntfp, final boolean is
    * We need to get this in MTP to tell the syncer the new meta replica count.
    */
   MetaLocationSyncer getMetaLocationSyncer();
+
+  /**
+   * Flush master local region
+   */
+  void flushMasterStore();
 }
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegion.java
@@ -154,6 +154,10 @@ public RegionScanner getRegionScanner(Scan scan) throws IOException {
     return region.getScanner(scan);
   }
 
+  public void rpcRequestFlush() {
+    flusherAndCompactor.requestFlush();
+  }
+
   public FlushResult flush(boolean force) throws IOException {
     return region.flush(force);
   }
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegionFlusherAndCompactor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/region/MasterRegionFlusherAndCompactor.java
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.hbase.master.region;
 
+import com.google.errorprone.annotations.RestrictedApi;
 import java.io.Closeable;
 import java.io.IOException;
 import java.util.concurrent.ExecutorService;
@@ -263,6 +264,12 @@ void requestFlush() {
     }
   }
 
+  @RestrictedApi(explanation = "Should only be called in tests", link = "",
+      allowedOnPath = ".*/src/test/.*")
+  public boolean getFlushRequest() {
+    return flushRequest;
+  }
+
   @Override
   public void close() {
     closed = true;
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java
@@ -503,6 +503,10 @@ public MetaLocationSyncer getMetaLocationSyncer() {
     return null;
   }
 
+  @Override
+  public void flushMasterStore() {
+  }
+
   @Override
   public long modifyTableStoreFileTracker(TableName tableName, String dstSFT, long nonceGroup,
     long nonce) throws IOException {
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/region/TestMasterRegionFlush.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/region/TestMasterRegionFlush.java
@@ -33,6 +33,7 @@
 import org.apache.hadoop.hbase.Abortable;
 import org.apache.hadoop.hbase.HBaseClassTestRule;
 import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HBaseTestingUtil;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.Waiter;
 import org.apache.hadoop.hbase.client.RegionInfoBuilder;
@@ -148,6 +149,16 @@ public void testPeriodicalFlush() throws InterruptedException {
     assertEquals(1, flushCalled.get());
     Thread.sleep(1000);
     assertEquals(2, flushCalled.get());
+  }
 
+  @Test
+  public void testRpcRequestFlush() throws Exception {
+    // Test for HBASE-27028
+    HBaseTestingUtil htu = new HBaseTestingUtil();
+    htu.startMiniCluster(1);
+    htu.getConnection().getAdmin().flushMasterStore();
+    assertTrue(htu.getMiniHBaseCluster().getMaster().getMasterRegion().flusherAndCompactor
+      .getFlushRequest());
+    htu.shutdownMiniCluster();
   }
 }
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestMetaRegionReplicaReplication.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestMetaRegionReplicaReplication.java
@@ -469,7 +469,7 @@ public void testHBaseMetaReplicaGets() throws Exception {
 
       RegionLocator locator = tableForGet.getRegionLocator();
 
-      for (int j = 0; j < numOfMetaReplica * 3; j ++) {
+      for (int j = 0; j < numOfMetaReplica * 3; j++) {
         locator.getAllRegionLocations();
       }
 
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdmin.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdmin.java
@@ -934,4 +934,9 @@ public Future<Void> modifyTableStoreFileTrackerAsync(TableName tableName, String
     throws IOException {
     return admin.modifyTableStoreFileTrackerAsync(tableName, dstSFT);
   }
+
+  @Override
+  public void flushMasterStore() throws IOException {
+    admin.flushMasterStore();
+  }
 }
diff --git a/hbase-shell/src/main/ruby/hbase/admin.rb b/hbase-shell/src/main/ruby/hbase/admin.rb
@@ -1867,11 +1867,17 @@ def modify_table_sft(tableName, sft)
       @admin.modifyTableStoreFileTracker(tableName, sft)
     end
 
-     #----------------------------------------------------------------------------------------------
+    #----------------------------------------------------------------------------------------------
     # Change table column family's sft
     def modify_table_family_sft(tableName, family_bytes, sft)
       @admin.modifyColumnFamilyStoreFileTracker(tableName, family_bytes, sft)
     end
+
+    #----------------------------------------------------------------------------------------------
+    # Flush master local region
+    def flush_master_store()
+      @admin.flushMasterStore()
+    end
   end
   # rubocop:enable Metrics/ClassLength
 end
diff --git a/hbase-shell/src/main/ruby/shell.rb b/hbase-shell/src/main/ruby/shell.rb
@@ -446,6 +446,7 @@ def self.exception_handler(hide_traceback)
     compact
     compaction_switch
     flush
+    flush_master_store
     get_balancer_decisions
     get_balancer_rejections
     get_slowlog_responses
diff --git a/hbase-shell/src/main/ruby/shell/commands/flush_master_store.rb b/hbase-shell/src/main/ruby/shell/commands/flush_master_store.rb
@@ -0,0 +1,37 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module Shell
+  module Commands
+    class FlushMasterStore < Command
+      def help
+        <<-EOF
+Flush master local region.
+For example:
+
+  hbase> flush_master_store
+EOF
+      end
+
+      def command()
+        admin.flush_master_store()
+      end
+    end
+  end
+end
diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java
@@ -1309,4 +1309,9 @@ public Future<Void> modifyTableStoreFileTrackerAsync(TableName tableName, String
     throw new NotImplementedException(
       "modifyTableStoreFileTrackerAsync not supported in ThriftAdmin");
   }
+
+  @Override
+  public void flushMasterStore() throws IOException {
+    throw new NotImplementedException("flushMasterStore not supported in ThriftAdmin");
+  }
 }

Original file line number	Diff line number	Diff line change
`@@ -1078,4 +1078,9 @@ public List<LogEntry> getLogEntries(Set<ServerName> serverNames, String logType,`
`1078`	`1078`	`ServerType serverType, int limit, Map<String, Object> filterParams) throws IOException {`
`1079`	`1079`	`return get(admin.getLogEntries(serverNames, logType, serverType, limit, filterParams));`
`1080`	`1080`	`}`
	`1081`	`+`
	`1082`	`+ @Override`
	`1083`	`+ public void flushMasterStore() throws IOException {`
	`1084`	`+ get(admin.flushMasterStore());`
	`1085`	`+ }`
`1081`	`1086`	`}`
Original file line number	Diff line number	Diff line change
`@@ -949,4 +949,9 @@ public CompletableFuture<List<LogEntry>> getLogEntries(Set<ServerName> serverNam`
`949`	`949`	`String logType, ServerType serverType, int limit, Map<String, Object> filterParams) {`
`950`	`950`	`return wrap(rawAdmin.getLogEntries(serverNames, logType, serverType, limit, filterParams));`
`951`	`951`	`}`
	`952`	`+`
	`953`	`+ @Override`
	`954`	`+ public CompletableFuture<Void> flushMasterStore() {`
	`955`	`+ return wrap(rawAdmin.flushMasterStore());`
	`956`	`+ }`
`952`	`957`	`}`
Original file line number	Diff line number	Diff line change
`@@ -4201,6 +4201,11 @@ public List<HRegionLocation> getMetaLocations() {`
`4201`	`4201`	`return metaRegionLocationCache.getMetaRegionLocations();`
`4202`	`4202`	`}`
`4203`	`4203`
	`4204`	`+ @Override`
	`4205`	`+ public void flushMasterStore() {`
	`4206`	`+ masterRegion.rpcRequestFlush();`
	`4207`	`+ }`
	`4208`	`+`
`4204`	`4209`	`public Collection<ServerName> getLiveRegionServers() {`
`4205`	`4210`	`return regionServerTracker.getRegionServers();`
`4206`	`4211`	`}`
Original file line number	Diff line number	Diff line change
`@@ -154,6 +154,10 @@ public RegionScanner getRegionScanner(Scan scan) throws IOException {`
`154`	`154`	`return region.getScanner(scan);`
`155`	`155`	`}`
`156`	`156`
	`157`	`+ public void rpcRequestFlush() {`
	`158`	`+ flusherAndCompactor.requestFlush();`
	`159`	`+ }`
	`160`	`+`
`157`	`161`	`public FlushResult flush(boolean force) throws IOException {`
`158`	`162`	`return region.flush(force);`
`159`	`163`	`}`