HBase 22537

hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/TimeoutExecutorThread.java

@@ -78,6 +78,10 @@ public void add(InlineChore chore) {
   }
 
   public void add(Procedure<TEnvironment> procedure) {
+    // On the assert, we expect WAITING_TIMEOUT but timing could make it so lock gets released by
+    // time we get here and in those cases the state could be back to RUNNABLE. Let it
+    assert procedure.getState() == ProcedureState.WAITING_TIMEOUT ||
+      procedure.getState() == ProcedureState.RUNNABLE;
     LOG.info("ADDED {}; timeout={}, timestamp={}", procedure, procedure.getTimeout(),
       procedure.getTimeoutTimestamp());
     queue.add(new DelayedProcedure<>(procedure));

hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/UnassignProcedure.java

@@ -231,6 +231,7 @@ protected void finishTransition(final MasterProcedureEnv env, final RegionStateN
     } else {
       // Remove from in-memory states
       am.getRegionStates().deleteRegion(regionInfo);
+      am.getRegionStates().removeRegionFromServer(regionNode.getRegionLocation(), regionNode);
       env.getMasterServices().getServerManager().removeRegion(regionInfo);
       FavoredNodesManager fnm = env.getMasterServices().getFavoredNodesManager();
       if (fnm != null) {

hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSplitOrMergeStatus.java

@@ -30,7 +30,15 @@
 import org.apache.hadoop.hbase.HBaseClassTestRule;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.master.assignment.AssignmentTestingUtil;
+import org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure;
+import org.apache.hadoop.hbase.master.procedure.DeleteTableProcedure;
+import org.apache.hadoop.hbase.master.procedure.DisableTableProcedure;
+import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
+import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
+import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
 import org.apache.hadoop.hbase.testclassification.ClientTests;
 import org.apache.hadoop.hbase.testclassification.MediumTests;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -162,6 +170,64 @@ public void testMultiSwitches() throws IOException {
     admin.close();
   }
 
+  @Test
+  public void testSplitRegionReplicaRitRecovery() throws Exception {
+    int startRowNum = 11;
+    int rowCount = 60;
+    final TableName tableName = TableName.valueOf(name.getMethodName());
+    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
+    TEST_UTIL.getAdmin().createTable(
+      TableDescriptorBuilder.newBuilder(tableName)
+        .setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY)).setRegionReplication(2)
+        .build());
+    TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
+    ServerName serverName =
+      RegionReplicaTestHelper.getRSCarryingReplica(TEST_UTIL, tableName, 1).get();
+    List<RegionInfo> regions = TEST_UTIL.getAdmin().getRegions(tableName);
+    insertData(tableName, startRowNum, rowCount);
+    int splitRowNum = startRowNum + rowCount / 2;
+    byte[] splitKey = Bytes.toBytes("" + splitRowNum);
+    // Split region of the table
+    long procId =
+      procExec.submitProcedure(new SplitTableRegionProcedure(procExec.getEnvironment(), regions
+        .get(0), splitKey));
+    // Wait the completion
+    ProcedureTestingUtility.waitProcedure(procExec, procId);
+    // Disable the table
+    long procId1 =
+      procExec.submitProcedure(new DisableTableProcedure(procExec.getEnvironment(), tableName,
+        false));
+    // Wait the completion
+    ProcedureTestingUtility.waitProcedure(procExec, procId1);
+    //Delete Table
+    long procId2 =
+      procExec.submitProcedure(new DeleteTableProcedure(procExec.getEnvironment(), tableName));
+    // Wait the completion
+    ProcedureTestingUtility.waitProcedure(procExec, procId2);
+    AssignmentTestingUtil.killRs(TEST_UTIL, serverName);
+    Threads.sleepWithoutInterrupt(5000);
+    boolean hasRegionsInTransition =
+      TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates()
+        .hasRegionsInTransition();
+    assertEquals(false, hasRegionsInTransition);
+
+  }
+
+  private ProcedureExecutor<MasterProcedureEnv> getMasterProcedureExecutor() {
+    return TEST_UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor();
+  }
+
+  private void insertData(final TableName tableName, int startRow, int rowCount)
+    throws IOException {
+    Table t = TEST_UTIL.getConnection().getTable(tableName);
+    Put p;
+    for (int i= 0; i < rowCount; i++) {
+      p = new Put(Bytes.toBytes("" + (startRow + i)));
+      p.addColumn(FAMILY, Bytes.toBytes("q1"), Bytes.toBytes(i));
+      t.put(p);
+    }
+  }
+
   private void initSwitchStatus(Admin admin) throws IOException {
     if (!admin.isSplitOrMergeEnabled(MasterSwitchType.SPLIT)) {
       admin.setSplitOrMergeEnabled(true, false, MasterSwitchType.SPLIT);