diff --git a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/TimeoutExecutorThread.java b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/TimeoutExecutorThread.java index 1c97bcccca3c..94d3f65b53be 100644 --- a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/TimeoutExecutorThread.java +++ b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/TimeoutExecutorThread.java @@ -78,6 +78,10 @@ public void add(InlineChore chore) { } public void add(Procedure procedure) { + // On the assert, we expect WAITING_TIMEOUT but timing could make it so lock gets released by + // time we get here and in those cases the state could be back to RUNNABLE. Let it + assert procedure.getState() == ProcedureState.WAITING_TIMEOUT || + procedure.getState() == ProcedureState.RUNNABLE; LOG.info("ADDED {}; timeout={}, timestamp={}", procedure, procedure.getTimeout(), procedure.getTimeoutTimestamp()); queue.add(new DelayedProcedure<>(procedure)); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/UnassignProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/UnassignProcedure.java index 23b2de776049..8ad4b74a28af 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/UnassignProcedure.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/UnassignProcedure.java @@ -231,6 +231,7 @@ protected void finishTransition(final MasterProcedureEnv env, final RegionStateN } else { // Remove from in-memory states am.getRegionStates().deleteRegion(regionInfo); + am.getRegionStates().removeRegionFromServer(regionNode.getRegionLocation(), regionNode); env.getMasterServices().getServerManager().removeRegion(regionInfo); FavoredNodesManager fnm = env.getMasterServices().getFavoredNodesManager(); if (fnm != null) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSplitOrMergeStatus.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSplitOrMergeStatus.java index 11f53a9671f0..98f7106f4704 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSplitOrMergeStatus.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSplitOrMergeStatus.java @@ -30,7 +30,15 @@ import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.master.assignment.AssignmentTestingUtil; +import org.apache.hadoop.hbase.master.assignment.SplitTableRegionProcedure; +import org.apache.hadoop.hbase.master.procedure.DeleteTableProcedure; +import org.apache.hadoop.hbase.master.procedure.DisableTableProcedure; +import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; +import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; +import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; import org.apache.hadoop.hbase.testclassification.ClientTests; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.util.Bytes; @@ -162,6 +170,64 @@ public void testMultiSwitches() throws IOException { admin.close(); } + @Test + public void testSplitRegionReplicaRitRecovery() throws Exception { + int startRowNum = 11; + int rowCount = 60; + final TableName tableName = TableName.valueOf(name.getMethodName()); + final ProcedureExecutor procExec = getMasterProcedureExecutor(); + TEST_UTIL.getAdmin().createTable( + TableDescriptorBuilder.newBuilder(tableName) + .setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY)).setRegionReplication(2) + .build()); + TEST_UTIL.waitUntilAllRegionsAssigned(tableName); + ServerName serverName = + RegionReplicaTestHelper.getRSCarryingReplica(TEST_UTIL, tableName, 1).get(); + List regions = TEST_UTIL.getAdmin().getRegions(tableName); + insertData(tableName, startRowNum, rowCount); + int splitRowNum = startRowNum + rowCount / 2; + byte[] splitKey = Bytes.toBytes("" + splitRowNum); + // Split region of the table + long procId = + procExec.submitProcedure(new SplitTableRegionProcedure(procExec.getEnvironment(), regions + .get(0), splitKey)); + // Wait the completion + ProcedureTestingUtility.waitProcedure(procExec, procId); + // Disable the table + long procId1 = + procExec.submitProcedure(new DisableTableProcedure(procExec.getEnvironment(), tableName, + false)); + // Wait the completion + ProcedureTestingUtility.waitProcedure(procExec, procId1); + //Delete Table + long procId2 = + procExec.submitProcedure(new DeleteTableProcedure(procExec.getEnvironment(), tableName)); + // Wait the completion + ProcedureTestingUtility.waitProcedure(procExec, procId2); + AssignmentTestingUtil.killRs(TEST_UTIL, serverName); + Threads.sleepWithoutInterrupt(5000); + boolean hasRegionsInTransition = + TEST_UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager().getRegionStates() + .hasRegionsInTransition(); + assertEquals(false, hasRegionsInTransition); + + } + + private ProcedureExecutor getMasterProcedureExecutor() { + return TEST_UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor(); + } + + private void insertData(final TableName tableName, int startRow, int rowCount) + throws IOException { + Table t = TEST_UTIL.getConnection().getTable(tableName); + Put p; + for (int i= 0; i < rowCount; i++) { + p = new Put(Bytes.toBytes("" + (startRow + i))); + p.addColumn(FAMILY, Bytes.toBytes("q1"), Bytes.toBytes(i)); + t.put(p); + } + } + private void initSwitchStatus(Admin admin) throws IOException { if (!admin.isSplitOrMergeEnabled(MasterSwitchType.SPLIT)) { admin.setSplitOrMergeEnabled(true, false, MasterSwitchType.SPLIT);