Skip to content

HBASE-23652 Move the unsupported procedure type check before migratin… #1018

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Apache9 merged 1 commit into from
Jan 16, 2020
Merged

HBASE-23652 Move the unsupported procedure type check before migratin… #1018

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 1 addition & 48 deletions hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,14 +98,11 @@
import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
import org.apache.hadoop.hbase.log.HBaseMarkers;
import org.apache.hadoop.hbase.master.MasterRpcServices.BalanceSwitchMode;
import org.apache.hadoop.hbase.master.assignment.AssignProcedure;
import org.apache.hadoop.hbase.master.assignment.AssignmentManager;
import org.apache.hadoop.hbase.master.assignment.MergeTableRegionsProcedure;
import org.apache.hadoop.hbase.master.assignment.MoveRegionProcedure;
import org.apache.hadoop.hbase.master.assignment.RegionStateNode;
import org.apache.hadoop.hbase.master.assignment.RegionStates;
import org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure;
import org.apache.hadoop.hbase.master.assignment.UnassignProcedure;
import org.apache.hadoop.hbase.master.balancer.BalancerChore;
import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer;
import org.apache.hadoop.hbase.master.balancer.ClusterStatusChore;
Expand Down Expand Up @@ -135,7 +132,6 @@
import org.apache.hadoop.hbase.master.procedure.ModifyTableProcedure;
import org.apache.hadoop.hbase.master.procedure.ProcedurePrepareLatch;
import org.apache.hadoop.hbase.master.procedure.ProcedureSyncWait;
import org.apache.hadoop.hbase.master.procedure.RecoverMetaProcedure;
import org.apache.hadoop.hbase.master.procedure.ReopenTableRegionsProcedure;
import org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure;
import org.apache.hadoop.hbase.master.procedure.TruncateTableProcedure;
Expand Down Expand Up @@ -225,7 +221,6 @@
import org.slf4j.LoggerFactory;

import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableSet;
import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
import org.apache.hbase.thirdparty.com.google.common.collect.Maps;

Expand Down Expand Up @@ -833,45 +828,6 @@ protected void initializeZKBasedSystemTrackers()
this.mpmHost.initialize(this, this.metricsMaster);
}

private static final ImmutableSet<Class<? extends Procedure>> UNSUPPORTED_PROCEDURES =
ImmutableSet.of(RecoverMetaProcedure.class, AssignProcedure.class, UnassignProcedure.class,
MoveRegionProcedure.class);

/**
* In HBASE-20811, we have introduced a new TRSP to assign/unassign/move regions, and it is
* incompatible with the old AssignProcedure/UnassignProcedure/MoveRegionProcedure. So we need to
* make sure that there are none these procedures when upgrading. If there are, the master will
* quit, you need to go back to the old version to finish these procedures first before upgrading.
*/
private void checkUnsupportedProcedure(
Map<Class<? extends Procedure>, List<Procedure<MasterProcedureEnv>>> procsByType)
throws HBaseIOException {
// Confirm that we do not have unfinished assign/unassign related procedures. It is not easy to
// support both the old assign/unassign procedures and the new TransitRegionStateProcedure as
// there will be conflict in the code for AM. We should finish all these procedures before
// upgrading.
for (Class<? extends Procedure> clazz : UNSUPPORTED_PROCEDURES) {
List<Procedure<MasterProcedureEnv>> procs = procsByType.get(clazz);
if (procs != null) {
LOG.error(
"Unsupported procedure type {} found, please rollback your master to the old" +
" version to finish them, and then try to upgrade again. The full procedure list: {}",
clazz, procs);
throw new HBaseIOException("Unsupported procedure type " + clazz + " found");
}
}
// A special check for SCP, as we do not support RecoverMetaProcedure any more so we need to
// make sure that no one will try to schedule it but SCP does have a state which will schedule
// it.
if (procsByType.getOrDefault(ServerCrashProcedure.class, Collections.emptyList()).stream()
.map(p -> (ServerCrashProcedure) p).anyMatch(ServerCrashProcedure::isInRecoverMetaState)) {
LOG.error("At least one ServerCrashProcedure is going to schedule a RecoverMetaProcedure," +
" which is not supported any more. Please rollback your master to the old version to" +
" finish them, and then try to upgrade again.");
throw new HBaseIOException("Unsupported procedure state found for ServerCrashProcedure");
}
}

// Will be overriden in test to inject customized AssignmentManager
@VisibleForTesting
protected AssignmentManager createAssignmentManager(MasterServices master) {
Expand Down Expand Up @@ -965,13 +921,10 @@ private void finishActiveMasterInitialization(MonitoredTask status) throws IOExc
this.splitWALManager = new SplitWALManager(this);
}
createProcedureExecutor();
@SuppressWarnings("rawtypes")
Map<Class<? extends Procedure>, List<Procedure<MasterProcedureEnv>>> procsByType =
Map<Class<?>, List<Procedure<MasterProcedureEnv>>> procsByType =
procedureExecutor.getActiveProceduresNoCopy().stream()
.collect(Collectors.groupingBy(p -> p.getClass()));

checkUnsupportedProcedure(procsByType);

// Create Assignment Manager
this.assignmentManager = createAssignmentManager(this);
this.assignmentManager.start();
Expand Down
8 changes: 8 additions & 0 deletions hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignProcedure.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@
import org.apache.hadoop.hbase.procedure2.RemoteProcedureDispatcher.RemoteOperation;
import org.apache.yetus.audience.InterfaceAudience;

import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;

import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.AssignRegionStateData;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.RegionTransitionState;
Expand Down Expand Up @@ -140,4 +142,10 @@ public void toStringClassDetails(StringBuilder sb) {
protected ProcedureMetrics getProcedureMetrics(MasterProcedureEnv env) {
return env.getAssignmentManager().getAssignmentManagerMetrics().getAssignProcMetrics();
}

@VisibleForTesting
@Override
public void setProcId(long procId) {
super.setProcId(procId);
}
}
3 changes: 2 additions & 1 deletion ...er/src/main/java/org/apache/hadoop/hbase/master/assignment/RegionTransitionProcedure.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,8 @@ public RegionInfo getRegionInfo() {
return regionInfo;
}

protected void setRegionInfo(final RegionInfo regionInfo) {
@VisibleForTesting
public void setRegionInfo(final RegionInfo regionInfo) {
this.regionInfo = regionInfo;
}

Expand Down
77 changes: 70 additions & 7 deletions ...r/src/main/java/org/apache/hadoop/hbase/procedure2/store/region/RegionProcedureStore.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,10 @@
import java.io.UncheckedIOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.commons.lang3.mutable.MutableLong;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
Expand All @@ -45,6 +48,11 @@
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.log.HBaseMarkers;
import org.apache.hadoop.hbase.master.assignment.AssignProcedure;
import org.apache.hadoop.hbase.master.assignment.MoveRegionProcedure;
import org.apache.hadoop.hbase.master.assignment.UnassignProcedure;
import org.apache.hadoop.hbase.master.procedure.RecoverMetaProcedure;
import org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure;
import org.apache.hadoop.hbase.procedure2.Procedure;
import org.apache.hadoop.hbase.procedure2.ProcedureUtil;
import org.apache.hadoop.hbase.procedure2.store.LeaseRecovery;
Expand All @@ -65,6 +73,7 @@
import org.slf4j.LoggerFactory;

import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableSet;
import org.apache.hbase.thirdparty.com.google.common.math.IntMath;

import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos;
Expand Down Expand Up @@ -298,6 +307,46 @@ private HRegion open(Configuration conf, FileSystem fs, Path rootDir) throws IOE
null);
}

@SuppressWarnings("deprecation")
private static final ImmutableSet<Class<?>> UNSUPPORTED_PROCEDURES =
ImmutableSet.of(RecoverMetaProcedure.class, AssignProcedure.class, UnassignProcedure.class,
MoveRegionProcedure.class);

/**
* In HBASE-20811, we have introduced a new TRSP to assign/unassign/move regions, and it is
* incompatible with the old AssignProcedure/UnassignProcedure/MoveRegionProcedure. So we need to
* make sure that there are none these procedures when upgrading. If there are, the master will
* quit, you need to go back to the old version to finish these procedures first before upgrading.
*/
private void checkUnsupportedProcedure(Map<Class<?>, List<Procedure<?>>> procsByType)
throws HBaseIOException {
// Confirm that we do not have unfinished assign/unassign related procedures. It is not easy to
// support both the old assign/unassign procedures and the new TransitRegionStateProcedure as
// there will be conflict in the code for AM. We should finish all these procedures before
// upgrading.
for (Class<?> clazz : UNSUPPORTED_PROCEDURES) {
List<Procedure<?>> procs = procsByType.get(clazz);
if (procs != null) {
LOG.error("Unsupported procedure type {} found, please rollback your master to the old" +
" version to finish them, and then try to upgrade again." +
" See https://hbase.apache.org/book.html#upgrade2.2 for more details." +
" The full procedure list: {}", clazz, procs);
throw new HBaseIOException("Unsupported procedure type " + clazz + " found");
}
}
// A special check for SCP, as we do not support RecoverMetaProcedure any more so we need to
// make sure that no one will try to schedule it but SCP does have a state which will schedule
// it.
if (procsByType.getOrDefault(ServerCrashProcedure.class, Collections.emptyList()).stream()
Copy link
Member

@ndimiduk ndimiduk Jan 14, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: this if check reads a bit easier with some newlines

if (procsByType.getOrDefault(ServerCrashProcedure.class, Collections.emptyList())
       .stream()
       .map(p -> (ServerCrashProcedure) p)
       .anyMatch(ServerCrashProcedure::isInRecoverMetaState)) {

Copy link
Contributor Author

@Apache9 Apache9 Jan 16, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just use eclipse formatter and it will make them in the same line if possible...

Copy link
Member

@ndimiduk ndimiduk Jan 16, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

::nod::

I disagree with eclipse formatter. I don't think it's been updated since java8.

.map(p -> (ServerCrashProcedure) p).anyMatch(ServerCrashProcedure::isInRecoverMetaState)) {
LOG.error("At least one ServerCrashProcedure is going to schedule a RecoverMetaProcedure," +
" which is not supported any more. Please rollback your master to the old version to" +
" finish them, and then try to upgrade again." +
" See https://hbase.apache.org/book.html#upgrade2.2 for more details.");
throw new HBaseIOException("Unsupported procedure state found for ServerCrashProcedure");
}
}

@SuppressWarnings("deprecation")
private void tryMigrate(FileSystem fs) throws IOException {
Configuration conf = server.getConfiguration();
Expand All @@ -311,7 +360,8 @@ private void tryMigrate(FileSystem fs) throws IOException {
store.start(numThreads);
store.recoverLease();
MutableLong maxProcIdSet = new MutableLong(-1);
MutableLong maxProcIdFromProcs = new MutableLong(-1);
List<Procedure<?>> procs = new ArrayList<>();
Map<Class<?>, List<Procedure<?>>> activeProcsByType = new HashMap<>();
store.load(new ProcedureLoader() {

@Override
Expand All @@ -321,16 +371,13 @@ public void setMaxProcId(long maxProcId) {

@Override
public void load(ProcedureIterator procIter) throws IOException {
long procCount = 0;
while (procIter.hasNext()) {
Procedure<?> proc = procIter.next();
update(proc);
procCount++;
if (proc.getProcId() > maxProcIdFromProcs.longValue()) {
maxProcIdFromProcs.setValue(proc.getProcId());
procs.add(proc);
if (!proc.isFinished()) {
activeProcsByType.computeIfAbsent(proc.getClass(), k -> new ArrayList<>()).add(proc);
}
}
LOG.info("Migrated {} procedures", procCount);
}

@Override
Expand All @@ -347,6 +394,22 @@ public void handleCorrupted(ProcedureIterator procIter) throws IOException {
}
}
});

// check whether there are unsupported procedures, this could happen when we are migrating from
// 2.1-. We used to do this in HMaster, after loading all the procedures from procedure store,
// but here we have to do it before migrating, otherwise, if we find some unsupported
// procedures, the users can not go back to 2.1 to finish them any more, as all the data are now
// in the new region based procedure store, which is not supported in 2.1-.
checkUnsupportedProcedure(activeProcsByType);

MutableLong maxProcIdFromProcs = new MutableLong(-1);
for (Procedure<?> proc : procs) {
update(proc);
if (proc.getProcId() > maxProcIdFromProcs.longValue()) {
maxProcIdFromProcs.setValue(proc.getProcId());
}
}
LOG.info("Migrated {} existing procedures from the old storage format.", procs.size());
LOG.info("The WALProcedureStore max pid is {}, and the max pid of all loaded procedures is {}",
maxProcIdSet.longValue(), maxProcIdFromProcs.longValue());
// Theoretically, the maxProcIdSet should be greater than or equal to maxProcIdFromProcs, but
Expand Down
22 changes: 22 additions & 0 deletions ...va/org/apache/hadoop/hbase/procedure2/store/region/TestRegionProcedureStoreMigration.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@
*/
package org.apache.hadoop.hbase.procedure2.store.region;

import static org.hamcrest.CoreMatchers.startsWith;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.fail;
Expand All @@ -32,6 +34,10 @@
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseCommonTestingUtility;
import org.apache.hadoop.hbase.HBaseIOException;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.RegionInfoBuilder;
import org.apache.hadoop.hbase.master.assignment.AssignProcedure;
import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.LoadCounter;
import org.apache.hadoop.hbase.procedure2.store.LeaseRecovery;
import org.apache.hadoop.hbase.procedure2.store.ProcedureStore.ProcedureIterator;
Expand Down Expand Up @@ -140,4 +146,20 @@ public void handleCorrupted(ProcedureIterator procIter) throws IOException {
// make sure the old proc wal directory has been deleted.
assertFalse(fs.exists(oldProcWALDir));
}

@Test
public void testMigrateWithUnsupportedProcedures() throws IOException {
AssignProcedure assignProc = new AssignProcedure();
assignProc.setProcId(1L);
assignProc.setRegionInfo(RegionInfoBuilder.newBuilder(TableName.valueOf("table")).build());
walStore.insert(assignProc, null);
walStore.stop(true);

try {
store = RegionProcedureStoreTestHelper.createStore(htu.getConfiguration(), new LoadCounter());
fail("Should fail since AssignProcedure is not supported");
} catch (HBaseIOException e) {
assertThat(e.getMessage(), startsWith("Unsupported"));
}
}
}