From ec915802fe1bada0d6abf6cfa39f241a33d1e802 Mon Sep 17 00:00:00 2001 From: Viraj Jasani Date: Tue, 29 Jun 2021 15:54:07 +0530 Subject: [PATCH 1/4] HBASE-22923 min version of RegionServer to move system table regions --- .../master/assignment/AssignmentManager.java | 59 ++++++++++++++++++- 1 file changed, 58 insertions(+), 1 deletion(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java index 700c91e735a5..7440f7627a71 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java @@ -163,6 +163,28 @@ public class AssignmentManager { private final RegionStates regionStates = new RegionStates(); private final RegionStateStore regionStateStore; + /** + * Min version to consider for moving system tables regions to higher + * versioned RS. If RS has higher version than rest of the cluster but that + * version is less than this value, we should not move system table regions + * to that RS. If RS has higher version than rest of the cluster but that + * version is greater than or equal to this value, we should move system + * table regions to that RS. This is optional config and default value is + * empty string ({@link #DEFAULT_MIN_VERSION_MOVE_SYS_TABLES_CONFIG}). + * For instance, if we do not want meta region to be moved to RS with higher + * version until that version is >= 2.0.0, then we can configure + * "hbase.min.version.move.system.tables" as "2.0.0". + * When operator uses this config, it should be used with care, meaning + * we should be confident that even if user table regions come to RS with + * higher version (that rest of cluster), it would not cause any + * incompatibility issues. + */ + private final String minVersionToMoveSysTables; + + private static final String MIN_VERSION_MOVE_SYS_TABLES_CONFIG = + "hbase.min.version.move.system.tables"; + private static final String DEFAULT_MIN_VERSION_MOVE_SYS_TABLES_CONFIG = ""; + private final Map> rsReports = new HashMap<>(); private final boolean shouldAssignRegionsWithFavoredNodes; @@ -211,6 +233,8 @@ public AssignmentManager(final MasterServices master) { } else { this.deadMetricChore = null; } + minVersionToMoveSysTables = conf.get(MIN_VERSION_MOVE_SYS_TABLES_CONFIG, + DEFAULT_MIN_VERSION_MOVE_SYS_TABLES_CONFIG); } public void start() throws IOException, KeeperException { @@ -550,7 +574,7 @@ public void checkIfShouldMoveSystemRegionAsync() { List plans = new ArrayList<>(); // TODO: I don't think this code does a good job if all servers in cluster have same // version. It looks like it will schedule unnecessary moves. - for (ServerName server : getExcludedServersForSystemTable()) { + for (ServerName server : getExcludedServersForSystemTableUnlessAllowed()) { if (master.getServerManager().isServerDead(server)) { // TODO: See HBASE-18494 and HBASE-18495. Though getExcludedServersForSystemTable() // considers only online servers, the server could be queued for dead server @@ -2298,6 +2322,39 @@ public List getExcludedServersForSystemTable() { .collect(Collectors.toList()); } + /** + * Get a list of servers that this region can not assign to. + * For system table, we must assign them to a server with highest version. + * This method is same as {@link #getExcludedServersForSystemTable()} with + * the only difference is we can disable this exclusion using config: + * "hbase.min.version.move.system.tables". + * + * @return List of Excluded servers for System table regions. + */ + private List getExcludedServersForSystemTableUnlessAllowed() { + List> serverList = master.getServerManager().getOnlineServersList() + .stream() + .map(s->new Pair<>(s, master.getRegionServerVersion(s))) + .collect(Collectors.toList()); + if (serverList.isEmpty()) { + return Collections.emptyList(); + } + String highestVersion = Collections.max(serverList, + (o1, o2) -> VersionInfo.compareVersion(o1.getSecond(), o2.getSecond())).getSecond(); + if (!DEFAULT_MIN_VERSION_MOVE_SYS_TABLES_CONFIG.equals(minVersionToMoveSysTables)) { + int comparedValue = + VersionInfo.compareVersion(minVersionToMoveSysTables, highestVersion); + if (comparedValue > 0) { + return Collections.emptyList(); + } + } + return serverList.stream() + .filter(pair -> !pair.getSecond().equals(highestVersion)) + .map(Pair::getFirst) + .collect(Collectors.toList()); + } + + MasterServices getMaster() { return master; } From f816f2ef843403c2a0e013ed910140627f8582e9 Mon Sep 17 00:00:00 2001 From: Viraj Jasani Date: Wed, 30 Jun 2021 21:07:48 +0530 Subject: [PATCH 2/4] addressing comments by @apurtell and @bharathv --- .../master/assignment/AssignmentManager.java | 62 +++++++------------ 1 file changed, 22 insertions(+), 40 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java index 7440f7627a71..c0465cd57074 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java @@ -164,20 +164,11 @@ public class AssignmentManager { private final RegionStateStore regionStateStore; /** - * Min version to consider for moving system tables regions to higher - * versioned RS. If RS has higher version than rest of the cluster but that - * version is less than this value, we should not move system table regions - * to that RS. If RS has higher version than rest of the cluster but that - * version is greater than or equal to this value, we should move system - * table regions to that RS. This is optional config and default value is - * empty string ({@link #DEFAULT_MIN_VERSION_MOVE_SYS_TABLES_CONFIG}). - * For instance, if we do not want meta region to be moved to RS with higher - * version until that version is >= 2.0.0, then we can configure - * "hbase.min.version.move.system.tables" as "2.0.0". - * When operator uses this config, it should be used with care, meaning - * we should be confident that even if user table regions come to RS with - * higher version (that rest of cluster), it would not cause any - * incompatibility issues. + * When the operator uses this configuration option, any version between + * the current version and the new value of + * "hbase.min.version.move.system.tables" does not trigger any region movement. + * It is assumed that the configured range of versions do not require special + * handling of moving system table regions to higher versioned RegionServer. */ private final String minVersionToMoveSysTables; @@ -574,7 +565,7 @@ public void checkIfShouldMoveSystemRegionAsync() { List plans = new ArrayList<>(); // TODO: I don't think this code does a good job if all servers in cluster have same // version. It looks like it will schedule unnecessary moves. - for (ServerName server : getExcludedServersForSystemTableUnlessAllowed()) { + for (ServerName server : getExcludedServersForSystemTable(true)) { if (master.getServerManager().isServerDead(server)) { // TODO: See HBASE-18494 and HBASE-18495. Though getExcludedServersForSystemTable() // considers only online servers, the server could be queued for dead server @@ -2304,34 +2295,24 @@ private void addToPendingAssignment(final HashMap r * For system tables, we must assign them to a server with highest version. */ public List getExcludedServersForSystemTable() { - // TODO: This should be a cached list kept by the ServerManager rather than calculated on each - // move or system region assign. The RegionServerTracker keeps list of online Servers with - // RegionServerInfo that includes Version. - List> serverList = master.getServerManager().getOnlineServersList() - .stream() - .map((s)->new Pair<>(s, master.getRegionServerVersion(s))) - .collect(Collectors.toList()); - if (serverList.isEmpty()) { - return Collections.emptyList(); - } - String highestVersion = Collections.max(serverList, - (o1, o2) -> VersionInfo.compareVersion(o1.getSecond(), o2.getSecond())).getSecond(); - return serverList.stream() - .filter((p)->!p.getSecond().equals(highestVersion)) - .map(Pair::getFirst) - .collect(Collectors.toList()); + return getExcludedServersForSystemTable(false); } /** * Get a list of servers that this region can not assign to. * For system table, we must assign them to a server with highest version. - * This method is same as {@link #getExcludedServersForSystemTable()} with - * the only difference is we can disable this exclusion using config: - * "hbase.min.version.move.system.tables". + * We can disable this exclusion using config: + * "hbase.min.version.move.system.tables" if checkForMinVersion is true. * + * @param checkForMinVersion if true, check for minVersionToMoveSysTables + * and decide moving system table regions accordingly. * @return List of Excluded servers for System table regions. */ - private List getExcludedServersForSystemTableUnlessAllowed() { + private List getExcludedServersForSystemTable( + boolean checkForMinVersion) { + // TODO: This should be a cached list kept by the ServerManager rather than calculated on each + // move or system region assign. The RegionServerTracker keeps list of online Servers with + // RegionServerInfo that includes Version. List> serverList = master.getServerManager().getOnlineServersList() .stream() .map(s->new Pair<>(s, master.getRegionServerVersion(s))) @@ -2341,11 +2322,12 @@ private List getExcludedServersForSystemTableUnlessAllowed() { } String highestVersion = Collections.max(serverList, (o1, o2) -> VersionInfo.compareVersion(o1.getSecond(), o2.getSecond())).getSecond(); - if (!DEFAULT_MIN_VERSION_MOVE_SYS_TABLES_CONFIG.equals(minVersionToMoveSysTables)) { - int comparedValue = - VersionInfo.compareVersion(minVersionToMoveSysTables, highestVersion); - if (comparedValue > 0) { - return Collections.emptyList(); + if (checkForMinVersion) { + if (!DEFAULT_MIN_VERSION_MOVE_SYS_TABLES_CONFIG.equals(minVersionToMoveSysTables)) { + int comparedValue = VersionInfo.compareVersion(minVersionToMoveSysTables, highestVersion); + if (comparedValue > 0) { + return Collections.emptyList(); + } } } return serverList.stream() From ef556983060bd034432f55cd1b375a3cd8cf8a22 Mon Sep 17 00:00:00 2001 From: Viraj Jasani Date: Wed, 30 Jun 2021 23:57:38 +0530 Subject: [PATCH 3/4] addressing comments by @saintstack --- .../master/assignment/AssignmentManager.java | 33 ++++++++++++++----- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java index c0465cd57074..a869ba9a80f7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java @@ -165,10 +165,22 @@ public class AssignmentManager { /** * When the operator uses this configuration option, any version between - * the current version and the new value of - * "hbase.min.version.move.system.tables" does not trigger any region movement. - * It is assumed that the configured range of versions do not require special + * the current cluster version and the value of "hbase.min.version.move.system.tables" + * does not trigger any auto-region movement. Auto-region movement here + * refers to auto-migration of system table regions to newer server versions. + * It is assumed that the configured range of versions does not require special * handling of moving system table regions to higher versioned RegionServer. + * This auto-migration is done by {@link #checkIfShouldMoveSystemRegionAsync()}. + * Example: Let's assume the cluster is on version 1.4.0 and we have + * set "hbase.min.version.move.system.tables" as "2.0.0". Now if we upgrade + * one RegionServer on 1.4.0 cluster to 1.6.0 (< 2.0.0), then AssignmentManager will + * not move hbase:meta, hbase:namespace and other system table regions + * to newly brought up RegionServer 1.6.0 as part of auto-migration. + * However, if we upgrade one RegionServer on 1.4.0 cluster to 2.2.0 (> 2.0.0), + * then AssignmentManager will move all system table regions to newly brought + * up RegionServer 2.2.0 as part of auto-migration done by + * {@link #checkIfShouldMoveSystemRegionAsync()}. + * "hbase.min.version.move.system.tables" is introduced as part of HBASE-22923. */ private final String minVersionToMoveSysTables; @@ -2291,18 +2303,23 @@ private void addToPendingAssignment(final HashMap r } /** - * Get a list of servers that this region cannot be assigned to. - * For system tables, we must assign them to a server with highest version. + * For a given cluster with mixed versions of servers, get a list of + * servers with lower versions, where system table regions should not be + * assigned to. + * For system table, we must assign regions to a server with highest version. */ public List getExcludedServersForSystemTable() { return getExcludedServersForSystemTable(false); } /** - * Get a list of servers that this region can not assign to. - * For system table, we must assign them to a server with highest version. - * We can disable this exclusion using config: + * For a given cluster with mixed versions of servers, get a list of + * servers with lower versions, where system table regions should not be + * assigned to. + * For system table, we must assign regions to a server with highest version. + * However, we can disable this exclusion using config: * "hbase.min.version.move.system.tables" if checkForMinVersion is true. + * Detailed explanation available with definition of minVersionToMoveSysTables. * * @param checkForMinVersion if true, check for minVersionToMoveSysTables * and decide moving system table regions accordingly. From b817a7d48a5a8fcb26164f066361c86dfe2789e7 Mon Sep 17 00:00:00 2001 From: Viraj Jasani Date: Thu, 1 Jul 2021 00:04:51 +0530 Subject: [PATCH 4/4] addendum --- .../hadoop/hbase/master/assignment/AssignmentManager.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java index a869ba9a80f7..c00bf7a8c14b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java @@ -2321,8 +2321,11 @@ public List getExcludedServersForSystemTable() { * "hbase.min.version.move.system.tables" if checkForMinVersion is true. * Detailed explanation available with definition of minVersionToMoveSysTables. * - * @param checkForMinVersion if true, check for minVersionToMoveSysTables - * and decide moving system table regions accordingly. + * @param checkForMinVersion If false, return a list of servers with lower version. If true, + * compare higher version with minVersionToMoveSysTables. Only if higher version is greater + * than minVersionToMoveSysTables, this method returns list of servers with lower version. If + * higher version is less than or equal to minVersionToMoveSysTables, returns empty list. + * An example is provided with definition of minVersionToMoveSysTables. * @return List of Excluded servers for System table regions. */ private List getExcludedServersForSystemTable(