diff --git a/hbase-tools/README.md b/hbase-tools/README.md index 5bbd0d9177..74a55ac2ab 100644 --- a/hbase-tools/README.md +++ b/hbase-tools/README.md @@ -29,7 +29,7 @@ module are: - RegionsMerger; - MissingRegionDirsRepairTool; - +- RegionsOnUnknownServersRecoverer; ## Setup Make sure HBase tools jar is added to HBase classpath: @@ -138,4 +138,35 @@ the affected regions, it copies the entire region dir to a region hfiles to a `HBASE_ROOT_DIR/.missing_dirs_repair/TS/TBL_NAME/bulkload` dir, renaming these files with the pattern `REGION_NAME-FILENAME`. For a given table, all affected regions would then have all its files under same directory for bulkload. _MissingRegionDirsRepairTool_ then uses -_LoadIncrementalHFiles_ to load all files for a given table at once. \ No newline at end of file +_LoadIncrementalHFiles_ to load all files for a given table at once. + +## RegionsOnUnknownServersRecoverer - Tool for recovering regions on "unknown servers." + +_RegionsOnUnknownServersRecoverer_ parses the master log to identify `unknown servers` +holding regions. This condition may happen in the event of recovering previously destroyed clusters, +where new Master/RS names completely differ from the previous ones currently +stored in meta table (see HBASE-24286). + +``` +NOTE: This tool is useful for clusters runing hbase versions lower than 2.2.7, 2.3.5 and 2.4.7. +For any of these versions or higher, HBCK2 'recoverUnknown' option can be used as a much simpler solution. +``` + +### Usage + +This tool requires the master logs path as parameter. Assuming classpath is properly set, +can be run as follows: + +``` +$ hbase org.apache.hbase.RegionsOnUnknownServersRecoverer PATH_TO_MASTER_LOGS [dryRun] +``` + +The `dryRun` optional parameter will just parse the logs and print the list of unknown servers, +without invoking `hbck2 scheduleRecoveries` command. + + +### Implementation Details + +_RegionsOnUnknownServersRecoverer_ parses master log file searching for specific messages mentioning + "unknown servers". Once "unknown servers" are found, it then uses `HBCK2.scheduleRecoveries` to + submit SCPs for each of these "unknown servers". \ No newline at end of file diff --git a/hbase-tools/src/main/java/org/apache/hbase/RegionsOnUnknownServersRecoverer.java b/hbase-tools/src/main/java/org/apache/hbase/RegionsOnUnknownServersRecoverer.java new file mode 100644 index 0000000000..1a83ca29d8 --- /dev/null +++ b/hbase-tools/src/main/java/org/apache/hbase/RegionsOnUnknownServersRecoverer.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hbase; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.util.HashSet; +import java.util.Set; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +/** + * Tool for identifying Unknown Servers from master logs and schedule SCPs for each of those using + * HBCK2 'scheduleRecoveries' option. This is useful for clusters running hbase versions lower than + * 2.2.7, 2.3.5 and 2.4.7. For any of these versions or higher, use HBCK2 'recoverUnknown' option. + */ +public class RegionsOnUnknownServersRecoverer extends Configured implements Tool { + + private static final Logger LOG = + LoggerFactory.getLogger(RegionsOnUnknownServersRecoverer.class.getName()); + + private static final String CATALOG_JANITOR = "CatalogJanitor: hole="; + + private static final String UNKNOWN_SERVER = "unknown_server="; + + private Configuration conf; + + private Set unknownServers = new HashSet<>(); + + private boolean dryRun = false; + + public RegionsOnUnknownServersRecoverer(Configuration conf){ + this.conf = conf; + } + + @Override + public int run(String[] args) throws Exception { + String logPath = null; + if(args.length>=1 && args.length<3) { + logPath = args[0]; + if(args.length==2) { + dryRun = args[1].equals("dryRun"); + } + } else { + LOG.error("Wrong number of arguments. " + + "Arguments are: [dryRun]"); + return 1; + } + BufferedReader reader = null; + try(Connection conn = ConnectionFactory.createConnection(conf)) { + reader = new BufferedReader(new FileReader(new File(logPath))); + String line = null; + while((line = reader.readLine()) != null){ + if(line.contains(CATALOG_JANITOR)){ + String[] servers = line.split(UNKNOWN_SERVER); + for(int i=1; i builder.append(s).append("\n")); + LOG.info(builder.toString()); + } else { + HBCK2 hbck2 = new HBCK2(conf); + LOG.info("Submitting SCPs for the found unknown servers with " + + "HBCK2 scheduleRecoveries option."); + hbck2.scheduleRecoveries(conn.getHbck(), unknownServers.toArray(new String[] {})); + } + } catch(Exception e){ + LOG.error("Recovering unknown servers failed:", e); + return 2; + } finally { + reader.close(); + } + return 0; + } + + public static void main(String [] args) throws Exception { + Configuration conf = HBaseConfiguration.create(); + int errCode = ToolRunner.run(new RegionsOnUnknownServersRecoverer(conf), args); + if (errCode != 0) { + System.exit(errCode); + } + } +} +