diff --git a/hbase-hbck2/README.md b/hbase-hbck2/README.md index c8ebb79579..b895195258 100644 --- a/hbase-hbck2/README.md +++ b/hbase-hbck2/README.md @@ -91,7 +91,9 @@ The above command with no options or arguments passed will dump out the _HBCK2_ usage: HBCK2 [OPTIONS] COMMAND Options: -d,--debug run with debug output + -i, --inputfiles take one or more files to read the args from -h,--help output this help message + -i,--inputFiles take one or more encoded region names -p,--hbase.zookeeper.property.clientPort port of hbase ensemble -q,--hbase.zookeeper.quorum hbase ensemble -s,--skip skip hbase version check @@ -123,11 +125,14 @@ Command: Returns HBCK2 an 'assigns' command with all re-inserted regions. SEE ALSO: reportMissingRegionsInMeta SEE ALSO: fixMeta + If -i or --inputFiles is specified, pass one or more input file names. + Each file contains , one per line. For example: + For example: + $ HBCK2 -i addFsRegionsMissingInMeta fileName1 fileName2 assigns [OPTIONS] ... Options: -o,--override override ownership by another procedure - -i,--inputFiles take one or more encoded region names A 'raw' assign that can be used even during Master initialization (if the -skip flag is specified). Skirts Coprocessors. Pass one or more encoded region names. 1588230740 is the hard-coded name for the @@ -137,7 +142,7 @@ Command: Returns the pid(s) of the created AssignProcedure(s) or -1 if none. If -i or --inputFiles is specified, pass one or more input file names. Each file contains encoded region names, one per line. For example: - $ HBCK2 assigns -i fileName1 fileName2 + $ HBCK2 -i assigns fileName1 fileName2 bypass [OPTIONS] ... Options: -o,--override override if procedure is running/stuck @@ -150,6 +155,9 @@ Command: procedure has children. Add 'recursive' if all you have is a parent pid to finish parent and children. This is SLOW, and dangerous so use selectively. Does not always work. + If -i or --inputFiles is specified, pass one or more input file names. + Each file contains PID's, one per line. For example: + $ HBCK2 -i bypass fileName1 fileName2 extraRegionsInMeta ... Options: @@ -173,6 +181,9 @@ Command: $ HBCK2 extraRegionsInMeta default:table_1 ns1 Returns list of extra regions for each table passed as parameter, or for each table on namespaces specified as parameter. + If -i or --inputFiles is specified, pass one or more input file names. + Each file contains , one per line. For example: + $ HBCK2 -i extraRegionsInMeta fileName1 fileName2 filesystem [OPTIONS] [...] Options: @@ -183,7 +194,9 @@ Command: more tablenames to narrow checkup. Default checks all tables and restores 'hbase.version' if missing. Interacts with the filesystem only! Modified regions need to be reopened to pick-up changes. - + If -i or --inputFiles is specified, pass one or more input file names. + Each file contains , one per line. For example: + $ HBCK2 -i extraRegionsInMeta fileName1 fileName2 fixMeta Do a server-side fix of bad or inconsistent state in hbase:meta. Available in hbase 2.2.1/2.1.6 or newer versions. Master UI has @@ -225,7 +238,9 @@ Command: -f, --fix fix any replication issues found. Looks for undeleted replication queues and deletes them if passed the '--fix' option. Pass a table name to check for replication barrier and - purge if '--fix'. + purge if '--fix'. If -i or --inputFiles is specified, pass one or more input file names. + Each file contains , one per line. For example: + $ HBCK2 -i replication fileName1 fileName2 reportMissingRegionsInMeta ... To be used when regions missing from hbase:meta but directories @@ -252,6 +267,9 @@ Command: $ HBCK2 reportMissingRegionsInMeta default:table_1 ns1 Returns list of missing regions for each table passed as parameter, or for each table on namespaces specified as parameter. + If -i or --inputFiles is specified, pass one or more input file names. + Each file contains , one per line. For example: + $ HBCK2 -i reportMissingRegionsInMeta fileName1 fileName2 setRegionState Possible region states: @@ -270,6 +288,10 @@ Command: setting region 'de00010733901a05f5a2a3a382e27dd4' to CLOSING: $ HBCK2 setRegionState de00010733901a05f5a2a3a382e27dd4 CLOSING Returns "0" if region state changed and "1" otherwise. + If -i or --inputFiles is specified, pass one or more input file names. + Each file contains , one pair per line. + For example: + $ HBCK2 -i setRegionState fileName1 fileName2 setTableState Possible table states: ENABLED, DISABLED, DISABLING, ENABLING @@ -280,8 +302,13 @@ Command: An example making table name 'user' ENABLED: $ HBCK2 setTableState users ENABLED Returns whatever the previous table state was. + If -i or --inputFiles is specified, pass one or more input file names. + Each file contains , one pair per line. + For example: + $ HBCK2 -i setTableState fileName1 fileName2 scheduleRecoveries ... + -i, --inputFile take one or more files to read the args from Schedule ServerCrashProcedure(SCP) for list of RegionServers. Format server name as ',,' (See HBase UI/logs). Example using RegionServer 'a.example.org,29100,1540348649479': @@ -289,6 +316,9 @@ Command: Returns the pid(s) of the created ServerCrashProcedure(s) or -1 if no procedure created (see master logs for why not). Command support added in hbase versions 2.0.3, 2.1.2, 2.2.0 or newer. + If -i or --inputFiles is specified, pass one or more input file names. + Each file contains , one per line. For example: + $ HBCK2 -i scheduleRecoveries fileName1 fileName2 unassigns ... Options: @@ -300,6 +330,9 @@ Command: of what a userspace encoded region name looks like. For example: $ HBCK2 unassign 1588230740 de00010733901a05f5a2a3a382e27dd4 Returns the pid(s) of the created UnassignProcedure(s) or -1 if none. + If -i or --inputFiles is specified, pass one or more input file names. + Each file contains encoded region names, one per line. For example: + $ HBCK2 -i unassigns fileName1 fileName2 SEE ALSO, org.apache.hbase.hbck1.OfflineMetaRepair, the offline hbase:meta tool. See the HBCK2 README for how to use. diff --git a/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java b/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java index 84dc8341e9..1d0ed20210 100644 --- a/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java +++ b/hbase-hbck2/src/main/java/org/apache/hbase/HBCK2.java @@ -128,6 +128,7 @@ public class HBCK2 extends Configured implements org.apache.hadoop.util.Tool { private Configuration conf; static final String [] MINIMUM_HBCK2_VERSION = {"2.0.3", "2.1.1", "2.2.0", "3.0.0"}; private boolean skipCheck = false; + private boolean getFromFile = false; /** * Wait 1ms on lock by default. @@ -190,6 +191,18 @@ TableState setTableState(Hbck hbck, TableName tableName, TableState.State state) return hbck.setTableStateInMeta(new TableState(tableName, state)); } + TableState setTableState(Hbck hbck, String[] args) + throws IOException { + if (args == null || args.length < 2) { + showErrorMessage(SET_TABLE_STATE + + " takes tablename and state arguments: e.g. user ENABLED, you entered: " + + Arrays.toString(args)); + return null; + } + return setTableState(hbck, TableName.valueOf(args[0]), + TableState.State.valueOf(args[1])); + } + int setRegionState(ClusterConnection connection, String region, RegionState.State newState) throws IOException { @@ -238,13 +251,22 @@ int setRegionState(ClusterConnection connection, String region, int replicaId, return EXIT_FAILURE; } + int setRegionState(ClusterConnection connection, String[] args) throws IOException { + if (args == null || args.length < 3) { + return EXIT_FAILURE; + } + RegionState.State state = RegionState.State.valueOf(args[2]); + int replicaId = Integer.valueOf(args[1]); + return setRegionState(connection, args[0], replicaId, state); + } + Map> reportTablesWithMissingRegionsInMeta(String... nameSpaceOrTable) throws IOException { Map> report; try (final FsRegionsMetaRecoverer fsRegionsMetaRecoverer = new FsRegionsMetaRecoverer(this.conf)) { - report = fsRegionsMetaRecoverer.reportTablesMissingRegions( - formatNameSpaceTableParam(nameSpaceOrTable)); + report = fsRegionsMetaRecoverer.reportTablesMissingRegions(getFromArgsOrFiles( + formatNameSpaceTableParam(nameSpaceOrTable))); } catch (IOException e) { LOG.error("Error reporting missing regions: ", e); throw e; @@ -268,7 +290,8 @@ Map> extraRegionsInMeta(String[] args) Map> result = new HashMap<>(); try (final FsRegionsMetaRecoverer fsRegionsMetaRecoverer = new FsRegionsMetaRecoverer(this.conf)) { - List namespacesTables = formatNameSpaceTableParam(commandLine.getArgs()); + List namespacesTables = + getFromArgsOrFiles(formatNameSpaceTableParam(commandLine.getArgs())); Map> reportMap = fsRegionsMetaRecoverer.reportTablesExtraRegions(namespacesTables); final List toFix = new ArrayList<>(); @@ -305,16 +328,12 @@ Map> extraRegionsInMeta(String[] args) return result; } - private List formatNameSpaceTableParam(String... nameSpaceOrTable) { - return nameSpaceOrTable != null ? Arrays.asList(nameSpaceOrTable) : null; - } - List>> addMissingRegionsInMetaForTables(String... nameSpaceOrTable) throws IOException { try (final FsRegionsMetaRecoverer fsRegionsMetaRecoverer = new FsRegionsMetaRecoverer(this.conf)) { - return fsRegionsMetaRecoverer.addMissingRegionsInMetaForTables( - formatNameSpaceTableParam(nameSpaceOrTable)); + return fsRegionsMetaRecoverer.addMissingRegionsInMetaForTables(getFromArgsOrFiles( + formatNameSpaceTableParam(nameSpaceOrTable))); } catch (IOException e) { LOG.error("Error adding missing regions: ", e); throw e; @@ -324,9 +343,7 @@ List>> addMissingRegionsInMetaForTables(String... List assigns(Hbck hbck, String[] args) throws IOException { Options options = new Options(); Option override = Option.builder("o").longOpt("override").build(); - Option inputFile = Option.builder("i").longOpt("inputFiles").build(); options.addOption(override); - options.addOption(inputFile); // Parse command-line. CommandLineParser parser = new DefaultParser(); CommandLine commandLine; @@ -337,21 +354,8 @@ List assigns(Hbck hbck, String[] args) throws IOException { return null; } boolean overrideFlag = commandLine.hasOption(override.getOpt()); - List argList = commandLine.getArgList(); - if (!commandLine.hasOption(inputFile.getOpt())) { - return hbck.assigns(argList, overrideFlag); - } - List assignmentList = new ArrayList<>(); - for (String filePath : argList) { - try (InputStream fileStream = new FileInputStream(filePath)){ - LineIterator it = IOUtils.lineIterator(fileStream, "UTF-8"); - while (it.hasNext()) { - assignmentList.add(it.nextLine().trim()); - } - } - } - return hbck.assigns(assignmentList, overrideFlag); + return hbck.assigns(this.getFromArgsOrFiles(argList), overrideFlag); } List unassigns(Hbck hbck, String [] args) throws IOException { @@ -368,7 +372,37 @@ List unassigns(Hbck hbck, String [] args) throws IOException { return null; } boolean overrideFlag = commandLine.hasOption(override.getOpt()); - return hbck.unassigns(commandLine.getArgList(), overrideFlag); + return hbck.unassigns(getFromArgsOrFiles(commandLine.getArgList()), overrideFlag); + } + + private List formatNameSpaceTableParam(String... nameSpaceOrTable) { + return nameSpaceOrTable != null ? Arrays.asList(nameSpaceOrTable) : null; + } + + /** + * @return Read arguments from args or a list of input files + */ + private List getFromArgsOrFiles(List args) throws IOException { + if (!getFromFile || args == null) { + return args; + } + return getFromFiles(args); + } + + /** + * @return Read arguments from a list of input files + */ + private List getFromFiles(List args) throws IOException { + List argList = new ArrayList<>(); + for (String filePath : args) { + try (InputStream fileStream = new FileInputStream(filePath)){ + LineIterator it = IOUtils.lineIterator(fileStream, "UTF-8"); + while (it.hasNext()) { + argList.add(it.nextLine().trim()); + } + } + } + return argList; } /** @@ -397,14 +431,14 @@ private List bypass(String[] args) throws IOException { if (commandLine.hasOption(wait.getOpt())) { lockWait = Integer.parseInt(commandLine.getOptionValue(wait.getOpt())); } - String[] pidStrs = commandLine.getArgs(); - if (pidStrs == null || pidStrs.length <= 0) { + List pidStrs = getFromArgsOrFiles(commandLine.getArgList()); + if (pidStrs == null || pidStrs.size() <= 0) { showErrorMessage("No pids supplied."); return null; } boolean overrideFlag = commandLine.hasOption(override.getOpt()); boolean recursiveFlag = commandLine.hasOption(recursive.getOpt()); - List pids = Arrays.stream(pidStrs).map(Long::valueOf).collect(Collectors.toList()); + List pids = pidStrs.stream().map(Long::valueOf).collect(Collectors.toList()); try (ClusterConnection connection = connect(); Hbck hbck = connection.getHbck()) { checkFunctionSupported(connection, BYPASS); return hbck.bypassProcedure(pids, lockWait, overrideFlag, recursiveFlag); @@ -412,8 +446,9 @@ private List bypass(String[] args) throws IOException { } List scheduleRecoveries(Hbck hbck, String[] args) throws IOException { + List arglist = getFromArgsOrFiles(Arrays.asList(args)); List serverNames = new ArrayList<>(); - for (String serverName: args) { + for (String serverName: arglist) { serverNames.add(parseServerName(serverName)); } return hbck.scheduleServerCrashProcedure(serverNames); @@ -498,7 +533,7 @@ private static String getCommandUsage() { private static void usageAddFsRegionsMissingInMeta(PrintWriter writer) { writer.println(" " + ADD_MISSING_REGIONS_IN_META_FOR_TABLES + " ..."); + + "NAMESPACE:TABLENAME|INPUTFILE_FOR_>..."); writer.println(" Options:"); writer.println(" -d,--force_disable aborts fix for table if disable fails."); writer.println(" To be used when regions missing from hbase:meta but directories"); @@ -522,27 +557,31 @@ private static void usageAddFsRegionsMissingInMeta(PrintWriter writer) { writer.println(" Returns HBCK2 an 'assigns' command with all re-inserted regions."); writer.println(" SEE ALSO: " + REPORT_MISSING_REGIONS_IN_META); writer.println(" SEE ALSO: " + FIX_META); + writer.println(" If -i or --inputFiles is specified, pass one or more input file names."); + writer.println(" Each file contains , one per line."); + writer.println(" For example:"); + writer.println(" $ HBCK2 -i" + ADD_MISSING_REGIONS_IN_META_FOR_TABLES + + "fileName1 fileName2"); } private static void usageAssigns(PrintWriter writer) { - writer.println(" " + ASSIGNS + " [OPTIONS] ..."); + writer.println(" " + ASSIGNS + " [OPTIONS] ..."); writer.println(" Options:"); writer.println(" -o,--override override ownership by another procedure"); - writer.println(" -i,--inputFiles take one or more files of encoded region names"); writer.println(" A 'raw' assign that can be used even during Master initialization (if"); writer.println(" the -skip flag is specified). Skirts Coprocessors. Pass one or more"); writer.println(" encoded region names. 1588230740 is the hard-coded name for the"); writer.println(" hbase:meta region and de00010733901a05f5a2a3a382e27dd4 is an example of"); writer.println(" what a user-space encoded region name looks like. For example:"); - writer.println(" $ HBCK2 assigns 1588230740 de00010733901a05f5a2a3a382e27dd4"); + writer.println(" $ HBCK2 " + ASSIGNS + " 1588230740 de00010733901a05f5a2a3a382e27dd4"); writer.println(" Returns the pid(s) of the created AssignProcedure(s) or -1 if none."); writer.println(" If -i or --inputFiles is specified, pass one or more input file names."); writer.println(" Each file contains encoded region names, one per line. For example:"); - writer.println(" $ HBCK2 assigns -i fileName1 fileName2"); + writer.println(" $ HBCK2 -i " + ASSIGNS + " fileName1 fileName2"); } private static void usageBypass(PrintWriter writer) { - writer.println(" " + BYPASS + " [OPTIONS] ..."); + writer.println(" " + BYPASS + " [OPTIONS] |..."); writer.println(" Options:"); writer.println(" -o,--override override if procedure is running/stuck"); writer.println(" -r,--recursive bypass parent and its children. SLOW! EXPENSIVE!"); @@ -554,10 +593,13 @@ private static void usageBypass(PrintWriter writer) { writer.println(" procedure has children. Add 'recursive' if all you have is a parent pid"); writer.println(" to finish parent and children. This is SLOW, and dangerous so use"); writer.println(" selectively. Does not always work."); + writer.println(" If -i or --inputFiles is specified, pass one or more input file names."); + writer.println(" Each file contains pids, one per line. For example:"); + writer.println(" $ HBCK2 -i" + BYPASS + "fileName1 fileName2"); } private static void usageFilesystem(PrintWriter writer) { - writer.println(" " + FILESYSTEM + " [OPTIONS] [...]"); + writer.println(" " + FILESYSTEM + " [OPTIONS] [...]"); writer.println(" Options:"); writer.println(" -f, --fix sideline corrupt hfiles, bad links, and references."); writer.println(" Report on corrupt hfiles, references, broken links, and integrity."); @@ -566,6 +608,9 @@ private static void usageFilesystem(PrintWriter writer) { writer.println(" more tablenames to narrow checkup. Default checks all tables and"); writer.println(" restores 'hbase.version' if missing. Interacts with the filesystem"); writer.println(" only! Modified regions need to be reopened to pick-up changes."); + writer.println(" If -i or --inputFiles is specified, pass one or more input file names."); + writer.println(" Each file contains table names, one per line. For example:"); + writer.println(" $ HBCK2 -i" + FILESYSTEM + "fileName1 fileName2"); } private static void usageFixMeta(PrintWriter writer) { @@ -620,7 +665,7 @@ private static void usageReplication(PrintWriter writer) { private static void usageExtraRegionsInMeta(PrintWriter writer) { writer.println(" " + EXTRA_REGIONS_IN_META + " ..."); + + "NAMESPACE:TABLENAME|INPUTFILE_FOR_>..."); writer.println(" Options:"); writer.println(" -f, --fix fix meta by removing all extra regions found."); writer.println(" Reports regions present on hbase:meta, but with no related "); @@ -637,17 +682,21 @@ private static void usageExtraRegionsInMeta(PrintWriter writer) { writer.println(" An example triggering extra regions report for tables 'table_1'"); writer.println(" and 'table_2', under default namespace:"); writer.println(" $ HBCK2 " + EXTRA_REGIONS_IN_META + - " default:table_1 default:table_2"); + " default:table_1 default:table_2"); writer.println(" An example triggering missing regions report for table 'table_1'"); writer.println(" under default namespace, and for all tables from namespace 'ns1':"); writer.println(" $ HBCK2 " + EXTRA_REGIONS_IN_META + " default:table_1 ns1"); writer.println(" Returns list of extra regions for each table passed as parameter, or"); writer.println(" for each table on namespaces specified as parameter."); + writer.println(" If -i or --inputFiles is specified, pass one or more input file names."); + writer.println(" Each file contains , one per line."); + writer.println(" For example:"); + writer.println(" $ HBCK2 -i" + EXTRA_REGIONS_IN_META + "fileName1 fileName2"); } private static void usageReportMissingRegionsInMeta(PrintWriter writer) { writer.println(" " + REPORT_MISSING_REGIONS_IN_META + " ..."); + + "NAMESPACE:TABLENAME|INPUTFILE_FOR_>..."); writer.println(" To be used when regions missing from hbase:meta but directories"); writer.println(" are present still in HDFS. Can happen if user has run _hbck1_"); writer.println(" 'OfflineMetaRepair' against an hbase-2.x cluster. This is a CHECK only"); @@ -667,20 +716,23 @@ private static void usageReportMissingRegionsInMeta(PrintWriter writer) { writer.println(" namespace, otherwise it will assume as a namespace value."); writer.println(" An example triggering missing regions execute for tables 'table_1'"); writer.println(" and 'table_2', under default namespace:"); - writer.println(" $ HBCK2 reportMissingRegionsInMeta default:table_1 default:table_2"); + writer.println(" $ HBCK2 " + REPORT_MISSING_REGIONS_IN_META + + " default:table_1 default:table_2"); writer.println(" An example triggering missing regions execute for table 'table_1'"); writer.println(" under default namespace, and for all tables from namespace 'ns1':"); - writer.println(" $ HBCK2 reportMissingRegionsInMeta default:table_1 ns1"); + writer.println(" $ HBCK2 " + REPORT_MISSING_REGIONS_IN_META + " default:table_1 ns1"); writer.println(" Returns list of missing regions for each table passed as parameter, or"); writer.println(" for each table on namespaces specified as parameter."); + writer.println(" If -i or --inputFiles is specified, pass one or more input file names."); + writer.println(" Each file contains , one per line.);" + + "For example:"); + writer.println(" $ HBCK2 -i " + REPORT_MISSING_REGIONS_IN_META + " fileName1 fileName2"); } private static void usageSetRegionState(PrintWriter writer) { writer.println(" " + SET_REGION_STATE + " "); writer.println(" To set the replica region's state, it needs the primary region's "); - writer.println(" encoded regionname and replica id. The command will be "); - writer.println(" " + SET_REGION_STATE + " , "); - writer.println(" Possible region states:"); + writer.println(" encoded regionname and replica id. The states will be "); writer.println(" OFFLINE, OPENING, OPEN, CLOSING, CLOSED, SPLITTING, SPLIT,"); writer.println(" FAILED_OPEN, FAILED_CLOSE, MERGING, MERGED, SPLITTING_NEW,"); writer.println(" MERGING_NEW, ABNORMALLY_CLOSED"); @@ -694,8 +746,13 @@ private static void usageSetRegionState(PrintWriter writer) { writer.println(" such as 'assign' or 'split'. You can get a view of running procedures"); writer.println(" in the hbase shell using the 'list_procedures' command. An example"); writer.println(" setting region 'de00010733901a05f5a2a3a382e27dd4' to CLOSING:"); - writer.println(" $ HBCK2 setRegionState de00010733901a05f5a2a3a382e27dd4 CLOSING"); + writer.println(" $ HBCK2 " + SET_REGION_STATE + + " de00010733901a05f5a2a3a382e27dd4 CLOSING"); writer.println(" Returns \"0\" if region state changed and \"1\" otherwise."); + writer.println(" If -i or --inputFiles is specified, pass one or more input file names."); + writer.println(" Each file contains , one pair per line.);" + + "For example:"); + writer.println(" $ HBCK2 -i " + SET_REGION_STATE + " fileName1 fileName2"); } private static void usageSetTableState(PrintWriter writer) { @@ -709,10 +766,14 @@ private static void usageSetTableState(PrintWriter writer) { writer.println(" An example making table name 'user' ENABLED:"); writer.println(" $ HBCK2 setTableState users ENABLED"); writer.println(" Returns whatever the previous table state was."); + writer.println(" If -i or --inputFiles is specified, pass one or more input file names."); + writer.println(" Each file contains , one pair per line.);" + + "For example:"); + writer.println(" $ HBCK2 -i " + SET_TABLE_STATE + " fileName1 fileName2"); } private static void usageScheduleRecoveries(PrintWriter writer) { - writer.println(" " + SCHEDULE_RECOVERIES + " ..."); + writer.println(" " + SCHEDULE_RECOVERIES + " ..."); writer.println(" Schedule ServerCrashProcedure(SCP) for list of RegionServers. Format"); writer.println(" server name as ',,' (See HBase UI/logs)."); writer.println(" Example using RegionServer 'a.example.org,29100,1540348649479':"); @@ -720,6 +781,9 @@ private static void usageScheduleRecoveries(PrintWriter writer) { writer.println(" Returns the pid(s) of the created ServerCrashProcedure(s) or -1 if"); writer.println(" no procedure created (see master logs for why not)."); writer.println(" Command support added in hbase versions 2.0.3, 2.1.2, 2.2.0 or newer."); + writer.println(" If -i or --inputFiles is specified, pass one or more input file names."); + writer.println(" Each file contains server names, one per line. For example:"); + writer.println(" $ HBCK2 -i " + SCHEDULE_RECOVERIES + " fileName1 fileName2"); } private static void usageRecoverUnknown(PrintWriter writer) { @@ -733,7 +797,7 @@ private static void usageRecoverUnknown(PrintWriter writer) { } private static void usageUnassigns(PrintWriter writer) { - writer.println(" " + UNASSIGNS + " ..."); + writer.println(" " + UNASSIGNS + " ..."); writer.println(" Options:"); writer.println(" -o,--override override ownership by another procedure"); writer.println(" A 'raw' unassign that can be used even during Master initialization"); @@ -743,6 +807,9 @@ private static void usageUnassigns(PrintWriter writer) { writer.println(" of what a userspace encoded region name looks like. For example:"); writer.println(" $ HBCK2 unassigns 1588230740 de00010733901a05f5a2a3a382e27dd4"); writer.println(" Returns the pid(s) of the created UnassignProcedure(s) or -1 if none."); + writer.println(" If -i or --inputFiles is specified, pass one or more input file names."); + writer.println(" Each file contains encoded region names, one per line. For example:"); + writer.println(" $ HBCK2 -i " + UNASSIGNS + " fileName1 fileName2"); writer.println(); writer.println(" SEE ALSO, org.apache.hbase.hbck1.OfflineMetaRepair, the offline"); writer.println(" hbase:meta tool. See the HBCK2 README for how to use."); @@ -818,6 +885,9 @@ public int run(String[] args) throws IOException { Option skip = Option.builder("s").longOpt("skip"). desc("skip hbase version check (PleaseHoldException)").build(); options.addOption(skip); + Option inputFiles = Option.builder("i").longOpt("inputFiles") + .desc("take one or more files to read the args from").build(); + options.addOption(inputFiles); // Parse command-line. CommandLineParser parser = new DefaultParser(); @@ -868,6 +938,9 @@ public int run(String[] args) throws IOException { if (commandLine.hasOption(skip.getOpt())) { skipCheck = true; } + if (commandLine.hasOption(inputFiles.getOpt())) { + getFromFile = true; + } return doCommandLine(commandLine, options); } @@ -892,17 +965,32 @@ private int doCommandLine(CommandLine commandLine, Options options) throws IOExc // Case handlers all have same format. Check first that the server supports // the feature FIRST, then move to process the command. case SET_TABLE_STATE: - if (commands.length < 3) { - showErrorMessage(command + " takes tablename and state arguments: e.g. user ENABLED"); - return EXIT_FAILURE; + if (getFromFile){ + if (commands.length < 2) { + showErrorMessage(command + " takes a list of file names"); + return EXIT_FAILURE; + } + List argList = getFromFiles(Arrays.asList(purgeFirst(commands))); + try (ClusterConnection connection = connect(); Hbck hbck = connection.getHbck()) { + checkFunctionSupported(connection, command); + for (String line : argList) { + String[] args = line.split("\\s+"); + System.out.println(setTableState(hbck, args)); + } + } } - try (ClusterConnection connection = connect(); Hbck hbck = connection.getHbck()) { - checkFunctionSupported(connection, command); - System.out.println(setTableState(hbck, TableName.valueOf(commands[1]), - TableState.State.valueOf(commands[2]))); + else { + if (commands.length < 3) { + showErrorMessage(SET_TABLE_STATE + + " takes tablename and state arguments: e.g. user ENABLED"); + return EXIT_FAILURE; + } + try (ClusterConnection connection = connect(); Hbck hbck = connection.getHbck()) { + checkFunctionSupported(connection, command); + System.out.println(setTableState(hbck, purgeFirst(commands))); + } } break; - case ASSIGNS: if (commands.length < 2) { showErrorMessage(command + " takes one or more encoded region names"); @@ -944,31 +1032,29 @@ private int doCommandLine(CommandLine commandLine, Options options) throws IOExc break; case SET_REGION_STATE: - if (commands.length < 3) { - showErrorMessage(command + " takes region encoded name and state arguments: e.g. " - + "35f30b0ce922c34bf5c284eff33ba8b3 CLOSING"); - return EXIT_FAILURE; - } - RegionState.State state = RegionState.State.valueOf(commands[2]); - - int replicaId = 0; - String region = commands[1]; - int separatorIndex = commands[1].indexOf(","); - if (separatorIndex > 0) { - region = commands[1].substring(0, separatorIndex); - replicaId = Integer.getInteger(commands[1].substring(separatorIndex + 1)); - } - - if (replicaId > 0) { - System.out.println("Change state for replica reigon " + replicaId + - " for primary region " + region); - } - - try (ClusterConnection connection = connect()) { - checkHBCKSupport(connection, command); - return setRegionState(connection, region, replicaId, state); + if (getFromFile) { + if (commands.length < 2) { + showErrorMessage(command + " takes a list of file names"); + return EXIT_FAILURE; + } + List argList = getFromFiles(Arrays.asList(purgeFirst(commands))); + try (ClusterConnection connection = connect()) { + checkHBCKSupport(connection, command); + for (String line : argList) { + String[] args = formatSetRegionStateCommand(line.split("\\s+")); + if (setRegionState(connection, args) == EXIT_FAILURE) { + showErrorMessage(command + " failed to set " + args); + } + } + } + break; + } else { + String[] args = formatSetRegionStateCommand(purgeFirst(commands)); + try (ClusterConnection connection = connect()) { + checkHBCKSupport(connection, command); + return setRegionState(connection, args); + } } - case FILESYSTEM: try (ClusterConnection connection = connect()) { checkHBCKSupport(connection, command); @@ -1191,6 +1277,31 @@ private static String[] purgeFirst(String[] args) { return result; } + /** + * @return arguements for SET_REGION_STATE command + */ + private String[] formatSetRegionStateCommand(String[] commands) { + if (commands.length < 2) { + showErrorMessage("setRegionState takes region encoded name and state arguments: e.g. " + + "35f30b0ce922c34bf5c284eff33ba8b3 CLOSING"); + return null; + } + Integer replicaId = 0; + String region = commands[0]; + int separatorIndex = commands[0].indexOf(","); + if (separatorIndex > 0) { + region = commands[0].substring(0, separatorIndex); + replicaId = Integer.getInteger(commands[0].substring(separatorIndex + 1)); + } + + if (replicaId > 0) { + System.out.println("Change state for replica reigon " + replicaId + + " for primary region " + region); + } + RegionState.State state = RegionState.State.valueOf(commands[1]); + return new String[]{region, replicaId.toString(), state.name()}; + } + HBCK2(Configuration conf) { super(conf); } diff --git a/hbase-hbck2/src/test/java/org/apache/hbase/TestHBCK2.java b/hbase-hbck2/src/test/java/org/apache/hbase/TestHBCK2.java index 840f9be38a..d0361b9255 100644 --- a/hbase-hbck2/src/test/java/org/apache/hbase/TestHBCK2.java +++ b/hbase-hbck2/src/test/java/org/apache/hbase/TestHBCK2.java @@ -76,6 +76,9 @@ public class TestHBCK2 { valueOf(TestHBCK2.class.getSimpleName() + "-REGIONS_STATES"); private final static String ASSIGNS = "assigns"; private static final String EXTRA_REGIONS_IN_META = "extraRegionsInMeta"; + private final static String UNASSIGNS = "unassigns"; + private final static String SET_REGION_STATE = "setRegionState"; + private final static String SET_TABLE_STATE = "setTableState"; @Rule public TestName testName = new TestName(); @@ -121,6 +124,43 @@ public void testSetTableStateInMeta() throws IOException { // Restore the state. state = this.hbck2.setTableState(hbck, TABLE_NAME, state.getState()); assertTrue("Found=" + state.getState(), state.isDisabled()); + + // Test the new method with arg list + String[] args = new String[]{TABLE_NAME.getNameAsString(), "DISABLED"}; + state = this.hbck2.setTableState(hbck, args); + assertTrue("Found=" + state.getState(), state.isEnabled()); + } + } + + @Test + public void testSetTableStateWithInputFiles() throws IOException { + File testFile = new File(TEST_UTIL.getDataTestDir().toString(), "inputForSetTableTest"); + writeStringsToAFile(testFile, new String[]{TABLE_NAME.getNameAsString() + " DISABLED" }); + String result = testRunWithArgs(new String[]{"-i", SET_TABLE_STATE, testFile.toString()}); + assertTrue(result.contains("tableName=TestHBCK2, state=ENABLED")); + + // Restore the state. + try (ClusterConnection connection = this.hbck2.connect(); Hbck hbck = connection.getHbck()) { + TableState state = this.hbck2.setTableState(hbck, TABLE_NAME, TableState.State.ENABLED); + assertTrue("Found=" + state.getState(), state.isDisabled()); + } + } + + @Test + public void testUnAssigns() throws IOException { + try (Admin admin = TEST_UTIL.getConnection().getAdmin()) { + List regions = admin.getRegions(TABLE_NAME); + for (RegionInfo ri : regions) { + RegionState rs = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager(). + getRegionStates().getRegionState(ri.getEncodedName()); + LOG.info("RS: {}", rs.toString()); + } + String[] regionStrsArray = + regions.stream().map(RegionInfo::getEncodedName).toArray(String[]::new); + File testFile = new File(TEST_UTIL.getDataTestDir().toString(), "inputForUnAssignsTest"); + writeStringsToAFile(testFile, regionStrsArray); + String result = testRunWithArgs(new String[]{"-i", UNASSIGNS, testFile.toString()}); + validateRegionEndState(getPidsFromResult(result), regions, false); } } @@ -139,8 +179,7 @@ public void testAssigns() throws IOException { try (ClusterConnection connection = this.hbck2.connect(); Hbck hbck = connection.getHbck()) { unassigns(regions, regionStrsArray); List pids = this.hbck2.assigns(hbck, regionStrsArray); - waitOnPids(pids); - validateOpen(regions); + validateRegionEndState(pids,regions, true); // What happens if crappy region list passed? pids = this.hbck2.assigns(hbck, Arrays.stream(new String[]{"a", "some rubbish name"}). collect(Collectors.toList()).toArray(new String[]{})); @@ -151,20 +190,9 @@ public void testAssigns() throws IOException { // test input files unassigns(regions, regionStrsArray); File testFile = new File(TEST_UTIL.getDataTestDir().toString(), "inputForAssignsTest"); - try (FileOutputStream output = new FileOutputStream(testFile, false)) { - for (String regionStr : regionStrsArray) { - output.write((regionStr + System.lineSeparator()).getBytes()); - } - } - String result = testRunWithArgs(new String[]{ASSIGNS, "-i", testFile.toString()}); - Scanner scanner = new Scanner(result).useDelimiter("[\\D]+"); - pids = new ArrayList<>(); - while (scanner.hasNext()) { - pids.add(scanner.nextLong()); - } - scanner.close(); - waitOnPids(pids); - validateOpen(regions); + writeStringsToAFile(testFile, regionStrsArray); + String result = testRunWithArgs(new String[]{"-i", ASSIGNS, testFile.toString()}); + validateRegionEndState(getPidsFromResult(result), regions, true); } } } @@ -186,6 +214,49 @@ public void testSetRegionState() throws IOException { } } + @Test + public void testSetRegionStateWithArgsList() throws IOException { + TEST_UTIL.createTable(REGION_STATES_TABLE_NAME, Bytes.toBytes("family1")); + try (Admin admin = TEST_UTIL.getConnection().getAdmin()) { + List regions = admin.getRegions(REGION_STATES_TABLE_NAME); + RegionInfo info = regions.get(0); + assertEquals(RegionState.State.OPEN, getCurrentRegionState(info)); + String region = info.getEncodedName(); + String[] args = new String[]{region, "0", "CLOSING"}; + try (ClusterConnection connection = this.hbck2.connect()) { + this.hbck2.setRegionState(connection, args); + } + assertEquals(RegionState.State.CLOSING, getCurrentRegionState(info)); + } finally { + TEST_UTIL.deleteTable(REGION_STATES_TABLE_NAME); + } + } + + @Test + public void testSetRegionStateInputFiles() throws IOException { + TEST_UTIL.createTable(REGION_STATES_TABLE_NAME, Bytes.toBytes("family1")); + try (Admin admin = TEST_UTIL.getConnection().getAdmin()) { + List regions = admin.getRegions(REGION_STATES_TABLE_NAME); + String[] input = new String[regions.size()]; + for (int i = 0; i < regions.size(); i++) { + RegionInfo info = regions.get(i); + assertEquals(RegionState.State.OPEN, getCurrentRegionState(info)); + String region = info.getEncodedName(); + input[i] = region + " CLOSING"; + } + + File testFile = new File(TEST_UTIL.getDataTestDir().toString(), "inputForSetRegionStateTest"); + writeStringsToAFile(testFile, input); + testRunWithArgs(new String[]{"-i", SET_REGION_STATE, testFile.toString()}); + + for (RegionInfo info : regions) { + assertEquals(RegionState.State.CLOSING, getCurrentRegionState(info)); + } + } finally { + TEST_UTIL.deleteTable(REGION_STATES_TABLE_NAME); + } + } + @Test public void testSetReplicaRegionState() throws IOException, InterruptedException { TEST_UTIL.createTable(REGION_STATES_TABLE_NAME, Bytes.toBytes("family1")); @@ -302,6 +373,23 @@ public void testFormatReportMissingInMetaOneMissing() throws IOException { assertTrue(result.contains(expectedResult)); } + private void writeStringsToAFile(File testFile, String[] strs) throws IOException { + try (FileOutputStream output = new FileOutputStream(testFile, false)) { + for (String regionStr : strs) { + output.write((regionStr + System.lineSeparator()).getBytes()); + } + } + } + private List getPidsFromResult(String result) { + Scanner scanner = new Scanner(result).useDelimiter("[\\D]+"); + List pids = new ArrayList<>(); + while (scanner.hasNext()) { + pids.add(scanner.nextLong()); + } + scanner.close(); + return pids; + } + private void unassigns(List regions, String[] regionStrsArray) throws IOException { try (ClusterConnection connection = this.hbck2.connect(); Hbck hbck = connection.getHbck()) { List pids = this.hbck2.unassigns(hbck, regionStrsArray); @@ -315,13 +403,13 @@ private void unassigns(List regions, String[] regionStrsArray) throw } } - - private void validateOpen(List regions) { + private void validateRegionEndState(List pids, List regions, boolean open) { + waitOnPids(pids); for (RegionInfo ri : regions) { RegionState rs = TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager(). getRegionStates().getRegionState(ri.getEncodedName()); LOG.info("RS: {}", rs.toString()); - assertTrue(rs.toString(), rs.isOpened()); + assertTrue(rs.toString(), open? rs.isOpened() : rs.isClosed()); } } diff --git a/hbase-hbck2/src/test/java/org/apache/hbase/TestHBCKCommandLineParsing.java b/hbase-hbck2/src/test/java/org/apache/hbase/TestHBCKCommandLineParsing.java index da37648a84..ae7cc42082 100644 --- a/hbase-hbck2/src/test/java/org/apache/hbase/TestHBCKCommandLineParsing.java +++ b/hbase-hbck2/src/test/java/org/apache/hbase/TestHBCKCommandLineParsing.java @@ -21,6 +21,7 @@ import static org.junit.Assert.assertTrue; import java.io.ByteArrayOutputStream; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.io.PrintStream; @@ -66,10 +67,19 @@ public void testCommandWithOptions() throws IOException { hbck.run(new String[]{"bypass", "--lockWait=3", "--override", "--recursive", "x"}); } + @Test (expected=FileNotFoundException.class) + public void testInputFileOption() throws IOException { + HBCK2 hbck = new HBCK2(TEST_UTIL.getConfiguration()); + // The 'x' below should cause the io exception for file not found. + // The Options should all be good. + hbck.run(new String[]{"--inputFile", "bypass", "--override", "x"}); + } + @Test (expected=IllegalArgumentException.class) public void testSetRegionStateCommandInvalidState() throws IOException { HBCK2 hbck = new HBCK2(TEST_UTIL.getConfiguration()); - // The 'x' below should cause the IllegalArgumentException. The Options should all be good. + // The 'INVALID_STATE' below should cause the IllegalArgumentException. + // The Options should all be good. hbck.run(new String[]{"setRegionState", "region_encoded", "INVALID_STATE"}); }