From 7f22704367eb1dc5e1e35cee62ae7a7ddc25be4f Mon Sep 17 00:00:00 2001 From: Dieter De Paepe Date: Wed, 27 Mar 2024 16:01:46 +0100 Subject: [PATCH 1/2] HBASE-28460 Full backup restore failed on empty HFiles --- .../backup/mapreduce/MapReduceRestoreJob.java | 5 +-- .../hadoop/hbase/backup/util/RestoreTool.java | 12 +++++- .../hadoop/hbase/backup/TestFullRestore.java | 39 +++++++++++++++++++ 3 files changed, 50 insertions(+), 6 deletions(-) diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceRestoreJob.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceRestoreJob.java index 5d654c0d85b5..7a2fce4c418a 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceRestoreJob.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceRestoreJob.java @@ -87,10 +87,7 @@ public void run(Path[] dirPaths, TableName[] tableNames, Path restoreRootDir, LOG.debug("Restoring HFiles from directory " + bulkOutputPath); } - if (loader.bulkLoad(newTableNames[i], bulkOutputPath).isEmpty()) { - throw new IOException("Can not restore from backup directory " + dirs - + " (check Hadoop and HBase logs). Bulk loader returns null"); - } + loader.bulkLoad(newTableNames[i], bulkOutputPath); } else { throw new IOException("Can not restore from backup directory " + dirs + " (check Hadoop/MR and HBase logs). Player return code =" + result); diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/RestoreTool.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/RestoreTool.java index 8ca80d1301f6..0194bd42cc65 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/RestoreTool.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/RestoreTool.java @@ -437,6 +437,10 @@ byte[][] generateBoundaryKeys(ArrayList regionDirList) throws IOException HFile.Reader reader = HFile.createReader(fs, hfile, conf); final byte[] first, last; try { + if (reader.getEntries() == 0) { + LOG.debug("Skipping hfile with 0 entries: " + hfile); + continue; + } first = reader.getFirstRowKey().get(); last = reader.getLastRowKey().get(); LOG.debug("Trying to figure out region boundaries hfile=" + hfile + " first=" @@ -491,8 +495,12 @@ private void checkAndCreateTable(Connection conn, TableName targetTableName, admin.createTable(htd); } else { keys = generateBoundaryKeys(regionDirList); - // create table using table descriptor and region boundaries - admin.createTable(htd, keys); + if (keys.length > 0) { + // create table using table descriptor and region boundaries + admin.createTable(htd, keys); + } else { + admin.createTable(htd); + } } } catch (NamespaceNotFoundException e) { LOG.warn("There was no namespace and the same will be created"); diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestFullRestore.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestFullRestore.java index 385a6b3c5193..e72b23ae7dd1 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestFullRestore.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestFullRestore.java @@ -27,6 +27,7 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.backup.util.BackupUtils; import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.util.ToolRunner; import org.junit.ClassRule; @@ -71,6 +72,44 @@ public void testFullRestoreSingle() throws Exception { hba.close(); } + @Test + public void testFullRestoreSingleWithRegion() throws Exception { + LOG.info("test full restore on a single table empty table that has a region"); + + // This test creates its own table so other tests are not affected (we adjust it in this test) + TableName tableName = TableName.valueOf("table-full-restore-single-region"); + TEST_UTIL.createTable(tableName, famName); + + Admin admin = TEST_UTIL.getAdmin(); + + // Add & remove data to ensure a region is active, but functionally empty + Table table = TEST_UTIL.getConnection().getTable(tableName); + loadTable(table); + admin.flush(tableName); + TEST_UTIL.deleteTableData(tableName); + admin.flush(tableName); + + admin.majorCompact(tableName); + Thread.sleep(5_000); + + List tables = Lists.newArrayList(tableName); + String backupId = fullTableBackup(tables); + assertTrue(checkSucceeded(backupId)); + + LOG.info("backup complete"); + + TEST_UTIL.deleteTable(tableName); + + TableName[] tableset = new TableName[] { tableName }; + TableName[] tablemap = new TableName[] { tableName }; + BackupAdmin client = getBackupAdmin(); + client.restore(BackupUtils.createRestoreRequest(BACKUP_ROOT_DIR, backupId, false, tableset, + tablemap, false)); + assertTrue(admin.tableExists(tableName)); + TEST_UTIL.deleteTable(tableName); + admin.close(); + } + @Test public void testFullRestoreSingleCommand() throws Exception { LOG.info("test full restore on a single table empty table: command-line"); From 55df619a3a4d8cd13abfcfcaa9916f6bc7223c18 Mon Sep 17 00:00:00 2001 From: Dieter De Paepe Date: Tue, 2 Apr 2024 11:17:26 +0200 Subject: [PATCH 2/2] Process review comments --- .../java/org/apache/hadoop/hbase/backup/util/RestoreTool.java | 2 +- .../java/org/apache/hadoop/hbase/backup/TestFullRestore.java | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/RestoreTool.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/RestoreTool.java index 0194bd42cc65..ff4e2672f7a2 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/RestoreTool.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/RestoreTool.java @@ -61,7 +61,7 @@ */ @InterfaceAudience.Private public class RestoreTool { - public static final Logger LOG = LoggerFactory.getLogger(BackupUtils.class); + public static final Logger LOG = LoggerFactory.getLogger(RestoreTool.class); private final static long TABLE_AVAILABILITY_WAIT_TIME = 180000; private final String[] ignoreDirs = { HConstants.RECOVERED_EDITS_DIR }; diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestFullRestore.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestFullRestore.java index e72b23ae7dd1..d16d7af75014 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestFullRestore.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestFullRestore.java @@ -89,8 +89,7 @@ public void testFullRestoreSingleWithRegion() throws Exception { TEST_UTIL.deleteTableData(tableName); admin.flush(tableName); - admin.majorCompact(tableName); - Thread.sleep(5_000); + TEST_UTIL.compact(tableName, true); List tables = Lists.newArrayList(tableName); String backupId = fullTableBackup(tables);