diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconInsightsForDeletedDirectories.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconInsightsForDeletedDirectories.java index ca8fcae6643b..8656682cd151 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconInsightsForDeletedDirectories.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconInsightsForDeletedDirectories.java @@ -476,7 +476,7 @@ private boolean assertTableRowCount(int expectedCount, return count.get() == expectedCount; } - private void syncDataFromOM() { + private void syncDataFromOM() throws IOException { // Sync data from Ozone Manager to Recon. OzoneManagerServiceProviderImpl impl = (OzoneManagerServiceProviderImpl) cluster.getReconServer().getOzoneManagerServiceProvider(); diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/OzoneManagerServiceProviderImpl.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/OzoneManagerServiceProviderImpl.java index bde89eea1da8..491d631249cd 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/OzoneManagerServiceProviderImpl.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/OzoneManagerServiceProviderImpl.java @@ -38,9 +38,12 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; +import com.google.common.collect.Iterators; import com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.hadoop.hdds.recon.ReconConfigKeys; import org.apache.hadoop.hdds.utils.db.RocksDatabase; +import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.hdds.utils.db.TableIterator; import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteBatch; import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteOptions; import org.apache.hadoop.hdfs.web.URLConnectionFactory; @@ -260,6 +263,23 @@ public void start() { omMetadataManager.start(configuration); } catch (IOException ioEx) { LOG.error("Error starting Recon OM Metadata Manager.", ioEx); + } catch (RuntimeException runtimeException) { + LOG.warn("Unexpected runtime error starting Recon OM Metadata Manager.", runtimeException); + LOG.warn("Trying to delete existing recon OM snapshot DB and fetch new one."); + metrics.incrNumSnapshotRequests(); + LOG.info("Fetching full snapshot from Ozone Manager"); + // Update local Recon OM DB to new snapshot. + try { + boolean success = updateReconOmDBWithNewSnapshot(); + if (success) { + LOG.info("Successfully fetched a full snapshot from Ozone Manager"); + } else { + LOG.error("Failed fetching a full snapshot from Ozone Manager"); + } + } catch (IOException e) { + LOG.error("Unexpected IOException occurred while trying to fetch a full snapshot: {}", e); + throw new RuntimeException(runtimeException); + } } reconTaskController.start(); long initialDelay = configuration.getTimeDuration( @@ -537,6 +557,7 @@ boolean innerGetAndApplyDeltaUpdatesFromOM(long fromSequenceNumber, /** * Based on current state of Recon's OM DB, we either get delta updates or * full snapshot from Ozone Manager. + * @return true or false if sync operation between Recon and OM was successful or failed. */ @VisibleForTesting public boolean syncDataFromOM() { @@ -613,6 +634,7 @@ public boolean syncDataFromOM() { reconContext.updateErrors(ReconContext.ErrorCode.GET_OM_DB_SNAPSHOT_FAILED); } } + printOMDBMetaInfo(); } finally { isSyncDataFromOMRunning.set(false); } @@ -623,6 +645,27 @@ public boolean syncDataFromOM() { return true; } + private void printOMDBMetaInfo() { + printTableCount("fileTable"); + printTableCount("keyTable"); + } + + private void printTableCount(String tableName) { + Table table = omMetadataManager.getTable(tableName); + if (table == null) { + LOG.error("Table {} not found in OM Metadata.", tableName); + return; + } + if (LOG.isDebugEnabled()) { + try (TableIterator> iterator = table.iterator()) { + long count = Iterators.size(iterator); + LOG.debug("{} Table count: {}", tableName, count); + } catch (IOException ioException) { + LOG.error("Unexpected error while iterating table for table count: {}", tableName); + } + } + } + public void checkAndValidateReconDbPermissions() { File dbDir = new File(reconDbDir.getPath()); if (!dbDir.exists()) {