Skip to content

HDDS-11231. Make Recon start more resilient #6987

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ArafatKhan2198 merged 7 commits into from
Jul 31, 2024
Merged

HDDS-11231. Make Recon start more resilient #6987

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
7f0c846
HDDS-11231. Ozone Recon - Make Recon restart more resilient and handl…
Jul 25, 2024
196de86
HDDS-11231. Ozone Recon - Make Recon restart more resilient and handl…
Jul 25, 2024
38e6e1a
HDDS-11231. Handled review comments.
Jul 26, 2024
65d3a3d
HDDS-11231. Handled review comments.
Jul 29, 2024
09bd998
HDDS-11231. Handled review comments.
Jul 29, 2024
dc9df05
HDDS-11231. Handled review comments.
Jul 29, 2024
eb236de
HDDS-11231. Handled review comments.
Jul 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion ...t/src/test/java/org/apache/hadoop/ozone/recon/TestReconInsightsForDeletedDirectories.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -476,7 +476,7 @@ private boolean assertTableRowCount(int expectedCount,
return count.get() == expectedCount;
}

private void syncDataFromOM() {
private void syncDataFromOM() throws IOException {
// Sync data from Ozone Manager to Recon.
OzoneManagerServiceProviderImpl impl = (OzoneManagerServiceProviderImpl)
cluster.getReconServer().getOzoneManagerServiceProvider();
Expand Down
43 changes: 43 additions & 0 deletions ...src/main/java/org/apache/hadoop/ozone/recon/spi/impl/OzoneManagerServiceProviderImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,9 +38,12 @@
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.stream.Collectors;

import com.google.common.collect.Iterators;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import org.apache.hadoop.hdds.recon.ReconConfigKeys;
import org.apache.hadoop.hdds.utils.db.RocksDatabase;
import org.apache.hadoop.hdds.utils.db.Table;
import org.apache.hadoop.hdds.utils.db.TableIterator;
import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteBatch;
import org.apache.hadoop.hdds.utils.db.managed.ManagedWriteOptions;
import org.apache.hadoop.hdfs.web.URLConnectionFactory;
Expand Down Expand Up @@ -260,6 +263,23 @@ public void start() {
omMetadataManager.start(configuration);
} catch (IOException ioEx) {
LOG.error("Error starting Recon OM Metadata Manager.", ioEx);
} catch (RuntimeException runtimeException) {
LOG.warn("Unexpected runtime error starting Recon OM Metadata Manager.", runtimeException);
LOG.warn("Trying to delete existing recon OM snapshot DB and fetch new one.");
metrics.incrNumSnapshotRequests();
LOG.info("Fetching full snapshot from Ozone Manager");
// Update local Recon OM DB to new snapshot.
try {
boolean success = updateReconOmDBWithNewSnapshot();
if (success) {
LOG.info("Successfully fetched a full snapshot from Ozone Manager");
} else {
LOG.error("Failed fetching a full snapshot from Ozone Manager");
}
} catch (IOException e) {
LOG.error("Unexpected IOException occurred while trying to fetch a full snapshot: {}", e);
throw new RuntimeException(runtimeException);
}
}
reconTaskController.start();
long initialDelay = configuration.getTimeDuration(
Expand Down Expand Up @@ -537,6 +557,7 @@ boolean innerGetAndApplyDeltaUpdatesFromOM(long fromSequenceNumber,
/**
* Based on current state of Recon's OM DB, we either get delta updates or
* full snapshot from Ozone Manager.
* @return true or false if sync operation between Recon and OM was successful or failed.
*/
@VisibleForTesting
public boolean syncDataFromOM() {
Expand Down Expand Up @@ -613,6 +634,7 @@ public boolean syncDataFromOM() {
reconContext.updateErrors(ReconContext.ErrorCode.GET_OM_DB_SNAPSHOT_FAILED);
}
}
printOMDBMetaInfo();
} finally {
isSyncDataFromOMRunning.set(false);
}
Expand All @@ -623,6 +645,27 @@ public boolean syncDataFromOM() {
return true;
}

private void printOMDBMetaInfo() {
printTableCount("fileTable");
printTableCount("keyTable");
}

private void printTableCount(String tableName) {
Table table = omMetadataManager.getTable(tableName);
if (table == null) {
LOG.error("Table {} not found in OM Metadata.", tableName);
return;
}
if (LOG.isDebugEnabled()) {
try (TableIterator<String, ? extends Table.KeyValue<String, ?>> iterator = table.iterator()) {
long count = Iterators.size(iterator);
LOG.debug("{} Table count: {}", tableName, count);
} catch (IOException ioException) {
LOG.error("Unexpected error while iterating table for table count: {}", tableName);
}
}
}

public void checkAndValidateReconDbPermissions() {
File dbDir = new File(reconDbDir.getPath());
if (!dbDir.exists()) {
Expand Down