Skip to content

HDDS-8565. Recon memory leak in NSSummary #8823

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ArafatKhan2198 merged 9 commits into from
Jul 22, 2025
Merged

HDDS-8565. Recon memory leak in NSSummary #8823

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
2cc3de2
HDDS-8565. Recon - memory leak in NSSummary in Recon.
Jul 17, 2025
b708fdc
HDDS-8565. Recon - memory leak in NSSummary in Recon.
Jul 17, 2025
fc5f946
HDDS-8565. Recon - memory leak in NSSummary in Recon.
Jul 18, 2025
eecef8d
HDDS-8565. Recon - memory leak in NSSummary in Recon.
Jul 18, 2025
c6d1471
HDDS-8565. Recon - memory leak in NSSummary in Recon.
Jul 18, 2025
b4cc98d
HDDS-8565. Fixing review comments and findbugs of Recon - memory lea…
Jul 18, 2025
93c9b8a
HDDS-8565. Fixing failing test cased of Recon - memory leak in NSSumm…
Jul 22, 2025
ee481ab
HDDS-8565. Fixing checkstyle of Recon - memory leak in NSSummary in R…
Jul 22, 2025
22b7004
HDDS-8565. Fixing failed test case of Recon - memory leak in NSSummar…
Jul 22, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions ...rc/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconNamespaceSummaryManagerImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

package org.apache.hadoop.ozone.recon.spi.impl;

import static org.apache.hadoop.hdds.utils.db.DBDefinition.LOG;
import static org.apache.hadoop.ozone.recon.spi.impl.ReconDBDefinition.NAMESPACE_SUMMARY;
import static org.apache.hadoop.ozone.recon.spi.impl.ReconDBProvider.truncateTable;

Expand Down Expand Up @@ -65,6 +66,7 @@ public void batchStoreNSSummaries(BatchOperation batch,
long objectId, NSSummary nsSummary)
throws IOException {
nsSummaryTable.putWithBatch(batch, objectId, nsSummary);
LOG.info("put in NSSummaryTable with objectId: {}, nsSummary: {}", objectId, nsSummary);
Comment thread
sumitagrawl marked this conversation as resolved.
Outdated
}

@Override
Expand Down
2 changes: 1 addition & 1 deletion hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTask.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -210,7 +210,7 @@ public TaskResult reprocess(OMMetadataManager omMetadataManager) {
TimeUnit.NANOSECONDS.toMillis(endTime - startTime);

// Log performance metrics
LOG.debug("Task execution time: {} milliseconds", durationInMillis);
LOG.info("Task execution time: {} milliseconds", durationInMillis);
}

return buildTaskResult(true);
Expand Down
226 changes: 164 additions & 62 deletions ...p-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTaskWithFSO.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@

package org.apache.hadoop.ozone.recon.tasks;

import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.DELETED_DIR_TABLE;
import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.DELETED_TABLE;
Comment thread
sumitagrawl marked this conversation as resolved.
Outdated
import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.DIRECTORY_TABLE;
import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.FILE_TABLE;

Expand Down Expand Up @@ -60,9 +62,9 @@ public NSSummaryTaskWithFSO(ReconNamespaceSummaryManager
this.nsSummaryFlushToDBMaxThreshold = nsSummaryFlushToDBMaxThreshold;
}

// We only listen to updates from FSO-enabled KeyTable(FileTable) and DirTable
// We listen to updates from FSO-enabled FileTable, DirTable, DeletedTable and DeletedDirTable
public Collection<String> getTaskTables() {
return Arrays.asList(FILE_TABLE, DIRECTORY_TABLE);
return Arrays.asList(FILE_TABLE, DIRECTORY_TABLE, DELETED_TABLE, DELETED_DIR_TABLE);
}

public Pair<Integer, Boolean> processWithFSO(OMUpdateEventBatch events,
Expand All @@ -83,9 +85,11 @@ public Pair<Integer, Boolean> processWithFSO(OMUpdateEventBatch events,
OMDBUpdateEvent.OMDBUpdateAction action = omdbUpdateEvent.getAction();
eventCounter++;

// we only process updates on OM's FileTable and Dirtable
// we process updates on OM's FileTable, DirTable, DeletedTable and DeletedDirTable
String table = omdbUpdateEvent.getTable();
boolean updateOnFileTable = table.equals(FILE_TABLE);
boolean updateOnDeletedTable = table.equals(DELETED_TABLE);
boolean updateOnDeletedDirTable = table.equals(DELETED_DIR_TABLE);
if (!taskTables.contains(table)) {
continue;
}
Expand All @@ -94,68 +98,19 @@ public Pair<Integer, Boolean> processWithFSO(OMUpdateEventBatch events,

try {
if (updateOnFileTable) {
// key update on fileTable
OMDBUpdateEvent<String, OmKeyInfo> keyTableUpdateEvent =
(OMDBUpdateEvent<String, OmKeyInfo>) omdbUpdateEvent;
OmKeyInfo updatedKeyInfo = keyTableUpdateEvent.getValue();
OmKeyInfo oldKeyInfo = keyTableUpdateEvent.getOldValue();

switch (action) {
case PUT:
handlePutKeyEvent(updatedKeyInfo, nsSummaryMap);
break;

case DELETE:
handleDeleteKeyEvent(updatedKeyInfo, nsSummaryMap);
break;

case UPDATE:
if (oldKeyInfo != null) {
// delete first, then put
handleDeleteKeyEvent(oldKeyInfo, nsSummaryMap);
} else {
LOG.warn("Update event does not have the old keyInfo for {}.",
updatedKey);
}
handlePutKeyEvent(updatedKeyInfo, nsSummaryMap);
break;
handleUpdateOnFileTable(omdbUpdateEvent, action, nsSummaryMap, updatedKey);

default:
LOG.debug("Skipping DB update event : {}",
omdbUpdateEvent.getAction());
}
} else if (updateOnDeletedTable) {
// Hard delete from deletedTable - cleanup memory leak for files
handleUpdateOnDeletedTable(omdbUpdateEvent, action, nsSummaryMap);

} else if (updateOnDeletedDirTable) {
// Hard delete from deletedDirectoryTable - cleanup memory leak for directories
handleUpdateOnDeletedDirTable((OMDBUpdateEvent<String, OmKeyInfo>) omdbUpdateEvent, action, nsSummaryMap);

} else {
// directory update on DirTable
OMDBUpdateEvent<String, OmDirectoryInfo> dirTableUpdateEvent =
(OMDBUpdateEvent<String, OmDirectoryInfo>) omdbUpdateEvent;
OmDirectoryInfo updatedDirectoryInfo = dirTableUpdateEvent.getValue();
OmDirectoryInfo oldDirectoryInfo = dirTableUpdateEvent.getOldValue();

switch (action) {
case PUT:
handlePutDirEvent(updatedDirectoryInfo, nsSummaryMap);
break;

case DELETE:
handleDeleteDirEvent(updatedDirectoryInfo, nsSummaryMap);
break;

case UPDATE:
if (oldDirectoryInfo != null) {
// delete first, then put
handleDeleteDirEvent(oldDirectoryInfo, nsSummaryMap);
} else {
LOG.warn("Update event does not have the old dirInfo for {}.",
updatedKey);
}
handlePutDirEvent(updatedDirectoryInfo, nsSummaryMap);
break;

default:
LOG.debug("Skipping DB update event : {}",
omdbUpdateEvent.getAction());
}
handleUpdateOnDirTable(omdbUpdateEvent, action, nsSummaryMap, updatedKey);
}
} catch (IOException ioEx) {
LOG.error("Unable to process Namespace Summary data in Recon DB. ",
Expand All @@ -179,6 +134,152 @@ public Pair<Integer, Boolean> processWithFSO(OMUpdateEventBatch events,
return new ImmutablePair<>(seekPos, true);
}

private void handleUpdateOnDirTable(OMDBUpdateEvent<String, ? extends WithParentObjectId> omdbUpdateEvent,
OMDBUpdateEvent.OMDBUpdateAction action, Map<Long, NSSummary> nsSummaryMap, String updatedKey)
throws IOException {
OMDBUpdateEvent<String, OmDirectoryInfo> dirTableUpdateEvent =
(OMDBUpdateEvent<String, OmDirectoryInfo>) omdbUpdateEvent;
OmDirectoryInfo updatedDirectoryInfo = dirTableUpdateEvent.getValue();
OmDirectoryInfo oldDirectoryInfo = dirTableUpdateEvent.getOldValue();

switch (action) {
case PUT:
handlePutDirEvent(updatedDirectoryInfo, nsSummaryMap);
break;

case DELETE:
handleDeleteDirEvent(updatedDirectoryInfo, nsSummaryMap);
break;

case UPDATE:
if (oldDirectoryInfo != null) {
// delete first, then put
handleDeleteDirEvent(oldDirectoryInfo, nsSummaryMap);
} else {
LOG.warn("Update event does not have the old dirInfo for {}.",
updatedKey);
}
handlePutDirEvent(updatedDirectoryInfo, nsSummaryMap);
break;

default:
LOG.debug("Skipping DB update event : {}",
omdbUpdateEvent.getAction());
}
}

private void handleUpdateOnDeletedDirTable(OMDBUpdateEvent<String, OmKeyInfo> omdbUpdateEvent,
OMDBUpdateEvent.OMDBUpdateAction action, Map<Long, NSSummary> nsSummaryMap) {
OMDBUpdateEvent<String, OmKeyInfo> deletedDirTableUpdateEvent =
omdbUpdateEvent;
OmKeyInfo deletedKeyInfo = deletedDirTableUpdateEvent.getValue();

switch (action) {
case DELETE:
// When entry is removed from deletedDirTable, remove from nsSummaryMap to prevent memory leak
if (deletedKeyInfo != null) {
long objectId = deletedKeyInfo.getObjectID();
nsSummaryMap.remove(objectId);
LOG.info("Removed hard deleted directory with objectId {} from nsSummaryMap", objectId);

// Also remove the parent directory's entry that references this deleted directory
long parentObjectId = deletedKeyInfo.getParentObjectID();
Comment thread
sumitagrawl marked this conversation as resolved.
Outdated
NSSummary parentSummary = nsSummaryMap.get(parentObjectId);
if (parentSummary != null) {
parentSummary.removeChildDir(objectId);
nsSummaryMap.put(parentObjectId, parentSummary);
LOG.info("Updated parent directory {} to remove child directory {}", parentObjectId, objectId);
}

// Delete the NSSummary entry from the database to prevent memory leak
try {
getReconNamespaceSummaryManager().deleteNSSummary(objectId);
Comment thread
sumitagrawl marked this conversation as resolved.
Outdated
LOG.info("Deleted NSSummary entry for objectId {} from database", objectId);
} catch (Exception e) {
LOG.error("Failed to delete NSSummary entry for objectId {} from database", objectId, e);
}
}
break;

default:
LOG.info("Skipping DB update event on deletedDirTable: {}", action);
Comment thread
sumitagrawl marked this conversation as resolved.
Outdated
}
}

private void handleUpdateOnDeletedTable(OMDBUpdateEvent<String, ? extends WithParentObjectId> omdbUpdateEvent,
Comment thread
sumitagrawl marked this conversation as resolved.
Outdated
OMDBUpdateEvent.OMDBUpdateAction action, Map<Long, NSSummary> nsSummaryMap) {
OMDBUpdateEvent<String, ?> deletedTableUpdateEvent = omdbUpdateEvent;
Object value = deletedTableUpdateEvent.getValue();

switch (action) {
case DELETE:
// When entry is removed from deletedTable, update parent directory in nsSummaryMap
if (value instanceof OmKeyInfo) {
OmKeyInfo deletedKeyInfo = (OmKeyInfo) value;
long parentObjectId = deletedKeyInfo.getParentObjectID();

// Update the parent directory's NSSummary to reflect the hard deleted file
NSSummary parentSummary = nsSummaryMap.get(parentObjectId);
if (parentSummary != null) {
// Decrement file count and size
parentSummary.setNumOfFiles(parentSummary.getNumOfFiles() - 1);
parentSummary.setSizeOfFiles(parentSummary.getSizeOfFiles() - deletedKeyInfo.getDataSize());

// Update file size bucket
int[] fileBucket = parentSummary.getFileSizeBucket();
int binIndex = org.apache.hadoop.ozone.recon.ReconUtils.getFileSizeBinIndex(deletedKeyInfo.getDataSize());
if (binIndex >= 0 && binIndex < fileBucket.length) {
fileBucket[binIndex] = Math.max(0, fileBucket[binIndex] - 1);
}
parentSummary.setFileSizeBucket(fileBucket);

nsSummaryMap.put(parentObjectId, parentSummary);
LOG.debug("Updated parent directory {} after hard deleted file with objectId {}", parentObjectId,
deletedKeyInfo.getObjectID());
}
}
break;

default:
LOG.info("Skipping DB update event on deletedTable: {}", action);
}
}

private void handleUpdateOnFileTable(OMDBUpdateEvent<String, ? extends WithParentObjectId> omdbUpdateEvent,
OMDBUpdateEvent.OMDBUpdateAction action, Map<Long, NSSummary> nsSummaryMap, String updatedKey)
throws IOException {
// key update on fileTable
OMDBUpdateEvent<String, OmKeyInfo> keyTableUpdateEvent =
(OMDBUpdateEvent<String, OmKeyInfo>) omdbUpdateEvent;
OmKeyInfo updatedKeyInfo = keyTableUpdateEvent.getValue();
OmKeyInfo oldKeyInfo = keyTableUpdateEvent.getOldValue();

switch (action) {
case PUT:
handlePutKeyEvent(updatedKeyInfo, nsSummaryMap);
break;

case DELETE:
handleDeleteKeyEvent(updatedKeyInfo, nsSummaryMap);
break;

case UPDATE:
if (oldKeyInfo != null) {
// delete first, then put
handleDeleteKeyEvent(oldKeyInfo, nsSummaryMap);
} else {
LOG.warn("Update event does not have the old keyInfo for {}.",
updatedKey);
}
handlePutKeyEvent(updatedKeyInfo, nsSummaryMap);
break;

default:
LOG.debug("Skipping DB update event : {}",
omdbUpdateEvent.getAction());
}
}

public boolean reprocessWithFSO(OMMetadataManager omMetadataManager) {
Map<Long, NSSummary> nsSummaryMap = new HashMap<>();

Expand Down Expand Up @@ -225,9 +326,10 @@ public boolean reprocessWithFSO(OMMetadataManager omMetadataManager) {
}
// flush and commit left out keys at end
if (!flushAndCommitNSToDB(nsSummaryMap)) {
LOG.info("flushAndCommitNSToDB failed during reprocessWithFSO.");
return false;
}
LOG.debug("Completed a reprocess run of NSSummaryTaskWithFSO");
LOG.info("Completed a reprocess run of NSSummaryTaskWithFSO");
return true;
}
}
Loading
Loading