Skip to content

HDDS-11660. Recon List Key API: Reduce object creation and buffering memory #7405

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
sodonnel merged 1 commit into from
Nov 8, 2024
Merged

HDDS-11660. Recon List Key API: Reduce object creation and buffering memory #7405

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 27 additions & 6 deletions hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -291,11 +291,34 @@ public void untarCheckpointFile(File tarFile, Path destPath)
*/
public static String constructFullPath(OmKeyInfo omKeyInfo,
ReconNamespaceSummaryManager reconNamespaceSummaryManager,
ReconOMMetadataManager omMetadataManager)
throws IOException {
ReconOMMetadataManager omMetadataManager) throws IOException {
return constructFullPath(omKeyInfo.getKeyName(), omKeyInfo.getParentObjectID(), omKeyInfo.getVolumeName(),
omKeyInfo.getBucketName(), reconNamespaceSummaryManager, omMetadataManager);
}

StringBuilder fullPath = new StringBuilder(omKeyInfo.getKeyName());
long parentId = omKeyInfo.getParentObjectID();
/**
* Constructs the full path of a key from its key name and parent ID using a bottom-up approach, starting from the
* leaf node.
*
* The method begins with the leaf node (the key itself) and recursively prepends parent directory names, fetched
* via NSSummary objects, until reaching the parent bucket (parentId is -1). It effectively builds the path from
* bottom to top, finally prepending the volume and bucket names to complete the full path. If the directory structure
* is currently being rebuilt (indicated by the rebuildTriggered flag), this method returns an empty string to signify
* that path construction is temporarily unavailable.
*
* @param keyName The name of the key
* @param initialParentId The parent ID of the key
* @param volumeName The name of the volume
* @param bucketName The name of the bucket
* @return The constructed full path of the key as a String, or an empty string if a rebuild is in progress and
* the path cannot be constructed at this time.
* @throws IOException
*/
public static String constructFullPath(String keyName, long initialParentId, String volumeName, String bucketName,
ReconNamespaceSummaryManager reconNamespaceSummaryManager,
ReconOMMetadataManager omMetadataManager) throws IOException {
StringBuilder fullPath = new StringBuilder(keyName);
long parentId = initialParentId;
boolean isDirectoryPresent = false;

while (parentId != 0) {
Expand All @@ -320,8 +343,6 @@ public static String constructFullPath(OmKeyInfo omKeyInfo,
}

// Prepend the volume and bucket to the constructed path
String volumeName = omKeyInfo.getVolumeName();
String bucketName = omKeyInfo.getBucketName();
fullPath.insert(0, volumeName + OM_KEY_PREFIX + bucketName + OM_KEY_PREFIX);
if (isDirectoryPresent) {
return OmUtils.normalizeKey(fullPath.toString(), true);
Expand Down
95 changes: 39 additions & 56 deletions hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/OMDBInsightEndpoint.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import org.apache.hadoop.ozone.recon.ReconUtils;
import org.apache.hadoop.ozone.recon.api.handlers.BucketHandler;
import org.apache.hadoop.ozone.recon.api.types.KeyEntityInfo;
import org.apache.hadoop.ozone.recon.api.types.KeyEntityInfoProtoWrapper;
import org.apache.hadoop.ozone.recon.api.types.KeyInsightInfoResponse;
import org.apache.hadoop.ozone.recon.api.types.ListKeysResponse;
import org.apache.hadoop.ozone.recon.api.types.NSSummary;
Expand All @@ -58,7 +59,6 @@
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;

Expand Down Expand Up @@ -989,7 +989,7 @@ public Response listKeys(@QueryParam("replicationType") String replicationType,
listKeysResponse = (ListKeysResponse) response.getEntity();
}

List<KeyEntityInfo> keyInfoList = listKeysResponse.getKeys();
List<KeyEntityInfoProtoWrapper> keyInfoList = listKeysResponse.getKeys();
if (!keyInfoList.isEmpty()) {
listKeysResponse.setLastKey(keyInfoList.get(keyInfoList.size() - 1).getKey());
}
Expand All @@ -1003,66 +1003,49 @@ private Response getListKeysResponse(ParamInfo paramInfo) {
listKeysResponse.setPath(paramInfo.getStartPrefix());
long replicatedTotal = 0;
long unreplicatedTotal = 0;
boolean keysFound = false; // Flag to track if any keys are found

// Search keys from non-FSO layout.
Map<String, OmKeyInfo> obsKeys;
Table<String, OmKeyInfo> keyTable =
omMetadataManager.getKeyTable(BucketLayout.LEGACY);
obsKeys = retrieveKeysFromTable(keyTable, paramInfo);
for (Map.Entry<String, OmKeyInfo> entry : obsKeys.entrySet()) {
keysFound = true;
KeyEntityInfo keyEntityInfo =
createKeyEntityInfoFromOmKeyInfo(entry.getKey(), entry.getValue());

listKeysResponse.getKeys().add(keyEntityInfo);
replicatedTotal += entry.getValue().getReplicatedSize();
unreplicatedTotal += entry.getValue().getDataSize();
}
Table<String, KeyEntityInfoProtoWrapper> keyTable =
omMetadataManager.getKeyTableLite(BucketLayout.LEGACY);
retrieveKeysFromTable(keyTable, paramInfo, listKeysResponse.getKeys());


// Search keys from FSO layout.
Map<String, OmKeyInfo> fsoKeys = searchKeysInFSO(paramInfo);
for (Map.Entry<String, OmKeyInfo> entry : fsoKeys.entrySet()) {
keysFound = true;
KeyEntityInfo keyEntityInfo =
createKeyEntityInfoFromOmKeyInfo(entry.getKey(), entry.getValue());

listKeysResponse.getKeys().add(keyEntityInfo);
replicatedTotal += entry.getValue().getReplicatedSize();
unreplicatedTotal += entry.getValue().getDataSize();
}
searchKeysInFSO(paramInfo, listKeysResponse.getKeys());

// If no keys were found, return a response indicating that no keys matched
if (!keysFound) {
if (listKeysResponse.getKeys().isEmpty()) {
return ReconResponseUtils.noMatchedKeysResponse(paramInfo.getStartPrefix());
}

for (KeyEntityInfoProtoWrapper keyEntityInfo : listKeysResponse.getKeys()) {
replicatedTotal += keyEntityInfo.getReplicatedSize();
unreplicatedTotal += keyEntityInfo.getSize();
}

// Set the aggregated totals in the response
listKeysResponse.setReplicatedDataSize(replicatedTotal);
listKeysResponse.setUnReplicatedDataSize(unreplicatedTotal);

return Response.ok(listKeysResponse).build();
} catch (IOException e) {
return ReconResponseUtils.createInternalServerErrorResponse(
"Error listing keys from OM DB: " + e.getMessage());
} catch (RuntimeException e) {
LOG.error("Error generating listKeys response", e);
return ReconResponseUtils.createInternalServerErrorResponse(
"Unexpected runtime error while searching keys in OM DB: " + e.getMessage());
} catch (Exception e) {
LOG.error("Error generating listKeys response", e);
return ReconResponseUtils.createInternalServerErrorResponse(
"Error listing keys from OM DB: " + e.getMessage());
}
}

public Map<String, OmKeyInfo> searchKeysInFSO(ParamInfo paramInfo)
public void searchKeysInFSO(ParamInfo paramInfo, List<KeyEntityInfoProtoWrapper> results)
throws IOException {
int originalLimit = paramInfo.getLimit();
Map<String, OmKeyInfo> matchedKeys = new LinkedHashMap<>();
// Convert the search prefix to an object path for FSO buckets
String startPrefixObjectPath = convertStartPrefixPathToObjectIdPath(paramInfo.getStartPrefix());
String[] names = parseRequestPath(startPrefixObjectPath);
Table<String, OmKeyInfo> fileTable =
omMetadataManager.getKeyTable(BucketLayout.FILE_SYSTEM_OPTIMIZED);
Table<String, KeyEntityInfoProtoWrapper> fileTable =
omMetadataManager.getKeyTableLite(BucketLayout.FILE_SYSTEM_OPTIMIZED);

// If names.length > 2, then the search prefix is at the level above bucket level hence
// no need to find parent or extract id's or find subpaths as the fileTable is
Expand All @@ -1075,7 +1058,7 @@ public Map<String, OmKeyInfo> searchKeysInFSO(ParamInfo paramInfo)
NSSummary parentSummary =
reconNamespaceSummaryManager.getNSSummary(parentId);
if (parentSummary == null) {
return matchedKeys;
return;
}
List<String> subPaths = new ArrayList<>();
// Add the initial search prefix object path because it can have both files and subdirectories with files.
Expand All @@ -1087,21 +1070,17 @@ public Map<String, OmKeyInfo> searchKeysInFSO(ParamInfo paramInfo)
// Iterate over the subpaths and retrieve the files
for (String subPath : subPaths) {
paramInfo.setStartPrefix(subPath);
matchedKeys.putAll(
retrieveKeysFromTable(fileTable, paramInfo));
paramInfo.setLimit(originalLimit - matchedKeys.size());
if (matchedKeys.size() >= originalLimit) {
retrieveKeysFromTable(fileTable, paramInfo, results);
if (results.size() >= paramInfo.getLimit()) {
break;
}
}
return matchedKeys;
return;
}

paramInfo.setStartPrefix(startPrefixObjectPath);
// Iterate over for bucket and volume level search
matchedKeys.putAll(
retrieveKeysFromTable(fileTable, paramInfo));
return matchedKeys;
retrieveKeysFromTable(fileTable, paramInfo, results);
}


Expand Down Expand Up @@ -1174,32 +1153,31 @@ public String convertStartPrefixPathToObjectIdPath(String startPrefixPath)
* @return A map of keys and their corresponding OmKeyInfo objects.
* @throws IOException If there are problems accessing the table.
*/
private Map<String, OmKeyInfo> retrieveKeysFromTable(
Table<String, OmKeyInfo> table, ParamInfo paramInfo)
private void retrieveKeysFromTable(
Table<String, KeyEntityInfoProtoWrapper> table, ParamInfo paramInfo, List<KeyEntityInfoProtoWrapper> results)
throws IOException {
boolean skipPrevKey = false;
String seekKey = paramInfo.getPrevKey();
Map<String, OmKeyInfo> matchedKeys = new LinkedHashMap<>();
try (
TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>> keyIter = table.iterator()) {
TableIterator<String, ? extends Table.KeyValue<String, KeyEntityInfoProtoWrapper>> keyIter = table.iterator()) {

if (!paramInfo.isSkipPrevKeyDone() && isNotBlank(seekKey)) {
skipPrevKey = true;
Table.KeyValue<String, OmKeyInfo> seekKeyValue =
Table.KeyValue<String, KeyEntityInfoProtoWrapper> seekKeyValue =
keyIter.seek(seekKey);

// check if RocksDB was able to seek correctly to the given key prefix
// if not, then return empty result
// In case of an empty prevKeyPrefix, all the keys are returned
if (seekKeyValue == null || (!seekKeyValue.getKey().equals(paramInfo.getPrevKey()))) {
return matchedKeys;
return;
}
} else {
keyIter.seek(paramInfo.getStartPrefix());
}

while (keyIter.hasNext()) {
Table.KeyValue<String, OmKeyInfo> entry = keyIter.next();
Table.KeyValue<String, KeyEntityInfoProtoWrapper> entry = keyIter.next();
String dbKey = entry.getKey();
if (!dbKey.startsWith(paramInfo.getStartPrefix())) {
break; // Exit the loop if the key no longer matches the prefix
Expand All @@ -1209,9 +1187,14 @@ private Map<String, OmKeyInfo> retrieveKeysFromTable(
continue;
}
if (applyFilters(entry, paramInfo)) {
matchedKeys.put(dbKey, entry.getValue());
KeyEntityInfoProtoWrapper keyEntityInfo = entry.getValue();
keyEntityInfo.setKey(dbKey);
keyEntityInfo.setPath(ReconUtils.constructFullPath(keyEntityInfo.getKeyName(), keyEntityInfo.getParentId(),
keyEntityInfo.getVolumeName(), keyEntityInfo.getBucketName(), reconNamespaceSummaryManager,
omMetadataManager));
results.add(keyEntityInfo);
paramInfo.setLastKey(dbKey);
if (matchedKeys.size() >= paramInfo.getLimit()) {
if (results.size() >= paramInfo.getLimit()) {
break;
}
}
Expand All @@ -1220,10 +1203,10 @@ private Map<String, OmKeyInfo> retrieveKeysFromTable(
LOG.error("Error retrieving keys from table for path: {}", paramInfo.getStartPrefix(), exception);
throw exception;
}
return matchedKeys;
}

private boolean applyFilters(Table.KeyValue<String, OmKeyInfo> entry, ParamInfo paramInfo) throws IOException {
private boolean applyFilters(Table.KeyValue<String, KeyEntityInfoProtoWrapper> entry, ParamInfo paramInfo)
throws IOException {

LOG.debug("Applying filters on : {}", entry.getKey());

Expand All @@ -1238,7 +1221,7 @@ private boolean applyFilters(Table.KeyValue<String, OmKeyInfo> entry, ParamInfo
return false;
}

return entry.getValue().getDataSize() >= paramInfo.getKeySize();
return entry.getValue().getSize() >= paramInfo.getKeySize();
}

/**
Expand Down
Loading