Skip to content

HDDS-6278 Improve memory profile for listStatus API call. #3053

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
adoroszlai merged 2 commits into from
Feb 10, 2022
Merged

HDDS-6278 Improve memory profile for listStatus API call. #3053

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3fccb3e
HDDS-6278 Improve memory profile for listStatus API call.
kerneltime Feb 8, 2022
77fb113
Update hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozo…
kerneltime Feb 10, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 25 additions & 23 deletions hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1397,13 +1397,20 @@ public void refresh(OmKeyInfo key) throws IOException {
refreshPipeline(Arrays.asList(key));
}

private boolean isKeyDeleted(String key, Table keyTable) {
CacheValue<OmKeyInfo> omKeyInfoCacheValue
= keyTable.getCacheValue(new CacheKey(key));
return omKeyInfoCacheValue != null
&& omKeyInfoCacheValue.getCacheValue() == null;
}

/**
* Helper function for listStatus to find key in TableCache.
*/
private void listStatusFindKeyInTableCache(
Iterator<Map.Entry<CacheKey<String>, CacheValue<OmKeyInfo>>> cacheIter,
String keyArgs, String startCacheKey, boolean recursive,
TreeMap<String, OzoneFileStatus> cacheKeyMap, Set<String> deletedKeySet) {
TreeMap<String, OzoneFileStatus> cacheKeyMap) {

while (cacheIter.hasNext()) {
Map.Entry<CacheKey<String>, CacheValue<OmKeyInfo>> entry =
Expand All @@ -1414,23 +1421,20 @@ private void listStatusFindKeyInTableCache(
}
OmKeyInfo cacheOmKeyInfo = entry.getValue().getCacheValue();
// cacheOmKeyInfo is null if an entry is deleted in cache
if (cacheOmKeyInfo != null) {
if (cacheKey.startsWith(startCacheKey) &&
cacheKey.compareTo(startCacheKey) >= 0) {
if (!recursive) {
String remainingKey = StringUtils.stripEnd(cacheKey.substring(
startCacheKey.length()), OZONE_URI_DELIMITER);
// For non-recursive, the remaining part of key can't have '/'
if (remainingKey.contains(OZONE_URI_DELIMITER)) {
continue;
}
if (cacheOmKeyInfo != null
&& cacheKey.startsWith(startCacheKey)
&& cacheKey.compareTo(startCacheKey) >= 0) {
if (!recursive) {
String remainingKey = StringUtils.stripEnd(cacheKey.substring(
startCacheKey.length()), OZONE_URI_DELIMITER);
// For non-recursive, the remaining part of key can't have '/'
if (remainingKey.contains(OZONE_URI_DELIMITER)) {
continue;
}
OzoneFileStatus fileStatus = new OzoneFileStatus(
cacheOmKeyInfo, scmBlockSize, !OzoneFSUtils.isFile(cacheKey));
cacheKeyMap.put(cacheKey, fileStatus);
}
} else {
deletedKeySet.add(cacheKey);
OzoneFileStatus fileStatus = new OzoneFileStatus(
cacheOmKeyInfo, scmBlockSize, !OzoneFSUtils.isFile(cacheKey));
cacheKeyMap.put(cacheKey, fileStatus);
}
}
}
Expand Down Expand Up @@ -1488,8 +1492,6 @@ public List<OzoneFileStatus> listStatus(OmKeyArgs args, boolean recursive,
String keyName = args.getKeyName();
// A map sorted by OmKey to combine results from TableCache and DB.
TreeMap<String, OzoneFileStatus> cacheKeyMap = new TreeMap<>();
// A set to keep track of keys deleted in cache but not flushed to DB.
Set<String> deletedKeySet = new TreeSet<>();

if (Strings.isNullOrEmpty(startKey)) {
OzoneFileStatus fileStatus = getFileStatus(args, clientAddress);
Expand Down Expand Up @@ -1519,7 +1521,7 @@ public List<OzoneFileStatus> listStatus(OmKeyArgs args, boolean recursive,

// First, find key in TableCache
listStatusFindKeyInTableCache(cacheIter, keyArgs, startCacheKey,
recursive, cacheKeyMap, deletedKeySet);
recursive, cacheKeyMap);
iterator = keyTable.iterator();
} finally {
metadataManager.getLock().releaseReadLock(BUCKET_LOCK, volumeName,
Expand All @@ -1545,7 +1547,8 @@ public List<OzoneFileStatus> listStatus(OmKeyArgs args, boolean recursive,
String entryKeyName = omKeyInfo.getKeyName();
if (recursive) {
// for recursive list all the entries
if (!deletedKeySet.contains(entryInDb)) {

if (!isKeyDeleted(entryInDb, keyTable)) {
cacheKeyMap.put(entryInDb, new OzoneFileStatus(omKeyInfo,
scmBlockSize, !OzoneFSUtils.isFile(entryKeyName)));
countEntries++;
Expand All @@ -1559,14 +1562,14 @@ public List<OzoneFileStatus> listStatus(OmKeyArgs args, boolean recursive,
.getImmediateChild(entryKeyName, keyName);
boolean isFile = OzoneFSUtils.isFile(immediateChild);
if (isFile) {
if (!deletedKeySet.contains(entryInDb)) {
if (!isKeyDeleted(entryInDb, keyTable)) {
cacheKeyMap.put(entryInDb,
new OzoneFileStatus(omKeyInfo, scmBlockSize, !isFile));
countEntries++;
}
} else {
// if entry is a directory
if (!deletedKeySet.contains(entryInDb)) {
if (!isKeyDeleted(entryInDb, keyTable)) {
if (!entryKeyName.equals(immediateChild)) {
OmKeyInfo fakeDirEntry = createDirectoryKey(
omKeyInfo.getVolumeName(), omKeyInfo.getBucketName(),
Expand Down Expand Up @@ -1605,7 +1608,6 @@ public List<OzoneFileStatus> listStatus(OmKeyArgs args, boolean recursive,
}
// Clean up temp map and set
cacheKeyMap.clear();
deletedKeySet.clear();

List<OmKeyInfo> keyInfoList = new ArrayList<>(fileStatusList.size());
fileStatusList.stream().map(s -> s.getKeyInfo()).forEach(keyInfoList::add);
Expand Down