Skip to content

HDDS-8877. Intermittent failure in TestOzoneFileSystem#testListStatusOnKeyNameContainDelimiter #5093

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
chungen0126 wants to merge 9 commits into from
Closed

HDDS-8877. Intermittent failure in TestOzoneFileSystem#testListStatusOnKeyNameContainDelimiter #5093

Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
5070474
fix listStatus in table cache.
chungen0126 Jul 19, 2023
38bab15
fix list status in table cache with recursive
chungen0126 Jul 19, 2023
65b7845
add javadoc
chungen0126 Jul 28, 2023
cbd4ddf
Merge branch 'master' into HDDS-8877
chungen0126 Aug 31, 2023
205b2b4
fix consistent between cache and DB
chungen0126 Sep 5, 2023
1cd77af
clean up isKeyDeleted
chungen0126 Sep 5, 2023
230b930
fix conflict
chungen0126 Sep 18, 2023
612fc3a
Merge branch 'master' into HDDS-8877
chungen0126 Sep 25, 2023
035157a
restore OzoneListStatusHelper
chungen0126 Sep 25, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
131 changes: 69 additions & 62 deletions hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1405,12 +1405,22 @@ public static boolean isKeyDeleted(String key, Table keyTable) {

/**
* Helper function for listStatus to find key in TableCache.
* If enableFileSystemPaths is false, create a fakeDir and add to cacheKeyMap
* @param recursive For a directory if true all the descendants of a
* particular directory are listed
* @param startKeyInTable Key from which listing needs to start. If startKey
* exists its status is included in the final list.
* @param keyName Key name of Key args.
* @param cacheKeyMap A map of keys to list.
* @param keyArgs A string converted from Key args.
* @param keyTable Key table from metadata manager.
*/
private void listStatusFindKeyInTableCache(
Iterator<Map.Entry<CacheKey<String>, CacheValue<OmKeyInfo>>> cacheIter,
String keyArgs, String startCacheKey, boolean recursive,
TreeMap<String, OzoneFileStatus> cacheKeyMap) {

boolean recursive, String startKeyInTable, String keyName,
TreeMap<String, OzoneFileStatus> cacheKeyMap, String keyArgs,
Table<String, OmKeyInfo> keyTable) throws IOException {
Iterator<Map.Entry<CacheKey<String>, CacheValue<OmKeyInfo>>>
cacheIter = keyTable.cacheIterator();
while (cacheIter.hasNext()) {
Map.Entry<CacheKey<String>, CacheValue<OmKeyInfo>> entry =
cacheIter.next();
Expand All @@ -1421,19 +1431,39 @@ private void listStatusFindKeyInTableCache(
OmKeyInfo cacheOmKeyInfo = entry.getValue().getCacheValue();
// cacheOmKeyInfo is null if an entry is deleted in cache
if (cacheOmKeyInfo != null
&& cacheKey.startsWith(startCacheKey)
&& cacheKey.compareTo(startCacheKey) >= 0) {
&& cacheKey.startsWith(keyArgs)
&& cacheKey.compareTo(startKeyInTable) >= 0) {
OzoneFileStatus fileStatus = new OzoneFileStatus(
cacheOmKeyInfo, scmBlockSize, !OzoneFSUtils.isFile(cacheKey));
if (!recursive) {
String remainingKey = StringUtils.stripEnd(cacheKey.substring(
startCacheKey.length()), OZONE_URI_DELIMITER);
// For non-recursive, the remaining part of key can't have '/'
if (remainingKey.contains(OZONE_URI_DELIMITER)) {
continue;
if (enableFileSystemPaths) {
String remainingKey = StringUtils.stripEnd(cacheKey.substring(
keyArgs.length()), OZONE_URI_DELIMITER);
// For non-recursive, the remaining part of key can't have '/'
if (remainingKey.contains(OZONE_URI_DELIMITER)) {
continue;
}
cacheKeyMap.putIfAbsent(cacheKey, fileStatus);
} else {
String entryKeyName = cacheOmKeyInfo.getKeyName();
String immediateChild = OzoneFSUtils.getImmediateChild(
entryKeyName, keyName);
if (entryKeyName.equals(immediateChild)) {
cacheKeyMap.putIfAbsent(cacheKey, fileStatus);
} else {
//If immediateChild isn't equal entryKeyName, it is a fakeDir.
OmKeyInfo fakeDirEntry = createDirectoryKey(
cacheOmKeyInfo, immediateChild);
String fakeDirKey = metadataManager.getOzoneDirKey(
fakeDirEntry.getVolumeName(), fakeDirEntry.getBucketName(),
immediateChild);
cacheKeyMap.putIfAbsent(fakeDirKey, new OzoneFileStatus(
fakeDirEntry, scmBlockSize, true));
}
}
} else {
cacheKeyMap.putIfAbsent(cacheKey, fileStatus);
}
OzoneFileStatus fileStatus = new OzoneFileStatus(
cacheOmKeyInfo, scmBlockSize, !OzoneFSUtils.isFile(cacheKey));
cacheKeyMap.putIfAbsent(cacheKey, fileStatus);
// This else block has been added to capture deleted entries in cache.
// Adding deleted entries in cacheKeyMap as there is a possible race
// condition where table cache iterator is flushed already when
Expand Down Expand Up @@ -1489,14 +1519,15 @@ public List<OzoneFileStatus> listStatus(OmKeyArgs args, boolean recursive,
String startKey, long numEntries, String clientAddress,
boolean allowPartialPrefixes) throws IOException {
Preconditions.checkNotNull(args, "Key args can not be null");
String volName = args.getVolumeName();
String buckName = args.getBucketName();
String volumeName = args.getVolumeName();
String bucketName = args.getBucketName();
String keyName = args.getKeyName();
List<OzoneFileStatus> fileStatusList = new ArrayList<>();
if (numEntries <= 0) {
return fileStatusList;
}

if (isBucketFSOptimized(volName, buckName)) {
if (isBucketFSOptimized(volumeName, bucketName)) {
Preconditions.checkArgument(!recursive);
OzoneListStatusHelper statusHelper =
new OzoneListStatusHelper(metadataManager, scmBlockSize,
Expand All @@ -1508,9 +1539,6 @@ public List<OzoneFileStatus> listStatus(OmKeyArgs args, boolean recursive,
return buildFinalStatusList(statuses, args, clientAddress);
}

String volumeName = args.getVolumeName();
String bucketName = args.getBucketName();
String keyName = args.getKeyName();
// A map sorted by OmKey to combine results from TableCache and DB.
TreeMap<String, OzoneFileStatus> cacheKeyMap = new TreeMap<>();

Expand All @@ -1527,25 +1555,20 @@ public List<OzoneFileStatus> listStatus(OmKeyArgs args, boolean recursive,
String keyArgs = OzoneFSUtils.addTrailingSlashIfNeeded(
metadataManager.getOzoneKey(volumeName, bucketName, keyName));

TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>> iterator;
Table<String, OmKeyInfo> keyTable;
metadataManager.getLock().acquireReadLock(BUCKET_LOCK, volumeName,
bucketName);
try {
keyTable = metadataManager
.getKeyTable(getBucketLayout(metadataManager, volName, buckName));
iterator = getIteratorForKeyInTableCache(recursive, startKey,
volumeName, bucketName, cacheKeyMap, keyArgs, keyTable);
Table<String, OmKeyInfo> keyTable = metadataManager.getKeyTable(
getBucketLayout(metadataManager, volumeName, bucketName));
String startKeyInTable = metadataManager.getOzoneKey(
volumeName, bucketName, startKey);
listStatusFindKeyInTableCache(recursive, startKeyInTable, keyName,
cacheKeyMap, keyArgs, keyTable);
listStatusFindKeyInDb(recursive, startKeyInTable, numEntries,
keyName, cacheKeyMap, keyArgs, keyTable);
} finally {
metadataManager.getLock().releaseReadLock(BUCKET_LOCK, volumeName,
bucketName);
}

try {
findKeyInDbWithIterator(recursive, startKey, numEntries, volumeName,
bucketName, keyName, cacheKeyMap, keyArgs, keyTable, iterator);
} finally {
iterator.close();
bucketName);
}

int countEntries;
Expand Down Expand Up @@ -1584,36 +1607,15 @@ public List<OzoneFileStatus> listStatus(OmKeyArgs args, boolean recursive,
return fileStatusList;
}

private TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>>
getIteratorForKeyInTableCache(
boolean recursive, String startKey, String volumeName, String bucketName,
TreeMap<String, OzoneFileStatus> cacheKeyMap, String keyArgs,
Table<String, OmKeyInfo> keyTable) throws IOException {
TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>> iterator;
Iterator<Map.Entry<CacheKey<String>, CacheValue<OmKeyInfo>>>
cacheIter = keyTable.cacheIterator();
String startCacheKey = OZONE_URI_DELIMITER + volumeName +
OZONE_URI_DELIMITER + bucketName + OZONE_URI_DELIMITER +
((startKey.equals(OZONE_URI_DELIMITER)) ? "" : startKey);

// First, find key in TableCache
listStatusFindKeyInTableCache(cacheIter, keyArgs, startCacheKey,
recursive, cacheKeyMap);
iterator = keyTable.iterator();
return iterator;
}

@SuppressWarnings("parameternumber")
private void findKeyInDbWithIterator(boolean recursive, String startKey,
long numEntries, String volumeName, String bucketName, String keyName,
TreeMap<String, OzoneFileStatus> cacheKeyMap, String keyArgs,
Table<String, OmKeyInfo> keyTable,
TableIterator<String,
? extends Table.KeyValue<String, OmKeyInfo>> iterator)
private void listStatusFindKeyInDb(
boolean recursive, String seekKeyInDb, long numEntries, String keyName,
TreeMap<String, OzoneFileStatus>
cacheKeyMap, String keyArgs, Table<String, OmKeyInfo> keyTable)
throws IOException {
TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>> iterator
= keyTable.iterator();
// Then, find key in DB
String seekKeyInDb =
metadataManager.getOzoneKey(volumeName, bucketName, startKey);
Table.KeyValue<String, OmKeyInfo> entry = iterator.seek(seekKeyInDb);
int countEntries = 0;
if (iterator.hasNext()) {
Expand All @@ -1626,6 +1628,8 @@ private void findKeyInDbWithIterator(boolean recursive, String startKey,
entry = iterator.next();
String entryInDb = entry.getKey();
OmKeyInfo omKeyInfo = entry.getValue();
String volumeName = omKeyInfo.getVolumeName();
String bucketName = omKeyInfo.getBucketName();
if (entryInDb.startsWith(keyArgs)) {
String entryKeyName = omKeyInfo.getKeyName();
if (recursive) {
Expand Down Expand Up @@ -1655,7 +1659,9 @@ private void findKeyInDbWithIterator(boolean recursive, String startKey,
if (!entryKeyName.equals(immediateChild)) {
OmKeyInfo fakeDirEntry = createDirectoryKey(
omKeyInfo, immediateChild);
cacheKeyMap.put(entryInDb,
String fakeDirKey = metadataManager.getOzoneDirKey(
volumeName, bucketName, immediateChild);
cacheKeyMap.put(fakeDirKey,
new OzoneFileStatus(fakeDirEntry,
scmBlockSize, true));
} else {
Expand All @@ -1675,6 +1681,7 @@ private void findKeyInDbWithIterator(boolean recursive, String startKey,
}
}
}
iterator.close();
}

private List<OzoneFileStatus> buildFinalStatusList(
Expand Down