Skip to content

HDDS-4364: [FSO]List FileStatus : startKey can be a non-existed path #3481

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rakeshadr merged 4 commits into from
Jun 6, 2022
Merged

HDDS-4364: [FSO]List FileStatus : startKey can be a non-existed path #3481

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
4d7ea44
HDDS-4364: [FSO]List FileStatus : startKey can be a non-existed path
Jun 3, 2022
203d2ab
Fixed static tools warn
Jun 3, 2022
cc16af9
Handled startKey ending with trailing slash and added test cases
Jun 4, 2022
e8dad3e
Fixed review comments
Jun 6, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
118 changes: 72 additions & 46 deletions hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@

import static org.apache.hadoop.ozone.OzoneConsts.QUOTA_RESET;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.FILE_NOT_FOUND;

/**
* A class that encapsulates OzoneBucket.
Expand Down Expand Up @@ -1107,7 +1108,8 @@ private void getSeekPathsBetweenKeyPrefixAndStartKey(String keyPrefix,

String parentStartKeyPath = OzoneFSUtils.getParentDir(startKey);

if (StringUtils.compare(parentStartKeyPath, keyPrefix) >= 0) {
if (StringUtils.isNotBlank(startKey) &&
StringUtils.compare(parentStartKeyPath, keyPrefix) >= 0) {
seekPaths.add(new ImmutablePair<>(parentStartKeyPath, startKey));

// recursively fetch all the sub-paths between keyPrefix and prevKey
Expand All @@ -1132,46 +1134,65 @@ List<OzoneKey> getNextListOfKeys(String prevKey) throws IOException {
}
setKeyPrefix(keyPrefixName);

if (StringUtils.isNotBlank(prevKey) &&
StringUtils.startsWith(prevKey, getKeyPrefix())) {
// 1. Prepare all the seekKeys after the prefixKey.
// Example case: prefixKey="a1", startKey="a1/b2/d2/f3/f31.tx"
// Now, stack should be build with all the levels after prefixKey
// Stack format => <keyPrefix and startKey>, startKey should be an
// immediate child of keyPrefix.
// _______________________________________
// Stack=> top | < a1/b2/d2/f3, a1/b2/d2/f3/f31.tx > |
// |-------------------------------------|
// | < a1/b2/d2, a1/b2/d2/f3 > |
// |-------------------------------------|
// | < a1/b2, a1/b2/d2 > |
// |-------------------------------------|
// bottom | < a1, a1/b2 > |
// --------------------------------------|
List<Pair<String, String>> seekPaths = new ArrayList<>();

if (StringUtils.isNotBlank(getKeyPrefix())) {
String parentStartKeyPath = OzoneFSUtils.getParentDir(prevKey);
if (StringUtils.compare(parentStartKeyPath, getKeyPrefix()) >= 0) {
// Add the leaf node to the seek path. The idea is to search for
// sub-paths if the given start key is a directory.
if (StringUtils.isNotBlank(prevKey)) {
if (StringUtils.startsWith(prevKey, getKeyPrefix())) {
// 1. Prepare all the seekKeys after the prefixKey.
// Example case: prefixKey="a1", startKey="a1/b2/d2/f3/f31.tx"
// Now, stack should be build with all the levels after prefixKey
// Stack format => <keyPrefix and startKey>, startKey should be an
// immediate child of keyPrefix.
// _______________________________________
// Stack=> top | < a1/b2/d2/f3, a1/b2/d2/f3/f31.tx > |
// |-------------------------------------|
// | < a1/b2/d2, a1/b2/d2/f3 > |
// |-------------------------------------|
// | < a1/b2, a1/b2/d2 > |
// |-------------------------------------|
// bottom | < a1, a1/b2 > |
// --------------------------------------|
List<Pair<String, String>> seekPaths = new ArrayList<>();

if (StringUtils.isNotBlank(getKeyPrefix())) {
String parentStartKeyPath = OzoneFSUtils.getParentDir(prevKey);
if (StringUtils.compare(parentStartKeyPath, getKeyPrefix()) >=
0) {
// Add the leaf node to the seek path. The idea is to search for
// sub-paths if the given start key is a directory.
seekPaths.add(new ImmutablePair<>(prevKey, ""));
removeStartKey = prevKey;
getSeekPathsBetweenKeyPrefixAndStartKey(getKeyPrefix(), prevKey,
seekPaths);
} else if (StringUtils.compare(prevKey, getKeyPrefix()) >= 0) {
// Add the leaf node to the seek path. The idea is to search for
// sub-paths if the given start key is a directory.
seekPaths.add(new ImmutablePair<>(prevKey, ""));
removeStartKey = prevKey;
}
} else {
// Key Prefix is Blank. The seek all the keys with startKey.
seekPaths.add(new ImmutablePair<>(prevKey, ""));
removeStartKey = prevKey;
getSeekPathsBetweenKeyPrefixAndStartKey(getKeyPrefix(), prevKey,
seekPaths);
} else if (StringUtils.compare(prevKey, getKeyPrefix()) >= 0) {
// Add the leaf node to the seek path. The idea is to search for
// sub-paths if the given start key is a directory.
seekPaths.add(new ImmutablePair<>(prevKey, ""));
removeStartKey = prevKey;
}
}

// 2. Push elements in reverse order so that the FS tree traversal
// will occur in left-to-right fashion[Depth-First Search]
for (int index = seekPaths.size() - 1; index >= 0; index--) {
Pair<String, String> seekDirPath = seekPaths.get(index);
stack.push(seekDirPath);
// 2. Push elements in reverse order so that the FS tree traversal
// will occur in left-to-right fashion[Depth-First Search]
for (int index = seekPaths.size() - 1; index >= 0; index--) {
Pair<String, String> seekDirPath = seekPaths.get(index);
stack.push(seekDirPath);
}
} else if (StringUtils.isNotBlank(getKeyPrefix())) {
if (!OzoneFSUtils.isSibling(prevKey, getKeyPrefix())) {
// Case-1 - sibling: keyPrefix="a1/b2", startKey="a0/b123Invalid"
// Skip traversing, if the startKey is not a sibling.
return new ArrayList<>();
} else if (StringUtils.compare(prevKey, getKeyPrefix()) < 0) {
// Case-2 - compare: keyPrefix="a1/b2", startKey="a1/b123Invalid"
// Since startKey is lexographically behind keyPrefix,
// the seek precedence goes to keyPrefix.
stack.push(new ImmutablePair<>(getKeyPrefix(), ""));
}
}
}
}
Expand Down Expand Up @@ -1374,28 +1395,33 @@ private void addKeyPrefixInfoToResultList(String keyPrefix,
return;
}

// TODO: HDDS-4859 will fix the case where startKey not started with
// keyPrefix.

OzoneFileStatus status = proxy.getOzoneFileStatus(volumeName, name,
keyPrefix);
OzoneFileStatus status = null;
try {
status = proxy.getOzoneFileStatus(volumeName, name,
keyPrefix);
} catch (OMException ome) {
if (ome.getResult() == FILE_NOT_FOUND) {
// keyPrefix path can't be found and skip adding it to result list
return;
}
}

if (status != null) {
OmKeyInfo keyInfo = status.getKeyInfo();
String keyName = keyInfo.getKeyName();

// removeStartKey - as the startKey is a placeholder, which is
// managed internally to traverse leaf node's sub-paths.
if (StringUtils.equals(keyName, removeStartKey)) {
return;
}

if (status.isDirectory()) {
// add trailing slash to represent directory
keyName =
OzoneFSUtils.addTrailingSlashIfNeeded(keyInfo.getKeyName());
}

// removeStartKey - as the startKey is a placeholder, which is
// managed internally to traverse leaf node's sub-paths.
if (StringUtils.equals(keyName, removeStartKey)) {
return;
}

OzoneKey ozoneKey = new OzoneKey(keyInfo.getVolumeName(),
keyInfo.getBucketName(), keyName,
keyInfo.getDataSize(), keyInfo.getCreationTime(),
Expand Down
4 changes: 4 additions & 0 deletions hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFSUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,10 @@ public static boolean isSibling(String parentKey, String childKey) {
java.nio.file.Path childParent = childPath.getParent();
java.nio.file.Path parentParent = parentPath.getParent();

if (childParent != null && parentParent != null) {
return childParent.equals(parentParent);
}

return childParent == parentParent;
}

Expand Down
Loading