Skip to content

HDDS-4859. [FSO]ListKeys: seek all the files/dirs from startKey to keyPrefix #3466

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rakeshadr merged 2 commits into from
Jun 2, 2022
Merged

HDDS-4859. [FSO]ListKeys: seek all the files/dirs from startKey to keyPrefix #3466

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
fa793b0
HDDS-4859. [FSO]ListKeys: seek all the files/dirs from startKey to ke…
May 31, 2022
855fd7b
Fixed test failure and static tool warns
May 31, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
121 changes: 106 additions & 15 deletions hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.hdds.client.DefaultReplicationConfig;
import org.apache.hadoop.hdds.client.OzoneQuota;
import org.apache.hadoop.hdds.client.ReplicationConfig;
Expand Down Expand Up @@ -1050,9 +1052,10 @@ List<OzoneKey> getNextListOfKeys(String prevKey) throws
*/
private class KeyIteratorWithFSO extends KeyIterator {

private Stack<String> stack;
private Stack<Pair<String, String>> stack;
private List<OzoneKey> pendingItemsToBeBatched;
private boolean addedKeyPrefix;
private String removeStartKey = "";

/**
* Creates an Iterator to iterate over all keys after prevKey in the bucket.
Expand All @@ -1066,6 +1069,31 @@ private class KeyIteratorWithFSO extends KeyIterator {
super(keyPrefix, prevKey);
}

/**
* keyPrefix="a1" and startKey="a1/b2/d2/f3/f31.tx".
* Now, this function will prepare and return a list :
* <"a1/b2/d2", "a1/b2/d2/f3">
* <"a1/b2", "a1/b2/d2">
* <"a1", "a1/b2">
*
* @param keyPrefix keyPrefix
* @param startKey startKey
* @param seekPaths list of seek paths between keyPrefix and startKey
*/
private void getSeekPathsBetweenKeyPrefixAndStartKey(String keyPrefix,
String startKey, List<Pair<String, String>> seekPaths) {

String parentStartKeyPath = OzoneFSUtils.getParentDir(startKey);

if (StringUtils.compare(parentStartKeyPath, keyPrefix) >= 0) {
seekPaths.add(new ImmutablePair<>(parentStartKeyPath, startKey));

// recursively fetch all the sub-paths between keyPrefix and prevKey
getSeekPathsBetweenKeyPrefixAndStartKey(keyPrefix, parentStartKeyPath,
seekPaths);
}
}

@Override
List<OzoneKey> getNextListOfKeys(String prevKey) throws IOException {
if (stack == null) {
Expand All @@ -1081,14 +1109,62 @@ List<OzoneKey> getNextListOfKeys(String prevKey) throws IOException {
keyPrefixName = OmUtils.normalizeKey(getKeyPrefix(), true);
}
setKeyPrefix(keyPrefixName);

if (StringUtils.isNotBlank(prevKey) &&
StringUtils.startsWith(prevKey, getKeyPrefix())) {
// 1. Prepare all the seekKeys after the prefixKey.
// Example case: prefixKey="a1", startKey="a1/b2/d2/f3/f31.tx"
// Now, stack should be build with all the levels after prefixKey
// Stack format => <keyPrefix and startKey>, startKey should be an
// immediate child of keyPrefix.
// _______________________________________
// Stack=> top | < a1/b2/d2/f3, a1/b2/d2/f3/f31.tx > |
// |-------------------------------------|
// | < a1/b2/d2, a1/b2/d2/f3 > |
// |-------------------------------------|
// | < a1/b2, a1/b2/d2 > |
// |-------------------------------------|
// bottom | < a1, a1/b2 > |
// --------------------------------------|
List<Pair<String, String>> seekPaths = new ArrayList<>();

if (StringUtils.isNotBlank(getKeyPrefix())) {
String parentStartKeyPath = OzoneFSUtils.getParentDir(prevKey);
if (StringUtils.compare(parentStartKeyPath, getKeyPrefix()) >= 0) {
// Add the leaf node to the seek path. The idea is to search for
// sub-paths if the given start key is a directory.
seekPaths.add(new ImmutablePair<>(prevKey, ""));
removeStartKey = prevKey;
getSeekPathsBetweenKeyPrefixAndStartKey(getKeyPrefix(), prevKey,
seekPaths);
} else if (StringUtils.compare(prevKey, getKeyPrefix()) >= 0) {
// Add the leaf node to the seek path. The idea is to search for
// sub-paths if the given start key is a directory.
seekPaths.add(new ImmutablePair<>(prevKey, ""));
removeStartKey = prevKey;
}
}

// 2. Push elements in reverse order so that the FS tree traversal
// will occur in left-to-right fashion[Depth-First Search]
for (int index = seekPaths.size() - 1; index >= 0; index--) {
Pair<String, String> seekDirPath = seekPaths.get(index);
stack.push(seekDirPath);
}
}
}

// Get immediate children
// 3. Pop out top pair and get its immediate children
List<OzoneKey> keysResultList = new ArrayList<>();
getChildrenKeys(getKeyPrefix(), prevKey, keysResultList);

// TODO: Back and Forth seek all the files & dirs, starting from
// startKey till keyPrefix.
if (stack.isEmpty()) {
// case: startKey is empty
getChildrenKeys(getKeyPrefix(), prevKey, keysResultList);
} else {
// case: startKey is non-empty
Pair<String, String> keyPrefixPath = stack.pop();
getChildrenKeys(keyPrefixPath.getLeft(), keyPrefixPath.getRight(),
keysResultList);
}

return keysResultList;
}
Expand Down Expand Up @@ -1201,7 +1277,7 @@ private boolean getChildrenKeys(String keyPrefix, String startKey,
// occur in left-to-right fashion.
for (int indx = dirList.size() - 1; indx >= 0; indx--) {
String dirPathComponent = dirList.get(indx);
stack.push(dirPathComponent);
stack.push(new ImmutablePair<>(dirPathComponent, ""));
}

if (reachedLimitCacheSize) {
Expand All @@ -1211,8 +1287,9 @@ private boolean getChildrenKeys(String keyPrefix, String startKey,
// 7. Pop element and seek for its sub-child path(s). Basically moving
// seek pointer to next level(depth) in FS tree.
while (!stack.isEmpty()) {
keyPrefix = stack.pop();
if (getChildrenKeys(keyPrefix, "", keysResultList)) {
Pair<String, String> keyPrefixPath = stack.pop();
if (getChildrenKeys(keyPrefixPath.getLeft(), keyPrefixPath.getRight(),
keysResultList)) {
// reached limit batch size.
return true;
}
Expand All @@ -1224,14 +1301,21 @@ private boolean getChildrenKeys(String keyPrefix, String startKey,
private void removeStartKeyIfExistsInStatusList(String startKey,
List<OzoneFileStatus> statuses) {

if (StringUtils.isNotBlank(startKey) && !statuses.isEmpty()) {
if (!statuses.isEmpty()) {
String firstElement = statuses.get(0).getKeyInfo().getKeyName();
String startKeyPath = startKey;
if (startKey.endsWith(OZONE_URI_DELIMITER)) {
startKeyPath = OzoneFSUtils.removeTrailingSlashIfNeeded(startKey);
if (StringUtils.isNotBlank(startKey)) {
if (startKey.endsWith(OZONE_URI_DELIMITER)) {
startKeyPath = OzoneFSUtils.removeTrailingSlashIfNeeded(startKey);
}
}
if (StringUtils.equals(statuses.get(0).getKeyInfo().getKeyName(),
startKeyPath)) {
// remove the duplicateKey from the list.

// case-1) remove the startKey from the list as it should be skipped.
// case-2) removeStartKey - as the startKey is a placeholder, which is
// managed internally to traverse leaf node's sub-paths.
if (StringUtils.equals(firstElement, startKeyPath) ||
StringUtils.equals(firstElement, removeStartKey)) {

statuses.remove(0);
}
}
Expand Down Expand Up @@ -1277,6 +1361,13 @@ private void addKeyPrefixInfoToResultList(String keyPrefix,
if (status != null) {
OmKeyInfo keyInfo = status.getKeyInfo();
String keyName = keyInfo.getKeyName();

// removeStartKey - as the startKey is a placeholder, which is
// managed internally to traverse leaf node's sub-paths.
if (StringUtils.equals(keyName, removeStartKey)) {
return;
}

if (status.isDirectory()) {
// add trailing slash to represent directory
keyName =
Expand Down
Loading