Skip to content

HDDS-8773. [S3G] Improve list performance in FSO bucket #4868

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ChenSammi merged 1 commit into from
Nov 9, 2023
Merged

HDDS-8773. [S3G] Improve list performance in FSO bucket #4868

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
271 changes: 198 additions & 73 deletions hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1072,6 +1072,10 @@ private class KeyIterator implements Iterator<OzoneKey> {
private boolean addedKeyPrefix;
private String delimiterKeyPrefix;

boolean shallow() {
return shallow;
}

String getKeyPrefix() {
return keyPrefix;
}
Expand All @@ -1088,6 +1092,14 @@ void setAddedKeyPrefix(boolean addedKeyPrefix) {
this.addedKeyPrefix = addedKeyPrefix;
}

String getDelimiterKeyPrefix() {
return delimiterKeyPrefix;
}

void setDelimiterKeyPrefix(String delimiterKeyPrefix) {
this.delimiterKeyPrefix = delimiterKeyPrefix;
}

/**
* Creates an Iterator to iterate over all keys after prevKey in the bucket.
* If prevKey is null it iterates from the first key in the bucket.
Expand Down Expand Up @@ -1142,8 +1154,8 @@ List<OzoneKey> getNextListOfKeys(String prevKey) throws
}

/**
* Using listStatus instead of listKeys avoiding listing all children keys.
* Giving the structure of keys delimited by "/":
* Using listStatusLight instead of listKeys avoiding listing all children
* keys. Giving the structure of keys delimited by "/":
*
* buck-1
* |
Expand Down Expand Up @@ -1192,15 +1204,16 @@ List<OzoneKey> getNextListOfKeys(String prevKey) throws
* In implementation, the keyPrefix "a/b" can be identified in listKeys,
* but cannot be identified in listStatus. Therefore, keyPrefix "a/b"
* needs to be split into keyPrefix "a" and call listKeys method to get
* the next one key as the startKey in listStatus.
* the next one key as the startKey in listStatusLight.
*/
protected List<OzoneKey> getNextShallowListOfKeys(String prevKey)
List<OzoneKey> getNextShallowListOfKeys(String prevKey)
throws IOException {
List<OzoneKey> resultList = new ArrayList<>();
String startKey = prevKey;

// handle for first round
// 1. Get first element as startKey
if (!addedKeyPrefix) {
initDelimiterKeyPrefix();
// prepare startKey
List<OzoneKey> nextOneKeys =
proxy.listKeys(volumeName, name, getKeyPrefix(), prevKey, 1);
Expand All @@ -1218,16 +1231,9 @@ protected List<OzoneKey> getNextShallowListOfKeys(String prevKey)
startKey.equals(getKeyPrefix())) {
resultList.add(nextOneKeys.get(0));
}

// prepare delimiterKeyPrefix
delimiterKeyPrefix = getKeyPrefix();
if (!getKeyPrefix().endsWith(OZONE_URI_DELIMITER)) {
delimiterKeyPrefix = OzoneFSUtils.getParentDir(getKeyPrefix());
}
}

// Elements in statuses must be sorted after startKey,
// which means they come after the keyPrefix.
// 2. Get immediate children by listStatusLight method
List<OzoneFileStatusLight> statuses =
proxy.listStatusLight(volumeName, name, delimiterKeyPrefix, false,
startKey, listCacheSize, false);
Expand All @@ -1239,16 +1245,20 @@ protected List<OzoneKey> getNextShallowListOfKeys(String prevKey)
setAddedKeyPrefix(true);
}

List<OzoneKey> ozoneKeys = buildOzoneKeysFromFileStatus(statuses)
.stream()
.filter(key -> StringUtils.startsWith(key.getName(), getKeyPrefix()))
.collect(Collectors.toList());
List<OzoneKey> ozoneKeys = buildKeysWithKeyPrefix(statuses);

resultList.addAll(ozoneKeys);
return resultList;
}

private List<OzoneKey> buildOzoneKeysFromFileStatus(
protected void initDelimiterKeyPrefix() {
setDelimiterKeyPrefix(getKeyPrefix());
if (!getKeyPrefix().endsWith(OZONE_URI_DELIMITER)) {
setDelimiterKeyPrefix(OzoneFSUtils.getParentDir(getKeyPrefix()));
}
}

protected List<OzoneKey> buildKeysWithKeyPrefix(
List<OzoneFileStatusLight> statuses) {
return statuses.stream()
.map(status -> {
Expand All @@ -1264,6 +1274,7 @@ private List<OzoneKey> buildOzoneKeysFromFileStatus(
keyInfo.getModificationTime(),
keyInfo.getReplicationConfig(), keyInfo.isFile());
})
.filter(key -> StringUtils.startsWith(key.getName(), getKeyPrefix()))
.collect(Collectors.toList());
}

Expand Down Expand Up @@ -1354,63 +1365,14 @@ List<OzoneKey> getNextListOfKeys(String prevKey) throws IOException {
stack = new Stack();
}

if (shallow()) {
return getNextShallowListOfKeys(prevKey);
}

// normalize paths
if (!addedKeyPrefix()) {
prevKey = OmUtils.normalizeKey(prevKey, true);
String keyPrefixName = "";
if (StringUtils.isNotBlank(getKeyPrefix())) {
keyPrefixName = OmUtils.normalizeKey(getKeyPrefix(), true);
}
setKeyPrefix(keyPrefixName);

if (StringUtils.isNotBlank(prevKey)) {
if (StringUtils.startsWith(prevKey, getKeyPrefix())) {
// 1. Prepare all the seekKeys after the prefixKey.
// Example case: prefixKey="a1", startKey="a1/b2/d2/f3/f31.tx"
// Now, stack should be build with all the levels after prefixKey
// Stack format => <keyPrefix and startKey>, startKey should be an
// immediate child of keyPrefix.
// _______________________________________
// Stack=> top | < a1/b2/d2/f3, a1/b2/d2/f3/f31.tx > |
// |-------------------------------------|
// | < a1/b2/d2, a1/b2/d2/f3 > |
// |-------------------------------------|
// | < a1/b2, a1/b2/d2 > |
// |-------------------------------------|
// bottom | < a1, a1/b2 > |
// --------------------------------------|
List<Pair<String, String>> seekPaths = new ArrayList<>();

if (StringUtils.isNotBlank(getKeyPrefix())) {
// If the prev key is a dir then seek its sub-paths
// Say, prevKey="a1/b2/d2"
addPrevDirectoryToSeekPath(prevKey, seekPaths);
}

// Key Prefix is Blank. The seek all the keys with startKey.
removeStartKey = prevKey;
getSeekPathsBetweenKeyPrefixAndStartKey(getKeyPrefix(), prevKey,
seekPaths);

// 2. Push elements in reverse order so that the FS tree traversal
// will occur in left-to-right fashion[Depth-First Search]
for (int index = seekPaths.size() - 1; index >= 0; index--) {
Pair<String, String> seekDirPath = seekPaths.get(index);
stack.push(seekDirPath);
}
} else if (StringUtils.isNotBlank(getKeyPrefix())) {
if (!OzoneFSUtils.isAncestorPath(getKeyPrefix(), prevKey)) {
// Case-1 - sibling: keyPrefix="a1/b2", startKey="a0/b123Invalid"
// Skip traversing, if the startKey is not a sibling.
// "a1/b", "a1/b1/e/"
return new ArrayList<>();
} else if (StringUtils.compare(prevKey, getKeyPrefix()) < 0) {
// Case-2 - compare: keyPrefix="a1/b2", startKey="a1/b123Invalid"
// Since startKey is lexographically behind keyPrefix,
// the seek precedence goes to keyPrefix.
stack.push(new ImmutablePair<>(getKeyPrefix(), ""));
}
}
if (!prepareStack(prevKey)) {
return new ArrayList<>();
}
}

Expand All @@ -1437,6 +1399,169 @@ List<OzoneKey> getNextListOfKeys(String prevKey) throws IOException {
return keysResultList;
}

@Override
List<OzoneKey> getNextShallowListOfKeys(String prevKey)
throws IOException {
List<OzoneKey> resultList = new ArrayList<>();
String startKey = prevKey;
boolean findFirstStartKey = false;

if (!addedKeyPrefix()) {
initDelimiterKeyPrefix();

if (!prepareStack(prevKey)) {
return new ArrayList<>();
}

// 1. Get first element as startKey.
List<OzoneKey> firstKeyResult = new ArrayList<>();
if (stack.isEmpty()) {
// Case: startKey is empty
getChildrenKeys(getKeyPrefix(), prevKey, firstKeyResult);
} else {
// Case: startKey is non-empty
while (!stack.isEmpty()) {
Pair<String, String> keyPrefixPath = stack.pop();
getChildrenKeys(keyPrefixPath.getLeft(), keyPrefixPath.getRight(),
firstKeyResult);
if (!firstKeyResult.isEmpty()) {
break;
}
}
}
if (!firstKeyResult.isEmpty()) {
startKey = firstKeyResult.get(0).getName();
findFirstStartKey = true;
}

// A specific case where findFirstStartKey is false does not mean that
// the final result is empty because also need to determine whether
// keyPrefix is an existing key. Consider the following structure:
// te/
// test1/
// test1/file1
// test1/file2
// test2/
// when keyPrefix='te' and prevKey='test1/file2', findFirstStartKey
// will be false because 'test1/file2' is the last key in dir
// 'test1/'. In the correct result for this case, 'test2/' is expected
// in the results and "test1/" should be excluded.
//
if (!findFirstStartKey) {
if (StringUtils.isBlank(prevKey) || !keyPrefixExist()
|| !StringUtils.startsWith(prevKey, getKeyPrefix())) {
return new ArrayList<>();
}
}
// A special case where keyPrefix element should present in the
// resultList. See the annotation of #addKeyPrefixInfoToResultList
if (getKeyPrefix().equals(startKey) && findFirstStartKey
&& !firstKeyResult.get(0).isFile()) {
resultList.add(firstKeyResult.get(0));
}
// Note that the startKey needs to be an immediate child of the
// keyPrefix or black before calling listStatus.
startKey = adjustStartKey(startKey);
}

// 2. Get immediate children by listStatus method.
List<OzoneFileStatusLight> statuses =
proxy.listStatusLight(volumeName, name, getDelimiterKeyPrefix(),
false, startKey, listCacheSize, false);

if (!statuses.isEmpty()) {
// If findFirstStartKey is false, indicates that the keyPrefix is an
// existing key and the prevKey is the last element within its
// directory. In this case, the result should not include the
// startKey itself.
if (!findFirstStartKey && addedKeyPrefix()) {
statuses.remove(0);
}
List<OzoneKey> ozoneKeys = buildKeysWithKeyPrefix(statuses);
resultList.addAll(ozoneKeys);
}
return resultList;
}

private boolean prepareStack(String prevKey) throws IOException {
prevKey = OmUtils.normalizeKey(prevKey, true);
String keyPrefixName = "";
if (StringUtils.isNotBlank(getKeyPrefix())) {
keyPrefixName = OmUtils.normalizeKey(getKeyPrefix(), true);
}
setKeyPrefix(keyPrefixName);

if (StringUtils.isNotBlank(prevKey)) {
if (StringUtils.startsWith(prevKey, getKeyPrefix())) {
// 1. Prepare all the seekKeys after the prefixKey.
// Example case: prefixKey="a1", startKey="a1/b2/d2/f3/f31.tx"
// Now, stack should be build with all the levels after prefixKey
// Stack format => <keyPrefix and startKey>, startKey should be an
// immediate child of keyPrefix.
// _______________________________________
// Stack=> top | < a1/b2/d2/f3, a1/b2/d2/f3/f31.tx > |
// |-------------------------------------|
// | < a1/b2/d2, a1/b2/d2/f3 > |
// |-------------------------------------|
// | < a1/b2, a1/b2/d2 > |
// |-------------------------------------|
// bottom | < a1, a1/b2 > |
// --------------------------------------|
List<Pair<String, String>> seekPaths = new ArrayList<>();

if (StringUtils.isNotBlank(getKeyPrefix())) {
// If the prev key is a dir then seek its sub-paths
// Say, prevKey="a1/b2/d2"
addPrevDirectoryToSeekPath(prevKey, seekPaths);
}

// Key Prefix is Blank. The seek all the keys with startKey.
removeStartKey = prevKey;
getSeekPathsBetweenKeyPrefixAndStartKey(getKeyPrefix(), prevKey,
seekPaths);

// 2. Push elements in reverse order so that the FS tree traversal
// will occur in left-to-right fashion[Depth-First Search]
for (int index = seekPaths.size() - 1; index >= 0; index--) {
Pair<String, String> seekDirPath = seekPaths.get(index);
stack.push(seekDirPath);
}
} else if (StringUtils.isNotBlank(getKeyPrefix())) {
if (!OzoneFSUtils.isAncestorPath(getKeyPrefix(), prevKey)) {
// Case-1 - sibling: keyPrefix="a1/b2", startKey="a0/b123Invalid"
// Skip traversing, if the startKey is not a sibling.
// "a1/b", "a1/b1/e/"
return false;
} else if (StringUtils.compare(prevKey, getKeyPrefix()) < 0) {
// Case-2 - compare: keyPrefix="a1/b2", startKey="a1/b123Invalid"
// Since startKey is lexographically behind keyPrefix,
// the seek precedence goes to keyPrefix.
stack.push(new ImmutablePair<>(getKeyPrefix(), ""));
}
}
}
return true;
}

private String adjustStartKey(String startKey) {
if (getKeyPrefix().endsWith(OZONE_URI_DELIMITER) &&
getKeyPrefix().equals(startKey)) {
return "";
}
return OzoneFSUtils.getImmediateChild(startKey, getDelimiterKeyPrefix());
}

private boolean keyPrefixExist() throws IOException {
OzoneFileStatus keyPrefixStatus = null;
try {
keyPrefixStatus =
proxy.getOzoneFileStatus(volumeName, name, getKeyPrefix());
} catch (OMException ome) {
// ignore exception
}
return keyPrefixStatus != null;
}

private void addPrevDirectoryToSeekPath(String prevKey,
List<Pair<String, String>> seekPaths)
throws IOException {
Expand Down
Loading