Skip to content

HDDS-10399. Fixed error in shallow listing an empty directory LEGACY/OBS buckets. #6259

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
adoroszlai merged 14 commits into from
Feb 25, 2024
Merged

HDDS-10399. Fixed error in shallow listing an empty directory LEGACY/OBS buckets. #6259

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
0b1ea27
HDDS-7035. Rebased.
Jul 26, 2023
be5016b
HDDS-7035. Removed custom variables. Moved String to Sign generation …
Aug 29, 2023
1a05023
HDDS-7035. Fixed checkstyle errors.
Aug 29, 2023
8a11cbc
HDDS-7035. Added licenses to new classes.
Aug 29, 2023
5cae64b
HDDS-7035. Fixed acceptance test errors.
Sep 6, 2023
96cf320
HDDS-7035. Added more Unit tests.
Sep 6, 2023
1fdc955
Merge branch 'apache:master' into master
SaketaChalamchala Oct 24, 2023
9a3c72a
Merge branch 'apache:master' into master
SaketaChalamchala Nov 1, 2023
a474f3d
Merge branch 'apache:master' into master
SaketaChalamchala Nov 14, 2023
5387064
Merge branch 'apache:master' into master
SaketaChalamchala Nov 16, 2023
4c8b5a3
Merge branch 'apache:master' into master
SaketaChalamchala Nov 27, 2023
01dc90d
Merge branch 'apache:master' into master
SaketaChalamchala Jan 20, 2024
75668c9
Merge branch 'apache:master' into master
SaketaChalamchala Feb 21, 2024
7b87a33
HDDS-10399. Fixed index error when shallow listing empty directories …
Feb 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1253,7 +1253,7 @@ List<OzoneKey> getNextShallowListOfKeys(String prevKey)
proxy.listStatusLight(volumeName, name, delimiterKeyPrefix, false,
startKey, listCacheSize, false);

if (addedKeyPrefix) {
if (addedKeyPrefix && statuses.size() > 0) {
// previous round already include the startKey, so remove it
statuses.remove(0);
} else {
Expand Down
114 changes: 94 additions & 20 deletions hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestListKeys.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,8 @@ public class TestListKeys {
private static OzoneConfiguration conf;

private static OzoneBucket legacyOzoneBucket;

private static OzoneBucket obsOzoneBucket;
private static OzoneClient client;

/**
Expand All @@ -86,6 +88,10 @@ public static void init() throws Exception {
legacyOzoneBucket = TestDataUtil
.createVolumeAndBucket(client, BucketLayout.LEGACY);

// create a volume and a OBJECT_STORE bucket
obsOzoneBucket = TestDataUtil
.createVolumeAndBucket(client, BucketLayout.OBJECT_STORE);

initFSNameSpace();
}

Expand All @@ -99,6 +105,7 @@ public static void teardownClass() {

private static void initFSNameSpace() throws Exception {
buildNameSpaceTree(legacyOzoneBucket);
buildNameSpaceTree(obsOzoneBucket);
}

/**
Expand All @@ -108,9 +115,9 @@ private static void initFSNameSpace() throws Exception {
* |
* a1
* |
* -----------------------------------
* | | |
* b1 b2 b3
* --------------------------------------------------------
* | | | |
* b1 b2 b3 b4
* ------- --------- -----------
* | | | | | | | |
* c1 c2 d1 d2 d3 e1 e2 e3
Expand All @@ -125,25 +132,27 @@ private static void initFSNameSpace() throws Exception {
private static void buildNameSpaceTree(OzoneBucket ozoneBucket)
throws Exception {
LinkedList<String> keys = new LinkedList<>();
keys.add("/a1/b1/c1111.tx");
keys.add("/a1/b1/c1222.tx");
keys.add("/a1/b1/c1333.tx");
keys.add("/a1/b1/c1444.tx");
keys.add("/a1/b1/c1555.tx");
keys.add("/a1/b1/c1/c1.tx");
keys.add("/a1/b1/c12/c2.tx");
keys.add("/a1/b1/c12/c3.tx");

keys.add("/a1/b2/d1/d11.tx");
keys.add("/a1/b2/d2/d21.tx");
keys.add("/a1/b2/d2/d22.tx");
keys.add("/a1/b2/d3/d31.tx");

keys.add("/a1/b3/e1/e11.tx");
keys.add("/a1/b3/e2/e21.tx");
keys.add("/a1/b3/e3/e31.tx");
keys.add("a1/b1/c1111.tx");
keys.add("a1/b1/c1222.tx");
keys.add("a1/b1/c1333.tx");
keys.add("a1/b1/c1444.tx");
keys.add("a1/b1/c1555.tx");
keys.add("a1/b1/c1/c1.tx");
keys.add("a1/b1/c12/c2.tx");
keys.add("a1/b1/c12/c3.tx");

keys.add("a1/b2/d1/d11.tx");
keys.add("a1/b2/d2/d21.tx");
keys.add("a1/b2/d2/d22.tx");
keys.add("a1/b2/d3/d31.tx");

keys.add("a1/b3/e1/e11.tx");
keys.add("a1/b3/e2/e21.tx");
keys.add("a1/b3/e3/e31.tx");

createKeys(ozoneBucket, keys);

ozoneBucket.createDirectory("a1/b4/");
}

private static Stream<Arguments> shallowListDataWithTrailingSlash() {
Expand Down Expand Up @@ -186,6 +195,58 @@ private static Stream<Arguments> shallowListDataWithTrailingSlash() {
"a1/b1/c1333.tx",
"a1/b1/c1444.tx",
"a1/b1/c1555.tx"
))),

// Case-7: StartKey is empty, return key that is same as keyPrefix.
of("a1/b4/", "", newLinkedList(Arrays.asList(
"a1/b4/"
)))
);
}

private static Stream<Arguments> shallowListObsDataWithTrailingSlash() {
return Stream.of(

// Case-1: StartKey is less than prefixKey, return emptyList.
of("a1/b2/", "a1", newLinkedList(Collections.emptyList())),

// Case-2: StartKey is empty, return all immediate node.
of("a1/b2/", "", newLinkedList(Arrays.asList(
"a1/b2/d1/",
"a1/b2/d2/",
"a1/b2/d3/"
))),

// Case-3: StartKey is same as prefixKey, return all immediate nodes.
of("a1/b2/", "a1/b2", newLinkedList(Arrays.asList(
"a1/b2/d1/",
"a1/b2/d2/",
"a1/b2/d3/"
))),

// Case-4: StartKey is greater than prefixKey
of("a1/b2/", "a1/b2/d2/d21.tx", newLinkedList(Arrays.asList(
"a1/b2/d2/",
"a1/b2/d3/"
))),

// Case-5: StartKey reaches last element, return emptyList
of("a1/b2/", "a1/b2/d3/d31.tx", newLinkedList(
Collections.emptyList()
)),

// Case-6: Mix result
of("a1/b1/", "a1/b1/c12", newLinkedList(Arrays.asList(
"a1/b1/c12/",
"a1/b1/c1222.tx",
"a1/b1/c1333.tx",
"a1/b1/c1444.tx",
"a1/b1/c1555.tx"
))),

// Case-7: StartKey is empty, return key that is same as keyPrefix.
of("a1/b4/", "", newLinkedList(Arrays.asList(
"a1/b4/"
)))
);
}
Expand Down Expand Up @@ -252,6 +313,11 @@ private static Stream<Arguments> shallowListDataWithoutTrailingSlash() {
of("a1/b1/c12", "", newLinkedList(Arrays.asList(
"a1/b1/c12/",
"a1/b1/c1222.tx"
))),

// Case-10:
of("a1/b4", "", newLinkedList(Arrays.asList(
"a1/b4/"
)))

);
Expand All @@ -264,11 +330,19 @@ public void testShallowListKeysWithPrefixTrailingSlash(String keyPrefix,
checkKeyShallowList(keyPrefix, startKey, expectedKeys, legacyOzoneBucket);
}

@ParameterizedTest
@MethodSource("shallowListObsDataWithTrailingSlash")
public void testShallowListObsKeysWithPrefixTrailingSlash(String keyPrefix,
String startKey, List<String> expectedKeys) throws Exception {
checkKeyShallowList(keyPrefix, startKey, expectedKeys, obsOzoneBucket);
}

@ParameterizedTest
@MethodSource("shallowListDataWithoutTrailingSlash")
public void testShallowListKeysWithoutPrefixTrailingSlash(String keyPrefix,
String startKey, List<String> expectedKeys) throws Exception {
checkKeyShallowList(keyPrefix, startKey, expectedKeys, legacyOzoneBucket);
checkKeyShallowList(keyPrefix, startKey, expectedKeys, obsOzoneBucket);
}

private void checkKeyShallowList(String keyPrefix, String startKey,
Expand Down