diff --git a/hadoop-ozone/dist/src/main/compose/ozone-ha/test.sh b/hadoop-ozone/dist/src/main/compose/ozone-ha/test.sh index 1361a4c0c335..976e490d32ca 100755 --- a/hadoop-ozone/dist/src/main/compose/ozone-ha/test.sh +++ b/hadoop-ozone/dist/src/main/compose/ozone-ha/test.sh @@ -38,9 +38,11 @@ execute_robot_test ${SCM} -v SCHEME:ofs -v BUCKET_TYPE:link -N ozonefs-ofs-link ## Exclude virtual-host tests. This is tested separately as it requires additional config. exclude="--exclude virtual-host" for bucket in generated; do - execute_robot_test ${SCM} -v BUCKET:${bucket} -N s3-${bucket} ${exclude} s3 - # some tests are independent of the bucket type, only need to be run once - exclude="--exclude virtual-host --exclude no-bucket-type" + for layout in OBJECT_STORE LEGACY FILE_SYSTEM_OPTIMIZED; do + execute_robot_test ${SCM} -v BUCKET:${bucket} -v BUCKET_LAYOUT:${layout} -N s3-${layout}-${bucket} ${exclude} s3 + # some tests are independent of the bucket type, only need to be run once + exclude="--exclude virtual-host --exclude no-bucket-type" + done done execute_robot_test ${SCM} freon diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot b/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot index 840fb963d8d1..b20537014dd1 100644 --- a/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot +++ b/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot @@ -23,6 +23,7 @@ ${ENDPOINT_URL} http://s3g:9878 ${OZONE_S3_HEADER_VERSION} v4 ${OZONE_S3_SET_CREDENTIALS} true ${BUCKET} generated +${BUCKET_LAYOUT} OBJECT_STORE ${KEY_NAME} key1 ${OZONE_S3_TESTS_SET_UP} ${FALSE} ${OZONE_AWS_ACCESS_KEY_ID} ${EMPTY} @@ -127,16 +128,12 @@ Create bucket with name ${result} = Execute AWSS3APICli create-bucket --bucket ${bucket} Should contain ${result} Location Should contain ${result} ${bucket} -Create legacy bucket - ${postfix} = Generate Ozone String - ${legacy_bucket} = Set Variable legacy-bucket-${postfix} - ${result} = Execute and checkrc ozone sh bucket create -l LEGACY s3v/${legacy_bucket} 0 - [Return] ${legacy_bucket} -Create obs bucket +Create bucket with layout + [Arguments] ${layout} ${postfix} = Generate Ozone String - ${bucket} = Set Variable obs-bucket-${postfix} - ${result} = Execute and checkrc ozone sh bucket create -l OBJECT_STORE s3v/${bucket} 0 + ${bucket} = Set Variable bucket-${postfix} + ${result} = Execute ozone sh bucket create --layout ${layout} s3v/${bucket} [Return] ${bucket} Setup s3 tests @@ -144,7 +141,7 @@ Setup s3 tests Run Keyword Generate random prefix Run Keyword Install aws cli Run Keyword if '${OZONE_S3_SET_CREDENTIALS}' == 'true' Setup v4 headers - Run Keyword if '${BUCKET}' == 'generated' Create generated bucket + Run Keyword if '${BUCKET}' == 'generated' Create generated bucket ${BUCKET_LAYOUT} Run Keyword if '${BUCKET}' == 'link' Setup links for S3 tests Run Keyword if '${BUCKET}' == 'encrypted' Create encrypted bucket Run Keyword if '${BUCKET}' == 'erasure' Create EC bucket @@ -154,18 +151,19 @@ Setup links for S3 tests ${exists} = Bucket Exists o3://${OM_SERVICE_ID}/s3v/link Return From Keyword If ${exists} Execute ozone sh volume create o3://${OM_SERVICE_ID}/legacy - Execute ozone sh bucket create o3://${OM_SERVICE_ID}/legacy/source-bucket + Execute ozone sh bucket create --layout ${BUCKET_LAYOUT} o3://${OM_SERVICE_ID}/legacy/source-bucket Create link link Create generated bucket - ${BUCKET} = Create bucket + [Arguments] ${layout}=OBJECT_STORE + ${BUCKET} = Create bucket with layout ${layout} Set Global Variable ${BUCKET} Create encrypted bucket Return From Keyword if '${SECURITY_ENABLED}' == 'false' ${exists} = Bucket Exists o3://${OM_SERVICE_ID}/s3v/encrypted Return From Keyword If ${exists} - Execute ozone sh bucket create -k ${KEY_NAME} o3://${OM_SERVICE_ID}/s3v/encrypted + Execute ozone sh bucket create -k ${KEY_NAME} --layout ${BUCKET_LAYOUT} o3://${OM_SERVICE_ID}/s3v/encrypted Create link [arguments] ${bucket} @@ -175,7 +173,7 @@ Create link Create EC bucket ${exists} = Bucket Exists o3://${OM_SERVICE_ID}/s3v/erasure Return From Keyword If ${exists} - Execute ozone sh bucket create --replication rs-3-2-1024k --type EC o3://${OM_SERVICE_ID}/s3v/erasure + Execute ozone sh bucket create --replication rs-3-2-1024k --type EC --layout ${BUCKET_LAYOUT} o3://${OM_SERVICE_ID}/s3v/erasure Generate random prefix ${random} = Generate Ozone String diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/objecthead.robot b/hadoop-ozone/dist/src/main/smoketest/s3/objecthead.robot index be0582edd1f2..66f3461b01dd 100644 --- a/hadoop-ozone/dist/src/main/smoketest/s3/objecthead.robot +++ b/hadoop-ozone/dist/src/main/smoketest/s3/objecthead.robot @@ -40,22 +40,23 @@ Head object in non existing bucket ${result} = Execute AWSS3APICli and checkrc head-object --bucket ${BUCKET}-non-existent --key ${PREFIX}/headobject/key=value/f1 255 Should contain ${result} 404 Should contain ${result} Not Found + Head object where path is a directory - ${legacy-bucket} = Create legacy bucket - ${result} = Execute AWSS3APICli and checkrc put-object --bucket ${legacy-bucket} --key ${PREFIX}/headobject/keyvalue/f1 --body /tmp/testfile 0 - ${result} = Execute AWSS3APICli and checkrc head-object --bucket ${legacy-bucket} --key ${PREFIX}/headobject/keyvalue/ 255 + Pass Execution If '${BUCKET_LAYOUT}' == 'FILE_SYSTEM_OPTIMIZED' does not apply to FSO buckets + ${result} = Execute AWSS3APICli and checkrc put-object --bucket ${BUCKET} --key ${PREFIX}/headobject/keyvalue/f1 --body /tmp/testfile 0 + ${result} = Execute AWSS3APICli and checkrc head-object --bucket ${BUCKET} --key ${PREFIX}/headobject/keyvalue/ 255 Should contain ${result} 404 Should contain ${result} Not Found Head directory objects - ${obs-bucket} = Create obs bucket - ${result} = Execute AWSS3APICli and checkrc put-object --bucket ${obs-bucket} --key ${PREFIX}/mydir/ --body /tmp/testfile 0 - ${result} = Execute AWSS3APICli and checkrc head-object --bucket ${obs-bucket} --key ${PREFIX}/mydir 255 + Pass Execution If '${BUCKET_LAYOUT}' == 'FILE_SYSTEM_OPTIMIZED' does not apply to FSO buckets + ${result} = Execute AWSS3APICli and checkrc put-object --bucket ${BUCKET} --key ${PREFIX}/mydir/ --body /tmp/testfile 0 + ${result} = Execute AWSS3APICli and checkrc head-object --bucket ${BUCKET} --key ${PREFIX}/mydir 255 Should contain ${result} 404 Should contain ${result} Not Found - ${result} = Execute AWSS3APICli and checkrc head-object --bucket ${obs-bucket} --key ${PREFIX}/mydir/ 0 + ${result} = Execute AWSS3APICli and checkrc head-object --bucket ${BUCKET} --key ${PREFIX}/mydir/ 0 Head non existing key ${result} = Execute AWSS3APICli and checkrc head-object --bucket ${BUCKET} --key ${PREFIX}/non-existent 255 Should contain ${result} 404 - Should contain ${result} Not Found \ No newline at end of file + Should contain ${result} Not Found diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot b/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot index 1b44360d6bed..05348fbcba4b 100644 --- a/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot +++ b/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot @@ -44,6 +44,8 @@ Put object to s3 Get object from s3 ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/f1 /tmp/testfile.result Compare files /tmp/testfile /tmp/testfile.result + ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/zerobyte /tmp/zerobyte.result + Compare files /tmp/zerobyte /tmp/zerobyte.result #This test depends on the previous test case. Can't be executed alone Get object with wrong signature @@ -151,34 +153,14 @@ Incorrect values for end and start offset Should Be Equal ${expectedData} ${actualData} Zero byte file - ${result} = Execute ozone sh bucket info /s3v/${BUCKET} - ${linked} = Execute echo '${result}' | jq -j '.sourceVolume,"/",.sourceBucket' - ${eval} = Evaluate "source" in """${linked}""" - IF ${eval} == ${True} - ${result} = Execute ozone sh bucket info ${linked} - END - ${fsolayout} = Evaluate "OPTIMIZED" in """${result}""" - ${result} = Execute AWSS3APICli and checkrc get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/zerobyte --range bytes=0-0 /tmp/testfile2.result 255 - IF ${fsolayout} == ${True} - Should contain ${result} NoSuchKey - ELSE Should contain ${result} InvalidRange - END ${result} = Execute AWSS3APICli and checkrc get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/zerobyte --range bytes=0-1 /tmp/testfile2.result 255 - IF ${fsolayout} == ${True} - Should contain ${result} NoSuchKey - ELSE Should contain ${result} InvalidRange - END ${result} = Execute AWSS3APICli and checkrc get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/key=value/zerobyte --range bytes=0-10000 /tmp/testfile2.result 255 - IF ${fsolayout} == ${True} - Should contain ${result} NoSuchKey - ELSE Should contain ${result} InvalidRange - END Create file with user defined metadata Execute echo "Randomtext" > /tmp/testfile2 @@ -257,4 +239,4 @@ Create key twice with different content and expect different ETags # clean up Execute AWSS3Cli rm s3://${BUCKET}/test_key_to_check_etag_differences Execute rm -rf /tmp/file1 - Execute rm -rf /tmp/file2 \ No newline at end of file + Execute rm -rf /tmp/file2 diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java index f163d3eac335..8b7db9f061d5 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java @@ -274,7 +274,9 @@ public Response put( boolean hasAmzDecodedLengthZero = amzDecodedLength != null && Long.parseLong(amzDecodedLength) == 0; if (canCreateDirectory && - (length == 0 || hasAmzDecodedLengthZero)) { + (length == 0 || hasAmzDecodedLengthZero) && + StringUtils.endsWith(keyPath, "/") + ) { s3GAction = S3GAction.CREATE_DIRECTORY; getClientProtocol() .createDirectory(volume.getName(), bucketName, keyPath); diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java index 32bcb0a2c9e4..17c3cba304c5 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java @@ -95,7 +95,9 @@ static Stream argumentsForPutObject() { ReplicationConfig ratis3 = RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE); ECReplicationConfig ec = new ECReplicationConfig("rs-3-2-1024K"); return Stream.of( + Arguments.of(0, ratis3), Arguments.of(10, ratis3), + Arguments.of(0, ec), Arguments.of(10, ec) ); } @@ -410,7 +412,7 @@ void testEmptyStorageType() throws IOException, OS3Exception { void testDirectoryCreation() throws IOException, OS3Exception { // GIVEN - final String path = "dir"; + final String path = "dir/"; // WHEN try (Response response = objectEndpoint.put(fsoBucket.getName(), path, @@ -426,7 +428,7 @@ void testDirectoryCreation() throws IOException, @Test void testDirectoryCreationOverFile() throws IOException, OS3Exception { // GIVEN - final String path = "dir"; + final String path = "key"; final ByteArrayInputStream body = new ByteArrayInputStream(CONTENT.getBytes(UTF_8)); objectEndpoint.put(FSO_BUCKET_NAME, path, CONTENT.length(), 0, "", body); @@ -434,7 +436,7 @@ void testDirectoryCreationOverFile() throws IOException, OS3Exception { // WHEN final OS3Exception exception = assertThrows(OS3Exception.class, () -> objectEndpoint - .put(FSO_BUCKET_NAME, path, 0, 0, "", null) + .put(FSO_BUCKET_NAME, path + "/", 0, 0, "", null) .close()); // THEN