diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot b/hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot index 8f86a67885ed..e05a706a0405 100644 --- a/hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot +++ b/hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot @@ -21,13 +21,13 @@ Resource ../commonlib.robot Resource commonawslib.robot Test Timeout 5 minutes Suite Setup Setup s3 tests +Test Setup Generate random prefix *** Keywords *** Create Random file [arguments] ${size_in_megabytes} Execute dd if=/dev/urandom of=/tmp/part1 bs=1048576 count=${size_in_megabytes} - *** Variables *** ${ENDPOINT_URL} http://s3g:9878 ${BUCKET} generated @@ -35,16 +35,16 @@ ${BUCKET} generated *** Test Cases *** Test Multipart Upload - ${result} = Execute AWSS3APICli create-multipart-upload --bucket ${BUCKET} --key multipartKey + ${result} = Execute AWSS3APICli create-multipart-upload --bucket ${BUCKET} --key ${PREFIX}/multipartKey ${uploadID} = Execute and checkrc echo '${result}' | jq -r '.UploadId' 0 Should contain ${result} ${BUCKET} - Should contain ${result} multipartKey + Should contain ${result} ${PREFIX}/multipartKey Should contain ${result} UploadId # initiate again - ${result} = Execute AWSS3APICli create-multipart-upload --bucket ${BUCKET} --key multipartKey + ${result} = Execute AWSS3APICli create-multipart-upload --bucket ${BUCKET} --key ${PREFIX}/multipartKey ${nextUploadID} = Execute and checkrc echo '${result}' | jq -r '.UploadId' 0 Should contain ${result} ${BUCKET} - Should contain ${result} multipartKey + Should contain ${result} ${PREFIX}/multipartKey Should contain ${result} UploadId Should Not Be Equal ${uploadID} ${nextUploadID} @@ -54,149 +54,149 @@ Test Multipart Upload # multipart upload, uploading each part as 5MB file, exception is for last part Run Keyword Create Random file 5 - ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey --part-number 1 --body /tmp/part1 --upload-id ${nextUploadID} + ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key ${PREFIX}/multipartKey --part-number 1 --body /tmp/part1 --upload-id ${nextUploadID} Should contain ${result} ETag # override part Run Keyword Create Random file 5 - ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey --part-number 1 --body /tmp/part1 --upload-id ${nextUploadID} + ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key ${PREFIX}/multipartKey --part-number 1 --body /tmp/part1 --upload-id ${nextUploadID} Should contain ${result} ETag Test Multipart Upload Complete - ${result} = Execute AWSS3APICli create-multipart-upload --bucket ${BUCKET} --key multipartKey1 + ${result} = Execute AWSS3APICli create-multipart-upload --bucket ${BUCKET} --key ${PREFIX}/multipartKey1 ${uploadID} = Execute and checkrc echo '${result}' | jq -r '.UploadId' 0 Should contain ${result} ${BUCKET} - Should contain ${result} multipartKey + Should contain ${result} ${PREFIX}/multipartKey Should contain ${result} UploadId #upload parts Run Keyword Create Random file 5 - ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey1 --part-number 1 --body /tmp/part1 --upload-id ${uploadID} + ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key ${PREFIX}/multipartKey1 --part-number 1 --body /tmp/part1 --upload-id ${uploadID} ${eTag1} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0 Should contain ${result} ETag Execute echo "Part2" > /tmp/part2 - ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey1 --part-number 2 --body /tmp/part2 --upload-id ${uploadID} + ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key ${PREFIX}/multipartKey1 --part-number 2 --body /tmp/part2 --upload-id ${uploadID} ${eTag2} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0 Should contain ${result} ETag #complete multipart upload - ${result} = Execute AWSS3APICli complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key multipartKey1 --multipart-upload 'Parts=[{ETag=${eTag1},PartNumber=1},{ETag=${eTag2},PartNumber=2}]' + ${result} = Execute AWSS3APICli complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key ${PREFIX}/multipartKey1 --multipart-upload 'Parts=[{ETag=${eTag1},PartNumber=1},{ETag=${eTag2},PartNumber=2}]' Should contain ${result} ${BUCKET} - Should contain ${result} multipartKey1 + Should contain ${result} ${PREFIX}/multipartKey1 Should contain ${result} ETag #read file and check the key - ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key multipartKey1 /tmp/multipartKey1.result - Execute cat /tmp/part1 /tmp/part2 > /tmp/multipartKey1 - Compare files /tmp/multipartKey1 /tmp/multipartKey1.result + ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key ${PREFIX}/multipartKey1 /tmp/${PREFIX}-multipartKey1.result + Execute cat /tmp/part1 /tmp/part2 > /tmp/${PREFIX}-multipartKey1 + Compare files /tmp/${PREFIX}-multipartKey1 /tmp/${PREFIX}-multipartKey1.result Test Multipart Upload Complete Entity too small - ${result} = Execute AWSS3APICli create-multipart-upload --bucket ${BUCKET} --key multipartKey2 + ${result} = Execute AWSS3APICli create-multipart-upload --bucket ${BUCKET} --key ${PREFIX}/multipartKey2 ${uploadID} = Execute and checkrc echo '${result}' | jq -r '.UploadId' 0 Should contain ${result} ${BUCKET} - Should contain ${result} multipartKey + Should contain ${result} ${PREFIX}/multipartKey Should contain ${result} UploadId #upload parts Execute echo "Part1" > /tmp/part1 - ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey2 --part-number 1 --body /tmp/part1 --upload-id ${uploadID} + ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key ${PREFIX}/multipartKey2 --part-number 1 --body /tmp/part1 --upload-id ${uploadID} ${eTag1} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0 Should contain ${result} ETag Execute echo "Part2" > /tmp/part2 - ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey2 --part-number 2 --body /tmp/part2 --upload-id ${uploadID} + ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key ${PREFIX}/multipartKey2 --part-number 2 --body /tmp/part2 --upload-id ${uploadID} ${eTag2} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0 Should contain ${result} ETag #complete multipart upload - ${result} = Execute AWSS3APICli and checkrc complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key multipartKey2 --multipart-upload 'Parts=[{ETag=${eTag1},PartNumber=1},{ETag=${eTag2},PartNumber=2}]' 255 + ${result} = Execute AWSS3APICli and checkrc complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key ${PREFIX}/multipartKey2 --multipart-upload 'Parts=[{ETag=${eTag1},PartNumber=1},{ETag=${eTag2},PartNumber=2}]' 255 Should contain ${result} EntityTooSmall Test Multipart Upload Complete Invalid part errors and complete mpu with few parts - ${result} = Execute AWSS3APICli create-multipart-upload --bucket ${BUCKET} --key multipartKey3 + ${result} = Execute AWSS3APICli create-multipart-upload --bucket ${BUCKET} --key ${PREFIX}/multipartKey3 ${uploadID} = Execute and checkrc echo '${result}' | jq -r '.UploadId' 0 Should contain ${result} ${BUCKET} - Should contain ${result} multipartKey + Should contain ${result} ${PREFIX}/multipartKey Should contain ${result} UploadId #complete multipart upload when no parts uploaded - ${result} = Execute AWSS3APICli and checkrc complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key multipartKey3 --multipart-upload 'Parts=[{ETag=etag1,PartNumber=1},{ETag=etag2,PartNumber=2}]' 255 + ${result} = Execute AWSS3APICli and checkrc complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key ${PREFIX}/multipartKey3 --multipart-upload 'Parts=[{ETag=etag1,PartNumber=1},{ETag=etag2,PartNumber=2}]' 255 Should contain ${result} InvalidPart - ${result} = Execute AWSS3APICli and checkrc complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key multipartKey3 --multipart-upload 'Parts=[{ETag=etag1,PartNumber=2},{ETag=etag2,PartNumber=1}]' 255 + ${result} = Execute AWSS3APICli and checkrc complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key ${PREFIX}/multipartKey3 --multipart-upload 'Parts=[{ETag=etag1,PartNumber=2},{ETag=etag2,PartNumber=1}]' 255 Should contain ${result} InvalidPart #upload parts Run Keyword Create Random file 5 - ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey3 --part-number 1 --body /tmp/part1 --upload-id ${uploadID} + ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key ${PREFIX}/multipartKey3 --part-number 1 --body /tmp/part1 --upload-id ${uploadID} ${eTag1} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0 Should contain ${result} ETag - ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey3 --part-number 2 --body /tmp/part1 --upload-id ${uploadID} + ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key ${PREFIX}/multipartKey3 --part-number 2 --body /tmp/part1 --upload-id ${uploadID} ${eTag2} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0 Should contain ${result} ETag Execute echo "Part3" > /tmp/part3 - ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey3 --part-number 3 --body /tmp/part3 --upload-id ${uploadID} + ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key ${PREFIX}/multipartKey3 --part-number 3 --body /tmp/part3 --upload-id ${uploadID} ${eTag3} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0 Should contain ${result} ETag #complete multipart upload - ${result} = Execute AWSS3APICli and checkrc complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key multipartKey3 --multipart-upload 'Parts=[{ETag=etag1,PartNumber=1},{ETag=etag2,PartNumber=2}]' 255 + ${result} = Execute AWSS3APICli and checkrc complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key ${PREFIX}/multipartKey3 --multipart-upload 'Parts=[{ETag=etag1,PartNumber=1},{ETag=etag2,PartNumber=2}]' 255 Should contain ${result} InvalidPart - ${result} = Execute AWSS3APICli and checkrc complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key multipartKey3 --multipart-upload 'Parts=[{ETag=${eTag1},PartNumber=1},{ETag=etag2,PartNumber=2}]' 255 + ${result} = Execute AWSS3APICli and checkrc complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key ${PREFIX}/multipartKey3 --multipart-upload 'Parts=[{ETag=${eTag1},PartNumber=1},{ETag=etag2,PartNumber=2}]' 255 Should contain ${result} InvalidPart - ${result} = Execute AWSS3APICli and checkrc complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key multipartKey3 --multipart-upload 'Parts=[{ETag=${eTag1},PartNumber=4},{ETag=etag2,PartNumber=2}]' 255 + ${result} = Execute AWSS3APICli and checkrc complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key ${PREFIX}/multipartKey3 --multipart-upload 'Parts=[{ETag=${eTag1},PartNumber=4},{ETag=etag2,PartNumber=2}]' 255 Should contain ${result} InvalidPartOrder #complete multipart upload(merge with few parts) - ${result} = Execute AWSS3APICli complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key multipartKey3 --multipart-upload 'Parts=[{ETag=${eTag1},PartNumber=1},{ETag=${eTag3},PartNumber=3}]' + ${result} = Execute AWSS3APICli complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key ${PREFIX}/multipartKey3 --multipart-upload 'Parts=[{ETag=${eTag1},PartNumber=1},{ETag=${eTag3},PartNumber=3}]' Should contain ${result} ${BUCKET} - Should contain ${result} multipartKey3 + Should contain ${result} ${PREFIX}/multipartKey3 Should contain ${result} ETag - ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key multipartKey3 /tmp/multipartKey3.result - Execute cat /tmp/part1 /tmp/part3 > /tmp/multipartKey3 - Compare files /tmp/multipartKey3 /tmp/multipartKey3.result + ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key ${PREFIX}/multipartKey3 /tmp/${PREFIX}-multipartKey3.result + Execute cat /tmp/part1 /tmp/part3 > /tmp/${PREFIX}-multipartKey3 + Compare files /tmp/${PREFIX}-multipartKey3 /tmp/${PREFIX}-multipartKey3.result Test abort Multipart upload - ${result} = Execute AWSS3APICli create-multipart-upload --bucket ${BUCKET} --key multipartKey4 --storage-class REDUCED_REDUNDANCY + ${result} = Execute AWSS3APICli create-multipart-upload --bucket ${BUCKET} --key ${PREFIX}/multipartKey4 --storage-class REDUCED_REDUNDANCY ${uploadID} = Execute and checkrc echo '${result}' | jq -r '.UploadId' 0 Should contain ${result} ${BUCKET} - Should contain ${result} multipartKey + Should contain ${result} ${PREFIX}/multipartKey Should contain ${result} UploadId - ${result} = Execute AWSS3APICli and checkrc abort-multipart-upload --bucket ${BUCKET} --key multipartKey4 --upload-id ${uploadID} 0 + ${result} = Execute AWSS3APICli and checkrc abort-multipart-upload --bucket ${BUCKET} --key ${PREFIX}/multipartKey4 --upload-id ${uploadID} 0 Test abort Multipart upload with invalid uploadId - ${result} = Execute AWSS3APICli and checkrc abort-multipart-upload --bucket ${BUCKET} --key multipartKey5 --upload-id "random" 255 + ${result} = Execute AWSS3APICli and checkrc abort-multipart-upload --bucket ${BUCKET} --key ${PREFIX}/multipartKey5 --upload-id "random" 255 Upload part with Incorrect uploadID Execute echo "Multipart upload" > /tmp/testfile - ${result} = Execute AWSS3APICli and checkrc upload-part --bucket ${BUCKET} --key multipartKey --part-number 1 --body /tmp/testfile --upload-id "random" 255 + ${result} = Execute AWSS3APICli and checkrc upload-part --bucket ${BUCKET} --key ${PREFIX}/multipartKey --part-number 1 --body /tmp/testfile --upload-id "random" 255 Should contain ${result} NoSuchUpload Test list parts #initiate multipart upload - ${result} = Execute AWSS3APICli create-multipart-upload --bucket ${BUCKET} --key multipartKey5 + ${result} = Execute AWSS3APICli create-multipart-upload --bucket ${BUCKET} --key ${PREFIX}/multipartKey5 ${uploadID} = Execute and checkrc echo '${result}' | jq -r '.UploadId' 0 Should contain ${result} ${BUCKET} - Should contain ${result} multipartKey + Should contain ${result} ${PREFIX}/multipartKey Should contain ${result} UploadId #upload parts Run Keyword Create Random file 5 - ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey5 --part-number 1 --body /tmp/part1 --upload-id ${uploadID} + ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key ${PREFIX}/multipartKey5 --part-number 1 --body /tmp/part1 --upload-id ${uploadID} ${eTag1} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0 Should contain ${result} ETag Execute echo "Part2" > /tmp/part2 - ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey5 --part-number 2 --body /tmp/part2 --upload-id ${uploadID} + ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key ${PREFIX}/multipartKey5 --part-number 2 --body /tmp/part2 --upload-id ${uploadID} ${eTag2} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0 Should contain ${result} ETag #list parts - ${result} = Execute AWSS3APICli list-parts --bucket ${BUCKET} --key multipartKey5 --upload-id ${uploadID} + ${result} = Execute AWSS3APICli list-parts --bucket ${BUCKET} --key ${PREFIX}/multipartKey5 --upload-id ${uploadID} ${part1} = Execute and checkrc echo '${result}' | jq -r '.Parts[0].ETag' 0 ${part2} = Execute and checkrc echo '${result}' | jq -r '.Parts[1].ETag' 0 Should Be equal ${part1} ${eTag1} @@ -204,19 +204,19 @@ Test list parts Should contain ${result} STANDARD #list parts with max-items and next token - ${result} = Execute AWSS3APICli list-parts --bucket ${BUCKET} --key multipartKey5 --upload-id ${uploadID} --max-items 1 + ${result} = Execute AWSS3APICli list-parts --bucket ${BUCKET} --key ${PREFIX}/multipartKey5 --upload-id ${uploadID} --max-items 1 ${part1} = Execute and checkrc echo '${result}' | jq -r '.Parts[0].ETag' 0 ${token} = Execute and checkrc echo '${result}' | jq -r '.NextToken' 0 Should Be equal ${part1} ${eTag1} Should contain ${result} STANDARD - ${result} = Execute AWSS3APICli list-parts --bucket ${BUCKET} --key multipartKey5 --upload-id ${uploadID} --max-items 1 --starting-token ${token} + ${result} = Execute AWSS3APICli list-parts --bucket ${BUCKET} --key ${PREFIX}/multipartKey5 --upload-id ${uploadID} --max-items 1 --starting-token ${token} ${part2} = Execute and checkrc echo '${result}' | jq -r '.Parts[0].ETag' 0 Should Be equal ${part2} ${eTag2} Should contain ${result} STANDARD #finally abort it - ${result} = Execute AWSS3APICli and checkrc abort-multipart-upload --bucket ${BUCKET} --key multipartKey5 --upload-id ${uploadID} 0 + ${result} = Execute AWSS3APICli and checkrc abort-multipart-upload --bucket ${BUCKET} --key ${PREFIX}/multipartKey5 --upload-id ${uploadID} 0 Test Multipart Upload with the simplified aws s3 cp API Create Random file 22 @@ -227,67 +227,67 @@ Test Multipart Upload with the simplified aws s3 cp API Test Multipart Upload Put With Copy Run Keyword Create Random file 5 - ${result} = Execute AWSS3APICli put-object --bucket ${BUCKET} --key copytest/source --body /tmp/part1 + ${result} = Execute AWSS3APICli put-object --bucket ${BUCKET} --key ${PREFIX}/copytest/source --body /tmp/part1 - ${result} = Execute AWSS3APICli create-multipart-upload --bucket ${BUCKET} --key copytest/destination + ${result} = Execute AWSS3APICli create-multipart-upload --bucket ${BUCKET} --key ${PREFIX}/copytest/destination ${uploadID} = Execute and checkrc echo '${result}' | jq -r '.UploadId' 0 Should contain ${result} ${BUCKET} Should contain ${result} UploadId - ${result} = Execute AWSS3APICli upload-part-copy --bucket ${BUCKET} --key copytest/destination --upload-id ${uploadID} --part-number 1 --copy-source ${BUCKET}/copytest/source + ${result} = Execute AWSS3APICli upload-part-copy --bucket ${BUCKET} --key ${PREFIX}/copytest/destination --upload-id ${uploadID} --part-number 1 --copy-source ${BUCKET}/${PREFIX}/copytest/source Should contain ${result} ETag Should contain ${result} LastModified ${eTag1} = Execute and checkrc echo '${result}' | jq -r '.CopyPartResult.ETag' 0 - Execute AWSS3APICli complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key copytest/destination --multipart-upload 'Parts=[{ETag=${eTag1},PartNumber=1}]' - Execute AWSS3APICli get-object --bucket ${BUCKET} --key copytest/destination /tmp/part-result + Execute AWSS3APICli complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key ${PREFIX}/copytest/destination --multipart-upload 'Parts=[{ETag=${eTag1},PartNumber=1}]' + Execute AWSS3APICli get-object --bucket ${BUCKET} --key ${PREFIX}/copytest/destination /tmp/part-result Compare files /tmp/part1 /tmp/part-result Test Multipart Upload Put With Copy and range Run Keyword Create Random file 10 - ${result} = Execute AWSS3APICli put-object --bucket ${BUCKET} --key copyrange/source --body /tmp/part1 + ${result} = Execute AWSS3APICli put-object --bucket ${BUCKET} --key ${PREFIX}/copyrange/source --body /tmp/part1 - ${result} = Execute AWSS3APICli create-multipart-upload --bucket ${BUCKET} --key copyrange/destination + ${result} = Execute AWSS3APICli create-multipart-upload --bucket ${BUCKET} --key ${PREFIX}/copyrange/destination ${uploadID} = Execute and checkrc echo '${result}' | jq -r '.UploadId' 0 Should contain ${result} ${BUCKET} Should contain ${result} UploadId - ${result} = Execute AWSS3APICli upload-part-copy --bucket ${BUCKET} --key copyrange/destination --upload-id ${uploadID} --part-number 1 --copy-source ${BUCKET}/copyrange/source --copy-source-range bytes=0-10485757 + ${result} = Execute AWSS3APICli upload-part-copy --bucket ${BUCKET} --key ${PREFIX}/copyrange/destination --upload-id ${uploadID} --part-number 1 --copy-source ${BUCKET}/${PREFIX}/copyrange/source --copy-source-range bytes=0-10485757 Should contain ${result} ETag Should contain ${result} LastModified ${eTag1} = Execute and checkrc echo '${result}' | jq -r '.CopyPartResult.ETag' 0 - ${result} = Execute AWSS3APICli upload-part-copy --bucket ${BUCKET} --key copyrange/destination --upload-id ${uploadID} --part-number 2 --copy-source ${BUCKET}/copyrange/source --copy-source-range bytes=10485758-10485759 + ${result} = Execute AWSS3APICli upload-part-copy --bucket ${BUCKET} --key ${PREFIX}/copyrange/destination --upload-id ${uploadID} --part-number 2 --copy-source ${BUCKET}/${PREFIX}/copyrange/source --copy-source-range bytes=10485758-10485759 Should contain ${result} ETag Should contain ${result} LastModified ${eTag2} = Execute and checkrc echo '${result}' | jq -r '.CopyPartResult.ETag' 0 - Execute AWSS3APICli complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key copyrange/destination --multipart-upload 'Parts=[{ETag=${eTag1},PartNumber=1},{ETag=${eTag2},PartNumber=2}]' - Execute AWSS3APICli get-object --bucket ${BUCKET} --key copyrange/destination /tmp/part-result + Execute AWSS3APICli complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key ${PREFIX}/copyrange/destination --multipart-upload 'Parts=[{ETag=${eTag1},PartNumber=1},{ETag=${eTag2},PartNumber=2}]' + Execute AWSS3APICli get-object --bucket ${BUCKET} --key ${PREFIX}/copyrange/destination /tmp/part-result Compare files /tmp/part1 /tmp/part-result Test Multipart Upload list - ${result} = Execute AWSS3APICli create-multipart-upload --bucket ${BUCKET} --key listtest/key1 + ${result} = Execute AWSS3APICli create-multipart-upload --bucket ${BUCKET} --key ${PREFIX}/listtest/key1 ${uploadID1} = Execute and checkrc echo '${result}' | jq -r '.UploadId' 0 Should contain ${result} ${BUCKET} - Should contain ${result} listtest/key1 + Should contain ${result} ${PREFIX}/listtest/key1 Should contain ${result} UploadId - ${result} = Execute AWSS3APICli create-multipart-upload --bucket ${BUCKET} --key listtest/key2 + ${result} = Execute AWSS3APICli create-multipart-upload --bucket ${BUCKET} --key ${PREFIX}/listtest/key2 ${uploadID2} = Execute and checkrc echo '${result}' | jq -r '.UploadId' 0 Should contain ${result} ${BUCKET} - Should contain ${result} listtest/key2 + Should contain ${result} ${PREFIX}/listtest/key2 Should contain ${result} UploadId - ${result} = Execute AWSS3APICli list-multipart-uploads --bucket ${BUCKET} --prefix listtest + ${result} = Execute AWSS3APICli list-multipart-uploads --bucket ${BUCKET} --prefix ${PREFIX}/listtest Should contain ${result} ${uploadID1} Should contain ${result} ${uploadID2} diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot b/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot index 74dba38657d6..505e0d1d5bf1 100644 --- a/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot +++ b/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot @@ -91,6 +91,7 @@ Create bucket with name Should contain ${result} ${ENDPOINT_URL}/${bucket} Setup s3 tests + Run Keyword Generate random prefix Run Keyword Install aws cli Run Keyword if '${OZONE_S3_SET_CREDENTIALS}' == 'true' Setup v4 headers ${BUCKET} = Run Keyword if '${BUCKET}' == 'generated' Create bucket @@ -109,3 +110,7 @@ Create link [arguments] ${bucket} Execute ozone sh bucket link o3://${OM_SERVICE_ID}/legacy/source-bucket o3://${OM_SERVICE_ID}/s3v/${bucket} [return] ${bucket} + +Generate random prefix + ${random} = Generate Random String 5 [NUMBERS] + Set Suite Variable ${PREFIX} ${random} diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/objectcopy.robot b/hadoop-ozone/dist/src/main/smoketest/s3/objectcopy.robot index c1b4953e152a..588ead7d2b5e 100644 --- a/hadoop-ozone/dist/src/main/smoketest/s3/objectcopy.robot +++ b/hadoop-ozone/dist/src/main/smoketest/s3/objectcopy.robot @@ -27,41 +27,38 @@ ${ENDPOINT_URL} http://s3g:9878 ${BUCKET} generated ${DESTBUCKET} generated1 - *** Keywords *** Create Dest Bucket - ${postfix} = Generate Random String 5 [NUMBERS] Set Suite Variable ${DESTBUCKET} destbucket-${postfix} Execute AWSS3APICli create-bucket --bucket ${DESTBUCKET} - *** Test Cases *** Copy Object Happy Scenario Run Keyword if '${DESTBUCKET}' == 'generated1' Create Dest Bucket Execute date > /tmp/copyfile - ${result} = Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key copyobject/f1 --body /tmp/copyfile - ${result} = Execute AWSS3ApiCli list-objects --bucket ${BUCKET} --prefix copyobject/ + ${result} = Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key ${PREFIX}/copyobject/f1 --body /tmp/copyfile + ${result} = Execute AWSS3ApiCli list-objects --bucket ${BUCKET} --prefix ${PREFIX}/copyobject/ Should contain ${result} f1 - ${result} = Execute AWSS3ApiCli copy-object --bucket ${DESTBUCKET} --key copyobject/f1 --copy-source ${BUCKET}/copyobject/f1 - ${result} = Execute AWSS3ApiCli list-objects --bucket ${DESTBUCKET} --prefix copyobject/ + ${result} = Execute AWSS3ApiCli copy-object --bucket ${DESTBUCKET} --key ${PREFIX}/copyobject/f1 --copy-source ${BUCKET}/${PREFIX}/copyobject/f1 + ${result} = Execute AWSS3ApiCli list-objects --bucket ${DESTBUCKET} --prefix ${PREFIX}/copyobject/ Should contain ${result} f1 #copying again will not throw error - ${result} = Execute AWSS3ApiCli copy-object --bucket ${DESTBUCKET} --key copyobject/f1 --copy-source ${BUCKET}/copyobject/f1 - ${result} = Execute AWSS3ApiCli list-objects --bucket ${DESTBUCKET} --prefix copyobject/ + ${result} = Execute AWSS3ApiCli copy-object --bucket ${DESTBUCKET} --key ${PREFIX}/copyobject/f1 --copy-source ${BUCKET}/${PREFIX}/copyobject/f1 + ${result} = Execute AWSS3ApiCli list-objects --bucket ${DESTBUCKET} --prefix ${PREFIX}/copyobject/ Should contain ${result} f1 Copy Object Where Bucket is not available - ${result} = Execute AWSS3APICli and checkrc copy-object --bucket dfdfdfdfdfnonexistent --key copyobject/f1 --copy-source ${BUCKET}/copyobject/f1 255 + ${result} = Execute AWSS3APICli and checkrc copy-object --bucket dfdfdfdfdfnonexistent --key ${PREFIX}/copyobject/f1 --copy-source ${BUCKET}/${PREFIX}/copyobject/f1 255 Should contain ${result} NoSuchBucket - ${result} = Execute AWSS3APICli and checkrc copy-object --bucket ${DESTBUCKET} --key copyobject/f1 --copy-source dfdfdfdfdfnonexistent/copyobject/f1 255 + ${result} = Execute AWSS3APICli and checkrc copy-object --bucket ${DESTBUCKET} --key ${PREFIX}/copyobject/f1 --copy-source dfdfdfdfdfnonexistent/${PREFIX}/copyobject/f1 255 Should contain ${result} NoSuchBucket Copy Object Where both source and dest are same with change to storageclass - ${result} = Execute AWSS3APICli copy-object --storage-class REDUCED_REDUNDANCY --bucket ${DESTBUCKET} --key copyobject/f1 --copy-source ${DESTBUCKET}/copyobject/f1 + ${result} = Execute AWSS3APICli copy-object --storage-class REDUCED_REDUNDANCY --bucket ${DESTBUCKET} --key ${PREFIX}/copyobject/f1 --copy-source ${DESTBUCKET}/${PREFIX}/copyobject/f1 Should contain ${result} ETag Copy Object Where Key not available - ${result} = Execute AWSS3APICli and checkrc copy-object --bucket ${DESTBUCKET} --key copyobject/f1 --copy-source ${BUCKET}/nonnonexistentkey 255 + ${result} = Execute AWSS3APICli and checkrc copy-object --bucket ${DESTBUCKET} --key ${PREFIX}/copyobject/f1 --copy-source ${BUCKET}/nonnonexistentkey 255 Should contain ${result} NoSuchKey diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/objectdelete.robot b/hadoop-ozone/dist/src/main/smoketest/s3/objectdelete.robot index b3faf7e1aa32..ca3a5a471158 100644 --- a/hadoop-ozone/dist/src/main/smoketest/s3/objectdelete.robot +++ b/hadoop-ozone/dist/src/main/smoketest/s3/objectdelete.robot @@ -29,11 +29,11 @@ ${BUCKET} generated *** Test Cases *** Delete file with s3api Execute date > /tmp/testfile - ${result} = Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key deletetestapi/f1 --body /tmp/testfile - ${result} = Execute AWSS3ApiCli list-objects --bucket ${BUCKET} --prefix deletetestapi/ + ${result} = Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key ${PREFIX}/deletetestapi/f1 --body /tmp/testfile + ${result} = Execute AWSS3ApiCli list-objects --bucket ${BUCKET} --prefix ${PREFIX}/deletetestapi/ Should contain ${result} f1 - ${result} = Execute AWSS3APICli delete-object --bucket ${BUCKET} --key deletetestapi/f1 - ${result} = Execute AWSS3ApiCli list-objects --bucket ${BUCKET} --prefix deletetestapi/ + ${result} = Execute AWSS3APICli delete-object --bucket ${BUCKET} --key ${PREFIX}/deletetestapi/f1 + ${result} = Execute AWSS3ApiCli list-objects --bucket ${BUCKET} --prefix ${PREFIX}/deletetestapi/ Should not contain ${result} f1 #In case of HTTP 500, the error code is printed out to the console. Should not contain ${result} 500 @@ -47,24 +47,24 @@ Delete file with s3api, file doesn't exist Delete dir with s3api Execute date > /tmp/testfile - ${result} = Execute AWSS3Cli cp /tmp/testfile s3://${BUCKET}/deletetestapidir/f1 - ${result} = Execute AWSS3Cli ls s3://${BUCKET}/deletetestapidir/ + ${result} = Execute AWSS3Cli cp /tmp/testfile s3://${BUCKET}/${PREFIX}/deletetestapidir/f1 + ${result} = Execute AWSS3Cli ls s3://${BUCKET}/${PREFIX}/deletetestapidir/ Should contain ${result} f1 - ${result} = Execute AWSS3APICli delete-object --bucket ${BUCKET} --key deletetestapidir/ - ${result} = Execute AWSS3Cli ls s3://${BUCKET}/deletetestapidir/ + ${result} = Execute AWSS3APICli delete-object --bucket ${BUCKET} --key ${PREFIX}/deletetestapidir/ + ${result} = Execute AWSS3Cli ls s3://${BUCKET}/${PREFIX}/deletetestapidir/ Should contain ${result} f1 - ${result} = Execute AWSS3APICli delete-object --bucket ${BUCKET} --key deletetestapidir/f1 + ${result} = Execute AWSS3APICli delete-object --bucket ${BUCKET} --key ${PREFIX}/deletetestapidir/f1 Delete file with s3api, file doesn't exist, prefix of a real file Execute date > /tmp/testfile - ${result} = Execute AWSS3Cli cp /tmp/testfile s3://${BUCKET}/deletetestapiprefix/filefile - ${result} = Execute AWSS3Cli ls s3://${BUCKET}/deletetestapiprefix/ + ${result} = Execute AWSS3Cli cp /tmp/testfile s3://${BUCKET}/${PREFIX}/deletetestapiprefix/filefile + ${result} = Execute AWSS3Cli ls s3://${BUCKET}/${PREFIX}/deletetestapiprefix/ Should contain ${result} filefile - ${result} = Execute AWSS3APICli delete-object --bucket ${BUCKET} --key deletetestapiprefix/file - ${result} = Execute AWSS3Cli ls s3://${BUCKET}/deletetestapiprefix/ + ${result} = Execute AWSS3APICli delete-object --bucket ${BUCKET} --key ${PREFIX}/deletetestapiprefix/file + ${result} = Execute AWSS3Cli ls s3://${BUCKET}/${PREFIX}/deletetestapiprefix/ Should contain ${result} filefile - ${result} = Execute AWSS3APICli delete-object --bucket ${BUCKET} --key deletetestapiprefix/filefile + ${result} = Execute AWSS3APICli delete-object --bucket ${BUCKET} --key ${PREFIX}/deletetestapiprefix/filefile diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/objectmultidelete.robot b/hadoop-ozone/dist/src/main/smoketest/s3/objectmultidelete.robot index 6e22d4cfb709..cc49129ee3c9 100644 --- a/hadoop-ozone/dist/src/main/smoketest/s3/objectmultidelete.robot +++ b/hadoop-ozone/dist/src/main/smoketest/s3/objectmultidelete.robot @@ -30,20 +30,20 @@ ${BUCKET} generated Delete file with multi delete Execute date > /tmp/testfile - ${result} = Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key multidelete/f1 --body /tmp/testfile - ${result} = Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key multidelete/f2 --body /tmp/testfile - ${result} = Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key multidelete/f3 --body /tmp/testfile - ${result} = Execute AWSS3ApiCli list-objects --bucket ${BUCKET} --prefix multidelete/ - Should contain ${result} multidelete/f1 - Should contain ${result} multidelete/f2 - Should contain ${result} multidelete/f3 + ${result} = Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key ${PREFIX}/multidelete/f1 --body /tmp/testfile + ${result} = Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key ${PREFIX}/multidelete/f2 --body /tmp/testfile + ${result} = Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key ${PREFIX}/multidelete/f3 --body /tmp/testfile + ${result} = Execute AWSS3ApiCli list-objects --bucket ${BUCKET} --prefix ${PREFIX}/multidelete/ + Should contain ${result} ${PREFIX}/multidelete/f1 + Should contain ${result} ${PREFIX}/multidelete/f2 + Should contain ${result} ${PREFIX}/multidelete/f3 Should contain ${result} STANDARD Should not contain ${result} REDUCED_REDUNDANCY - ${result} = Execute AWSS3APICli delete-objects --bucket ${BUCKET} --delete 'Objects=[{Key=multidelete/f1},{Key=multidelete/f2},{Key=multidelete/f4}]' + ${result} = Execute AWSS3APICli delete-objects --bucket ${BUCKET} --delete 'Objects=[{Key=${PREFIX}/multidelete/f1},{Key=${PREFIX}/multidelete/f2},{Key=${PREFIX}/multidelete/f4}]' Should not contain ${result} Error - ${result} = Execute AWSS3ApiCli list-objects --bucket ${BUCKET} --prefix multidelete/ - Should not contain ${result} multidelete/f1 - Should not contain ${result} multidelete/f2 - Should contain ${result} multidelete/f3 + ${result} = Execute AWSS3ApiCli list-objects --bucket ${BUCKET} --prefix ${PREFIX}/multidelete/ + Should not contain ${result} ${PREFIX}/multidelete/f1 + Should not contain ${result} ${PREFIX}/multidelete/f2 + Should contain ${result} ${PREFIX}/multidelete/f3 Should contain ${result} STANDARD Should not contain ${result} REDUCED_REDUNDANCY diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot b/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot index 40bcccb42de6..e1896e22a26a 100644 --- a/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot +++ b/hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot @@ -31,22 +31,22 @@ ${BUCKET} generated Put object to s3 Execute echo "Randomtext" > /tmp/testfile - ${result} = Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key putobject/f1 --body /tmp/testfile - ${result} = Execute AWSS3ApiCli list-objects --bucket ${BUCKET} --prefix putobject/ + ${result} = Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key ${PREFIX}/putobject/f1 --body /tmp/testfile + ${result} = Execute AWSS3ApiCli list-objects --bucket ${BUCKET} --prefix ${PREFIX}/putobject/ Should contain ${result} f1 Execute touch -f /tmp/zerobyte - ${result} = Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key putobject/zerobyte --body /tmp/zerobyte - ${result} = Execute AWSS3ApiCli list-objects --bucket ${BUCKET} --prefix putobject/ + ${result} = Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key ${PREFIX}/putobject/zerobyte --body /tmp/zerobyte + ${result} = Execute AWSS3ApiCli list-objects --bucket ${BUCKET} --prefix ${PREFIX}/putobject/ Should contain ${result} zerobyte #This test depends on the previous test case. Can't be executes alone Get object from s3 - ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key putobject/f1 /tmp/testfile.result + ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/f1 /tmp/testfile.result Compare files /tmp/testfile /tmp/testfile.result Get Partial object from s3 with both start and endoffset - ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key putobject/f1 --range bytes=0-4 /tmp/testfile1.result + ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/f1 --range bytes=0-4 /tmp/testfile1.result Should contain ${result} ContentRange Should contain ${result} bytes 0-4/11 Should contain ${result} AcceptRanges @@ -54,7 +54,7 @@ Get Partial object from s3 with both start and endoffset ${actualData} = Execute cat /tmp/testfile1.result Should Be Equal ${expectedData} ${actualData} - ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key putobject/f1 --range bytes=2-4 /tmp/testfile1.result1 + ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/f1 --range bytes=2-4 /tmp/testfile1.result1 Should contain ${result} ContentRange Should contain ${result} bytes 2-4/11 Should contain ${result} AcceptRanges @@ -63,7 +63,7 @@ Get Partial object from s3 with both start and endoffset Should Be Equal ${expectedData} ${actualData} # end offset greater than file size and start with in file length - ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key putobject/f1 --range bytes=2-1000 /tmp/testfile1.result2 + ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/f1 --range bytes=2-1000 /tmp/testfile1.result2 Should contain ${result} ContentRange Should contain ${result} bytes 2-10/11 Should contain ${result} AcceptRanges @@ -72,12 +72,12 @@ Get Partial object from s3 with both start and endoffset Should Be Equal ${expectedData} ${actualData} Get Partial object from s3 with both start and endoffset(start offset and endoffset is greater than file size) - ${result} = Execute AWSS3APICli and checkrc get-object --bucket ${BUCKET} --key putobject/f1 --range bytes=10000-10000 /tmp/testfile2.result 255 + ${result} = Execute AWSS3APICli and checkrc get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/f1 --range bytes=10000-10000 /tmp/testfile2.result 255 Should contain ${result} InvalidRange Get Partial object from s3 with both start and endoffset(end offset is greater than file size) - ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key putobject/f1 --range bytes=0-10000 /tmp/testfile2.result + ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/f1 --range bytes=0-10000 /tmp/testfile2.result Should contain ${result} ContentRange Should contain ${result} bytes 0-10/11 Should contain ${result} AcceptRanges @@ -86,7 +86,7 @@ Get Partial object from s3 with both start and endoffset(end offset is greater t Should Be Equal ${expectedData} ${actualData} Get Partial object from s3 with only start offset - ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key putobject/f1 --range bytes=0- /tmp/testfile3.result + ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/f1 --range bytes=0- /tmp/testfile3.result Should contain ${result} ContentRange Should contain ${result} bytes 0-10/11 Should contain ${result} AcceptRanges @@ -95,7 +95,7 @@ Get Partial object from s3 with only start offset Should Be Equal ${expectedData} ${actualData} Get Partial object from s3 with both start and endoffset which are equal - ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key putobject/f1 --range bytes=0-0 /tmp/testfile4.result + ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/f1 --range bytes=0-0 /tmp/testfile4.result Should contain ${result} ContentRange Should contain ${result} bytes 0-0/11 Should contain ${result} AcceptRanges @@ -103,7 +103,7 @@ Get Partial object from s3 with both start and endoffset which are equal ${actualData} = Execute cat /tmp/testfile4.result Should Be Equal ${expectedData} ${actualData} - ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key putobject/f1 --range bytes=4-4 /tmp/testfile5.result + ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/f1 --range bytes=4-4 /tmp/testfile5.result Should contain ${result} ContentRange Should contain ${result} bytes 4-4/11 Should contain ${result} AcceptRanges @@ -112,7 +112,7 @@ Get Partial object from s3 with both start and endoffset which are equal Should Be Equal ${expectedData} ${actualData} Get Partial object from s3 to get last n bytes - ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key putobject/f1 --range bytes=-4 /tmp/testfile6.result + ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/f1 --range bytes=-4 /tmp/testfile6.result Should contain ${result} ContentRange Should contain ${result} bytes 7-10/11 Should contain ${result} AcceptRanges @@ -121,7 +121,7 @@ Get Partial object from s3 to get last n bytes Should Be Equal ${expectedData} ${actualData} # if end is greater than file length, returns whole file - ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key putobject/f1 --range bytes=-10000 /tmp/testfile7.result + ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/f1 --range bytes=-10000 /tmp/testfile7.result Should contain ${result} ContentRange Should contain ${result} bytes 0-10/11 Should contain ${result} AcceptRanges @@ -130,14 +130,14 @@ Get Partial object from s3 to get last n bytes Should Be Equal ${expectedData} ${actualData} Incorrect values for end and start offset - ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key putobject/f1 --range bytes=-11-10000 /tmp/testfile8.result + ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/f1 --range bytes=-11-10000 /tmp/testfile8.result Should not contain ${result} ContentRange Should contain ${result} AcceptRanges ${expectedData} = Execute cat /tmp/testfile ${actualData} = Execute cat /tmp/testfile8.result Should Be Equal ${expectedData} ${actualData} - ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key putobject/f1 --range bytes=11-8 /tmp/testfile9.result + ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/f1 --range bytes=11-8 /tmp/testfile9.result Should not contain ${result} ContentRange Should contain ${result} AcceptRanges ${expectedData} = Execute cat /tmp/testfile @@ -145,11 +145,11 @@ Incorrect values for end and start offset Should Be Equal ${expectedData} ${actualData} Zero byte file - ${result} = Execute AWSS3APICli and checkrc get-object --bucket ${BUCKET} --key putobject/zerobyte --range bytes=0-0 /tmp/testfile2.result 255 + ${result} = Execute AWSS3APICli and checkrc get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/zerobyte --range bytes=0-0 /tmp/testfile2.result 255 Should contain ${result} InvalidRange - ${result} = Execute AWSS3APICli and checkrc get-object --bucket ${BUCKET} --key putobject/zerobyte --range bytes=0-1 /tmp/testfile2.result 255 + ${result} = Execute AWSS3APICli and checkrc get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/zerobyte --range bytes=0-1 /tmp/testfile2.result 255 Should contain ${result} InvalidRange - ${result} = Execute AWSS3APICli and checkrc get-object --bucket ${BUCKET} --key putobject/zerobyte --range bytes=0-10000 /tmp/testfile2.result 255 + ${result} = Execute AWSS3APICli and checkrc get-object --bucket ${BUCKET} --key ${PREFIX}/putobject/zerobyte --range bytes=0-10000 /tmp/testfile2.result 255 Should contain ${result} InvalidRange \ No newline at end of file diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/s3_compatbility_check.sh b/hadoop-ozone/dist/src/main/smoketest/s3/s3_compatbility_check.sh new file mode 100755 index 000000000000..3153bd7fbd2a --- /dev/null +++ b/hadoop-ozone/dist/src/main/smoketest/s3/s3_compatbility_check.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# shellcheck disable=SC2086 + +set -e + +# This script helps to execute S3 robot test against real AWS s3 endpoint +# To make sure that all of our defined tests cases copies the behavior of AWS + +: ${OZONE_TEST_S3_BUCKET1:?Please define test bucket} +: ${OZONE_TEST_S3_BUCKET2:?Please define second test bucket} +: ${OZONE_TEST_S3_REGION:?Please define the S3 region for test buckets} + +run_robot_test() { + TEST_NAME=$1 + robot \ + --nostatusrc \ + -v ENDPOINT_URL:https://s3.$OZONE_TEST_S3_REGION.amazonaws.com \ + -v BUCKET:$OZONE_TEST_S3_BUCKET1 \ + -v DESTBUCKET:$OZONE_TEST_S3_BUCKET2 \ + -v OZONE_S3_SET_CREDENTIALS:false \ + -o results/$TEST_NAME.xml \ + $TEST_NAME.robot +} + +mkdir -p results + +run_robot_test objectputget +run_robot_test objectdelete +run_robot_test objectcopy +run_robot_test objectmultidelete +run_robot_test MultipartUpload + +rebot --outputdir results/ results/*.xml