diff --git a/hadoop-ozone/dist/src/main/compose/common/ec-test.sh b/hadoop-ozone/dist/src/main/compose/common/ec-test.sh index 65a659563e59..04df2b2787d3 100755 --- a/hadoop-ozone/dist/src/main/compose/common/ec-test.sh +++ b/hadoop-ozone/dist/src/main/compose/common/ec-test.sh @@ -20,6 +20,8 @@ start_docker_env 5 ## Exclude virtual-host tests. This is tested separately as it requires additional config. execute_robot_test scm -v BUCKET:erasure --exclude virtual-host s3 +execute_robot_test scm ec/rewrite.robot + prefix=${RANDOM} execute_robot_test scm -v PREFIX:${prefix} ec/basic.robot docker-compose up -d --no-recreate --scale datanode=4 diff --git a/hadoop-ozone/dist/src/main/smoketest/ec/rewrite.robot b/hadoop-ozone/dist/src/main/smoketest/ec/rewrite.robot new file mode 100644 index 000000000000..5b5df209eced --- /dev/null +++ b/hadoop-ozone/dist/src/main/smoketest/ec/rewrite.robot @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*** Settings *** +Documentation Test EC shell commands +Library OperatingSystem +Resource ../commonlib.robot +Resource ../lib/os.robot +Resource ../ozone-lib/shell.robot +Resource ../s3/commonawslib.robot +Resource lib.resource +Suite Setup Run Keyword if '${SECURITY_ENABLED}' == 'true' Kinit test user testuser testuser.keytab + + +*** Variables *** +${ENDPOINT_URL} http://s3g:9878 + + +*** Test Cases *** + +Rewrite Multipart Key + [setup] Setup v4 headers + ${bucket} = Create bucket with layout /s3v OBJECT_STORE + ${key} = Set Variable multipart.key + ${file} = Create Random File MB 12 + Execute AWSS3Cli cp ${file} s3://${bucket}/${key} + Key Should Match Local File /s3v/${bucket}/${key} ${file} + Verify Key Replica Replication Config /s3v/${bucket}/${key} RATIS THREE + + Execute ozone sh key rewrite -t EC -r rs-3-2-1024k /s3v/${bucket}/${key} + + Key Should Match Local File /s3v/${bucket}/${key} ${file} + Verify Key EC Replication Config /s3v/${bucket}/${key} RS 3 2 1048576 diff --git a/hadoop-ozone/dist/src/main/smoketest/lib/os.robot b/hadoop-ozone/dist/src/main/smoketest/lib/os.robot index cc20d6ee286e..98f836ff8bf7 100644 --- a/hadoop-ozone/dist/src/main/smoketest/lib/os.robot +++ b/hadoop-ozone/dist/src/main/smoketest/lib/os.robot @@ -40,13 +40,28 @@ Compare files ${checksumafter} = Execute md5sum ${file2} | awk '{print $1}' Should Be Equal ${checksumbefore} ${checksumafter} +Create Random File MB + [arguments] ${size_in_megabytes} ${path}=${EMPTY} + ${path} = Create Random File ${size_in_megabytes} 1048576 ${path} + [return] ${path} + +Create Random File KB + [arguments] ${size_in_kilobytes} ${path}=${EMPTY} + ${path} = Create Random File ${size_in_kilobytes} 1024 ${path} + [return] ${path} + Create Random File - ${postfix} = Generate Random String 5 [NUMBERS] + [arguments] ${block_count} ${block_size} ${path}=${EMPTY} + ${path} = Run Keyword If '${path}' == '${EMPTY}' Get Random Filename + ... ELSE Set Variable ${path} + Execute dd if=/dev/urandom of=${path} bs=${block_size} count=${block_count} status=none + [return] ${path} + +Get Random Filename + ${postfix} = Generate Random String 10 [LOWER] ${tmpfile} = Set Variable /tmp/tempfile-${postfix} File Should Not Exist ${tmpfile} - ${content} = Set Variable "Random string" - Create File ${tmpfile} ${content} - [Return] ${tmpfile} + [return] ${tmpfile} List All Processes ${output} = Execute ps aux diff --git a/hadoop-ozone/dist/src/main/smoketest/ozone-lib/shell.robot b/hadoop-ozone/dist/src/main/smoketest/ozone-lib/shell.robot index b255d516da56..ffefda833349 100644 --- a/hadoop-ozone/dist/src/main/smoketest/ozone-lib/shell.robot +++ b/hadoop-ozone/dist/src/main/smoketest/ozone-lib/shell.robot @@ -56,6 +56,13 @@ Find Jars Dir ${dir} = Execute ozone envvars | grep 'HDDS_LIB_JARS_DIR' | cut -f2 -d= | sed -e "s/'//g" -e 's/"//g' Set Environment Variable HDDS_LIB_JARS_DIR ${dir} +Create bucket with layout + [Arguments] ${volume} ${layout} + ${postfix} = Generate Random String 10 [LOWER] + ${bucket} = Set Variable bucket-${postfix} + ${result} = Execute ozone sh bucket create --layout ${layout} ${volume}/${bucket} + [Return] ${bucket} + Create Key [arguments] ${key} ${file} ${args}=${EMPTY} ${output} = Execute ozone sh key put ${args} ${key} ${file} diff --git a/hadoop-ozone/dist/src/main/smoketest/ozone-lib/shell_tests.robot b/hadoop-ozone/dist/src/main/smoketest/ozone-lib/shell_tests.robot index 56fbcf8b61f0..22805efcb1bb 100644 --- a/hadoop-ozone/dist/src/main/smoketest/ozone-lib/shell_tests.robot +++ b/hadoop-ozone/dist/src/main/smoketest/ozone-lib/shell_tests.robot @@ -48,7 +48,7 @@ Key Should Match Local File Key Should Match Local File o3://${OM_SERVICE_ID}/vol1/bucket/passwd /etc/passwd Compare Key With Local File with Different File - ${random_file} = Create Random File + ${random_file} = Create Random File KB 42 ${matches} = Compare Key With Local File o3://${OM_SERVICE_ID}/vol1/bucket/passwd ${random_file} Should Be Equal ${matches} ${FALSE} [Teardown] Remove File ${random_file} diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot b/hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot index 96feec2f8133..55bc880cd61a 100644 --- a/hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot +++ b/hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot @@ -22,6 +22,7 @@ Resource ../commonlib.robot Resource commonawslib.robot Test Timeout 5 minutes Suite Setup Setup Multipart Tests +Suite Teardown Teardown Multipart Tests Test Setup Generate random prefix *** Keywords *** @@ -29,18 +30,19 @@ Setup Multipart Tests Setup s3 tests # 5MB + a bit - Create Random File KB /tmp/part1 5121 + Create Random File KB 5121 /tmp/part1 # 1MB - a bit - Create Random File KB /tmp/part2 1023 + Create Random File KB 1023 /tmp/part2 -Create Random file - [arguments] ${size_in_megabytes} - Execute dd if=/dev/urandom of=/tmp/part1 bs=1048576 count=${size_in_megabytes} status=none + Create Random File MB 10 /tmp/10mb + Create Random File MB 22 /tmp/22mb + Create Random File KB 10 /tmp/10kb + + +Teardown Multipart Tests + Remove Files /tmp/part1 /tmp/part2 /tmp/10mb /tmp/22mb /tmp/10kb -Create Random File KB - [arguments] ${file} ${size_in_kilobytes} - Execute dd if=/dev/urandom of=${file} bs=1024 count=${size_in_kilobytes} status=none Wait Til Date Past [arguments] ${date} @@ -77,11 +79,9 @@ Test Multipart Upload # upload we get error entity too small. So, considering further complete # multipart upload, uploading each part as 5MB file, exception is for last part - Run Keyword Create Random file 5 ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key ${PREFIX}/multipartKey --part-number 1 --body /tmp/part1 --upload-id ${nextUploadID} Should contain ${result} ETag # override part - Run Keyword Create Random file 5 ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key ${PREFIX}/multipartKey --part-number 1 --body /tmp/part1 --upload-id ${nextUploadID} Should contain ${result} ETag @@ -94,7 +94,6 @@ Test Multipart Upload Complete Should contain ${result} UploadId #upload parts - Run Keyword Create Random file 5 ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key ${PREFIX}/multipartKey1 --part-number 1 --body /tmp/part1 --upload-id ${uploadID} ${eTag1} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0 Should contain ${result} ETag @@ -152,13 +151,11 @@ Test Multipart Upload Complete Entity too small Should contain ${result} UploadId #upload parts - Execute echo "Part1" > /tmp/part1 - ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key ${PREFIX}/multipartKey2 --part-number 1 --body /tmp/part1 --upload-id ${uploadID} + ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key ${PREFIX}/multipartKey2 --part-number 1 --body /tmp/10kb --upload-id ${uploadID} ${eTag1} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0 Should contain ${result} ETag - Execute echo "Part2" > /tmp/part2 - ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key ${PREFIX}/multipartKey2 --part-number 2 --body /tmp/part2 --upload-id ${uploadID} + ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key ${PREFIX}/multipartKey2 --part-number 2 --body /tmp/10kb --upload-id ${uploadID} ${eTag2} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0 Should contain ${result} ETag @@ -180,7 +177,6 @@ Test Multipart Upload Complete Invalid part errors and complete mpu with few par ${result} = Execute AWSS3APICli and checkrc complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key ${PREFIX}/multipartKey3 --multipart-upload 'Parts=[{ETag=etag1,PartNumber=2},{ETag=etag2,PartNumber=1}]' 255 Should contain ${result} InvalidPart #upload parts - Run Keyword Create Random file 5 ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key ${PREFIX}/multipartKey3 --part-number 1 --body /tmp/part1 --upload-id ${uploadID} ${eTag1} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0 Should contain ${result} ETag @@ -245,7 +241,6 @@ Test list parts Should contain ${result} UploadId #upload parts - Run Keyword Create Random file 5 ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key ${PREFIX}/multipartKey5 --part-number 1 --body /tmp/part1 --upload-id ${uploadID} ${eTag1} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0 Should contain ${result} ETag @@ -279,14 +274,12 @@ Test list parts ${result} = Execute AWSS3APICli and checkrc abort-multipart-upload --bucket ${BUCKET} --key ${PREFIX}/multipartKey5 --upload-id ${uploadID} 0 Test Multipart Upload with the simplified aws s3 cp API - Create Random file 22 - Execute AWSS3Cli cp /tmp/part1 s3://${BUCKET}/mpyawscli - Execute AWSS3Cli cp s3://${BUCKET}/mpyawscli /tmp/part1.result + Execute AWSS3Cli cp /tmp/22mb s3://${BUCKET}/mpyawscli + Execute AWSS3Cli cp s3://${BUCKET}/mpyawscli /tmp/22mb.result Execute AWSS3Cli rm s3://${BUCKET}/mpyawscli - Compare files /tmp/part1 /tmp/part1.result + Compare files /tmp/22mb /tmp/22mb.result Test Multipart Upload Put With Copy - Run Keyword Create Random file 5 ${result} = Execute AWSS3APICli put-object --bucket ${BUCKET} --key ${PREFIX}/copytest/source --body /tmp/part1 @@ -308,8 +301,7 @@ Test Multipart Upload Put With Copy Compare files /tmp/part1 /tmp/part-result Test Multipart Upload Put With Copy and range - Run Keyword Create Random file 10 - ${result} = Execute AWSS3APICli put-object --bucket ${BUCKET} --key ${PREFIX}/copyrange/source --body /tmp/part1 + ${result} = Execute AWSS3APICli put-object --bucket ${BUCKET} --key ${PREFIX}/copyrange/source --body /tmp/10mb ${result} = Execute AWSS3APICli create-multipart-upload --bucket ${BUCKET} --key ${PREFIX}/copyrange/destination @@ -332,15 +324,14 @@ Test Multipart Upload Put With Copy and range Execute AWSS3APICli complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key ${PREFIX}/copyrange/destination --multipart-upload 'Parts=[{ETag=${eTag1},PartNumber=1},{ETag=${eTag2},PartNumber=2}]' Execute AWSS3APICli get-object --bucket ${BUCKET} --key ${PREFIX}/copyrange/destination /tmp/part-result - Compare files /tmp/part1 /tmp/part-result + Compare files /tmp/10mb /tmp/part-result Test Multipart Upload Put With Copy and range with IfModifiedSince - Run Keyword Create Random file 10 ${curDate} = Get Current Date ${beforeCreate} = Subtract Time From Date ${curDate} 1 day ${tomorrow} = Add Time To Date ${curDate} 1 day - ${result} = Execute AWSS3APICli put-object --bucket ${BUCKET} --key ${PREFIX}/copyrange/source --body /tmp/part1 + ${result} = Execute AWSS3APICli put-object --bucket ${BUCKET} --key ${PREFIX}/copyrange/source --body /tmp/10mb ${result} = Execute AWSS3APICli create-multipart-upload --bucket ${BUCKET} --key ${PREFIX}/copyrange/destination @@ -385,7 +376,7 @@ Test Multipart Upload Put With Copy and range with IfModifiedSince Execute AWSS3APICli complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key ${PREFIX}/copyrange/destination --multipart-upload 'Parts=[{ETag=${eTag1},PartNumber=1},{ETag=${eTag2},PartNumber=2}]' Execute AWSS3APICli get-object --bucket ${BUCKET} --key ${PREFIX}/copyrange/destination /tmp/part-result - Compare files /tmp/part1 /tmp/part-result + Compare files /tmp/10mb /tmp/part-result Test Multipart Upload list ${result} = Execute AWSS3APICli create-multipart-upload --bucket ${BUCKET} --key ${PREFIX}/listtest/key1 diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot b/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot index b20537014dd1..45dee9270bd8 100644 --- a/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot +++ b/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot @@ -129,13 +129,6 @@ Create bucket with name Should contain ${result} Location Should contain ${result} ${bucket} -Create bucket with layout - [Arguments] ${layout} - ${postfix} = Generate Ozone String - ${bucket} = Set Variable bucket-${postfix} - ${result} = Execute ozone sh bucket create --layout ${layout} s3v/${bucket} - [Return] ${bucket} - Setup s3 tests Return From Keyword if ${OZONE_S3_TESTS_SET_UP} Run Keyword Generate random prefix @@ -156,7 +149,7 @@ Setup links for S3 tests Create generated bucket [Arguments] ${layout}=OBJECT_STORE - ${BUCKET} = Create bucket with layout ${layout} + ${BUCKET} = Create bucket with layout s3v ${layout} Set Global Variable ${BUCKET} Create encrypted bucket