HDDS-10947. Add robot test for rewrite of multipart key

hadoop-ozone/dist/src/main/compose/common/ec-test.sh

@@ -20,6 +20,8 @@ start_docker_env 5
 ## Exclude virtual-host tests. This is tested separately as it requires additional config.
 execute_robot_test scm -v BUCKET:erasure --exclude virtual-host s3
 
+execute_robot_test scm ec/rewrite.robot
+
 prefix=${RANDOM}
 execute_robot_test scm -v PREFIX:${prefix} ec/basic.robot
 docker-compose up -d --no-recreate --scale datanode=4

hadoop-ozone/dist/src/main/smoketest/ec/rewrite.robot

@@ -0,0 +1,45 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+Documentation       Test EC shell commands
+Library             OperatingSystem
+Resource            ../commonlib.robot
+Resource            ../lib/os.robot
+Resource            ../ozone-lib/shell.robot
+Resource            ../s3/commonawslib.robot
+Resource            lib.resource
+Suite Setup         Run Keyword if    '${SECURITY_ENABLED}' == 'true'    Kinit test user     testuser     testuser.keytab
+
+
+*** Variables ***
+${ENDPOINT_URL}       http://s3g:9878
+
+
+*** Test Cases ***
+
+Rewrite Multipart Key
+    [setup]    Setup v4 headers
+    ${bucket} =    Create bucket with layout    /s3v    OBJECT_STORE
+    ${key} =    Set Variable    multipart.key
+    ${file} =    Create Random File MB    12
+    Execute AWSS3Cli    cp ${file} s3://${bucket}/${key}
+    Key Should Match Local File    /s3v/${bucket}/${key}    ${file}
+    Verify Key Replica Replication Config    /s3v/${bucket}/${key}    RATIS    THREE
+
+    Execute    ozone sh key rewrite -t EC -r rs-3-2-1024k /s3v/${bucket}/${key}
+
+    Key Should Match Local File    /s3v/${bucket}/${key}    ${file}
+    Verify Key EC Replication Config    /s3v/${bucket}/${key}    RS    3    2    1048576

hadoop-ozone/dist/src/main/smoketest/lib/os.robot

@@ -40,13 +40,28 @@ Compare files
     ${checksumafter} =          Execute                    md5sum ${file2} | awk '{print $1}'
                                 Should Be Equal            ${checksumbefore}            ${checksumafter}
 
+Create Random File MB
+    [arguments]    ${size_in_megabytes}    ${path}=${EMPTY}
+    ${path} =      Create Random File      ${size_in_megabytes}    1048576    ${path}
+    [return]       ${path}
+
+Create Random File KB
+    [arguments]    ${size_in_kilobytes}    ${path}=${EMPTY}
+    ${path} =      Create Random File      ${size_in_kilobytes}    1024    ${path}
+    [return]       ${path}
+
 Create Random File
-    ${postfix} =             Generate Random String  5  [NUMBERS]
+    [arguments]    ${block_count}    ${block_size}    ${path}=${EMPTY}
+    ${path} =      Run Keyword If   '${path}' == '${EMPTY}'    Get Random Filename
+    ...            ELSE             Set Variable    ${path}
+    Execute        dd if=/dev/urandom of=${path} bs=${block_size} count=${block_count} status=none
+    [return]       ${path}
+
+Get Random Filename
+    ${postfix} =             Generate Random String  10  [LOWER]
     ${tmpfile} =             Set Variable   /tmp/tempfile-${postfix}
     File Should Not Exist    ${tmpfile}
-    ${content} =             Set Variable   "Random string"
-    Create File              ${tmpfile}    ${content}
-    [Return]                 ${tmpfile}
+    [return]                 ${tmpfile}
 
 List All Processes
     ${output} =    Execute    ps aux

hadoop-ozone/dist/src/main/smoketest/ozone-lib/shell.robot

@@ -56,6 +56,13 @@ Find Jars Dir
     ${dir} =    Execute    ozone envvars | grep 'HDDS_LIB_JARS_DIR' | cut -f2 -d= | sed -e "s/'//g" -e 's/"//g'
     Set Environment Variable    HDDS_LIB_JARS_DIR    ${dir}
 
+Create bucket with layout
+    [Arguments]          ${volume}    ${layout}
+    ${postfix} =         Generate Random String    10    [LOWER]
+    ${bucket} =          Set Variable    bucket-${postfix}
+    ${result} =          Execute         ozone sh bucket create --layout ${layout} ${volume}/${bucket}
+    [Return]             ${bucket}
+
 Create Key
     [arguments]    ${key}    ${file}    ${args}=${EMPTY}
     ${output} =    Execute          ozone sh key put ${args} ${key} ${file}

hadoop-ozone/dist/src/main/smoketest/ozone-lib/shell_tests.robot

@@ -48,7 +48,7 @@ Key Should Match Local File
     Key Should Match Local File     o3://${OM_SERVICE_ID}/vol1/bucket/passwd    /etc/passwd
 
 Compare Key With Local File with Different File
-    ${random_file} =            Create Random File
+    ${random_file} =            Create Random File KB    42
     ${matches} =                Compare Key With Local File     o3://${OM_SERVICE_ID}/vol1/bucket/passwd    ${random_file}
     Should Be Equal             ${matches}     ${FALSE}
     [Teardown]                  Remove File    ${random_file}

hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot

@@ -22,25 +22,27 @@ Resource            ../commonlib.robot
 Resource            commonawslib.robot
 Test Timeout        5 minutes
 Suite Setup         Setup Multipart Tests
+Suite Teardown      Teardown Multipart Tests
 Test Setup          Generate random prefix
 
 *** Keywords ***
 Setup Multipart Tests
     Setup s3 tests
 
     # 5MB + a bit
-    Create Random File KB    /tmp/part1    5121
+    Create Random File KB    5121    /tmp/part1
 
     # 1MB - a bit
-    Create Random File KB    /tmp/part2    1023
+    Create Random File KB    1023    /tmp/part2
 
-Create Random file
-    [arguments]             ${size_in_megabytes}
-    Execute                 dd if=/dev/urandom of=/tmp/part1 bs=1048576 count=${size_in_megabytes} status=none
+    Create Random File MB    10      /tmp/10mb
+    Create Random File MB    22      /tmp/22mb
+    Create Random File KB    10      /tmp/10kb
+
+
+Teardown Multipart Tests
+    Remove Files    /tmp/part1 /tmp/part2 /tmp/10mb /tmp/22mb /tmp/10kb
 
-Create Random File KB
-    [arguments]             ${file}    ${size_in_kilobytes}
-    Execute                 dd if=/dev/urandom of=${file} bs=1024 count=${size_in_kilobytes} status=none
 
 Wait Til Date Past
     [arguments]         ${date}
@@ -77,11 +79,9 @@ Test Multipart Upload
 # upload we get error entity too small. So, considering further complete
 # multipart upload, uploading each part as 5MB file, exception is for last part
 
-    Run Keyword         Create Random file      5
     ${result} =         Execute AWSS3APICli     upload-part --bucket ${BUCKET} --key ${PREFIX}/multipartKey --part-number 1 --body /tmp/part1 --upload-id ${nextUploadID}
                         Should contain          ${result}    ETag
 # override part
-    Run Keyword         Create Random file      5
     ${result} =         Execute AWSS3APICli     upload-part --bucket ${BUCKET} --key ${PREFIX}/multipartKey --part-number 1 --body /tmp/part1 --upload-id ${nextUploadID}
                         Should contain          ${result}    ETag
 
@@ -94,7 +94,6 @@ Test Multipart Upload Complete
                         Should contain          ${result}    UploadId
 
 #upload parts
-    Run Keyword         Create Random file            5
     ${result} =         Execute AWSS3APICli           upload-part --bucket ${BUCKET} --key ${PREFIX}/multipartKey1 --part-number 1 --body /tmp/part1 --upload-id ${uploadID}
     ${eTag1} =          Execute and checkrc           echo '${result}' | jq -r '.ETag'   0
                         Should contain                ${result}    ETag
@@ -152,13 +151,11 @@ Test Multipart Upload Complete Entity too small
                         Should contain          ${result}    UploadId
 
 #upload parts
-                        Execute                 echo "Part1" > /tmp/part1
-    ${result} =         Execute AWSS3APICli     upload-part --bucket ${BUCKET} --key ${PREFIX}/multipartKey2 --part-number 1 --body /tmp/part1 --upload-id ${uploadID}
+    ${result} =         Execute AWSS3APICli     upload-part --bucket ${BUCKET} --key ${PREFIX}/multipartKey2 --part-number 1 --body /tmp/10kb --upload-id ${uploadID}
     ${eTag1} =          Execute and checkrc     echo '${result}' | jq -r '.ETag'   0
                         Should contain          ${result}    ETag
 
-                        Execute                 echo "Part2" > /tmp/part2
-    ${result} =         Execute AWSS3APICli     upload-part --bucket ${BUCKET} --key ${PREFIX}/multipartKey2 --part-number 2 --body /tmp/part2 --upload-id ${uploadID}
+    ${result} =         Execute AWSS3APICli     upload-part --bucket ${BUCKET} --key ${PREFIX}/multipartKey2 --part-number 2 --body /tmp/10kb --upload-id ${uploadID}
     ${eTag2} =          Execute and checkrc     echo '${result}' | jq -r '.ETag'   0
                         Should contain          ${result}    ETag
 
@@ -180,7 +177,6 @@ Test Multipart Upload Complete Invalid part errors and complete mpu with few par
     ${result} =         Execute AWSS3APICli and checkrc  complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key ${PREFIX}/multipartKey3 --multipart-upload 'Parts=[{ETag=etag1,PartNumber=2},{ETag=etag2,PartNumber=1}]'    255
                         Should contain          ${result}    InvalidPart
 #upload parts
-                        Run Keyword             Create Random file      5
     ${result} =         Execute AWSS3APICli     upload-part --bucket ${BUCKET} --key ${PREFIX}/multipartKey3 --part-number 1 --body /tmp/part1 --upload-id ${uploadID}
     ${eTag1} =          Execute and checkrc     echo '${result}' | jq -r '.ETag'   0
                         Should contain          ${result}    ETag
@@ -245,7 +241,6 @@ Test list parts
                         Should contain          ${result}    UploadId
 
 #upload parts
-    Run Keyword         Create Random file      5
     ${result} =         Execute AWSS3APICli     upload-part --bucket ${BUCKET} --key ${PREFIX}/multipartKey5 --part-number 1 --body /tmp/part1 --upload-id ${uploadID}
     ${eTag1} =          Execute and checkrc     echo '${result}' | jq -r '.ETag'   0
                         Should contain          ${result}    ETag
@@ -279,14 +274,12 @@ Test list parts
     ${result} =         Execute AWSS3APICli and checkrc    abort-multipart-upload --bucket ${BUCKET} --key ${PREFIX}/multipartKey5 --upload-id ${uploadID}    0
 
 Test Multipart Upload with the simplified aws s3 cp API
-                        Create Random file      22
-                        Execute AWSS3Cli        cp /tmp/part1 s3://${BUCKET}/mpyawscli
-                        Execute AWSS3Cli        cp s3://${BUCKET}/mpyawscli /tmp/part1.result
+                        Execute AWSS3Cli        cp /tmp/22mb s3://${BUCKET}/mpyawscli
+                        Execute AWSS3Cli        cp s3://${BUCKET}/mpyawscli /tmp/22mb.result
                         Execute AWSS3Cli        rm s3://${BUCKET}/mpyawscli
-                        Compare files           /tmp/part1        /tmp/part1.result
+                        Compare files           /tmp/22mb        /tmp/22mb.result
 
 Test Multipart Upload Put With Copy
-    Run Keyword         Create Random file      5
     ${result} =         Execute AWSS3APICli     put-object --bucket ${BUCKET} --key ${PREFIX}/copytest/source --body /tmp/part1
 
 
@@ -308,8 +301,7 @@ Test Multipart Upload Put With Copy
                         Compare files           /tmp/part1        /tmp/part-result
 
 Test Multipart Upload Put With Copy and range
-    Run Keyword         Create Random file      10
-    ${result} =         Execute AWSS3APICli     put-object --bucket ${BUCKET} --key ${PREFIX}/copyrange/source --body /tmp/part1
+    ${result} =         Execute AWSS3APICli     put-object --bucket ${BUCKET} --key ${PREFIX}/copyrange/source --body /tmp/10mb
 
 
     ${result} =         Execute AWSS3APICli     create-multipart-upload --bucket ${BUCKET} --key ${PREFIX}/copyrange/destination
@@ -332,15 +324,14 @@ Test Multipart Upload Put With Copy and range
                         Execute AWSS3APICli     complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key ${PREFIX}/copyrange/destination --multipart-upload 'Parts=[{ETag=${eTag1},PartNumber=1},{ETag=${eTag2},PartNumber=2}]'
                         Execute AWSS3APICli     get-object --bucket ${BUCKET} --key ${PREFIX}/copyrange/destination /tmp/part-result
 
-                        Compare files           /tmp/part1        /tmp/part-result
+                        Compare files           /tmp/10mb        /tmp/part-result
 
 Test Multipart Upload Put With Copy and range with IfModifiedSince
-    Run Keyword         Create Random file      10
     ${curDate} =        Get Current Date
     ${beforeCreate} =   Subtract Time From Date     ${curDate}  1 day
     ${tomorrow} =       Add Time To Date            ${curDate}  1 day
 
-    ${result} =         Execute AWSS3APICli     put-object --bucket ${BUCKET} --key ${PREFIX}/copyrange/source --body /tmp/part1
+    ${result} =         Execute AWSS3APICli     put-object --bucket ${BUCKET} --key ${PREFIX}/copyrange/source --body /tmp/10mb
 
     ${result} =         Execute AWSS3APICli     create-multipart-upload --bucket ${BUCKET} --key ${PREFIX}/copyrange/destination
 
@@ -385,7 +376,7 @@ Test Multipart Upload Put With Copy and range with IfModifiedSince
                         Execute AWSS3APICli     complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key ${PREFIX}/copyrange/destination --multipart-upload 'Parts=[{ETag=${eTag1},PartNumber=1},{ETag=${eTag2},PartNumber=2}]'
                         Execute AWSS3APICli     get-object --bucket ${BUCKET} --key ${PREFIX}/copyrange/destination /tmp/part-result
 
-                        Compare files           /tmp/part1        /tmp/part-result
+                        Compare files           /tmp/10mb        /tmp/part-result
 
 Test Multipart Upload list
     ${result} =         Execute AWSS3APICli     create-multipart-upload --bucket ${BUCKET} --key ${PREFIX}/listtest/key1

hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot

@@ -129,13 +129,6 @@ Create bucket with name
                          Should contain              ${result}         Location
                          Should contain              ${result}         ${bucket}
 
-Create bucket with layout
-    [Arguments]          ${layout}
-    ${postfix} =         Generate Ozone String
-    ${bucket} =          Set Variable    bucket-${postfix}
-    ${result} =          Execute         ozone sh bucket create --layout ${layout} s3v/${bucket}
-    [Return]             ${bucket}
-
 Setup s3 tests
     Return From Keyword if    ${OZONE_S3_TESTS_SET_UP}
     Run Keyword        Generate random prefix
@@ -156,7 +149,7 @@ Setup links for S3 tests
 
 Create generated bucket
     [Arguments]          ${layout}=OBJECT_STORE
-    ${BUCKET} =          Create bucket with layout    ${layout}
+    ${BUCKET} =          Create bucket with layout    s3v    ${layout}
     Set Global Variable   ${BUCKET}
 
 Create encrypted bucket