Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion hadoop-ozone/dist/src/main/smoketest/commonlib.robot
Original file line number Diff line number Diff line change
Expand Up @@ -28,4 +28,10 @@ Execute and checkrc
${rc} ${output} = Run And Return Rc And Output ${command}
Log ${output}
Should Be Equal As Integers ${rc} ${expected_error_code}
[return] ${output}
[return] ${output}

Compare files
[arguments] ${file1} ${file2}
${checksumbefore} = Execute md5sum ${file1} | awk '{print $1}'
${checksumafter} = Execute md5sum ${file2} | awk '{print $1}'
Should Be Equal ${checksumbefore} ${checksumafter}
135 changes: 65 additions & 70 deletions hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,9 @@ Resource commonawslib.robot
Test Setup Setup s3 tests

*** Keywords ***
Create Random file for mac
Execute dd if=/dev/urandom of=/tmp/part1 bs=1m count=5

Create Random file for linux
Execute dd if=/dev/urandom of=/tmp/part1 bs=1M count=5
Create Random file
[arguments] ${size_in_megabytes}
Execute dd if=/dev/urandom of=/tmp/part1 bs=1048576 count=${size_in_megabytes}


*** Variables ***
Expand Down Expand Up @@ -54,16 +52,13 @@ Test Multipart Upload
# upload we get error entity too small. So, considering further complete
# multipart upload, uploading each part as 5MB file, exception is for last part

${system} = Evaluate platform.system() platform
Run Keyword if '${system}' == 'Darwin' Create Random file for mac
Run Keyword if '${system}' == 'Linux' Create Random file for linux
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey --part-number 1 --body /tmp/part1 --upload-id ${nextUploadID}
Should contain ${result} ETag
Run Keyword Create Random file 5
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey --part-number 1 --body /tmp/part1 --upload-id ${nextUploadID}
Should contain ${result} ETag
# override part
Run Keyword if '${system}' == 'Darwin' Create Random file for mac
Run Keyword if '${system}' == 'Linux' Create Random file for linux
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey --part-number 1 --body /tmp/part1 --upload-id ${nextUploadID}
Should contain ${result} ETag
Run Keyword Create Random file 5
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey --part-number 1 --body /tmp/part1 --upload-id ${nextUploadID}
Should contain ${result} ETag


Test Multipart Upload Complete
Expand All @@ -74,17 +69,15 @@ Test Multipart Upload Complete
Should contain ${result} UploadId

#upload parts
${system} = Evaluate platform.system() platform
Run Keyword if '${system}' == 'Darwin' Create Random file for mac
Run Keyword if '${system}' == 'Linux' Create Random file for linux
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey1 --part-number 1 --body /tmp/part1 --upload-id ${uploadID}
${eTag1} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
Should contain ${result} ETag
Run Keyword Create Random file 5
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey1 --part-number 1 --body /tmp/part1 --upload-id ${uploadID}
${eTag1} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
Should contain ${result} ETag

Execute echo "Part2" > /tmp/part2
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey1 --part-number 2 --body /tmp/part2 --upload-id ${uploadID}
${eTag2} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
Should contain ${result} ETag
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey1 --part-number 2 --body /tmp/part2 --upload-id ${uploadID}
${eTag2} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
Should contain ${result} ETag
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Are these unintended changes? As I see the change is indentation.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yetus asked me to not use tab. Without fixing the tabs the patch would cause a whitespace mismatch (my new lines with spaces and the old lines with tabs). So it's intentional, but I can move to a different


#complete multipart upload
${result} = Execute AWSS3APICli complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key multipartKey1 --multipart-upload 'Parts=[{ETag=${eTag1},PartNumber=1},{ETag=${eTag2},PartNumber=2}]'
Expand All @@ -94,11 +87,8 @@ Test Multipart Upload Complete

#read file and check the key
${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key multipartKey1 /tmp/multipartKey1.result
Execute cat /tmp/part1 /tmp/part2 >> /tmp/multipartkey1
${checksumbefore} = Execute md5sum /tmp/multipartkey1 | awk '{print $1}'
${checksumafter} = Execute md5sum /tmp/multipartKey1.result | awk '{print $1}'
Should Be Equal ${checksumbefore} ${checksumafter}

Execute cat /tmp/part1 /tmp/part2 >> /tmp/multipartKey1
Compare files /tmp/multipartKey1 /tmp/multipartKey1.result

Test Multipart Upload Complete Entity too small
${result} = Execute AWSS3APICli create-multipart-upload --bucket ${BUCKET} --key multipartKey2
Expand All @@ -109,14 +99,14 @@ Test Multipart Upload Complete Entity too small

#upload parts
Execute echo "Part1" > /tmp/part1
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey2 --part-number 1 --body /tmp/part1 --upload-id ${uploadID}
${eTag1} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
Should contain ${result} ETag
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey2 --part-number 1 --body /tmp/part1 --upload-id ${uploadID}
${eTag1} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
Should contain ${result} ETag

Execute echo "Part2" > /tmp/part2
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey2 --part-number 2 --body /tmp/part2 --upload-id ${uploadID}
${eTag2} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
Should contain ${result} ETag
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey2 --part-number 2 --body /tmp/part2 --upload-id ${uploadID}
${eTag2} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
Should contain ${result} ETag
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we remove these indentation changes?


#complete multipart upload
${result} = Execute AWSS3APICli and checkrc complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key multipartKey2 --multipart-upload 'Parts=[{ETag=${eTag1},PartNumber=1},{ETag=${eTag2},PartNumber=2}]' 255
Expand All @@ -132,14 +122,14 @@ Test Multipart Upload Complete Invalid part

#upload parts
Execute echo "Part1" > /tmp/part1
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey3 --part-number 1 --body /tmp/part1 --upload-id ${uploadID}
${eTag1} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
Should contain ${result} ETag
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey3 --part-number 1 --body /tmp/part1 --upload-id ${uploadID}
${eTag1} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
Should contain ${result} ETag

Execute echo "Part2" > /tmp/part2
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey3 --part-number 2 --body /tmp/part2 --upload-id ${uploadID}
${eTag2} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
Should contain ${result} ETag
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey3 --part-number 2 --body /tmp/part2 --upload-id ${uploadID}
${eTag2} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
Should contain ${result} ETag

#complete multipart upload
${result} = Execute AWSS3APICli and checkrc complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key multipartKey3 --multipart-upload 'Parts=[{ETag=etag1,PartNumber=1},{ETag=etag2,PartNumber=2}]' 255
Expand All @@ -158,9 +148,9 @@ Test abort Multipart upload with invalid uploadId
${result} = Execute AWSS3APICli and checkrc abort-multipart-upload --bucket ${BUCKET} --key multipartKey5 --upload-id "random" 255

Upload part with Incorrect uploadID
Execute echo "Multipart upload" > /tmp/testfile
${result} = Execute AWSS3APICli and checkrc upload-part --bucket ${BUCKET} --key multipartKey --part-number 1 --body /tmp/testfile --upload-id "random" 255
Should contain ${result} NoSuchUpload
Execute echo "Multipart upload" > /tmp/testfile
${result} = Execute AWSS3APICli and checkrc upload-part --bucket ${BUCKET} --key multipartKey --part-number 1 --body /tmp/testfile --upload-id "random" 255
Should contain ${result} NoSuchUpload

Test list parts
#initiate multipart upload
Expand All @@ -171,37 +161,42 @@ Test list parts
Should contain ${result} UploadId

#upload parts
${system} = Evaluate platform.system() platform
Run Keyword if '${system}' == 'Darwin' Create Random file for mac
Run Keyword if '${system}' == 'Linux' Create Random file for linux
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey5 --part-number 1 --body /tmp/part1 --upload-id ${uploadID}
${eTag1} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
Should contain ${result} ETag

Execute echo "Part2" > /tmp/part2
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey5 --part-number 2 --body /tmp/part2 --upload-id ${uploadID}
${eTag2} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
Should contain ${result} ETag
Run Keyword Create Random file 5
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey5 --part-number 1 --body /tmp/part1 --upload-id ${uploadID}
${eTag1} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
Should contain ${result} ETag

Execute echo "Part2" > /tmp/part2
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key multipartKey5 --part-number 2 --body /tmp/part2 --upload-id ${uploadID}
${eTag2} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
Should contain ${result} ETag

#list parts
${result} = Execute AWSS3APICli list-parts --bucket ${BUCKET} --key multipartKey5 --upload-id ${uploadID}
${part1} = Execute and checkrc echo '${result}' | jq -r '.Parts[0].ETag' 0
${part2} = Execute and checkrc echo '${result}' | jq -r '.Parts[1].ETag' 0
Should Be equal ${part1} ${eTag1}
Should contain ${part2} ${eTag2}
Should contain ${result} STANDARD
${result} = Execute AWSS3APICli list-parts --bucket ${BUCKET} --key multipartKey5 --upload-id ${uploadID}
${part1} = Execute and checkrc echo '${result}' | jq -r '.Parts[0].ETag' 0
${part2} = Execute and checkrc echo '${result}' | jq -r '.Parts[1].ETag' 0
Should Be equal ${part1} ${eTag1}
Should contain ${part2} ${eTag2}
Should contain ${result} STANDARD

#list parts with max-items and next token
${result} = Execute AWSS3APICli list-parts --bucket ${BUCKET} --key multipartKey5 --upload-id ${uploadID} --max-items 1
${part1} = Execute and checkrc echo '${result}' | jq -r '.Parts[0].ETag' 0
${token} = Execute and checkrc echo '${result}' | jq -r '.NextToken' 0
Should Be equal ${part1} ${eTag1}
Should contain ${result} STANDARD
${result} = Execute AWSS3APICli list-parts --bucket ${BUCKET} --key multipartKey5 --upload-id ${uploadID} --max-items 1
${part1} = Execute and checkrc echo '${result}' | jq -r '.Parts[0].ETag' 0
${token} = Execute and checkrc echo '${result}' | jq -r '.NextToken' 0
Should Be equal ${part1} ${eTag1}
Should contain ${result} STANDARD

${result} = Execute AWSS3APICli list-parts --bucket ${BUCKET} --key multipartKey5 --upload-id ${uploadID} --max-items 1 --starting-token ${token}
${part2} = Execute and checkrc echo '${result}' | jq -r '.Parts[0].ETag' 0
Should Be equal ${part2} ${eTag2}
Should contain ${result} STANDARD
${result} = Execute AWSS3APICli list-parts --bucket ${BUCKET} --key multipartKey5 --upload-id ${uploadID} --max-items 1 --starting-token ${token}
${part2} = Execute and checkrc echo '${result}' | jq -r '.Parts[0].ETag' 0
Should Be equal ${part2} ${eTag2}
Should contain ${result} STANDARD

#finally abort it
${result} = Execute AWSS3APICli and checkrc abort-multipart-upload --bucket ${BUCKET} --key multipartKey5 --upload-id ${uploadID} 0
${result} = Execute AWSS3APICli and checkrc abort-multipart-upload --bucket ${BUCKET} --key multipartKey5 --upload-id ${uploadID} 0

Test Multipart Upload with the simplified aws s3 cp API
Create Random file 22
Execute AWSS3Cli cp /tmp/part1 s3://${BUCKET}/mpyawscli
Execute AWSS3Cli cp s3://${BUCKET}/mpyawscli /tmp/part1.result
Execute AWSS3Cli rm s3://${BUCKET}/mpyawscli
Compare files /tmp/part1 /tmp/part1.result
4 changes: 1 addition & 3 deletions hadoop-ozone/dist/src/main/smoketest/s3/objectputget.robot
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,7 @@ Put object to s3
#This test depends on the previous test case. Can't be executes alone
Get object from s3
${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key putobject/f1 /tmp/testfile.result
${checksumbefore} = Execute md5sum /tmp/testfile | awk '{print $1}'
${checksumafter} = Execute md5sum /tmp/testfile.result | awk '{print $1}'
Should Be Equal ${checksumbefore} ${checksumafter}
Compare files /tmp/testfile /tmp/testfile.result

Get Partial object from s3 with both start and endoffset
${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key putobject/f1 --range bytes=0-4 /tmp/testfile1.result
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.ozone.s3.endpoint;

import javax.ws.rs.Consumes;
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.MultivaluedMap;
import javax.ws.rs.ext.MessageBodyReader;
import javax.ws.rs.ext.Provider;
import java.io.IOException;
import java.io.InputStream;
import java.lang.annotation.Annotation;
import java.lang.reflect.Type;

/**
* Body reader to accept plain text MPU.
* <p>
* Aws s3 api sends a multipartupload request with the content type
* 'text/plain' in case of using 'aws s3 cp' (instead of aws s3api).
* <p>
* Our generic ObjectEndpoint.multipartUpload has a
* CompleteMultipartUploadRequest parameter, which is required only for the
* completion request.
* <p>
* But JaxRS tries to parse it from the body for the requests and in case of
* text/plain requests this parsing is failed. This simple BodyReader enables
* to parse an empty text/plain message and return with an empty completion
* request.
*/
@Provider
@Consumes("text/plain")
public class PlainTextMultipartUploadReader
implements MessageBodyReader<CompleteMultipartUploadRequest> {

@Override
public boolean isReadable(Class<?> type, Type genericType,
Annotation[] annotations, MediaType mediaType) {
return type.equals(CompleteMultipartUploadRequest.class)
&& mediaType.equals(MediaType.TEXT_PLAIN_TYPE);
}

@Override
public CompleteMultipartUploadRequest readFrom(
Class<CompleteMultipartUploadRequest> type, Type genericType,
Annotation[] annotations, MediaType mediaType,
MultivaluedMap<String, String> httpHeaders, InputStream entityStream)
throws IOException, WebApplicationException {
return new CompleteMultipartUploadRequest();
}
}