Skip to content

HDDS-10645. Support x-amz-metadata-directive in CopyObject #6490

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
adoroszlai merged 4 commits into from
Apr 10, 2024
Merged

HDDS-10645. Support x-amz-metadata-directive in CopyObject #6490

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
28ddba7
HDDS-10645. Support x-amz-metadata-directive in CopyObject
ivandika3 Apr 7, 2024
2cdbcde
Add unit test
ivandika3 Apr 7, 2024
4522a5e
Fix acceptance test
ivandika3 Apr 7, 2024
2adff15
Typo and comment update
ivandika3 Apr 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 29 additions & 3 deletions hadoop-ozone/dist/src/main/smoketest/s3/objectcopy.robot
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,26 +39,42 @@ Copy Object Happy Scenario
Execute date > /tmp/copyfile
${file_checksum} = Execute md5sum /tmp/copyfile | awk '{print $1}'

${result} = Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key ${PREFIX}/copyobject/key=value/f1 --body /tmp/copyfile
${result} = Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key ${PREFIX}/copyobject/key=value/f1 --body /tmp/copyfile --metadata="custom-key1=custom-value1,custom-key2=custom-value2,gdprEnabled=true"
${eTag} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
Should Be Equal ${eTag} \"${file_checksum}\"

${result} = Execute AWSS3ApiCli list-objects --bucket ${BUCKET} --prefix ${PREFIX}/copyobject/key=value/
Should contain ${result} f1

${result} = Execute AWSS3ApiCli copy-object --bucket ${DESTBUCKET} --key ${PREFIX}/copyobject/key=value/f1 --copy-source ${BUCKET}/${PREFIX}/copyobject/key=value/f1
${result} = Execute AWSS3ApiCli copy-object --bucket ${DESTBUCKET} --key ${PREFIX}/copyobject/key=value/f1 --copy-source ${BUCKET}/${PREFIX}/copyobject/key=value/f1 --metadata="custom-key3=custom-value3,custom-key4=custom-value4"
${eTag} = Execute and checkrc echo '${result}' | jq -r '.CopyObjectResult.ETag' 0
Should Be Equal ${eTag} \"${file_checksum}\"

${result} = Execute AWSS3ApiCli list-objects --bucket ${DESTBUCKET} --prefix ${PREFIX}/copyobject/key=value/
Should contain ${result} f1

#check that the custom metadata of the source key has been copied to the destination key (default copy directive is COPY)
${result} = Execute AWSS3ApiCli head-object --bucket ${BUCKET} --key ${PREFIX}/copyobject/key=value/f1
Should contain ${result} \"custom-key1\": \"custom-value1\"
Should contain ${result} \"custom-key2\": \"custom-value2\"
# COPY directive ignores any metadata specified in the copy object request
Should Not contain ${result} \"custom-key3\": \"custom-value3\"
Should Not contain ${result} \"custom-key4\": \"custom-value4\"

#copying again will not throw error
${result} = Execute AWSS3ApiCli copy-object --bucket ${DESTBUCKET} --key ${PREFIX}/copyobject/key=value/f1 --copy-source ${BUCKET}/${PREFIX}/copyobject/key=value/f1
#also uses the REPLACE copy directive
${result} = Execute AWSS3ApiCli copy-object --bucket ${DESTBUCKET} --key ${PREFIX}/copyobject/key=value/f1 --copy-source ${BUCKET}/${PREFIX}/copyobject/key=value/f1 --metadata="custom-key3=custom-value3,custom-key4=custom-value4" --metadata-directive REPLACE
${eTag} = Execute and checkrc echo '${result}' | jq -r '.CopyObjectResult.ETag' 0
Should Be Equal ${eTag} \"${file_checksum}\"

${result} = Execute AWSS3ApiCli list-objects --bucket ${DESTBUCKET} --prefix ${PREFIX}/copyobject/key=value/
Should contain ${result} f1
${result} = Execute AWSS3ApiCli head-object --bucket ${DESTBUCKET} --key ${PREFIX}/copyobject/key=value/f1
Should contain ${result} \"custom-key3\": \"custom-value3\"
Should contain ${result} \"custom-key4\": \"custom-value4\"
# REPLACE directive uses the custom metadata specified in the request instead of the source key's custom metadata
Should Not contain ${result} \"custom-key1\": \"custom-value1\"
Should Not contain ${result} \"custom-key2\": \"custom-value2\"

Copy Object Where Bucket is not available
${result} = Execute AWSS3APICli and checkrc copy-object --bucket dfdfdfdfdfnonexistent --key ${PREFIX}/copyobject/key=value/f1 --copy-source ${BUCKET}/${PREFIX}/copyobject/key=value/f1 255
Expand All @@ -76,3 +92,13 @@ Copy Object Where both source and dest are same with change to storageclass
Copy Object Where Key not available
${result} = Execute AWSS3APICli and checkrc copy-object --bucket ${DESTBUCKET} --key ${PREFIX}/copyobject/key=value/f1 --copy-source ${BUCKET}/nonnonexistentkey 255
Should contain ${result} NoSuchKey

Copy Object using an invalid copy directive
${result} = Execute AWSS3ApiCli and checkrc copy-object --bucket ${DESTBUCKET} --key ${PREFIX}/copyobject/key=value/f1 --copy-source ${BUCKET}/${PREFIX}/copyobject/key=value/f1 --metadata-directive INVALID 255
Should contain ${result} InvalidArgument

Copy Object with user defined metadata size larger than 2 KB
Execute echo "Randomtext" > /tmp/testfile2
${custom_metadata_value} = Execute printf 'v%.0s' {1..3000}
${result} = Execute AWSS3ApiCli and checkrc copy-object --bucket ${DESTBUCKET} --key ${PREFIX}/copyobject/key=value/f1 --copy-source ${BUCKET}/${PREFIX}/copyobject/key=value/f1 --metadata="custom-key1=${custom_metadata_value}" --metadata-directive REPLACE 255
Should contain ${result} MetadataTooLarge
22 changes: 21 additions & 1 deletion hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,7 @@
import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.PRECOND_FAILED;
import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.newError;
import static org.apache.hadoop.ozone.s3.util.S3Consts.ACCEPT_RANGE_HEADER;
import static org.apache.hadoop.ozone.s3.util.S3Consts.CUSTOM_METADATA_COPY_DIRECTIVE_HEADER;
import static org.apache.hadoop.ozone.s3.util.S3Consts.DECODED_CONTENT_LENGTH_HEADER;
import static org.apache.hadoop.ozone.s3.util.S3Consts.CONTENT_RANGE_HEADER;
import static org.apache.hadoop.ozone.s3.util.S3Consts.COPY_SOURCE_HEADER;
Expand All @@ -135,6 +136,7 @@
import static org.apache.hadoop.ozone.s3.util.S3Consts.RANGE_HEADER;
import static org.apache.hadoop.ozone.s3.util.S3Consts.RANGE_HEADER_SUPPORTED_UNIT;
import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CLASS_HEADER;
import static org.apache.hadoop.ozone.s3.util.S3Consts.CopyDirective;
import static org.apache.hadoop.ozone.s3.util.S3Utils.urlDecode;

/**
Expand Down Expand Up @@ -1208,12 +1210,30 @@ private CopyObjectResponse copyObject(OzoneVolume volume,
}
}
long sourceKeyLen = sourceKeyDetails.getDataSize();

// Custom metadata in copyObject with metadata directive
Map<String, String> customMetadata;
String metadataCopyDirective = headers.getHeaderString(CUSTOM_METADATA_COPY_DIRECTIVE_HEADER);
if (StringUtils.isEmpty(metadataCopyDirective) || metadataCopyDirective.equals(CopyDirective.COPY.name())) {
// The custom metadata will be copied from the source key
customMetadata = sourceKeyDetails.getMetadata();
} else if (metadataCopyDirective.equals(CopyDirective.REPLACE.name())) {
// Replace the metadata with the metadata form the request headers
customMetadata = getCustomMetadataFromHeaders(headers.getRequestHeaders());
} else {
OS3Exception ex = newError(INVALID_ARGUMENT, metadataCopyDirective);
ex.setErrorMessage("An error occurred (InvalidArgument) " +
"when calling the CopyObject operation: " +
"The metadata directive specified is invalid. Valid values are COPY or REPLACE.");
throw ex;
}

try (OzoneInputStream src = getClientProtocol().getKey(volume.getName(),
sourceBucket, sourceKey)) {
getMetrics().updateCopyKeyMetadataStats(startNanos);
sourceDigestInputStream = new DigestInputStream(src, getMessageDigestInstance());
copy(volume, sourceDigestInputStream, sourceKeyLen, destkey, destBucket, replicationConfig,
sourceKeyDetails.getMetadata(), perf, startNanos);
customMetadata, perf, startNanos);
}

final OzoneKeyDetails destKeyDetails = getClientProtocol().getKeyDetails(
Expand Down
10 changes: 10 additions & 0 deletions hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,10 +62,20 @@ private S3Consts() {
public static final String S3_XML_NAMESPACE = "http://s3.amazonaws" +
".com/doc/2006-03-01/";

// Constants related to custom metadata
public static final String CUSTOM_METADATA_HEADER_PREFIX = "x-amz-meta-";
public static final String CUSTOM_METADATA_COPY_DIRECTIVE_HEADER = "x-amz-metadata-directive";


public static final String DECODED_CONTENT_LENGTH_HEADER =
"x-amz-decoded-content-length";

/**
* Copy directive for metadata and tags.
*/
public enum CopyDirective {
COPY, // Default directive
REPLACE
}

}
63 changes: 62 additions & 1 deletion hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@
import java.io.OutputStream;
import java.security.MessageDigest;
import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.MultivaluedHashMap;
import javax.ws.rs.core.MultivaluedMap;
import javax.ws.rs.core.Response;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.RandomStringUtils;
Expand Down Expand Up @@ -56,6 +58,8 @@
import org.mockito.MockedStatic;

import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.hadoop.ozone.s3.util.S3Consts.CUSTOM_METADATA_COPY_DIRECTIVE_HEADER;
import static org.apache.hadoop.ozone.s3.util.S3Consts.CUSTOM_METADATA_HEADER_PREFIX;
import static org.apache.hadoop.ozone.s3.util.S3Consts.DECODED_CONTENT_LENGTH_HEADER;
import static org.apache.hadoop.ozone.s3.util.S3Consts.COPY_SOURCE_HEADER;
import static org.apache.hadoop.ozone.s3.util.S3Consts.STORAGE_CLASS_HEADER;
Expand Down Expand Up @@ -254,6 +258,14 @@ void testCopyObject() throws IOException, OS3Exception {
ByteArrayInputStream body =
new ByteArrayInputStream(CONTENT.getBytes(UTF_8));

// Add some custom metadata
MultivaluedMap<String, String> metadataHeaders = new MultivaluedHashMap<>();
metadataHeaders.putSingle(CUSTOM_METADATA_HEADER_PREFIX + "custom-key-1", "custom-value-1");
metadataHeaders.putSingle(CUSTOM_METADATA_HEADER_PREFIX + "custom-key-2", "custom-value-2");
when(headers.getRequestHeaders()).thenReturn(metadataHeaders);
// Add COPY metadata directive (default)
when(headers.getHeaderString(CUSTOM_METADATA_COPY_DIRECTIVE_HEADER)).thenReturn("COPY");

Response response = objectEndpoint.put(BUCKET_NAME, KEY_NAME,
CONTENT.length(), 1, null, body);

Expand All @@ -268,9 +280,14 @@ void testCopyObject() throws IOException, OS3Exception {
assertEquals(CONTENT, keyContent);
assertNotNull(keyDetails.getMetadata());
assertThat(keyDetails.getMetadata().get(OzoneConsts.ETAG)).isNotEmpty();
assertThat(keyDetails.getMetadata().get("custom-key-1")).isEqualTo("custom-value-1");
assertThat(keyDetails.getMetadata().get("custom-key-2")).isEqualTo("custom-value-2");

String sourceETag = keyDetails.getMetadata().get(OzoneConsts.ETAG);

// This will be ignored since the copy directive is COPY
metadataHeaders.putSingle(CUSTOM_METADATA_HEADER_PREFIX + "custom-key-3", "custom-value-3");

// Add copy header, and then call put
when(headers.getHeaderString(COPY_SOURCE_HEADER)).thenReturn(
BUCKET_NAME + "/" + urlEncode(KEY_NAME));
Expand All @@ -296,9 +313,53 @@ void testCopyObject() throws IOException, OS3Exception {
// the same Etag since the key content is the same
assertEquals(sourceETag, sourceKeyDetails.getMetadata().get(OzoneConsts.ETAG));
assertEquals(sourceETag, destKeyDetails.getMetadata().get(OzoneConsts.ETAG));
assertThat(destKeyDetails.getMetadata().get("custom-key-1")).isEqualTo("custom-value-1");
assertThat(destKeyDetails.getMetadata().get("custom-key-2")).isEqualTo("custom-value-2");
assertThat(destKeyDetails.getMetadata().containsKey("custom-key-3")).isFalse();

// source and dest same
// Now use REPLACE metadata directive (default) and remove some custom metadata used in the source key
when(headers.getHeaderString(CUSTOM_METADATA_COPY_DIRECTIVE_HEADER)).thenReturn("REPLACE");
metadataHeaders.remove(CUSTOM_METADATA_HEADER_PREFIX + "custom-key-1");
metadataHeaders.remove(CUSTOM_METADATA_HEADER_PREFIX + "custom-key-2");

response = objectEndpoint.put(DEST_BUCKET_NAME, DEST_KEY, CONTENT.length(), 1,
null, body);

ozoneInputStream = clientStub.getObjectStore().getS3Bucket(DEST_BUCKET_NAME)
.readKey(DEST_KEY);

keyContent = IOUtils.toString(ozoneInputStream, UTF_8);
sourceKeyDetails = clientStub.getObjectStore()
.getS3Bucket(BUCKET_NAME).getKey(KEY_NAME);
destKeyDetails = clientStub.getObjectStore()
.getS3Bucket(DEST_BUCKET_NAME).getKey(DEST_KEY);

assertEquals(200, response.getStatus());
assertEquals(CONTENT, keyContent);
assertNotNull(keyDetails.getMetadata());
assertThat(keyDetails.getMetadata().get(OzoneConsts.ETAG)).isNotEmpty();
// Source key eTag should remain unchanged and the dest key should have
// the same Etag since the key content is the same
assertEquals(sourceETag, sourceKeyDetails.getMetadata().get(OzoneConsts.ETAG));
assertEquals(sourceETag, destKeyDetails.getMetadata().get(OzoneConsts.ETAG));
assertThat(destKeyDetails.getMetadata().containsKey("custom-key-1")).isFalse();
assertThat(destKeyDetails.getMetadata().containsKey("custom-key-2")).isFalse();
assertThat(destKeyDetails.getMetadata().get("custom-key-3")).isEqualTo("custom-value-3");


// wrong copy metadata directive
when(headers.getHeaderString(CUSTOM_METADATA_COPY_DIRECTIVE_HEADER)).thenReturn("INVALID");
OS3Exception e = assertThrows(OS3Exception.class, () -> objectEndpoint.put(
DEST_BUCKET_NAME, DEST_KEY, CONTENT.length(), 1, null, body),
"test copy object failed");
assertThat(e.getHttpCode()).isEqualTo(400);
assertThat(e.getCode()).isEqualTo("InvalidArgument");
assertThat(e.getErrorMessage()).contains("The metadata directive specified is invalid");

when(headers.getHeaderString(CUSTOM_METADATA_COPY_DIRECTIVE_HEADER)).thenReturn("COPY");

// source and dest same
e = assertThrows(OS3Exception.class, () -> objectEndpoint.put(
BUCKET_NAME, KEY_NAME, CONTENT.length(), 1, null, body),
"test copy object failed");
assertThat(e.getErrorMessage()).contains("This copy request is illegal");
Expand Down