HDDS-13737. S3 ETag JSON should be quoted

hadoop-ozone/dist/src/main/smoketest/s3/mpu_lib.robot

@@ -42,6 +42,7 @@ Upload MPU part
     IF    '${expected_rc}' == '0'
         Should contain    ${result}    ETag
         ${etag} =      Execute    echo '${result}' | jq -r '.ETag'
+        ${etag} =      Replace String    ${etag}    \"    ${EMPTY}
         ${md5sum} =    Execute    md5sum ${file} | awk '{print $1}'
         Should Be Equal As Strings    ${etag}    ${md5sum}
         RETURN    ${etag}

hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java

@@ -118,6 +118,7 @@
 import org.apache.hadoop.ozone.s3.MultiS3GatewayService;
 import org.apache.hadoop.ozone.s3.S3ClientFactory;
 import org.apache.hadoop.ozone.s3.awssdk.S3SDKTestUtils;
+import org.apache.hadoop.ozone.s3.endpoint.ObjectEndpoint;
 import org.apache.hadoop.ozone.s3.endpoint.S3Owner;
 import org.apache.hadoop.ozone.s3.util.S3Consts;
 import org.apache.hadoop.security.UserGroupInformation;
@@ -1231,7 +1232,7 @@ private void completeMPU(String keyName, String uploadId, List<PartETag> complet
       for (PartETag part : completedParts) {
         completionXml.append("  <Part>\n");
         completionXml.append("    <PartNumber>").append(part.getPartNumber()).append("</PartNumber>\n");
-        completionXml.append("    <ETag>").append(part.getETag()).append("</ETag>\n");
+        completionXml.append("    <ETag>").append(ObjectEndpoint.stripQuotes(part.getETag())).append("</ETag>\n");
         completionXml.append("  </Part>\n");
       }
       completionXml.append("</CompleteMultipartUpload>");

hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java

@@ -68,6 +68,7 @@
 import org.apache.hadoop.ozone.s3.MultiS3GatewayService;
 import org.apache.hadoop.ozone.s3.S3ClientFactory;
 import org.apache.hadoop.ozone.s3.awssdk.S3SDKTestUtils;
+import org.apache.hadoop.ozone.s3.endpoint.ObjectEndpoint;
 import org.apache.hadoop.ozone.s3.endpoint.S3Owner;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.ozone.test.OzoneTestBase;
@@ -970,7 +971,7 @@ private String buildCompleteMultipartUploadXml(List<CompletedPart> parts) {
       for (CompletedPart part : parts) {
         xml.append("  <Part>\n");
         xml.append("    <PartNumber>").append(part.partNumber()).append("</PartNumber>\n");
-        xml.append("    <ETag>").append(part.eTag()).append("</ETag>\n");
+        xml.append("    <ETag>").append(ObjectEndpoint.stripQuotes(part.eTag())).append("</ETag>\n");
         xml.append("  </Part>\n");
       }
       xml.append("</CompleteMultipartUpload>");
@@ -1142,11 +1143,11 @@ private List<CompletedPart> uploadParts(String bucketName, String key, String up
             RequestBody.fromByteBuffer(bb));
 
         assertEquals(DatatypeConverter.printHexBinary(
-            calculateDigest(fileInputStream, 0, partSize)).toLowerCase(), partResponse.eTag());
+            calculateDigest(fileInputStream, 0, partSize)).toLowerCase(), ObjectEndpoint.stripQuotes(partResponse.eTag()));
 
         CompletedPart part = CompletedPart.builder()
             .partNumber(partNumber)
-            .eTag(partResponse.eTag())
+            .eTag(ObjectEndpoint.stripQuotes(partResponse.eTag()))
             .build();
         completedParts.add(part);
 
@@ -1643,7 +1644,7 @@ public void testCompleteMultipartUpload() {
 
         CompletedMultipartUpload completedUpload = CompletedMultipartUpload.builder()
             .parts(
-                CompletedPart.builder().partNumber(1).eTag(uploadPartResponse.eTag()).build()
+                CompletedPart.builder().partNumber(1).eTag(ObjectEndpoint.stripQuotes(uploadPartResponse.eTag())).build()
             ).build();
 
 

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java

@@ -905,7 +905,7 @@ public Response completeMultipartUpload(@PathParam("bucket") String bucket,
       S3Owner.verifyBucketOwnerCondition(headers, bucket, ozoneBucket.getOwner());
 
       for (CompleteMultipartUploadRequest.Part part : partList) {
-        partsMap.put(part.getPartNumber(), part.getETag());
+        partsMap.put(part.getPartNumber(), stripQuotes(part.getETag()));
       }
       if (LOG.isDebugEnabled()) {
         LOG.debug("Parts map {}", partsMap);
@@ -1065,6 +1065,10 @@ private Response createMultipartKey(OzoneVolume volume, OzoneBucket ozoneBucket,
                   new byte[getIOBufferSize(length)]);
               ozoneOutputStream.getMetadata()
                   .putAll(sourceKeyDetails.getMetadata());
+              String raw = ozoneOutputStream.getMetadata().get(ETAG);
+              if (raw != null) {
+                ozoneOutputStream.getMetadata().put(ETAG, stripQuotes(raw));
+              }
               outputStream = ozoneOutputStream;
             }
           }
@@ -1099,6 +1103,7 @@ private Response createMultipartKey(OzoneVolume volume, OzoneBucket ozoneBucket,
       if (StringUtils.isEmpty(eTag)) {
         eTag = omMultipartCommitUploadPartInfo.getPartName();
       }
+      eTag = wrapInQuotes(eTag);
 
       if (copyHeader != null) {
         getMetrics().updateCopyObjectSuccessStats(startNanos);
@@ -1518,6 +1523,13 @@ public boolean isDatastreamEnabled() {
     return datastreamEnabled;
   }
 
+  public static String stripQuotes(String value) {
+    if (value == null) {
+      return null;
+    }
+    return value.replaceAll("^\\\"|\\\"$", "");
+  }
+
   static String wrapInQuotes(String value) {
     return "\"" + value + "\"";
   }

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java

@@ -189,6 +189,8 @@ public static Response createMultipartKey(OzoneBucket ozoneBucket, String key,
       }
       throw ex;
     }
-    return Response.ok().header(OzoneConsts.ETAG, eTag).build();
+    return Response.ok()
+        .header(OzoneConsts.ETAG, ObjectEndpoint.wrapInQuotes(eTag))
+        .build();
   }
 }