HDDS-10395. Fix eTag compatibility issues during MPU

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneMultipartUploadPartListParts.java

@@ -98,13 +98,13 @@ public ReplicationConfig getReplicationConfig() {
   /**
    * Class that represents each Part information of a multipart upload part.
    */
-  public static class PartInfo {
+  public static final class PartInfo {
 
-    private int partNumber;
-    private String partName;
-    private long modificationTime;
-    private long size;
-    private String eTag;
+    private final int partNumber;
+    private final String partName;
+    private final long modificationTime;
+    private final long size;
+    private final String eTag;
 
     public PartInfo(int number, String name, long time, long size,
                     String eTag) {

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmPartInfo.java

@@ -23,12 +23,12 @@
 /**
  * Class that defines information about each part of a multipart upload key.
  */
-public class OmPartInfo {
-  private int partNumber;
-  private String partName;
-  private long modificationTime;
-  private long size;
-  private String eTag;
+public final class OmPartInfo {
+  private final int partNumber;
+  private final String partName;
+  private final long modificationTime;
+  private final long size;
+  private final String eTag;
 
   public OmPartInfo(int number, String name, long time, long size,
                     String eTag) {
@@ -60,8 +60,14 @@ public String getETag() {
   }
 
   public PartInfo getProto() {
-    return PartInfo.newBuilder().setPartNumber(partNumber).setPartName(partName)
-       .setModificationTime(modificationTime)
-       .setSize(size).setETag(eTag).build();
+    PartInfo.Builder builder = PartInfo.newBuilder()
+        .setPartNumber(partNumber)
+        .setPartName(partName)
+        .setModificationTime(modificationTime)
+        .setSize(size);
+    if (eTag != null) {
+      builder.setETag(eTag);
+    }
+    return builder.build();
   }
 }

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java

@@ -32,6 +32,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.Set;
 import java.util.Stack;
 import java.util.TreeMap;
@@ -47,6 +48,7 @@
 import org.apache.hadoop.hdds.client.ReplicationConfig;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.protocol.DatanodeDetails;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
 import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
 import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
 import org.apache.hadoop.hdds.scm.storage.BlockLocationInfo;
@@ -822,13 +824,19 @@ public OmMultipartUploadListParts listParts(String volumeName,
           if (nextPartNumberMarker > partNumberMarker) {
             String partName = getPartName(partKeyInfo, volumeName, bucketName,
                 keyName);
+            // Before HDDS-9680, MPU part does not have eTag metadata, for
+            // this case, we return null. The S3G will handle this case by
+            // using the MPU part name as the eTag field instead.
+            Optional<HddsProtos.KeyValue> eTag = partKeyInfo.getPartKeyInfo()
+                .getMetadataList()
+                .stream()
+                .filter(keyValue -> keyValue.getKey().equals(ETAG))
+                .findFirst();
             OmPartInfo omPartInfo = new OmPartInfo(partKeyInfo.getPartNumber(),
                 partName,
                 partKeyInfo.getPartKeyInfo().getModificationTime(),
                 partKeyInfo.getPartKeyInfo().getDataSize(),
-                partKeyInfo.getPartKeyInfo().getMetadataList().stream()
-                    .filter(keyValue -> keyValue.getKey().equals(ETAG))
-                    .findFirst().get().getValue());
+                eTag.map(HddsProtos.KeyValue::getValue).orElse(null));
             omPartInfoList.add(omPartInfo);
 
             //if there are parts, use replication type from one of the parts

hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java

@@ -65,6 +65,7 @@
 import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadList;
 import org.apache.hadoop.ozone.om.helpers.OmMultipartUploadListParts;
 import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
+import org.apache.hadoop.ozone.om.helpers.OpenKeySession;
 import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
 import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
 import org.apache.hadoop.ozone.om.request.OMRequestTestUtils;
@@ -161,6 +162,60 @@ public void listMultipartUploadPartsWithZeroUpload() throws IOException {
         omMultipartUploadListParts.getPartInfoList().size());
   }
 
+  @Test
+  public void listMultipartUploadPartsWithoutEtagField() throws IOException {
+    // For backward compatibility reasons
+    final String volume = volumeName();
+    final String bucket = "bucketForEtag";
+    final String key = "dir/key1";
+    createBucket(metadataManager, volume, bucket);
+    OmMultipartInfo omMultipartInfo =
+        initMultipartUpload(writeClient, volume, bucket, key);
+
+
+    // Commit some MPU parts without eTag field
+    for (int i = 1; i <= 5; i++) {
+      OmKeyArgs partKeyArgs =
+          new OmKeyArgs.Builder()
+              .setVolumeName(volume)
+              .setBucketName(bucket)
+              .setKeyName(key)
+              .setIsMultipartKey(true)
+              .setMultipartUploadID(omMultipartInfo.getUploadID())
+              .setMultipartUploadPartNumber(i)
+              .setAcls(Collections.emptyList())
+              .setReplicationConfig(
+                  RatisReplicationConfig.getInstance(ReplicationFactor.THREE))
+              .build();
+
+      OpenKeySession openKey = writeClient.openKey(partKeyArgs);
+
+      OmKeyArgs commitPartKeyArgs =
+          new OmKeyArgs.Builder()
+              .setVolumeName(volume)
+              .setBucketName(bucket)
+              .setKeyName(key)
+              .setIsMultipartKey(true)
+              .setMultipartUploadID(omMultipartInfo.getUploadID())
+              .setMultipartUploadPartNumber(i)
+              .setAcls(Collections.emptyList())
+              .setReplicationConfig(
+                  RatisReplicationConfig.getInstance(ReplicationFactor.THREE))
+              .setLocationInfoList(Collections.emptyList())
+              .build();
+
+      writeClient.commitMultipartUploadPart(commitPartKeyArgs, openKey.getId());
+    }
+
+
+    OmMultipartUploadListParts omMultipartUploadListParts = keyManager
+        .listParts(volume, bucket, key, omMultipartInfo.getUploadID(),
+            0, 10);
+    assertEquals(5,
+        omMultipartUploadListParts.getPartInfoList().size());
+
+  }
+
   private String volumeName() {
     return getTestName();
   }

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java

@@ -999,6 +999,12 @@ private Response createMultipartKey(OzoneVolume volume, String bucket,
       OmMultipartCommitUploadPartInfo omMultipartCommitUploadPartInfo =
           keyOutputStream.getCommitUploadPartInfo();
       String eTag = omMultipartCommitUploadPartInfo.getETag();
+      // If the OmMultipartCommitUploadPartInfo does not contain eTag,
+      // fall back to MPU part name for compatibility in case the (old) OM
+      // does not return the eTag field
+      if (StringUtils.isEmpty(eTag)) {
+        eTag = omMultipartCommitUploadPartInfo.getPartName();
+      }
 
       if (copyHeader != null) {
         getMetrics().updateCopyObjectSuccessStats(startNanos);
@@ -1069,7 +1075,10 @@ private Response listParts(String bucket, String key, String uploadID,
       ozoneMultipartUploadPartListParts.getPartInfoList().forEach(partInfo -> {
         ListPartsResponse.Part part = new ListPartsResponse.Part();
         part.setPartNumber(partInfo.getPartNumber());
-        part.setETag(partInfo.getETag());
+        // If the ETag field does not exist, use MPU part name for backward
+        // compatibility
+        part.setETag(StringUtils.isNotEmpty(partInfo.getETag()) ?
+            partInfo.getETag() : partInfo.getPartName());
         part.setSize(partInfo.getSize());
         part.setLastModified(Instant.ofEpochMilli(
             partInfo.getModificationTime()));