diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmMultipartKeyInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmMultipartKeyInfo.java index 80123fd37d73..c16de1a41c9d 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmMultipartKeyInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmMultipartKeyInfo.java @@ -16,10 +16,10 @@ */ package org.apache.hadoop.ozone.om.helpers; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos - .MultipartKeyInfo; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos - .PartKeyInfo; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.MultipartKeyInfo; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PartKeyInfo; import java.util.HashMap; import java.util.Map; @@ -30,15 +30,24 @@ * upload part information of the key. */ public class OmMultipartKeyInfo { - private String uploadID; + private final String uploadID; + private final long creationTime; + private final ReplicationType replicationType; + private final ReplicationFactor replicationFactor; private TreeMap partKeyInfoList; /** * Construct OmMultipartKeyInfo object which holds multipart upload * information for a key. */ - public OmMultipartKeyInfo(String id, Map list) { + public OmMultipartKeyInfo(String id, long creationTime, + ReplicationType replicationType, + ReplicationFactor replicationFactor, + Map list) { this.uploadID = id; + this.creationTime = creationTime; + this.replicationType = replicationType; + this.replicationFactor = replicationFactor; this.partKeyInfoList = new TreeMap<>(list); } @@ -50,6 +59,10 @@ public String getUploadID() { return uploadID; } + public long getCreationTime() { + return creationTime; + } + public TreeMap getPartKeyInfoMap() { return partKeyInfoList; } @@ -62,18 +75,27 @@ public PartKeyInfo getPartKeyInfo(int partNumber) { return partKeyInfoList.get(partNumber); } + public ReplicationType getReplicationType() { + return replicationType; + } + + public ReplicationFactor getReplicationFactor() { + return replicationFactor; + } /** * Construct OmMultipartInfo from MultipartKeyInfo proto object. * @param multipartKeyInfo * @return OmMultipartKeyInfo */ - public static OmMultipartKeyInfo getFromProto(MultipartKeyInfo - multipartKeyInfo) { + public static OmMultipartKeyInfo getFromProto( + MultipartKeyInfo multipartKeyInfo) { Map list = new HashMap<>(); - multipartKeyInfo.getPartKeyInfoListList().stream().forEach(partKeyInfo - -> list.put(partKeyInfo.getPartNumber(), partKeyInfo)); - return new OmMultipartKeyInfo(multipartKeyInfo.getUploadID(), list); + multipartKeyInfo.getPartKeyInfoListList().forEach(partKeyInfo -> + list.put(partKeyInfo.getPartNumber(), partKeyInfo)); + return new OmMultipartKeyInfo(multipartKeyInfo.getUploadID(), + multipartKeyInfo.getCreationTime(), multipartKeyInfo.getType(), + multipartKeyInfo.getFactor(), list); } /** @@ -82,7 +104,10 @@ public static OmMultipartKeyInfo getFromProto(MultipartKeyInfo */ public MultipartKeyInfo getProto() { MultipartKeyInfo.Builder builder = MultipartKeyInfo.newBuilder() - .setUploadID(uploadID); + .setUploadID(uploadID) + .setCreationTime(creationTime) + .setType(replicationType) + .setFactor(replicationFactor); partKeyInfoList.forEach((key, value) -> builder.addPartKeyInfoList(value)); return builder.build(); } diff --git a/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto b/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto index d82fdf2a8e26..ba53bd0edde7 100644 --- a/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto +++ b/hadoop-ozone/common/src/main/proto/OzoneManagerProtocol.proto @@ -969,7 +969,10 @@ message MultipartInfoInitiateResponse { } message MultipartKeyInfo { - required string uploadID = 4; + required string uploadID = 1; + required uint64 creationTime = 2; + required hadoop.hdds.ReplicationType type = 3; + required hadoop.hdds.ReplicationFactor factor = 4; repeated PartKeyInfo partKeyInfoList = 5; } diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/codec/TestOmMultipartKeyInfoCodec.java b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/codec/TestOmMultipartKeyInfoCodec.java index 7a537c0d5023..852514dd4faa 100644 --- a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/codec/TestOmMultipartKeyInfoCodec.java +++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/codec/TestOmMultipartKeyInfoCodec.java @@ -18,8 +18,10 @@ package org.apache.hadoop.ozone.om.codec; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo; import org.apache.hadoop.test.GenericTestUtils; +import org.apache.hadoop.util.Time; import org.junit.Assert; import org.junit.Test; @@ -34,8 +36,10 @@ public class TestOmMultipartKeyInfoCodec { @Test public void testOmMultipartKeyInfoCodec() { OmMultipartKeyInfoCodec codec = new OmMultipartKeyInfoCodec(); - OmMultipartKeyInfo omMultipartKeyInfo = new OmMultipartKeyInfo(UUID - .randomUUID().toString(), new HashMap<>()); + OmMultipartKeyInfo omMultipartKeyInfo = new OmMultipartKeyInfo( + UUID.randomUUID().toString(), Time.now(), + HddsProtos.ReplicationType.RATIS, HddsProtos.ReplicationFactor.THREE, + new HashMap<>()); byte[] data = new byte[0]; try { data = codec.toPersistedFormat(omMultipartKeyInfo); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index 90bcfb3bc944..c504a753ee19 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -893,17 +893,18 @@ private OmMultipartInfo createMultipartInfo(OmKeyArgs keyArgs, // Not checking if there is an already key for this in the keyTable, as // during final complete multipart upload we take care of this. - + long currentTime = Time.now(); Map partKeyInfoMap = new HashMap<>(); OmMultipartKeyInfo multipartKeyInfo = new OmMultipartKeyInfo( - multipartUploadID, partKeyInfoMap); + multipartUploadID, currentTime, keyArgs.getType(), + keyArgs.getFactor(), partKeyInfoMap); List locations = new ArrayList<>(); OmKeyInfo omKeyInfo = new OmKeyInfo.Builder() .setVolumeName(keyArgs.getVolumeName()) .setBucketName(keyArgs.getBucketName()) .setKeyName(keyArgs.getKeyName()) - .setCreationTime(Time.now()) - .setModificationTime(Time.now()) + .setCreationTime(currentTime) + .setModificationTime(currentTime) .setReplicationType(keyArgs.getType()) .setReplicationFactor(keyArgs.getFactor()) .setOmKeyLocationInfos(Collections.singletonList( @@ -1323,29 +1324,29 @@ public OmMultipartUploadList listMultipartUploads(String volumeName, List collect = multipartUploadKeys.stream() .map(OmMultipartUpload::from) - .map(upload -> { + .peek(upload -> { String dbKey = metadataManager .getOzoneKey(upload.getVolumeName(), upload.getBucketName(), upload.getKeyName()); try { - Table openKeyTable = - metadataManager.getOpenKeyTable(); + Table keyInfoTable = + metadataManager.getMultipartInfoTable(); - OmKeyInfo omKeyInfo = - openKeyTable.get(upload.getDbKey()); + OmMultipartKeyInfo multipartKeyInfo = + keyInfoTable.get(upload.getDbKey()); upload.setCreationTime( - Instant.ofEpochMilli(omKeyInfo.getCreationTime())); - - upload.setReplicationType(omKeyInfo.getType()); - upload.setReplicationFactor(omKeyInfo.getFactor()); + Instant.ofEpochMilli(multipartKeyInfo.getCreationTime())); + upload.setReplicationType( + multipartKeyInfo.getReplicationType()); + upload.setReplicationFactor( + multipartKeyInfo.getReplicationFactor()); } catch (IOException e) { LOG.warn( "Open key entry for multipart upload record can be read {}", dbKey); } - return upload; }) .collect(Collectors.toList()); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java index df0e168e2e0f..0b934a05c176 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java @@ -144,7 +144,8 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, multipartKeyInfo = new OmMultipartKeyInfo( - keyArgs.getMultipartUploadID(), new HashMap<>()); + keyArgs.getMultipartUploadID(), keyArgs.getModificationTime(), + keyArgs.getType(), keyArgs.getFactor(), new HashMap<>()); omKeyInfo = new OmKeyInfo.Builder() .setVolumeName(keyArgs.getVolumeName()) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartResponse.java index 09b028bef47c..e9cb8b4312fa 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3MultipartResponse.java @@ -75,7 +75,9 @@ public S3InitiateMultipartUploadResponse createS3InitiateMPUResponse( String volumeName, String bucketName, String keyName, String multipartUploadID) { OmMultipartKeyInfo multipartKeyInfo = new OmMultipartKeyInfo( - multipartUploadID, new HashMap<>()); + multipartUploadID, Time.now(), + HddsProtos.ReplicationType.RATIS, HddsProtos.ReplicationFactor.ONE, + new HashMap<>()); OmKeyInfo omKeyInfo = new OmKeyInfo.Builder() .setVolumeName(volumeName)