From c463b7fb6a8a9f05c96f0eebe37b0677aa1feb1a Mon Sep 17 00:00:00 2001 From: ceekay Date: Sun, 21 Jan 2024 23:49:32 -0800 Subject: [PATCH 1/3] HDDS-8649. Remove duplicate helper methods getDBMultipartOpenKey() --- ...TestOzoneClientMultipartUploadWithFSO.java | 33 ++----------------- .../hadoop/ozone/om/OMMetadataManager.java | 13 ++++++++ .../hadoop/ozone/om/KeyManagerImpl.java | 10 +----- .../ozone/om/OmMetadataManagerImpl.java | 16 +++++++++ .../key/OMKeyCreateRequestWithFSO.java | 14 +------- ...MultipartUploadCompleteRequestWithFSO.java | 17 +--------- .../request/util/OMMultipartUploadUtils.java | 22 +------------ 7 files changed, 35 insertions(+), 90 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadWithFSO.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadWithFSO.java index 4867be49066f..2de5e83a4e9f 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadWithFSO.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadWithFSO.java @@ -58,7 +58,6 @@ import org.apache.hadoop.ozone.om.helpers.QuotaUtil; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; -import org.apache.hadoop.ozone.om.request.file.OMFileRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeEach; @@ -568,8 +567,7 @@ public void testAbortUploadSuccessWithParts() throws Exception { bucket.abortMultipartUpload(keyName, uploadID); String multipartOpenKey = - getMultipartOpenKey(uploadID, volumeName, bucketName, keyName, - metadataMgr); + metadataMgr.getMultipartKeyFSO(volumeName, bucketName, keyName, uploadID); OmKeyInfo omKeyInfo = metadataMgr.getOpenKeyTable(bucketLayout).get(multipartOpenKey); OmMultipartKeyInfo omMultipartKeyInfo = @@ -853,8 +851,7 @@ private String verifyUploadedPart(String uploadID, String partName, ozoneManager.getMetadataManager().getBucketTable().get(buckKey); BucketLayout bucketLayout = buckInfo.getBucketLayout(); String multipartOpenKey = - getMultipartOpenKey(uploadID, volumeName, bucketName, keyName, - metadataMgr); + metadataMgr.getMultipartKeyFSO(volumeName, bucketName, keyName, uploadID); String multipartKey = metadataMgr.getMultipartKey(volumeName, bucketName, keyName, uploadID); @@ -881,32 +878,6 @@ private String verifyUploadedPart(String uploadID, String partName, return multipartKey; } - private String getMultipartOpenKey(String multipartUploadID, - String volName, String buckName, String kName, - OMMetadataManager omMetadataManager) throws IOException { - - String fileName = OzoneFSUtils.getFileName(kName); - final long volumeId = omMetadataManager.getVolumeId(volName); - final long bucketId = omMetadataManager.getBucketId(volName, - buckName); - long parentID = getParentID(volName, buckName, kName, - omMetadataManager); - - String multipartKey = omMetadataManager.getMultipartKey(volumeId, bucketId, - parentID, fileName, multipartUploadID); - - return multipartKey; - } - - private long getParentID(String volName, String buckName, - String kName, OMMetadataManager omMetadataManager) throws IOException { - final long volumeId = omMetadataManager.getVolumeId(volName); - final long bucketId = omMetadataManager.getBucketId(volName, - buckName); - return OMFileRequest.getParentID(volumeId, bucketId, - kName, omMetadataManager); - } - private String initiateMultipartUpload(OzoneBucket oBucket, String kName, ReplicationType replicationType, ReplicationFactor replicationFactor) throws IOException { diff --git a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java index 68c5cf758eb5..9651c16175a9 100644 --- a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java +++ b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java @@ -391,6 +391,19 @@ List getExpiredMultipartUploads( String getMultipartKey(String volume, String bucket, String key, String uploadId); + /** + * Returns the DB key name of a multipart upload key in OM metadata store + * for FSO-enabled buckets. + * + * @param volume - volume name + * @param bucket - bucket name + * @param key - key name + * @param uploadId - the upload id for this key + * @return bytes of DB key. + */ + String getMultipartKeyFSO(String volume, String bucket, String key, String + uploadId) throws IOException; + /** * Gets the multipart info table which holds the information about diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index 9cfd4043146a..d8243d731008 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -80,7 +80,6 @@ import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.request.OMClientRequest; import org.apache.hadoop.ozone.om.request.file.OMFileRequest; -import org.apache.hadoop.ozone.om.request.util.OMMultipartUploadUtils; import org.apache.hadoop.ozone.om.service.DirectoryDeletingService; import org.apache.hadoop.ozone.om.service.KeyDeletingService; import org.apache.hadoop.ozone.om.service.MultipartUploadCleanupService; @@ -836,7 +835,7 @@ public OmMultipartUploadListParts listParts(String volumeName, //if there are no parts, use the replicationType from the open key. if (isBucketFSOptimized(volumeName, bucketName)) { multipartKey = - getMultipartOpenKeyFSO(volumeName, bucketName, keyName, + metadataManager.getMultipartKeyFSO(volumeName, bucketName, keyName, uploadID); } OmKeyInfo omKeyInfo = @@ -907,13 +906,6 @@ private String getPartName(PartKeyInfo partKeyInfo, String volName, return partName; } - private String getMultipartOpenKeyFSO(String volumeName, String bucketName, - String keyName, String uploadID) throws IOException { - OMMetadataManager metaMgr = metadataManager; - return OMMultipartUploadUtils.getMultipartOpenKeyFSO( - volumeName, bucketName, keyName, uploadID, metaMgr); - } - /** * Returns list of ACLs for given Ozone object. * diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 1f8c3ba3cd97..f17a27d6bff1 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -86,6 +86,7 @@ import org.apache.hadoop.ozone.om.lock.OzoneManagerLock; import org.apache.hadoop.hdds.utils.TransactionInfo; import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolClientSideTranslatorPB; +import org.apache.hadoop.ozone.om.request.file.OMFileRequest; import org.apache.hadoop.ozone.om.request.util.OMMultipartUploadUtils; import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted; import org.apache.hadoop.ozone.om.snapshot.SnapshotCache; @@ -113,6 +114,7 @@ import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.FILE_NOT_FOUND; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.VOLUME_NOT_FOUND; import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DIR; +import static org.apache.hadoop.ozone.om.request.file.OMFileRequest.getParentId; import static org.apache.hadoop.ozone.om.service.SnapshotDeletingService.isBlockLocationInfoSame; import static org.apache.hadoop.ozone.om.snapshot.SnapshotUtils.checkSnapshotDirExist; @@ -863,6 +865,20 @@ public String getMultipartKey(String volume, String bucket, String key, return OmMultipartUpload.getDbKey(volume, bucket, key, uploadId); } + @Override + public String getMultipartKeyFSO(String volume, String bucket, String key, String uploadId) throws IOException { + final long volumeId = getVolumeId(volume); + final long bucketId = getBucketId(volume, + bucket); + long parentId = + OMFileRequest.getParentID(volumeId, bucketId, key, this); + + String fileName = OzoneFSUtils.getFileName(key); + + return getMultipartKey(volumeId, bucketId, parentId, + fileName, uploadId); + } + /** * Returns the OzoneManagerLock used on Metadata DB. * diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequestWithFSO.java index 65a485305d3f..0dec9fa459f6 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequestWithFSO.java @@ -29,7 +29,6 @@ import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; -import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; import org.apache.hadoop.ozone.om.request.file.OMDirectoryCreateRequestWithFSO; @@ -54,7 +53,6 @@ import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.BUCKET_LOCK; import static org.apache.hadoop.ozone.om.request.file.OMFileRequest.OMDirectoryResult.DIRECTORY_EXISTS; import static org.apache.hadoop.ozone.om.request.file.OMFileRequest.OMDirectoryResult.FILE_EXISTS_IN_GIVENPATH; -import static org.apache.hadoop.ozone.om.request.file.OMFileRequest.getParentId; /** * Handles CreateKey request layout version1. @@ -253,16 +251,6 @@ protected String getDBMultipartOpenKey(String volumeName, String bucketName, String keyName, String uploadID, OMMetadataManager omMetadataManager) throws IOException { - - final long volumeId = omMetadataManager.getVolumeId(volumeName); - final long bucketId = omMetadataManager.getBucketId(volumeName, - bucketName); - long parentId = - getParentId(omMetadataManager, volumeName, bucketName, keyName); - - String fileName = OzoneFSUtils.getFileName(keyName); - - return omMetadataManager.getMultipartKey(volumeId, bucketId, parentId, - fileName, uploadID); + return omMetadataManager.getMultipartKeyFSO(volumeName, bucketName, keyName, uploadID); } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequestWithFSO.java index 35867bb84e85..c224786b108a 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequestWithFSO.java @@ -130,22 +130,7 @@ protected String getDBOzoneKey(OMMetadataManager omMetadataManager, protected String getDBMultipartOpenKey(String volumeName, String bucketName, String keyName, String uploadID, OMMetadataManager omMetadataManager) throws IOException { - - long parentId = - getParentId(omMetadataManager, volumeName, bucketName, keyName); - - String fileName = keyName; - Path filePath = Paths.get(keyName).getFileName(); - if (filePath != null) { - fileName = filePath.toString(); - } - - final long volumeId = omMetadataManager.getVolumeId(volumeName); - final long bucketId = omMetadataManager.getBucketId(volumeName, - bucketName); - - return omMetadataManager.getMultipartKey(volumeId, bucketId, - parentId, fileName, uploadID); + return omMetadataManager.getMultipartKeyFSO(volumeName, bucketName, keyName, uploadID); } @Override diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/util/OMMultipartUploadUtils.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/util/OMMultipartUploadUtils.java index 4d8e466fd34d..201ae90fe72d 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/util/OMMultipartUploadUtils.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/util/OMMultipartUploadUtils.java @@ -23,8 +23,6 @@ import org.apache.hadoop.ozone.om.OMMetadataManager; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; -import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; -import org.apache.hadoop.ozone.om.request.file.OMFileRequest; import java.io.IOException; import java.util.UUID; @@ -93,8 +91,7 @@ public static String getMultipartOpenKey(String volumeName, OMMetadataManager omMetadataManager, BucketLayout bucketLayout) throws IOException { if (bucketLayout == BucketLayout.FILE_SYSTEM_OPTIMIZED) { - return getMultipartOpenKeyFSO(volumeName, bucketName, - keyName, multipartUploadId, omMetadataManager); + return omMetadataManager.getMultipartKeyFSO(volumeName, bucketName, keyName, multipartUploadId); } else { return getMultipartOpenKey(volumeName, bucketName, keyName, multipartUploadId, omMetadataManager); @@ -108,23 +105,6 @@ public static String getMultipartOpenKey(String volumeName, volumeName, bucketName, keyName, multipartUploadId); } - public static String getMultipartOpenKeyFSO(String volumeName, - String bucketName, String keyName, String uploadID, - OMMetadataManager metaMgr) throws IOException { - String fileName = OzoneFSUtils.getFileName(keyName); - - final long volumeId = metaMgr.getVolumeId(volumeName); - final long bucketId = metaMgr.getBucketId(volumeName, bucketName); - long parentID = - OMFileRequest.getParentID(volumeId, bucketId, - keyName, metaMgr); - - String multipartKey = metaMgr.getMultipartKey(volumeId, bucketId, - parentID, fileName, uploadID); - - return multipartKey; - } - /** * Check whether key's isMultipartKey flag is set. From c5ec0edeafcf603de512be240fd0b92270f93f47 Mon Sep 17 00:00:00 2001 From: ceekay Date: Mon, 22 Jan 2024 00:00:52 -0800 Subject: [PATCH 2/3] HDDS-8649. Fix checkstyle --- .../java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index f17a27d6bff1..c3ed18d25484 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -114,7 +114,6 @@ import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.FILE_NOT_FOUND; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.VOLUME_NOT_FOUND; import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DIR; -import static org.apache.hadoop.ozone.om.request.file.OMFileRequest.getParentId; import static org.apache.hadoop.ozone.om.service.SnapshotDeletingService.isBlockLocationInfoSame; import static org.apache.hadoop.ozone.om.snapshot.SnapshotUtils.checkSnapshotDirExist; From bc858943bef48671c827d43f199c291de56aad84 Mon Sep 17 00:00:00 2001 From: ceekay Date: Mon, 22 Jan 2024 23:01:35 -0800 Subject: [PATCH 3/3] HDDS-8649. Address review comments --- .../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 5 +++-- .../ozone/om/request/util/OMMultipartUploadUtils.java | 10 +--------- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index 2c241c5ce696..18cfa8010c0c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -80,6 +80,7 @@ import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.request.OMClientRequest; import org.apache.hadoop.ozone.om.request.file.OMFileRequest; +import org.apache.hadoop.ozone.om.request.util.OMMultipartUploadUtils; import org.apache.hadoop.ozone.om.service.DirectoryDeletingService; import org.apache.hadoop.ozone.om.service.KeyDeletingService; import org.apache.hadoop.ozone.om.service.MultipartUploadCleanupService; @@ -805,8 +806,8 @@ public OmMultipartUploadListParts listParts(String volumeName, //if there are no parts, use the replicationType from the open key. if (isBucketFSOptimized(volumeName, bucketName)) { multipartKey = - metadataManager.getMultipartKeyFSO(volumeName, bucketName, keyName, - uploadID); + OMMultipartUploadUtils.getMultipartOpenKey(volumeName, bucketName, keyName, uploadID, + metadataManager, BucketLayout.FILE_SYSTEM_OPTIMIZED); } OmKeyInfo omKeyInfo = metadataManager.getOpenKeyTable(bucketLayout) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/util/OMMultipartUploadUtils.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/util/OMMultipartUploadUtils.java index 201ae90fe72d..42c651887823 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/util/OMMultipartUploadUtils.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/util/OMMultipartUploadUtils.java @@ -93,18 +93,10 @@ public static String getMultipartOpenKey(String volumeName, if (bucketLayout == BucketLayout.FILE_SYSTEM_OPTIMIZED) { return omMetadataManager.getMultipartKeyFSO(volumeName, bucketName, keyName, multipartUploadId); } else { - return getMultipartOpenKey(volumeName, bucketName, - keyName, multipartUploadId, omMetadataManager); + return omMetadataManager.getMultipartKey(volumeName, bucketName, keyName, multipartUploadId); } } - public static String getMultipartOpenKey(String volumeName, - String bucketName, String keyName, String multipartUploadId, - OMMetadataManager omMetadataManager) { - return omMetadataManager.getMultipartKey( - volumeName, bucketName, keyName, multipartUploadId); - } - /** * Check whether key's isMultipartKey flag is set.