From 1bb689ea3666c283cc0f42c4a029e7565d121675 Mon Sep 17 00:00:00 2001 From: Sadanand Shenoy Date: Wed, 10 Jul 2024 16:41:24 +0530 Subject: [PATCH 1/4] HDDS-11133. Update the OLDER_CLIENT_REQUEST validations to check against bucket types. --- .../src/main/java/org/apache/hadoop/ozone/ClientVersion.java | 4 ++++ .../ozone/om/request/validation/ValidationCondition.java | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java index f3bd1a96b662..3e0ffcff7d0b 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java @@ -74,6 +74,10 @@ public int toProtoValue() { return version; } + public int getVersion() { + return version; + } + public static ClientVersion fromProtoValue(int value) { return BY_PROTO_VALUE.getOrDefault(value, FUTURE_VERSION); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidationCondition.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidationCondition.java index 9630500cbd53..e2bd4579994d 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidationCondition.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidationCondition.java @@ -47,7 +47,7 @@ public boolean shouldApply(OMRequest req, ValidationContext ctx) { OLDER_CLIENT_REQUESTS { @Override public boolean shouldApply(OMRequest req, ValidationContext ctx) { - return req.getVersion() < ClientVersion.CURRENT_VERSION; + return req.getVersion() < ClientVersion.BUCKET_LAYOUT_SUPPORT.getVersion(); } }; From 100fe0ae751e36cc97e89c40c581f76cb40f1736 Mon Sep 17 00:00:00 2001 From: Sadanand Shenoy Date: Wed, 10 Jul 2024 16:57:50 +0530 Subject: [PATCH 2/4] rename condition --- .../request/bucket/OMBucketCreateRequest.java | 2 +- .../request/bucket/OMBucketDeleteRequest.java | 2 +- .../file/OMDirectoryCreateRequest.java | 2 +- .../om/request/file/OMFileCreateRequest.java | 2 +- .../request/key/OMAllocateBlockRequest.java | 2 +- .../om/request/key/OMKeyCommitRequest.java | 2 +- .../om/request/key/OMKeyCreateRequest.java | 2 +- .../om/request/key/OMKeyDeleteRequest.java | 2 +- .../om/request/key/OMKeyRenameRequest.java | 2 +- .../om/request/key/OMKeysDeleteRequest.java | 2 +- .../om/request/key/OMKeysRenameRequest.java | 2 +- .../request/key/acl/OMKeyAddAclRequest.java | 2 +- .../key/acl/OMKeyRemoveAclRequest.java | 2 +- .../request/key/acl/OMKeySetAclRequest.java | 2 +- .../S3InitiateMultipartUploadRequest.java | 2 +- .../S3MultipartUploadAbortRequest.java | 2 +- .../S3MultipartUploadCommitPartRequest.java | 2 +- .../S3MultipartUploadCompleteRequest.java | 2 +- .../validation/ValidationCondition.java | 5 ++-- .../OzoneManagerRequestHandler.java | 24 +++++++++---------- .../TestRequestFeatureValidatorProcessor.java | 2 +- .../validation/TestValidatorRegistry.java | 17 ++++++------- .../GeneralValidatorsForTesting.java | 17 ++++++------- ...ValidatorsForOnlyOldClientValidations.java | 4 ++-- 24 files changed, 54 insertions(+), 51 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java index f2c343e0d161..e44c5fc3b573 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java @@ -460,7 +460,7 @@ public static OMRequest handleCreateBucketWithBucketLayoutDuringPreFinalize( * they do not understand. */ @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + conditions = ValidationCondition.OLDER_BUCKET_LAYOUT_CLIENT_REQUESTS, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CreateBucket ) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketDeleteRequest.java index 48a3c807b9a3..7a60b00ce186 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketDeleteRequest.java @@ -281,7 +281,7 @@ private boolean bucketContainsSnapshotInCache( * @throws OMException if the request is invalid */ @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + conditions = ValidationCondition.OLDER_BUCKET_LAYOUT_CLIENT_REQUESTS, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.DeleteBucket ) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java index 6071110a315d..890dbb941d8b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java @@ -432,7 +432,7 @@ public static OMRequest disallowCreateDirectoryWithECReplicationConfig( * @throws OMException if the request is invalid */ @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + conditions = ValidationCondition.OLDER_BUCKET_LAYOUT_CLIENT_REQUESTS, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CreateDirectory ) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java index 9b9fb4e7cc5c..98c76b61bb17 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java @@ -425,7 +425,7 @@ public static OMRequest disallowCreateFileWithECReplicationConfig( * @throws OMException if the request is invalid */ @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + conditions = ValidationCondition.OLDER_BUCKET_LAYOUT_CLIENT_REQUESTS, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CreateFile ) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java index ac4d9ab6244f..1ac43c8f58b4 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java @@ -298,7 +298,7 @@ public static OMRequest disallowAllocateBlockWithECReplicationConfig( * @throws OMException if the request is invalid */ @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + conditions = ValidationCondition.OLDER_BUCKET_LAYOUT_CLIENT_REQUESTS, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.AllocateBlock ) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java index 5c3593b07839..9383c09ebb3b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java @@ -457,7 +457,7 @@ public static OMRequest disallowCommitKeyWithECReplicationConfig( * @throws OMException if the request is invalid */ @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + conditions = ValidationCondition.OLDER_BUCKET_LAYOUT_CLIENT_REQUESTS, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CommitKey ) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java index e9a9f007197a..6a8309d984d2 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java @@ -423,7 +423,7 @@ public static OMRequest disallowCreateKeyWithECReplicationConfig( * @throws OMException if the request is invalid */ @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + conditions = ValidationCondition.OLDER_BUCKET_LAYOUT_CLIENT_REQUESTS, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CreateKey ) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java index 61e5976f8052..88fa497890e6 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java @@ -233,7 +233,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn * @throws OMException if the request is invalid */ @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + conditions = ValidationCondition.OLDER_BUCKET_LAYOUT_CLIENT_REQUESTS, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.DeleteKey ) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java index db08951831cb..89620b75d4a5 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java @@ -270,7 +270,7 @@ private Map buildAuditMap( * @throws OMException if the request is invalid */ @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + conditions = ValidationCondition.OLDER_BUCKET_LAYOUT_CLIENT_REQUESTS, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.RenameKey ) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java index 19e9ed716e4a..c750597fd6f3 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java @@ -351,7 +351,7 @@ protected static void addDeletedKeys(Map auditMap, * @throws OMException if the request is invalid */ @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + conditions = ValidationCondition.OLDER_BUCKET_LAYOUT_CLIENT_REQUESTS, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.DeleteKeys ) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysRenameRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysRenameRequest.java index 0b93dd57946e..3ef5ef8d36a5 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysRenameRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysRenameRequest.java @@ -297,7 +297,7 @@ private Map buildAuditMap(Map auditMap, * @throws OMException if the request is invalid */ @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + conditions = ValidationCondition.OLDER_BUCKET_LAYOUT_CLIENT_REQUESTS, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.RenameKeys ) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAddAclRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAddAclRequest.java index 31fa245fa55b..8acc8d76d0b1 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAddAclRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAddAclRequest.java @@ -165,7 +165,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn * @throws OMException if the request is invalid */ @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + conditions = ValidationCondition.OLDER_BUCKET_LAYOUT_CLIENT_REQUESTS, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.AddAcl ) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyRemoveAclRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyRemoveAclRequest.java index 7e4fc04eb39c..a3de88d2548c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyRemoveAclRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyRemoveAclRequest.java @@ -166,7 +166,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn * @throws OMException if the request is invalid */ @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + conditions = ValidationCondition.OLDER_BUCKET_LAYOUT_CLIENT_REQUESTS, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.RemoveAcl ) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeySetAclRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeySetAclRequest.java index 0e5e1cf3362a..2a36ad9c1876 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeySetAclRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeySetAclRequest.java @@ -162,7 +162,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn * @throws OMException if the request is invalid */ @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + conditions = ValidationCondition.OLDER_BUCKET_LAYOUT_CLIENT_REQUESTS, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.SetAcl ) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java index f16ef9f8f423..dd34622873e6 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java @@ -320,7 +320,7 @@ protected void logResult(OzoneManager ozoneManager, * @throws OMException if the request is invalid */ @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + conditions = ValidationCondition.OLDER_BUCKET_LAYOUT_CLIENT_REQUESTS, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.InitiateMultiPartUpload ) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java index 400291d7a448..e2a87fe6bb89 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java @@ -289,7 +289,7 @@ public static OMRequest disallowAbortMultiPartUploadWithECReplicationConfig( * @throws OMException if the request is invalid */ @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + conditions = ValidationCondition.OLDER_BUCKET_LAYOUT_CLIENT_REQUESTS, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.AbortMultiPartUpload ) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java index 1123b8dba7a6..3d0838c5a7dc 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java @@ -380,7 +380,7 @@ public static OMRequest disallowCommitMultiPartUploadWithECReplicationConfig( * @throws OMException if the request is invalid */ @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + conditions = ValidationCondition.OLDER_BUCKET_LAYOUT_CLIENT_REQUESTS, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CommitMultiPartUpload ) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java index 1dab110e783a..957f6cfc8921 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java @@ -742,7 +742,7 @@ private void updateCache(OMMetadataManager omMetadataManager, * @throws OMException if the request is invalid */ @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + conditions = ValidationCondition.OLDER_BUCKET_LAYOUT_CLIENT_REQUESTS, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CompleteMultiPartUpload ) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidationCondition.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidationCondition.java index e2bd4579994d..14a520064334 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidationCondition.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidationCondition.java @@ -42,9 +42,10 @@ public boolean shouldApply(OMRequest req, ValidationContext ctx) { /** * Classifies validations that has to run, when the client uses an older - * protocol version than the server. + * protocol version than the bucket layout version i.e the version where + * bucket types where introduced. */ - OLDER_CLIENT_REQUESTS { + OLDER_BUCKET_LAYOUT_CLIENT_REQUESTS { @Override public boolean shouldApply(OMRequest req, ValidationContext ctx) { return req.getVersion() < ClientVersion.BUCKET_LAYOUT_SUPPORT.getVersion(); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java index 5b8365e5a75b..c5ab0ba6fb8b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java @@ -629,7 +629,7 @@ private GetKeyInfoResponse getKeyInfo(GetKeyInfoRequest request, } @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + conditions = ValidationCondition.OLDER_BUCKET_LAYOUT_CLIENT_REQUESTS, processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.LookupKey ) @@ -656,7 +656,7 @@ public static OMResponse disallowLookupKeyResponseWithECReplicationConfig( } @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + conditions = ValidationCondition.OLDER_BUCKET_LAYOUT_CLIENT_REQUESTS, processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.LookupKey ) @@ -743,7 +743,7 @@ private ListKeysLightResponse listKeysLight(ListKeysRequest request) } @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + conditions = ValidationCondition.OLDER_BUCKET_LAYOUT_CLIENT_REQUESTS, processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.ListKeys ) @@ -770,7 +770,7 @@ public static OMResponse disallowListKeysResponseWithECReplicationConfig( } @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + conditions = ValidationCondition.OLDER_BUCKET_LAYOUT_CLIENT_REQUESTS, processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.ListKeys ) @@ -831,7 +831,7 @@ private ListTrashResponse listTrash(ListTrashRequest request, } @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + conditions = ValidationCondition.OLDER_BUCKET_LAYOUT_CLIENT_REQUESTS, processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.ListTrash ) @@ -861,7 +861,7 @@ public static OMResponse disallowListTrashWithECReplicationConfig( } @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + conditions = ValidationCondition.OLDER_BUCKET_LAYOUT_CLIENT_REQUESTS, processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.ListTrash ) @@ -1048,7 +1048,7 @@ private RefetchSecretKeyResponse refetchSecretKey() { } @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + conditions = ValidationCondition.OLDER_BUCKET_LAYOUT_CLIENT_REQUESTS, processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.GetFileStatus ) @@ -1078,7 +1078,7 @@ public static OMResponse disallowGetFileStatusWithECReplicationConfig( @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + conditions = ValidationCondition.OLDER_BUCKET_LAYOUT_CLIENT_REQUESTS, processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.GetFileStatus ) @@ -1127,7 +1127,7 @@ private LookupFileResponse lookupFile(LookupFileRequest request, } @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + conditions = ValidationCondition.OLDER_BUCKET_LAYOUT_CLIENT_REQUESTS, processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.LookupFile ) @@ -1155,7 +1155,7 @@ public static OMResponse disallowLookupFileWithECReplicationConfig( } @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + conditions = ValidationCondition.OLDER_BUCKET_LAYOUT_CLIENT_REQUESTS, processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.LookupFile ) @@ -1237,7 +1237,7 @@ private ListStatusLightResponse listStatusLight( } @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + conditions = ValidationCondition.OLDER_BUCKET_LAYOUT_CLIENT_REQUESTS, processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.ListStatus ) @@ -1265,7 +1265,7 @@ public static OMResponse disallowListStatusResponseWithECReplicationConfig( } @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + conditions = ValidationCondition.OLDER_BUCKET_LAYOUT_CLIENT_REQUESTS, processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.ListStatus ) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestRequestFeatureValidatorProcessor.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestRequestFeatureValidatorProcessor.java index 48947518abd4..68d284d39620 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestRequestFeatureValidatorProcessor.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestRequestFeatureValidatorProcessor.java @@ -365,7 +365,7 @@ private Compilation compile(List source) { private ValidationCondition[] someConditions() { return - new ValidationCondition[] {ValidationCondition.OLDER_CLIENT_REQUESTS}; + new ValidationCondition[] {ValidationCondition.OLDER_BUCKET_LAYOUT_CLIENT_REQUESTS }; } private ValidationCondition[] emptyConditions() { diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestValidatorRegistry.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestValidatorRegistry.java index 076ba0187272..74d54c7aa225 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestValidatorRegistry.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestValidatorRegistry.java @@ -34,7 +34,7 @@ import static org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase.POST_PROCESS; import static org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase.PRE_PROCESS; import static org.apache.hadoop.ozone.om.request.validation.ValidationCondition.CLUSTER_NEEDS_FINALIZATION; -import static org.apache.hadoop.ozone.om.request.validation.ValidationCondition.OLDER_CLIENT_REQUESTS; +import static org.apache.hadoop.ozone.om.request.validation.ValidationCondition.OLDER_BUCKET_LAYOUT_CLIENT_REQUESTS; import static org.apache.hadoop.ozone.om.request.validation.testvalidatorset1.GeneralValidatorsForTesting.startValidatorTest; import static org.apache.hadoop.ozone.om.request.validation.testvalidatorset1.GeneralValidatorsForTesting.finishValidatorTest; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.CreateDirectory; @@ -107,7 +107,7 @@ public void testRegistryHasTheOldClientPreProcessCreateKeyValidator() { ValidatorRegistry registry = new ValidatorRegistry(PACKAGE); List validators = registry.validationsFor( - asList(OLDER_CLIENT_REQUESTS), CreateKey, PRE_PROCESS); + asList(OLDER_BUCKET_LAYOUT_CLIENT_REQUESTS), CreateKey, PRE_PROCESS); assertEquals(2, validators.size()); List methodNames = @@ -121,7 +121,7 @@ public void testRegistryHasTheOldClientPostProcessCreateKeyValidator() { ValidatorRegistry registry = new ValidatorRegistry(PACKAGE); List validators = registry.validationsFor( - asList(OLDER_CLIENT_REQUESTS), CreateKey, POST_PROCESS); + asList(OLDER_BUCKET_LAYOUT_CLIENT_REQUESTS), CreateKey, POST_PROCESS); assertEquals(2, validators.size()); List methodNames = @@ -138,7 +138,7 @@ public void testRegistryHasTheMultiPurposePreProcessCreateVolumeValidator() { asList(CLUSTER_NEEDS_FINALIZATION), CreateVolume, PRE_PROCESS); List newClientValidators = registry.validationsFor( - asList(OLDER_CLIENT_REQUESTS), CreateVolume, PRE_PROCESS); + asList(OLDER_BUCKET_LAYOUT_CLIENT_REQUESTS), CreateVolume, PRE_PROCESS); assertEquals(1, preFinalizeValidators.size()); assertEquals(1, newClientValidators.size()); @@ -155,7 +155,7 @@ public void testRegistryHasTheMultiPurposePostProcessCreateVolumeValidator() { asList(CLUSTER_NEEDS_FINALIZATION), CreateVolume, POST_PROCESS); List oldClientValidators = registry.validationsFor( - asList(OLDER_CLIENT_REQUESTS), CreateVolume, POST_PROCESS); + asList(OLDER_BUCKET_LAYOUT_CLIENT_REQUESTS), CreateVolume, POST_PROCESS); assertEquals(1, preFinalizeValidators.size()); assertEquals(1, oldClientValidators.size()); @@ -169,7 +169,8 @@ public void testValidatorsAreReturnedForMultiCondition() { ValidatorRegistry registry = new ValidatorRegistry(PACKAGE); List validators = registry.validationsFor( - asList(CLUSTER_NEEDS_FINALIZATION, OLDER_CLIENT_REQUESTS), + asList(CLUSTER_NEEDS_FINALIZATION, + OLDER_BUCKET_LAYOUT_CLIENT_REQUESTS), CreateKey, POST_PROCESS); assertEquals(3, validators.size()); @@ -185,7 +186,7 @@ public void testNoValidatorForRequestsAtAllReturnsEmptyList() { ValidatorRegistry registry = new ValidatorRegistry(PACKAGE_WO_VALIDATORS); assertTrue(registry.validationsFor( - asList(OLDER_CLIENT_REQUESTS), CreateKey, PRE_PROCESS).isEmpty()); + asList(OLDER_BUCKET_LAYOUT_CLIENT_REQUESTS), CreateKey, PRE_PROCESS).isEmpty()); } @Test @@ -207,7 +208,7 @@ public void testNoDefinedValidationForRequestReturnsEmptyList() { ValidatorRegistry registry = new ValidatorRegistry(PACKAGE); assertTrue(registry.validationsFor( - asList(OLDER_CLIENT_REQUESTS), CreateDirectory, null).isEmpty()); + asList(OLDER_BUCKET_LAYOUT_CLIENT_REQUESTS), CreateDirectory, null).isEmpty()); } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset1/GeneralValidatorsForTesting.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset1/GeneralValidatorsForTesting.java index 35c3afa4cf96..145670e13322 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset1/GeneralValidatorsForTesting.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset1/GeneralValidatorsForTesting.java @@ -29,7 +29,7 @@ import static org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase.POST_PROCESS; import static org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase.PRE_PROCESS; import static org.apache.hadoop.ozone.om.request.validation.ValidationCondition.CLUSTER_NEEDS_FINALIZATION; -import static org.apache.hadoop.ozone.om.request.validation.ValidationCondition.OLDER_CLIENT_REQUESTS; +import static org.apache.hadoop.ozone.om.request.validation.ValidationCondition.OLDER_BUCKET_LAYOUT_CLIENT_REQUESTS; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.CreateKey; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.CreateVolume; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.DeleteKeys; @@ -109,7 +109,7 @@ public static OMResponse preFinalizePostProcessCreateKeyValidator( } @RequestFeatureValidator( - conditions = { OLDER_CLIENT_REQUESTS }, + conditions = { OLDER_BUCKET_LAYOUT_CLIENT_REQUESTS }, processingPhase = PRE_PROCESS, requestType = CreateKey) public static OMRequest oldClientPreProcessCreateKeyValidator( @@ -119,7 +119,7 @@ public static OMRequest oldClientPreProcessCreateKeyValidator( } @RequestFeatureValidator( - conditions = { OLDER_CLIENT_REQUESTS }, + conditions = { OLDER_BUCKET_LAYOUT_CLIENT_REQUESTS }, processingPhase = POST_PROCESS, requestType = CreateKey) public static OMResponse oldClientPostProcessCreateKeyValidator( @@ -129,7 +129,8 @@ public static OMResponse oldClientPostProcessCreateKeyValidator( } @RequestFeatureValidator( - conditions = { CLUSTER_NEEDS_FINALIZATION, OLDER_CLIENT_REQUESTS }, + conditions = { CLUSTER_NEEDS_FINALIZATION, + OLDER_BUCKET_LAYOUT_CLIENT_REQUESTS }, processingPhase = PRE_PROCESS, requestType = CreateVolume) public static OMRequest multiPurposePreProcessCreateVolumeValidator( @@ -139,7 +140,7 @@ public static OMRequest multiPurposePreProcessCreateVolumeValidator( } @RequestFeatureValidator( - conditions = { OLDER_CLIENT_REQUESTS, CLUSTER_NEEDS_FINALIZATION }, + conditions = { OLDER_BUCKET_LAYOUT_CLIENT_REQUESTS, CLUSTER_NEEDS_FINALIZATION }, processingPhase = POST_PROCESS, requestType = CreateVolume) public static OMResponse multiPurposePostProcessCreateVolumeValidator( @@ -149,7 +150,7 @@ public static OMResponse multiPurposePostProcessCreateVolumeValidator( } @RequestFeatureValidator( - conditions = { OLDER_CLIENT_REQUESTS }, + conditions = { OLDER_BUCKET_LAYOUT_CLIENT_REQUESTS }, processingPhase = POST_PROCESS, requestType = CreateKey) public static OMResponse oldClientPostProcessCreateKeyValidator2( @@ -159,7 +160,7 @@ public static OMResponse oldClientPostProcessCreateKeyValidator2( } @RequestFeatureValidator( - conditions = {OLDER_CLIENT_REQUESTS}, + conditions = { OLDER_BUCKET_LAYOUT_CLIENT_REQUESTS }, processingPhase = PRE_PROCESS, requestType = DeleteKeys ) @@ -173,7 +174,7 @@ public static OMRequest throwingPreProcessValidator( } @RequestFeatureValidator( - conditions = {OLDER_CLIENT_REQUESTS}, + conditions = { OLDER_BUCKET_LAYOUT_CLIENT_REQUESTS }, processingPhase = POST_PROCESS, requestType = DeleteKeys ) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset2/ValidatorsForOnlyOldClientValidations.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset2/ValidatorsForOnlyOldClientValidations.java index 447b9ab0260b..7736862bae43 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset2/ValidatorsForOnlyOldClientValidations.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset2/ValidatorsForOnlyOldClientValidations.java @@ -21,7 +21,7 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import static org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase.PRE_PROCESS; -import static org.apache.hadoop.ozone.om.request.validation.ValidationCondition.OLDER_CLIENT_REQUESTS; +import static org.apache.hadoop.ozone.om.request.validation.ValidationCondition.OLDER_BUCKET_LAYOUT_CLIENT_REQUESTS; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.CreateKey; /** @@ -33,7 +33,7 @@ public final class ValidatorsForOnlyOldClientValidations { private ValidatorsForOnlyOldClientValidations() { } @RequestFeatureValidator( - conditions = { OLDER_CLIENT_REQUESTS }, + conditions = { OLDER_BUCKET_LAYOUT_CLIENT_REQUESTS }, processingPhase = PRE_PROCESS, requestType = CreateKey) public static OMRequest oldClientPreProcessCreateKeyValidator2( From c011772eb128b1822b52edc0cdba13e404154c8c Mon Sep 17 00:00:00 2001 From: Sadanand Shenoy Date: Wed, 10 Jul 2024 19:36:22 +0530 Subject: [PATCH 3/4] fix CI failure and add unit test --- ...ManagerProtocolClientSideTranslatorPB.java | 11 ++- .../ozone/om/TestObjectStoreWithFSO.java | 86 +++++++++++++++++++ .../validation/TestRequestValidations.java | 3 +- 3 files changed, 98 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java index 3e21494ac61a..c95155b1d8c8 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java @@ -301,13 +301,22 @@ private OMRequest.Builder createOMRequest(Type cmdType) { .setClientId(clientID); } + @VisibleForTesting + public OMRequest.Builder createOMRequest(Type cmdType,int clientVersion) { + return OMRequest.newBuilder() + .setCmdType(cmdType) + .setVersion(clientVersion) + .setClientId(clientID); + } + /** * Submits client request to OM server. * @param omRequest client request * @return response from OM * @throws IOException thrown if any Protobuf service exception occurs */ - private OMResponse submitRequest(OMRequest omRequest) + @VisibleForTesting + public OMResponse submitRequest(OMRequest omRequest) throws IOException { OMRequest.Builder builder = OMRequest.newBuilder(omRequest); // Insert S3 Authentication information for each request. diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithFSO.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithFSO.java index 2e58b6dbb731..5a7d7e693127 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithFSO.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithFSO.java @@ -38,14 +38,20 @@ import org.apache.hadoop.ozone.client.OzoneKey; import org.apache.hadoop.ozone.client.OzoneKeyDetails; import org.apache.hadoop.ozone.client.OzoneVolume; +import org.apache.hadoop.ozone.ClientVersion; import org.apache.hadoop.ozone.client.BucketArgs; import org.apache.hadoop.ozone.client.io.KeyOutputStream; import org.apache.hadoop.ozone.client.io.OzoneInputStream; import org.apache.hadoop.ozone.client.io.OzoneOutputStream; +import org.apache.hadoop.ozone.client.rpc.RpcClient; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; +import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; +import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolClientSideTranslatorPB; import org.apache.hadoop.util.StringUtils; import org.apache.ozone.test.GenericTestUtils; import org.junit.jupiter.api.AfterEach; @@ -53,6 +59,9 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.Arguments; import java.io.IOException; import java.net.URI; @@ -64,6 +73,7 @@ import java.util.List; import java.util.UUID; import java.util.concurrent.TimeoutException; +import java.util.stream.Stream; import static org.apache.hadoop.hdds.client.ReplicationFactor.ONE; import static org.apache.hadoop.hdds.client.ReplicationType.RATIS; @@ -72,6 +82,7 @@ import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_SCHEME; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_ALREADY_EXISTS; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND; +import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.OK; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; @@ -94,6 +105,8 @@ public class TestObjectStoreWithFSO { private static FileSystem fs; private static OzoneClient client; + private static OzoneManagerProtocolClientSideTranslatorPB ozoneManagerClient; + /** * Create a MiniDFSCluster for testing. *

@@ -108,6 +121,9 @@ public static void init() throws Exception { cluster = MiniOzoneCluster.newBuilder(conf).build(); cluster.waitForClusterToBeReady(); client = cluster.newClient(); + ozoneManagerClient = + (OzoneManagerProtocolClientSideTranslatorPB) + ((RpcClient) client.getObjectStore().getClientProxy()).getOzoneManagerClient(); // create a volume and a bucket to be used by OzoneFileSystem OzoneBucket bucket = TestDataUtil .createVolumeAndBucket(client, BucketLayout.FILE_SYSTEM_OPTIMIZED); @@ -210,6 +226,76 @@ public void testCreateKey() throws Exception { dirPathC.getObjectID(), true); } + public OzoneFileStatus getFileStatus(OmKeyArgs args, int clientVersion) + throws IOException { + OzoneManagerProtocolProtos.KeyArgs keyArgs = + OzoneManagerProtocolProtos.KeyArgs.newBuilder() + .setVolumeName(args.getVolumeName()) + .setBucketName(args.getBucketName()).setKeyName(args.getKeyName()) + .build(); + OzoneManagerProtocolProtos.GetFileStatusRequest req = + OzoneManagerProtocolProtos.GetFileStatusRequest.newBuilder() + .setKeyArgs(keyArgs).build(); + OzoneManagerProtocolProtos.OMRequest omRequest = + ozoneManagerClient.createOMRequest( + OzoneManagerProtocolProtos.Type.GetFileStatus, clientVersion) + .setGetFileStatusRequest(req).build(); + OzoneManagerProtocolProtos.GetFileStatusResponse resp; + try { + OzoneManagerProtocolProtos.OMResponse omResponse = + ozoneManagerClient.submitRequest(omRequest); + if (omResponse.getStatus() != OK) { + throw new OMException(omResponse.getMessage(), + OMException.ResultCodes.values()[omResponse.getStatus().ordinal()]); + } + resp = omResponse.getGetFileStatusResponse(); + } catch (IOException e) { + throw e; + } + OzoneFileStatus getFileStatusResp = + OzoneFileStatus.getFromProtobuf(resp.getStatus()); + return getFileStatusResp; + } + + + private static Stream allClientVersions() { + return Arrays.stream(ClientVersion.values()) + .map(clientVersion -> Arguments.of(clientVersion.getVersion())); + } + @ParameterizedTest + @MethodSource("allClientVersions") + public void testGetFileStatusFromOlderClients(int clientVersion) throws Exception { + // try to create key from new client + String parent = "a/b/c/"; + String file = "key1" + RandomStringUtils.randomNumeric(5); + String key = parent + file; + + ObjectStore objectStore = client.getObjectStore(); + OzoneVolume ozoneVolume = objectStore.getVolume(volumeName); + assertEquals(volumeName, ozoneVolume.getName()); + OzoneBucket ozoneBucket = ozoneVolume.getBucket(bucketName); + assertEquals(bucketName, ozoneBucket.getName()); + String data = "random data"; + try(OzoneOutputStream ozoneOutputStream = ozoneBucket.createKey(key, + data.length(), ReplicationType.RATIS, ReplicationFactor.ONE, + new HashMap<>())){ + ozoneOutputStream.write(data.getBytes(StandardCharsets.UTF_8)); + } + // try to read from older client + OmKeyArgs keyArgs = new OmKeyArgs.Builder() + .setVolumeName(ozoneBucket.getVolumeName()) + .setBucketName(ozoneBucket.getName()) + .setKeyName(key) + .build(); + if (clientVersion < ClientVersion.BUCKET_LAYOUT_SUPPORT.getVersion()) { + assertThrows(OMException.class, + () -> getFileStatus(keyArgs, clientVersion), + "Expecting Unsupported Operation Exception"); + } else { + assertNotNull(getFileStatus(keyArgs,clientVersion)); + } + } + /** * Tests bucket deletion behaviour. Buckets should not be allowed to be * deleted if they contain files or directories under them. diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestRequestValidations.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestRequestValidations.java index fdb332f66042..9057b23b077f 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestRequestValidations.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestRequestValidations.java @@ -284,7 +284,8 @@ private RequestValidations loadEmptyValidations(ValidationContext ctx) { } private int olderClientVersion() { - return ClientVersion.CURRENT_VERSION - 1; + // older than bucket type version + return ClientVersion.BUCKET_LAYOUT_SUPPORT.getVersion() - 1; } private int currentClientVersion() { From 5d1f73ae260137b96c5ecfff0481fbe6d7682e49 Mon Sep 17 00:00:00 2001 From: Sadanand Shenoy Date: Wed, 10 Jul 2024 19:40:09 +0530 Subject: [PATCH 4/4] fix checkstyle --- .../OzoneManagerProtocolClientSideTranslatorPB.java | 6 ++---- .../org/apache/hadoop/ozone/om/TestObjectStoreWithFSO.java | 6 +++--- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java index c95155b1d8c8..52ccc2bee529 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java @@ -302,10 +302,8 @@ private OMRequest.Builder createOMRequest(Type cmdType) { } @VisibleForTesting - public OMRequest.Builder createOMRequest(Type cmdType,int clientVersion) { - return OMRequest.newBuilder() - .setCmdType(cmdType) - .setVersion(clientVersion) + public OMRequest.Builder createOMRequest(Type cmdType, int clientVersion) { + return OMRequest.newBuilder().setCmdType(cmdType).setVersion(clientVersion) .setClientId(clientID); } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithFSO.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithFSO.java index 5a7d7e693127..5ca19c22db62 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithFSO.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithFSO.java @@ -276,9 +276,9 @@ public void testGetFileStatusFromOlderClients(int clientVersion) throws Exceptio OzoneBucket ozoneBucket = ozoneVolume.getBucket(bucketName); assertEquals(bucketName, ozoneBucket.getName()); String data = "random data"; - try(OzoneOutputStream ozoneOutputStream = ozoneBucket.createKey(key, + try (OzoneOutputStream ozoneOutputStream = ozoneBucket.createKey(key, data.length(), ReplicationType.RATIS, ReplicationFactor.ONE, - new HashMap<>())){ + new HashMap<>())) { ozoneOutputStream.write(data.getBytes(StandardCharsets.UTF_8)); } // try to read from older client @@ -292,7 +292,7 @@ public void testGetFileStatusFromOlderClients(int clientVersion) throws Exceptio () -> getFileStatus(keyArgs, clientVersion), "Expecting Unsupported Operation Exception"); } else { - assertNotNull(getFileStatus(keyArgs,clientVersion)); + assertNotNull(getFileStatus(keyArgs, clientVersion)); } }