diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/Versioned.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/Versioned.java index 136edeaf2a0a..bffb418403a3 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/Versioned.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/Versioned.java @@ -17,9 +17,17 @@ package org.apache.hadoop.ozone; +import java.util.Comparator; + /** * Base class defining the version in the entire system. */ public interface Versioned { + Comparator VERSIONED_COMPARATOR = Comparator.comparingInt(Versioned::version); + int version(); + + static Comparator versionComparator() { + return VERSIONED_COMPARATOR; + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java index d66b5d370814..511ae1263f07 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java @@ -52,8 +52,8 @@ import org.apache.hadoop.ozone.om.helpers.OzoneAclUtil; import org.apache.hadoop.ozone.om.request.OMClientRequest; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; -import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; -import org.apache.hadoop.ozone.om.request.validation.ValidationCondition; +import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; +import org.apache.hadoop.ozone.om.request.validation.OMLayoutVersionValidator; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.om.response.bucket.OMBucketCreateResponse; @@ -405,8 +405,8 @@ public boolean checkQuotaBytesValid(OMMetadataManager metadataManager, } - @RequestFeatureValidator( - conditions = ValidationCondition.CLUSTER_NEEDS_FINALIZATION, + @OMLayoutVersionValidator( + applyBefore = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CreateBucket ) @@ -428,8 +428,8 @@ public static OMRequest disallowCreateBucketWithECReplicationConfig( return req; } - @RequestFeatureValidator( - conditions = ValidationCondition.CLUSTER_NEEDS_FINALIZATION, + @OMLayoutVersionValidator( + applyBefore = OMLayoutFeature.BUCKET_LAYOUT_SUPPORT, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CreateBucket ) @@ -469,8 +469,8 @@ public static OMRequest handleCreateBucketWithBucketLayoutDuringPreFinalize( * write to them, instead of using the server default which may be in a layout * they do not understand. */ - @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + @OMClientVersionValidator( + applyBefore = ClientVersion.BUCKET_LAYOUT_SUPPORT, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CreateBucket ) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketDeleteRequest.java index 3c5f028bb5d9..5d0b0083d478 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketDeleteRequest.java @@ -30,6 +30,7 @@ import org.apache.hadoop.hdds.utils.db.TableIterator; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; +import org.apache.hadoop.ozone.ClientVersion; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.audit.AuditLogger; import org.apache.hadoop.ozone.audit.OMAction; @@ -45,8 +46,7 @@ import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.request.OMClientRequest; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; -import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; -import org.apache.hadoop.ozone.om.request.validation.ValidationCondition; +import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.om.response.bucket.OMBucketDeleteResponse; @@ -275,8 +275,8 @@ private boolean bucketContainsSnapshotInCache( * @return the validated request * @throws OMException if the request is invalid */ - @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + @OMClientVersionValidator( + applyBefore = ClientVersion.BUCKET_LAYOUT_SUPPORT, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.DeleteBucket ) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketSetPropertyRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketSetPropertyRequest.java index a88e5fb73334..ce350ad76140 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketSetPropertyRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketSetPropertyRequest.java @@ -44,8 +44,7 @@ import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.request.OMClientRequest; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; -import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; -import org.apache.hadoop.ozone.om.request.validation.ValidationCondition; +import org.apache.hadoop.ozone.om.request.validation.OMLayoutVersionValidator; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.om.response.bucket.OMBucketSetPropertyResponse; @@ -352,8 +351,8 @@ public boolean checkQuotaNamespaceValid(OmVolumeArgs omVolumeArgs, return true; } - @RequestFeatureValidator( - conditions = ValidationCondition.CLUSTER_NEEDS_FINALIZATION, + @OMLayoutVersionValidator( + applyBefore = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.SetBucketProperty ) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java index 97e3d612c8ec..4fa6b21eef6c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java @@ -31,6 +31,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import org.apache.hadoop.ozone.ClientVersion; import org.apache.hadoop.ozone.OmUtils; import org.apache.hadoop.ozone.audit.AuditLogger; import org.apache.hadoop.ozone.audit.OMAction; @@ -44,8 +45,8 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.request.key.OMKeyRequest; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; -import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; -import org.apache.hadoop.ozone.om.request.validation.ValidationCondition; +import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; +import org.apache.hadoop.ozone.om.request.validation.OMLayoutVersionValidator; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.om.response.file.OMDirectoryCreateResponse; @@ -262,8 +263,8 @@ private void logResult(CreateDirectoryRequest createDirectoryRequest, } } - @RequestFeatureValidator( - conditions = ValidationCondition.CLUSTER_NEEDS_FINALIZATION, + @OMLayoutVersionValidator( + applyBefore = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CreateDirectory ) @@ -295,8 +296,8 @@ public static OMRequest disallowCreateDirectoryWithECReplicationConfig( * @return the validated request * @throws OMException if the request is invalid */ - @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + @OMClientVersionValidator( + applyBefore = ClientVersion.BUCKET_LAYOUT_SUPPORT, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CreateDirectory ) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java index ecb863008350..fa6dae8685d6 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java @@ -35,6 +35,7 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList; import org.apache.hadoop.hdds.utils.UniqueId; +import org.apache.hadoop.ozone.ClientVersion; import org.apache.hadoop.ozone.OmUtils; import org.apache.hadoop.ozone.audit.OMAction; import org.apache.hadoop.ozone.om.OMMetadataManager; @@ -49,8 +50,8 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; import org.apache.hadoop.ozone.om.request.key.OMKeyRequest; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; -import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; -import org.apache.hadoop.ozone.om.request.validation.ValidationCondition; +import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; +import org.apache.hadoop.ozone.om.request.validation.OMLayoutVersionValidator; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.om.response.file.OMFileCreateResponse; @@ -380,8 +381,8 @@ protected void checkAllParentsExist(KeyArgs keyArgs, } } - @RequestFeatureValidator( - conditions = ValidationCondition.CLUSTER_NEEDS_FINALIZATION, + @OMLayoutVersionValidator( + applyBefore = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = CreateFile ) @@ -412,8 +413,8 @@ public static OMRequest disallowCreateFileWithECReplicationConfig( * @return the validated request * @throws OMException if the request is invalid */ - @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + @OMClientVersionValidator( + applyBefore = ClientVersion.BUCKET_LAYOUT_SUPPORT, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CreateFile ) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java index b692cf9d55eb..1ad1cf7e8f9a 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java @@ -31,6 +31,7 @@ import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; +import org.apache.hadoop.ozone.ClientVersion; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.audit.AuditLogger; import org.apache.hadoop.ozone.audit.OMAction; @@ -45,8 +46,8 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; import org.apache.hadoop.ozone.om.helpers.QuotaUtil; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; -import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; -import org.apache.hadoop.ozone.om.request.validation.ValidationCondition; +import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; +import org.apache.hadoop.ozone.om.request.validation.OMLayoutVersionValidator; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.om.response.key.OMAllocateBlockResponse; @@ -276,8 +277,8 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut return omClientResponse; } - @RequestFeatureValidator( - conditions = ValidationCondition.CLUSTER_NEEDS_FINALIZATION, + @OMLayoutVersionValidator( + applyBefore = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.AllocateBlock ) @@ -306,8 +307,8 @@ public static OMRequest disallowAllocateBlockWithECReplicationConfig( * @return the validated request * @throws OMException if the request is invalid */ - @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + @OMClientVersionValidator( + applyBefore = ClientVersion.BUCKET_LAYOUT_SUPPORT, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.AllocateBlock ) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java index be0935d909d8..a90b342f6cb0 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java @@ -34,6 +34,7 @@ import java.util.Map; import java.util.Objects; import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.ozone.ClientVersion; import org.apache.hadoop.ozone.OmUtils; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.OzoneManagerVersion; @@ -55,8 +56,8 @@ import org.apache.hadoop.ozone.om.helpers.WithMetadata; import org.apache.hadoop.ozone.om.request.util.OmKeyHSyncUtil; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; -import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; -import org.apache.hadoop.ozone.om.request.validation.ValidationCondition; +import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; +import org.apache.hadoop.ozone.om.request.validation.OMLayoutVersionValidator; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.om.response.key.OMKeyCommitResponse; @@ -505,8 +506,8 @@ protected void processResult(CommitKeyRequest commitKeyRequest, } } - @RequestFeatureValidator( - conditions = ValidationCondition.CLUSTER_NEEDS_FINALIZATION, + @OMLayoutVersionValidator( + applyBefore = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CommitKey ) @@ -535,8 +536,8 @@ public static OMRequest disallowCommitKeyWithECReplicationConfig( * @return the validated request * @throws OMException if the request is invalid */ - @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + @OMClientVersionValidator( + applyBefore = ClientVersion.BUCKET_LAYOUT_SUPPORT, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CommitKey ) @@ -554,8 +555,8 @@ public static OMRequest blockCommitKeyWithBucketLayoutFromOldClient( return req; } - @RequestFeatureValidator( - conditions = ValidationCondition.CLUSTER_NEEDS_FINALIZATION, + @OMLayoutVersionValidator( + applyBefore = OMLayoutFeature.HBASE_SUPPORT, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CommitKey ) @@ -587,8 +588,8 @@ public static OMRequest disallowHsync( * @return the validated request * @throws OMException if the request is invalid */ - @RequestFeatureValidator( - conditions = ValidationCondition.CLUSTER_NEEDS_FINALIZATION, + @OMLayoutVersionValidator( + applyBefore = OMLayoutFeature.HBASE_SUPPORT, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CommitKey ) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java index 345ac492a048..97ff720266af 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java @@ -34,6 +34,7 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList; import org.apache.hadoop.hdds.utils.UniqueId; +import org.apache.hadoop.ozone.ClientVersion; import org.apache.hadoop.ozone.OmUtils; import org.apache.hadoop.ozone.OzoneManagerVersion; import org.apache.hadoop.ozone.audit.OMAction; @@ -51,8 +52,8 @@ import org.apache.hadoop.ozone.om.lock.OzoneLockStrategy; import org.apache.hadoop.ozone.om.request.file.OMFileRequest; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; -import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; -import org.apache.hadoop.ozone.om.request.validation.ValidationCondition; +import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; +import org.apache.hadoop.ozone.om.request.validation.OMLayoutVersionValidator; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.om.response.key.OMKeyCreateResponse; @@ -178,7 +179,7 @@ public OMRequest preExecute(OzoneManager ozoneManager) throws IOException { KeyArgs.Builder finalNewKeyArgs = newKeyArgs; KeyArgs resolvedKeyArgs = - captureLatencyNs(perfMetrics.getCreateKeyResolveBucketAndAclCheckLatencyNs(), + captureLatencyNs(perfMetrics.getCreateKeyResolveBucketAndAclCheckLatencyNs(), () -> resolveBucketAndCheckKeyAcls(finalNewKeyArgs.build(), ozoneManager, IAccessAuthorizer.ACLType.CREATE)); newCreateKeyRequest = @@ -358,7 +359,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut } else { perfMetrics.addCreateKeyFailureLatencyNs(createKeyLatency); } - + if (acquireLock) { mergeOmLockDetails(ozoneLockStrategy .releaseWriteLock(omMetadataManager, volumeName, @@ -408,8 +409,8 @@ protected void logResult(CreateKeyRequest createKeyRequest, } } - @RequestFeatureValidator( - conditions = ValidationCondition.CLUSTER_NEEDS_FINALIZATION, + @OMLayoutVersionValidator( + applyBefore = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CreateKey ) @@ -438,8 +439,8 @@ public static OMRequest disallowCreateKeyWithECReplicationConfig( * @return the validated request * @throws OMException if the request is invalid */ - @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + @OMClientVersionValidator( + applyBefore = ClientVersion.BUCKET_LAYOUT_SUPPORT, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CreateKey ) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java index 4726d4af2d5f..3fc06d0a970c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java @@ -29,6 +29,7 @@ import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; +import org.apache.hadoop.ozone.ClientVersion; import org.apache.hadoop.ozone.OmUtils; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.audit.AuditLogger; @@ -43,8 +44,7 @@ import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; -import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; -import org.apache.hadoop.ozone.om.request.validation.ValidationCondition; +import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.om.response.key.OMKeyDeleteResponse; @@ -247,8 +247,8 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut * @return the validated request * @throws OMException if the request is invalid */ - @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + @OMClientVersionValidator( + applyBefore = ClientVersion.BUCKET_LAYOUT_SUPPORT, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.DeleteKey ) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java index 850f111a913f..6ff41120b3b6 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java @@ -27,6 +27,7 @@ import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; +import org.apache.hadoop.ozone.ClientVersion; import org.apache.hadoop.ozone.OmUtils; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.audit.AuditLogger; @@ -39,8 +40,7 @@ import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; -import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; -import org.apache.hadoop.ozone.om.request.validation.ValidationCondition; +import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.om.response.key.OMKeyRenameResponse; @@ -259,8 +259,8 @@ private Map buildAuditMap( * @return the validated request * @throws OMException if the request is invalid */ - @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + @OMClientVersionValidator( + applyBefore = ClientVersion.BUCKET_LAYOUT_SUPPORT, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.RenameKey ) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java index 2f8562642982..d64bf86e77e6 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java @@ -42,6 +42,7 @@ import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; +import org.apache.hadoop.ozone.ClientVersion; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.audit.AuditLogger; import org.apache.hadoop.ozone.om.OMMetadataManager; @@ -57,8 +58,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; -import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; -import org.apache.hadoop.ozone.om.request.validation.ValidationCondition; +import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.om.response.key.OMKeysDeleteResponse; @@ -383,8 +383,8 @@ protected static void addDeletedKeys(Map auditMap, * @return the validated request * @throws OMException if the request is invalid */ - @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + @OMClientVersionValidator( + applyBefore = ClientVersion.BUCKET_LAYOUT_SUPPORT, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.DeleteKeys ) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysRenameRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysRenameRequest.java index 3da0849f44f4..08e50801f0d2 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysRenameRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysRenameRequest.java @@ -34,6 +34,7 @@ import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; +import org.apache.hadoop.ozone.ClientVersion; import org.apache.hadoop.ozone.audit.AuditLogger; import org.apache.hadoop.ozone.audit.OMAction; import org.apache.hadoop.ozone.om.OMMetadataManager; @@ -46,8 +47,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmRenameKeys; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; -import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; -import org.apache.hadoop.ozone.om.request.validation.ValidationCondition; +import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.om.response.key.OMKeysRenameResponse; @@ -296,8 +296,8 @@ private Map buildAuditMap(Map auditMap, * @return the validated request * @throws OMException if the request is invalid */ - @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + @OMClientVersionValidator( + applyBefore = ClientVersion.BUCKET_LAYOUT_SUPPORT, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.RenameKeys ) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAddAclRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAddAclRequest.java index 367e3c87db68..5f877de8e027 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAddAclRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAddAclRequest.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.util.List; import java.util.Map; +import org.apache.hadoop.ozone.ClientVersion; import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.audit.AuditLogger; @@ -32,8 +33,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.request.util.ObjectParser; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; -import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; -import org.apache.hadoop.ozone.om.request.validation.ValidationCondition; +import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.om.response.key.acl.OMKeyAclResponse; @@ -161,8 +161,8 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut * @return the validated request * @throws OMException if the request is invalid */ - @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + @OMClientVersionValidator( + applyBefore = ClientVersion.BUCKET_LAYOUT_SUPPORT, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.AddAcl ) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyRemoveAclRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyRemoveAclRequest.java index 0de996fd28ad..32d8c0454f56 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyRemoveAclRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyRemoveAclRequest.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.util.List; import java.util.Map; +import org.apache.hadoop.ozone.ClientVersion; import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.audit.AuditLogger; @@ -32,8 +33,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.request.util.ObjectParser; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; -import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; -import org.apache.hadoop.ozone.om.request.validation.ValidationCondition; +import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.om.response.key.acl.OMKeyAclResponse; @@ -162,8 +162,8 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut * @return the validated request * @throws OMException if the request is invalid */ - @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + @OMClientVersionValidator( + applyBefore = ClientVersion.BUCKET_LAYOUT_SUPPORT, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.RemoveAcl ) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeySetAclRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeySetAclRequest.java index b0bd2f8fe528..79ac89aa7d8b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeySetAclRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeySetAclRequest.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.util.List; import java.util.Map; +import org.apache.hadoop.ozone.ClientVersion; import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.audit.AuditLogger; @@ -33,8 +34,7 @@ import org.apache.hadoop.ozone.om.helpers.OzoneAclUtil; import org.apache.hadoop.ozone.om.request.util.ObjectParser; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; -import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; -import org.apache.hadoop.ozone.om.request.validation.ValidationCondition; +import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.om.response.key.acl.OMKeyAclResponse; @@ -158,8 +158,8 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut * @return the validated request * @throws OMException if the request is invalid */ - @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + @OMClientVersionValidator( + applyBefore = ClientVersion.BUCKET_LAYOUT_SUPPORT, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.SetAcl ) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java index 22f470c80c7b..a1a50997d4fd 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java @@ -29,6 +29,7 @@ import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; +import org.apache.hadoop.ozone.ClientVersion; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.audit.OMAction; import org.apache.hadoop.ozone.om.OMMetadataManager; @@ -46,8 +47,8 @@ import org.apache.hadoop.ozone.om.request.key.OMKeyRequest; import org.apache.hadoop.ozone.om.request.util.OMMultipartUploadUtils; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; -import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; -import org.apache.hadoop.ozone.om.request.validation.ValidationCondition; +import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; +import org.apache.hadoop.ozone.om.request.validation.OMLayoutVersionValidator; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.om.response.s3.multipart.S3InitiateMultipartUploadResponse; @@ -284,8 +285,8 @@ protected void logResult(OzoneManager ozoneManager, } } - @RequestFeatureValidator( - conditions = ValidationCondition.CLUSTER_NEEDS_FINALIZATION, + @OMLayoutVersionValidator( + applyBefore = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.InitiateMultiPartUpload ) @@ -317,8 +318,8 @@ protected void logResult(OzoneManager ozoneManager, * @return the validated request * @throws OMException if the request is invalid */ - @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + @OMClientVersionValidator( + applyBefore = ClientVersion.BUCKET_LAYOUT_SUPPORT, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.InitiateMultiPartUpload ) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java index 9645a0c2e240..8a8cfd40f660 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java @@ -24,6 +24,7 @@ import java.util.Map; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; +import org.apache.hadoop.ozone.ClientVersion; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.audit.OMAction; import org.apache.hadoop.ozone.om.OMMetadataManager; @@ -38,8 +39,8 @@ import org.apache.hadoop.ozone.om.request.key.OMKeyRequest; import org.apache.hadoop.ozone.om.request.util.OMMultipartUploadUtils; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; -import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; -import org.apache.hadoop.ozone.om.request.validation.ValidationCondition; +import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; +import org.apache.hadoop.ozone.om.request.validation.OMLayoutVersionValidator; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.om.response.s3.multipart.S3MultipartUploadAbortResponse; @@ -255,8 +256,8 @@ protected String getMultipartOpenKey(String multipartUploadID, getBucketLayout()); } - @RequestFeatureValidator( - conditions = ValidationCondition.CLUSTER_NEEDS_FINALIZATION, + @OMLayoutVersionValidator( + applyBefore = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.AbortMultiPartUpload ) @@ -286,8 +287,8 @@ public static OMRequest disallowAbortMultiPartUploadWithECReplicationConfig( * @return the validated request * @throws OMException if the request is invalid */ - @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + @OMClientVersionValidator( + applyBefore = ClientVersion.BUCKET_LAYOUT_SUPPORT, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.AbortMultiPartUpload ) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java index ac123ff680ac..4a7d80600df2 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java @@ -29,6 +29,7 @@ import java.util.stream.Collectors; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; +import org.apache.hadoop.ozone.ClientVersion; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.audit.OMAction; import org.apache.hadoop.ozone.om.OMMetadataManager; @@ -44,8 +45,8 @@ import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.request.key.OMKeyRequest; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; -import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; -import org.apache.hadoop.ozone.om.request.validation.ValidationCondition; +import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; +import org.apache.hadoop.ozone.om.request.validation.OMLayoutVersionValidator; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.om.response.s3.multipart.S3MultipartUploadCommitPartResponse; @@ -370,8 +371,8 @@ private String getMultipartKey(String volumeName, String bucketName, keyName, uploadID); } - @RequestFeatureValidator( - conditions = ValidationCondition.CLUSTER_NEEDS_FINALIZATION, + @OMLayoutVersionValidator( + applyBefore = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CommitMultiPartUpload ) @@ -401,8 +402,8 @@ public static OMRequest disallowCommitMultiPartUploadWithECReplicationConfig( * @return the validated request * @throws OMException if the request is invalid */ - @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + @OMClientVersionValidator( + applyBefore = ClientVersion.BUCKET_LAYOUT_SUPPORT, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CommitMultiPartUpload ) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java index cadd7f80b62f..0e2c93d3103b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java @@ -36,6 +36,7 @@ import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; +import org.apache.hadoop.ozone.ClientVersion; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.audit.OMAction; import org.apache.hadoop.ozone.om.OMMetadataManager; @@ -55,8 +56,8 @@ import org.apache.hadoop.ozone.om.request.file.OMFileRequest; import org.apache.hadoop.ozone.om.request.key.OMKeyRequest; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; -import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; -import org.apache.hadoop.ozone.om.request.validation.ValidationCondition; +import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; +import org.apache.hadoop.ozone.om.request.validation.OMLayoutVersionValidator; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.om.response.s3.multipart.S3MultipartUploadCompleteResponse; @@ -683,8 +684,8 @@ private void updateCache(OMMetadataManager omMetadataManager, } } - @RequestFeatureValidator( - conditions = ValidationCondition.CLUSTER_NEEDS_FINALIZATION, + @OMLayoutVersionValidator( + applyBefore = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CompleteMultiPartUpload ) @@ -715,8 +716,8 @@ private void updateCache(OMMetadataManager omMetadataManager, * @return the validated request * @throws OMException if the request is invalid */ - @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + @OMClientVersionValidator( + applyBefore = ClientVersion.BUCKET_LAYOUT_SUPPORT, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CompleteMultiPartUpload ) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestFeatureValidator.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestFeatureValidator.java deleted file mode 100644 index 2aca1cf771b3..000000000000 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestFeatureValidator.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.om.request.validation; - -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; -import org.apache.hadoop.ozone.request.validation.RequestProcessingPhase; - -/** - * An annotation to mark methods that do certain request validations. - * - * The methods annotated with this annotation are collected by the - * {@link ValidatorRegistry} class during the initialization of the server. - * - * The conditions specify the specific use case in which the validator should be - * applied to the request. See {@link ValidationCondition} for more details - * on the specific conditions. - * The validator method should be applied to just one specific request type - * to help keep these methods simple and straightforward. If you want to use - * the same validation for different request types, use inheritance, and - * annotate the override method that just calls super. - * Note that the aim is to have these validators together with the request - * processing code, so the handling of these specific situations are easy to - * find. - * - * The annotated methods have to have a fixed signature. - * A {@link RequestProcessingPhase#PRE_PROCESS} phase method is running before - * the request is processed by the regular code. - * Its signature has to be the following: - * - it has to be static and idempotent - * - it has to have two parameters - * - the first parameter it is an - * {@link - * org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest} - * - the second parameter of type {@link ValidationContext} - * - the method has to return the modified request, or throw a ServiceException - * in case the request is considered to be invalid - * - the method does not need to care about preserving the request it gets, - * the original request is captured and saved by the calling environment. - * - * A {@link RequestProcessingPhase#POST_PROCESS} phase method is running once - * the - * {@link - * org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse} - * is calculated for a given request. - * Its signature has to be the following: - * - it has to be static and idempotent - * - it has three parameters - * - similalry to the pre-processing validators, first parameter is the - * OMRequest, the second parameter is the OMResponse, and the third - * parameter is a ValidationContext. - * - the method has to return the modified OMResponse or throw a - * ServiceException if the request is considered invalid based on response. - * - the method gets the request object that was supplied for the general - * request processing code, not the original request, while it gets a copy - * of the original response object provided by the general request processing - * code. - */ -@Retention(RetentionPolicy.RUNTIME) -@Target(ElementType.METHOD) -public @interface RequestFeatureValidator { - - /** - * Runtime conditions in which a validator should run. - * @return a list of conditions when the validator should be applied - */ - ValidationCondition[] conditions(); - - /** - * Defines if the validation has to run before or after the general request - * processing. - * @return if this is a pre or post processing validator - */ - RequestProcessingPhase processingPhase(); - - /** - * The type of the request handled by this validator method. - * @return the requestType to whihc the validator shoudl be applied - */ - Type requestType(); - -} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestValidations.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestValidations.java index a2be5a2d97cb..721913a2ebcd 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestValidations.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestValidations.java @@ -20,15 +20,24 @@ import static org.apache.hadoop.ozone.request.validation.RequestProcessingPhase.POST_PROCESS; import static org.apache.hadoop.ozone.request.validation.RequestProcessingPhase.PRE_PROCESS; +import com.google.common.collect.Sets; import com.google.protobuf.ServiceException; +import java.lang.annotation.Annotation; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.util.Arrays; import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; import java.util.stream.Collectors; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.ozone.Versioned; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; +import org.apache.hadoop.ozone.request.validation.RequestProcessingPhase; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -40,10 +49,11 @@ public class RequestValidations { static final Logger LOG = LoggerFactory.getLogger(RequestValidations.class); private static final String DEFAULT_PACKAGE = "org.apache.hadoop.ozone"; - + private static final Set ALLOWED_REQUEST_PROCESSING_PHASES = + Sets.immutableEnumSet(PRE_PROCESS, POST_PROCESS); private String validationsPackageName = DEFAULT_PACKAGE; private ValidationContext context = null; - private ValidatorRegistry registry = null; + private ValidatorRegistry registry = null; public synchronized RequestValidations fromPackage(String packageName) { validationsPackageName = packageName; @@ -56,15 +66,16 @@ public RequestValidations withinContext(ValidationContext validationContext) { } public synchronized RequestValidations load() { - registry = new ValidatorRegistry(validationsPackageName); + registry = new ValidatorRegistry<>(Type.class, validationsPackageName, + Arrays.stream(VersionExtractor.values()).map(VersionExtractor::getValidatorClass).collect(Collectors.toSet()), + ALLOWED_REQUEST_PROCESSING_PHASES); return this; } public OMRequest validateRequest(OMRequest request) throws Exception { - List validations = registry.validationsFor( - conditions(request), request.getCmdType(), PRE_PROCESS); + List validations = registry.validationsFor(request.getCmdType(), PRE_PROCESS, getVersions(request)); OMRequest validatedRequest = request; try { for (Method m : validations) { @@ -87,10 +98,10 @@ public OMRequest validateRequest(OMRequest request) public OMResponse validateResponse(OMRequest request, OMResponse response) throws ServiceException { - List validations = registry.validationsFor( - conditions(request), request.getCmdType(), POST_PROCESS); - OMResponse validatedResponse = response; + List validations = registry.validationsFor(request.getCmdType(), POST_PROCESS, this.getVersions(request)); + + OMResponse validatedResponse = response.toBuilder().build(); try { for (Method m : validations) { LOG.debug("Running the {} request post-process validation from {}.{}", @@ -105,9 +116,11 @@ public OMResponse validateResponse(OMRequest request, OMResponse response) return validatedResponse; } - private List conditions(OMRequest request) { - return Arrays.stream(ValidationCondition.values()) - .filter(c -> c.shouldApply(request, context)) - .collect(Collectors.toList()); + private Map, Versioned> getVersions(OMRequest request) { + return Arrays.stream(VersionExtractor.values()) + .map(versionExtractor -> Pair.of(versionExtractor.getValidatorClass(), + versionExtractor.extractVersion(request, context))) + .filter(pair -> Objects.nonNull(pair.getValue())) + .collect(Collectors.toMap(Pair::getKey, Pair::getValue)); } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidationCondition.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidationCondition.java deleted file mode 100644 index 87f2188ea1e2..000000000000 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidationCondition.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.om.request.validation; - -import org.apache.hadoop.ozone.ClientVersion; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; - -/** - * Defines conditions for which validators can be assigned to. - * - * These conditions describe a situation where special request handling might - * be necessary. In these cases we do not override the actual request handling - * code, but based on certain request properties we might reject a request - * early, or we might modify the request, or the response received/sent from/to - * the client. - */ -public enum ValidationCondition { - /** - * Classifies validations that has to run after an upgrade until the cluster - * is in a pre-finalized state. - */ - CLUSTER_NEEDS_FINALIZATION { - @Override - public boolean shouldApply(OMRequest req, ValidationContext ctx) { - return ctx.versionManager().needsFinalization(); - } - }, - - /** - * Classifies validations that has to run, when the client uses an older - * protocol version than the server. - */ - OLDER_CLIENT_REQUESTS { - @Override - public boolean shouldApply(OMRequest req, ValidationContext ctx) { - return req.getVersion() < ClientVersion.CURRENT_VERSION; - } - }; - - public abstract boolean shouldApply(OMRequest req, ValidationContext ctx); -} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidatorRegistry.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidatorRegistry.java index ebc60aab7682..473314e0c8c2 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidatorRegistry.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidatorRegistry.java @@ -17,21 +17,29 @@ package org.apache.hadoop.ozone.om.request.validation; -import static org.apache.hadoop.ozone.request.validation.RequestProcessingPhase.POST_PROCESS; -import static org.apache.hadoop.ozone.request.validation.RequestProcessingPhase.PRE_PROCESS; - +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Sets; +import java.lang.annotation.Annotation; +import java.lang.reflect.Array; +import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.net.URL; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.Comparator; import java.util.EnumMap; -import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.NavigableMap; +import java.util.Optional; import java.util.Set; -import java.util.function.Supplier; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; +import java.util.TreeMap; +import java.util.function.Function; +import java.util.stream.Collectors; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.ozone.Versioned; +import org.apache.hadoop.ozone.request.validation.RegisterValidator; import org.apache.hadoop.ozone.request.validation.RequestProcessingPhase; import org.reflections.Reflections; import org.reflections.scanners.Scanners; @@ -42,156 +50,265 @@ * Registry that loads and stores the request validators to be applied by * a service. */ -public class ValidatorRegistry { +public class ValidatorRegistry> { - private final EnumMap>>> - validators = new EnumMap<>(ValidationCondition.class); + /** + * A validator registered should have the following parameters: + * applyBeforeVersion: Enum extending Version + * RequestType: Enum signifying the type of request. + * RequestProcessingPhase: Signifying if the validator is supposed to run pre or post submitting the request. + * Based on the afforementioned parameters a complete map is built which stores the validators in a sorted order of + * the applyBeforeVersion value of the validator method. + * Thus when a request comes with a certain version value, all validators containing `applyBeforeVersion` parameter + * greater than the request versions get triggered. + * {@link #validationsFor(Enum, RequestProcessingPhase, Class, Versioned)} + */ + private final Map, EnumMap>>> indexedValidatorMap; /** * Creates a {@link ValidatorRegistry} instance that discovers validation * methods in the provided package and the packages in the same resource. - * A validation method is recognized by the {@link RequestFeatureValidator} - * annotation that contains important information about how and when to use - * the validator. - * @param validatorPackage the main package inside which validatiors should - * be discovered. + * A validation method is recognized by all the annotations classes which + * are annotated by {@link RegisterValidator} annotation that contains + * important information about how and when to use the validator. + * + * @param requestType class of request type enum. + * @param validatorPackage the main package inside which validatiors should + * be discovered. + * @param allowedValidators a set containing the various types of version allowed to be registered. + * @param allowedProcessingPhases set of request processing phases which would be allowed to be registered to + * registry. */ - ValidatorRegistry(String validatorPackage) { - this(ClasspathHelper.forPackage(validatorPackage)); + public ValidatorRegistry(Class requestType, String validatorPackage, + Set> allowedValidators, Set allowedProcessingPhases) { + this(requestType, ClasspathHelper.forPackage(validatorPackage), allowedValidators, allowedProcessingPhases); } /** * Creates a {@link ValidatorRegistry} instance that discovers validation * methods under the provided URL. - * A validation method is recognized by the {@link RequestFeatureValidator} + * A validation method is recognized by all annotations annotated by the {@link RegisterValidator} * annotation that contains important information about how and when to use * the validator. - * @param searchUrls the path in which the annotated methods are searched. + * + * @param requestType class of request type enum. + * @param searchUrls the path in which the annotated methods are searched. + * @param allowedValidators a set containing the various types of validator annotation allowed to be registered. + * @param allowedProcessingPhases set of request processing phases which would be allowed to be registered to + * the registry. */ - ValidatorRegistry(Collection searchUrls) { + ValidatorRegistry(Class requestType, Collection searchUrls, + Set> allowedValidators, + Set allowedProcessingPhases) { + Class requestArrayClass = (Class) Array.newInstance(requestType, 0) + .getClass(); + Set> validatorsToBeRegistered = + new Reflections(new ConfigurationBuilder().setUrls(ClasspathHelper.forPackage("org.apache.hadoop")) + .setScanners(Scanners.TypesAnnotated) + .setParallel(true)).getTypesAnnotatedWith(RegisterValidator.class).stream() + .filter(allowedValidators::contains) + .filter(annotationClass -> getReturnTypeOfAnnotationMethod((Class) annotationClass, + RegisterValidator.REQUEST_TYPE_METHOD_NAME) + .equals(requestArrayClass)) + .map(annotationClass -> (Class) annotationClass) + .collect(Collectors.toSet()); + this.indexedValidatorMap = allowedValidators.stream().collect(ImmutableMap.toImmutableMap(Function.identity(), + validatorClass -> new EnumMap<>(requestType))); Reflections reflections = new Reflections(new ConfigurationBuilder() .setUrls(searchUrls) .setScanners(Scanners.MethodsAnnotated) .setParallel(true) ); - - Set describedValidators = - reflections.getMethodsAnnotatedWith(RequestFeatureValidator.class); - initMaps(describedValidators); + initMaps(requestArrayClass, allowedProcessingPhases, validatorsToBeRegistered, reflections); } /** - * Get the validators that has to be run in the given list of - * {@link ValidationCondition}s, for the given requestType and + * Get the validators that has to be run in the given list of, + * for the given requestType and for the given request versions. * {@link RequestProcessingPhase}. * - * @param conditions conditions that are present for the request - * @param requestType the type of the protocol message - * @param phase the request processing phase + * @param requestType the type of the protocol message + * @param phase the request processing phase + * @param requestVersions different versions extracted from the request. * @return the list of validation methods that has to run. */ - List validationsFor( - List conditions, - Type requestType, - RequestProcessingPhase phase) { + public List validationsFor(RequestType requestType, RequestProcessingPhase phase, + Map, ? extends Versioned> requestVersions) { + return requestVersions.entrySet().stream() + .flatMap(requestVersion -> this.validationsFor(requestType, phase, requestVersion.getKey(), + requestVersion.getValue()).stream()) + .distinct().collect(Collectors.toList()); + } - if (conditions.isEmpty() || validators.isEmpty()) { - return Collections.emptyList(); - } + /** + * Get the validators that has to be run in the given list of, + * for the given requestType and for the given request versions. + * {@link RequestProcessingPhase}. + * + * @param requestType the type of the protocol message + * @param phase the request processing phase + * @param validatorClass annotation class of the validator + * @param requestVersion version extracted corresponding to the request. + * @return the list of validation methods that has to run. + */ + public List validationsFor(RequestType requestType, + RequestProcessingPhase phase, Class validatorClass, V requestVersion) { - Set returnValue = new HashSet<>(); + return Optional.ofNullable(this.indexedValidatorMap.get(validatorClass)) + .map(requestTypeMap -> requestTypeMap.get(requestType)) + .map(phaseMap -> phaseMap.get(phase)) + .map(indexedMethods -> requestVersion.version() < 0 ? + indexedMethods.getItemsEqualToIdx(requestVersion) : + indexedMethods.getItemsGreaterThanIdx(requestVersion)) + .orElse(Collections.emptyList()); - for (ValidationCondition condition: conditions) { - returnValue.addAll(validationsFor(condition, requestType, phase)); - } - return new ArrayList<>(returnValue); } /** - * Grabs validations for one particular condition. - * - * @param condition conditions that are present for the request - * @param requestType the type of the protocol message - * @param phase the request processing phase - * @return the list of validation methods that has to run. + * Calls a specified method on the validator. + * @throws IllegalArgumentException when the specified method in the validator is invalid. */ - private List validationsFor( - ValidationCondition condition, - Type requestType, - RequestProcessingPhase phase) { - - EnumMap>> - requestTypeMap = validators.get(condition); - if (requestTypeMap == null || requestTypeMap.isEmpty()) { - return Collections.emptyList(); + private static ReturnValue callAnnotationMethod( + Validator validator, String methodName, Class returnValueType) { + try { + return returnValueType.cast(validator.getClass().getMethod(methodName).invoke(validator)); + } catch (NoSuchMethodException e) { + throw new IllegalArgumentException("Method " + methodName + " not found in class:" + + validator.getClass().getCanonicalName(), e); + } catch (InvocationTargetException | IllegalAccessException e) { + throw new IllegalArgumentException("Error while invoking Method " + methodName + " from " + + validator.getClass().getCanonicalName(), e); } + } - EnumMap> phases = - requestTypeMap.get(requestType); - if (phases == null) { - return Collections.emptyList(); + private static Class getReturnTypeOfAnnotationMethod(Class clzz, String methodName) { + try { + return clzz.getMethod(methodName).getReturnType(); + } catch (NoSuchMethodException e) { + throw new IllegalArgumentException("Method " + methodName + " not found in class:" + clzz.getCanonicalName()); } + } - List validatorsForPhase = phases.get(phase); - if (validatorsForPhase == null) { - return Collections.emptyList(); + private static Versioned getApplyBeforeVersion(Validator validator) { + return callAnnotationMethod(validator, RegisterValidator.APPLY_BEFORE_METHOD_NAME, Versioned.class); + } + + private static RequestProcessingPhase getRequestPhase(Validator validator) { + return callAnnotationMethod(validator, RegisterValidator.PROCESSING_PHASE_METHOD_NAME, + RequestProcessingPhase.class); + } + + private RequestType[] getRequestType(Validator validator, + Class requestType) { + return callAnnotationMethod(validator, RegisterValidator.REQUEST_TYPE_METHOD_NAME, requestType); + } + + private static void checkAllowedAnnotationValues(Set values, V value, String valueName, String methodName) { + if (!values.contains(value)) { + throw new IllegalArgumentException( + String.format("Invalid %1$s defined at annotation defined for method : %2$s, Annotation value : %3$s " + + "Allowed versionType: %4$s", valueName, methodName, value.toString(), values)); } - return validatorsForPhase; } /** * Initializes the internal request validator store. * The requests are stored in the following structure: - * - An EnumMap with the {@link ValidationCondition} as the key, and in which - * - values are an EnumMap with the request type as the key, and in which - * - values are Pair of lists, in which - * - left side is the pre-processing validations list - * - right side is the post-processing validations list - * @param describedValidators collection of the annotated methods to process. + * - An EnumMap with the RequestType as the key, and in which + * - values are an EnumMap with the request processing phase as the key, and in which + * - values is an {@link IndexedItems } containing the validation list + * + * @param validatorsToBeRegistered collection of the annotated validtors to process. */ - void initMaps(Collection describedValidators) { - for (Method m : describedValidators) { - RequestFeatureValidator descriptor = - m.getAnnotation(RequestFeatureValidator.class); - m.setAccessible(true); - - for (ValidationCondition condition : descriptor.conditions()) { - EnumMap>> - requestTypeMap = getAndInitialize( - condition, this::newTypeMap, validators); - EnumMap> phases = getAndInitialize( - descriptor.requestType(), this::newPhaseMap, requestTypeMap); - if (isPreProcessValidator(descriptor)) { - getAndInitialize(PRE_PROCESS, ArrayList::new, phases).add(m); - } else if (isPostProcessValidator(descriptor)) { - getAndInitialize(POST_PROCESS, ArrayList::new, phases).add(m); - } - } + private void initMaps(Class requestType, + Set allowedPhases, + Collection> validatorsToBeRegistered, + Reflections reflections) { + for (Class validator : validatorsToBeRegistered) { + registerValidator(requestType, allowedPhases, validator, reflections); } } - private EnumMap>> newTypeMap() { - return new EnumMap<>(Type.class); + private void registerValidator(Class requestType, + Set allowedPhases, + Class validatorToBeRegistered, + Reflections reflections) { + Collection methods = reflections.getMethodsAnnotatedWith(validatorToBeRegistered); + List> sortedMethodsByApplyBeforeVersion = methods.stream() + .map(method -> Pair.of(method.getAnnotation(validatorToBeRegistered), method)) + .sorted(Comparator.comparing(validatorMethodPair -> getApplyBeforeVersion(validatorMethodPair.getKey()), + Versioned.versionComparator())) + .collect(Collectors.toList()); + for (Pair validatorMethodPair : sortedMethodsByApplyBeforeVersion) { + Annotation validator = validatorMethodPair.getKey(); + Method method = validatorMethodPair.getValue(); + Versioned applyBeforeVersion = this.getApplyBeforeVersion(validator); + RequestProcessingPhase phase = this.getRequestPhase(validator); + checkAllowedAnnotationValues(allowedPhases, phase, RegisterValidator.PROCESSING_PHASE_METHOD_NAME, + method.getName()); + Set types = Sets.newHashSet(this.getRequestType(validator, requestType)); + method.setAccessible(true); + for (RequestType type : types) { + EnumMap>> requestMap = + this.indexedValidatorMap.get(validatorToBeRegistered); + EnumMap> phaseMap = + requestMap.computeIfAbsent(type, k -> new EnumMap<>(RequestProcessingPhase.class)); + phaseMap.computeIfAbsent(phase, k -> new IndexedItems<>(Versioned.versionComparator())) + .add(method, applyBeforeVersion); + } + } } - private EnumMap> newPhaseMap() { - return new EnumMap<>(RequestProcessingPhase.class); - } + /** + * Class responsible for maintaining indexs of items. Here each item should have an index corresponding to it. + * The class implements functions for efficiently fetching range gets on the items added to the data structure. + * + * @param Refers to the Type of the item in the data structure + * @param Type of the index of an item added in the data structure. It is important that the index is + * comparable to each other. + */ + private static final class IndexedItems { + private final List items; + private final NavigableMap indexMap; - private V getAndInitialize(K key, Supplier defaultSupplier, Map from) { - return from.computeIfAbsent(key, k -> defaultSupplier.get()); - } + private IndexedItems(Comparator comparator) { + this.items = new ArrayList<>(); + this.indexMap = new TreeMap<>(comparator); + } - private boolean isPreProcessValidator(RequestFeatureValidator descriptor) { - return descriptor.processingPhase() - .equals(PRE_PROCESS); - } + /** + * Add an item to the collection and update index if required. The order of items added should have their index + * sorted in increasing order. + * + * @param item + * @param idx + */ + public void add(T item, IDX idx) { + indexMap.putIfAbsent(idx, items.size()); + items.add(item); + } - private boolean isPostProcessValidator(RequestFeatureValidator descriptor) { - return descriptor.processingPhase() - .equals(POST_PROCESS); + /** + * @param indexValue Given index value. + * @return All the items which has an index value greater than given index value. + */ + public List getItemsGreaterThanIdx(IDX indexValue) { + return Optional.ofNullable(indexMap.higherEntry(indexValue)) + .map(Map.Entry::getValue) + .map(startIndex -> items.subList(startIndex, items.size())).orElse(Collections.emptyList()); + } + + /** + * @param indexValue Given index value. + * @return All the items that have an index value equal to the given index value. + */ + public List getItemsEqualToIdx(IDX indexValue) { + return Optional.ofNullable(indexMap.get(indexValue)) + .map(startIndex -> items.subList(startIndex, Optional.ofNullable(indexMap.higherEntry(indexValue)) + .map(Map.Entry::getValue).orElse(items.size()))) + .orElse(Collections.emptyList()); + } } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/package-info.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/package-info.java index d3d70fe6ff76..a2fc73da167f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/package-info.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/package-info.java @@ -16,7 +16,7 @@ */ /** - * Request's feature validation handling. + * Request's validation handling. * * This package holds facilities to add new situation specific behaviour to * request handling without cluttering the basic logic of the request handler @@ -28,25 +28,29 @@ * request type * - a client connects to the server but uses an older version of the protocol * - a client connects to the server but uses a newer version of the protocol + * - a client connects to the server and performs an operation corresponding + * to a feature the server hasn't finalized for which these requests might have + * to be rejected. * - the code can handle certain checks that have to run all the time, but at * first we do not see a general use case that we would pull in immediately. - * These are the current - * {@link org.apache.hadoop.ozone.om.request.validation.ValidationCondition}s - * but this list might be extended later on if we see other use cases. + * These are the current registered + * {@link org.apache.hadoop.ozone.om.request.validation.VersionExtractor}s + * which would be extracted out of the om request and all validators + * fulfilling the condition would be run. * - * The system uses a reflection based discovery to find methods that are + * The system uses a reflection based discovery to find annotations that are * annotated with the - * {@link org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator} + * {@link org.apache.hadoop.ozone.request.validation.RegisterValidator} * annotation. - * This annotation is used to specify the condition in which a certain validator - * has to be used, the request type to which the validation should be applied, - * and the request processing phase in which we apply the validation. - * - * One validator can be applied in multiple - * {@link org.apache.hadoop.ozone.om.request.validation.ValidationCondition} - * but a validator has to handle strictly just one - * {@link org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type - * }. + * This annotation is used to register a particular annotation which in turn would be used to specify + * the request type to which the validation should be applied, + * and the request processing phase in which we apply the validation and the maxVersion corresponding to which this + * is supposed to run. + * + * One validator can be applied based on multiple trigger conditions, E.g. + * A validator registered can have both {@link org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator}, + * {@link org.apache.hadoop.ozone.om.request.validation.OMLayoutVersionValidator} + * * The main reason to avoid validating multiple request types with the same * validator, is that these validators have to be simple methods without state * any complex validation has to happen in the reql request handling. diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java index bea7785bfbc2..7c7b41e01e24 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java @@ -60,6 +60,7 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos.UpgradeFinalizationStatus; import org.apache.hadoop.hdds.scm.protocolPB.OzonePBHelper; import org.apache.hadoop.hdds.utils.FaultInjector; +import org.apache.hadoop.ozone.ClientVersion; import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.OzoneManagerPrepareState; @@ -94,8 +95,7 @@ import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; import org.apache.hadoop.ozone.om.request.OMClientRequest; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; -import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; -import org.apache.hadoop.ozone.om.request.validation.ValidationCondition; +import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.om.upgrade.DisallowedUntilLayoutVersion; @@ -657,10 +657,10 @@ private GetKeyInfoResponse getKeyInfo(GetKeyInfoRequest request, return keyInfo.toProtobuf(clientVersion); } - @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + @OMClientVersionValidator( processingPhase = RequestProcessingPhase.POST_PROCESS, - requestType = Type.LookupKey + requestType = Type.LookupKey, + applyBefore = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMResponse disallowLookupKeyResponseWithECReplicationConfig( OMRequest req, OMResponse resp, ValidationContext ctx) @@ -684,10 +684,10 @@ public static OMResponse disallowLookupKeyResponseWithECReplicationConfig( return resp; } - @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + @OMClientVersionValidator( processingPhase = RequestProcessingPhase.POST_PROCESS, - requestType = Type.LookupKey + requestType = Type.LookupKey, + applyBefore = ClientVersion.BUCKET_LAYOUT_SUPPORT ) public static OMResponse disallowLookupKeyWithBucketLayout( OMRequest req, OMResponse resp, ValidationContext ctx) @@ -771,8 +771,8 @@ private ListKeysLightResponse listKeysLight(ListKeysRequest request) return resp.build(); } - @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + @OMClientVersionValidator( + applyBefore = ClientVersion.ERASURE_CODING_SUPPORT, processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.ListKeys ) @@ -798,8 +798,8 @@ public static OMResponse disallowListKeysResponseWithECReplicationConfig( return resp; } - @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + @OMClientVersionValidator( + applyBefore = ClientVersion.BUCKET_LAYOUT_SUPPORT, processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.ListKeys ) @@ -839,8 +839,8 @@ public static OMResponse disallowListKeysWithBucketLayout( return resp; } - @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + @OMClientVersionValidator( + applyBefore = ClientVersion.ERASURE_CODING_SUPPORT, processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.ListTrash ) @@ -869,8 +869,8 @@ public static OMResponse disallowListTrashWithECReplicationConfig( return resp; } - @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + @OMClientVersionValidator( + applyBefore = ClientVersion.BUCKET_LAYOUT_SUPPORT, processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.ListTrash ) @@ -1085,8 +1085,8 @@ private RefetchSecretKeyResponse refetchSecretKey() { return response; } - @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + @OMClientVersionValidator( + applyBefore = ClientVersion.ERASURE_CODING_SUPPORT, processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.GetFileStatus ) @@ -1114,8 +1114,8 @@ public static OMResponse disallowGetFileStatusWithECReplicationConfig( return resp; } - @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + @OMClientVersionValidator( + applyBefore = ClientVersion.BUCKET_LAYOUT_SUPPORT, processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.GetFileStatus ) @@ -1163,8 +1163,8 @@ private LookupFileResponse lookupFile(LookupFileRequest request, .build(); } - @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + @OMClientVersionValidator( + applyBefore = ClientVersion.ERASURE_CODING_SUPPORT, processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.LookupFile ) @@ -1191,8 +1191,8 @@ public static OMResponse disallowLookupFileWithECReplicationConfig( return resp; } - @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + @OMClientVersionValidator( + applyBefore = ClientVersion.BUCKET_LAYOUT_SUPPORT, processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.LookupFile ) @@ -1273,8 +1273,8 @@ private ListStatusLightResponse listStatusLight( return listStatusLightResponseBuilder.build(); } - @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + @OMClientVersionValidator( + applyBefore = ClientVersion.ERASURE_CODING_SUPPORT, processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.ListStatus ) @@ -1301,8 +1301,8 @@ public static OMResponse disallowListStatusResponseWithECReplicationConfig( return resp; } - @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + @OMClientVersionValidator( + applyBefore = ClientVersion.BUCKET_LAYOUT_SUPPORT, processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.ListStatus ) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestRequestValidations.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestRequestValidations.java index dca87f88733c..444762b2d573 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestRequestValidations.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestRequestValidations.java @@ -27,6 +27,8 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -40,6 +42,7 @@ import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.request.validation.testvalidatorset1.GeneralValidatorsForTesting; import org.apache.hadoop.ozone.om.request.validation.testvalidatorset1.GeneralValidatorsForTesting.ValidationListener; +import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; @@ -159,7 +162,7 @@ public void testPreProcessorExceptionHandling() throws Exception { ValidationContext ctx = of(aFinalizedVersionManager(), metadataManager); RequestValidations validations = loadValidations(ctx); assertThrows(Exception.class, - () -> validations.validateRequest(aDeleteKeysRequest(olderClientVersion()))); + () -> validations.validateRequest(aDeleteKeysRequest(ClientVersion.ERASURE_CODING_SUPPORT))); validationListener.assertNumOfEvents(1); validationListener.assertExactListOfValidatorsCalled( @@ -171,7 +174,8 @@ public void testPostProcessorExceptionHandling() { ValidationContext ctx = of(aFinalizedVersionManager(), metadataManager); RequestValidations validations = loadValidations(ctx); assertThrows(Exception.class, - () -> validations.validateResponse(aDeleteKeysRequest(olderClientVersion()), aDeleteKeysResponse())); + () -> validations.validateResponse(aDeleteKeysRequest(ClientVersion.ERASURE_CODING_SUPPORT), + aDeleteKeysResponse())); validationListener.assertNumOfEvents(1); validationListener.assertExactListOfValidatorsCalled( @@ -184,11 +188,11 @@ public void testOldClientConditionIsRecognizedAndPreValidatorsApplied() ValidationContext ctx = of(aFinalizedVersionManager(), metadataManager); RequestValidations validations = loadValidations(ctx); - validations.validateRequest(aCreateKeyRequest(olderClientVersion())); + validations.validateRequest(aCreateKeyRequest(ClientVersion.ERASURE_CODING_SUPPORT)); validationListener.assertNumOfEvents(1); validationListener.assertExactListOfValidatorsCalled( - "oldClientPreProcessCreateKeyValidator"); + "preProcessCreateKeyBucketLayoutClientValidator"); } @Test @@ -198,12 +202,11 @@ public void testOldClientConditionIsRecognizedAndPostValidatorsApplied() RequestValidations validations = loadValidations(ctx); validations.validateResponse( - aCreateKeyRequest(olderClientVersion()), aCreateKeyResponse()); + aCreateKeyRequest(ClientVersion.ERASURE_CODING_SUPPORT), aCreateKeyResponse()); - validationListener.assertNumOfEvents(2); + validationListener.assertNumOfEvents(1); validationListener.assertExactListOfValidatorsCalled( - "oldClientPostProcessCreateKeyValidator", - "oldClientPostProcessCreateKeyValidator2"); + "postProcessCreateKeyBucketLayoutClientValidator"); } @Test @@ -212,12 +215,12 @@ public void testPreFinalizedWithOldClientConditionPreProcValidatorsApplied() ValidationContext ctx = of(anUnfinalizedVersionManager(), metadataManager); RequestValidations validations = loadValidations(ctx); - validations.validateRequest(aCreateKeyRequest(olderClientVersion())); + validations.validateRequest(aCreateKeyRequest(ClientVersion.ERASURE_CODING_SUPPORT)); validationListener.assertNumOfEvents(2); validationListener.assertExactListOfValidatorsCalled( - "preFinalizePreProcessCreateKeyValidator", - "oldClientPreProcessCreateKeyValidator"); + "preProcessCreateKeyQuotaLayoutValidator", + "preProcessCreateKeyBucketLayoutClientValidator"); } @Test @@ -227,13 +230,12 @@ public void testPreFinalizedWithOldClientConditionPostProcValidatorsApplied() RequestValidations validations = loadValidations(ctx); validations.validateResponse( - aCreateKeyRequest(olderClientVersion()), aCreateKeyResponse()); + aCreateKeyRequest(ClientVersion.ERASURE_CODING_SUPPORT), aCreateKeyResponse()); - validationListener.assertNumOfEvents(3); + validationListener.assertNumOfEvents(2); validationListener.assertExactListOfValidatorsCalled( - "preFinalizePostProcessCreateKeyValidator", - "oldClientPostProcessCreateKeyValidator", - "oldClientPostProcessCreateKeyValidator2"); + "postProcessCreateKeyQuotaLayoutValidator", + "postProcessCreateKeyBucketLayoutClientValidator"); } /** @@ -283,10 +285,6 @@ private RequestValidations loadEmptyValidations(ValidationContext ctx) { .load(); } - private int olderClientVersion() { - return ClientVersion.CURRENT_VERSION - 1; - } - private int currentClientVersion() { return ClientVersion.CURRENT_VERSION; } @@ -295,6 +293,14 @@ private OMRequest aCreateKeyRequest(int clientVersion) { return aRequest(CreateKey, clientVersion); } + private OMRequest aCreateKeyRequest(ClientVersion clientVersion) { + return aRequest(CreateKey, clientVersion.toProtoValue()); + } + + private OMRequest aDeleteKeysRequest(ClientVersion clientVersion) { + return aDeleteKeysRequest(clientVersion.toProtoValue()); + } + private OMRequest aDeleteKeysRequest(int clientVersion) { return aRequest(DeleteKeys, clientVersion); } @@ -333,12 +339,16 @@ private OMResponse aResponse(Type type) { private LayoutVersionManager aFinalizedVersionManager() { LayoutVersionManager vm = mock(LayoutVersionManager.class); when(vm.needsFinalization()).thenReturn(false); + when(vm.getFeature(anyString())).thenReturn(OMLayoutFeature.QUOTA); + when(vm.getFeature(anyInt())).thenReturn(OMLayoutFeature.QUOTA); return vm; } private LayoutVersionManager anUnfinalizedVersionManager() { LayoutVersionManager vm = mock(LayoutVersionManager.class); when(vm.needsFinalization()).thenReturn(true); + when(vm.getFeature(anyString())).thenReturn(OMLayoutFeature.HSYNC); + when(vm.getFeature(anyInt())).thenReturn(OMLayoutFeature.HSYNC); return vm; } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestValidatorRegistry.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestValidatorRegistry.java index db37843b837d..280046280f70 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestValidatorRegistry.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestValidatorRegistry.java @@ -17,10 +17,8 @@ package org.apache.hadoop.ozone.om.request.validation; -import static java.util.Arrays.asList; -import static java.util.Collections.emptyList; -import static org.apache.hadoop.ozone.om.request.validation.ValidationCondition.CLUSTER_NEEDS_FINALIZATION; -import static org.apache.hadoop.ozone.om.request.validation.ValidationCondition.OLDER_CLIENT_REQUESTS; +import static org.apache.hadoop.ozone.om.request.validation.VersionExtractor.CLIENT_VERSION_EXTRACTOR; +import static org.apache.hadoop.ozone.om.request.validation.VersionExtractor.LAYOUT_VERSION_EXTRACTOR; import static org.apache.hadoop.ozone.om.request.validation.testvalidatorset1.GeneralValidatorsForTesting.finishValidatorTest; import static org.apache.hadoop.ozone.om.request.validation.testvalidatorset1.GeneralValidatorsForTesting.startValidatorTest; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.CreateDirectory; @@ -32,13 +30,24 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Sets; import java.lang.reflect.Method; import java.net.MalformedURLException; import java.net.URL; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; +import org.apache.hadoop.ozone.ClientVersion; +import org.apache.hadoop.ozone.Versioned; +import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; +import org.apache.hadoop.ozone.request.validation.RequestProcessingPhase; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -59,6 +68,9 @@ public class TestValidatorRegistry { private static final String PACKAGE_WO_VALIDATORS = "org.apache.hadoop.hdds.annotation"; + private static final Set REQUEST_PROCESSING_PHASES = + Sets.immutableEnumSet(PRE_PROCESS, POST_PROCESS); + @BeforeEach public void setup() { startValidatorTest(); @@ -69,123 +81,129 @@ public void tearDown() { finishValidatorTest(); } + private List getValidatorsForRequest(OzoneManagerProtocolProtos.Type requestType, + RequestProcessingPhase phase, VersionExtractor extractor, Versioned versioned) { + return getValidatorsForRequest(PACKAGE, requestType, phase, Collections.singletonMap(extractor, versioned)); + } + + private List getValidatorsForRequest(OzoneManagerProtocolProtos.Type requestType, + RequestProcessingPhase phase, Map requestVersions) { + return getValidatorsForRequest(PACKAGE, requestType, phase, requestVersions); + } + + private List getValidatorsForRequest(String packageName, OzoneManagerProtocolProtos.Type requestType, + RequestProcessingPhase phase, Map requestVersions) { + ValidatorRegistry registry = + new ValidatorRegistry<>(OzoneManagerProtocolProtos.Type.class, packageName, + Arrays.stream(VersionExtractor.values()).map(VersionExtractor::getValidatorClass) + .collect(Collectors.toSet()), REQUEST_PROCESSING_PHASES); + return registry.validationsFor(requestType, phase, requestVersions.entrySet().stream() + .collect(Collectors.toMap(e -> e.getKey().getValidatorClass(), Map.Entry::getValue))); + } + @Test public void testNoValidatorsReturnedForEmptyConditionList() { - ValidatorRegistry registry = new ValidatorRegistry(PACKAGE); - List validators = - registry.validationsFor(emptyList(), CreateKey, PRE_PROCESS); - + List validators = getValidatorsForRequest(CreateKey, PRE_PROCESS, + CLIENT_VERSION_EXTRACTOR, ClientVersion.CURRENT); assertTrue(validators.isEmpty()); } @Test public void testRegistryHasThePreFinalizePreProcessCreateKeyValidator() { - ValidatorRegistry registry = new ValidatorRegistry(PACKAGE); - List validators = - registry.validationsFor( - asList(CLUSTER_NEEDS_FINALIZATION), CreateKey, PRE_PROCESS); + List validators = getValidatorsForRequest(CreateKey, PRE_PROCESS, + ImmutableMap.of(CLIENT_VERSION_EXTRACTOR, ClientVersion.CURRENT, + LAYOUT_VERSION_EXTRACTOR, OMLayoutFeature.FILESYSTEM_SNAPSHOT)); assertEquals(1, validators.size()); - String expectedMethodName = "preFinalizePreProcessCreateKeyValidator"; + String expectedMethodName = "preProcessCreateKeyQuotaLayoutValidator"; assertEquals(expectedMethodName, validators.get(0).getName()); } @Test public void testRegistryHasThePreFinalizePostProcessCreateKeyValidator() { - ValidatorRegistry registry = new ValidatorRegistry(PACKAGE); - List validators = - registry.validationsFor( - asList(CLUSTER_NEEDS_FINALIZATION), CreateKey, POST_PROCESS); + List validators = getValidatorsForRequest(CreateKey, POST_PROCESS, + ImmutableMap.of(CLIENT_VERSION_EXTRACTOR, ClientVersion.CURRENT, + LAYOUT_VERSION_EXTRACTOR, OMLayoutFeature.BUCKET_LAYOUT_SUPPORT)); assertEquals(1, validators.size()); - String expectedMethodName = "preFinalizePostProcessCreateKeyValidator"; + String expectedMethodName = "postProcessCreateKeyQuotaLayoutValidator"; assertEquals(expectedMethodName, validators.get(0).getName()); } @Test public void testRegistryHasTheOldClientPreProcessCreateKeyValidator() { - ValidatorRegistry registry = new ValidatorRegistry(PACKAGE); - List validators = - registry.validationsFor( - asList(OLDER_CLIENT_REQUESTS), CreateKey, PRE_PROCESS); + List validators = getValidatorsForRequest(CreateKey, PRE_PROCESS, CLIENT_VERSION_EXTRACTOR, + ClientVersion.ERASURE_CODING_SUPPORT); assertEquals(2, validators.size()); List methodNames = validators.stream().map(Method::getName).collect(Collectors.toList()); - assertThat(methodNames).contains("oldClientPreProcessCreateKeyValidator"); - assertThat(methodNames).contains("oldClientPreProcessCreateKeyValidator2"); + assertEquals(Arrays.asList("preProcessCreateKeyBucketLayoutClientValidator", + "preProcessCreateKeyBucketLayoutClientValidator"), methodNames); } @Test public void testRegistryHasTheOldClientPostProcessCreateKeyValidator() { - ValidatorRegistry registry = new ValidatorRegistry(PACKAGE); - List validators = - registry.validationsFor( - asList(OLDER_CLIENT_REQUESTS), CreateKey, POST_PROCESS); + List validators = getValidatorsForRequest(CreateKey, POST_PROCESS, + CLIENT_VERSION_EXTRACTOR, ClientVersion.ERASURE_CODING_SUPPORT); - assertEquals(2, validators.size()); + assertEquals(1, validators.size()); List methodNames = validators.stream().map(Method::getName).collect(Collectors.toList()); - assertThat(methodNames).contains("oldClientPostProcessCreateKeyValidator"); - assertThat(methodNames).contains("oldClientPostProcessCreateKeyValidator2"); + assertThat(methodNames).contains("postProcessCreateKeyBucketLayoutClientValidator"); } @Test public void testRegistryHasTheMultiPurposePreProcessCreateVolumeValidator() { - ValidatorRegistry registry = new ValidatorRegistry(PACKAGE); - List preFinalizeValidators = - registry.validationsFor( - asList(CLUSTER_NEEDS_FINALIZATION), CreateVolume, PRE_PROCESS); - List newClientValidators = - registry.validationsFor( - asList(OLDER_CLIENT_REQUESTS), CreateVolume, PRE_PROCESS); + List preFinalizeValidators = getValidatorsForRequest(CreateVolume, PRE_PROCESS, LAYOUT_VERSION_EXTRACTOR, + OMLayoutFeature.HSYNC); + List newClientValidators = getValidatorsForRequest(CreateVolume, PRE_PROCESS, CLIENT_VERSION_EXTRACTOR, + ClientVersion.ERASURE_CODING_SUPPORT); assertEquals(1, preFinalizeValidators.size()); assertEquals(1, newClientValidators.size()); - String expectedMethodName = "multiPurposePreProcessCreateVolumeValidator"; + String expectedMethodName = "multiPurposePreProcessCreateVolumeBucketLayoutCLientQuotaLayoutValidator"; assertEquals(expectedMethodName, preFinalizeValidators.get(0).getName()); assertEquals(expectedMethodName, newClientValidators.get(0).getName()); } @Test public void testRegistryHasTheMultiPurposePostProcessCreateVolumeValidator() { - ValidatorRegistry registry = new ValidatorRegistry(PACKAGE); - List preFinalizeValidators = - registry.validationsFor( - asList(CLUSTER_NEEDS_FINALIZATION), CreateVolume, POST_PROCESS); - List oldClientValidators = - registry.validationsFor( - asList(OLDER_CLIENT_REQUESTS), CreateVolume, POST_PROCESS); + List preFinalizeValidators = getValidatorsForRequest(CreateVolume, POST_PROCESS, LAYOUT_VERSION_EXTRACTOR, + OMLayoutFeature.HSYNC); + List oldClientValidators = getValidatorsForRequest(CreateVolume, POST_PROCESS, CLIENT_VERSION_EXTRACTOR, + ClientVersion.ERASURE_CODING_SUPPORT); + List combinedValidators = getValidatorsForRequest(CreateVolume, POST_PROCESS, + ImmutableMap.of(LAYOUT_VERSION_EXTRACTOR, OMLayoutFeature.HSYNC, + CLIENT_VERSION_EXTRACTOR, ClientVersion.ERASURE_CODING_SUPPORT)); assertEquals(1, preFinalizeValidators.size()); assertEquals(1, oldClientValidators.size()); - String expectedMethodName = "multiPurposePostProcessCreateVolumeValidator"; + assertEquals(1, combinedValidators.size()); + String expectedMethodName = "multiPurposePostProcessCreateVolumeBucketLayoutCLientQuotaLayoutValidator"; assertEquals(expectedMethodName, preFinalizeValidators.get(0).getName()); assertEquals(expectedMethodName, oldClientValidators.get(0).getName()); + assertEquals(expectedMethodName, combinedValidators.get(0).getName()); } @Test public void testValidatorsAreReturnedForMultiCondition() { - ValidatorRegistry registry = new ValidatorRegistry(PACKAGE); - List validators = - registry.validationsFor( - asList(CLUSTER_NEEDS_FINALIZATION, OLDER_CLIENT_REQUESTS), - CreateKey, POST_PROCESS); + List validators = getValidatorsForRequest(CreateKey, POST_PROCESS, + ImmutableMap.of(CLIENT_VERSION_EXTRACTOR, ClientVersion.ERASURE_CODING_SUPPORT, + LAYOUT_VERSION_EXTRACTOR, OMLayoutFeature.HSYNC)); - assertEquals(3, validators.size()); + assertEquals(2, validators.size()); List methodNames = validators.stream().map(Method::getName).collect(Collectors.toList()); - assertThat(methodNames).contains("preFinalizePostProcessCreateKeyValidator"); - assertThat(methodNames).contains("oldClientPostProcessCreateKeyValidator"); - assertThat(methodNames).contains("oldClientPostProcessCreateKeyValidator2"); + assertThat(methodNames).contains("postProcessCreateKeyQuotaLayoutValidator"); + assertThat(methodNames).contains("postProcessCreateKeyBucketLayoutClientValidator"); } @Test public void testNoValidatorForRequestsAtAllReturnsEmptyList() { - ValidatorRegistry registry = new ValidatorRegistry(PACKAGE_WO_VALIDATORS); - - assertTrue(registry.validationsFor( - asList(OLDER_CLIENT_REQUESTS), CreateKey, PRE_PROCESS).isEmpty()); + assertTrue(getValidatorsForRequest(PACKAGE_WO_VALIDATORS, CreateKey, PRE_PROCESS, + ImmutableMap.of(CLIENT_VERSION_EXTRACTOR, ClientVersion.ERASURE_CODING_SUPPORT, + LAYOUT_VERSION_EXTRACTOR, OMLayoutFeature.HSYNC)).isEmpty()); } @Test @@ -196,18 +214,31 @@ public void testNoValidatorForConditionReturnsEmptyList() for (URL url : urls) { urlsToUse.add(new URL(url, PACKAGE2.replaceAll("\\.", "/"))); } - ValidatorRegistry registry = new ValidatorRegistry(urlsToUse); - - assertTrue(registry.validationsFor( - asList(CLUSTER_NEEDS_FINALIZATION), CreateKey, PRE_PROCESS).isEmpty()); + ValidatorRegistry registry = new ValidatorRegistry<>( + OzoneManagerProtocolProtos.Type.class, urlsToUse, + Arrays.stream(VersionExtractor.values()).map(VersionExtractor::getValidatorClass).collect(Collectors.toSet()), + REQUEST_PROCESSING_PHASES); + + assertTrue(registry.validationsFor(CreateKey, PRE_PROCESS, + ImmutableMap.of(CLIENT_VERSION_EXTRACTOR.getValidatorClass(), ClientVersion.CURRENT, + LAYOUT_VERSION_EXTRACTOR.getValidatorClass(), OMLayoutFeature.BUCKET_LAYOUT_SUPPORT)).isEmpty()); } @Test public void testNoDefinedValidationForRequestReturnsEmptyList() { - ValidatorRegistry registry = new ValidatorRegistry(PACKAGE); + assertTrue(getValidatorsForRequest(CreateDirectory, null, CLIENT_VERSION_EXTRACTOR, + ClientVersion.ERASURE_CODING_SUPPORT).isEmpty()); + } - assertTrue(registry.validationsFor( - asList(OLDER_CLIENT_REQUESTS), CreateDirectory, null).isEmpty()); + @Test + public void testFutureVersionForRequestReturnsOnlyFutureVersionValidators() { + List validators = getValidatorsForRequest(CreateKey, PRE_PROCESS, + CLIENT_VERSION_EXTRACTOR, ClientVersion.FUTURE_VERSION); + + assertEquals(1, validators.size()); + List methodNames = + validators.stream().map(Method::getName).collect(Collectors.toList()); + assertThat(methodNames).contains("preProcessCreateKeyFutureClientValidator"); } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset1/GeneralValidatorsForTesting.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset1/GeneralValidatorsForTesting.java index f0895ac47d7f..18184196b0fa 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset1/GeneralValidatorsForTesting.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset1/GeneralValidatorsForTesting.java @@ -17,8 +17,6 @@ package org.apache.hadoop.ozone.om.request.validation.testvalidatorset1; -import static org.apache.hadoop.ozone.om.request.validation.ValidationCondition.CLUSTER_NEEDS_FINALIZATION; -import static org.apache.hadoop.ozone.om.request.validation.ValidationCondition.OLDER_CLIENT_REQUESTS; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.CreateKey; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.CreateVolume; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.DeleteKeys; @@ -28,9 +26,12 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; -import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; +import org.apache.hadoop.ozone.ClientVersion; +import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; +import org.apache.hadoop.ozone.om.request.validation.OMLayoutVersionValidator; import org.apache.hadoop.ozone.om.request.validation.TestRequestValidations; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; +import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; @@ -88,82 +89,90 @@ private static void fireValidationEvent(String calledMethodName) { listeners.forEach(l -> l.validationCalled(calledMethodName)); } - @RequestFeatureValidator( - conditions = { CLUSTER_NEEDS_FINALIZATION }, + @OMLayoutVersionValidator( + applyBefore = OMLayoutFeature.QUOTA, processingPhase = PRE_PROCESS, requestType = CreateKey) - public static OMRequest preFinalizePreProcessCreateKeyValidator( + public static OMRequest preProcessCreateKeyQuotaLayoutValidator( OMRequest req, ValidationContext ctx) { - fireValidationEvent("preFinalizePreProcessCreateKeyValidator"); + fireValidationEvent("preProcessCreateKeyQuotaLayoutValidator"); return req; } - @RequestFeatureValidator( - conditions = { CLUSTER_NEEDS_FINALIZATION }, + @OMClientVersionValidator( + applyBefore = ClientVersion.FUTURE_VERSION, + processingPhase = PRE_PROCESS, + requestType = CreateKey) + public static OMRequest preProcessCreateKeyFutureClientValidator( + OMRequest req, ValidationContext ctx) { + fireValidationEvent("preProcessCreateKeyFutureClientValidator"); + return req; + } + + @OMLayoutVersionValidator( + applyBefore = OMLayoutFeature.QUOTA, processingPhase = POST_PROCESS, requestType = CreateKey) - public static OMResponse preFinalizePostProcessCreateKeyValidator( + public static OMResponse postProcessCreateKeyQuotaLayoutValidator( OMRequest req, OMResponse resp, ValidationContext ctx) { - fireValidationEvent("preFinalizePostProcessCreateKeyValidator"); + fireValidationEvent("postProcessCreateKeyQuotaLayoutValidator"); return resp; } - @RequestFeatureValidator( - conditions = { OLDER_CLIENT_REQUESTS }, + @OMClientVersionValidator( processingPhase = PRE_PROCESS, - requestType = CreateKey) - public static OMRequest oldClientPreProcessCreateKeyValidator( + requestType = CreateKey, + applyBefore = ClientVersion.BUCKET_LAYOUT_SUPPORT) + public static OMRequest preProcessCreateKeyBucketLayoutClientValidator( OMRequest req, ValidationContext ctx) { - fireValidationEvent("oldClientPreProcessCreateKeyValidator"); + fireValidationEvent("preProcessCreateKeyBucketLayoutClientValidator"); return req; } - @RequestFeatureValidator( - conditions = { OLDER_CLIENT_REQUESTS }, + @OMClientVersionValidator( + applyBefore = ClientVersion.BUCKET_LAYOUT_SUPPORT, processingPhase = POST_PROCESS, requestType = CreateKey) - public static OMResponse oldClientPostProcessCreateKeyValidator( + public static OMResponse postProcessCreateKeyBucketLayoutClientValidator( OMRequest req, OMResponse resp, ValidationContext ctx) { - fireValidationEvent("oldClientPostProcessCreateKeyValidator"); + fireValidationEvent("postProcessCreateKeyBucketLayoutClientValidator"); return resp; } - @RequestFeatureValidator( - conditions = { CLUSTER_NEEDS_FINALIZATION, OLDER_CLIENT_REQUESTS }, + @OMClientVersionValidator( + applyBefore = ClientVersion.BUCKET_LAYOUT_SUPPORT, processingPhase = PRE_PROCESS, requestType = CreateVolume) - public static OMRequest multiPurposePreProcessCreateVolumeValidator( + @OMLayoutVersionValidator( + applyBefore = OMLayoutFeature.QUOTA, + processingPhase = PRE_PROCESS, + requestType = CreateVolume) + public static OMRequest multiPurposePreProcessCreateVolumeBucketLayoutCLientQuotaLayoutValidator( OMRequest req, ValidationContext ctx) { - fireValidationEvent("multiPurposePreProcessCreateVolumeValidator"); + fireValidationEvent("multiPurposePreProcessCreateVolumeBucketLayoutCLientQuotaLayoutValidator"); return req; } - @RequestFeatureValidator( - conditions = { OLDER_CLIENT_REQUESTS, CLUSTER_NEEDS_FINALIZATION }, + @OMClientVersionValidator( + applyBefore = ClientVersion.BUCKET_LAYOUT_SUPPORT, processingPhase = POST_PROCESS, requestType = CreateVolume) - public static OMResponse multiPurposePostProcessCreateVolumeValidator( - OMRequest req, OMResponse resp, ValidationContext ctx) { - fireValidationEvent("multiPurposePostProcessCreateVolumeValidator"); - return resp; - } - - @RequestFeatureValidator( - conditions = { OLDER_CLIENT_REQUESTS }, + @OMLayoutVersionValidator( + applyBefore = OMLayoutFeature.QUOTA, processingPhase = POST_PROCESS, - requestType = CreateKey) - public static OMResponse oldClientPostProcessCreateKeyValidator2( + requestType = CreateVolume) + public static OMResponse multiPurposePostProcessCreateVolumeBucketLayoutCLientQuotaLayoutValidator( OMRequest req, OMResponse resp, ValidationContext ctx) { - fireValidationEvent("oldClientPostProcessCreateKeyValidator2"); + fireValidationEvent("multiPurposePostProcessCreateVolumeBucketLayoutCLientQuotaLayoutValidator"); return resp; } - @RequestFeatureValidator( - conditions = {OLDER_CLIENT_REQUESTS}, + @OMClientVersionValidator( + applyBefore = ClientVersion.BUCKET_LAYOUT_SUPPORT, processingPhase = PRE_PROCESS, requestType = DeleteKeys ) - public static OMRequest throwingPreProcessValidator( + public static OMRequest throwingPreProcessDeleteKeyBucketLayoutClientValidator( OMRequest req, ValidationContext ctx) throws IOException { fireValidationEvent("throwingPreProcessValidator"); if (validatorTestsRunning) { @@ -172,12 +181,12 @@ public static OMRequest throwingPreProcessValidator( return req; } - @RequestFeatureValidator( - conditions = {OLDER_CLIENT_REQUESTS}, + @OMClientVersionValidator( + applyBefore = ClientVersion.BUCKET_LAYOUT_SUPPORT, processingPhase = POST_PROCESS, requestType = DeleteKeys ) - public static OMResponse throwingPostProcessValidator( + public static OMResponse throwingPostProcessDeleteKeyBucketLayoutClientValidator( OMRequest req, OMResponse resp, ValidationContext ctx) throws IOException { fireValidationEvent("throwingPostProcessValidator"); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset2/ValidatorsForOnlyOldClientValidations.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset2/ValidatorsForOnlyOldClientValidations.java index 52faefe57a10..15bfa00ed37b 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset2/ValidatorsForOnlyOldClientValidations.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset2/ValidatorsForOnlyOldClientValidations.java @@ -17,11 +17,11 @@ package org.apache.hadoop.ozone.om.request.validation.testvalidatorset2; -import static org.apache.hadoop.ozone.om.request.validation.ValidationCondition.OLDER_CLIENT_REQUESTS; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.CreateKey; import static org.apache.hadoop.ozone.request.validation.RequestProcessingPhase.PRE_PROCESS; -import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; +import org.apache.hadoop.ozone.ClientVersion; +import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; @@ -33,11 +33,11 @@ public final class ValidatorsForOnlyOldClientValidations { private ValidatorsForOnlyOldClientValidations() { } - @RequestFeatureValidator( - conditions = { OLDER_CLIENT_REQUESTS }, + @OMClientVersionValidator( + applyBefore = ClientVersion.BUCKET_LAYOUT_SUPPORT, processingPhase = PRE_PROCESS, requestType = CreateKey) - public static OMRequest oldClientPreProcessCreateKeyValidator2( + public static OMRequest preProcessCreateKeyBucketLayoutClientValidator( OMRequest req, ValidationContext ctx) { return req; }