From bde95f71a58aad4814c8313cdd9f368cdbf79038 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Thu, 11 Jul 2024 21:15:24 -0700 Subject: [PATCH 01/28] HDDS-11149. Add a maximum client version in the RequestFeatureValidator to validate clients older than the specified version Change-Id: Ifcbb239994e7357032934d104ab18063c8d64251 --- .../RequestFeatureValidatorProcessor.java | 48 +++++++-- .../request/bucket/OMBucketCreateRequest.java | 5 +- .../request/bucket/OMBucketDeleteRequest.java | 7 +- .../file/OMDirectoryCreateRequest.java | 6 +- .../om/request/file/OMFileCreateRequest.java | 6 +- .../request/key/OMAllocateBlockRequest.java | 6 +- .../om/request/key/OMKeyCommitRequest.java | 6 +- .../om/request/key/OMKeyCreateRequest.java | 6 +- .../om/request/key/OMKeyDeleteRequest.java | 7 +- .../om/request/key/OMKeyRenameRequest.java | 7 +- .../om/request/key/OMKeysDeleteRequest.java | 7 +- .../om/request/key/OMKeysRenameRequest.java | 7 +- .../request/key/acl/OMKeyAddAclRequest.java | 7 +- .../key/acl/OMKeyRemoveAclRequest.java | 7 +- .../request/key/acl/OMKeySetAclRequest.java | 7 +- .../S3InitiateMultipartUploadRequest.java | 6 +- .../S3MultipartUploadAbortRequest.java | 6 +- .../S3MultipartUploadCommitPartRequest.java | 6 +- .../S3MultipartUploadCompleteRequest.java | 6 +- .../validation/RequestFeatureValidator.java | 12 +++ .../validation/RequestValidations.java | 4 +- .../validation/ValidationCondition.java | 1 + .../request/validation/ValidatorRegistry.java | 102 +++++++++++++++--- .../OzoneManagerRequestHandler.java | 62 ++++++----- .../TestRequestFeatureValidatorProcessor.java | 50 +++++++-- .../validation/TestValidatorRegistry.java | 42 ++++---- .../GeneralValidatorsForTesting.java | 37 ++++--- ...ValidatorsForOnlyOldClientValidations.java | 7 +- 28 files changed, 341 insertions(+), 139 deletions(-) diff --git a/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/RequestFeatureValidatorProcessor.java b/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/RequestFeatureValidatorProcessor.java index 830706e9a540..45538f5fa374 100644 --- a/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/RequestFeatureValidatorProcessor.java +++ b/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/RequestFeatureValidatorProcessor.java @@ -28,6 +28,7 @@ import javax.lang.model.element.ElementKind; import javax.lang.model.element.ExecutableElement; import javax.lang.model.element.Modifier; +import javax.lang.model.element.Name; import javax.lang.model.element.TypeElement; import javax.lang.model.element.VariableElement; import javax.lang.model.type.ExecutableType; @@ -56,8 +57,8 @@ public class RequestFeatureValidatorProcessor extends AbstractProcessor { public static final String ERROR_CONDITION_IS_EMPTY = - "RequestFeatureValidator has an empty condition list. Please define the" - + " ValidationCondition in which the validator has to be applied."; + "RequestFeatureValidator has both an empty condition list & undefined max client version. Please define either" + + " ValidationCondition or max client version in which the validator has to be applied."; public static final String ERROR_ANNOTATED_ELEMENT_IS_NOT_A_METHOD = "RequestFeatureValidator annotation is not applied to a method."; public static final String ERROR_VALIDATOR_METHOD_HAS_TO_BE_STATIC = @@ -94,12 +95,15 @@ public class RequestFeatureValidatorProcessor extends AbstractProcessor { public static final String ANNOTATION_CONDITIONS_PROPERTY_NAME = "conditions"; public static final String ANNOTATION_PROCESSING_PHASE_PROPERTY_NAME = "processingPhase"; + public static final String ANNOTATION_MAX_CLIENT_VERSION_PROPERTY_NAME = "maxClientVersion"; public static final String PROCESSING_PHASE_PRE_PROCESS = "PRE_PROCESS"; public static final String PROCESSING_PHASE_POST_PROCESS = "POST_PROCESS"; public static final String ERROR_NO_PROCESSING_PHASE_DEFINED = "RequestFeatureValidator has an invalid ProcessingPhase defined."; + public static final String MAX_CLIENT_VERSION_FUTURE_VERSION = "FUTURE_VERSION"; + @Override public boolean process(Set annotations, RoundEnvironment roundEnv) { @@ -199,16 +203,22 @@ private void checkMethodIsAnnotated(Element elem) { private boolean checkAndEvaluateAnnotation( AnnotationMirror methodAnnotation) { boolean isPreprocessor = false; + boolean hasInvalidConditions = false; + boolean hasValidMaxClientVersions = false; for (Entry entry : methodAnnotation.getElementValues().entrySet()) { - - if (hasInvalidValidationCondition(entry)) { - emitErrorMsg(ERROR_CONDITION_IS_EMPTY); - } + + hasInvalidConditions = hasInvalidConditions || hasInvalidValidationCondition(entry); + if (isProcessingPhaseValue(entry)) { isPreprocessor = evaluateProcessingPhase(entry); } + hasValidMaxClientVersions = hasValidMaxClientVersions || hasValidMaxClientVersion(entry); + } + if (!hasValidMaxClientVersions && hasInvalidConditions) { + emitErrorMsg(ERROR_CONDITION_IS_EMPTY); } + return isPreprocessor; } @@ -223,11 +233,25 @@ private boolean evaluateProcessingPhase( return false; } + private boolean hasValidMaxClientVersion( + Entry entry) { + if (isPropertyNamedAs(entry, ANNOTATION_MAX_CLIENT_VERSION_PROPERTY_NAME)) { + Name maxClientVersion = visit(entry, new MaxClientVersionVisitor()); + return !maxClientVersion.contentEquals(MAX_CLIENT_VERSION_FUTURE_VERSION); + } + return false; + } + private boolean isProcessingPhaseValue( Entry entry) { return isPropertyNamedAs(entry, ANNOTATION_PROCESSING_PHASE_PROPERTY_NAME); } + private boolean isMaxClientVersionValue( + Entry entry) { + return isPropertyNamedAs(entry, ANNOTATION_MAX_CLIENT_VERSION_PROPERTY_NAME); + } + private boolean hasInvalidValidationCondition( Entry entry) { return isPropertyNamedAs(entry, ANNOTATION_CONDITIONS_PROPERTY_NAME) @@ -264,6 +288,18 @@ public Boolean visitArray(List vals, } + private static class MaxClientVersionVisitor + extends SimpleAnnotationValueVisitor8 { + + MaxClientVersionVisitor() { + } + + @Override + public Name visitEnumConstant(VariableElement c, Void unused) { + return c.getSimpleName(); + } + } + private static class ProcessingPhaseVisitor extends SimpleAnnotationValueVisitor8 { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java index f2c343e0d161..f9e63cb182d8 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java @@ -460,9 +460,10 @@ public static OMRequest handleCreateBucketWithBucketLayoutDuringPreFinalize( * they do not understand. */ @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + conditions = {}, processingPhase = RequestProcessingPhase.PRE_PROCESS, - requestType = Type.CreateBucket + requestType = Type.CreateBucket, + maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest setDefaultBucketLayoutForOlderClients(OMRequest req, ValidationContext ctx) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketDeleteRequest.java index 48a3c807b9a3..74b94948ce40 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketDeleteRequest.java @@ -23,6 +23,7 @@ import java.util.Iterator; import java.util.Map; +import org.apache.hadoop.ozone.ClientVersion; import org.apache.ratis.server.protocol.TermIndex; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.TableIterator; @@ -34,7 +35,6 @@ import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; -import org.apache.hadoop.ozone.om.request.validation.ValidationCondition; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -281,9 +281,10 @@ private boolean bucketContainsSnapshotInCache( * @throws OMException if the request is invalid */ @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + conditions = {}, processingPhase = RequestProcessingPhase.PRE_PROCESS, - requestType = Type.DeleteBucket + requestType = Type.DeleteBucket, + maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockBucketDeleteWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java index 6071110a315d..eba409448658 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java @@ -31,6 +31,7 @@ import org.apache.hadoop.hdds.client.ECReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.ozone.ClientVersion; import org.apache.ratis.server.protocol.TermIndex; import org.apache.hadoop.ozone.OmUtils; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; @@ -432,9 +433,10 @@ public static OMRequest disallowCreateDirectoryWithECReplicationConfig( * @throws OMException if the request is invalid */ @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + conditions = {}, processingPhase = RequestProcessingPhase.PRE_PROCESS, - requestType = Type.CreateDirectory + requestType = Type.CreateDirectory, + maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockCreateDirectoryWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java index 9b9fb4e7cc5c..a07f0cd057e4 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java @@ -29,6 +29,7 @@ import com.google.common.base.Preconditions; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hdds.client.ReplicationConfig; +import org.apache.hadoop.ozone.ClientVersion; import org.apache.ratis.server.protocol.TermIndex; import org.apache.hadoop.ozone.OmUtils; import org.apache.hadoop.ozone.OzoneConsts; @@ -425,9 +426,10 @@ public static OMRequest disallowCreateFileWithECReplicationConfig( * @throws OMException if the request is invalid */ @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + conditions = {}, processingPhase = RequestProcessingPhase.PRE_PROCESS, - requestType = Type.CreateFile + requestType = Type.CreateFile, + maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockCreateFileWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java index ac4d9ab6244f..f02cece6f9fe 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java @@ -26,6 +26,7 @@ import com.google.common.base.Preconditions; import org.apache.hadoop.hdds.client.ReplicationConfig; +import org.apache.hadoop.ozone.ClientVersion; import org.apache.ratis.server.protocol.TermIndex; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; @@ -298,9 +299,10 @@ public static OMRequest disallowAllocateBlockWithECReplicationConfig( * @throws OMException if the request is invalid */ @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + conditions = {}, processingPhase = RequestProcessingPhase.PRE_PROCESS, - requestType = Type.AllocateBlock + requestType = Type.AllocateBlock, + maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockAllocateBlockWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java index 7d31422c80e3..f2964d573b02 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java @@ -28,6 +28,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.ozone.ClientVersion; import org.apache.ratis.server.protocol.TermIndex; import org.apache.hadoop.ozone.OmUtils; import org.apache.hadoop.ozone.OzoneConfigKeys; @@ -463,9 +464,10 @@ public static OMRequest disallowCommitKeyWithECReplicationConfig( * @throws OMException if the request is invalid */ @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + conditions = {}, processingPhase = RequestProcessingPhase.PRE_PROCESS, - requestType = Type.CommitKey + requestType = Type.CommitKey, + maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockCommitKeyWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java index 36f2a8c31be1..2e90b5e5e6f4 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java @@ -28,6 +28,7 @@ import com.google.common.base.Preconditions; import org.apache.hadoop.hdds.client.ReplicationConfig; +import org.apache.hadoop.ozone.ClientVersion; import org.apache.ratis.server.protocol.TermIndex; import org.apache.hadoop.ozone.OmUtils; import org.apache.hadoop.ozone.om.OMConfigKeys; @@ -424,9 +425,10 @@ public static OMRequest disallowCreateKeyWithECReplicationConfig( * @throws OMException if the request is invalid */ @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + conditions = {}, processingPhase = RequestProcessingPhase.PRE_PROCESS, - requestType = Type.CreateKey + requestType = Type.CreateKey, + maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockCreateKeyWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java index 61e5976f8052..f0a53d9cd6bc 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java @@ -23,6 +23,7 @@ import java.util.Map; import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.ozone.ClientVersion; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.ratis.server.protocol.TermIndex; import org.apache.hadoop.ozone.OmUtils; @@ -31,7 +32,6 @@ import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; -import org.apache.hadoop.ozone.om.request.validation.ValidationCondition; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType; import org.slf4j.Logger; @@ -233,9 +233,10 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn * @throws OMException if the request is invalid */ @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + conditions = {}, processingPhase = RequestProcessingPhase.PRE_PROCESS, - requestType = Type.DeleteKey + requestType = Type.DeleteKey, + maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockDeleteKeyWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java index db08951831cb..c11b30412487 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java @@ -23,6 +23,7 @@ import java.util.Map; import com.google.common.base.Preconditions; +import org.apache.hadoop.ozone.ClientVersion; import org.apache.ratis.server.protocol.TermIndex; import org.apache.hadoop.ozone.OmUtils; import org.apache.hadoop.ozone.OzoneConsts; @@ -31,7 +32,6 @@ import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; -import org.apache.hadoop.ozone.om.request.validation.ValidationCondition; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer; import org.apache.hadoop.ozone.security.acl.OzoneObj; @@ -270,9 +270,10 @@ private Map buildAuditMap( * @throws OMException if the request is invalid */ @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + conditions = {}, processingPhase = RequestProcessingPhase.PRE_PROCESS, - requestType = Type.RenameKey + requestType = Type.RenameKey, + maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockRenameKeyWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java index 19e9ed716e4a..342b0c633364 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java @@ -20,6 +20,7 @@ import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.ozone.ClientVersion; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.ratis.server.protocol.TermIndex; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; @@ -38,7 +39,6 @@ import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; -import org.apache.hadoop.ozone.om.request.validation.ValidationCondition; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.om.response.key.OMKeysDeleteResponse; @@ -351,9 +351,10 @@ protected static void addDeletedKeys(Map auditMap, * @throws OMException if the request is invalid */ @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + conditions = {}, processingPhase = RequestProcessingPhase.PRE_PROCESS, - requestType = Type.DeleteKeys + requestType = Type.DeleteKeys, + maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockDeleteKeysWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysRenameRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysRenameRequest.java index 0b93dd57946e..1ffa1650782c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysRenameRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysRenameRequest.java @@ -19,6 +19,7 @@ package org.apache.hadoop.ozone.om.request.key; import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.ozone.ClientVersion; import org.apache.ratis.server.protocol.TermIndex; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; @@ -36,7 +37,6 @@ import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; -import org.apache.hadoop.ozone.om.request.validation.ValidationCondition; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.om.response.key.OMKeysRenameResponse; @@ -297,9 +297,10 @@ private Map buildAuditMap(Map auditMap, * @throws OMException if the request is invalid */ @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + conditions = {}, processingPhase = RequestProcessingPhase.PRE_PROCESS, - requestType = Type.RenameKeys + requestType = Type.RenameKeys, + maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockRenameKeysWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAddAclRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAddAclRequest.java index 31fa245fa55b..0fd43a657ad8 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAddAclRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAddAclRequest.java @@ -23,6 +23,7 @@ import java.util.Map; import com.google.common.collect.Lists; +import org.apache.hadoop.ozone.ClientVersion; import org.apache.ratis.server.protocol.TermIndex; import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.OzoneConsts; @@ -36,7 +37,6 @@ import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; -import org.apache.hadoop.ozone.om.request.validation.ValidationCondition; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.response.key.acl.OMKeyAclResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; @@ -165,9 +165,10 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn * @throws OMException if the request is invalid */ @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + conditions = {}, processingPhase = RequestProcessingPhase.PRE_PROCESS, - requestType = Type.AddAcl + requestType = Type.AddAcl, + maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockAddAclWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyRemoveAclRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyRemoveAclRequest.java index 7e4fc04eb39c..3618c8fae198 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyRemoveAclRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyRemoveAclRequest.java @@ -23,6 +23,7 @@ import java.util.Map; import com.google.common.collect.Lists; +import org.apache.hadoop.ozone.ClientVersion; import org.apache.ratis.server.protocol.TermIndex; import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.OzoneConsts; @@ -36,7 +37,6 @@ import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; -import org.apache.hadoop.ozone.om.request.validation.ValidationCondition; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.response.key.acl.OMKeyAclResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; @@ -166,9 +166,10 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn * @throws OMException if the request is invalid */ @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + conditions = {}, processingPhase = RequestProcessingPhase.PRE_PROCESS, - requestType = Type.RemoveAcl + requestType = Type.RemoveAcl, + maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockRemoveAclWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeySetAclRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeySetAclRequest.java index 0e5e1cf3362a..6abca95b7e3b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeySetAclRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeySetAclRequest.java @@ -23,6 +23,7 @@ import java.util.Map; import com.google.common.collect.Lists; +import org.apache.hadoop.ozone.ClientVersion; import org.apache.ratis.server.protocol.TermIndex; import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.OzoneConsts; @@ -37,7 +38,6 @@ import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; -import org.apache.hadoop.ozone.om.request.validation.ValidationCondition; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.response.key.acl.OMKeyAclResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; @@ -162,9 +162,10 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn * @throws OMException if the request is invalid */ @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + conditions = {}, processingPhase = RequestProcessingPhase.PRE_PROCESS, - requestType = Type.SetAcl + requestType = Type.SetAcl, + maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockSetAclWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java index f16ef9f8f423..5afa5958ef29 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java @@ -20,6 +20,7 @@ import com.google.common.base.Preconditions; import org.apache.hadoop.hdds.client.ReplicationConfig; +import org.apache.hadoop.ozone.ClientVersion; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.om.helpers.KeyValueUtil; import org.apache.ratis.server.protocol.TermIndex; @@ -320,9 +321,10 @@ protected void logResult(OzoneManager ozoneManager, * @throws OMException if the request is invalid */ @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + conditions = {}, processingPhase = RequestProcessingPhase.PRE_PROCESS, - requestType = Type.InitiateMultiPartUpload + requestType = Type.InitiateMultiPartUpload, + maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockInitiateMPUWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java index 400291d7a448..a688a7f44b50 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java @@ -22,6 +22,7 @@ import java.nio.file.InvalidPathException; import java.util.Map; +import org.apache.hadoop.ozone.ClientVersion; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.ratis.server.protocol.TermIndex; import org.apache.hadoop.ozone.om.helpers.BucketLayout; @@ -289,9 +290,10 @@ public static OMRequest disallowAbortMultiPartUploadWithECReplicationConfig( * @throws OMException if the request is invalid */ @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + conditions = {}, processingPhase = RequestProcessingPhase.PRE_PROCESS, - requestType = Type.AbortMultiPartUpload + requestType = Type.AbortMultiPartUpload, + maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockMPUAbortWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java index 1123b8dba7a6..0a619d54c144 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java @@ -19,6 +19,7 @@ package org.apache.hadoop.ozone.om.request.s3.multipart; import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.ozone.ClientVersion; import org.apache.ratis.server.protocol.TermIndex; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.audit.OMAction; @@ -380,9 +381,10 @@ public static OMRequest disallowCommitMultiPartUploadWithECReplicationConfig( * @throws OMException if the request is invalid */ @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + conditions = {}, processingPhase = RequestProcessingPhase.PRE_PROCESS, - requestType = Type.CommitMultiPartUpload + requestType = Type.CommitMultiPartUpload, + maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockMPUCommitWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java index 1dab110e783a..873bf51339da 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java @@ -32,6 +32,7 @@ import java.util.function.BiFunction; import org.apache.hadoop.hdds.client.ReplicationConfig; +import org.apache.hadoop.ozone.ClientVersion; import org.apache.hadoop.ozone.om.OzoneConfigUtil; import org.apache.hadoop.ozone.om.request.file.OMDirectoryCreateRequestWithFSO; import org.apache.hadoop.ozone.om.request.file.OMFileRequest; @@ -742,9 +743,10 @@ private void updateCache(OMMetadataManager omMetadataManager, * @throws OMException if the request is invalid */ @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + conditions = {}, processingPhase = RequestProcessingPhase.PRE_PROCESS, - requestType = Type.CompleteMultiPartUpload + requestType = Type.CompleteMultiPartUpload, + maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockMPUCompleteWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestFeatureValidator.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestFeatureValidator.java index 5f1f08266b85..a960f23b90da 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestFeatureValidator.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestFeatureValidator.java @@ -16,6 +16,7 @@ */ package org.apache.hadoop.ozone.om.request.validation; +import org.apache.hadoop.ozone.ClientVersion; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; import java.lang.annotation.ElementType; @@ -96,4 +97,15 @@ */ Type requestType(); + + /** + * The max client version for which the validator would run. This condition happens in parallel with the specified + * validation conditions. The validator would run either one of the specified conditions apply or the request + * client version is equal to or older than the specified version. The value defaults to FUTURE_VERSION so the + * validator would never run for any of the clients, the validator could still run based on the specified + * conditions though. + * @returns the max required client version for which the validator runs for older clients. + */ + ClientVersion maxClientVersion() default ClientVersion.FUTURE_VERSION; + } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestValidations.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestValidations.java index 248505cd472e..aa3a6a7efda1 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestValidations.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestValidations.java @@ -63,7 +63,7 @@ public synchronized RequestValidations load() { public OMRequest validateRequest(OMRequest request) throws Exception { List validations = registry.validationsFor( - conditions(request), request.getCmdType(), PRE_PROCESS); + conditions(request), request.getCmdType(), PRE_PROCESS, request.getVersion()); OMRequest validatedRequest = request.toBuilder().build(); try { @@ -88,7 +88,7 @@ public OMRequest validateRequest(OMRequest request) public OMResponse validateResponse(OMRequest request, OMResponse response) throws ServiceException { List validations = registry.validationsFor( - conditions(request), request.getCmdType(), POST_PROCESS); + conditions(request), request.getCmdType(), POST_PROCESS, request.getVersion()); OMResponse validatedResponse = response.toBuilder().build(); try { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidationCondition.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidationCondition.java index 9630500cbd53..58eacff7771e 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidationCondition.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidationCondition.java @@ -44,6 +44,7 @@ public boolean shouldApply(OMRequest req, ValidationContext ctx) { * Classifies validations that has to run, when the client uses an older * protocol version than the server. */ + @Deprecated OLDER_CLIENT_REQUESTS { @Override public boolean shouldApply(OMRequest req, ValidationContext ctx) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidatorRegistry.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidatorRegistry.java index 610949e0f8a4..4c8053b4b471 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidatorRegistry.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidatorRegistry.java @@ -16,6 +16,8 @@ */ package org.apache.hadoop.ozone.om.request.validation; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.ozone.ClientVersion; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; import org.reflections.Reflections; import org.reflections.scanners.MethodAnnotationsScanner; @@ -28,10 +30,15 @@ import java.util.Collection; import java.util.Collections; import java.util.EnumMap; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; +import java.util.TreeMap; +import java.util.function.Supplier; +import java.util.stream.Collectors; import static org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase.POST_PROCESS; import static org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase.PRE_PROCESS; @@ -45,6 +52,9 @@ public class ValidatorRegistry { private final EnumMap>>> validators = new EnumMap<>(ValidationCondition.class); + private final Map, Pair, TreeMap>> + maxAllowedVersionValidatorMap = new HashMap<>(Type.values().length * RequestProcessingPhase.values().length, + 1.0f); /** * Creates a {@link ValidatorRegistry} instance that discovers validation @@ -87,15 +97,60 @@ public class ValidatorRegistry { * @param conditions conditions that are present for the request * @param requestType the type of the protocol message * @param phase the request processing phase + * @param requestClientVersion the client version of the protocol message. * @return the list of validation methods that has to run. */ - List validationsFor( + List validationsFor(List conditions, + Type requestType, + RequestProcessingPhase phase, + int requestClientVersion) { + Set methodsToRun = new HashSet<>(validationsFor(conditions, requestType, phase)); + methodsToRun.addAll(validationsFor(requestType, phase, requestClientVersion)); + return new ArrayList<>(methodsToRun); + } + + /** + * Get the validators that has to run + * that require {@Link ClientVersion}. minimum client version newer than the request client version, for the given + * requestType and {@link RequestProcessingPhase}. + * + * @param requestType the type of the protocol message + * @param phase the request processing phase + * @param requestClientVersion the client version of the protocol message. + * @return the list of validation methods that has to run. + */ + private List validationsFor(Type requestType, + RequestProcessingPhase phase, + int requestClientVersion) { + Pair key = Pair.of(requestType, phase); + if (this.maxAllowedVersionValidatorMap.containsKey(key)) { + Pair, TreeMap> value = this.maxAllowedVersionValidatorMap.get(key); + return Optional.ofNullable(value.getRight().ceilingEntry(requestClientVersion)) + .map(Map.Entry::getValue) + .map(startIndex -> value.getKey().subList(startIndex, value.getKey().size())) + .orElse(Collections.emptyList()); + } + return Collections.emptyList(); + } + + + /** + * Get the validators that has to be run in the given list of + * {@link ValidationCondition}s, for the given requestType and + * {@link RequestProcessingPhase}. + * + * @param conditions conditions that are present for the request + * @param requestType the type of the protocol message + * @param phase the request processing phase + * @return the list of validation methods that has to run. + */ + private Set validationsFor( List conditions, Type requestType, RequestProcessingPhase phase) { if (conditions.isEmpty() || validators.isEmpty()) { - return Collections.emptyList(); + return Collections.emptySet(); } Set returnValue = new HashSet<>(); @@ -103,7 +158,7 @@ List validationsFor( for (ValidationCondition condition: conditions) { returnValue.addAll(validationsFor(condition, requestType, phase)); } - return new ArrayList<>(returnValue); + return returnValue; } /** @@ -138,6 +193,13 @@ private List validationsFor( return validatorsForPhase; } + private int getMaxAllowedClientVersion(Method method) { + RequestFeatureValidator descriptor = method.getAnnotation(RequestFeatureValidator.class); + ClientVersion maxAllowedClientVersion = descriptor.maxClientVersion(); + return maxAllowedClientVersion == null ? (ClientVersion.DEFAULT_VERSION.toProtoValue() - 1) : + maxAllowedClientVersion.toProtoValue(); + } + /** * Initializes the internal request validator store. * The requests are stored in the following structure: @@ -149,23 +211,34 @@ private List validationsFor( * @param describedValidators collection of the annotated methods to process. */ void initMaps(Collection describedValidators) { - for (Method m : describedValidators) { - RequestFeatureValidator descriptor = - m.getAnnotation(RequestFeatureValidator.class); + List sortedMethodsByMaxVersion = describedValidators.stream() + .sorted((method1, method2) -> Integer.compare(getMaxAllowedClientVersion(method1), + getMaxAllowedClientVersion(method2))) + .collect(Collectors.toList()); + for (Method m : sortedMethodsByMaxVersion) { + RequestFeatureValidator descriptor = m.getAnnotation(RequestFeatureValidator.class); m.setAccessible(true); + Type requestType = descriptor.requestType(); for (ValidationCondition condition : descriptor.conditions()) { EnumMap>> requestTypeMap = getAndInitialize( - condition, newTypeMap(), validators); + condition, this::newTypeMap, validators); EnumMap> phases = getAndInitialize( - descriptor.requestType(), newPhaseMap(), requestTypeMap); + requestType, this::newPhaseMap, requestTypeMap); if (isPreProcessValidator(descriptor)) { - getAndInitialize(PRE_PROCESS, new ArrayList<>(), phases).add(m); + getAndInitialize(PRE_PROCESS, ArrayList::new, phases).add(m); } else if (isPostProcessValidator(descriptor)) { - getAndInitialize(POST_PROCESS, new ArrayList<>(), phases).add(m); + getAndInitialize(POST_PROCESS, ArrayList::new, phases).add(m); } } + Pair indexKey = Pair.of(descriptor.requestType(), descriptor.processingPhase()); + int maxAllowedClientVersion = this.getMaxAllowedClientVersion(m); + Pair, TreeMap> value = + getAndInitialize(indexKey, () -> Pair.of(new ArrayList<>(), new TreeMap<>()), maxAllowedVersionValidatorMap); + List methods = value.getKey(); + value.getRight().putIfAbsent(maxAllowedClientVersion, methods.size()); + methods.add(m); } } @@ -178,13 +251,8 @@ private EnumMap> newPhaseMap() { return new EnumMap<>(RequestProcessingPhase.class); } - private V getAndInitialize(K key, V defaultValue, Map from) { - V inMapValue = from.get(key); - if (inMapValue == null || !from.containsKey(key)) { - from.put(key, defaultValue); - return defaultValue; - } - return inMapValue; + private V getAndInitialize(K key, Supplier defaultValue, Map from) { + return from.compute(key, (k, v) -> v == null ? defaultValue.get() : v); } private boolean isPreProcessValidator(RequestFeatureValidator descriptor) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java index 5b8365e5a75b..bc8da9d17ea2 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java @@ -40,6 +40,7 @@ import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.scm.protocolPB.OzonePBHelper; import org.apache.hadoop.hdds.utils.FaultInjector; +import org.apache.hadoop.ozone.ClientVersion; import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.util.PayloadUtils; import org.apache.hadoop.ozone.om.OzoneManager; @@ -73,7 +74,6 @@ import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; -import org.apache.hadoop.ozone.om.request.validation.ValidationCondition; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.om.upgrade.DisallowedUntilLayoutVersion; @@ -629,9 +629,10 @@ private GetKeyInfoResponse getKeyInfo(GetKeyInfoRequest request, } @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + conditions = {}, processingPhase = RequestProcessingPhase.POST_PROCESS, - requestType = Type.LookupKey + requestType = Type.LookupKey, + maxClientVersion = ClientVersion.VERSION_HANDLES_UNKNOWN_DN_PORTS ) public static OMResponse disallowLookupKeyResponseWithECReplicationConfig( OMRequest req, OMResponse resp, ValidationContext ctx) @@ -656,9 +657,10 @@ public static OMResponse disallowLookupKeyResponseWithECReplicationConfig( } @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + conditions = {}, processingPhase = RequestProcessingPhase.POST_PROCESS, - requestType = Type.LookupKey + requestType = Type.LookupKey, + maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMResponse disallowLookupKeyWithBucketLayout( OMRequest req, OMResponse resp, ValidationContext ctx) @@ -743,9 +745,10 @@ private ListKeysLightResponse listKeysLight(ListKeysRequest request) } @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + conditions = {}, processingPhase = RequestProcessingPhase.POST_PROCESS, - requestType = Type.ListKeys + requestType = Type.ListKeys, + maxClientVersion = ClientVersion.VERSION_HANDLES_UNKNOWN_DN_PORTS ) public static OMResponse disallowListKeysResponseWithECReplicationConfig( OMRequest req, OMResponse resp, ValidationContext ctx) @@ -770,9 +773,10 @@ public static OMResponse disallowListKeysResponseWithECReplicationConfig( } @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + conditions = {}, processingPhase = RequestProcessingPhase.POST_PROCESS, - requestType = Type.ListKeys + requestType = Type.ListKeys, + maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMResponse disallowListKeysWithBucketLayout( OMRequest req, OMResponse resp, ValidationContext ctx) @@ -831,9 +835,10 @@ private ListTrashResponse listTrash(ListTrashRequest request, } @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + conditions = {}, processingPhase = RequestProcessingPhase.POST_PROCESS, - requestType = Type.ListTrash + requestType = Type.ListTrash, + maxClientVersion = ClientVersion.VERSION_HANDLES_UNKNOWN_DN_PORTS ) public static OMResponse disallowListTrashWithECReplicationConfig( OMRequest req, OMResponse resp, ValidationContext ctx) @@ -861,9 +866,10 @@ public static OMResponse disallowListTrashWithECReplicationConfig( } @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + conditions = {}, processingPhase = RequestProcessingPhase.POST_PROCESS, - requestType = Type.ListTrash + requestType = Type.ListTrash, + maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMResponse disallowListTrashWithBucketLayout( OMRequest req, OMResponse resp, ValidationContext ctx) @@ -1048,9 +1054,10 @@ private RefetchSecretKeyResponse refetchSecretKey() { } @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + conditions = {}, processingPhase = RequestProcessingPhase.POST_PROCESS, - requestType = Type.GetFileStatus + requestType = Type.GetFileStatus, + maxClientVersion = ClientVersion.VERSION_HANDLES_UNKNOWN_DN_PORTS ) public static OMResponse disallowGetFileStatusWithECReplicationConfig( OMRequest req, OMResponse resp, ValidationContext ctx) @@ -1078,9 +1085,10 @@ public static OMResponse disallowGetFileStatusWithECReplicationConfig( @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + conditions = {}, processingPhase = RequestProcessingPhase.POST_PROCESS, - requestType = Type.GetFileStatus + requestType = Type.GetFileStatus, + maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMResponse disallowGetFileStatusWithBucketLayout( OMRequest req, OMResponse resp, ValidationContext ctx) @@ -1127,9 +1135,10 @@ private LookupFileResponse lookupFile(LookupFileRequest request, } @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + conditions = {}, processingPhase = RequestProcessingPhase.POST_PROCESS, - requestType = Type.LookupFile + requestType = Type.LookupFile, + maxClientVersion = ClientVersion.VERSION_HANDLES_UNKNOWN_DN_PORTS ) public static OMResponse disallowLookupFileWithECReplicationConfig( OMRequest req, OMResponse resp, ValidationContext ctx) @@ -1155,9 +1164,10 @@ public static OMResponse disallowLookupFileWithECReplicationConfig( } @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + conditions = {}, processingPhase = RequestProcessingPhase.POST_PROCESS, - requestType = Type.LookupFile + requestType = Type.LookupFile, + maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMResponse disallowLookupFileWithBucketLayout( OMRequest req, OMResponse resp, ValidationContext ctx) @@ -1237,9 +1247,10 @@ private ListStatusLightResponse listStatusLight( } @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + conditions = {}, processingPhase = RequestProcessingPhase.POST_PROCESS, - requestType = Type.ListStatus + requestType = Type.ListStatus, + maxClientVersion = ClientVersion.VERSION_HANDLES_UNKNOWN_DN_PORTS ) public static OMResponse disallowListStatusResponseWithECReplicationConfig( OMRequest req, OMResponse resp, ValidationContext ctx) @@ -1265,9 +1276,10 @@ public static OMResponse disallowListStatusResponseWithECReplicationConfig( } @RequestFeatureValidator( - conditions = ValidationCondition.OLDER_CLIENT_REQUESTS, + conditions = {}, processingPhase = RequestProcessingPhase.POST_PROCESS, - requestType = Type.ListStatus + requestType = Type.ListStatus, + maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMResponse disallowListStatusResponseWithBucketLayout( OMRequest req, OMResponse resp, ValidationContext ctx) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestRequestFeatureValidatorProcessor.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestRequestFeatureValidatorProcessor.java index 48947518abd4..682b645b284a 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestRequestFeatureValidatorProcessor.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestRequestFeatureValidatorProcessor.java @@ -18,15 +18,21 @@ import com.google.testing.compile.Compilation; import com.google.testing.compile.JavaFileObjects; +import org.apache.hadoop.ozone.ClientVersion; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; import org.apache.ozone.annotations.RequestFeatureValidatorProcessor; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import java.lang.annotation.Annotation; import java.lang.annotation.ElementType; import java.lang.annotation.Target; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; +import java.util.stream.Collectors; import static com.google.testing.compile.CompilationSubject.assertThat; import static com.google.testing.compile.Compiler.javac; @@ -332,17 +338,27 @@ public void testInvalidProcessingPhase() { assertThat(compile(source)).failed(); } - @Test - public void testMultipleErrorMessages() { + private static List getClientVersions() { + List clientVersions = + Arrays.stream(ClientVersion.values()).map(Arguments::of).collect(Collectors.toList()); + clientVersions.add(null); + return clientVersions; + } + + @ParameterizedTest + @MethodSource("getClientVersions") + public void testMultipleErrorMessages(ClientVersion clientVersion) { List source = generateSourceOfValidatorMethodWith( - annotationOf(emptyConditions(), postProcess(), aReqType()), + annotationOf(emptyConditions(), postProcess(), aReqType(), clientVersion), modifiers(), returnValue("String"), parameters("String rq", "int rp", "String ctx"), exceptions()); Compilation compilation = compile(source); - assertThat(compilation).hadErrorContaining(ERROR_CONDITION_IS_EMPTY); + if (ClientVersion.FUTURE_VERSION.equals(clientVersion)) { + assertThat(compilation).hadErrorContaining(ERROR_CONDITION_IS_EMPTY); + } assertThat(compilation) .hadErrorContaining(ERROR_VALIDATOR_METHOD_HAS_TO_BE_STATIC); assertThat(compilation) @@ -365,7 +381,7 @@ private Compilation compile(List source) { private ValidationCondition[] someConditions() { return - new ValidationCondition[] {ValidationCondition.OLDER_CLIENT_REQUESTS}; + new ValidationCondition[] {ValidationCondition.CLUSTER_NEEDS_FINALIZATION}; } private ValidationCondition[] emptyConditions() { @@ -428,13 +444,29 @@ private String annotationOf( ValidationCondition[] conditions, RequestProcessingPhase phase, Type reqType) { - return annotationOf(conditions, phase.name(), reqType); + return annotationOf(conditions, phase.name(), reqType, null); + } + + private String annotationOf( + ValidationCondition[] conditions, + RequestProcessingPhase phase, + Type reqType, + ClientVersion clientVersion) { + return annotationOf(conditions, phase.name(), reqType, clientVersion); } private String annotationOf( ValidationCondition[] conditions, String phase, Type reqType) { + return annotationOf(conditions, phase, reqType, null); + } + + private String annotationOf( + ValidationCondition[] conditions, + String phase, + Type reqType, + ClientVersion clientVersion) { StringBuilder annotation = new StringBuilder(); annotation.append("@RequestFeatureValidator("); StringBuilder conditionsArray = new StringBuilder(); @@ -451,6 +483,9 @@ private String annotationOf( annotation.append(" }"); annotation.append(", processingPhase = ").append(phase); annotation.append(", requestType = ").append(reqType.name()); + if (clientVersion != null) { + annotation.append(", maxClientVersion = ").append(clientVersion.name()); + } annotation.append(" )"); return annotation.toString(); } @@ -478,6 +513,9 @@ private List allImports() { imports.add("import static org.apache.hadoop.ozone.protocol.proto" + ".OzoneManagerProtocolProtos.Type." + reqType.name() + ";"); } + for (ClientVersion clientVersion : ClientVersion.values()) { + imports.add("import static org.apache.hadoop.ozone.ClientVersion." + clientVersion.name() + ";"); + } return imports; } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestValidatorRegistry.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestValidatorRegistry.java index 076ba0187272..dbc763efe8ce 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestValidatorRegistry.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestValidatorRegistry.java @@ -16,6 +16,7 @@ */ package org.apache.hadoop.ozone.om.request.validation; +import org.apache.hadoop.ozone.ClientVersion; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -26,6 +27,7 @@ import java.net.URL; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -34,7 +36,6 @@ import static org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase.POST_PROCESS; import static org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase.PRE_PROCESS; import static org.apache.hadoop.ozone.om.request.validation.ValidationCondition.CLUSTER_NEEDS_FINALIZATION; -import static org.apache.hadoop.ozone.om.request.validation.ValidationCondition.OLDER_CLIENT_REQUESTS; import static org.apache.hadoop.ozone.om.request.validation.testvalidatorset1.GeneralValidatorsForTesting.startValidatorTest; import static org.apache.hadoop.ozone.om.request.validation.testvalidatorset1.GeneralValidatorsForTesting.finishValidatorTest; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.CreateDirectory; @@ -72,8 +73,8 @@ public void tearDown() { @Test public void testNoValidatorsReturnedForEmptyConditionList() { ValidatorRegistry registry = new ValidatorRegistry(PACKAGE); - List validators = - registry.validationsFor(emptyList(), CreateKey, PRE_PROCESS); + List validators = registry.validationsFor(emptyList(), CreateKey, PRE_PROCESS, + ClientVersion.CURRENT_VERSION); assertTrue(validators.isEmpty()); } @@ -82,8 +83,8 @@ public void testNoValidatorsReturnedForEmptyConditionList() { public void testRegistryHasThePreFinalizePreProcessCreateKeyValidator() { ValidatorRegistry registry = new ValidatorRegistry(PACKAGE); List validators = - registry.validationsFor( - asList(CLUSTER_NEEDS_FINALIZATION), CreateKey, PRE_PROCESS); + registry.validationsFor(asList(CLUSTER_NEEDS_FINALIZATION), CreateKey, PRE_PROCESS, + ClientVersion.CURRENT_VERSION); assertEquals(1, validators.size()); String expectedMethodName = "preFinalizePreProcessCreateKeyValidator"; @@ -95,7 +96,7 @@ public void testRegistryHasThePreFinalizePostProcessCreateKeyValidator() { ValidatorRegistry registry = new ValidatorRegistry(PACKAGE); List validators = registry.validationsFor( - asList(CLUSTER_NEEDS_FINALIZATION), CreateKey, POST_PROCESS); + asList(CLUSTER_NEEDS_FINALIZATION), CreateKey, POST_PROCESS, ClientVersion.CURRENT_VERSION); assertEquals(1, validators.size()); String expectedMethodName = "preFinalizePostProcessCreateKeyValidator"; @@ -107,7 +108,7 @@ public void testRegistryHasTheOldClientPreProcessCreateKeyValidator() { ValidatorRegistry registry = new ValidatorRegistry(PACKAGE); List validators = registry.validationsFor( - asList(OLDER_CLIENT_REQUESTS), CreateKey, PRE_PROCESS); + Collections.emptyList(), CreateKey, PRE_PROCESS, ClientVersion.ERASURE_CODING_SUPPORT.toProtoValue()); assertEquals(2, validators.size()); List methodNames = @@ -119,9 +120,8 @@ public void testRegistryHasTheOldClientPreProcessCreateKeyValidator() { @Test public void testRegistryHasTheOldClientPostProcessCreateKeyValidator() { ValidatorRegistry registry = new ValidatorRegistry(PACKAGE); - List validators = - registry.validationsFor( - asList(OLDER_CLIENT_REQUESTS), CreateKey, POST_PROCESS); + List validators = registry.validationsFor(Collections.emptyList(), CreateKey, POST_PROCESS, + ClientVersion.ERASURE_CODING_SUPPORT.toProtoValue()); assertEquals(2, validators.size()); List methodNames = @@ -135,10 +135,10 @@ public void testRegistryHasTheMultiPurposePreProcessCreateVolumeValidator() { ValidatorRegistry registry = new ValidatorRegistry(PACKAGE); List preFinalizeValidators = registry.validationsFor( - asList(CLUSTER_NEEDS_FINALIZATION), CreateVolume, PRE_PROCESS); + asList(CLUSTER_NEEDS_FINALIZATION), CreateVolume, PRE_PROCESS, ClientVersion.CURRENT_VERSION); List newClientValidators = registry.validationsFor( - asList(OLDER_CLIENT_REQUESTS), CreateVolume, PRE_PROCESS); + Collections.emptyList(), CreateVolume, PRE_PROCESS, ClientVersion.ERASURE_CODING_SUPPORT.toProtoValue()); assertEquals(1, preFinalizeValidators.size()); assertEquals(1, newClientValidators.size()); @@ -152,10 +152,10 @@ public void testRegistryHasTheMultiPurposePostProcessCreateVolumeValidator() { ValidatorRegistry registry = new ValidatorRegistry(PACKAGE); List preFinalizeValidators = registry.validationsFor( - asList(CLUSTER_NEEDS_FINALIZATION), CreateVolume, POST_PROCESS); + asList(CLUSTER_NEEDS_FINALIZATION), CreateVolume, POST_PROCESS, ClientVersion.CURRENT_VERSION); List oldClientValidators = registry.validationsFor( - asList(OLDER_CLIENT_REQUESTS), CreateVolume, POST_PROCESS); + Collections.emptyList(), CreateVolume, POST_PROCESS, ClientVersion.ERASURE_CODING_SUPPORT.toProtoValue()); assertEquals(1, preFinalizeValidators.size()); assertEquals(1, oldClientValidators.size()); @@ -169,8 +169,8 @@ public void testValidatorsAreReturnedForMultiCondition() { ValidatorRegistry registry = new ValidatorRegistry(PACKAGE); List validators = registry.validationsFor( - asList(CLUSTER_NEEDS_FINALIZATION, OLDER_CLIENT_REQUESTS), - CreateKey, POST_PROCESS); + asList(CLUSTER_NEEDS_FINALIZATION), + CreateKey, POST_PROCESS, ClientVersion.ERASURE_CODING_SUPPORT.toProtoValue()); assertEquals(3, validators.size()); List methodNames = @@ -183,9 +183,8 @@ public void testValidatorsAreReturnedForMultiCondition() { @Test public void testNoValidatorForRequestsAtAllReturnsEmptyList() { ValidatorRegistry registry = new ValidatorRegistry(PACKAGE_WO_VALIDATORS); - - assertTrue(registry.validationsFor( - asList(OLDER_CLIENT_REQUESTS), CreateKey, PRE_PROCESS).isEmpty()); + assertTrue(registry.validationsFor(Collections.emptyList(), CreateKey, PRE_PROCESS, + ClientVersion.ERASURE_CODING_SUPPORT.toProtoValue()).isEmpty()); } @Test @@ -199,7 +198,7 @@ public void testNoValidatorForConditionReturnsEmptyList() ValidatorRegistry registry = new ValidatorRegistry(urlsToUse); assertTrue(registry.validationsFor( - asList(CLUSTER_NEEDS_FINALIZATION), CreateKey, PRE_PROCESS).isEmpty()); + asList(CLUSTER_NEEDS_FINALIZATION), CreateKey, PRE_PROCESS, ClientVersion.CURRENT_VERSION).isEmpty()); } @Test @@ -207,7 +206,8 @@ public void testNoDefinedValidationForRequestReturnsEmptyList() { ValidatorRegistry registry = new ValidatorRegistry(PACKAGE); assertTrue(registry.validationsFor( - asList(OLDER_CLIENT_REQUESTS), CreateDirectory, null).isEmpty()); + Collections.emptyList(), CreateDirectory, null, ClientVersion.ERASURE_CODING_SUPPORT.toProtoValue()) + .isEmpty()); } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset1/GeneralValidatorsForTesting.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset1/GeneralValidatorsForTesting.java index 35c3afa4cf96..2c240284a0a6 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset1/GeneralValidatorsForTesting.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset1/GeneralValidatorsForTesting.java @@ -16,6 +16,7 @@ */ package org.apache.hadoop.ozone.om.request.validation.testvalidatorset1; +import org.apache.hadoop.ozone.ClientVersion; import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; import org.apache.hadoop.ozone.om.request.validation.TestRequestValidations; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; @@ -29,7 +30,6 @@ import static org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase.POST_PROCESS; import static org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase.PRE_PROCESS; import static org.apache.hadoop.ozone.om.request.validation.ValidationCondition.CLUSTER_NEEDS_FINALIZATION; -import static org.apache.hadoop.ozone.om.request.validation.ValidationCondition.OLDER_CLIENT_REQUESTS; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.CreateKey; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.CreateVolume; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.DeleteKeys; @@ -109,9 +109,10 @@ public static OMResponse preFinalizePostProcessCreateKeyValidator( } @RequestFeatureValidator( - conditions = { OLDER_CLIENT_REQUESTS }, + conditions = {}, processingPhase = PRE_PROCESS, - requestType = CreateKey) + requestType = CreateKey, + maxClientVersion = ClientVersion.BUCKET_LAYOUT_SUPPORT) public static OMRequest oldClientPreProcessCreateKeyValidator( OMRequest req, ValidationContext ctx) { fireValidationEvent("oldClientPreProcessCreateKeyValidator"); @@ -119,9 +120,10 @@ public static OMRequest oldClientPreProcessCreateKeyValidator( } @RequestFeatureValidator( - conditions = { OLDER_CLIENT_REQUESTS }, + conditions = {}, processingPhase = POST_PROCESS, - requestType = CreateKey) + requestType = CreateKey, + maxClientVersion = ClientVersion.BUCKET_LAYOUT_SUPPORT) public static OMResponse oldClientPostProcessCreateKeyValidator( OMRequest req, OMResponse resp, ValidationContext ctx) { fireValidationEvent("oldClientPostProcessCreateKeyValidator"); @@ -129,9 +131,10 @@ public static OMResponse oldClientPostProcessCreateKeyValidator( } @RequestFeatureValidator( - conditions = { CLUSTER_NEEDS_FINALIZATION, OLDER_CLIENT_REQUESTS }, + conditions = { CLUSTER_NEEDS_FINALIZATION }, processingPhase = PRE_PROCESS, - requestType = CreateVolume) + requestType = CreateVolume, + maxClientVersion = ClientVersion.BUCKET_LAYOUT_SUPPORT) public static OMRequest multiPurposePreProcessCreateVolumeValidator( OMRequest req, ValidationContext ctx) { fireValidationEvent("multiPurposePreProcessCreateVolumeValidator"); @@ -139,9 +142,10 @@ public static OMRequest multiPurposePreProcessCreateVolumeValidator( } @RequestFeatureValidator( - conditions = { OLDER_CLIENT_REQUESTS, CLUSTER_NEEDS_FINALIZATION }, + conditions = { CLUSTER_NEEDS_FINALIZATION }, processingPhase = POST_PROCESS, - requestType = CreateVolume) + requestType = CreateVolume, + maxClientVersion = ClientVersion.BUCKET_LAYOUT_SUPPORT) public static OMResponse multiPurposePostProcessCreateVolumeValidator( OMRequest req, OMResponse resp, ValidationContext ctx) { fireValidationEvent("multiPurposePostProcessCreateVolumeValidator"); @@ -149,9 +153,10 @@ public static OMResponse multiPurposePostProcessCreateVolumeValidator( } @RequestFeatureValidator( - conditions = { OLDER_CLIENT_REQUESTS }, + conditions = {}, processingPhase = POST_PROCESS, - requestType = CreateKey) + requestType = CreateKey, + maxClientVersion = ClientVersion.BUCKET_LAYOUT_SUPPORT) public static OMResponse oldClientPostProcessCreateKeyValidator2( OMRequest req, OMResponse resp, ValidationContext ctx) { fireValidationEvent("oldClientPostProcessCreateKeyValidator2"); @@ -159,9 +164,10 @@ public static OMResponse oldClientPostProcessCreateKeyValidator2( } @RequestFeatureValidator( - conditions = {OLDER_CLIENT_REQUESTS}, + conditions = {}, processingPhase = PRE_PROCESS, - requestType = DeleteKeys + requestType = DeleteKeys, + maxClientVersion = ClientVersion.BUCKET_LAYOUT_SUPPORT ) public static OMRequest throwingPreProcessValidator( OMRequest req, ValidationContext ctx) throws IOException { @@ -173,9 +179,10 @@ public static OMRequest throwingPreProcessValidator( } @RequestFeatureValidator( - conditions = {OLDER_CLIENT_REQUESTS}, + conditions = {}, processingPhase = POST_PROCESS, - requestType = DeleteKeys + requestType = DeleteKeys, + maxClientVersion = ClientVersion.BUCKET_LAYOUT_SUPPORT ) public static OMResponse throwingPostProcessValidator( OMRequest req, OMResponse resp, ValidationContext ctx) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset2/ValidatorsForOnlyOldClientValidations.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset2/ValidatorsForOnlyOldClientValidations.java index 447b9ab0260b..b4ea709c3bdf 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset2/ValidatorsForOnlyOldClientValidations.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset2/ValidatorsForOnlyOldClientValidations.java @@ -16,12 +16,12 @@ */ package org.apache.hadoop.ozone.om.request.validation.testvalidatorset2; +import org.apache.hadoop.ozone.ClientVersion; import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import static org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase.PRE_PROCESS; -import static org.apache.hadoop.ozone.om.request.validation.ValidationCondition.OLDER_CLIENT_REQUESTS; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.CreateKey; /** @@ -33,9 +33,10 @@ public final class ValidatorsForOnlyOldClientValidations { private ValidatorsForOnlyOldClientValidations() { } @RequestFeatureValidator( - conditions = { OLDER_CLIENT_REQUESTS }, + conditions = {}, processingPhase = PRE_PROCESS, - requestType = CreateKey) + requestType = CreateKey, + maxClientVersion = ClientVersion.BUCKET_LAYOUT_SUPPORT) public static OMRequest oldClientPreProcessCreateKeyValidator2( OMRequest req, ValidationContext ctx) { return req; From f1676be68a2f57f7b6f91fd508ab46fc08e6d0f4 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Thu, 11 Jul 2024 22:28:06 -0700 Subject: [PATCH 02/28] HDDS-11149. Set conditions default value to {} Change-Id: Ife98bca526abf9d82d383f7e65f4140ba73e21a2 --- .../RequestFeatureValidatorProcessor.java | 12 +++--- .../request/bucket/OMBucketCreateRequest.java | 1 - .../request/bucket/OMBucketDeleteRequest.java | 1 - .../file/OMDirectoryCreateRequest.java | 1 - .../om/request/file/OMFileCreateRequest.java | 1 - .../request/key/OMAllocateBlockRequest.java | 1 - .../om/request/key/OMKeyCommitRequest.java | 1 - .../om/request/key/OMKeyCreateRequest.java | 1 - .../om/request/key/OMKeyDeleteRequest.java | 1 - .../om/request/key/OMKeyRenameRequest.java | 1 - .../om/request/key/OMKeysDeleteRequest.java | 1 - .../om/request/key/OMKeysRenameRequest.java | 1 - .../request/key/acl/OMKeyAddAclRequest.java | 1 - .../key/acl/OMKeyRemoveAclRequest.java | 1 - .../request/key/acl/OMKeySetAclRequest.java | 1 - .../S3InitiateMultipartUploadRequest.java | 1 - .../S3MultipartUploadAbortRequest.java | 1 - .../S3MultipartUploadCommitPartRequest.java | 1 - .../S3MultipartUploadCompleteRequest.java | 1 - .../validation/RequestFeatureValidator.java | 2 +- .../OzoneManagerRequestHandler.java | 12 ------ .../TestRequestFeatureValidatorProcessor.java | 38 ++++++++++++++----- .../GeneralValidatorsForTesting.java | 13 ++----- ...ValidatorsForOnlyOldClientValidations.java | 1 - 24 files changed, 39 insertions(+), 57 deletions(-) diff --git a/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/RequestFeatureValidatorProcessor.java b/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/RequestFeatureValidatorProcessor.java index 45538f5fa374..97bb721afc45 100644 --- a/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/RequestFeatureValidatorProcessor.java +++ b/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/RequestFeatureValidatorProcessor.java @@ -203,19 +203,19 @@ private void checkMethodIsAnnotated(Element elem) { private boolean checkAndEvaluateAnnotation( AnnotationMirror methodAnnotation) { boolean isPreprocessor = false; - boolean hasInvalidConditions = false; + boolean hasValidConditions = false; boolean hasValidMaxClientVersions = false; for (Entry entry : methodAnnotation.getElementValues().entrySet()) { - hasInvalidConditions = hasInvalidConditions || hasInvalidValidationCondition(entry); + hasValidConditions = hasValidConditions || hasValidValidationCondition(entry); if (isProcessingPhaseValue(entry)) { isPreprocessor = evaluateProcessingPhase(entry); } hasValidMaxClientVersions = hasValidMaxClientVersions || hasValidMaxClientVersion(entry); } - if (!hasValidMaxClientVersions && hasInvalidConditions) { + if (!hasValidMaxClientVersions && !hasValidConditions) { emitErrorMsg(ERROR_CONDITION_IS_EMPTY); } @@ -252,10 +252,10 @@ private boolean isMaxClientVersionValue( return isPropertyNamedAs(entry, ANNOTATION_MAX_CLIENT_VERSION_PROPERTY_NAME); } - private boolean hasInvalidValidationCondition( + private boolean hasValidValidationCondition( Entry entry) { return isPropertyNamedAs(entry, ANNOTATION_CONDITIONS_PROPERTY_NAME) - && !visit(entry, new ConditionValidator()); + && visit(entry, new ConditionValidator()); } private boolean isPropertyNamedAs( @@ -274,7 +274,7 @@ private static class ConditionValidator extends SimpleAnnotationValueVisitor8 { ConditionValidator() { - super(Boolean.TRUE); + super(Boolean.FALSE); } @Override diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java index f9e63cb182d8..74f626cebc8a 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java @@ -460,7 +460,6 @@ public static OMRequest handleCreateBucketWithBucketLayoutDuringPreFinalize( * they do not understand. */ @RequestFeatureValidator( - conditions = {}, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CreateBucket, maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketDeleteRequest.java index 74b94948ce40..8be06c4b8f59 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketDeleteRequest.java @@ -281,7 +281,6 @@ private boolean bucketContainsSnapshotInCache( * @throws OMException if the request is invalid */ @RequestFeatureValidator( - conditions = {}, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.DeleteBucket, maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java index eba409448658..504176c4c8ee 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java @@ -433,7 +433,6 @@ public static OMRequest disallowCreateDirectoryWithECReplicationConfig( * @throws OMException if the request is invalid */ @RequestFeatureValidator( - conditions = {}, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CreateDirectory, maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java index a07f0cd057e4..42bdf7c2ac8a 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java @@ -426,7 +426,6 @@ public static OMRequest disallowCreateFileWithECReplicationConfig( * @throws OMException if the request is invalid */ @RequestFeatureValidator( - conditions = {}, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CreateFile, maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java index f02cece6f9fe..9dff7040d59b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java @@ -299,7 +299,6 @@ public static OMRequest disallowAllocateBlockWithECReplicationConfig( * @throws OMException if the request is invalid */ @RequestFeatureValidator( - conditions = {}, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.AllocateBlock, maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java index f2964d573b02..22040f2ae272 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java @@ -464,7 +464,6 @@ public static OMRequest disallowCommitKeyWithECReplicationConfig( * @throws OMException if the request is invalid */ @RequestFeatureValidator( - conditions = {}, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CommitKey, maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java index 2e90b5e5e6f4..1b6ae159db88 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java @@ -425,7 +425,6 @@ public static OMRequest disallowCreateKeyWithECReplicationConfig( * @throws OMException if the request is invalid */ @RequestFeatureValidator( - conditions = {}, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CreateKey, maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java index f0a53d9cd6bc..87740b4887ab 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java @@ -233,7 +233,6 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn * @throws OMException if the request is invalid */ @RequestFeatureValidator( - conditions = {}, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.DeleteKey, maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java index c11b30412487..c6adacbe7650 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java @@ -270,7 +270,6 @@ private Map buildAuditMap( * @throws OMException if the request is invalid */ @RequestFeatureValidator( - conditions = {}, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.RenameKey, maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java index 342b0c633364..4212826cf29d 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java @@ -351,7 +351,6 @@ protected static void addDeletedKeys(Map auditMap, * @throws OMException if the request is invalid */ @RequestFeatureValidator( - conditions = {}, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.DeleteKeys, maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysRenameRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysRenameRequest.java index 1ffa1650782c..6884c8a88a36 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysRenameRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysRenameRequest.java @@ -297,7 +297,6 @@ private Map buildAuditMap(Map auditMap, * @throws OMException if the request is invalid */ @RequestFeatureValidator( - conditions = {}, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.RenameKeys, maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAddAclRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAddAclRequest.java index 0fd43a657ad8..cd3ee13a4185 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAddAclRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAddAclRequest.java @@ -165,7 +165,6 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn * @throws OMException if the request is invalid */ @RequestFeatureValidator( - conditions = {}, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.AddAcl, maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyRemoveAclRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyRemoveAclRequest.java index 3618c8fae198..122962e1db58 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyRemoveAclRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyRemoveAclRequest.java @@ -166,7 +166,6 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn * @throws OMException if the request is invalid */ @RequestFeatureValidator( - conditions = {}, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.RemoveAcl, maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeySetAclRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeySetAclRequest.java index 6abca95b7e3b..188fc474199c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeySetAclRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeySetAclRequest.java @@ -162,7 +162,6 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn * @throws OMException if the request is invalid */ @RequestFeatureValidator( - conditions = {}, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.SetAcl, maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java index 5afa5958ef29..bac283238bee 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java @@ -321,7 +321,6 @@ protected void logResult(OzoneManager ozoneManager, * @throws OMException if the request is invalid */ @RequestFeatureValidator( - conditions = {}, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.InitiateMultiPartUpload, maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java index a688a7f44b50..3b751f655552 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java @@ -290,7 +290,6 @@ public static OMRequest disallowAbortMultiPartUploadWithECReplicationConfig( * @throws OMException if the request is invalid */ @RequestFeatureValidator( - conditions = {}, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.AbortMultiPartUpload, maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java index 0a619d54c144..68bef1831870 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java @@ -381,7 +381,6 @@ public static OMRequest disallowCommitMultiPartUploadWithECReplicationConfig( * @throws OMException if the request is invalid */ @RequestFeatureValidator( - conditions = {}, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CommitMultiPartUpload, maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java index 873bf51339da..da3543bcfcd3 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java @@ -743,7 +743,6 @@ private void updateCache(OMMetadataManager omMetadataManager, * @throws OMException if the request is invalid */ @RequestFeatureValidator( - conditions = {}, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CompleteMultiPartUpload, maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestFeatureValidator.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestFeatureValidator.java index a960f23b90da..4a8f65684b4a 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestFeatureValidator.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestFeatureValidator.java @@ -82,7 +82,7 @@ * Runtime conditions in which a validator should run. * @return a list of conditions when the validator should be applied */ - ValidationCondition[] conditions(); + ValidationCondition[] conditions() default {}; /** * Defines if the validation has to run before or after the general request diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java index bc8da9d17ea2..13d0904985f4 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java @@ -629,7 +629,6 @@ private GetKeyInfoResponse getKeyInfo(GetKeyInfoRequest request, } @RequestFeatureValidator( - conditions = {}, processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.LookupKey, maxClientVersion = ClientVersion.VERSION_HANDLES_UNKNOWN_DN_PORTS @@ -657,7 +656,6 @@ public static OMResponse disallowLookupKeyResponseWithECReplicationConfig( } @RequestFeatureValidator( - conditions = {}, processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.LookupKey, maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT @@ -745,7 +743,6 @@ private ListKeysLightResponse listKeysLight(ListKeysRequest request) } @RequestFeatureValidator( - conditions = {}, processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.ListKeys, maxClientVersion = ClientVersion.VERSION_HANDLES_UNKNOWN_DN_PORTS @@ -773,7 +770,6 @@ public static OMResponse disallowListKeysResponseWithECReplicationConfig( } @RequestFeatureValidator( - conditions = {}, processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.ListKeys, maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT @@ -835,7 +831,6 @@ private ListTrashResponse listTrash(ListTrashRequest request, } @RequestFeatureValidator( - conditions = {}, processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.ListTrash, maxClientVersion = ClientVersion.VERSION_HANDLES_UNKNOWN_DN_PORTS @@ -866,7 +861,6 @@ public static OMResponse disallowListTrashWithECReplicationConfig( } @RequestFeatureValidator( - conditions = {}, processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.ListTrash, maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT @@ -1054,7 +1048,6 @@ private RefetchSecretKeyResponse refetchSecretKey() { } @RequestFeatureValidator( - conditions = {}, processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.GetFileStatus, maxClientVersion = ClientVersion.VERSION_HANDLES_UNKNOWN_DN_PORTS @@ -1085,7 +1078,6 @@ public static OMResponse disallowGetFileStatusWithECReplicationConfig( @RequestFeatureValidator( - conditions = {}, processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.GetFileStatus, maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT @@ -1135,7 +1127,6 @@ private LookupFileResponse lookupFile(LookupFileRequest request, } @RequestFeatureValidator( - conditions = {}, processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.LookupFile, maxClientVersion = ClientVersion.VERSION_HANDLES_UNKNOWN_DN_PORTS @@ -1164,7 +1155,6 @@ public static OMResponse disallowLookupFileWithECReplicationConfig( } @RequestFeatureValidator( - conditions = {}, processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.LookupFile, maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT @@ -1247,7 +1237,6 @@ private ListStatusLightResponse listStatusLight( } @RequestFeatureValidator( - conditions = {}, processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.ListStatus, maxClientVersion = ClientVersion.VERSION_HANDLES_UNKNOWN_DN_PORTS @@ -1276,7 +1265,6 @@ public static OMResponse disallowListStatusResponseWithECReplicationConfig( } @RequestFeatureValidator( - conditions = {}, processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.ListStatus, maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestRequestFeatureValidatorProcessor.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestRequestFeatureValidatorProcessor.java index 682b645b284a..95a62d349a2e 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestRequestFeatureValidatorProcessor.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestRequestFeatureValidatorProcessor.java @@ -178,6 +178,18 @@ public void testEmptyValidationConditionListDoesNotCompile() { assertThat(compile(source)).hadErrorContaining(ERROR_CONDITION_IS_EMPTY); } + @Test + public void testNullValidationConditionListDoesNotCompile() { + List source = generateSourceOfValidatorMethodWith( + annotationOf(nullConditions(), preProcess(), aReqType()), + modifiers("public", "static"), + returnValue("OMRequest"), + parameters("OMRequest rq", "ValidationContext ctx"), + exceptions()); + + assertThat(compile(source)).hadErrorContaining(ERROR_CONDITION_IS_EMPTY); + } + @Test public void testNotEnoughParametersForPreProcess() { List source = generateSourceOfValidatorMethodWith( @@ -388,6 +400,10 @@ private ValidationCondition[] emptyConditions() { return new ValidationCondition[] {}; } + private ValidationCondition[] nullConditions() { + return null; + } + private RequestProcessingPhase preProcess() { return RequestProcessingPhase.PRE_PROCESS; } @@ -470,18 +486,20 @@ private String annotationOf( StringBuilder annotation = new StringBuilder(); annotation.append("@RequestFeatureValidator("); StringBuilder conditionsArray = new StringBuilder(); - conditionsArray.append("conditions = { "); - if (conditions.length > 0) { - for (ValidationCondition condition : conditions) { - conditionsArray.append(condition.name()).append(", "); + if (conditions != null) { + conditionsArray.append("conditions = { "); + if (conditions.length > 0) { + for (ValidationCondition condition : conditions) { + conditionsArray.append(condition.name()).append(", "); + } + annotation + .append(conditionsArray.substring(0, conditionsArray.length() - 2)); + } else { + annotation.append(conditionsArray); } - annotation - .append(conditionsArray.substring(0, conditionsArray.length() - 2)); - } else { - annotation.append(conditionsArray); + annotation.append(" }, "); } - annotation.append(" }"); - annotation.append(", processingPhase = ").append(phase); + annotation.append("processingPhase = ").append(phase); annotation.append(", requestType = ").append(reqType.name()); if (clientVersion != null) { annotation.append(", maxClientVersion = ").append(clientVersion.name()); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset1/GeneralValidatorsForTesting.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset1/GeneralValidatorsForTesting.java index 2c240284a0a6..80bbae1340ce 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset1/GeneralValidatorsForTesting.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset1/GeneralValidatorsForTesting.java @@ -89,7 +89,7 @@ private static void fireValidationEvent(String calledMethodName) { } @RequestFeatureValidator( - conditions = { CLUSTER_NEEDS_FINALIZATION }, + conditions = CLUSTER_NEEDS_FINALIZATION, processingPhase = PRE_PROCESS, requestType = CreateKey) public static OMRequest preFinalizePreProcessCreateKeyValidator( @@ -99,7 +99,7 @@ public static OMRequest preFinalizePreProcessCreateKeyValidator( } @RequestFeatureValidator( - conditions = { CLUSTER_NEEDS_FINALIZATION }, + conditions = CLUSTER_NEEDS_FINALIZATION, processingPhase = POST_PROCESS, requestType = CreateKey) public static OMResponse preFinalizePostProcessCreateKeyValidator( @@ -109,7 +109,6 @@ public static OMResponse preFinalizePostProcessCreateKeyValidator( } @RequestFeatureValidator( - conditions = {}, processingPhase = PRE_PROCESS, requestType = CreateKey, maxClientVersion = ClientVersion.BUCKET_LAYOUT_SUPPORT) @@ -120,7 +119,6 @@ public static OMRequest oldClientPreProcessCreateKeyValidator( } @RequestFeatureValidator( - conditions = {}, processingPhase = POST_PROCESS, requestType = CreateKey, maxClientVersion = ClientVersion.BUCKET_LAYOUT_SUPPORT) @@ -131,7 +129,7 @@ public static OMResponse oldClientPostProcessCreateKeyValidator( } @RequestFeatureValidator( - conditions = { CLUSTER_NEEDS_FINALIZATION }, + conditions = CLUSTER_NEEDS_FINALIZATION, processingPhase = PRE_PROCESS, requestType = CreateVolume, maxClientVersion = ClientVersion.BUCKET_LAYOUT_SUPPORT) @@ -142,7 +140,7 @@ public static OMRequest multiPurposePreProcessCreateVolumeValidator( } @RequestFeatureValidator( - conditions = { CLUSTER_NEEDS_FINALIZATION }, + conditions = CLUSTER_NEEDS_FINALIZATION, processingPhase = POST_PROCESS, requestType = CreateVolume, maxClientVersion = ClientVersion.BUCKET_LAYOUT_SUPPORT) @@ -153,7 +151,6 @@ public static OMResponse multiPurposePostProcessCreateVolumeValidator( } @RequestFeatureValidator( - conditions = {}, processingPhase = POST_PROCESS, requestType = CreateKey, maxClientVersion = ClientVersion.BUCKET_LAYOUT_SUPPORT) @@ -164,7 +161,6 @@ public static OMResponse oldClientPostProcessCreateKeyValidator2( } @RequestFeatureValidator( - conditions = {}, processingPhase = PRE_PROCESS, requestType = DeleteKeys, maxClientVersion = ClientVersion.BUCKET_LAYOUT_SUPPORT @@ -179,7 +175,6 @@ public static OMRequest throwingPreProcessValidator( } @RequestFeatureValidator( - conditions = {}, processingPhase = POST_PROCESS, requestType = DeleteKeys, maxClientVersion = ClientVersion.BUCKET_LAYOUT_SUPPORT diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset2/ValidatorsForOnlyOldClientValidations.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset2/ValidatorsForOnlyOldClientValidations.java index b4ea709c3bdf..a042523227b5 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset2/ValidatorsForOnlyOldClientValidations.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset2/ValidatorsForOnlyOldClientValidations.java @@ -33,7 +33,6 @@ public final class ValidatorsForOnlyOldClientValidations { private ValidatorsForOnlyOldClientValidations() { } @RequestFeatureValidator( - conditions = {}, processingPhase = PRE_PROCESS, requestType = CreateKey, maxClientVersion = ClientVersion.BUCKET_LAYOUT_SUPPORT) From 561d0106f69321ca27a287eaf931d3522cd67f93 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Mon, 15 Jul 2024 01:46:41 -0700 Subject: [PATCH 03/28] HDDS-11149. Create Base validator for generic version validation Change-Id: Ib1742e6a78f908598021ed0bb80f71d3c1527735 --- ...> OmRequestFeatureValidatorProcessor.java} | 31 +- .../RegisterValidatorProcessor.java | 115 +++++++ .../apache/hadoop/hdds/ComponentVersion.java | 9 +- .../apache/hadoop/ozone/ClientVersion.java | 4 +- .../java/org/apache/hadoop/ozone/Version.java | 26 ++ .../hadoop/ozone/upgrade/LayoutFeature.java | 9 +- .../javax.annotation.processing.Processor | 3 +- .../request/bucket/OMBucketCreateRequest.java | 21 +- .../request/bucket/OMBucketDeleteRequest.java | 6 +- .../bucket/OMBucketSetPropertyRequest.java | 11 +- .../file/OMDirectoryCreateRequest.java | 15 +- .../om/request/file/OMFileCreateRequest.java | 15 +- .../request/key/OMAllocateBlockRequest.java | 15 +- .../om/request/key/OMKeyCommitRequest.java | 21 +- .../om/request/key/OMKeyCreateRequest.java | 15 +- .../om/request/key/OMKeyDeleteRequest.java | 6 +- .../om/request/key/OMKeyRenameRequest.java | 6 +- .../om/request/key/OMKeysDeleteRequest.java | 6 +- .../om/request/key/OMKeysRenameRequest.java | 6 +- .../request/key/acl/OMKeyAddAclRequest.java | 6 +- .../key/acl/OMKeyRemoveAclRequest.java | 6 +- .../request/key/acl/OMKeySetAclRequest.java | 6 +- .../S3InitiateMultipartUploadRequest.java | 15 +- .../S3MultipartUploadAbortRequest.java | 15 +- .../S3MultipartUploadCommitPartRequest.java | 15 +- .../S3MultipartUploadCompleteRequest.java | 15 +- ...tor.java => OMClientVersionValidator.java} | 23 +- .../validation/OMLayoutVersionValidator.java | 103 ++++++ .../request/validation/RegisterValidator.java | 31 ++ .../validation/RequestValidations.java | 35 +- .../validation/ValidationCondition.java | 56 ---- .../request/validation/ValidatorRegistry.java | 307 +++++++++--------- .../request/validation/VersionExtractor.java | 62 ++++ .../om/request/validation/package-info.java | 6 +- .../ozone/om/upgrade/OMLayoutFeature.java | 2 + .../OzoneManagerRequestHandler.java | 50 +-- ...estOMClientVersionValidatorProcessor.java} | 142 +++----- .../validation/TestRequestValidations.java | 29 +- .../validation/TestValidatorRegistry.java | 113 ++++--- .../GeneralValidatorsForTesting.java | 50 +-- ...ValidatorsForOnlyOldClientValidations.java | 6 +- 41 files changed, 883 insertions(+), 550 deletions(-) rename hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/{RequestFeatureValidatorProcessor.java => OmRequestFeatureValidatorProcessor.java} (92%) create mode 100644 hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/RegisterValidatorProcessor.java create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/Version.java rename hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/{RequestFeatureValidator.java => OMClientVersionValidator.java} (82%) create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/OMLayoutVersionValidator.java create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RegisterValidator.java delete mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidationCondition.java create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/VersionExtractor.java rename hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/{TestRequestFeatureValidatorProcessor.java => TestOMClientVersionValidatorProcessor.java} (77%) diff --git a/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/RequestFeatureValidatorProcessor.java b/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/OmRequestFeatureValidatorProcessor.java similarity index 92% rename from hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/RequestFeatureValidatorProcessor.java rename to hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/OmRequestFeatureValidatorProcessor.java index 97bb721afc45..cb78efb7371e 100644 --- a/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/RequestFeatureValidatorProcessor.java +++ b/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/OmRequestFeatureValidatorProcessor.java @@ -16,6 +16,8 @@ */ package org.apache.ozone.annotations; +import jdk.nashorn.internal.ir.annotations.Immutable; + import javax.annotation.processing.AbstractProcessor; import javax.annotation.processing.RoundEnvironment; import javax.annotation.processing.SupportedAnnotationTypes; @@ -35,6 +37,8 @@ import javax.lang.model.type.TypeMirror; import javax.lang.model.util.SimpleAnnotationValueVisitor8; import javax.tools.Diagnostic; +import java.util.Arrays; +import java.util.HashSet; import java.util.List; import java.util.Map.Entry; import java.util.Set; @@ -51,14 +55,11 @@ * META-INF/services/javax.annotation.processing.Processor file in the module's * resources folder. */ -@SupportedAnnotationTypes( - "org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator") +@SupportedAnnotationTypes({ + "org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator", + "org.apache.hadoop.ozone.om.request.validation.OMLayoutVersionValidator"}) @SupportedSourceVersion(SourceVersion.RELEASE_8) -public class RequestFeatureValidatorProcessor extends AbstractProcessor { - - public static final String ERROR_CONDITION_IS_EMPTY = - "RequestFeatureValidator has both an empty condition list & undefined max client version. Please define either" - + " ValidationCondition or max client version in which the validator has to be applied."; +public class OmRequestFeatureValidatorProcessor extends AbstractProcessor { public static final String ERROR_ANNOTATED_ELEMENT_IS_NOT_A_METHOD = "RequestFeatureValidator annotation is not applied to a method."; public static final String ERROR_VALIDATOR_METHOD_HAS_TO_BE_STATIC = @@ -91,7 +92,9 @@ public class RequestFeatureValidatorProcessor extends AbstractProcessor { public static final String VALIDATION_CONTEXT_CLASS_NAME = "org.apache.hadoop.ozone.om.request.validation.ValidationContext"; - public static final String ANNOTATION_SIMPLE_NAME = "RequestFeatureValidator"; + + public static final List ANNOTATION_SIMPLE_NAMES = Arrays.asList("OMClientVersionValidator", + "OMLayoutVersionValidator"); public static final String ANNOTATION_CONDITIONS_PROPERTY_NAME = "conditions"; public static final String ANNOTATION_PROCESSING_PHASE_PROPERTY_NAME = "processingPhase"; @@ -108,7 +111,7 @@ public class RequestFeatureValidatorProcessor extends AbstractProcessor { public boolean process(Set annotations, RoundEnvironment roundEnv) { for (TypeElement annotation : annotations) { - if (!annotation.getSimpleName().contentEquals(ANNOTATION_SIMPLE_NAME)) { + if (!ANNOTATION_SIMPLE_NAMES.contains(annotation.getSimpleName().toString())) { continue; } processElements(roundEnv.getElementsAnnotatedWith(annotation)); @@ -176,7 +179,7 @@ private void ensurePostProcessorReturnsOMResponse( ExecutableElement elem, boolean isPreprocessor) { if (!isPreprocessor && !elem.getReturnType().toString() .equals(OM_RESPONSE_CLASS_NAME)) { - emitErrorMsg(ERROR_VALIDATOR_METHOD_HAS_TO_RETURN_OMRESPONSE); + emitErrorMsg(ERROR_VALIDATOR_METHOD_HAS_TO_RETURN_OMRESPONSE + " " + elem.getSimpleName()); } } @@ -203,20 +206,12 @@ private void checkMethodIsAnnotated(Element elem) { private boolean checkAndEvaluateAnnotation( AnnotationMirror methodAnnotation) { boolean isPreprocessor = false; - boolean hasValidConditions = false; - boolean hasValidMaxClientVersions = false; for (Entry entry : methodAnnotation.getElementValues().entrySet()) { - hasValidConditions = hasValidConditions || hasValidValidationCondition(entry); - if (isProcessingPhaseValue(entry)) { isPreprocessor = evaluateProcessingPhase(entry); } - hasValidMaxClientVersions = hasValidMaxClientVersions || hasValidMaxClientVersion(entry); - } - if (!hasValidMaxClientVersions && !hasValidConditions) { - emitErrorMsg(ERROR_CONDITION_IS_EMPTY); } return isPreprocessor; diff --git a/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/RegisterValidatorProcessor.java b/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/RegisterValidatorProcessor.java new file mode 100644 index 000000000000..f57eec33d09b --- /dev/null +++ b/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/RegisterValidatorProcessor.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ozone.annotations; + +import javax.annotation.processing.AbstractProcessor; +import javax.annotation.processing.RoundEnvironment; +import javax.annotation.processing.SupportedAnnotationTypes; +import javax.annotation.processing.SupportedSourceVersion; +import javax.lang.model.SourceVersion; +import javax.lang.model.element.Element; +import javax.lang.model.element.ElementKind; +import javax.lang.model.element.ExecutableElement; +import javax.lang.model.element.TypeElement; +import javax.lang.model.util.Elements; +import javax.lang.model.util.Types; +import javax.tools.Diagnostic; +import java.util.Set; + +@SupportedAnnotationTypes("org.apache.hadoop.ozone.om.request.validation.RegisterValidator") +@SupportedSourceVersion(SourceVersion.RELEASE_8) +public class RegisterValidatorProcessor extends AbstractProcessor { + + public static final String ANNOTATION_SIMPLE_NAME = "RegisterValidator"; + public static final String VERSION_CLASS_NAME = "org.apache.hadoop.ozone.Version"; + public static final String REQUEST_PROCESSING_PHASE_CLASS_NAME = "org.apache.hadoop.ozone.om.request.validation" + + ".RequestProcessingPhase"; + public static final String MAX_VERSION_METHOD_NAME = "maxVersion"; + public static final String REQUEST_TYPE_METHOD_NAME = "requestType"; + public static final String PROCESSING_PHASE_METHOD_NAME = "processingPhase"; + + public static final String MAX_VERSION_NOT_FOUND_ERROR_MESSAGE = "Method " + MAX_VERSION_METHOD_NAME + + " returning an enum implementing " + VERSION_CLASS_NAME + " not found"; + public static final String REQUEST_TYPE_NOT_FOUND_ERROR_MESSAGE = "Method " + REQUEST_TYPE_METHOD_NAME + + " returning an enum not found"; + public static final String PROCESSING_PHASE_NOT_FOUND_ERROR_MESSAGE = "Method " + PROCESSING_PHASE_METHOD_NAME + + " returning an enum implementing " + REQUEST_PROCESSING_PHASE_CLASS_NAME + " not found"; + + @Override + public boolean process(Set annotations, RoundEnvironment roundEnv) { + for (TypeElement annotation : annotations) { + if (!annotation.getSimpleName().contentEquals(ANNOTATION_SIMPLE_NAME)) { + continue; + } + processElements(roundEnv.getElementsAnnotatedWith(annotation)); + } + return false; + } + + private boolean validateMethod(ExecutableElement method, String expectedMethodName, ElementKind expectedReturnType, + String expectedReturnClass) { + Elements elementUtils = processingEnv.getElementUtils(); + Types types = processingEnv.getTypeUtils(); + TypeElement expectedReturnInterface = expectedReturnClass == null || expectedReturnClass.equals("") ? null : + elementUtils.getTypeElement(expectedReturnClass); + return method.getSimpleName().toString().equals(expectedMethodName) && (expectedReturnType == null || + types.asElement(method.getReturnType()) != null && + types.asElement(method.getReturnType()).getKind() == expectedReturnType) && + (expectedReturnInterface == null || + types.isAssignable(types.asElement(method.getReturnType()).asType(), expectedReturnInterface.asType())); + } + + private void processElements(Set annotatedElements) { + for (Element element : annotatedElements) { + if (element.getKind().equals(ElementKind.ANNOTATION_TYPE)) { + boolean hasMaxVersion = false; + boolean hasRequestType = false; + boolean hasRequestProcessPhase = false; + for (Element enclosedElement : element.getEnclosedElements()) { + // Check if the annotation has a method called "validatorName" returning a String + if (enclosedElement instanceof ExecutableElement) { + ExecutableElement method = (ExecutableElement) enclosedElement; + hasMaxVersion = hasMaxVersion || validateMethod(method, MAX_VERSION_METHOD_NAME, ElementKind.ENUM, + VERSION_CLASS_NAME); + hasRequestType = hasRequestType || validateMethod(method, REQUEST_TYPE_METHOD_NAME, ElementKind.ENUM, + null); + hasRequestProcessPhase = hasRequestProcessPhase || validateMethod(method, PROCESSING_PHASE_METHOD_NAME, + ElementKind.ENUM, REQUEST_PROCESSING_PHASE_CLASS_NAME); + } + } + if (!hasMaxVersion) { + emitErrorMsg(MAX_VERSION_NOT_FOUND_ERROR_MESSAGE + " for " + + element.getSimpleName().toString()); + } + if (!hasRequestType) { + emitErrorMsg(REQUEST_TYPE_NOT_FOUND_ERROR_MESSAGE + " for " + + element.getSimpleName().toString()); + } + if (!hasRequestProcessPhase) { + emitErrorMsg(PROCESSING_PHASE_NOT_FOUND_ERROR_MESSAGE + " for " + + element.getSimpleName().toString()); + } + } + } + } + + + private void emitErrorMsg(String s) { + processingEnv.getMessager().printMessage(Diagnostic.Kind.ERROR, s); + } +} diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ComponentVersion.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ComponentVersion.java index 9545869e1636..cc8951b2ad75 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ComponentVersion.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ComponentVersion.java @@ -17,10 +17,12 @@ */ package org.apache.hadoop.hdds; +import org.apache.hadoop.ozone.Version; + /** * Base type for component version enums. */ -public interface ComponentVersion { +public interface ComponentVersion extends Version { /** * Returns the description of the version enum value. @@ -34,4 +36,9 @@ public interface ComponentVersion { * @return the version associated with the enum value. */ int toProtoValue(); + + @Override + default int getVersion() { + return toProtoValue(); + } } diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java index f3bd1a96b662..df50cf9d2e57 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java @@ -49,6 +49,8 @@ public enum ClientVersion implements ComponentVersion { FUTURE_VERSION(-1, "Used internally when the server side is older and an" + " unknown client version has arrived from the client."); + public static final String ERASURE_CODING_SUPPORT_NAME = ERASURE_CODING_SUPPORT.name(); + public static final ClientVersion CURRENT = latest(); public static final int CURRENT_VERSION = CURRENT.version; @@ -57,7 +59,7 @@ public enum ClientVersion implements ComponentVersion { .collect(toMap(ClientVersion::toProtoValue, identity())); private final int version; - private final String description; + public final String description; ClientVersion(int version, String description) { this.version = version; diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/Version.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/Version.java new file mode 100644 index 000000000000..1e399b188b76 --- /dev/null +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/Version.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone; + + +/** + * Base class defining the version in the entire system. + */ +public interface Version { + int getVersion(); +} diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/LayoutFeature.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/LayoutFeature.java index 92dd706f4bb4..8d2faac6549d 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/LayoutFeature.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/LayoutFeature.java @@ -18,12 +18,14 @@ package org.apache.hadoop.ozone.upgrade; +import org.apache.hadoop.ozone.Version; + import java.util.Optional; /** * Generic Layout feature interface for Ozone. */ -public interface LayoutFeature { +public interface LayoutFeature extends Version { String name(); int layoutVersion(); @@ -48,6 +50,11 @@ default String name() { void execute(T arg) throws Exception; } + @Override + default int getVersion() { + return this.layoutVersion(); + } + /** * Phase of execution for this action. */ diff --git a/hadoop-hdds/common/src/main/resources/META-INF/services/javax.annotation.processing.Processor b/hadoop-hdds/common/src/main/resources/META-INF/services/javax.annotation.processing.Processor index 8d65dbc0764c..774933b13efd 100644 --- a/hadoop-hdds/common/src/main/resources/META-INF/services/javax.annotation.processing.Processor +++ b/hadoop-hdds/common/src/main/resources/META-INF/services/javax.annotation.processing.Processor @@ -14,5 +14,6 @@ # limitations under the License. org.apache.hadoop.hdds.conf.ConfigFileGenerator -org.apache.ozone.annotations.RequestFeatureValidatorProcessor +org.apache.ozone.annotations.OmRequestFeatureValidatorProcessor org.apache.ozone.annotations.ReplicateAnnotationProcessor +org.apache.ozone.annotations.RegisterValidatorProcessor \ No newline at end of file diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java index 74f626cebc8a..8708dc839d69 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java @@ -20,6 +20,7 @@ import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension; import org.apache.hadoop.hdds.client.DefaultReplicationConfig; +import org.apache.hadoop.ozone.om.request.validation.OMLayoutVersionValidator; import org.apache.ratis.server.protocol.TermIndex; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; @@ -42,9 +43,9 @@ import org.apache.hadoop.ozone.om.helpers.OzoneAclUtil; import org.apache.hadoop.ozone.om.request.OMClientRequest; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; -import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; +import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; -import org.apache.hadoop.ozone.om.request.validation.ValidationCondition; +import org.apache.hadoop.ozone.om.request.validation.VersionExtractor; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.om.response.bucket.OMBucketCreateResponse; @@ -395,10 +396,10 @@ public boolean checkQuotaBytesValid(OMMetadataManager metadataManager, } - @RequestFeatureValidator( - conditions = ValidationCondition.CLUSTER_NEEDS_FINALIZATION, + @OMLayoutVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, - requestType = Type.CreateBucket + requestType = Type.CreateBucket, + maxVersion = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT ) public static OMRequest disallowCreateBucketWithECReplicationConfig( OMRequest req, ValidationContext ctx) throws OMException { @@ -418,10 +419,10 @@ public static OMRequest disallowCreateBucketWithECReplicationConfig( return req; } - @RequestFeatureValidator( - conditions = ValidationCondition.CLUSTER_NEEDS_FINALIZATION, + @OMLayoutVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, - requestType = Type.CreateBucket + requestType = Type.CreateBucket, + maxVersion = OMLayoutFeature.BUCKET_LAYOUT_SUPPORT ) public static OMRequest handleCreateBucketWithBucketLayoutDuringPreFinalize( OMRequest req, ValidationContext ctx) throws OMException { @@ -459,10 +460,10 @@ public static OMRequest handleCreateBucketWithBucketLayoutDuringPreFinalize( * write to them, instead of using the server default which may be in a layout * they do not understand. */ - @RequestFeatureValidator( + @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CreateBucket, - maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT + maxVersion = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest setDefaultBucketLayoutForOlderClients(OMRequest req, ValidationContext ctx) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketDeleteRequest.java index 8be06c4b8f59..8bf4417906f2 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketDeleteRequest.java @@ -33,7 +33,7 @@ import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; -import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; +import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.slf4j.Logger; @@ -280,10 +280,10 @@ private boolean bucketContainsSnapshotInCache( * @return the validated request * @throws OMException if the request is invalid */ - @RequestFeatureValidator( + @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.DeleteBucket, - maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT + maxVersion = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockBucketDeleteWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketSetPropertyRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketSetPropertyRequest.java index 9c7ef1087c10..3fa3daddbe0d 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketSetPropertyRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketSetPropertyRequest.java @@ -25,6 +25,7 @@ import com.google.common.base.Preconditions; import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension; import org.apache.hadoop.hdds.client.DefaultReplicationConfig; +import org.apache.hadoop.ozone.om.request.validation.OMLayoutVersionValidator; import org.apache.ratis.server.protocol.TermIndex; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.audit.AuditLogger; @@ -33,9 +34,9 @@ import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.BucketEncryptionKeyInfo; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; -import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; +import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; -import org.apache.hadoop.ozone.om.request.validation.ValidationCondition; +import org.apache.hadoop.ozone.om.request.validation.VersionExtractor; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; @@ -357,10 +358,10 @@ public boolean checkQuotaNamespaceValid(OmVolumeArgs omVolumeArgs, return true; } - @RequestFeatureValidator( - conditions = ValidationCondition.CLUSTER_NEEDS_FINALIZATION, + @OMLayoutVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, - requestType = Type.SetBucketProperty + requestType = Type.SetBucketProperty, + maxVersion = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT ) public static OMRequest disallowSetBucketPropertyWithECReplicationConfig( OMRequest req, ValidationContext ctx) throws OMException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java index 504176c4c8ee..91a07df44db6 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java @@ -32,6 +32,7 @@ import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.ozone.ClientVersion; +import org.apache.hadoop.ozone.om.request.validation.OMLayoutVersionValidator; import org.apache.ratis.server.protocol.TermIndex; import org.apache.hadoop.ozone.OmUtils; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; @@ -40,9 +41,9 @@ import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; -import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; +import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; -import org.apache.hadoop.ozone.om.request.validation.ValidationCondition; +import org.apache.hadoop.ozone.om.request.validation.VersionExtractor; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; @@ -399,10 +400,10 @@ static long getMaxNumOfRecursiveDirs() { return MAX_NUM_OF_RECURSIVE_DIRS; } - @RequestFeatureValidator( - conditions = ValidationCondition.CLUSTER_NEEDS_FINALIZATION, + @OMLayoutVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, - requestType = Type.CreateDirectory + requestType = Type.CreateDirectory, + maxVersion = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT ) public static OMRequest disallowCreateDirectoryWithECReplicationConfig( OMRequest req, ValidationContext ctx) throws OMException { @@ -432,10 +433,10 @@ public static OMRequest disallowCreateDirectoryWithECReplicationConfig( * @return the validated request * @throws OMException if the request is invalid */ - @RequestFeatureValidator( + @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CreateDirectory, - maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT + maxVersion = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockCreateDirectoryWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java index 42bdf7c2ac8a..6727884d3448 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java @@ -30,6 +30,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.ozone.ClientVersion; +import org.apache.hadoop.ozone.om.request.validation.OMLayoutVersionValidator; import org.apache.ratis.server.protocol.TermIndex; import org.apache.hadoop.ozone.OmUtils; import org.apache.hadoop.ozone.OzoneConsts; @@ -37,9 +38,9 @@ import org.apache.hadoop.ozone.om.OzoneConfigUtil; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; -import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; +import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; -import org.apache.hadoop.ozone.om.request.validation.ValidationCondition; +import org.apache.hadoop.ozone.om.request.validation.VersionExtractor; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.response.file.OMFileCreateResponse; import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature; @@ -393,10 +394,10 @@ protected void checkAllParentsExist(KeyArgs keyArgs, } } - @RequestFeatureValidator( - conditions = ValidationCondition.CLUSTER_NEEDS_FINALIZATION, + @OMLayoutVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, - requestType = CreateFile + requestType = CreateFile, + maxVersion = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT ) public static OMRequest disallowCreateFileWithECReplicationConfig( OMRequest req, ValidationContext ctx) throws OMException { @@ -425,10 +426,10 @@ public static OMRequest disallowCreateFileWithECReplicationConfig( * @return the validated request * @throws OMException if the request is invalid */ - @RequestFeatureValidator( + @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CreateFile, - maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT + maxVersion = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockCreateFileWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java index 9dff7040d59b..716ef6cf0bb5 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java @@ -27,14 +27,15 @@ import com.google.common.base.Preconditions; import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.ozone.ClientVersion; +import org.apache.hadoop.ozone.om.request.validation.OMLayoutVersionValidator; import org.apache.ratis.server.protocol.TermIndex; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.QuotaUtil; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; -import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; +import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; -import org.apache.hadoop.ozone.om.request.validation.ValidationCondition; +import org.apache.hadoop.ozone.om.request.validation.VersionExtractor; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; @@ -268,10 +269,10 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn return omClientResponse; } - @RequestFeatureValidator( - conditions = ValidationCondition.CLUSTER_NEEDS_FINALIZATION, + @OMLayoutVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, - requestType = Type.AllocateBlock + requestType = Type.AllocateBlock, + maxVersion = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT ) public static OMRequest disallowAllocateBlockWithECReplicationConfig( OMRequest req, ValidationContext ctx) throws OMException { @@ -298,10 +299,10 @@ public static OMRequest disallowAllocateBlockWithECReplicationConfig( * @return the validated request * @throws OMException if the request is invalid */ - @RequestFeatureValidator( + @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.AllocateBlock, - maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT + maxVersion = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockAllocateBlockWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java index 22040f2ae272..883ee9e37c2f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java @@ -29,6 +29,7 @@ import com.google.common.base.Preconditions; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.ozone.ClientVersion; +import org.apache.hadoop.ozone.om.request.validation.OMLayoutVersionValidator; import org.apache.ratis.server.protocol.TermIndex; import org.apache.hadoop.ozone.OmUtils; import org.apache.hadoop.ozone.OzoneConfigKeys; @@ -43,9 +44,9 @@ import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.WithMetadata; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; -import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; +import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; -import org.apache.hadoop.ozone.om.request.validation.ValidationCondition; +import org.apache.hadoop.ozone.om.request.validation.VersionExtractor; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; @@ -433,10 +434,10 @@ protected void processResult(CommitKeyRequest commitKeyRequest, } } - @RequestFeatureValidator( - conditions = ValidationCondition.CLUSTER_NEEDS_FINALIZATION, + @OMLayoutVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, - requestType = Type.CommitKey + requestType = Type.CommitKey, + maxVersion = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT ) public static OMRequest disallowCommitKeyWithECReplicationConfig( OMRequest req, ValidationContext ctx) throws OMException { @@ -463,10 +464,10 @@ public static OMRequest disallowCommitKeyWithECReplicationConfig( * @return the validated request * @throws OMException if the request is invalid */ - @RequestFeatureValidator( + @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CommitKey, - maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT + maxVersion = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockCommitKeyWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { @@ -482,10 +483,10 @@ public static OMRequest blockCommitKeyWithBucketLayoutFromOldClient( return req; } - @RequestFeatureValidator( - conditions = ValidationCondition.CLUSTER_NEEDS_FINALIZATION, + @OMLayoutVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, - requestType = Type.CommitKey + requestType = Type.CommitKey, + maxVersion = OMLayoutFeature.HSYNC ) public static OMRequest disallowHsync( OMRequest req, ValidationContext ctx) throws OMException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java index 1b6ae159db88..1aebcc6aee62 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java @@ -29,6 +29,7 @@ import com.google.common.base.Preconditions; import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.ozone.ClientVersion; +import org.apache.hadoop.ozone.om.request.validation.OMLayoutVersionValidator; import org.apache.ratis.server.protocol.TermIndex; import org.apache.hadoop.ozone.OmUtils; import org.apache.hadoop.ozone.om.OMConfigKeys; @@ -39,9 +40,9 @@ import org.apache.hadoop.ozone.om.request.file.OMDirectoryCreateRequest; import org.apache.hadoop.ozone.om.request.file.OMFileRequest; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; -import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; +import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; -import org.apache.hadoop.ozone.om.request.validation.ValidationCondition; +import org.apache.hadoop.ozone.om.request.validation.VersionExtractor; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.UserInfo; @@ -394,10 +395,10 @@ protected void logResult(CreateKeyRequest createKeyRequest, } } - @RequestFeatureValidator( - conditions = ValidationCondition.CLUSTER_NEEDS_FINALIZATION, + @OMLayoutVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, - requestType = Type.CreateKey + requestType = Type.CreateKey, + maxVersion = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT ) public static OMRequest disallowCreateKeyWithECReplicationConfig( OMRequest req, ValidationContext ctx) throws OMException { @@ -424,10 +425,10 @@ public static OMRequest disallowCreateKeyWithECReplicationConfig( * @return the validated request * @throws OMException if the request is invalid */ - @RequestFeatureValidator( + @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CreateKey, - maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT + maxVersion = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockCreateKeyWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java index 87740b4887ab..612576fd91f2 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java @@ -30,7 +30,7 @@ import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; -import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; +import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType; @@ -232,10 +232,10 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn * @return the validated request * @throws OMException if the request is invalid */ - @RequestFeatureValidator( + @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.DeleteKey, - maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT + maxVersion = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockDeleteKeyWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java index c6adacbe7650..aec37a002300 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java @@ -30,7 +30,7 @@ import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; -import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; +import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer; @@ -269,10 +269,10 @@ private Map buildAuditMap( * @return the validated request * @throws OMException if the request is invalid */ - @RequestFeatureValidator( + @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.RenameKey, - maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT + maxVersion = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockRenameKeyWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java index 4212826cf29d..51971408cfd1 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java @@ -37,7 +37,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; -import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; +import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.response.OMClientResponse; @@ -350,10 +350,10 @@ protected static void addDeletedKeys(Map auditMap, * @return the validated request * @throws OMException if the request is invalid */ - @RequestFeatureValidator( + @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.DeleteKeys, - maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT + maxVersion = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockDeleteKeysWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysRenameRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysRenameRequest.java index 6884c8a88a36..e3ca76e1ee5d 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysRenameRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysRenameRequest.java @@ -35,7 +35,7 @@ import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; -import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; +import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.response.OMClientResponse; @@ -296,10 +296,10 @@ private Map buildAuditMap(Map auditMap, * @return the validated request * @throws OMException if the request is invalid */ - @RequestFeatureValidator( + @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.RenameKeys, - maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT + maxVersion = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockRenameKeysWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAddAclRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAddAclRequest.java index cd3ee13a4185..673ee26a82cd 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAddAclRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAddAclRequest.java @@ -35,7 +35,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.request.util.ObjectParser; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; -import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; +import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.response.key.acl.OMKeyAclResponse; @@ -164,10 +164,10 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn * @return the validated request * @throws OMException if the request is invalid */ - @RequestFeatureValidator( + @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.AddAcl, - maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT + maxVersion = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockAddAclWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyRemoveAclRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyRemoveAclRequest.java index 122962e1db58..8f31b7dad405 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyRemoveAclRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyRemoveAclRequest.java @@ -35,7 +35,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.request.util.ObjectParser; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; -import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; +import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.response.key.acl.OMKeyAclResponse; @@ -165,10 +165,10 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn * @return the validated request * @throws OMException if the request is invalid */ - @RequestFeatureValidator( + @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.RemoveAcl, - maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT + maxVersion = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockRemoveAclWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeySetAclRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeySetAclRequest.java index 188fc474199c..174a0542d6f0 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeySetAclRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeySetAclRequest.java @@ -36,7 +36,7 @@ import org.apache.hadoop.ozone.om.helpers.OzoneAclUtil; import org.apache.hadoop.ozone.om.request.util.ObjectParser; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; -import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; +import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.response.key.acl.OMKeyAclResponse; @@ -161,10 +161,10 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn * @return the validated request * @throws OMException if the request is invalid */ - @RequestFeatureValidator( + @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.SetAcl, - maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT + maxVersion = ClientVersion.BUCKET_LAYOUT_SUPPORT ) public static OMRequest blockSetAclWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java index bac283238bee..fe8d196f72a3 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java @@ -23,6 +23,7 @@ import org.apache.hadoop.ozone.ClientVersion; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.om.helpers.KeyValueUtil; +import org.apache.hadoop.ozone.om.request.validation.OMLayoutVersionValidator; import org.apache.ratis.server.protocol.TermIndex; import org.apache.hadoop.ozone.audit.OMAction; import org.apache.hadoop.ozone.om.OMMetadataManager; @@ -38,9 +39,9 @@ import org.apache.hadoop.ozone.om.request.key.OMKeyRequest; import org.apache.hadoop.ozone.om.request.util.OMMultipartUploadUtils; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; -import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; +import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; -import org.apache.hadoop.ozone.om.request.validation.ValidationCondition; +import org.apache.hadoop.ozone.om.request.validation.VersionExtractor; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.om.response.s3.multipart.S3InitiateMultipartUploadResponse; @@ -287,10 +288,10 @@ protected void logResult(OzoneManager ozoneManager, } } - @RequestFeatureValidator( - conditions = ValidationCondition.CLUSTER_NEEDS_FINALIZATION, + @OMLayoutVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, - requestType = Type.InitiateMultiPartUpload + requestType = Type.InitiateMultiPartUpload, + maxVersion = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT ) public static OMRequest disallowInitiateMultiPartUploadWithECReplicationConfig( @@ -320,10 +321,10 @@ protected void logResult(OzoneManager ozoneManager, * @return the validated request * @throws OMException if the request is invalid */ - @RequestFeatureValidator( + @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.InitiateMultiPartUpload, - maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT + maxVersion = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockInitiateMPUWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java index 3b751f655552..52bdcb8459ac 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java @@ -24,15 +24,16 @@ import org.apache.hadoop.ozone.ClientVersion; import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.om.request.validation.OMLayoutVersionValidator; import org.apache.ratis.server.protocol.TermIndex; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.QuotaUtil; import org.apache.hadoop.ozone.om.request.util.OMMultipartUploadUtils; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; -import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; +import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; -import org.apache.hadoop.ozone.om.request.validation.ValidationCondition; +import org.apache.hadoop.ozone.om.request.validation.VersionExtractor; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; @@ -258,10 +259,10 @@ protected String getMultipartOpenKey(String multipartUploadID, getBucketLayout()); } - @RequestFeatureValidator( - conditions = ValidationCondition.CLUSTER_NEEDS_FINALIZATION, + @OMLayoutVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, - requestType = Type.AbortMultiPartUpload + requestType = Type.AbortMultiPartUpload, + maxVersion = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT ) public static OMRequest disallowAbortMultiPartUploadWithECReplicationConfig( OMRequest req, ValidationContext ctx) throws OMException { @@ -289,10 +290,10 @@ public static OMRequest disallowAbortMultiPartUploadWithECReplicationConfig( * @return the validated request * @throws OMException if the request is invalid */ - @RequestFeatureValidator( + @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.AbortMultiPartUpload, - maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT + maxVersion = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockMPUAbortWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java index 68bef1831870..a3d90fcebabc 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java @@ -20,6 +20,7 @@ import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.ozone.ClientVersion; +import org.apache.hadoop.ozone.om.request.validation.OMLayoutVersionValidator; import org.apache.ratis.server.protocol.TermIndex; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.audit.OMAction; @@ -34,9 +35,9 @@ import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo; import org.apache.hadoop.ozone.om.request.key.OMKeyRequest; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; -import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; +import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; -import org.apache.hadoop.ozone.om.request.validation.ValidationCondition; +import org.apache.hadoop.ozone.om.request.validation.VersionExtractor; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.om.response.s3.multipart @@ -349,10 +350,10 @@ private String getMultipartKey(String volumeName, String bucketName, keyName, uploadID); } - @RequestFeatureValidator( - conditions = ValidationCondition.CLUSTER_NEEDS_FINALIZATION, + @OMLayoutVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, - requestType = Type.CommitMultiPartUpload + requestType = Type.CommitMultiPartUpload, + maxVersion = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT ) public static OMRequest disallowCommitMultiPartUploadWithECReplicationConfig( OMRequest req, ValidationContext ctx) throws OMException { @@ -380,10 +381,10 @@ public static OMRequest disallowCommitMultiPartUploadWithECReplicationConfig( * @return the validated request * @throws OMException if the request is invalid */ - @RequestFeatureValidator( + @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CommitMultiPartUpload, - maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT + maxVersion = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockMPUCommitWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java index da3543bcfcd3..484c8af369e8 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java @@ -36,6 +36,7 @@ import org.apache.hadoop.ozone.om.OzoneConfigUtil; import org.apache.hadoop.ozone.om.request.file.OMDirectoryCreateRequestWithFSO; import org.apache.hadoop.ozone.om.request.file.OMFileRequest; +import org.apache.hadoop.ozone.om.request.validation.OMLayoutVersionValidator; import org.apache.hadoop.ozone.protocolPB.OMPBHelper; import org.apache.ratis.server.protocol.TermIndex; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; @@ -56,9 +57,9 @@ import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; import org.apache.hadoop.ozone.om.request.key.OMKeyRequest; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; -import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; +import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; -import org.apache.hadoop.ozone.om.request.validation.ValidationCondition; +import org.apache.hadoop.ozone.om.request.validation.VersionExtractor; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.om.response.s3.multipart.S3MultipartUploadCompleteResponse; @@ -710,10 +711,10 @@ private void updateCache(OMMetadataManager omMetadataManager, } } - @RequestFeatureValidator( - conditions = ValidationCondition.CLUSTER_NEEDS_FINALIZATION, + @OMLayoutVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, - requestType = Type.CompleteMultiPartUpload + requestType = Type.CompleteMultiPartUpload, + maxVersion = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT ) public static OMRequest disallowCompleteMultiPartUploadWithECReplicationConfig( @@ -742,10 +743,10 @@ private void updateCache(OMMetadataManager omMetadataManager, * @return the validated request * @throws OMException if the request is invalid */ - @RequestFeatureValidator( + @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CompleteMultiPartUpload, - maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT + maxVersion = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockMPUCompleteWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestFeatureValidator.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/OMClientVersionValidator.java similarity index 82% rename from hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestFeatureValidator.java rename to hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/OMClientVersionValidator.java index 4a8f65684b4a..f0528e94f7e3 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestFeatureValidator.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/OMClientVersionValidator.java @@ -31,7 +31,7 @@ * {@link ValidatorRegistry} class during the initialization of the server. * * The conditions specify the specific use case in which the validator should be - * applied to the request. See {@link ValidationCondition} for more details + * applied to the request. See {@link VersionExtractor} for more details * on the specific conditions. * The validator method should be applied to just one specific request type * to help keep these methods simple and straightforward. If you want to use @@ -76,13 +76,8 @@ */ @Retention(RetentionPolicy.RUNTIME) @Target(ElementType.METHOD) -public @interface RequestFeatureValidator { - - /** - * Runtime conditions in which a validator should run. - * @return a list of conditions when the validator should be applied - */ - ValidationCondition[] conditions() default {}; +@RegisterValidator +public @interface OMClientVersionValidator { /** * Defines if the validation has to run before or after the general request @@ -97,15 +92,11 @@ */ Type requestType(); - /** - * The max client version for which the validator would run. This condition happens in parallel with the specified - * validation conditions. The validator would run either one of the specified conditions apply or the request - * client version is equal to or older than the specified version. The value defaults to FUTURE_VERSION so the - * validator would never run for any of the clients, the validator could still run based on the specified - * conditions though. - * @returns the max required client version for which the validator runs for older clients. + * The max version for which the validator would run. The validator would run for the request + * where the version is older than the excluding of the specified version. + * @returns the max required client version for which the validator runs for older version. */ - ClientVersion maxClientVersion() default ClientVersion.FUTURE_VERSION; + ClientVersion maxVersion(); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/OMLayoutVersionValidator.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/OMLayoutVersionValidator.java new file mode 100644 index 000000000000..db0f7e4996e6 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/OMLayoutVersionValidator.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.ozone.om.request.validation; + +import org.apache.hadoop.ozone.ClientVersion; +import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * An annotation to mark methods that do certain request validations. + * + * The methods annotated with this annotation are collected by the + * {@link ValidatorRegistry} class during the initialization of the server. + * + * The conditions specify the specific use case in which the validator should be + * applied to the request. See {@link VersionExtractor} for more details + * on the specific conditions. + * The validator method should be applied to just one specific request type + * to help keep these methods simple and straightforward. If you want to use + * the same validation for different request types, use inheritance, and + * annotate the override method that just calls super. + * Note that the aim is to have these validators together with the request + * processing code, so the handling of these specific situations are easy to + * find. + * + * The annotated methods have to have a fixed signature. + * A {@link RequestProcessingPhase#PRE_PROCESS} phase method is running before + * the request is processed by the regular code. + * Its signature has to be the following: + * - it has to be static and idempotent + * - it has to have two parameters + * - the first parameter it is an + * {@link + * org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest} + * - the second parameter of type {@link ValidationContext} + * - the method has to return the modified request, or throw a ServiceException + * in case the request is considered to be invalid + * - the method does not need to care about preserving the request it gets, + * the original request is captured and saved by the calling environment. + * + * A {@link RequestProcessingPhase#POST_PROCESS} phase method is running once + * the + * {@link + * org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse} + * is calculated for a given request. + * Its signature has to be the following: + * - it has to be static and idempotent + * - it has three parameters + * - similalry to the pre-processing validators, first parameter is the + * OMRequest, the second parameter is the OMResponse, and the third + * parameter is a ValidationContext. + * - the method has to return the modified OMResponse or throw a + * ServiceException if the request is considered invalid based on response. + * - the method gets the request object that was supplied for the general + * request processing code, not the original request, while it gets a copy + * of the original response object provided by the general request processing + * code. + */ +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.METHOD) +@RegisterValidator +public @interface OMLayoutVersionValidator { + + /** + * Defines if the validation has to run before or after the general request + * processing. + * @return if this is a pre or post processing validator + */ + RequestProcessingPhase processingPhase(); + + /** + * The type of the request handled by this validator method. + * @return the requestType to whihc the validator shoudl be applied + */ + Type requestType(); + + /** + * The max version for which the validator would run. The validator would run for the request + * where the version is older than the excluding of the specified version. + * @returns the max required client version for which the validator runs for older version. + */ + OMLayoutFeature maxVersion(); + +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RegisterValidator.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RegisterValidator.java new file mode 100644 index 000000000000..88de9a137215 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RegisterValidator.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.om.request.validation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.ANNOTATION_TYPE) +public @interface RegisterValidator { + public static final String MAX_VERSION_METHOD_NAME = "maxVersion"; + public static final String REQUEST_TYPE_METHOD_NAME = "requestType"; + public static final String PROCESSING_PHASE_METHOD_NAME = "processingPhase"; +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestValidations.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestValidations.java index 0a13feee8e1b..e286d3d93d2c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestValidations.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestValidations.java @@ -16,10 +16,13 @@ */ package org.apache.hadoop.ozone.om.request.validation; +import com.google.common.collect.Sets; import com.google.protobuf.ServiceException; +import org.apache.hadoop.ozone.Version; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -27,6 +30,8 @@ import java.lang.reflect.Method; import java.util.Arrays; import java.util.List; +import java.util.Objects; +import java.util.Set; import java.util.stream.Collectors; import static org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase.POST_PROCESS; @@ -40,10 +45,13 @@ public class RequestValidations { static final Logger LOG = LoggerFactory.getLogger(RequestValidations.class); private static final String DEFAULT_PACKAGE = "org.apache.hadoop.ozone"; - + private static final Set ALLOWED_REQUEST_PROCESSING_PHASES = + Sets.immutableEnumSet(PRE_PROCESS, POST_PROCESS); private String validationsPackageName = DEFAULT_PACKAGE; private ValidationContext context = null; - private ValidatorRegistry registry = null; + private ValidatorRegistry registry = null; + + public synchronized RequestValidations fromPackage(String packageName) { validationsPackageName = packageName; @@ -56,16 +64,18 @@ public RequestValidations withinContext(ValidationContext validationContext) { } public synchronized RequestValidations load() { - registry = new ValidatorRegistry(validationsPackageName); + registry = new ValidatorRegistry<>(Type.class, validationsPackageName, + Arrays.stream(VersionExtractor.values()).map(VersionExtractor::getVersionClass).collect(Collectors.toSet()), + ALLOWED_REQUEST_PROCESSING_PHASES); return this; } public OMRequest validateRequest(OMRequest request) throws Exception { - List validations = registry.validationsFor( - conditions(request), request.getCmdType(), PRE_PROCESS, request.getVersion()); - OMRequest validatedRequest = request; + List validations = registry.validationsFor(request.getCmdType(), PRE_PROCESS, + this.getVersions(request)); + OMRequest validatedRequest = request.toBuilder().build(); try { for (Method m : validations) { LOG.debug("Running the {} request pre-process validation from {}.{}", @@ -87,10 +97,10 @@ public OMRequest validateRequest(OMRequest request) public OMResponse validateResponse(OMRequest request, OMResponse response) throws ServiceException { - List validations = registry.validationsFor( - conditions(request), request.getCmdType(), POST_PROCESS, request.getVersion()); - OMResponse validatedResponse = response; + List validations = registry.validationsFor(request.getCmdType(), POST_PROCESS, this.getVersions(request)); + + OMResponse validatedResponse = response.toBuilder().build(); try { for (Method m : validations) { LOG.debug("Running the {} request post-process validation from {}.{}", @@ -105,9 +115,10 @@ public OMResponse validateResponse(OMRequest request, OMResponse response) return validatedResponse; } - private List conditions(OMRequest request) { - return Arrays.stream(ValidationCondition.values()) - .filter(c -> c.shouldApply(request, context)) + private List getVersions(OMRequest request) { + return Arrays.stream(VersionExtractor.values()) + .map(versionExtractor -> versionExtractor.extractVersion(request, context)) + .filter(Objects::nonNull) .collect(Collectors.toList()); } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidationCondition.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidationCondition.java deleted file mode 100644 index 58eacff7771e..000000000000 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidationCondition.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ -package org.apache.hadoop.ozone.om.request.validation; - -import org.apache.hadoop.ozone.ClientVersion; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; - -/** - * Defines conditions for which validators can be assigned to. - * - * These conditions describe a situation where special request handling might - * be necessary. In these cases we do not override the actual request handling - * code, but based on certain request properties we might reject a request - * early, or we might modify the request, or the response received/sent from/to - * the client. - */ -public enum ValidationCondition { - /** - * Classifies validations that has to run after an upgrade until the cluster - * is in a pre-finalized state. - */ - CLUSTER_NEEDS_FINALIZATION { - @Override - public boolean shouldApply(OMRequest req, ValidationContext ctx) { - return ctx.versionManager().needsFinalization(); - } - }, - - /** - * Classifies validations that has to run, when the client uses an older - * protocol version than the server. - */ - @Deprecated - OLDER_CLIENT_REQUESTS { - @Override - public boolean shouldApply(OMRequest req, ValidationContext ctx) { - return req.getVersion() < ClientVersion.CURRENT_VERSION; - } - }; - - public abstract boolean shouldApply(OMRequest req, ValidationContext ctx); -} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidatorRegistry.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidatorRegistry.java index 391ac41f454a..37bbc2ef81ff 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidatorRegistry.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidatorRegistry.java @@ -17,21 +17,21 @@ package org.apache.hadoop.ozone.om.request.validation; import org.apache.commons.lang3.tuple.Pair; -import org.apache.hadoop.ozone.ClientVersion; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; +import org.apache.commons.lang3.tuple.Triple; +import org.apache.hadoop.ozone.Version; import org.reflections.Reflections; import org.reflections.scanners.Scanners; import org.reflections.util.ClasspathHelper; import org.reflections.util.ConfigurationBuilder; +import java.lang.annotation.Annotation; +import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.net.URL; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.EnumMap; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -40,229 +40,240 @@ import java.util.function.Supplier; import java.util.stream.Collectors; -import static org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase.POST_PROCESS; -import static org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase.PRE_PROCESS; - /** * Registry that loads and stores the request validators to be applied by * a service. */ -public class ValidatorRegistry { - - private final EnumMap>>> - validators = new EnumMap<>(ValidationCondition.class); - private final Map, Pair, TreeMap>> - maxAllowedVersionValidatorMap = new HashMap<>(Type.values().length * RequestProcessingPhase.values().length, - 1.0f); +public class ValidatorRegistry> { + private final Map>, RequestType, RequestProcessingPhase>, + IndexedItems> indexedValidatorMap; /** * Creates a {@link ValidatorRegistry} instance that discovers validation * methods in the provided package and the packages in the same resource. - * A validation method is recognized by the {@link RequestFeatureValidator} + * A validation method is recognized by the {@link OMClientVersionValidator} * annotation that contains important information about how and when to use * the validator. * @param validatorPackage the main package inside which validatiors should * be discovered. */ - ValidatorRegistry(String validatorPackage) { - this(ClasspathHelper.forPackage(validatorPackage)); + ValidatorRegistry(Class requestType, + String validatorPackage, + Set>> allowedVersionTypes, + Set allowedProcessingPhases) { + this(requestType, ClasspathHelper.forPackage(validatorPackage), allowedVersionTypes, allowedProcessingPhases); + } + + private Class getReturnTypeOfAnnotationMethod(Class clzz, String methodName) { + try { + return clzz.getMethod(methodName).getReturnType(); + } catch (NoSuchMethodException e) { + throw new IllegalArgumentException("Method " + methodName + " not found in class:" + clzz.getCanonicalName()); + } } /** * Creates a {@link ValidatorRegistry} instance that discovers validation * methods under the provided URL. - * A validation method is recognized by the {@link RequestFeatureValidator} + * A validation method is recognized by the {@link OMClientVersionValidator} * annotation that contains important information about how and when to use * the validator. * @param searchUrls the path in which the annotated methods are searched. */ - ValidatorRegistry(Collection searchUrls) { + ValidatorRegistry(Class requestType, + Collection searchUrls, + Set>> allowedVersionTypes, + Set allowedProcessingPhases) { + Set> validatorsToBeRegistered = + new Reflections(new ConfigurationBuilder().setUrls(ClasspathHelper.forPackage("")) + .setScanners(Scanners.TypesAnnotated) + .setParallel(true)).getTypesAnnotatedWith(RegisterValidator.class).stream() + .filter(annotationClass -> + getReturnTypeOfAnnotationMethod((Class) annotationClass, RegisterValidator.REQUEST_TYPE_METHOD_NAME) + .equals(requestType)) + .filter(annotationClass -> allowedVersionTypes.contains(getReturnTypeOfAnnotationMethod( + (Class) annotationClass, + RegisterValidator.MAX_VERSION_METHOD_NAME))) + .map(annotationClass -> (Class) annotationClass) + .collect(Collectors.toSet()); + int expectedNumberOfKeys = requestType.getEnumConstants().length * allowedProcessingPhases.size() * + allowedVersionTypes.stream().mapToInt(clz -> clz.getEnumConstants().length).sum(); + this.indexedValidatorMap = new HashMap<>(expectedNumberOfKeys, 1.0f); Reflections reflections = new Reflections(new ConfigurationBuilder() .setUrls(searchUrls) .setScanners(Scanners.MethodsAnnotated) .setParallel(true) ); - - Set describedValidators = - reflections.getMethodsAnnotatedWith(RequestFeatureValidator.class); - initMaps(describedValidators); + initMaps(requestType, allowedProcessingPhases, validatorsToBeRegistered, reflections); } /** - * Get the validators that has to be run in the given list of - * {@link ValidationCondition}s, for the given requestType and + * Get the validators that has to be run in the given list of, + * for the given requestType and for the given request versions. * {@link RequestProcessingPhase}. * - * @param conditions conditions that are present for the request * @param requestType the type of the protocol message * @param phase the request processing phase - * @param requestClientVersion the client version of the protocol message. + * @param requestVersions different versions extracted from the request. * @return the list of validation methods that has to run. */ - List validationsFor(List conditions, - Type requestType, - RequestProcessingPhase phase, - int requestClientVersion) { - Set methodsToRun = new HashSet<>(validationsFor(conditions, requestType, phase)); - methodsToRun.addAll(validationsFor(requestType, phase, requestClientVersion)); - return new ArrayList<>(methodsToRun); + public List validationsFor(RequestType requestType, RequestProcessingPhase phase, + List requestVersions) { + return requestVersions.stream() + .flatMap(requestVersion -> this.validationsFor(requestType, phase, requestVersion).stream()) + .distinct().collect(Collectors.toList()); } /** - * Get the validators that has to run - * that require {@Link ClientVersion}. minimum client version newer than the request client version, for the given - * requestType and {@link RequestProcessingPhase}. + * Get the validators that has to be run in the given list of, + * for the given requestType and for the given request versions. + * {@link RequestProcessingPhase}. * * @param requestType the type of the protocol message * @param phase the request processing phase - * @param requestClientVersion the client version of the protocol message. + * @param requestVersion version extracted corresponding to the request. * @return the list of validation methods that has to run. */ - private List validationsFor(Type requestType, - RequestProcessingPhase phase, - int requestClientVersion) { - Pair key = Pair.of(requestType, phase); - if (this.maxAllowedVersionValidatorMap.containsKey(key)) { - Pair, TreeMap> value = this.maxAllowedVersionValidatorMap.get(key); - return Optional.ofNullable(value.getRight().ceilingEntry(requestClientVersion)) - .map(Map.Entry::getValue) - .map(startIndex -> value.getKey().subList(startIndex, value.getKey().size())) - .orElse(Collections.emptyList()); + public List validationsFor(RequestType requestType, RequestProcessingPhase phase, + Version requestVersion) { + Class> requestVersionClass = + (Class>) requestVersion.getClass(); + Triple>, RequestType, RequestProcessingPhase> key = + Triple.of(requestVersionClass, requestType, phase); + if (this.indexedValidatorMap.containsKey(key)) { + return this.indexedValidatorMap.get(key).getItemsGreaterThanIdx(requestVersion.getVersion()); } return Collections.emptyList(); } - /** - * Get the validators that has to be run in the given list of - * {@link ValidationCondition}s, for the given requestType and - * {@link RequestProcessingPhase}. - * - * @param conditions conditions that are present for the request - * @param requestType the type of the protocol message - * @param phase the request processing phase - * @return the list of validation methods that has to run. + * Calls a specified method on the validator. + * @Throws IllegalArgumentException when the specified method in the validator is invalid. */ - private Set validationsFor( - List conditions, - Type requestType, - RequestProcessingPhase phase) { - - if (conditions.isEmpty() || validators.isEmpty()) { - return Collections.emptySet(); + private ReturnValue callAnnotationMethod( + Validator validator, String methodName, Class returnValueType) { + try { + return (ReturnValue) validator.getClass().getMethod(methodName).invoke(validator); + } catch (NoSuchMethodException e) { + throw new IllegalArgumentException("Method " + methodName + " not found in class:" + + validator.getClass().getCanonicalName(), e); + } catch (InvocationTargetException | IllegalAccessException e) { + throw new IllegalArgumentException("Error while invoking Method " + methodName + " from " + + validator.getClass().getCanonicalName(), e); } + } - Set returnValue = new HashSet<>(); - - for (ValidationCondition condition: conditions) { - returnValue.addAll(validationsFor(condition, requestType, phase)); - } - return returnValue; + private Version getMaxVersion(Validator validator) { + return callAnnotationMethod(validator, RegisterValidator.MAX_VERSION_METHOD_NAME, Version.class); } - /** - * Grabs validations for one particular condition. - * - * @param condition conditions that are present for the request - * @param requestType the type of the protocol message - * @param phase the request processing phase - * @return the list of validation methods that has to run. - */ - private List validationsFor( - ValidationCondition condition, - Type requestType, - RequestProcessingPhase phase) { + private RequestProcessingPhase getRequestPhase(Validator validator) { + return callAnnotationMethod(validator, RegisterValidator.PROCESSING_PHASE_METHOD_NAME, + RequestProcessingPhase.class); + } - EnumMap>> - requestTypeMap = validators.get(condition); - if (requestTypeMap == null || requestTypeMap.isEmpty()) { - return Collections.emptyList(); - } + private RequestType getRequestType(Validator validator, + Class requestType) { + return callAnnotationMethod(validator, RegisterValidator.REQUEST_TYPE_METHOD_NAME, requestType); + } - EnumMap> phases = - requestTypeMap.get(requestType); - if (phases == null) { - return Collections.emptyList(); - } - List validatorsForPhase = phases.get(phase); - if (validatorsForPhase == null) { - return Collections.emptyList(); + private void checkAllowedAnnotationValues(Set values, V value, String valueName, String methodName) { + if (!values.contains(value)) { + throw new IllegalArgumentException( + String.format("Invalid %1$s defined at annotation defined for method : %2$s, Annotation value : %3$s " + + "Allowed versionType: %4$s", valueName, methodName, value.toString(), values)); } - return validatorsForPhase; - } - - private int getMaxAllowedClientVersion(Method method) { - RequestFeatureValidator descriptor = method.getAnnotation(RequestFeatureValidator.class); - ClientVersion maxAllowedClientVersion = descriptor.maxClientVersion(); - return maxAllowedClientVersion == null ? (ClientVersion.DEFAULT_VERSION.toProtoValue() - 1) : - maxAllowedClientVersion.toProtoValue(); } /** * Initializes the internal request validator store. * The requests are stored in the following structure: - * - An EnumMap with the {@link ValidationCondition} as the key, and in which + * - An EnumMap with the {@link VersionExtractor} as the key, and in which * - values are an EnumMap with the request type as the key, and in which * - values are Pair of lists, in which * - left side is the pre-processing validations list * - right side is the post-processing validations list - * @param describedValidators collection of the annotated methods to process. + * @param validatorsToBeRegistered collection of the annotated validtors to process. */ - void initMaps(Collection describedValidators) { - List sortedMethodsByMaxVersion = describedValidators.stream() - .sorted((method1, method2) -> Integer.compare(getMaxAllowedClientVersion(method1), - getMaxAllowedClientVersion(method2))) - .collect(Collectors.toList()); - for (Method m : sortedMethodsByMaxVersion) { - RequestFeatureValidator descriptor = m.getAnnotation(RequestFeatureValidator.class); - m.setAccessible(true); - Type requestType = descriptor.requestType(); - - for (ValidationCondition condition : descriptor.conditions()) { - EnumMap>> - requestTypeMap = getAndInitialize( - condition, this::newTypeMap, validators); - EnumMap> phases = getAndInitialize( - requestType, this::newPhaseMap, requestTypeMap); - if (isPreProcessValidator(descriptor)) { - getAndInitialize(PRE_PROCESS, ArrayList::new, phases).add(m); - } else if (isPostProcessValidator(descriptor)) { - getAndInitialize(POST_PROCESS, ArrayList::new, phases).add(m); - } - } - Pair indexKey = Pair.of(descriptor.requestType(), descriptor.processingPhase()); - int maxAllowedClientVersion = this.getMaxAllowedClientVersion(m); - Pair, TreeMap> value = - getAndInitialize(indexKey, () -> Pair.of(new ArrayList<>(), new TreeMap<>()), maxAllowedVersionValidatorMap); - List methods = value.getKey(); - value.getRight().putIfAbsent(maxAllowedClientVersion, methods.size()); - methods.add(m); + private void initMaps(Class requestType, + Set allowedPhases, + Collection> validatorsToBeRegistered, + Reflections reflections) { + for (Class validator : validatorsToBeRegistered) { + registerValidator(requestType, allowedPhases, validator, reflections); } } - private EnumMap>> newTypeMap() { - return new EnumMap<>(Type.class); - } - private EnumMap> newPhaseMap() { - return new EnumMap<>(RequestProcessingPhase.class); + + private void registerValidator(Class requestType, + Set allowedPhases, + Class validatorToBeRegistered, + Reflections reflections) { + Collection methods = reflections.getMethodsAnnotatedWith(validatorToBeRegistered); + Class> versionClass = (Class>) + this.getReturnTypeOfAnnotationMethod(validatorToBeRegistered, RegisterValidator.MAX_VERSION_METHOD_NAME); + List> sortedMethodsByMaxVersion = methods.stream() + .map(method -> Pair.of(method.getAnnotation(validatorToBeRegistered), method)) + .sorted((validatorMethodPair1, validatorMethodPair2) -> + Integer.compare( + this.getMaxVersion(validatorMethodPair1.getKey()).getVersion(), + this.getMaxVersion(validatorMethodPair2.getKey()).getVersion())) + .collect(Collectors.toList()); + for (Pair validatorMethodPair : sortedMethodsByMaxVersion) { + Annotation validator = validatorMethodPair.getKey(); + Method method = validatorMethodPair.getValue(); + Version maxVersion = this.getMaxVersion(validator); + RequestProcessingPhase phase = this.getRequestPhase(validator); + checkAllowedAnnotationValues(allowedPhases, phase, RegisterValidator.PROCESSING_PHASE_METHOD_NAME, + method.getName()); + RequestType type = this.getRequestType(validator, requestType); + method.setAccessible(true); + + Triple>, RequestType, RequestProcessingPhase> indexKey = + Triple.of(versionClass, type, phase); + this.getAndInitialize(indexKey, IndexedItems::new, this.indexedValidatorMap) + .add(method, maxVersion.getVersion()); + } + } private V getAndInitialize(K key, Supplier defaultSupplier, Map from) { return from.computeIfAbsent(key, k -> defaultSupplier.get()); } - private boolean isPreProcessValidator(RequestFeatureValidator descriptor) { - return descriptor.processingPhase() - .equals(PRE_PROCESS); - } + class IndexedItems> { + private final List items; + private final TreeMap indexMap; + + public IndexedItems() { + this.items = new ArrayList<>(); + this.indexMap = new TreeMap<>(); + } + + /** + * Add an item to the collection and update index if required. The order of items added should have their index + * sorted in increasing order. + * @param item + * @param idx + */ + public void add(T item, IDX idx) { + indexMap.putIfAbsent(idx, items.size()); + items.add(item); + } + + /** + * + * @param indexValue Given index value. + * @return All the items which has an index value greater than given index value. + */ + public List getItemsGreaterThanIdx(IDX indexValue) { + return Optional.ofNullable(indexMap.higherEntry(indexValue)) + .map(Map.Entry::getValue) + .map(startIndex -> items.subList(startIndex , items.size())).orElse(Collections.emptyList()); + } - private boolean isPostProcessValidator(RequestFeatureValidator descriptor) { - return descriptor.processingPhase() - .equals(POST_PROCESS); } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/VersionExtractor.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/VersionExtractor.java new file mode 100644 index 000000000000..3e95545f3e7b --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/VersionExtractor.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.ozone.om.request.validation; + +import org.apache.hadoop.ozone.ClientVersion; +import org.apache.hadoop.ozone.Version; +import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; +import org.apache.hadoop.ozone.upgrade.LayoutVersionManager; + +/** + * Class to extract version out of OM request. + */ +public enum VersionExtractor { + /** + * Extracts current software layout version. + */ + LAYOUT_VERSION_EXTRACTOR { + @Override + public Version extractVersion(OMRequest req, ValidationContext ctx) { + LayoutVersionManager layoutVersionManager = ctx.versionManager(); + return ctx.versionManager().getFeature(layoutVersionManager.getSoftwareLayoutVersion()); + } + + @Override + public Class> getVersionClass() { + return OMLayoutFeature.class; + } + }, + + /** + * Extracts client version from the OMRequests. + */ + CLIENT_VERSION_EXTRACTOR { + @Override + public Version extractVersion(OMRequest req, ValidationContext ctx) { + return ClientVersion.fromProtoValue(req.getVersion()); + } + + @Override + public Class> getVersionClass() { + return ClientVersion.class; + } + }; + + public abstract Version extractVersion(OMRequest req, ValidationContext ctx); + public abstract Class> getVersionClass(); +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/package-info.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/package-info.java index 8f5ac133b19f..c149734cc910 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/package-info.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/package-info.java @@ -31,19 +31,19 @@ * - the code can handle certain checks that have to run all the time, but at * first we do not see a general use case that we would pull in immediately. * These are the current - * {@link org.apache.hadoop.ozone.om.request.validation.ValidationCondition}s + * {@link org.apache.hadoop.ozone.om.request.validation.VersionExtractor}s * but this list might be extended later on if we see other use cases. * * The system uses a reflection based discovery to find methods that are * annotated with the - * {@link org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator} + * {@link org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator} * annotation. * This annotation is used to specify the condition in which a certain validator * has to be used, the request type to which the validation should be applied, * and the request processing phase in which we apply the validation. * * One validator can be applied in multiple - * {@link org.apache.hadoop.ozone.om.request.validation.ValidationCondition} + * {@link org.apache.hadoop.ozone.om.request.validation.VersionExtractor} * but a validator has to handle strictly just one * {@link org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type * }. diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java index d931c128a901..247a89e63666 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java @@ -28,6 +28,8 @@ * List of OM Layout features / versions. */ public enum OMLayoutFeature implements LayoutFeature { + + ////////////////////////////// ////////////////////////////// INITIAL_VERSION(0, "Initial Layout Version"), diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java index 13d0904985f4..3033718193ab 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java @@ -72,7 +72,7 @@ import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; import org.apache.hadoop.ozone.om.request.OMClientRequest; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; -import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; +import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.response.OMClientResponse; @@ -628,10 +628,10 @@ private GetKeyInfoResponse getKeyInfo(GetKeyInfoRequest request, return keyInfo.toProtobuf(clientVersion); } - @RequestFeatureValidator( + @OMClientVersionValidator( processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.LookupKey, - maxClientVersion = ClientVersion.VERSION_HANDLES_UNKNOWN_DN_PORTS + maxVersion = ClientVersion.VERSION_HANDLES_UNKNOWN_DN_PORTS ) public static OMResponse disallowLookupKeyResponseWithECReplicationConfig( OMRequest req, OMResponse resp, ValidationContext ctx) @@ -655,10 +655,10 @@ public static OMResponse disallowLookupKeyResponseWithECReplicationConfig( return resp; } - @RequestFeatureValidator( + @OMClientVersionValidator( processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.LookupKey, - maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT + maxVersion = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMResponse disallowLookupKeyWithBucketLayout( OMRequest req, OMResponse resp, ValidationContext ctx) @@ -742,10 +742,10 @@ private ListKeysLightResponse listKeysLight(ListKeysRequest request) return resp.build(); } - @RequestFeatureValidator( + @OMClientVersionValidator( processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.ListKeys, - maxClientVersion = ClientVersion.VERSION_HANDLES_UNKNOWN_DN_PORTS + maxVersion = ClientVersion.VERSION_HANDLES_UNKNOWN_DN_PORTS ) public static OMResponse disallowListKeysResponseWithECReplicationConfig( OMRequest req, OMResponse resp, ValidationContext ctx) @@ -769,10 +769,10 @@ public static OMResponse disallowListKeysResponseWithECReplicationConfig( return resp; } - @RequestFeatureValidator( + @OMClientVersionValidator( processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.ListKeys, - maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT + maxVersion = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMResponse disallowListKeysWithBucketLayout( OMRequest req, OMResponse resp, ValidationContext ctx) @@ -830,10 +830,10 @@ private ListTrashResponse listTrash(ListTrashRequest request, return resp.build(); } - @RequestFeatureValidator( + @OMClientVersionValidator( processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.ListTrash, - maxClientVersion = ClientVersion.VERSION_HANDLES_UNKNOWN_DN_PORTS + maxVersion = ClientVersion.VERSION_HANDLES_UNKNOWN_DN_PORTS ) public static OMResponse disallowListTrashWithECReplicationConfig( OMRequest req, OMResponse resp, ValidationContext ctx) @@ -860,10 +860,10 @@ public static OMResponse disallowListTrashWithECReplicationConfig( return resp; } - @RequestFeatureValidator( + @OMClientVersionValidator( processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.ListTrash, - maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT + maxVersion = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMResponse disallowListTrashWithBucketLayout( OMRequest req, OMResponse resp, ValidationContext ctx) @@ -1047,10 +1047,10 @@ private RefetchSecretKeyResponse refetchSecretKey() { return response; } - @RequestFeatureValidator( + @OMClientVersionValidator( processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.GetFileStatus, - maxClientVersion = ClientVersion.VERSION_HANDLES_UNKNOWN_DN_PORTS + maxVersion = ClientVersion.VERSION_HANDLES_UNKNOWN_DN_PORTS ) public static OMResponse disallowGetFileStatusWithECReplicationConfig( OMRequest req, OMResponse resp, ValidationContext ctx) @@ -1077,10 +1077,10 @@ public static OMResponse disallowGetFileStatusWithECReplicationConfig( } - @RequestFeatureValidator( + @OMClientVersionValidator( processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.GetFileStatus, - maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT + maxVersion = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMResponse disallowGetFileStatusWithBucketLayout( OMRequest req, OMResponse resp, ValidationContext ctx) @@ -1126,10 +1126,10 @@ private LookupFileResponse lookupFile(LookupFileRequest request, .build(); } - @RequestFeatureValidator( + @OMClientVersionValidator( processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.LookupFile, - maxClientVersion = ClientVersion.VERSION_HANDLES_UNKNOWN_DN_PORTS + maxVersion = ClientVersion.VERSION_HANDLES_UNKNOWN_DN_PORTS ) public static OMResponse disallowLookupFileWithECReplicationConfig( OMRequest req, OMResponse resp, ValidationContext ctx) @@ -1154,10 +1154,10 @@ public static OMResponse disallowLookupFileWithECReplicationConfig( return resp; } - @RequestFeatureValidator( + @OMClientVersionValidator( processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.LookupFile, - maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT + maxVersion = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMResponse disallowLookupFileWithBucketLayout( OMRequest req, OMResponse resp, ValidationContext ctx) @@ -1236,10 +1236,10 @@ private ListStatusLightResponse listStatusLight( return listStatusLightResponseBuilder.build(); } - @RequestFeatureValidator( + @OMClientVersionValidator( processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.ListStatus, - maxClientVersion = ClientVersion.VERSION_HANDLES_UNKNOWN_DN_PORTS + maxVersion = ClientVersion.VERSION_HANDLES_UNKNOWN_DN_PORTS ) public static OMResponse disallowListStatusResponseWithECReplicationConfig( OMRequest req, OMResponse resp, ValidationContext ctx) @@ -1264,10 +1264,10 @@ public static OMResponse disallowListStatusResponseWithECReplicationConfig( return resp; } - @RequestFeatureValidator( + @OMClientVersionValidator( processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.ListStatus, - maxClientVersion = ClientVersion.ERASURE_CODING_SUPPORT + maxVersion = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMResponse disallowListStatusResponseWithBucketLayout( OMRequest req, OMResponse resp, ValidationContext ctx) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestRequestFeatureValidatorProcessor.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestOMClientVersionValidatorProcessor.java similarity index 77% rename from hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestRequestFeatureValidatorProcessor.java rename to hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestOMClientVersionValidatorProcessor.java index 95a62d349a2e..d21134c95f4b 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestRequestFeatureValidatorProcessor.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestOMClientVersionValidatorProcessor.java @@ -20,7 +20,7 @@ import com.google.testing.compile.JavaFileObjects; import org.apache.hadoop.ozone.ClientVersion; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; -import org.apache.ozone.annotations.RequestFeatureValidatorProcessor; +import org.apache.ozone.annotations.OmRequestFeatureValidatorProcessor; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -36,31 +36,30 @@ import static com.google.testing.compile.CompilationSubject.assertThat; import static com.google.testing.compile.Compiler.javac; -import static org.apache.ozone.annotations.RequestFeatureValidatorProcessor.ERROR_CONDITION_IS_EMPTY; -import static org.apache.ozone.annotations.RequestFeatureValidatorProcessor.ERROR_FIRST_PARAM_HAS_TO_BE_OMREQUEST; -import static org.apache.ozone.annotations.RequestFeatureValidatorProcessor.ERROR_LAST_PARAM_HAS_TO_BE_VALIDATION_CONTEXT; -import static org.apache.ozone.annotations.RequestFeatureValidatorProcessor.ERROR_SECOND_PARAM_HAS_TO_BE_OMRESPONSE; -import static org.apache.ozone.annotations.RequestFeatureValidatorProcessor.ERROR_UNEXPECTED_PARAMETER_COUNT; -import static org.apache.ozone.annotations.RequestFeatureValidatorProcessor.ERROR_VALIDATOR_METHOD_HAS_TO_BE_STATIC; -import static org.apache.ozone.annotations.RequestFeatureValidatorProcessor.ERROR_VALIDATOR_METHOD_HAS_TO_RETURN_OMREQUEST; -import static org.apache.ozone.annotations.RequestFeatureValidatorProcessor.ERROR_VALIDATOR_METHOD_HAS_TO_RETURN_OMRESPONSE; +import static org.apache.ozone.annotations.OmRequestFeatureValidatorProcessor.ERROR_FIRST_PARAM_HAS_TO_BE_OMREQUEST; +import static org.apache.ozone.annotations.OmRequestFeatureValidatorProcessor.ERROR_LAST_PARAM_HAS_TO_BE_VALIDATION_CONTEXT; +import static org.apache.ozone.annotations.OmRequestFeatureValidatorProcessor.ERROR_SECOND_PARAM_HAS_TO_BE_OMRESPONSE; +import static org.apache.ozone.annotations.OmRequestFeatureValidatorProcessor.ERROR_UNEXPECTED_PARAMETER_COUNT; +import static org.apache.ozone.annotations.OmRequestFeatureValidatorProcessor.ERROR_VALIDATOR_METHOD_HAS_TO_BE_STATIC; +import static org.apache.ozone.annotations.OmRequestFeatureValidatorProcessor.ERROR_VALIDATOR_METHOD_HAS_TO_RETURN_OMREQUEST; +import static org.apache.ozone.annotations.OmRequestFeatureValidatorProcessor.ERROR_VALIDATOR_METHOD_HAS_TO_RETURN_OMRESPONSE; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertSame; /** * Compile tests against the annotation processor for the - * {@link RequestFeatureValidator} annotation. + * {@link OMClientVersionValidator} annotation. * * The processor should ensure the method signatures and return values, based * on annotation arguments provided. */ -public class TestRequestFeatureValidatorProcessor { +public class TestOMClientVersionValidatorProcessor { private static final String CLASSNAME = "Validation"; @Test public void testAnnotationCanOnlyBeAppliedOnMethods() { - Class c = RequestFeatureValidator.class; + Class c = OMClientVersionValidator.class; for (Annotation a : c.getAnnotations()) { if (a instanceof Target) { assertEquals(1, ((Target) a).value().length); @@ -72,7 +71,7 @@ public void testAnnotationCanOnlyBeAppliedOnMethods() { @Test public void testACorrectAnnotationSetupForPreProcessCompiles() { List source = generateSourceOfValidatorMethodWith( - annotationOf(someConditions(), preProcess(), aReqType()), + annotationOf(preProcess(), aReqType()), modifiers("public", "static"), returnValue("OMRequest"), parameters("OMRequest rq", "ValidationContext ctx"), @@ -84,7 +83,7 @@ public void testACorrectAnnotationSetupForPreProcessCompiles() { @Test public void testACorrectAnnotationSetupForPostProcessCompiles() { List source = generateSourceOfValidatorMethodWith( - annotationOf(someConditions(), postProcess(), aReqType()), + annotationOf(postProcess(), aReqType()), modifiers("public", "static"), returnValue("OMResponse"), parameters("OMRequest rq", "OMResponse rp", "ValidationContext ctx"), @@ -96,7 +95,7 @@ public void testACorrectAnnotationSetupForPostProcessCompiles() { @Test public void testValidatorDoesNotNecessarilyThrowsExceptions() { List source = generateSourceOfValidatorMethodWith( - annotationOf(someConditions(), preProcess(), aReqType()), + annotationOf(preProcess(), aReqType()), modifiers("public", "static"), returnValue("OMRequest"), parameters("OMRequest rq", "ValidationContext ctx"), @@ -108,7 +107,7 @@ public void testValidatorDoesNotNecessarilyThrowsExceptions() { @Test public void testNonStaticValidatorDoesNotCompile() { List source = generateSourceOfValidatorMethodWith( - annotationOf(someConditions(), preProcess(), aReqType()), + annotationOf(preProcess(), aReqType()), modifiers("public"), returnValue("OMRequest"), parameters("OMRequest rq", "ValidationContext ctx"), @@ -121,7 +120,7 @@ public void testNonStaticValidatorDoesNotCompile() { @Test public void testValidatorMethodCanBeFinal() { List source = generateSourceOfValidatorMethodWith( - annotationOf(someConditions(), preProcess(), aReqType()), + annotationOf(preProcess(), aReqType()), modifiers("public", "static", "final"), returnValue("OMRequest"), parameters("OMRequest rq", "ValidationContext ctx"), @@ -133,7 +132,7 @@ public void testValidatorMethodCanBeFinal() { @Test public void testValidatorMethodCanBePrivate() { List source = generateSourceOfValidatorMethodWith( - annotationOf(someConditions(), preProcess(), aReqType()), + annotationOf(preProcess(), aReqType()), modifiers("private", "static"), returnValue("OMRequest"), parameters("OMRequest rq", "ValidationContext ctx"), @@ -145,7 +144,7 @@ public void testValidatorMethodCanBePrivate() { @Test public void testValidatorMethodCanBeDefaultVisible() { List source = generateSourceOfValidatorMethodWith( - annotationOf(someConditions(), preProcess(), aReqType()), + annotationOf(preProcess(), aReqType()), modifiers("static"), returnValue("OMRequest"), parameters("OMRequest rq", "ValidationContext ctx"), @@ -157,7 +156,7 @@ public void testValidatorMethodCanBeDefaultVisible() { @Test public void testValidatorMethodCanBeProtected() { List source = generateSourceOfValidatorMethodWith( - annotationOf(someConditions(), preProcess(), aReqType()), + annotationOf(preProcess(), aReqType()), modifiers("protected", "static"), returnValue("OMRequest"), parameters("OMRequest rq", "ValidationContext ctx"), @@ -169,31 +168,27 @@ public void testValidatorMethodCanBeProtected() { @Test public void testEmptyValidationConditionListDoesNotCompile() { List source = generateSourceOfValidatorMethodWith( - annotationOf(emptyConditions(), preProcess(), aReqType()), + annotationOf(preProcess(), aReqType()), modifiers("public", "static"), returnValue("OMRequest"), parameters("OMRequest rq", "ValidationContext ctx"), exceptions()); - - assertThat(compile(source)).hadErrorContaining(ERROR_CONDITION_IS_EMPTY); } @Test public void testNullValidationConditionListDoesNotCompile() { List source = generateSourceOfValidatorMethodWith( - annotationOf(nullConditions(), preProcess(), aReqType()), + annotationOf(preProcess(), aReqType()), modifiers("public", "static"), returnValue("OMRequest"), parameters("OMRequest rq", "ValidationContext ctx"), exceptions()); - - assertThat(compile(source)).hadErrorContaining(ERROR_CONDITION_IS_EMPTY); } @Test public void testNotEnoughParametersForPreProcess() { List source = generateSourceOfValidatorMethodWith( - annotationOf(someConditions(), preProcess(), aReqType()), + annotationOf(preProcess(), aReqType()), modifiers("public", "static"), returnValue("OMRequest"), parameters("OMRequest rq"), @@ -207,7 +202,7 @@ public void testNotEnoughParametersForPreProcess() { @Test public void testTooManyParametersForPreProcess() { List source = generateSourceOfValidatorMethodWith( - annotationOf(someConditions(), preProcess(), aReqType()), + annotationOf(preProcess(), aReqType()), modifiers("public", "static"), returnValue("OMRequest"), parameters("OMRequest rq", "OMResponse rp", "ValidationContext ctx"), @@ -221,7 +216,7 @@ public void testTooManyParametersForPreProcess() { @Test public void testNotEnoughParametersForPostProcess() { List source = generateSourceOfValidatorMethodWith( - annotationOf(someConditions(), postProcess(), aReqType()), + annotationOf(postProcess(), aReqType()), modifiers("public", "static"), returnValue("OMResponse"), parameters("OMRequest rq", "OMResponse rp"), @@ -235,7 +230,7 @@ public void testNotEnoughParametersForPostProcess() { @Test public void testTooManyParametersForPostProcess() { List source = generateSourceOfValidatorMethodWith( - annotationOf(someConditions(), postProcess(), aReqType()), + annotationOf(postProcess(), aReqType()), modifiers("public", "static"), returnValue("OMResponse"), parameters("OMRequest rq", "OMResponse rp", "ValidationContext ctx", @@ -250,7 +245,7 @@ public void testTooManyParametersForPostProcess() { @Test public void testWrongReturnValueForPreProcess() { List source = generateSourceOfValidatorMethodWith( - annotationOf(someConditions(), preProcess(), aReqType()), + annotationOf(preProcess(), aReqType()), modifiers("public", "static"), returnValue("String"), parameters("OMRequest rq", "ValidationContext ctx"), @@ -263,7 +258,7 @@ public void testWrongReturnValueForPreProcess() { @Test public void testWrongReturnValueForPostProcess() { List source = generateSourceOfValidatorMethodWith( - annotationOf(someConditions(), postProcess(), aReqType()), + annotationOf(postProcess(), aReqType()), modifiers("public", "static"), returnValue("String"), parameters("OMRequest rq", "OMResponse rp", "ValidationContext ctx"), @@ -276,7 +271,7 @@ public void testWrongReturnValueForPostProcess() { @Test public void testWrongFirstArgumentForPreProcess() { List source = generateSourceOfValidatorMethodWith( - annotationOf(someConditions(), preProcess(), aReqType()), + annotationOf(preProcess(), aReqType()), modifiers("public", "static"), returnValue("OMRequest"), parameters("String rq", "ValidationContext ctx"), @@ -289,7 +284,7 @@ public void testWrongFirstArgumentForPreProcess() { @Test public void testWrongFirstArgumentForPostProcess() { List source = generateSourceOfValidatorMethodWith( - annotationOf(someConditions(), postProcess(), aReqType()), + annotationOf(postProcess(), aReqType()), modifiers("public", "static"), returnValue("OMResponse"), parameters("String rq", "OMResponse rp", "ValidationContext ctx"), @@ -302,7 +297,7 @@ public void testWrongFirstArgumentForPostProcess() { @Test public void testWrongSecondArgumentForPreProcess() { List source = generateSourceOfValidatorMethodWith( - annotationOf(someConditions(), preProcess(), aReqType()), + annotationOf(preProcess(), aReqType()), modifiers("public", "static"), returnValue("OMRequest"), parameters("OMRequest rq", "String ctx"), @@ -315,7 +310,7 @@ public void testWrongSecondArgumentForPreProcess() { @Test public void testWrongSecondArgumentForPostProcess() { List source = generateSourceOfValidatorMethodWith( - annotationOf(someConditions(), postProcess(), aReqType()), + annotationOf(postProcess(), aReqType()), modifiers("public", "static"), returnValue("OMResponse"), parameters("OMRequest rq", "String rp", "ValidationContext ctx"), @@ -328,7 +323,7 @@ public void testWrongSecondArgumentForPostProcess() { @Test public void testWrongThirdArgumentForPostProcess() { List source = generateSourceOfValidatorMethodWith( - annotationOf(someConditions(), postProcess(), aReqType()), + annotationOf(postProcess(), aReqType()), modifiers("public", "static"), returnValue("OMResponse"), parameters("OMRequest rq", "OMResponse rp", "String ctx"), @@ -341,7 +336,19 @@ public void testWrongThirdArgumentForPostProcess() { @Test public void testInvalidProcessingPhase() { List source = generateSourceOfValidatorMethodWith( - annotationOf(someConditions(), "INVALID", aReqType()), + annotationOf("INVALID", aReqType(), ClientVersion.CURRENT), + modifiers("public", "static"), + returnValue("OMResponse"), + parameters("OMRequest rq", "OMResponse rp", "ValidationContext ctx"), + exceptions("ServiceException")); + + assertThat(compile(source)).failed(); + } + + @Test + public void testInvalidClientVersion() { + List source = generateSourceOfValidatorMethodWith( + annotationOf(RequestProcessingPhase.PRE_PROCESS, aReqType(), null), modifiers("public", "static"), returnValue("OMResponse"), parameters("OMRequest rq", "OMResponse rp", "ValidationContext ctx"), @@ -353,7 +360,6 @@ public void testInvalidProcessingPhase() { private static List getClientVersions() { List clientVersions = Arrays.stream(ClientVersion.values()).map(Arguments::of).collect(Collectors.toList()); - clientVersions.add(null); return clientVersions; } @@ -361,16 +367,13 @@ private static List getClientVersions() { @MethodSource("getClientVersions") public void testMultipleErrorMessages(ClientVersion clientVersion) { List source = generateSourceOfValidatorMethodWith( - annotationOf(emptyConditions(), postProcess(), aReqType(), clientVersion), + annotationOf(postProcess(), aReqType(), clientVersion), modifiers(), returnValue("String"), parameters("String rq", "int rp", "String ctx"), exceptions()); Compilation compilation = compile(source); - if (ClientVersion.FUTURE_VERSION.equals(clientVersion)) { - assertThat(compilation).hadErrorContaining(ERROR_CONDITION_IS_EMPTY); - } assertThat(compilation) .hadErrorContaining(ERROR_VALIDATOR_METHOD_HAS_TO_BE_STATIC); assertThat(compilation) @@ -385,25 +388,12 @@ public void testMultipleErrorMessages(ClientVersion clientVersion) { private Compilation compile(List source) { Compilation c = javac() - .withProcessors(new RequestFeatureValidatorProcessor()) + .withProcessors(new OmRequestFeatureValidatorProcessor()) .compile(JavaFileObjects.forSourceLines(CLASSNAME, source)); c.diagnostics().forEach(System.out::println); return c; } - private ValidationCondition[] someConditions() { - return - new ValidationCondition[] {ValidationCondition.CLUSTER_NEEDS_FINALIZATION}; - } - - private ValidationCondition[] emptyConditions() { - return new ValidationCondition[] {}; - } - - private ValidationCondition[] nullConditions() { - return null; - } - private RequestProcessingPhase preProcess() { return RequestProcessingPhase.PRE_PROCESS; } @@ -457,52 +447,28 @@ private List generateSourceOfValidatorMethodWith( } private String annotationOf( - ValidationCondition[] conditions, RequestProcessingPhase phase, Type reqType) { - return annotationOf(conditions, phase.name(), reqType, null); + return annotationOf(phase.name(), reqType, ClientVersion.CURRENT); } private String annotationOf( - ValidationCondition[] conditions, RequestProcessingPhase phase, Type reqType, ClientVersion clientVersion) { - return annotationOf(conditions, phase.name(), reqType, clientVersion); - } - - private String annotationOf( - ValidationCondition[] conditions, - String phase, - Type reqType) { - return annotationOf(conditions, phase, reqType, null); + return annotationOf(phase.name(), reqType, clientVersion); } private String annotationOf( - ValidationCondition[] conditions, String phase, Type reqType, ClientVersion clientVersion) { StringBuilder annotation = new StringBuilder(); - annotation.append("@RequestFeatureValidator("); - StringBuilder conditionsArray = new StringBuilder(); - if (conditions != null) { - conditionsArray.append("conditions = { "); - if (conditions.length > 0) { - for (ValidationCondition condition : conditions) { - conditionsArray.append(condition.name()).append(", "); - } - annotation - .append(conditionsArray.substring(0, conditionsArray.length() - 2)); - } else { - annotation.append(conditionsArray); - } - annotation.append(" }, "); - } + annotation.append("@OMClientVersionValidator("); annotation.append("processingPhase = ").append(phase); annotation.append(", requestType = ").append(reqType.name()); if (clientVersion != null) { - annotation.append(", maxClientVersion = ").append(clientVersion.name()); + annotation.append(", maxVersion = ").append(clientVersion.name()); } annotation.append(" )"); return annotation.toString(); @@ -511,7 +477,7 @@ private String annotationOf( private List allImports() { List imports = new ArrayList<>(); imports.add("import org.apache.hadoop.ozone.om.request.validation" - + ".RequestFeatureValidator;"); + + ".OMClientVersionValidator;"); imports.add("import org.apache.hadoop.ozone.protocol.proto" + ".OzoneManagerProtocolProtos.OMRequest;"); imports.add("import org.apache.hadoop.ozone.protocol.proto" @@ -519,10 +485,6 @@ private List allImports() { imports.add("import org.apache.hadoop.ozone.om.request.validation" + ".ValidationContext;"); imports.add("import com.google.protobuf.ServiceException;"); - for (ValidationCondition condition : ValidationCondition.values()) { - imports.add("import static org.apache.hadoop.ozone.om.request.validation" - + ".ValidationCondition." + condition.name() + ";"); - } for (RequestProcessingPhase phase : RequestProcessingPhase.values()) { imports.add("import static org.apache.hadoop.ozone.om.request.validation" + ".RequestProcessingPhase." + phase.name() + ";"); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestRequestValidations.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestRequestValidations.java index fdb332f66042..b882c9eb6357 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestRequestValidations.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestRequestValidations.java @@ -24,6 +24,7 @@ import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.request.validation.testvalidatorset1.GeneralValidatorsForTesting; import org.apache.hadoop.ozone.om.request.validation.testvalidatorset1.GeneralValidatorsForTesting.ValidationListener; +import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; @@ -45,6 +46,9 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -159,7 +163,7 @@ public void testPreProcessorExceptionHandling() throws Exception { ValidationContext ctx = of(aFinalizedVersionManager(), metadataManager); RequestValidations validations = loadValidations(ctx); assertThrows(Exception.class, - () -> validations.validateRequest(aDeleteKeysRequest(olderClientVersion()))); + () -> validations.validateRequest(aDeleteKeysRequest(ClientVersion.ERASURE_CODING_SUPPORT))); validationListener.assertNumOfEvents(1); validationListener.assertExactListOfValidatorsCalled( @@ -171,7 +175,8 @@ public void testPostProcessorExceptionHandling() { ValidationContext ctx = of(aFinalizedVersionManager(), metadataManager); RequestValidations validations = loadValidations(ctx); assertThrows(Exception.class, - () -> validations.validateResponse(aDeleteKeysRequest(olderClientVersion()), aDeleteKeysResponse())); + () -> validations.validateResponse(aDeleteKeysRequest(ClientVersion.ERASURE_CODING_SUPPORT), + aDeleteKeysResponse())); validationListener.assertNumOfEvents(1); validationListener.assertExactListOfValidatorsCalled( @@ -184,7 +189,7 @@ public void testOldClientConditionIsRecognizedAndPreValidatorsApplied() ValidationContext ctx = of(aFinalizedVersionManager(), metadataManager); RequestValidations validations = loadValidations(ctx); - validations.validateRequest(aCreateKeyRequest(olderClientVersion())); + validations.validateRequest(aCreateKeyRequest(ClientVersion.ERASURE_CODING_SUPPORT)); validationListener.assertNumOfEvents(1); validationListener.assertExactListOfValidatorsCalled( @@ -198,7 +203,7 @@ public void testOldClientConditionIsRecognizedAndPostValidatorsApplied() RequestValidations validations = loadValidations(ctx); validations.validateResponse( - aCreateKeyRequest(olderClientVersion()), aCreateKeyResponse()); + aCreateKeyRequest(ClientVersion.ERASURE_CODING_SUPPORT), aCreateKeyResponse()); validationListener.assertNumOfEvents(2); validationListener.assertExactListOfValidatorsCalled( @@ -212,7 +217,7 @@ public void testPreFinalizedWithOldClientConditionPreProcValidatorsApplied() ValidationContext ctx = of(anUnfinalizedVersionManager(), metadataManager); RequestValidations validations = loadValidations(ctx); - validations.validateRequest(aCreateKeyRequest(olderClientVersion())); + validations.validateRequest(aCreateKeyRequest(ClientVersion.ERASURE_CODING_SUPPORT)); validationListener.assertNumOfEvents(2); validationListener.assertExactListOfValidatorsCalled( @@ -227,7 +232,7 @@ public void testPreFinalizedWithOldClientConditionPostProcValidatorsApplied() RequestValidations validations = loadValidations(ctx); validations.validateResponse( - aCreateKeyRequest(olderClientVersion()), aCreateKeyResponse()); + aCreateKeyRequest(ClientVersion.ERASURE_CODING_SUPPORT), aCreateKeyResponse()); validationListener.assertNumOfEvents(3); validationListener.assertExactListOfValidatorsCalled( @@ -295,6 +300,14 @@ private OMRequest aCreateKeyRequest(int clientVersion) { return aRequest(CreateKey, clientVersion); } + private OMRequest aCreateKeyRequest(ClientVersion clientVersion) { + return aRequest(CreateKey, clientVersion.toProtoValue()); + } + + private OMRequest aDeleteKeysRequest(ClientVersion clientVersion) { + return aDeleteKeysRequest(clientVersion.toProtoValue()); + } + private OMRequest aDeleteKeysRequest(int clientVersion) { return aRequest(DeleteKeys, clientVersion); } @@ -333,12 +346,16 @@ private OMResponse aResponse(Type type) { private LayoutVersionManager aFinalizedVersionManager() { LayoutVersionManager vm = mock(LayoutVersionManager.class); when(vm.needsFinalization()).thenReturn(false); + when(vm.getFeature(anyString())).thenReturn(OMLayoutFeature.QUOTA); + when(vm.getFeature(anyInt())).thenReturn(OMLayoutFeature.QUOTA); return vm; } private LayoutVersionManager anUnfinalizedVersionManager() { LayoutVersionManager vm = mock(LayoutVersionManager.class); when(vm.needsFinalization()).thenReturn(true); + when(vm.getFeature(anyString())).thenReturn(OMLayoutFeature.HSYNC); + when(vm.getFeature(anyInt())).thenReturn(OMLayoutFeature.HSYNC); return vm; } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestValidatorRegistry.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestValidatorRegistry.java index dbc763efe8ce..f5d9c7fc3335 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestValidatorRegistry.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestValidatorRegistry.java @@ -16,7 +16,11 @@ */ package org.apache.hadoop.ozone.om.request.validation; +import com.google.common.collect.Sets; import org.apache.hadoop.ozone.ClientVersion; +import org.apache.hadoop.ozone.Version; +import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -26,16 +30,18 @@ import java.net.MalformedURLException; import java.net.URL; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Set; import java.util.stream.Collectors; import static java.util.Arrays.asList; import static java.util.Collections.emptyList; import static org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase.POST_PROCESS; import static org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase.PRE_PROCESS; -import static org.apache.hadoop.ozone.om.request.validation.ValidationCondition.CLUSTER_NEEDS_FINALIZATION; +import static org.apache.hadoop.ozone.om.request.validation.VersionExtractor.LAYOUT_VERSION_EXTRACTOR; import static org.apache.hadoop.ozone.om.request.validation.testvalidatorset1.GeneralValidatorsForTesting.startValidatorTest; import static org.apache.hadoop.ozone.om.request.validation.testvalidatorset1.GeneralValidatorsForTesting.finishValidatorTest; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.CreateDirectory; @@ -60,6 +66,9 @@ public class TestValidatorRegistry { private static final String PACKAGE_WO_VALIDATORS = "org.apache.hadoop.hdds.annotation"; + private static final Set REQUEST_PROCESSING_PHASES = + Sets.immutableEnumSet(PRE_PROCESS, POST_PROCESS); + @BeforeEach public void setup() { startValidatorTest(); @@ -72,19 +81,25 @@ public void tearDown() { @Test public void testNoValidatorsReturnedForEmptyConditionList() { - ValidatorRegistry registry = new ValidatorRegistry(PACKAGE); - List validators = registry.validationsFor(emptyList(), CreateKey, PRE_PROCESS, - ClientVersion.CURRENT_VERSION); + ValidatorRegistry registry = + new ValidatorRegistry<>(OzoneManagerProtocolProtos.Type.class, PACKAGE, + Arrays.stream(VersionExtractor.values()).map(VersionExtractor::getVersionClass).collect(Collectors.toSet()), + REQUEST_PROCESSING_PHASES); + List validators = registry.validationsFor(CreateKey, PRE_PROCESS, + Collections.singletonList(ClientVersion.CURRENT)); assertTrue(validators.isEmpty()); } @Test public void testRegistryHasThePreFinalizePreProcessCreateKeyValidator() { - ValidatorRegistry registry = new ValidatorRegistry(PACKAGE); + ValidatorRegistry registry = new ValidatorRegistry<>( + OzoneManagerProtocolProtos.Type.class, PACKAGE, + Arrays.stream(VersionExtractor.values()).map(VersionExtractor::getVersionClass).collect(Collectors.toSet()), + REQUEST_PROCESSING_PHASES); List validators = - registry.validationsFor(asList(CLUSTER_NEEDS_FINALIZATION), CreateKey, PRE_PROCESS, - ClientVersion.CURRENT_VERSION); + registry.validationsFor(CreateKey, PRE_PROCESS, asList(ClientVersion.CURRENT, + OMLayoutFeature.FILESYSTEM_SNAPSHOT)); assertEquals(1, validators.size()); String expectedMethodName = "preFinalizePreProcessCreateKeyValidator"; @@ -93,10 +108,11 @@ public void testRegistryHasThePreFinalizePreProcessCreateKeyValidator() { @Test public void testRegistryHasThePreFinalizePostProcessCreateKeyValidator() { - ValidatorRegistry registry = new ValidatorRegistry(PACKAGE); - List validators = - registry.validationsFor( - asList(CLUSTER_NEEDS_FINALIZATION), CreateKey, POST_PROCESS, ClientVersion.CURRENT_VERSION); + ValidatorRegistry registry = new ValidatorRegistry<>(OzoneManagerProtocolProtos.Type.class, PACKAGE, + Arrays.stream(VersionExtractor.values()).map(VersionExtractor::getVersionClass).collect(Collectors.toSet()), + REQUEST_PROCESSING_PHASES); + List validators = registry.validationsFor(CreateKey, POST_PROCESS, + asList(ClientVersion.CURRENT, OMLayoutFeature.BUCKET_LAYOUT_SUPPORT)); assertEquals(1, validators.size()); String expectedMethodName = "preFinalizePostProcessCreateKeyValidator"; @@ -105,10 +121,11 @@ public void testRegistryHasThePreFinalizePostProcessCreateKeyValidator() { @Test public void testRegistryHasTheOldClientPreProcessCreateKeyValidator() { - ValidatorRegistry registry = new ValidatorRegistry(PACKAGE); + ValidatorRegistry registry = new ValidatorRegistry<>(OzoneManagerProtocolProtos.Type.class, PACKAGE, + Arrays.stream(VersionExtractor.values()).map(VersionExtractor::getVersionClass).collect(Collectors.toSet()), + REQUEST_PROCESSING_PHASES); List validators = - registry.validationsFor( - Collections.emptyList(), CreateKey, PRE_PROCESS, ClientVersion.ERASURE_CODING_SUPPORT.toProtoValue()); + registry.validationsFor(CreateKey, PRE_PROCESS, ClientVersion.ERASURE_CODING_SUPPORT); assertEquals(2, validators.size()); List methodNames = @@ -119,9 +136,11 @@ public void testRegistryHasTheOldClientPreProcessCreateKeyValidator() { @Test public void testRegistryHasTheOldClientPostProcessCreateKeyValidator() { - ValidatorRegistry registry = new ValidatorRegistry(PACKAGE); - List validators = registry.validationsFor(Collections.emptyList(), CreateKey, POST_PROCESS, - ClientVersion.ERASURE_CODING_SUPPORT.toProtoValue()); + ValidatorRegistry registry = new ValidatorRegistry<>(OzoneManagerProtocolProtos.Type.class, PACKAGE, + Arrays.stream(VersionExtractor.values()).map(VersionExtractor::getVersionClass).collect(Collectors.toSet()), + REQUEST_PROCESSING_PHASES); + List validators = registry.validationsFor(CreateKey, POST_PROCESS, + ClientVersion.ERASURE_CODING_SUPPORT); assertEquals(2, validators.size()); List methodNames = @@ -132,13 +151,13 @@ public void testRegistryHasTheOldClientPostProcessCreateKeyValidator() { @Test public void testRegistryHasTheMultiPurposePreProcessCreateVolumeValidator() { - ValidatorRegistry registry = new ValidatorRegistry(PACKAGE); + ValidatorRegistry registry = new ValidatorRegistry<>(OzoneManagerProtocolProtos.Type.class, PACKAGE, + Arrays.stream(VersionExtractor.values()).map(VersionExtractor::getVersionClass).collect(Collectors.toSet()), + REQUEST_PROCESSING_PHASES); List preFinalizeValidators = - registry.validationsFor( - asList(CLUSTER_NEEDS_FINALIZATION), CreateVolume, PRE_PROCESS, ClientVersion.CURRENT_VERSION); + registry.validationsFor(CreateVolume, PRE_PROCESS, OMLayoutFeature.HSYNC); List newClientValidators = - registry.validationsFor( - Collections.emptyList(), CreateVolume, PRE_PROCESS, ClientVersion.ERASURE_CODING_SUPPORT.toProtoValue()); + registry.validationsFor(CreateVolume, PRE_PROCESS, ClientVersion.ERASURE_CODING_SUPPORT); assertEquals(1, preFinalizeValidators.size()); assertEquals(1, newClientValidators.size()); @@ -149,13 +168,14 @@ public void testRegistryHasTheMultiPurposePreProcessCreateVolumeValidator() { @Test public void testRegistryHasTheMultiPurposePostProcessCreateVolumeValidator() { - ValidatorRegistry registry = new ValidatorRegistry(PACKAGE); + ValidatorRegistry registry = new ValidatorRegistry<>( + OzoneManagerProtocolProtos.Type.class, PACKAGE, + Arrays.stream(VersionExtractor.values()).map(VersionExtractor::getVersionClass).collect(Collectors.toSet()), + REQUEST_PROCESSING_PHASES); List preFinalizeValidators = - registry.validationsFor( - asList(CLUSTER_NEEDS_FINALIZATION), CreateVolume, POST_PROCESS, ClientVersion.CURRENT_VERSION); + registry.validationsFor(CreateVolume, POST_PROCESS, OMLayoutFeature.HSYNC); List oldClientValidators = - registry.validationsFor( - Collections.emptyList(), CreateVolume, POST_PROCESS, ClientVersion.ERASURE_CODING_SUPPORT.toProtoValue()); + registry.validationsFor(CreateVolume, POST_PROCESS, ClientVersion.ERASURE_CODING_SUPPORT); assertEquals(1, preFinalizeValidators.size()); assertEquals(1, oldClientValidators.size()); @@ -166,11 +186,12 @@ public void testRegistryHasTheMultiPurposePostProcessCreateVolumeValidator() { @Test public void testValidatorsAreReturnedForMultiCondition() { - ValidatorRegistry registry = new ValidatorRegistry(PACKAGE); - List validators = - registry.validationsFor( - asList(CLUSTER_NEEDS_FINALIZATION), - CreateKey, POST_PROCESS, ClientVersion.ERASURE_CODING_SUPPORT.toProtoValue()); + ValidatorRegistry registry = new ValidatorRegistry<>( + OzoneManagerProtocolProtos.Type.class, PACKAGE, + Arrays.stream(VersionExtractor.values()).map(VersionExtractor::getVersionClass).collect(Collectors.toSet()), + REQUEST_PROCESSING_PHASES); + List validators = registry.validationsFor(CreateKey, POST_PROCESS, + Arrays.asList(ClientVersion.ERASURE_CODING_SUPPORT, OMLayoutFeature.HSYNC)); assertEquals(3, validators.size()); List methodNames = @@ -182,9 +203,13 @@ public void testValidatorsAreReturnedForMultiCondition() { @Test public void testNoValidatorForRequestsAtAllReturnsEmptyList() { - ValidatorRegistry registry = new ValidatorRegistry(PACKAGE_WO_VALIDATORS); - assertTrue(registry.validationsFor(Collections.emptyList(), CreateKey, PRE_PROCESS, - ClientVersion.ERASURE_CODING_SUPPORT.toProtoValue()).isEmpty()); + + ValidatorRegistry registry = new ValidatorRegistry<>( + OzoneManagerProtocolProtos.Type.class, PACKAGE_WO_VALIDATORS, + Arrays.stream(VersionExtractor.values()).map(VersionExtractor::getVersionClass).collect(Collectors.toSet()), + REQUEST_PROCESSING_PHASES); + assertTrue(registry.validationsFor(CreateKey, PRE_PROCESS, + asList(ClientVersion.ERASURE_CODING_SUPPORT, OMLayoutFeature.HSYNC)).isEmpty()); } @Test @@ -195,19 +220,23 @@ public void testNoValidatorForConditionReturnsEmptyList() for (URL url : urls) { urlsToUse.add(new URL(url, PACKAGE2.replaceAll("\\.", "/"))); } - ValidatorRegistry registry = new ValidatorRegistry(urlsToUse); + ValidatorRegistry registry = new ValidatorRegistry<>( + OzoneManagerProtocolProtos.Type.class, urlsToUse, + Arrays.stream(VersionExtractor.values()).map(VersionExtractor::getVersionClass).collect(Collectors.toSet()), + REQUEST_PROCESSING_PHASES); - assertTrue(registry.validationsFor( - asList(CLUSTER_NEEDS_FINALIZATION), CreateKey, PRE_PROCESS, ClientVersion.CURRENT_VERSION).isEmpty()); + assertTrue(registry.validationsFor(CreateKey, PRE_PROCESS, asList(ClientVersion.CURRENT, + OMLayoutFeature.BUCKET_LAYOUT_SUPPORT)).isEmpty()); } @Test public void testNoDefinedValidationForRequestReturnsEmptyList() { - ValidatorRegistry registry = new ValidatorRegistry(PACKAGE); + ValidatorRegistry registry = new ValidatorRegistry<>( + OzoneManagerProtocolProtos.Type.class, PACKAGE, + Arrays.stream(VersionExtractor.values()).map(VersionExtractor::getVersionClass).collect(Collectors.toSet()), + REQUEST_PROCESSING_PHASES); - assertTrue(registry.validationsFor( - Collections.emptyList(), CreateDirectory, null, ClientVersion.ERASURE_CODING_SUPPORT.toProtoValue()) - .isEmpty()); + assertTrue(registry.validationsFor(CreateDirectory, null, ClientVersion.ERASURE_CODING_SUPPORT).isEmpty()); } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset1/GeneralValidatorsForTesting.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset1/GeneralValidatorsForTesting.java index 80bbae1340ce..e7312193a840 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset1/GeneralValidatorsForTesting.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset1/GeneralValidatorsForTesting.java @@ -17,9 +17,11 @@ package org.apache.hadoop.ozone.om.request.validation.testvalidatorset1; import org.apache.hadoop.ozone.ClientVersion; -import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; +import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; +import org.apache.hadoop.ozone.om.request.validation.OMLayoutVersionValidator; import org.apache.hadoop.ozone.om.request.validation.TestRequestValidations; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; +import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; @@ -29,7 +31,7 @@ import static org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase.POST_PROCESS; import static org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase.PRE_PROCESS; -import static org.apache.hadoop.ozone.om.request.validation.ValidationCondition.CLUSTER_NEEDS_FINALIZATION; +import static org.apache.hadoop.ozone.om.request.validation.VersionExtractor.LAYOUT_VERSION_EXTRACTOR; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.CreateKey; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.CreateVolume; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.DeleteKeys; @@ -88,8 +90,8 @@ private static void fireValidationEvent(String calledMethodName) { listeners.forEach(l -> l.validationCalled(calledMethodName)); } - @RequestFeatureValidator( - conditions = CLUSTER_NEEDS_FINALIZATION, + @OMLayoutVersionValidator( + maxVersion = OMLayoutFeature.QUOTA, processingPhase = PRE_PROCESS, requestType = CreateKey) public static OMRequest preFinalizePreProcessCreateKeyValidator( @@ -98,8 +100,8 @@ public static OMRequest preFinalizePreProcessCreateKeyValidator( return req; } - @RequestFeatureValidator( - conditions = CLUSTER_NEEDS_FINALIZATION, + @OMLayoutVersionValidator( + maxVersion = OMLayoutFeature.QUOTA, processingPhase = POST_PROCESS, requestType = CreateKey) public static OMResponse preFinalizePostProcessCreateKeyValidator( @@ -108,62 +110,66 @@ public static OMResponse preFinalizePostProcessCreateKeyValidator( return resp; } - @RequestFeatureValidator( + @OMClientVersionValidator( processingPhase = PRE_PROCESS, requestType = CreateKey, - maxClientVersion = ClientVersion.BUCKET_LAYOUT_SUPPORT) + maxVersion = ClientVersion.BUCKET_LAYOUT_SUPPORT) public static OMRequest oldClientPreProcessCreateKeyValidator( OMRequest req, ValidationContext ctx) { fireValidationEvent("oldClientPreProcessCreateKeyValidator"); return req; } - @RequestFeatureValidator( + @OMClientVersionValidator( processingPhase = POST_PROCESS, requestType = CreateKey, - maxClientVersion = ClientVersion.BUCKET_LAYOUT_SUPPORT) + maxVersion = ClientVersion.BUCKET_LAYOUT_SUPPORT) public static OMResponse oldClientPostProcessCreateKeyValidator( OMRequest req, OMResponse resp, ValidationContext ctx) { fireValidationEvent("oldClientPostProcessCreateKeyValidator"); return resp; } - @RequestFeatureValidator( - conditions = CLUSTER_NEEDS_FINALIZATION, + @OMClientVersionValidator( processingPhase = PRE_PROCESS, requestType = CreateVolume, - maxClientVersion = ClientVersion.BUCKET_LAYOUT_SUPPORT) + maxVersion = ClientVersion.BUCKET_LAYOUT_SUPPORT) + @OMLayoutVersionValidator(processingPhase = PRE_PROCESS, + requestType = CreateVolume, + maxVersion = OMLayoutFeature.QUOTA) public static OMRequest multiPurposePreProcessCreateVolumeValidator( OMRequest req, ValidationContext ctx) { fireValidationEvent("multiPurposePreProcessCreateVolumeValidator"); return req; } - @RequestFeatureValidator( - conditions = CLUSTER_NEEDS_FINALIZATION, + @OMClientVersionValidator( processingPhase = POST_PROCESS, requestType = CreateVolume, - maxClientVersion = ClientVersion.BUCKET_LAYOUT_SUPPORT) + maxVersion = ClientVersion.BUCKET_LAYOUT_SUPPORT) + @OMLayoutVersionValidator(processingPhase = POST_PROCESS, + requestType = CreateVolume, + maxVersion = OMLayoutFeature.QUOTA) public static OMResponse multiPurposePostProcessCreateVolumeValidator( OMRequest req, OMResponse resp, ValidationContext ctx) { fireValidationEvent("multiPurposePostProcessCreateVolumeValidator"); return resp; } - @RequestFeatureValidator( + @OMClientVersionValidator( processingPhase = POST_PROCESS, requestType = CreateKey, - maxClientVersion = ClientVersion.BUCKET_LAYOUT_SUPPORT) + maxVersion = ClientVersion.EC_REPLICA_INDEX_REQUIRED_IN_BLOCK_REQUEST) public static OMResponse oldClientPostProcessCreateKeyValidator2( OMRequest req, OMResponse resp, ValidationContext ctx) { fireValidationEvent("oldClientPostProcessCreateKeyValidator2"); return resp; } - @RequestFeatureValidator( + @OMClientVersionValidator( processingPhase = PRE_PROCESS, requestType = DeleteKeys, - maxClientVersion = ClientVersion.BUCKET_LAYOUT_SUPPORT + maxVersion = ClientVersion.BUCKET_LAYOUT_SUPPORT ) public static OMRequest throwingPreProcessValidator( OMRequest req, ValidationContext ctx) throws IOException { @@ -174,10 +180,10 @@ public static OMRequest throwingPreProcessValidator( return req; } - @RequestFeatureValidator( + @OMClientVersionValidator( processingPhase = POST_PROCESS, requestType = DeleteKeys, - maxClientVersion = ClientVersion.BUCKET_LAYOUT_SUPPORT + maxVersion = ClientVersion.BUCKET_LAYOUT_SUPPORT ) public static OMResponse throwingPostProcessValidator( OMRequest req, OMResponse resp, ValidationContext ctx) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset2/ValidatorsForOnlyOldClientValidations.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset2/ValidatorsForOnlyOldClientValidations.java index a042523227b5..b544c9a749d5 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset2/ValidatorsForOnlyOldClientValidations.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset2/ValidatorsForOnlyOldClientValidations.java @@ -17,7 +17,7 @@ package org.apache.hadoop.ozone.om.request.validation.testvalidatorset2; import org.apache.hadoop.ozone.ClientVersion; -import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; +import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; @@ -32,10 +32,10 @@ public final class ValidatorsForOnlyOldClientValidations { private ValidatorsForOnlyOldClientValidations() { } - @RequestFeatureValidator( + @OMClientVersionValidator( processingPhase = PRE_PROCESS, requestType = CreateKey, - maxClientVersion = ClientVersion.BUCKET_LAYOUT_SUPPORT) + maxVersion = ClientVersion.BUCKET_LAYOUT_SUPPORT) public static OMRequest oldClientPreProcessCreateKeyValidator2( OMRequest req, ValidationContext ctx) { return req; From e623fdb76866bcdfc28b2989907c76323d8b48a4 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Mon, 15 Jul 2024 01:53:21 -0700 Subject: [PATCH 04/28] HDDS-11149. Add register validator processor Change-Id: Iab5e022b6a1401998d905c68d491fe2a2a7413b7 --- hadoop-ozone/ozone-manager/pom.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/ozone-manager/pom.xml b/hadoop-ozone/ozone-manager/pom.xml index 5f6bc54c5af9..4d15fb1ee7a0 100644 --- a/hadoop-ozone/ozone-manager/pom.xml +++ b/hadoop-ozone/ozone-manager/pom.xml @@ -252,7 +252,8 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> org.apache.hadoop.hdds.conf.ConfigFileGenerator - org.apache.ozone.annotations.RequestFeatureValidatorProcessor + org.apache.ozone.annotations.OmRequestFeatureValidatorProcessor + org.apache.ozone.annotations.RegisterValidatorProcessor From b270bf6f1ec6ef93e34f0f21cf2880c203811b90 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Mon, 15 Jul 2024 02:10:34 -0700 Subject: [PATCH 05/28] HDDS-11149. Fix checkstyle Change-Id: Iba9e1caa6a66b033d39c95b20f78ebc8e23fba4d --- .../OmRequestFeatureValidatorProcessor.java | 2 -- .../annotations/RegisterValidatorProcessor.java | 11 +++++++++++ .../org/apache/hadoop/ozone/ClientVersion.java | 2 +- .../om/request/bucket/OMBucketCreateRequest.java | 1 - .../bucket/OMBucketSetPropertyRequest.java | 2 -- .../request/file/OMDirectoryCreateRequest.java | 1 - .../om/request/file/OMFileCreateRequest.java | 1 - .../om/request/key/OMAllocateBlockRequest.java | 1 - .../ozone/om/request/key/OMKeyCommitRequest.java | 1 - .../ozone/om/request/key/OMKeyCreateRequest.java | 1 - .../S3InitiateMultipartUploadRequest.java | 1 - .../multipart/S3MultipartUploadAbortRequest.java | 1 - .../S3MultipartUploadCommitPartRequest.java | 1 - .../S3MultipartUploadCompleteRequest.java | 1 - .../validation/OMLayoutVersionValidator.java | 1 - .../om/request/validation/RegisterValidator.java | 16 +++++++++++++--- .../om/request/validation/ValidatorRegistry.java | 10 +++++----- .../validation/TestRequestValidations.java | 1 - .../validation/TestValidatorRegistry.java | 15 ++++++++------- .../GeneralValidatorsForTesting.java | 1 - 20 files changed, 38 insertions(+), 33 deletions(-) diff --git a/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/OmRequestFeatureValidatorProcessor.java b/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/OmRequestFeatureValidatorProcessor.java index cb78efb7371e..8a56d9fec676 100644 --- a/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/OmRequestFeatureValidatorProcessor.java +++ b/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/OmRequestFeatureValidatorProcessor.java @@ -16,7 +16,6 @@ */ package org.apache.ozone.annotations; -import jdk.nashorn.internal.ir.annotations.Immutable; import javax.annotation.processing.AbstractProcessor; import javax.annotation.processing.RoundEnvironment; @@ -38,7 +37,6 @@ import javax.lang.model.util.SimpleAnnotationValueVisitor8; import javax.tools.Diagnostic; import java.util.Arrays; -import java.util.HashSet; import java.util.List; import java.util.Map.Entry; import java.util.Set; diff --git a/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/RegisterValidatorProcessor.java b/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/RegisterValidatorProcessor.java index f57eec33d09b..ecec031a58f2 100644 --- a/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/RegisterValidatorProcessor.java +++ b/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/RegisterValidatorProcessor.java @@ -31,6 +31,17 @@ import javax.tools.Diagnostic; import java.util.Set; +/** + * This class is an annotation processor that is hooked into the java compiler + * and is used to validate the RegisterValidator annotations in the + * codebase, to ensure that the annotated classes have the proper methods returning appropriate object types. + * + * The module is compiled in a different execution via Maven before anything + * else is compiled, and then javac picks this class up as an annotation + * processor from the classpath via a ServiceLoader, based on the + * META-INF/services/javax.annotation.processing.Processor file in the module's + * resources folder. + */ @SupportedAnnotationTypes("org.apache.hadoop.ozone.om.request.validation.RegisterValidator") @SupportedSourceVersion(SourceVersion.RELEASE_8) public class RegisterValidatorProcessor extends AbstractProcessor { diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java index df50cf9d2e57..75d1a17a4157 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java @@ -59,7 +59,7 @@ public enum ClientVersion implements ComponentVersion { .collect(toMap(ClientVersion::toProtoValue, identity())); private final int version; - public final String description; + private final String description; ClientVersion(int version, String description) { this.version = version; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java index 8708dc839d69..1c0d6db1b2df 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java @@ -45,7 +45,6 @@ import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; -import org.apache.hadoop.ozone.om.request.validation.VersionExtractor; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.om.response.bucket.OMBucketCreateResponse; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketSetPropertyRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketSetPropertyRequest.java index 3fa3daddbe0d..23acb390c31c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketSetPropertyRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketSetPropertyRequest.java @@ -34,9 +34,7 @@ import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.BucketEncryptionKeyInfo; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; -import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; -import org.apache.hadoop.ozone.om.request.validation.VersionExtractor; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java index 91a07df44db6..d796df1e67bb 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java @@ -43,7 +43,6 @@ import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; -import org.apache.hadoop.ozone.om.request.validation.VersionExtractor; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java index 6727884d3448..8dc05c84a106 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java @@ -40,7 +40,6 @@ import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; -import org.apache.hadoop.ozone.om.request.validation.VersionExtractor; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.response.file.OMFileCreateResponse; import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java index 716ef6cf0bb5..c8c43b264eeb 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java @@ -35,7 +35,6 @@ import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; -import org.apache.hadoop.ozone.om.request.validation.VersionExtractor; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java index 883ee9e37c2f..ef58bc565485 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java @@ -46,7 +46,6 @@ import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; -import org.apache.hadoop.ozone.om.request.validation.VersionExtractor; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java index 1aebcc6aee62..64745c93b135 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java @@ -42,7 +42,6 @@ import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; -import org.apache.hadoop.ozone.om.request.validation.VersionExtractor; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.UserInfo; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java index fe8d196f72a3..be9ce421a952 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java @@ -41,7 +41,6 @@ import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; -import org.apache.hadoop.ozone.om.request.validation.VersionExtractor; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.om.response.s3.multipart.S3InitiateMultipartUploadResponse; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java index 52bdcb8459ac..1d93e8721ade 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java @@ -33,7 +33,6 @@ import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; -import org.apache.hadoop.ozone.om.request.validation.VersionExtractor; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java index a3d90fcebabc..01a59b5b600f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java @@ -37,7 +37,6 @@ import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; -import org.apache.hadoop.ozone.om.request.validation.VersionExtractor; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.om.response.s3.multipart diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java index 484c8af369e8..df0046cfc0b0 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java @@ -59,7 +59,6 @@ import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; -import org.apache.hadoop.ozone.om.request.validation.VersionExtractor; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.om.response.s3.multipart.S3MultipartUploadCompleteResponse; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/OMLayoutVersionValidator.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/OMLayoutVersionValidator.java index db0f7e4996e6..50accaa28214 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/OMLayoutVersionValidator.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/OMLayoutVersionValidator.java @@ -16,7 +16,6 @@ */ package org.apache.hadoop.ozone.om.request.validation; -import org.apache.hadoop.ozone.ClientVersion; import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RegisterValidator.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RegisterValidator.java index 88de9a137215..29963caec0c6 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RegisterValidator.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RegisterValidator.java @@ -17,15 +17,25 @@ */ package org.apache.hadoop.ozone.om.request.validation; +import org.apache.hadoop.ozone.Version; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; + +/** + * Annotations to register a validator. {@link org.apache.ozone.annotations.RegisterValidatorProcessor} + * enforces other annotation to have the following methods: + * maxVersion : Returns an enum which implement {@link Version} + * requestType: Returns an Enum value. + * processingPhase: Returns {@link RequestProcessingPhase} + * + */ @Retention(RetentionPolicy.RUNTIME) @Target(ElementType.ANNOTATION_TYPE) public @interface RegisterValidator { - public static final String MAX_VERSION_METHOD_NAME = "maxVersion"; - public static final String REQUEST_TYPE_METHOD_NAME = "requestType"; - public static final String PROCESSING_PHASE_METHOD_NAME = "processingPhase"; + String MAX_VERSION_METHOD_NAME = "maxVersion"; + String REQUEST_TYPE_METHOD_NAME = "requestType"; + String PROCESSING_PHASE_METHOD_NAME = "processingPhase"; } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidatorRegistry.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidatorRegistry.java index 37bbc2ef81ff..08ba39470f90 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidatorRegistry.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidatorRegistry.java @@ -88,8 +88,8 @@ private Class getReturnTypeOfAnnotationMethod(Class clz new Reflections(new ConfigurationBuilder().setUrls(ClasspathHelper.forPackage("")) .setScanners(Scanners.TypesAnnotated) .setParallel(true)).getTypesAnnotatedWith(RegisterValidator.class).stream() - .filter(annotationClass -> - getReturnTypeOfAnnotationMethod((Class) annotationClass, RegisterValidator.REQUEST_TYPE_METHOD_NAME) + .filter(annotationClass -> getReturnTypeOfAnnotationMethod((Class) annotationClass, + RegisterValidator.REQUEST_TYPE_METHOD_NAME) .equals(requestType)) .filter(annotationClass -> allowedVersionTypes.contains(getReturnTypeOfAnnotationMethod( (Class) annotationClass, @@ -243,11 +243,11 @@ private V getAndInitialize(K key, Supplier defaultSupplier, Map return from.computeIfAbsent(key, k -> defaultSupplier.get()); } - class IndexedItems> { + static final class IndexedItems> { private final List items; private final TreeMap indexMap; - public IndexedItems() { + private IndexedItems() { this.items = new ArrayList<>(); this.indexMap = new TreeMap<>(); } @@ -271,7 +271,7 @@ public void add(T item, IDX idx) { public List getItemsGreaterThanIdx(IDX indexValue) { return Optional.ofNullable(indexMap.higherEntry(indexValue)) .map(Map.Entry::getValue) - .map(startIndex -> items.subList(startIndex , items.size())).orElse(Collections.emptyList()); + .map(startIndex -> items.subList(startIndex, items.size())).orElse(Collections.emptyList()); } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestRequestValidations.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestRequestValidations.java index b882c9eb6357..47fba0d83248 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestRequestValidations.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestRequestValidations.java @@ -46,7 +46,6 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; -import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestValidatorRegistry.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestValidatorRegistry.java index f5d9c7fc3335..762838165bb4 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestValidatorRegistry.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestValidatorRegistry.java @@ -18,7 +18,6 @@ import com.google.common.collect.Sets; import org.apache.hadoop.ozone.ClientVersion; -import org.apache.hadoop.ozone.Version; import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.junit.jupiter.api.AfterEach; @@ -38,10 +37,8 @@ import java.util.stream.Collectors; import static java.util.Arrays.asList; -import static java.util.Collections.emptyList; import static org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase.POST_PROCESS; import static org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase.PRE_PROCESS; -import static org.apache.hadoop.ozone.om.request.validation.VersionExtractor.LAYOUT_VERSION_EXTRACTOR; import static org.apache.hadoop.ozone.om.request.validation.testvalidatorset1.GeneralValidatorsForTesting.startValidatorTest; import static org.apache.hadoop.ozone.om.request.validation.testvalidatorset1.GeneralValidatorsForTesting.finishValidatorTest; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.CreateDirectory; @@ -108,7 +105,8 @@ public void testRegistryHasThePreFinalizePreProcessCreateKeyValidator() { @Test public void testRegistryHasThePreFinalizePostProcessCreateKeyValidator() { - ValidatorRegistry registry = new ValidatorRegistry<>(OzoneManagerProtocolProtos.Type.class, PACKAGE, + ValidatorRegistry registry = + new ValidatorRegistry<>(OzoneManagerProtocolProtos.Type.class, PACKAGE, Arrays.stream(VersionExtractor.values()).map(VersionExtractor::getVersionClass).collect(Collectors.toSet()), REQUEST_PROCESSING_PHASES); List validators = registry.validationsFor(CreateKey, POST_PROCESS, @@ -121,7 +119,8 @@ public void testRegistryHasThePreFinalizePostProcessCreateKeyValidator() { @Test public void testRegistryHasTheOldClientPreProcessCreateKeyValidator() { - ValidatorRegistry registry = new ValidatorRegistry<>(OzoneManagerProtocolProtos.Type.class, PACKAGE, + ValidatorRegistry registry = + new ValidatorRegistry<>(OzoneManagerProtocolProtos.Type.class, PACKAGE, Arrays.stream(VersionExtractor.values()).map(VersionExtractor::getVersionClass).collect(Collectors.toSet()), REQUEST_PROCESSING_PHASES); List validators = @@ -136,7 +135,8 @@ public void testRegistryHasTheOldClientPreProcessCreateKeyValidator() { @Test public void testRegistryHasTheOldClientPostProcessCreateKeyValidator() { - ValidatorRegistry registry = new ValidatorRegistry<>(OzoneManagerProtocolProtos.Type.class, PACKAGE, + ValidatorRegistry registry = + new ValidatorRegistry<>(OzoneManagerProtocolProtos.Type.class, PACKAGE, Arrays.stream(VersionExtractor.values()).map(VersionExtractor::getVersionClass).collect(Collectors.toSet()), REQUEST_PROCESSING_PHASES); List validators = registry.validationsFor(CreateKey, POST_PROCESS, @@ -151,7 +151,8 @@ public void testRegistryHasTheOldClientPostProcessCreateKeyValidator() { @Test public void testRegistryHasTheMultiPurposePreProcessCreateVolumeValidator() { - ValidatorRegistry registry = new ValidatorRegistry<>(OzoneManagerProtocolProtos.Type.class, PACKAGE, + ValidatorRegistry registry = + new ValidatorRegistry<>(OzoneManagerProtocolProtos.Type.class, PACKAGE, Arrays.stream(VersionExtractor.values()).map(VersionExtractor::getVersionClass).collect(Collectors.toSet()), REQUEST_PROCESSING_PHASES); List preFinalizeValidators = diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset1/GeneralValidatorsForTesting.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset1/GeneralValidatorsForTesting.java index e7312193a840..a19d3e9c983e 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset1/GeneralValidatorsForTesting.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset1/GeneralValidatorsForTesting.java @@ -31,7 +31,6 @@ import static org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase.POST_PROCESS; import static org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase.PRE_PROCESS; -import static org.apache.hadoop.ozone.om.request.validation.VersionExtractor.LAYOUT_VERSION_EXTRACTOR; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.CreateKey; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.CreateVolume; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.DeleteKeys; From ca3746d10de6060d3fc42622cfa04bd4e2db6116 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Tue, 16 Jul 2024 19:46:33 -0700 Subject: [PATCH 06/28] HDDS-11149. Fix findbugs & rat failures Change-Id: I05265cad5f26082351c6926c24866b20a50b26d2 --- .../OmRequestFeatureValidatorProcessor.java | 3 +-- .../apache/hadoop/ozone/ClientVersion.java | 7 +++---- .../javax.annotation.processing.Processor | 0 .../request/validation/VersionExtractor.java | 3 ++- ...TestOMClientVersionValidatorProcessor.java | 20 ------------------- 5 files changed, 6 insertions(+), 27 deletions(-) delete mode 100644 hadoop-hdds/common/src/main/resources/META-INF/services/javax.annotation.processing.Processor diff --git a/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/OmRequestFeatureValidatorProcessor.java b/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/OmRequestFeatureValidatorProcessor.java index 8a56d9fec676..30f1e74ebec1 100644 --- a/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/OmRequestFeatureValidatorProcessor.java +++ b/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/OmRequestFeatureValidatorProcessor.java @@ -90,8 +90,7 @@ public class OmRequestFeatureValidatorProcessor extends AbstractProcessor { public static final String VALIDATION_CONTEXT_CLASS_NAME = "org.apache.hadoop.ozone.om.request.validation.ValidationContext"; - - public static final List ANNOTATION_SIMPLE_NAMES = Arrays.asList("OMClientVersionValidator", + private static final List ANNOTATION_SIMPLE_NAMES = Arrays.asList("OMClientVersionValidator", "OMLayoutVersionValidator"); public static final String ANNOTATION_CONDITIONS_PROPERTY_NAME = "conditions"; public static final String ANNOTATION_PROCESSING_PHASE_PROPERTY_NAME = diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java index 75d1a17a4157..c5627ec30d89 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java @@ -20,6 +20,7 @@ import org.apache.hadoop.hdds.ComponentVersion; import java.util.Arrays; +import java.util.Comparator; import java.util.Map; import static java.util.function.Function.identity; @@ -49,8 +50,6 @@ public enum ClientVersion implements ComponentVersion { FUTURE_VERSION(-1, "Used internally when the server side is older and an" + " unknown client version has arrived from the client."); - public static final String ERASURE_CODING_SUPPORT_NAME = ERASURE_CODING_SUPPORT.name(); - public static final ClientVersion CURRENT = latest(); public static final int CURRENT_VERSION = CURRENT.version; @@ -81,8 +80,8 @@ public static ClientVersion fromProtoValue(int value) { } private static ClientVersion latest() { - ClientVersion[] versions = ClientVersion.values(); - return versions[versions.length - 2]; + return Arrays.stream(ClientVersion.values()) + .max(Comparator.comparingInt(ComponentVersion::toProtoValue)).orElse(null); } } diff --git a/hadoop-hdds/common/src/main/resources/META-INF/services/javax.annotation.processing.Processor b/hadoop-hdds/common/src/main/resources/META-INF/services/javax.annotation.processing.Processor deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/VersionExtractor.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/VersionExtractor.java index 3e95545f3e7b..8f4b34f5d3d2 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/VersionExtractor.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/VersionExtractor.java @@ -48,7 +48,8 @@ public Class> getVersionClass() { CLIENT_VERSION_EXTRACTOR { @Override public Version extractVersion(OMRequest req, ValidationContext ctx) { - return ClientVersion.fromProtoValue(req.getVersion()); + return req.getVersion() > ClientVersion.CURRENT_VERSION ? + ClientVersion.CURRENT : ClientVersion.fromProtoValue(req.getVersion()); } @Override diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestOMClientVersionValidatorProcessor.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestOMClientVersionValidatorProcessor.java index d21134c95f4b..eecfb46f8e39 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestOMClientVersionValidatorProcessor.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestOMClientVersionValidatorProcessor.java @@ -165,26 +165,6 @@ public void testValidatorMethodCanBeProtected() { assertThat(compile(source)).succeeded(); } - @Test - public void testEmptyValidationConditionListDoesNotCompile() { - List source = generateSourceOfValidatorMethodWith( - annotationOf(preProcess(), aReqType()), - modifiers("public", "static"), - returnValue("OMRequest"), - parameters("OMRequest rq", "ValidationContext ctx"), - exceptions()); - } - - @Test - public void testNullValidationConditionListDoesNotCompile() { - List source = generateSourceOfValidatorMethodWith( - annotationOf(preProcess(), aReqType()), - modifiers("public", "static"), - returnValue("OMRequest"), - parameters("OMRequest rq", "ValidationContext ctx"), - exceptions()); - } - @Test public void testNotEnoughParametersForPreProcess() { List source = generateSourceOfValidatorMethodWith( From a84749b997acc98ce658d2f4fb1c2479c870bfb2 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Thu, 18 Jul 2024 11:13:42 -0700 Subject: [PATCH 07/28] HDDS-11149. Move to enum maps Change-Id: I22ff08e5f4c43cbdda7a7db5f4d83417b55d481e --- .../request/validation/ValidatorRegistry.java | 53 ++++++++++--------- .../validation/TestValidatorRegistry.java | 4 +- 2 files changed, 28 insertions(+), 29 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidatorRegistry.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidatorRegistry.java index 08ba39470f90..3c3aab89fa87 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidatorRegistry.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidatorRegistry.java @@ -16,8 +16,8 @@ */ package org.apache.hadoop.ozone.om.request.validation; +import com.google.common.collect.ImmutableMap; import org.apache.commons.lang3.tuple.Pair; -import org.apache.commons.lang3.tuple.Triple; import org.apache.hadoop.ozone.Version; import org.reflections.Reflections; import org.reflections.scanners.Scanners; @@ -31,12 +31,13 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; +import java.util.EnumMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.TreeMap; +import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -45,8 +46,9 @@ * a service. */ public class ValidatorRegistry> { - private final Map>, RequestType, RequestProcessingPhase>, - IndexedItems> indexedValidatorMap; + + private final Map, EnumMap>>> indexedValidatorMap; /** * Creates a {@link ValidatorRegistry} instance that discovers validation @@ -59,7 +61,7 @@ public class ValidatorRegistry> { */ ValidatorRegistry(Class requestType, String validatorPackage, - Set>> allowedVersionTypes, + Set> allowedVersionTypes, Set allowedProcessingPhases) { this(requestType, ClasspathHelper.forPackage(validatorPackage), allowedVersionTypes, allowedProcessingPhases); } @@ -82,7 +84,7 @@ private Class getReturnTypeOfAnnotationMethod(Class clz */ ValidatorRegistry(Class requestType, Collection searchUrls, - Set>> allowedVersionTypes, + Set> allowedVersionTypes, Set allowedProcessingPhases) { Set> validatorsToBeRegistered = new Reflections(new ConfigurationBuilder().setUrls(ClasspathHelper.forPackage("")) @@ -96,9 +98,8 @@ private Class getReturnTypeOfAnnotationMethod(Class clz RegisterValidator.MAX_VERSION_METHOD_NAME))) .map(annotationClass -> (Class) annotationClass) .collect(Collectors.toSet()); - int expectedNumberOfKeys = requestType.getEnumConstants().length * allowedProcessingPhases.size() * - allowedVersionTypes.stream().mapToInt(clz -> clz.getEnumConstants().length).sum(); - this.indexedValidatorMap = new HashMap<>(expectedNumberOfKeys, 1.0f); + this.indexedValidatorMap = allowedVersionTypes.stream().collect(ImmutableMap.toImmutableMap(Function.identity(), + versionClass -> new EnumMap<>(requestType))); Reflections reflections = new Reflections(new ConfigurationBuilder() .setUrls(searchUrls) .setScanners(Scanners.MethodsAnnotated) @@ -117,8 +118,9 @@ private Class getReturnTypeOfAnnotationMethod(Class clz * @param requestVersions different versions extracted from the request. * @return the list of validation methods that has to run. */ - public List validationsFor(RequestType requestType, RequestProcessingPhase phase, - List requestVersions) { + public List validationsFor(RequestType requestType, + RequestProcessingPhase phase, + List requestVersions) { return requestVersions.stream() .flatMap(requestVersion -> this.validationsFor(requestType, phase, requestVersion).stream()) .distinct().collect(Collectors.toList()); @@ -134,16 +136,14 @@ public List validationsFor(RequestType requestType, RequestProcessingPha * @param requestVersion version extracted corresponding to the request. * @return the list of validation methods that has to run. */ - public List validationsFor(RequestType requestType, RequestProcessingPhase phase, - Version requestVersion) { - Class> requestVersionClass = - (Class>) requestVersion.getClass(); - Triple>, RequestType, RequestProcessingPhase> key = - Triple.of(requestVersionClass, requestType, phase); - if (this.indexedValidatorMap.containsKey(key)) { - return this.indexedValidatorMap.get(key).getItemsGreaterThanIdx(requestVersion.getVersion()); - } - return Collections.emptyList(); + public List validationsFor(RequestType requestType, + RequestProcessingPhase phase, + V requestVersion) { + return Optional.ofNullable(this.indexedValidatorMap.get(requestVersion.getClass())) + .map(requestTypeMap -> requestTypeMap.get(requestType)).map(phaseMap -> phaseMap.get(phase)) + .map(indexedMethods -> indexedMethods.getItemsGreaterThanIdx(requestVersion.getVersion())) + .orElse(Collections.emptyList()); + } /** @@ -212,7 +212,7 @@ private void registerValidator(Class requestType, Class validatorToBeRegistered, Reflections reflections) { Collection methods = reflections.getMethodsAnnotatedWith(validatorToBeRegistered); - Class> versionClass = (Class>) + Class versionClass = (Class) this.getReturnTypeOfAnnotationMethod(validatorToBeRegistered, RegisterValidator.MAX_VERSION_METHOD_NAME); List> sortedMethodsByMaxVersion = methods.stream() .map(method -> Pair.of(method.getAnnotation(validatorToBeRegistered), method)) @@ -231,10 +231,11 @@ private void registerValidator(Class requestType, RequestType type = this.getRequestType(validator, requestType); method.setAccessible(true); - Triple>, RequestType, RequestProcessingPhase> indexKey = - Triple.of(versionClass, type, phase); - this.getAndInitialize(indexKey, IndexedItems::new, this.indexedValidatorMap) - .add(method, maxVersion.getVersion()); + EnumMap>> requestMap = + this.getAndInitialize(versionClass, () -> new EnumMap<>(requestType), this.indexedValidatorMap); + EnumMap> phaseMap = + this.getAndInitialize(type, () -> new EnumMap<>(RequestProcessingPhase.class), requestMap); + this.getAndInitialize(phase, IndexedItems::new, phaseMap).add(method, maxVersion.getVersion()); } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestValidatorRegistry.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestValidatorRegistry.java index 762838165bb4..0a4eaa0c62fa 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestValidatorRegistry.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestValidatorRegistry.java @@ -31,7 +31,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.Collections; import java.util.List; import java.util.Set; import java.util.stream.Collectors; @@ -82,8 +81,7 @@ public void testNoValidatorsReturnedForEmptyConditionList() { new ValidatorRegistry<>(OzoneManagerProtocolProtos.Type.class, PACKAGE, Arrays.stream(VersionExtractor.values()).map(VersionExtractor::getVersionClass).collect(Collectors.toSet()), REQUEST_PROCESSING_PHASES); - List validators = registry.validationsFor(CreateKey, PRE_PROCESS, - Collections.singletonList(ClientVersion.CURRENT)); + List validators = registry.validationsFor(CreateKey, PRE_PROCESS, ClientVersion.CURRENT); assertTrue(validators.isEmpty()); } From fc57929908cbb05a6304366e67f64317dbf9505d Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Thu, 18 Jul 2024 23:57:38 -0700 Subject: [PATCH 08/28] HDDS-11149. Move validation to ozone-common and fix docs Change-Id: I551cb26f648bfedb7bc5bfee63b34731b56b4d9b --- .../OmRequestFeatureValidatorProcessor.java | 1 - hadoop-hdds/client/pom.xml | 3 +- hadoop-hdds/common/pom.xml | 3 +- hadoop-hdds/container-service/pom.xml | 3 +- hadoop-hdds/framework/pom.xml | 3 +- hadoop-hdds/server-scm/pom.xml | 3 +- hadoop-hdds/tools/pom.xml | 3 +- hadoop-ozone/client/pom.xml | 3 +- hadoop-ozone/common/pom.xml | 3 +- .../request/validation/RegisterValidator.java | 4 +- .../validation/RequestProcessingPhase.java | 2 +- .../request/validation/ValidatorRegistry.java | 24 +- .../request/validation/package-info.java | 40 +++ hadoop-ozone/csi/pom.xml | 3 +- hadoop-ozone/insight/pom.xml | 3 +- hadoop-ozone/interface-storage/pom.xml | 3 +- .../request/bucket/OMBucketCreateRequest.java | 2 +- .../request/bucket/OMBucketDeleteRequest.java | 2 +- .../bucket/OMBucketSetPropertyRequest.java | 2 +- .../file/OMDirectoryCreateRequest.java | 2 +- .../om/request/file/OMFileCreateRequest.java | 2 +- .../request/key/OMAllocateBlockRequest.java | 2 +- .../om/request/key/OMKeyCommitRequest.java | 2 +- .../om/request/key/OMKeyCreateRequest.java | 2 +- .../om/request/key/OMKeyDeleteRequest.java | 2 +- .../om/request/key/OMKeyRenameRequest.java | 2 +- .../om/request/key/OMKeysDeleteRequest.java | 2 +- .../om/request/key/OMKeysRenameRequest.java | 2 +- .../request/key/acl/OMKeyAddAclRequest.java | 2 +- .../key/acl/OMKeyRemoveAclRequest.java | 2 +- .../request/key/acl/OMKeySetAclRequest.java | 2 +- .../S3InitiateMultipartUploadRequest.java | 2 +- .../S3MultipartUploadAbortRequest.java | 2 +- .../S3MultipartUploadCommitPartRequest.java | 2 +- .../S3MultipartUploadCompleteRequest.java | 2 +- .../validation/OMClientVersionValidator.java | 10 +- .../validation/OMLayoutVersionValidator.java | 10 +- .../validation/RequestValidations.java | 6 +- .../request/validation/VersionExtractor.java | 6 +- .../om/request/validation/package-info.java | 24 +- .../OzoneManagerRequestHandler.java | 2 +- ...sor.java => TestOMValidatorProcessor.java} | 321 +++++++++++------- .../validation/TestValidatorRegistry.java | 6 +- .../GeneralValidatorsForTesting.java | 4 +- ...ValidatorsForOnlyOldClientValidations.java | 2 +- hadoop-ozone/recon-codegen/pom.xml | 3 +- hadoop-ozone/recon/pom.xml | 3 +- hadoop-ozone/s3gateway/pom.xml | 3 +- hadoop-ozone/tools/pom.xml | 3 +- pom.xml | 3 +- 50 files changed, 344 insertions(+), 204 deletions(-) rename hadoop-ozone/{ozone-manager/src/main/java/org/apache/hadoop/ozone/om => common/src/main/java/org/apache/hadoop/ozone}/request/validation/RegisterValidator.java (96%) rename hadoop-ozone/{ozone-manager/src/main/java/org/apache/hadoop/ozone/om => common/src/main/java/org/apache/hadoop/ozone}/request/validation/RequestProcessingPhase.java (95%) rename hadoop-ozone/{ozone-manager/src/main/java/org/apache/hadoop/ozone/om => common/src/main/java/org/apache/hadoop/ozone}/request/validation/ValidatorRegistry.java (93%) create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/package-info.java rename hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/{TestOMClientVersionValidatorProcessor.java => TestOMValidatorProcessor.java} (54%) diff --git a/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/OmRequestFeatureValidatorProcessor.java b/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/OmRequestFeatureValidatorProcessor.java index 30f1e74ebec1..f11977262f41 100644 --- a/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/OmRequestFeatureValidatorProcessor.java +++ b/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/OmRequestFeatureValidatorProcessor.java @@ -16,7 +16,6 @@ */ package org.apache.ozone.annotations; - import javax.annotation.processing.AbstractProcessor; import javax.annotation.processing.RoundEnvironment; import javax.annotation.processing.SupportedAnnotationTypes; diff --git a/hadoop-hdds/client/pom.xml b/hadoop-hdds/client/pom.xml index 5cd4ead18fbf..08b72dbb63d2 100644 --- a/hadoop-hdds/client/pom.xml +++ b/hadoop-hdds/client/pom.xml @@ -97,7 +97,8 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> Only selected annotation processors are enabled, see configuration of maven-compiler-plugin. - org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator + org.apache.hadoop.ozone.om.request.validation.OMLayoutVersionValidator + org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator org.apache.hadoop.hdds.scm.metadata.Replicate org.kohsuke.MetaInfServices diff --git a/hadoop-hdds/common/pom.xml b/hadoop-hdds/common/pom.xml index aeec60f97906..376a102a9737 100644 --- a/hadoop-hdds/common/pom.xml +++ b/hadoop-hdds/common/pom.xml @@ -284,7 +284,8 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> Only selected annotation processors are enabled, see configuration of maven-compiler-plugin. - org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator + org.apache.hadoop.ozone.om.request.validation.OMLayoutVersionValidator + org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator org.apache.hadoop.hdds.scm.metadata.Replicate org.kohsuke.MetaInfServices diff --git a/hadoop-hdds/container-service/pom.xml b/hadoop-hdds/container-service/pom.xml index 7a341bd66a8f..ef219c02e591 100644 --- a/hadoop-hdds/container-service/pom.xml +++ b/hadoop-hdds/container-service/pom.xml @@ -150,7 +150,8 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> Only selected annotation processors are enabled, see configuration of maven-compiler-plugin. - org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator + org.apache.hadoop.ozone.om.request.validation.OMLayoutVersionValidator + org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator org.apache.hadoop.hdds.scm.metadata.Replicate org.kohsuke.MetaInfServices diff --git a/hadoop-hdds/framework/pom.xml b/hadoop-hdds/framework/pom.xml index 5eb3d2071b70..b51ec28c1bcb 100644 --- a/hadoop-hdds/framework/pom.xml +++ b/hadoop-hdds/framework/pom.xml @@ -227,7 +227,8 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> Only selected annotation processors are enabled, see configuration of maven-compiler-plugin. - org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator + org.apache.hadoop.ozone.om.request.validation.OMLayoutVersionValidator + org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator org.kohsuke.MetaInfServices diff --git a/hadoop-hdds/server-scm/pom.xml b/hadoop-hdds/server-scm/pom.xml index f1a998f27cfb..08bdc90a3e10 100644 --- a/hadoop-hdds/server-scm/pom.xml +++ b/hadoop-hdds/server-scm/pom.xml @@ -154,7 +154,8 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> Only selected annotation processors are enabled, see configuration of maven-compiler-plugin. - org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator + org.apache.hadoop.ozone.om.request.validation.OMLayoutVersionValidator + org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator org.kohsuke.MetaInfServices diff --git a/hadoop-hdds/tools/pom.xml b/hadoop-hdds/tools/pom.xml index 5e7238f5f3eb..280131d3ca26 100644 --- a/hadoop-hdds/tools/pom.xml +++ b/hadoop-hdds/tools/pom.xml @@ -146,7 +146,8 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> Only selected annotation processors are enabled, see configuration of maven-compiler-plugin. - org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator + org.apache.hadoop.ozone.om.request.validation.OMLayoutVersionValidator + org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator org.apache.hadoop.hdds.scm.metadata.Replicate diff --git a/hadoop-ozone/client/pom.xml b/hadoop-ozone/client/pom.xml index dac609caa46d..e53d7ddf514d 100644 --- a/hadoop-ozone/client/pom.xml +++ b/hadoop-ozone/client/pom.xml @@ -106,7 +106,8 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> Only selected annotation processors are enabled, see configuration of maven-compiler-plugin. - org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator + org.apache.hadoop.ozone.om.request.validation.OMLayoutVersionValidator + org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator org.apache.hadoop.hdds.scm.metadata.Replicate org.kohsuke.MetaInfServices diff --git a/hadoop-ozone/common/pom.xml b/hadoop-ozone/common/pom.xml index 15d961e09491..09bd997f1faf 100644 --- a/hadoop-ozone/common/pom.xml +++ b/hadoop-ozone/common/pom.xml @@ -167,7 +167,8 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> Only selected annotation processors are enabled, see configuration of maven-compiler-plugin. - org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator + org.apache.hadoop.ozone.om.request.validation.OMLayoutVersionValidator + org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator org.apache.hadoop.hdds.scm.metadata.Replicate org.kohsuke.MetaInfServices diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RegisterValidator.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/RegisterValidator.java similarity index 96% rename from hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RegisterValidator.java rename to hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/RegisterValidator.java index 29963caec0c6..afaeb2ead150 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RegisterValidator.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/RegisterValidator.java @@ -15,9 +15,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.hadoop.ozone.om.request.validation; +package org.apache.hadoop.ozone.request.validation; import org.apache.hadoop.ozone.Version; + import java.lang.annotation.ElementType; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; @@ -30,7 +31,6 @@ * maxVersion : Returns an enum which implement {@link Version} * requestType: Returns an Enum value. * processingPhase: Returns {@link RequestProcessingPhase} - * */ @Retention(RetentionPolicy.RUNTIME) @Target(ElementType.ANNOTATION_TYPE) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestProcessingPhase.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/RequestProcessingPhase.java similarity index 95% rename from hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestProcessingPhase.java rename to hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/RequestProcessingPhase.java index 672156842914..225ac1a6d452 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestProcessingPhase.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/RequestProcessingPhase.java @@ -14,7 +14,7 @@ * License for the specific language governing permissions and limitations under * the License. */ -package org.apache.hadoop.ozone.om.request.validation; +package org.apache.hadoop.ozone.request.validation; /** * Processing phase defines when a request validator should run. diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidatorRegistry.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/ValidatorRegistry.java similarity index 93% rename from hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidatorRegistry.java rename to hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/ValidatorRegistry.java index 3c3aab89fa87..d5fb7654c2dd 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidatorRegistry.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/ValidatorRegistry.java @@ -14,7 +14,7 @@ * License for the specific language governing permissions and limitations under * the License. */ -package org.apache.hadoop.ozone.om.request.validation; +package org.apache.hadoop.ozone.request.validation; import com.google.common.collect.ImmutableMap; import org.apache.commons.lang3.tuple.Pair; @@ -53,13 +53,13 @@ public class ValidatorRegistry> { /** * Creates a {@link ValidatorRegistry} instance that discovers validation * methods in the provided package and the packages in the same resource. - * A validation method is recognized by the {@link OMClientVersionValidator} - * annotation that contains important information about how and when to use - * the validator. + * A validation method is recognized by all the annotations classes which + * are annotated by {@link RegisterValidator} annotation that contains + * important information about how and when to use the validator. * @param validatorPackage the main package inside which validatiors should * be discovered. */ - ValidatorRegistry(Class requestType, + public ValidatorRegistry(Class requestType, String validatorPackage, Set> allowedVersionTypes, Set allowedProcessingPhases) { @@ -77,12 +77,12 @@ private Class getReturnTypeOfAnnotationMethod(Class clz /** * Creates a {@link ValidatorRegistry} instance that discovers validation * methods under the provided URL. - * A validation method is recognized by the {@link OMClientVersionValidator} + * A validation method is recognized by all annotations having the annotated by {@link RegisterValidator} * annotation that contains important information about how and when to use * the validator. * @param searchUrls the path in which the annotated methods are searched. */ - ValidatorRegistry(Class requestType, + public ValidatorRegistry(Class requestType, Collection searchUrls, Set> allowedVersionTypes, Set allowedProcessingPhases) { @@ -189,11 +189,9 @@ private void checkAllowedAnnotationValues(Set values, V value, String val /** * Initializes the internal request validator store. * The requests are stored in the following structure: - * - An EnumMap with the {@link VersionExtractor} as the key, and in which - * - values are an EnumMap with the request type as the key, and in which - * - values are Pair of lists, in which - * - left side is the pre-processing validations list - * - right side is the post-processing validations list + * - An EnumMap with the RequestType as the key, and in which + * - values are an EnumMap with the request processing phase as the key, and in which + * - values is an {@link IndexedItems } containing the validation list * @param validatorsToBeRegistered collection of the annotated validtors to process. */ private void initMaps(Class requestType, @@ -232,7 +230,7 @@ private void registerValidator(Class requestType, method.setAccessible(true); EnumMap>> requestMap = - this.getAndInitialize(versionClass, () -> new EnumMap<>(requestType), this.indexedValidatorMap); + this.indexedValidatorMap.get(versionClass); EnumMap> phaseMap = this.getAndInitialize(type, () -> new EnumMap<>(RequestProcessingPhase.class), requestMap); this.getAndInitialize(phase, IndexedItems::new, phaseMap).add(method, maxVersion.getVersion()); diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/package-info.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/package-info.java new file mode 100644 index 000000000000..dcfde0312bc3 --- /dev/null +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/package-info.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Request's validation handling. + * + * This package holds facilities to add new situation specific behaviour to + * request handling without cluttering the basic logic of the request handler + * code for any server. + * {@link org.apache.hadoop.ozone.request.validation.RegisterValidator} + * is used to register any validator which has the following methods: + * - maxVersion : Returns an enum which implement {@link org.apache.hadoop.ozone.Version} + * - requestType: Returns an Enum value. + * - processingPhase: Returns {@link org.apache.hadoop.ozone.request.validation.RequestProcessingPhase} + * + * The system uses a reflection based discovery to find methods that are + * annotated with the + * {@link org.apache.hadoop.ozone.request.validation.RegisterValidator} + * annotation. + * This annotation is used to register a particular annotation which inturn would be used to specify conditions in + * which a certain validator has to be used, the request type to which the validation should be applied, + * and the request processing phase in which we apply the validation and the maxVersion corresponding to which this + * is supposed to run. + */ + +package org.apache.hadoop.ozone.request.validation; diff --git a/hadoop-ozone/csi/pom.xml b/hadoop-ozone/csi/pom.xml index 04c153f3988c..0e9a3642ae5d 100644 --- a/hadoop-ozone/csi/pom.xml +++ b/hadoop-ozone/csi/pom.xml @@ -235,7 +235,8 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd"> Only selected annotation processors are enabled, see configuration of maven-compiler-plugin. - org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator + org.apache.hadoop.ozone.om.request.validation.OMLayoutVersionValidator + org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator org.apache.hadoop.hdds.scm.metadata.Replicate org.kohsuke.MetaInfServices diff --git a/hadoop-ozone/insight/pom.xml b/hadoop-ozone/insight/pom.xml index ae0bc5cd3aab..2ba16a910dd9 100644 --- a/hadoop-ozone/insight/pom.xml +++ b/hadoop-ozone/insight/pom.xml @@ -127,7 +127,8 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> Only selected annotation processors are enabled, see configuration of maven-compiler-plugin. - org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator + org.apache.hadoop.ozone.om.request.validation.OMLayoutVersionValidator + org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator org.apache.hadoop.hdds.scm.metadata.Replicate org.kohsuke.MetaInfServices diff --git a/hadoop-ozone/interface-storage/pom.xml b/hadoop-ozone/interface-storage/pom.xml index ca7f14f1fa50..aad7d28dccd7 100644 --- a/hadoop-ozone/interface-storage/pom.xml +++ b/hadoop-ozone/interface-storage/pom.xml @@ -104,7 +104,8 @@ Only selected annotation processors are enabled, see configuration of maven-compiler-plugin. - org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator + org.apache.hadoop.ozone.om.request.validation.OMLayoutVersionValidator + org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator org.apache.hadoop.hdds.scm.metadata.Replicate org.kohsuke.MetaInfServices diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java index 1c0d6db1b2df..f3a15a25baa5 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java @@ -44,7 +44,7 @@ import org.apache.hadoop.ozone.om.request.OMClientRequest; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; -import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; +import org.apache.hadoop.ozone.request.validation.RequestProcessingPhase; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.om.response.bucket.OMBucketCreateResponse; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketDeleteRequest.java index 8bf4417906f2..d9aff9546293 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketDeleteRequest.java @@ -34,7 +34,7 @@ import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; -import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; +import org.apache.hadoop.ozone.request.validation.RequestProcessingPhase; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketSetPropertyRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketSetPropertyRequest.java index 23acb390c31c..319ad2ab168d 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketSetPropertyRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketSetPropertyRequest.java @@ -34,7 +34,7 @@ import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.BucketEncryptionKeyInfo; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; -import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; +import org.apache.hadoop.ozone.request.validation.RequestProcessingPhase; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java index d796df1e67bb..33ec419588fc 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java @@ -42,7 +42,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; -import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; +import org.apache.hadoop.ozone.request.validation.RequestProcessingPhase; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java index 8dc05c84a106..f91050c57dd6 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java @@ -39,7 +39,7 @@ import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; -import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; +import org.apache.hadoop.ozone.request.validation.RequestProcessingPhase; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.response.file.OMFileCreateResponse; import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java index c8c43b264eeb..07356080774c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java @@ -34,7 +34,7 @@ import org.apache.hadoop.ozone.om.helpers.QuotaUtil; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; -import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; +import org.apache.hadoop.ozone.request.validation.RequestProcessingPhase; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java index ef58bc565485..f31210989a00 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java @@ -45,7 +45,7 @@ import org.apache.hadoop.ozone.om.helpers.WithMetadata; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; -import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; +import org.apache.hadoop.ozone.request.validation.RequestProcessingPhase; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java index 64745c93b135..fa2d126bd269 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java @@ -41,7 +41,7 @@ import org.apache.hadoop.ozone.om.request.file.OMFileRequest; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; -import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; +import org.apache.hadoop.ozone.request.validation.RequestProcessingPhase; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.UserInfo; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java index 1725ec5caf81..94da69c3f745 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java @@ -32,7 +32,7 @@ import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; -import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; +import org.apache.hadoop.ozone.request.validation.RequestProcessingPhase; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType; import org.slf4j.Logger; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java index aec37a002300..6a42eecd5e9f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java @@ -31,7 +31,7 @@ import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; -import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; +import org.apache.hadoop.ozone.request.validation.RequestProcessingPhase; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer; import org.apache.hadoop.ozone.security.acl.OzoneObj; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java index 9a3f214e4338..975816cb6074 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java @@ -40,7 +40,7 @@ import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; -import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; +import org.apache.hadoop.ozone.request.validation.RequestProcessingPhase; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.om.response.key.OMKeysDeleteResponse; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysRenameRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysRenameRequest.java index e3ca76e1ee5d..f4ed06348375 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysRenameRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysRenameRequest.java @@ -36,7 +36,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; -import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; +import org.apache.hadoop.ozone.request.validation.RequestProcessingPhase; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.om.response.key.OMKeysRenameResponse; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAddAclRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAddAclRequest.java index 673ee26a82cd..713ab4dd0784 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAddAclRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAddAclRequest.java @@ -36,7 +36,7 @@ import org.apache.hadoop.ozone.om.request.util.ObjectParser; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; -import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; +import org.apache.hadoop.ozone.request.validation.RequestProcessingPhase; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.response.key.acl.OMKeyAclResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyRemoveAclRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyRemoveAclRequest.java index 8f31b7dad405..50e14ae0ef7c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyRemoveAclRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyRemoveAclRequest.java @@ -36,7 +36,7 @@ import org.apache.hadoop.ozone.om.request.util.ObjectParser; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; -import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; +import org.apache.hadoop.ozone.request.validation.RequestProcessingPhase; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.response.key.acl.OMKeyAclResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeySetAclRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeySetAclRequest.java index 174a0542d6f0..b0d32a0cb082 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeySetAclRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeySetAclRequest.java @@ -37,7 +37,7 @@ import org.apache.hadoop.ozone.om.request.util.ObjectParser; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; -import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; +import org.apache.hadoop.ozone.request.validation.RequestProcessingPhase; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.response.key.acl.OMKeyAclResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java index be9ce421a952..cace71b33af6 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java @@ -40,7 +40,7 @@ import org.apache.hadoop.ozone.om.request.util.OMMultipartUploadUtils; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; -import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; +import org.apache.hadoop.ozone.request.validation.RequestProcessingPhase; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.om.response.s3.multipart.S3InitiateMultipartUploadResponse; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java index 1d93e8721ade..e2aa9960ee15 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java @@ -32,7 +32,7 @@ import org.apache.hadoop.ozone.om.request.util.OMMultipartUploadUtils; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; -import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; +import org.apache.hadoop.ozone.request.validation.RequestProcessingPhase; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java index 01a59b5b600f..943b59e4b3d6 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java @@ -36,7 +36,7 @@ import org.apache.hadoop.ozone.om.request.key.OMKeyRequest; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; -import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; +import org.apache.hadoop.ozone.request.validation.RequestProcessingPhase; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.om.response.s3.multipart diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java index df0046cfc0b0..96c0c894b29c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java @@ -58,7 +58,7 @@ import org.apache.hadoop.ozone.om.request.key.OMKeyRequest; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; -import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; +import org.apache.hadoop.ozone.request.validation.RequestProcessingPhase; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.om.response.s3.multipart.S3MultipartUploadCompleteResponse; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/OMClientVersionValidator.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/OMClientVersionValidator.java index f0528e94f7e3..5ea5392fdf49 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/OMClientVersionValidator.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/OMClientVersionValidator.java @@ -18,6 +18,9 @@ import org.apache.hadoop.ozone.ClientVersion; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; +import org.apache.hadoop.ozone.request.validation.RegisterValidator; +import org.apache.hadoop.ozone.request.validation.RequestProcessingPhase; +import org.apache.hadoop.ozone.request.validation.ValidatorRegistry; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; @@ -25,14 +28,15 @@ import java.lang.annotation.Target; /** - * An annotation to mark methods that do certain request validations. + * An annotation to mark methods that do certain request validations based on the + * request protocol's client version. * * The methods annotated with this annotation are collected by the * {@link ValidatorRegistry} class during the initialization of the server. * * The conditions specify the specific use case in which the validator should be - * applied to the request. See {@link VersionExtractor} for more details - * on the specific conditions. + * applied to the request. See {@link VersionExtractor} for getting all the supported different + * {@link org.apache.hadoop.ozone.Version}. * The validator method should be applied to just one specific request type * to help keep these methods simple and straightforward. If you want to use * the same validation for different request types, use inheritance, and diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/OMLayoutVersionValidator.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/OMLayoutVersionValidator.java index 50accaa28214..e6cd6c6d5714 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/OMLayoutVersionValidator.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/OMLayoutVersionValidator.java @@ -18,6 +18,9 @@ import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; +import org.apache.hadoop.ozone.request.validation.RegisterValidator; +import org.apache.hadoop.ozone.request.validation.RequestProcessingPhase; +import org.apache.hadoop.ozone.request.validation.ValidatorRegistry; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; @@ -25,14 +28,15 @@ import java.lang.annotation.Target; /** - * An annotation to mark methods that do certain request validations. + * An annotation to mark methods that do certain request validations based on the + * server's layout version and capability to perform a certain operation. * * The methods annotated with this annotation are collected by the * {@link ValidatorRegistry} class during the initialization of the server. * * The conditions specify the specific use case in which the validator should be - * applied to the request. See {@link VersionExtractor} for more details - * on the specific conditions. + * applied to the request. See {@link VersionExtractor} for getting all the supported different + * {@link org.apache.hadoop.ozone.Version}. * The validator method should be applied to just one specific request type * to help keep these methods simple and straightforward. If you want to use * the same validation for different request types, use inheritance, and diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestValidations.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestValidations.java index e286d3d93d2c..a41411270781 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestValidations.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestValidations.java @@ -23,6 +23,8 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; +import org.apache.hadoop.ozone.request.validation.RequestProcessingPhase; +import org.apache.hadoop.ozone.request.validation.ValidatorRegistry; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,8 +36,8 @@ import java.util.Set; import java.util.stream.Collectors; -import static org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase.POST_PROCESS; -import static org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase.PRE_PROCESS; +import static org.apache.hadoop.ozone.request.validation.RequestProcessingPhase.POST_PROCESS; +import static org.apache.hadoop.ozone.request.validation.RequestProcessingPhase.PRE_PROCESS; /** * Main class to configure and set up and access the request/response diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/VersionExtractor.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/VersionExtractor.java index 8f4b34f5d3d2..de4ece51676c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/VersionExtractor.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/VersionExtractor.java @@ -37,7 +37,7 @@ public Version extractVersion(OMRequest req, ValidationContext ctx) { } @Override - public Class> getVersionClass() { + public Class getVersionClass() { return OMLayoutFeature.class; } }, @@ -53,11 +53,11 @@ public Version extractVersion(OMRequest req, ValidationContext ctx) { } @Override - public Class> getVersionClass() { + public Class getVersionClass() { return ClientVersion.class; } }; public abstract Version extractVersion(OMRequest req, ValidationContext ctx); - public abstract Class> getVersionClass(); + public abstract Class getVersionClass(); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/package-info.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/package-info.java index c149734cc910..897cd4353cb3 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/package-info.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/package-info.java @@ -16,7 +16,7 @@ */ /** - * Request's feature validation handling. + * Request's validation handling. * * This package holds facilities to add new situation specific behaviour to * request handling without cluttering the basic logic of the request handler @@ -28,22 +28,28 @@ * request type * - a client connects to the server but uses an older version of the protocol * - a client connects to the server but uses a newer version of the protocol + * - a client connects to the server and performs an operation corresponding + * to a feature the server hasn't finalized for which these requests might have + * to be rejected. * - the code can handle certain checks that have to run all the time, but at * first we do not see a general use case that we would pull in immediately. - * These are the current + * These are the current registered * {@link org.apache.hadoop.ozone.om.request.validation.VersionExtractor}s - * but this list might be extended later on if we see other use cases. + * which would be extracted out of the om request and all validators + * fulfilling the condition would be run. * * The system uses a reflection based discovery to find methods that are * annotated with the - * {@link org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator} + * {@link org.apache.hadoop.ozone.request.validation.RegisterValidator} * annotation. - * This annotation is used to specify the condition in which a certain validator - * has to be used, the request type to which the validation should be applied, - * and the request processing phase in which we apply the validation. + * This annotation is used to register a particular annotation which inturn would be used to specify conditions in + * which a certain validator has to be used, the request type to which the validation should be applied, + * and the request processing phase in which we apply the validation and the maxVersion corresponding to which this + * is supposed to run. * - * One validator can be applied in multiple - * {@link org.apache.hadoop.ozone.om.request.validation.VersionExtractor} + * One validator can be applied in multiple, E.g. + * {@link org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator}, + * {@link org.apache.hadoop.ozone.om.request.validation.OMLayoutVersionValidator} * but a validator has to handle strictly just one * {@link org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type * }. diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java index 3033718193ab..e53d1e3a6704 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java @@ -73,7 +73,7 @@ import org.apache.hadoop.ozone.om.request.OMClientRequest; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator; -import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; +import org.apache.hadoop.ozone.request.validation.RequestProcessingPhase; import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.om.upgrade.DisallowedUntilLayoutVersion; diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestOMClientVersionValidatorProcessor.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestOMValidatorProcessor.java similarity index 54% rename from hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestOMClientVersionValidatorProcessor.java rename to hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestOMValidatorProcessor.java index eecfb46f8e39..71c15f19d2be 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestOMClientVersionValidatorProcessor.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestOMValidatorProcessor.java @@ -18,13 +18,18 @@ import com.google.testing.compile.Compilation; import com.google.testing.compile.JavaFileObjects; -import org.apache.hadoop.ozone.ClientVersion; +import org.apache.hadoop.ozone.Version; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; +import org.apache.hadoop.ozone.request.validation.RegisterValidator; +import org.apache.hadoop.ozone.request.validation.RequestProcessingPhase; import org.apache.ozone.annotations.OmRequestFeatureValidatorProcessor; -import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; +import org.reflections.Reflections; +import org.reflections.scanners.Scanners; +import org.reflections.util.ClasspathHelper; +import org.reflections.util.ConfigurationBuilder; import java.lang.annotation.Annotation; import java.lang.annotation.ElementType; @@ -32,7 +37,10 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Map; +import java.util.function.Function; import java.util.stream.Collectors; +import java.util.stream.Stream; import static com.google.testing.compile.CompilationSubject.assertThat; import static com.google.testing.compile.Compiler.javac; @@ -47,20 +55,43 @@ import static org.junit.jupiter.api.Assertions.assertSame; /** - * Compile tests against the annotation processor for the - * {@link OMClientVersionValidator} annotation. + * Compile tests against the annotation processor for all validators annotated with + * {@link RegisterValidator} annotation. * * The processor should ensure the method signatures and return values, based * on annotation arguments provided. */ -public class TestOMClientVersionValidatorProcessor { +public class TestOMValidatorProcessor { private static final String CLASSNAME = "Validation"; + private static final Map, Class> ANNOTATION_VERSION_CLASS_MAP = new Reflections( + new ConfigurationBuilder().setUrls(ClasspathHelper.forPackage("")).setScanners(Scanners.TypesAnnotated) + .setParallel(true)).getTypesAnnotatedWith(RegisterValidator.class).stream().filter(annotationClass -> { + try { + return annotationClass.getMethod(RegisterValidator.REQUEST_TYPE_METHOD_NAME).getReturnType() + .equals(Type.class); + } catch (NoSuchMethodException e) { + throw new RuntimeException(e); + } + }) + .collect(Collectors.toMap(Function.identity(), annotationClass -> { + try { + return annotationClass.getMethod(RegisterValidator.MAX_VERSION_METHOD_NAME).getReturnType(); + } catch (NoSuchMethodException e) { + throw new RuntimeException(e); + } + })); + + private static Stream annotatedClasses() { + return ANNOTATION_VERSION_CLASS_MAP.entrySet().stream().flatMap(e -> + Arrays.stream(e.getValue().getEnumConstants()).map(version -> Arguments.of(e.getKey(), version))); + } - @Test - public void testAnnotationCanOnlyBeAppliedOnMethods() { - Class c = OMClientVersionValidator.class; - for (Annotation a : c.getAnnotations()) { + @ParameterizedTest + @MethodSource("annotatedClasses") + public & Version> void testAnnotationCanOnlyBeAppliedOnMethods(Class annotationClass, + V version) { + for (Annotation a : annotationClass.getAnnotations()) { if (a instanceof Target) { assertEquals(1, ((Target) a).value().length); assertSame(((Target) a).value()[0], ElementType.METHOD); @@ -68,290 +99,327 @@ public void testAnnotationCanOnlyBeAppliedOnMethods() { } } - @Test - public void testACorrectAnnotationSetupForPreProcessCompiles() { + @ParameterizedTest + @MethodSource("annotatedClasses") + public & Version> void testACorrectAnnotationSetupForPreProcessCompiles(Class annotationClass, + V version) { List source = generateSourceOfValidatorMethodWith( - annotationOf(preProcess(), aReqType()), + annotationOf(preProcess(), aReqType(), annotationClass, version), modifiers("public", "static"), returnValue("OMRequest"), parameters("OMRequest rq", "ValidationContext ctx"), - exceptions("ServiceException")); + exceptions("ServiceException"), annotationClass); assertThat(compile(source)).succeeded(); } - @Test - public void testACorrectAnnotationSetupForPostProcessCompiles() { + @ParameterizedTest + @MethodSource("annotatedClasses") + public & Version> void testACorrectAnnotationSetupForPostProcessCompiles(Class annotationClass, + V version) { List source = generateSourceOfValidatorMethodWith( - annotationOf(postProcess(), aReqType()), + annotationOf(postProcess(), aReqType(), annotationClass, version), modifiers("public", "static"), returnValue("OMResponse"), parameters("OMRequest rq", "OMResponse rp", "ValidationContext ctx"), - exceptions("ServiceException")); + exceptions("ServiceException"), annotationClass); assertThat(compile(source)).succeeded(); } - @Test - public void testValidatorDoesNotNecessarilyThrowsExceptions() { + @ParameterizedTest + @MethodSource("annotatedClasses") + public & Version> void testValidatorDoesNotNecessarilyThrowsExceptions(Class annotationClass, + V version) { List source = generateSourceOfValidatorMethodWith( - annotationOf(preProcess(), aReqType()), + annotationOf(preProcess(), aReqType(), annotationClass, version), modifiers("public", "static"), returnValue("OMRequest"), parameters("OMRequest rq", "ValidationContext ctx"), - exceptions()); + exceptions(), annotationClass); assertThat(compile(source)).succeeded(); } - @Test - public void testNonStaticValidatorDoesNotCompile() { + @ParameterizedTest + @MethodSource("annotatedClasses") + public & Version> void testNonStaticValidatorDoesNotCompile(Class annotationClass, + V version) { List source = generateSourceOfValidatorMethodWith( - annotationOf(preProcess(), aReqType()), + annotationOf(preProcess(), aReqType(), annotationClass, version), modifiers("public"), returnValue("OMRequest"), parameters("OMRequest rq", "ValidationContext ctx"), - exceptions()); + exceptions(), annotationClass); assertThat(compile(source)) .hadErrorContaining(ERROR_VALIDATOR_METHOD_HAS_TO_BE_STATIC); } - @Test - public void testValidatorMethodCanBeFinal() { + @ParameterizedTest + @MethodSource("annotatedClasses") + public & Version> void testValidatorMethodCanBeFinal(Class annotationClass, + V version) { List source = generateSourceOfValidatorMethodWith( - annotationOf(preProcess(), aReqType()), + annotationOf(preProcess(), aReqType(), annotationClass, version), modifiers("public", "static", "final"), returnValue("OMRequest"), parameters("OMRequest rq", "ValidationContext ctx"), - exceptions()); + exceptions(), annotationClass); assertThat(compile(source)).succeeded(); } - @Test - public void testValidatorMethodCanBePrivate() { + @ParameterizedTest + @MethodSource("annotatedClasses") + public & Version> void testValidatorMethodCanBePrivate(Class annotationClass, + V version) { List source = generateSourceOfValidatorMethodWith( - annotationOf(preProcess(), aReqType()), + annotationOf(preProcess(), aReqType(), annotationClass, version), modifiers("private", "static"), returnValue("OMRequest"), parameters("OMRequest rq", "ValidationContext ctx"), - exceptions()); + exceptions(), annotationClass); assertThat(compile(source)).succeeded(); } - @Test - public void testValidatorMethodCanBeDefaultVisible() { + @ParameterizedTest + @MethodSource("annotatedClasses") + public & Version> void testValidatorMethodCanBeDefaultVisible(Class annotationClass, + V version) { List source = generateSourceOfValidatorMethodWith( - annotationOf(preProcess(), aReqType()), + annotationOf(preProcess(), aReqType(), annotationClass, version), modifiers("static"), returnValue("OMRequest"), parameters("OMRequest rq", "ValidationContext ctx"), - exceptions()); + exceptions(), annotationClass); assertThat(compile(source)).succeeded(); } - @Test - public void testValidatorMethodCanBeProtected() { + @ParameterizedTest + @MethodSource("annotatedClasses") + public & Version> void testValidatorMethodCanBeProtected(Class annotationClass, + V version) { List source = generateSourceOfValidatorMethodWith( - annotationOf(preProcess(), aReqType()), + annotationOf(preProcess(), aReqType(), annotationClass, version), modifiers("protected", "static"), returnValue("OMRequest"), parameters("OMRequest rq", "ValidationContext ctx"), - exceptions()); + exceptions(), annotationClass); assertThat(compile(source)).succeeded(); } - @Test - public void testNotEnoughParametersForPreProcess() { + @ParameterizedTest + @MethodSource("annotatedClasses") + public & Version> void testNotEnoughParametersForPreProcess(Class annotationClass, + V version) { List source = generateSourceOfValidatorMethodWith( - annotationOf(preProcess(), aReqType()), + annotationOf(preProcess(), aReqType(), annotationClass, version), modifiers("public", "static"), returnValue("OMRequest"), parameters("OMRequest rq"), - exceptions()); + exceptions(), annotationClass); assertThat(compile(source)) .hadErrorContaining( String.format(ERROR_UNEXPECTED_PARAMETER_COUNT, 2, 1)); } - @Test - public void testTooManyParametersForPreProcess() { + @ParameterizedTest + @MethodSource("annotatedClasses") + public & Version> void testTooManyParametersForPreProcess(Class annotationClass, + V version) { List source = generateSourceOfValidatorMethodWith( - annotationOf(preProcess(), aReqType()), + annotationOf(preProcess(), aReqType(), annotationClass, version), modifiers("public", "static"), returnValue("OMRequest"), parameters("OMRequest rq", "OMResponse rp", "ValidationContext ctx"), - exceptions()); + exceptions(), annotationClass); assertThat(compile(source)) .hadErrorContaining( String.format(ERROR_UNEXPECTED_PARAMETER_COUNT, 2, 3)); } - @Test - public void testNotEnoughParametersForPostProcess() { + @ParameterizedTest + @MethodSource("annotatedClasses") + public & Version> void testNotEnoughParametersForPostProcess(Class annotationClass, + V version) { List source = generateSourceOfValidatorMethodWith( - annotationOf(postProcess(), aReqType()), + annotationOf(postProcess(), aReqType(), annotationClass, version), modifiers("public", "static"), returnValue("OMResponse"), parameters("OMRequest rq", "OMResponse rp"), - exceptions()); + exceptions(), annotationClass); assertThat(compile(source)) .hadErrorContaining( String.format(ERROR_UNEXPECTED_PARAMETER_COUNT, 3, 2)); } - @Test - public void testTooManyParametersForPostProcess() { + @ParameterizedTest + @MethodSource("annotatedClasses") + public & Version> void testTooManyParametersForPostProcess(Class annotationClass, + V version) { List source = generateSourceOfValidatorMethodWith( - annotationOf(postProcess(), aReqType()), + annotationOf(postProcess(), aReqType(), annotationClass, version), modifiers("public", "static"), returnValue("OMResponse"), parameters("OMRequest rq", "OMResponse rp", "ValidationContext ctx", "String name"), - exceptions()); + exceptions(), annotationClass); assertThat(compile(source)) .hadErrorContaining( String.format(ERROR_UNEXPECTED_PARAMETER_COUNT, 3, 4)); } - @Test - public void testWrongReturnValueForPreProcess() { + @ParameterizedTest + @MethodSource("annotatedClasses") + public & Version> void testWrongReturnValueForPreProcess(Class annotationClass, + V version) { List source = generateSourceOfValidatorMethodWith( - annotationOf(preProcess(), aReqType()), + annotationOf(preProcess(), aReqType(), annotationClass, version), modifiers("public", "static"), returnValue("String"), parameters("OMRequest rq", "ValidationContext ctx"), - exceptions()); + exceptions(), annotationClass); assertThat(compile(source)) .hadErrorContaining(ERROR_VALIDATOR_METHOD_HAS_TO_RETURN_OMREQUEST); } - @Test - public void testWrongReturnValueForPostProcess() { + @ParameterizedTest + @MethodSource("annotatedClasses") + public & Version> void testWrongReturnValueForPostProcess(Class annotationClass, + V version) { List source = generateSourceOfValidatorMethodWith( - annotationOf(postProcess(), aReqType()), + annotationOf(postProcess(), aReqType(), annotationClass, version), modifiers("public", "static"), returnValue("String"), parameters("OMRequest rq", "OMResponse rp", "ValidationContext ctx"), - exceptions()); + exceptions(), annotationClass); assertThat(compile(source)) .hadErrorContaining(ERROR_VALIDATOR_METHOD_HAS_TO_RETURN_OMRESPONSE); } - @Test - public void testWrongFirstArgumentForPreProcess() { + @ParameterizedTest + @MethodSource("annotatedClasses") + public & Version> void testWrongFirstArgumentForPreProcess(Class annotationClass, + V version) { List source = generateSourceOfValidatorMethodWith( - annotationOf(preProcess(), aReqType()), + annotationOf(preProcess(), aReqType(), annotationClass, version), modifiers("public", "static"), returnValue("OMRequest"), parameters("String rq", "ValidationContext ctx"), - exceptions()); + exceptions(), annotationClass); assertThat(compile(source)) .hadErrorContaining(ERROR_FIRST_PARAM_HAS_TO_BE_OMREQUEST); } - @Test - public void testWrongFirstArgumentForPostProcess() { + @ParameterizedTest + @MethodSource("annotatedClasses") + public & Version> void testWrongFirstArgumentForPostProcess(Class annotationClass, + V version) { List source = generateSourceOfValidatorMethodWith( - annotationOf(postProcess(), aReqType()), + annotationOf(postProcess(), aReqType(), annotationClass, version), modifiers("public", "static"), returnValue("OMResponse"), parameters("String rq", "OMResponse rp", "ValidationContext ctx"), - exceptions()); + exceptions(), annotationClass); assertThat(compile(source)) .hadErrorContaining(ERROR_FIRST_PARAM_HAS_TO_BE_OMREQUEST); } - @Test - public void testWrongSecondArgumentForPreProcess() { + @ParameterizedTest + @MethodSource("annotatedClasses") + public & Version> void testWrongSecondArgumentForPreProcess(Class annotationClass, + V version) { List source = generateSourceOfValidatorMethodWith( - annotationOf(preProcess(), aReqType()), + annotationOf(preProcess(), aReqType(), annotationClass, version), modifiers("public", "static"), returnValue("OMRequest"), parameters("OMRequest rq", "String ctx"), - exceptions()); + exceptions(), annotationClass); assertThat(compile(source)) .hadErrorContaining(ERROR_LAST_PARAM_HAS_TO_BE_VALIDATION_CONTEXT); } - @Test - public void testWrongSecondArgumentForPostProcess() { + @ParameterizedTest + @MethodSource("annotatedClasses") + public & Version> void testWrongSecondArgumentForPostProcess(Class annotationClass, + V version) { List source = generateSourceOfValidatorMethodWith( - annotationOf(postProcess(), aReqType()), + annotationOf(postProcess(), aReqType(), annotationClass, version), modifiers("public", "static"), returnValue("OMResponse"), parameters("OMRequest rq", "String rp", "ValidationContext ctx"), - exceptions()); + exceptions(), annotationClass); assertThat(compile(source)) .hadErrorContaining(ERROR_SECOND_PARAM_HAS_TO_BE_OMRESPONSE); } - @Test - public void testWrongThirdArgumentForPostProcess() { + @ParameterizedTest + @MethodSource("annotatedClasses") + public & Version> void testWrongThirdArgumentForPostProcess(Class annotationClass, + V version) { List source = generateSourceOfValidatorMethodWith( - annotationOf(postProcess(), aReqType()), + annotationOf(postProcess(), aReqType(), annotationClass, version), modifiers("public", "static"), returnValue("OMResponse"), parameters("OMRequest rq", "OMResponse rp", "String ctx"), - exceptions()); + exceptions(), annotationClass); assertThat(compile(source)) .hadErrorContaining(ERROR_LAST_PARAM_HAS_TO_BE_VALIDATION_CONTEXT); } - @Test - public void testInvalidProcessingPhase() { + @ParameterizedTest + @MethodSource("annotatedClasses") + public & Version> void testInvalidProcessingPhase(Class annotationClass, + V version) { List source = generateSourceOfValidatorMethodWith( - annotationOf("INVALID", aReqType(), ClientVersion.CURRENT), + annotationOf("INVALID", aReqType(), annotationClass, version), modifiers("public", "static"), returnValue("OMResponse"), parameters("OMRequest rq", "OMResponse rp", "ValidationContext ctx"), - exceptions("ServiceException")); + exceptions("ServiceException"), annotationClass); assertThat(compile(source)).failed(); } - @Test - public void testInvalidClientVersion() { + @ParameterizedTest + @MethodSource("annotatedClasses") + public & Version> void testInvalidClientVersion(Class annotationClass, + V version) { List source = generateSourceOfValidatorMethodWith( - annotationOf(RequestProcessingPhase.PRE_PROCESS, aReqType(), null), + annotationOf(RequestProcessingPhase.PRE_PROCESS, aReqType(), annotationClass, null), modifiers("public", "static"), returnValue("OMResponse"), parameters("OMRequest rq", "OMResponse rp", "ValidationContext ctx"), - exceptions("ServiceException")); + exceptions("ServiceException"), annotationClass); assertThat(compile(source)).failed(); } - private static List getClientVersions() { - List clientVersions = - Arrays.stream(ClientVersion.values()).map(Arguments::of).collect(Collectors.toList()); - return clientVersions; - } - @ParameterizedTest - @MethodSource("getClientVersions") - public void testMultipleErrorMessages(ClientVersion clientVersion) { + @MethodSource("annotatedClasses") + public & Version> void testMultipleErrorMessages(Class annotationClass, + V version) { List source = generateSourceOfValidatorMethodWith( - annotationOf(postProcess(), aReqType(), clientVersion), + annotationOf(postProcess(), aReqType(), annotationClass, version), modifiers(), returnValue("String"), parameters("String rq", "int rp", "String ctx"), - exceptions()); + exceptions(), annotationClass); Compilation compilation = compile(source); assertThat(compilation) @@ -407,8 +475,9 @@ private List generateSourceOfValidatorMethodWith( String[] modifiers, String returnType, String[] paramspecs, - String[] exceptions) { - List lines = new ArrayList<>(allImports()); + String[] exceptions, + Class annotationClass) { + List lines = new ArrayList<>(allImports(annotationClass)); lines.add(""); lines.add("public class " + CLASSNAME + " {"); lines.add(""); @@ -421,43 +490,40 @@ private List generateSourceOfValidatorMethodWith( lines.add("}"); lines.add(""); lines.stream() - .filter(s -> !s.startsWith("import")) +// .filter(s -> !s.startsWith("import")) .forEach(System.out::println); return lines; } - private String annotationOf( - RequestProcessingPhase phase, - Type reqType) { - return annotationOf(phase.name(), reqType, ClientVersion.CURRENT); + private & Version> String annotationOf(RequestProcessingPhase phase, + Type reqType, Class annotationClass) { + return annotationOf(phase.name(), reqType, annotationClass, + (V)ANNOTATION_VERSION_CLASS_MAP.get(annotationClass).getEnumConstants()[0]); } - private String annotationOf( - RequestProcessingPhase phase, - Type reqType, - ClientVersion clientVersion) { - return annotationOf(phase.name(), reqType, clientVersion); + private & Version> String annotationOf( + RequestProcessingPhase phase, Type reqType, Class annotationClass, V maxVersion) { + return annotationOf(phase.name(), reqType, annotationClass, maxVersion); } - private String annotationOf( + private & Version> String annotationOf( String phase, Type reqType, - ClientVersion clientVersion) { + Class annotationClass, + V maxVersion) { StringBuilder annotation = new StringBuilder(); - annotation.append("@OMClientVersionValidator("); + annotation.append("@" + annotationClass.getName() + "("); annotation.append("processingPhase = ").append(phase); annotation.append(", requestType = ").append(reqType.name()); - if (clientVersion != null) { - annotation.append(", maxVersion = ").append(clientVersion.name()); + if (maxVersion != null) { + annotation.append(", maxVersion = ").append(maxVersion.name()); } annotation.append(" )"); return annotation.toString(); } - private List allImports() { + private List allImports(Class annotationClass) { List imports = new ArrayList<>(); - imports.add("import org.apache.hadoop.ozone.om.request.validation" - + ".OMClientVersionValidator;"); imports.add("import org.apache.hadoop.ozone.protocol.proto" + ".OzoneManagerProtocolProtos.OMRequest;"); imports.add("import org.apache.hadoop.ozone.protocol.proto" @@ -466,15 +532,18 @@ private List allImports() { + ".ValidationContext;"); imports.add("import com.google.protobuf.ServiceException;"); for (RequestProcessingPhase phase : RequestProcessingPhase.values()) { - imports.add("import static org.apache.hadoop.ozone.om.request.validation" + imports.add("import static org.apache.hadoop.ozone.request.validation" + ".RequestProcessingPhase." + phase.name() + ";"); } for (Type reqType : Type.values()) { imports.add("import static org.apache.hadoop.ozone.protocol.proto" + ".OzoneManagerProtocolProtos.Type." + reqType.name() + ";"); } - for (ClientVersion clientVersion : ClientVersion.values()) { - imports.add("import static org.apache.hadoop.ozone.ClientVersion." + clientVersion.name() + ";"); + imports.add("import " + annotationClass.getName() + ";"); + + for (Object enumConstant : ANNOTATION_VERSION_CLASS_MAP.get(annotationClass).getEnumConstants()) { + imports.add("import static " + ANNOTATION_VERSION_CLASS_MAP.get(annotationClass).getCanonicalName() + "." + + ((Enum)enumConstant).name() + ";"); } return imports; } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestValidatorRegistry.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestValidatorRegistry.java index 0a4eaa0c62fa..cc191f446534 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestValidatorRegistry.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestValidatorRegistry.java @@ -20,6 +20,8 @@ import org.apache.hadoop.ozone.ClientVersion; import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; +import org.apache.hadoop.ozone.request.validation.RequestProcessingPhase; +import org.apache.hadoop.ozone.request.validation.ValidatorRegistry; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -36,8 +38,8 @@ import java.util.stream.Collectors; import static java.util.Arrays.asList; -import static org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase.POST_PROCESS; -import static org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase.PRE_PROCESS; +import static org.apache.hadoop.ozone.request.validation.RequestProcessingPhase.POST_PROCESS; +import static org.apache.hadoop.ozone.request.validation.RequestProcessingPhase.PRE_PROCESS; import static org.apache.hadoop.ozone.om.request.validation.testvalidatorset1.GeneralValidatorsForTesting.startValidatorTest; import static org.apache.hadoop.ozone.om.request.validation.testvalidatorset1.GeneralValidatorsForTesting.finishValidatorTest; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.CreateDirectory; diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset1/GeneralValidatorsForTesting.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset1/GeneralValidatorsForTesting.java index a19d3e9c983e..50c825ca6c13 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset1/GeneralValidatorsForTesting.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset1/GeneralValidatorsForTesting.java @@ -29,8 +29,8 @@ import java.util.ArrayList; import java.util.List; -import static org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase.POST_PROCESS; -import static org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase.PRE_PROCESS; +import static org.apache.hadoop.ozone.request.validation.RequestProcessingPhase.POST_PROCESS; +import static org.apache.hadoop.ozone.request.validation.RequestProcessingPhase.PRE_PROCESS; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.CreateKey; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.CreateVolume; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.DeleteKeys; diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset2/ValidatorsForOnlyOldClientValidations.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset2/ValidatorsForOnlyOldClientValidations.java index b544c9a749d5..3b8c1a870853 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset2/ValidatorsForOnlyOldClientValidations.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset2/ValidatorsForOnlyOldClientValidations.java @@ -21,7 +21,7 @@ import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; -import static org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase.PRE_PROCESS; +import static org.apache.hadoop.ozone.request.validation.RequestProcessingPhase.PRE_PROCESS; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.CreateKey; /** diff --git a/hadoop-ozone/recon-codegen/pom.xml b/hadoop-ozone/recon-codegen/pom.xml index 26a70c459748..05a161afc88e 100644 --- a/hadoop-ozone/recon-codegen/pom.xml +++ b/hadoop-ozone/recon-codegen/pom.xml @@ -87,7 +87,8 @@ Only selected annotation processors are enabled, see configuration of maven-compiler-plugin. - org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator + org.apache.hadoop.ozone.om.request.validation.OMLayoutVersionValidator + org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator org.apache.hadoop.hdds.scm.metadata.Replicate org.kohsuke.MetaInfServices diff --git a/hadoop-ozone/recon/pom.xml b/hadoop-ozone/recon/pom.xml index 0e75e0850b0a..7e2500af5633 100644 --- a/hadoop-ozone/recon/pom.xml +++ b/hadoop-ozone/recon/pom.xml @@ -63,7 +63,8 @@ Only selected annotation processors are enabled, see configuration of maven-compiler-plugin. - org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator + org.apache.hadoop.ozone.om.request.validation.OMLayoutVersionValidator + org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator org.apache.hadoop.hdds.scm.metadata.Replicate org.kohsuke.MetaInfServices diff --git a/hadoop-ozone/s3gateway/pom.xml b/hadoop-ozone/s3gateway/pom.xml index 01fe0a257bdb..5853b481ad1b 100644 --- a/hadoop-ozone/s3gateway/pom.xml +++ b/hadoop-ozone/s3gateway/pom.xml @@ -177,7 +177,8 @@ Only selected annotation processors are enabled, see configuration of maven-compiler-plugin. - org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator + org.apache.hadoop.ozone.om.request.validation.OMLayoutVersionValidator + org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator org.apache.hadoop.hdds.scm.metadata.Replicate org.kohsuke.MetaInfServices diff --git a/hadoop-ozone/tools/pom.xml b/hadoop-ozone/tools/pom.xml index 98a6fef13e17..cc44cde9f3a4 100644 --- a/hadoop-ozone/tools/pom.xml +++ b/hadoop-ozone/tools/pom.xml @@ -154,7 +154,8 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> Only selected annotation processors are enabled, see configuration of maven-compiler-plugin. - org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator + org.apache.hadoop.ozone.om.request.validation.OMLayoutVersionValidator + org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator org.apache.hadoop.hdds.scm.metadata.Replicate diff --git a/pom.xml b/pom.xml index b7e30a2e01b0..38a3e8cc7446 100644 --- a/pom.xml +++ b/pom.xml @@ -1546,7 +1546,8 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs org.apache.hadoop.hdds.conf.Config org.apache.hadoop.hdds.conf.ConfigGroup - org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator + org.apache.hadoop.ozone.om.request.validation.OMLayoutVersionValidator + org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator org.apache.hadoop.hdds.scm.metadata.Replicate org.kohsuke.MetaInfServices From 778229acf9065977e56114d8b6ab4b724ed5ab7b Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Sun, 28 Jul 2024 22:02:32 -0700 Subject: [PATCH 09/28] HDDS-11149. Fix typo while extracting version Change-Id: I5fb15b4e3559fcee51f1964a4b0a50c57222f409 --- .../hadoop/ozone/om/request/validation/VersionExtractor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/VersionExtractor.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/VersionExtractor.java index de4ece51676c..7c0925db3090 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/VersionExtractor.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/VersionExtractor.java @@ -33,7 +33,7 @@ public enum VersionExtractor { @Override public Version extractVersion(OMRequest req, ValidationContext ctx) { LayoutVersionManager layoutVersionManager = ctx.versionManager(); - return ctx.versionManager().getFeature(layoutVersionManager.getSoftwareLayoutVersion()); + return ctx.versionManager().getFeature(layoutVersionManager.getMetadataLayoutVersion()); } @Override From 9cf34be40477074583cec1548793f7ccd83f34d5 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Sun, 11 Aug 2024 19:26:30 -0700 Subject: [PATCH 10/28] HDDS-11149. Address review comments Change-Id: If7c47ea7242cfac29934c1a0eae6c57a46ccea58 --- .../RegisterValidatorProcessor.java | 30 +++++++-- .../apache/hadoop/hdds/ComponentVersion.java | 2 +- .../java/org/apache/hadoop/ozone/Version.java | 2 +- .../hadoop/ozone/upgrade/LayoutFeature.java | 2 +- .../request/validation/RegisterValidator.java | 4 +- .../request/validation/ValidatorRegistry.java | 67 ++++++++++++------- .../request/validation/package-info.java | 2 +- .../request/bucket/OMBucketCreateRequest.java | 6 +- .../request/bucket/OMBucketDeleteRequest.java | 2 +- .../bucket/OMBucketSetPropertyRequest.java | 2 +- .../file/OMDirectoryCreateRequest.java | 4 +- .../om/request/file/OMFileCreateRequest.java | 4 +- .../request/key/OMAllocateBlockRequest.java | 4 +- .../om/request/key/OMKeyCommitRequest.java | 6 +- .../om/request/key/OMKeyCreateRequest.java | 4 +- .../om/request/key/OMKeyDeleteRequest.java | 2 +- .../om/request/key/OMKeyRenameRequest.java | 2 +- .../om/request/key/OMKeysDeleteRequest.java | 2 +- .../om/request/key/OMKeysRenameRequest.java | 2 +- .../request/key/acl/OMKeyAddAclRequest.java | 2 +- .../key/acl/OMKeyRemoveAclRequest.java | 2 +- .../request/key/acl/OMKeySetAclRequest.java | 2 +- .../S3InitiateMultipartUploadRequest.java | 4 +- .../S3MultipartUploadAbortRequest.java | 4 +- .../S3MultipartUploadCommitPartRequest.java | 4 +- .../S3MultipartUploadCompleteRequest.java | 4 +- .../validation/OMClientVersionValidator.java | 16 ++--- .../validation/OMLayoutVersionValidator.java | 6 +- .../request/validation/VersionExtractor.java | 2 +- .../om/request/validation/package-info.java | 10 ++- .../ozone/om/upgrade/OMLayoutFeature.java | 2 - .../OzoneManagerRequestHandler.java | 24 +++---- .../validation/TestOMValidatorProcessor.java | 14 ++-- .../validation/TestRequestValidations.java | 16 ++--- .../validation/TestValidatorRegistry.java | 37 +++++++--- .../GeneralValidatorsForTesting.java | 64 ++++++++++-------- ...ValidatorsForOnlyOldClientValidations.java | 4 +- 37 files changed, 209 insertions(+), 157 deletions(-) diff --git a/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/RegisterValidatorProcessor.java b/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/RegisterValidatorProcessor.java index ecec031a58f2..ccc83438a133 100644 --- a/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/RegisterValidatorProcessor.java +++ b/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/RegisterValidatorProcessor.java @@ -26,6 +26,8 @@ import javax.lang.model.element.ElementKind; import javax.lang.model.element.ExecutableElement; import javax.lang.model.element.TypeElement; +import javax.lang.model.type.ArrayType; +import javax.lang.model.type.TypeKind; import javax.lang.model.util.Elements; import javax.lang.model.util.Types; import javax.tools.Diagnostic; @@ -42,7 +44,7 @@ * META-INF/services/javax.annotation.processing.Processor file in the module's * resources folder. */ -@SupportedAnnotationTypes("org.apache.hadoop.ozone.om.request.validation.RegisterValidator") +@SupportedAnnotationTypes("org.apache.hadoop.ozone.request.validation.RegisterValidator") @SupportedSourceVersion(SourceVersion.RELEASE_8) public class RegisterValidatorProcessor extends AbstractProcessor { @@ -50,11 +52,11 @@ public class RegisterValidatorProcessor extends AbstractProcessor { public static final String VERSION_CLASS_NAME = "org.apache.hadoop.ozone.Version"; public static final String REQUEST_PROCESSING_PHASE_CLASS_NAME = "org.apache.hadoop.ozone.om.request.validation" + ".RequestProcessingPhase"; - public static final String MAX_VERSION_METHOD_NAME = "maxVersion"; + public static final String APPLY_UNTIL_METHOD_NAME = "applyUntil"; public static final String REQUEST_TYPE_METHOD_NAME = "requestType"; public static final String PROCESSING_PHASE_METHOD_NAME = "processingPhase"; - public static final String MAX_VERSION_NOT_FOUND_ERROR_MESSAGE = "Method " + MAX_VERSION_METHOD_NAME + + public static final String MAX_VERSION_NOT_FOUND_ERROR_MESSAGE = "Method " + APPLY_UNTIL_METHOD_NAME + " returning an enum implementing " + VERSION_CLASS_NAME + " not found"; public static final String REQUEST_TYPE_NOT_FOUND_ERROR_MESSAGE = "Method " + REQUEST_TYPE_METHOD_NAME + " returning an enum not found"; @@ -72,6 +74,20 @@ public boolean process(Set annotations, RoundEnvironment return false; } + private boolean validateArrayMethod(ExecutableElement method, String expectedMethodName, + ElementKind expectedReturnType, + String expectedReturnClass) { + Elements elementUtils = processingEnv.getElementUtils(); + Types types = processingEnv.getTypeUtils(); + TypeElement expectedReturnInterface = expectedReturnClass == null || expectedReturnClass.equals("") ? null : + elementUtils.getTypeElement(expectedReturnClass); + return method.getSimpleName().toString().equals(expectedMethodName) && (expectedReturnType == null || + TypeKind.ARRAY.equals(method.getReturnType().getKind()) && + types.asElement(((ArrayType)method.getReturnType()).getComponentType()).getKind() == expectedReturnType) && + (expectedReturnInterface == null || + types.isAssignable(types.asElement(method.getReturnType()).asType(), expectedReturnInterface.asType())); + } + private boolean validateMethod(ExecutableElement method, String expectedMethodName, ElementKind expectedReturnType, String expectedReturnClass) { Elements elementUtils = processingEnv.getElementUtils(); @@ -88,22 +104,22 @@ private boolean validateMethod(ExecutableElement method, String expectedMethodNa private void processElements(Set annotatedElements) { for (Element element : annotatedElements) { if (element.getKind().equals(ElementKind.ANNOTATION_TYPE)) { - boolean hasMaxVersion = false; + boolean hasApplyUntilMethod = false; boolean hasRequestType = false; boolean hasRequestProcessPhase = false; for (Element enclosedElement : element.getEnclosedElements()) { // Check if the annotation has a method called "validatorName" returning a String if (enclosedElement instanceof ExecutableElement) { ExecutableElement method = (ExecutableElement) enclosedElement; - hasMaxVersion = hasMaxVersion || validateMethod(method, MAX_VERSION_METHOD_NAME, ElementKind.ENUM, + hasApplyUntilMethod = hasApplyUntilMethod || validateMethod(method, APPLY_UNTIL_METHOD_NAME, ElementKind.ENUM, VERSION_CLASS_NAME); - hasRequestType = hasRequestType || validateMethod(method, REQUEST_TYPE_METHOD_NAME, ElementKind.ENUM, + hasRequestType = hasRequestType || validateArrayMethod(method, REQUEST_TYPE_METHOD_NAME, ElementKind.ENUM, null); hasRequestProcessPhase = hasRequestProcessPhase || validateMethod(method, PROCESSING_PHASE_METHOD_NAME, ElementKind.ENUM, REQUEST_PROCESSING_PHASE_CLASS_NAME); } } - if (!hasMaxVersion) { + if (!hasApplyUntilMethod) { emitErrorMsg(MAX_VERSION_NOT_FOUND_ERROR_MESSAGE + " for " + element.getSimpleName().toString()); } diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ComponentVersion.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ComponentVersion.java index cc8951b2ad75..a86d07fa5869 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ComponentVersion.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ComponentVersion.java @@ -38,7 +38,7 @@ public interface ComponentVersion extends Version { int toProtoValue(); @Override - default int getVersion() { + default int version() { return toProtoValue(); } } diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/Version.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/Version.java index 1e399b188b76..b6dc636e248c 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/Version.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/Version.java @@ -22,5 +22,5 @@ * Base class defining the version in the entire system. */ public interface Version { - int getVersion(); + int version(); } diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/LayoutFeature.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/LayoutFeature.java index 8d2faac6549d..93c5ef0a0b73 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/LayoutFeature.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/LayoutFeature.java @@ -51,7 +51,7 @@ default String name() { } @Override - default int getVersion() { + default int version() { return this.layoutVersion(); } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/RegisterValidator.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/RegisterValidator.java index afaeb2ead150..eb5457ab5734 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/RegisterValidator.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/RegisterValidator.java @@ -28,14 +28,14 @@ /** * Annotations to register a validator. {@link org.apache.ozone.annotations.RegisterValidatorProcessor} * enforces other annotation to have the following methods: - * maxVersion : Returns an enum which implement {@link Version} + * applyUntil : Returns an enum which implement {@link Version} * requestType: Returns an Enum value. * processingPhase: Returns {@link RequestProcessingPhase} */ @Retention(RetentionPolicy.RUNTIME) @Target(ElementType.ANNOTATION_TYPE) public @interface RegisterValidator { - String MAX_VERSION_METHOD_NAME = "maxVersion"; + String APPLY_UNTIL_METHOD_NAME = "applyUntil"; String REQUEST_TYPE_METHOD_NAME = "requestType"; String PROCESSING_PHASE_METHOD_NAME = "processingPhase"; } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/ValidatorRegistry.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/ValidatorRegistry.java index d5fb7654c2dd..e9e974f31a37 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/ValidatorRegistry.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/ValidatorRegistry.java @@ -17,6 +17,7 @@ package org.apache.hadoop.ozone.request.validation; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Sets; import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.ozone.Version; import org.reflections.Reflections; @@ -25,6 +26,7 @@ import org.reflections.util.ConfigurationBuilder; import java.lang.annotation.Annotation; +import java.lang.reflect.Array; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.net.URL; @@ -86,16 +88,18 @@ public ValidatorRegistry(Class requestType, Collection searchUrls, Set> allowedVersionTypes, Set allowedProcessingPhases) { + Class requestArrayClass = (Class) Array.newInstance(requestType, 0) + .getClass(); Set> validatorsToBeRegistered = new Reflections(new ConfigurationBuilder().setUrls(ClasspathHelper.forPackage("")) .setScanners(Scanners.TypesAnnotated) .setParallel(true)).getTypesAnnotatedWith(RegisterValidator.class).stream() .filter(annotationClass -> getReturnTypeOfAnnotationMethod((Class) annotationClass, RegisterValidator.REQUEST_TYPE_METHOD_NAME) - .equals(requestType)) + .equals(requestArrayClass)) .filter(annotationClass -> allowedVersionTypes.contains(getReturnTypeOfAnnotationMethod( (Class) annotationClass, - RegisterValidator.MAX_VERSION_METHOD_NAME))) + RegisterValidator.APPLY_UNTIL_METHOD_NAME))) .map(annotationClass -> (Class) annotationClass) .collect(Collectors.toSet()); this.indexedValidatorMap = allowedVersionTypes.stream().collect(ImmutableMap.toImmutableMap(Function.identity(), @@ -105,7 +109,7 @@ public ValidatorRegistry(Class requestType, .setScanners(Scanners.MethodsAnnotated) .setParallel(true) ); - initMaps(requestType, allowedProcessingPhases, validatorsToBeRegistered, reflections); + initMaps(requestArrayClass, allowedProcessingPhases, validatorsToBeRegistered, reflections); } /** @@ -139,9 +143,12 @@ public List validationsFor(RequestType requestType, public List validationsFor(RequestType requestType, RequestProcessingPhase phase, V requestVersion) { + return Optional.ofNullable(this.indexedValidatorMap.get(requestVersion.getClass())) .map(requestTypeMap -> requestTypeMap.get(requestType)).map(phaseMap -> phaseMap.get(phase)) - .map(indexedMethods -> indexedMethods.getItemsGreaterThanIdx(requestVersion.getVersion())) + .map(indexedMethods -> requestVersion.version() < 0 ? + indexedMethods.getItemsEqualToIdx(requestVersion.version()) : + indexedMethods.getItemsGreaterThanIdx(requestVersion.version())) .orElse(Collections.emptyList()); } @@ -163,8 +170,8 @@ private ReturnValue callAnnotationMe } } - private Version getMaxVersion(Validator validator) { - return callAnnotationMethod(validator, RegisterValidator.MAX_VERSION_METHOD_NAME, Version.class); + private Version getApplyUntilVersion(Validator validator) { + return callAnnotationMethod(validator, RegisterValidator.APPLY_UNTIL_METHOD_NAME, Version.class); } private RequestProcessingPhase getRequestPhase(Validator validator) { @@ -172,8 +179,8 @@ private RequestProcessingPhase getRequestPhase(Va RequestProcessingPhase.class); } - private RequestType getRequestType(Validator validator, - Class requestType) { + private RequestType[] getRequestType(Validator validator, + Class requestType) { return callAnnotationMethod(validator, RegisterValidator.REQUEST_TYPE_METHOD_NAME, requestType); } @@ -194,7 +201,7 @@ private void checkAllowedAnnotationValues(Set values, V value, String val * - values is an {@link IndexedItems } containing the validation list * @param validatorsToBeRegistered collection of the annotated validtors to process. */ - private void initMaps(Class requestType, + private void initMaps(Class requestType, Set allowedPhases, Collection> validatorsToBeRegistered, Reflections reflections) { @@ -203,37 +210,37 @@ private void initMaps(Class requestType, } } - - - private void registerValidator(Class requestType, + private void registerValidator(Class requestType, Set allowedPhases, Class validatorToBeRegistered, Reflections reflections) { Collection methods = reflections.getMethodsAnnotatedWith(validatorToBeRegistered); Class versionClass = (Class) - this.getReturnTypeOfAnnotationMethod(validatorToBeRegistered, RegisterValidator.MAX_VERSION_METHOD_NAME); - List> sortedMethodsByMaxVersion = methods.stream() + this.getReturnTypeOfAnnotationMethod(validatorToBeRegistered, RegisterValidator.APPLY_UNTIL_METHOD_NAME); + List> sortedMethodsByApplyUntilVersion = methods.stream() .map(method -> Pair.of(method.getAnnotation(validatorToBeRegistered), method)) .sorted((validatorMethodPair1, validatorMethodPair2) -> Integer.compare( - this.getMaxVersion(validatorMethodPair1.getKey()).getVersion(), - this.getMaxVersion(validatorMethodPair2.getKey()).getVersion())) + this.getApplyUntilVersion(validatorMethodPair1.getKey()).version(), + this.getApplyUntilVersion(validatorMethodPair2.getKey()).version())) .collect(Collectors.toList()); - for (Pair validatorMethodPair : sortedMethodsByMaxVersion) { + for (Pair validatorMethodPair : sortedMethodsByApplyUntilVersion) { Annotation validator = validatorMethodPair.getKey(); Method method = validatorMethodPair.getValue(); - Version maxVersion = this.getMaxVersion(validator); + Version applyUntilVersion = this.getApplyUntilVersion(validator); RequestProcessingPhase phase = this.getRequestPhase(validator); checkAllowedAnnotationValues(allowedPhases, phase, RegisterValidator.PROCESSING_PHASE_METHOD_NAME, method.getName()); - RequestType type = this.getRequestType(validator, requestType); + Set types = Sets.newHashSet(this.getRequestType(validator, requestType)); method.setAccessible(true); + for (RequestType type : types) { + EnumMap>> requestMap = + this.indexedValidatorMap.get(versionClass); + EnumMap> phaseMap = + this.getAndInitialize(type, () -> new EnumMap<>(RequestProcessingPhase.class), requestMap); + this.getAndInitialize(phase, IndexedItems::new, phaseMap).add(method, applyUntilVersion.version()); + } - EnumMap>> requestMap = - this.indexedValidatorMap.get(versionClass); - EnumMap> phaseMap = - this.getAndInitialize(type, () -> new EnumMap<>(RequestProcessingPhase.class), requestMap); - this.getAndInitialize(phase, IndexedItems::new, phaseMap).add(method, maxVersion.getVersion()); } } @@ -273,6 +280,18 @@ public List getItemsGreaterThanIdx(IDX indexValue) { .map(startIndex -> items.subList(startIndex, items.size())).orElse(Collections.emptyList()); } + /** + * + * @param indexValue Given index value. + * @return All the items which has an index value greater than given index value. + */ + public List getItemsEqualToIdx(IDX indexValue) { + return Optional.ofNullable(indexMap.get(indexValue)) + .map(startIndex -> items.subList(startIndex, Optional.ofNullable(indexMap.higherEntry(indexValue)) + .map(Map.Entry::getValue).orElse(items.size()))) + .orElse(Collections.emptyList()); + } + } } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/package-info.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/package-info.java index dcfde0312bc3..cf7ad12d5471 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/package-info.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/package-info.java @@ -23,7 +23,7 @@ * code for any server. * {@link org.apache.hadoop.ozone.request.validation.RegisterValidator} * is used to register any validator which has the following methods: - * - maxVersion : Returns an enum which implement {@link org.apache.hadoop.ozone.Version} + * - applyUntil : Returns an enum which implement {@link org.apache.hadoop.ozone.Version} * - requestType: Returns an Enum value. * - processingPhase: Returns {@link org.apache.hadoop.ozone.request.validation.RequestProcessingPhase} * diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java index f3a15a25baa5..a1cb80d5c16f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java @@ -398,7 +398,7 @@ public boolean checkQuotaBytesValid(OMMetadataManager metadataManager, @OMLayoutVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CreateBucket, - maxVersion = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT + applyUntil = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT ) public static OMRequest disallowCreateBucketWithECReplicationConfig( OMRequest req, ValidationContext ctx) throws OMException { @@ -421,7 +421,7 @@ public static OMRequest disallowCreateBucketWithECReplicationConfig( @OMLayoutVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CreateBucket, - maxVersion = OMLayoutFeature.BUCKET_LAYOUT_SUPPORT + applyUntil = OMLayoutFeature.BUCKET_LAYOUT_SUPPORT ) public static OMRequest handleCreateBucketWithBucketLayoutDuringPreFinalize( OMRequest req, ValidationContext ctx) throws OMException { @@ -462,7 +462,7 @@ public static OMRequest handleCreateBucketWithBucketLayoutDuringPreFinalize( @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CreateBucket, - maxVersion = ClientVersion.ERASURE_CODING_SUPPORT + applyUntil = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest setDefaultBucketLayoutForOlderClients(OMRequest req, ValidationContext ctx) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketDeleteRequest.java index d9aff9546293..10ba3abc2808 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketDeleteRequest.java @@ -283,7 +283,7 @@ private boolean bucketContainsSnapshotInCache( @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.DeleteBucket, - maxVersion = ClientVersion.ERASURE_CODING_SUPPORT + applyUntil = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockBucketDeleteWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketSetPropertyRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketSetPropertyRequest.java index 319ad2ab168d..7336b4a0158a 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketSetPropertyRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketSetPropertyRequest.java @@ -359,7 +359,7 @@ public boolean checkQuotaNamespaceValid(OmVolumeArgs omVolumeArgs, @OMLayoutVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.SetBucketProperty, - maxVersion = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT + applyUntil = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT ) public static OMRequest disallowSetBucketPropertyWithECReplicationConfig( OMRequest req, ValidationContext ctx) throws OMException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java index 33ec419588fc..47aa53ee5cf7 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java @@ -402,7 +402,7 @@ static long getMaxNumOfRecursiveDirs() { @OMLayoutVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CreateDirectory, - maxVersion = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT + applyUntil = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT ) public static OMRequest disallowCreateDirectoryWithECReplicationConfig( OMRequest req, ValidationContext ctx) throws OMException { @@ -435,7 +435,7 @@ public static OMRequest disallowCreateDirectoryWithECReplicationConfig( @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CreateDirectory, - maxVersion = ClientVersion.ERASURE_CODING_SUPPORT + applyUntil = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockCreateDirectoryWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java index f91050c57dd6..aeaeb4da40c9 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java @@ -396,7 +396,7 @@ protected void checkAllParentsExist(KeyArgs keyArgs, @OMLayoutVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = CreateFile, - maxVersion = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT + applyUntil = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT ) public static OMRequest disallowCreateFileWithECReplicationConfig( OMRequest req, ValidationContext ctx) throws OMException { @@ -428,7 +428,7 @@ public static OMRequest disallowCreateFileWithECReplicationConfig( @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CreateFile, - maxVersion = ClientVersion.ERASURE_CODING_SUPPORT + applyUntil = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockCreateFileWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java index 07356080774c..2cd958f0fafd 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java @@ -271,7 +271,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn @OMLayoutVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.AllocateBlock, - maxVersion = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT + applyUntil = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT ) public static OMRequest disallowAllocateBlockWithECReplicationConfig( OMRequest req, ValidationContext ctx) throws OMException { @@ -301,7 +301,7 @@ public static OMRequest disallowAllocateBlockWithECReplicationConfig( @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.AllocateBlock, - maxVersion = ClientVersion.ERASURE_CODING_SUPPORT + applyUntil = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockAllocateBlockWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java index f31210989a00..d7104c35b0a5 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java @@ -436,7 +436,7 @@ protected void processResult(CommitKeyRequest commitKeyRequest, @OMLayoutVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CommitKey, - maxVersion = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT + applyUntil = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT ) public static OMRequest disallowCommitKeyWithECReplicationConfig( OMRequest req, ValidationContext ctx) throws OMException { @@ -466,7 +466,7 @@ public static OMRequest disallowCommitKeyWithECReplicationConfig( @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CommitKey, - maxVersion = ClientVersion.ERASURE_CODING_SUPPORT + applyUntil = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockCommitKeyWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { @@ -485,7 +485,7 @@ public static OMRequest blockCommitKeyWithBucketLayoutFromOldClient( @OMLayoutVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CommitKey, - maxVersion = OMLayoutFeature.HSYNC + applyUntil = OMLayoutFeature.HSYNC ) public static OMRequest disallowHsync( OMRequest req, ValidationContext ctx) throws OMException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java index fa2d126bd269..4de22ba819d8 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java @@ -397,7 +397,7 @@ protected void logResult(CreateKeyRequest createKeyRequest, @OMLayoutVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CreateKey, - maxVersion = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT + applyUntil = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT ) public static OMRequest disallowCreateKeyWithECReplicationConfig( OMRequest req, ValidationContext ctx) throws OMException { @@ -427,7 +427,7 @@ public static OMRequest disallowCreateKeyWithECReplicationConfig( @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CreateKey, - maxVersion = ClientVersion.ERASURE_CODING_SUPPORT + applyUntil = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockCreateKeyWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java index 94da69c3f745..8f66296c9b13 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java @@ -242,7 +242,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.DeleteKey, - maxVersion = ClientVersion.ERASURE_CODING_SUPPORT + applyUntil = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockDeleteKeyWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java index 6a42eecd5e9f..0c463392e429 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java @@ -272,7 +272,7 @@ private Map buildAuditMap( @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.RenameKey, - maxVersion = ClientVersion.ERASURE_CODING_SUPPORT + applyUntil = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockRenameKeyWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java index 975816cb6074..d98143713ed8 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java @@ -363,7 +363,7 @@ protected static void addDeletedKeys(Map auditMap, @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.DeleteKeys, - maxVersion = ClientVersion.ERASURE_CODING_SUPPORT + applyUntil = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockDeleteKeysWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysRenameRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysRenameRequest.java index f4ed06348375..2d56dac212dd 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysRenameRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysRenameRequest.java @@ -299,7 +299,7 @@ private Map buildAuditMap(Map auditMap, @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.RenameKeys, - maxVersion = ClientVersion.ERASURE_CODING_SUPPORT + applyUntil = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockRenameKeysWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAddAclRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAddAclRequest.java index 713ab4dd0784..725f83ba59d1 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAddAclRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAddAclRequest.java @@ -167,7 +167,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.AddAcl, - maxVersion = ClientVersion.ERASURE_CODING_SUPPORT + applyUntil = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockAddAclWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyRemoveAclRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyRemoveAclRequest.java index 50e14ae0ef7c..212a70422b15 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyRemoveAclRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyRemoveAclRequest.java @@ -168,7 +168,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.RemoveAcl, - maxVersion = ClientVersion.ERASURE_CODING_SUPPORT + applyUntil = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockRemoveAclWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeySetAclRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeySetAclRequest.java index b0d32a0cb082..f01fb8561d28 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeySetAclRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeySetAclRequest.java @@ -164,7 +164,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.SetAcl, - maxVersion = ClientVersion.BUCKET_LAYOUT_SUPPORT + applyUntil = ClientVersion.BUCKET_LAYOUT_SUPPORT ) public static OMRequest blockSetAclWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java index cace71b33af6..620e6e076753 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java @@ -290,7 +290,7 @@ protected void logResult(OzoneManager ozoneManager, @OMLayoutVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.InitiateMultiPartUpload, - maxVersion = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT + applyUntil = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT ) public static OMRequest disallowInitiateMultiPartUploadWithECReplicationConfig( @@ -323,7 +323,7 @@ protected void logResult(OzoneManager ozoneManager, @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.InitiateMultiPartUpload, - maxVersion = ClientVersion.ERASURE_CODING_SUPPORT + applyUntil = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockInitiateMPUWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java index e2aa9960ee15..2bb909884fb2 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java @@ -261,7 +261,7 @@ protected String getMultipartOpenKey(String multipartUploadID, @OMLayoutVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.AbortMultiPartUpload, - maxVersion = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT + applyUntil = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT ) public static OMRequest disallowAbortMultiPartUploadWithECReplicationConfig( OMRequest req, ValidationContext ctx) throws OMException { @@ -292,7 +292,7 @@ public static OMRequest disallowAbortMultiPartUploadWithECReplicationConfig( @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.AbortMultiPartUpload, - maxVersion = ClientVersion.ERASURE_CODING_SUPPORT + applyUntil = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockMPUAbortWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java index 943b59e4b3d6..79581b62b963 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java @@ -352,7 +352,7 @@ private String getMultipartKey(String volumeName, String bucketName, @OMLayoutVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CommitMultiPartUpload, - maxVersion = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT + applyUntil = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT ) public static OMRequest disallowCommitMultiPartUploadWithECReplicationConfig( OMRequest req, ValidationContext ctx) throws OMException { @@ -383,7 +383,7 @@ public static OMRequest disallowCommitMultiPartUploadWithECReplicationConfig( @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CommitMultiPartUpload, - maxVersion = ClientVersion.ERASURE_CODING_SUPPORT + applyUntil = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockMPUCommitWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java index 96c0c894b29c..11b217805de8 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java @@ -713,7 +713,7 @@ private void updateCache(OMMetadataManager omMetadataManager, @OMLayoutVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CompleteMultiPartUpload, - maxVersion = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT + applyUntil = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT ) public static OMRequest disallowCompleteMultiPartUploadWithECReplicationConfig( @@ -745,7 +745,7 @@ private void updateCache(OMMetadataManager omMetadataManager, @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CompleteMultiPartUpload, - maxVersion = ClientVersion.ERASURE_CODING_SUPPORT + applyUntil = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockMPUCompleteWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/OMClientVersionValidator.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/OMClientVersionValidator.java index 5ea5392fdf49..26f27b8803cd 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/OMClientVersionValidator.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/OMClientVersionValidator.java @@ -37,13 +37,9 @@ * The conditions specify the specific use case in which the validator should be * applied to the request. See {@link VersionExtractor} for getting all the supported different * {@link org.apache.hadoop.ozone.Version}. - * The validator method should be applied to just one specific request type + * The validator method should be applied the specified request types * to help keep these methods simple and straightforward. If you want to use - * the same validation for different request types, use inheritance, and - * annotate the override method that just calls super. - * Note that the aim is to have these validators together with the request - * processing code, so the handling of these specific situations are easy to - * find. + * the same validation for different requests just put it as part of the lists of request types. * * The annotated methods have to have a fixed signature. * A {@link RequestProcessingPhase#PRE_PROCESS} phase method is running before @@ -68,7 +64,7 @@ * Its signature has to be the following: * - it has to be static and idempotent * - it has three parameters - * - similalry to the pre-processing validators, first parameter is the + * - similarly to the pre-processing validators, first parameter is the * OMRequest, the second parameter is the OMResponse, and the third * parameter is a ValidationContext. * - the method has to return the modified OMResponse or throw a @@ -94,13 +90,13 @@ * The type of the request handled by this validator method. * @return the requestType to whihc the validator shoudl be applied */ - Type requestType(); + Type[] requestType(); /** * The max version for which the validator would run. The validator would run for the request * where the version is older than the excluding of the specified version. - * @returns the max required client version for which the validator runs for older version. + * @returns the max client version until which the validator runs excluding the specified version itself. */ - ClientVersion maxVersion(); + ClientVersion applyUntil(); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/OMLayoutVersionValidator.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/OMLayoutVersionValidator.java index e6cd6c6d5714..1ad1df607045 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/OMLayoutVersionValidator.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/OMLayoutVersionValidator.java @@ -94,13 +94,13 @@ * The type of the request handled by this validator method. * @return the requestType to whihc the validator shoudl be applied */ - Type requestType(); + Type[] requestType(); /** * The max version for which the validator would run. The validator would run for the request * where the version is older than the excluding of the specified version. - * @returns the max required client version for which the validator runs for older version. + * @returns the max layout version until which the validator runs excluding the specified version itself. */ - OMLayoutFeature maxVersion(); + OMLayoutFeature applyUntil(); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/VersionExtractor.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/VersionExtractor.java index 7c0925db3090..04e36930fef3 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/VersionExtractor.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/VersionExtractor.java @@ -49,7 +49,7 @@ public Class getVersionClass() { @Override public Version extractVersion(OMRequest req, ValidationContext ctx) { return req.getVersion() > ClientVersion.CURRENT_VERSION ? - ClientVersion.CURRENT : ClientVersion.fromProtoValue(req.getVersion()); + ClientVersion.FUTURE_VERSION : ClientVersion.fromProtoValue(req.getVersion()); } @Override diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/package-info.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/package-info.java index 897cd4353cb3..326abac63598 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/package-info.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/package-info.java @@ -38,21 +38,19 @@ * which would be extracted out of the om request and all validators * fulfilling the condition would be run. * - * The system uses a reflection based discovery to find methods that are + * The system uses a reflection based discovery to find annotations that are * annotated with the * {@link org.apache.hadoop.ozone.request.validation.RegisterValidator} * annotation. - * This annotation is used to register a particular annotation which inturn would be used to specify conditions in - * which a certain validator has to be used, the request type to which the validation should be applied, + * This annotation is used to register a particular annotation which in turn would be used to specify + * the request type to which the validation should be applied, * and the request processing phase in which we apply the validation and the maxVersion corresponding to which this * is supposed to run. * * One validator can be applied in multiple, E.g. * {@link org.apache.hadoop.ozone.om.request.validation.OMClientVersionValidator}, * {@link org.apache.hadoop.ozone.om.request.validation.OMLayoutVersionValidator} - * but a validator has to handle strictly just one - * {@link org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type - * }. + * * The main reason to avoid validating multiple request types with the same * validator, is that these validators have to be simple methods without state * any complex validation has to happen in the reql request handling. diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java index 247a89e63666..d931c128a901 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java @@ -28,8 +28,6 @@ * List of OM Layout features / versions. */ public enum OMLayoutFeature implements LayoutFeature { - - ////////////////////////////// ////////////////////////////// INITIAL_VERSION(0, "Initial Layout Version"), diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java index e53d1e3a6704..6eaf2697579c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java @@ -631,7 +631,7 @@ private GetKeyInfoResponse getKeyInfo(GetKeyInfoRequest request, @OMClientVersionValidator( processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.LookupKey, - maxVersion = ClientVersion.VERSION_HANDLES_UNKNOWN_DN_PORTS + applyUntil = ClientVersion.VERSION_HANDLES_UNKNOWN_DN_PORTS ) public static OMResponse disallowLookupKeyResponseWithECReplicationConfig( OMRequest req, OMResponse resp, ValidationContext ctx) @@ -658,7 +658,7 @@ public static OMResponse disallowLookupKeyResponseWithECReplicationConfig( @OMClientVersionValidator( processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.LookupKey, - maxVersion = ClientVersion.ERASURE_CODING_SUPPORT + applyUntil = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMResponse disallowLookupKeyWithBucketLayout( OMRequest req, OMResponse resp, ValidationContext ctx) @@ -745,7 +745,7 @@ private ListKeysLightResponse listKeysLight(ListKeysRequest request) @OMClientVersionValidator( processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.ListKeys, - maxVersion = ClientVersion.VERSION_HANDLES_UNKNOWN_DN_PORTS + applyUntil = ClientVersion.VERSION_HANDLES_UNKNOWN_DN_PORTS ) public static OMResponse disallowListKeysResponseWithECReplicationConfig( OMRequest req, OMResponse resp, ValidationContext ctx) @@ -772,7 +772,7 @@ public static OMResponse disallowListKeysResponseWithECReplicationConfig( @OMClientVersionValidator( processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.ListKeys, - maxVersion = ClientVersion.ERASURE_CODING_SUPPORT + applyUntil = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMResponse disallowListKeysWithBucketLayout( OMRequest req, OMResponse resp, ValidationContext ctx) @@ -833,7 +833,7 @@ private ListTrashResponse listTrash(ListTrashRequest request, @OMClientVersionValidator( processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.ListTrash, - maxVersion = ClientVersion.VERSION_HANDLES_UNKNOWN_DN_PORTS + applyUntil = ClientVersion.VERSION_HANDLES_UNKNOWN_DN_PORTS ) public static OMResponse disallowListTrashWithECReplicationConfig( OMRequest req, OMResponse resp, ValidationContext ctx) @@ -863,7 +863,7 @@ public static OMResponse disallowListTrashWithECReplicationConfig( @OMClientVersionValidator( processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.ListTrash, - maxVersion = ClientVersion.ERASURE_CODING_SUPPORT + applyUntil = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMResponse disallowListTrashWithBucketLayout( OMRequest req, OMResponse resp, ValidationContext ctx) @@ -1050,7 +1050,7 @@ private RefetchSecretKeyResponse refetchSecretKey() { @OMClientVersionValidator( processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.GetFileStatus, - maxVersion = ClientVersion.VERSION_HANDLES_UNKNOWN_DN_PORTS + applyUntil = ClientVersion.VERSION_HANDLES_UNKNOWN_DN_PORTS ) public static OMResponse disallowGetFileStatusWithECReplicationConfig( OMRequest req, OMResponse resp, ValidationContext ctx) @@ -1080,7 +1080,7 @@ public static OMResponse disallowGetFileStatusWithECReplicationConfig( @OMClientVersionValidator( processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.GetFileStatus, - maxVersion = ClientVersion.ERASURE_CODING_SUPPORT + applyUntil = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMResponse disallowGetFileStatusWithBucketLayout( OMRequest req, OMResponse resp, ValidationContext ctx) @@ -1129,7 +1129,7 @@ private LookupFileResponse lookupFile(LookupFileRequest request, @OMClientVersionValidator( processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.LookupFile, - maxVersion = ClientVersion.VERSION_HANDLES_UNKNOWN_DN_PORTS + applyUntil = ClientVersion.VERSION_HANDLES_UNKNOWN_DN_PORTS ) public static OMResponse disallowLookupFileWithECReplicationConfig( OMRequest req, OMResponse resp, ValidationContext ctx) @@ -1157,7 +1157,7 @@ public static OMResponse disallowLookupFileWithECReplicationConfig( @OMClientVersionValidator( processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.LookupFile, - maxVersion = ClientVersion.ERASURE_CODING_SUPPORT + applyUntil = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMResponse disallowLookupFileWithBucketLayout( OMRequest req, OMResponse resp, ValidationContext ctx) @@ -1239,7 +1239,7 @@ private ListStatusLightResponse listStatusLight( @OMClientVersionValidator( processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.ListStatus, - maxVersion = ClientVersion.VERSION_HANDLES_UNKNOWN_DN_PORTS + applyUntil = ClientVersion.VERSION_HANDLES_UNKNOWN_DN_PORTS ) public static OMResponse disallowListStatusResponseWithECReplicationConfig( OMRequest req, OMResponse resp, ValidationContext ctx) @@ -1267,7 +1267,7 @@ public static OMResponse disallowListStatusResponseWithECReplicationConfig( @OMClientVersionValidator( processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.ListStatus, - maxVersion = ClientVersion.ERASURE_CODING_SUPPORT + applyUntil = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMResponse disallowListStatusResponseWithBucketLayout( OMRequest req, OMResponse resp, ValidationContext ctx) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestOMValidatorProcessor.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestOMValidatorProcessor.java index 71c15f19d2be..7aeead51f5b6 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestOMValidatorProcessor.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestOMValidatorProcessor.java @@ -69,14 +69,14 @@ public class TestOMValidatorProcessor { .setParallel(true)).getTypesAnnotatedWith(RegisterValidator.class).stream().filter(annotationClass -> { try { return annotationClass.getMethod(RegisterValidator.REQUEST_TYPE_METHOD_NAME).getReturnType() - .equals(Type.class); + .equals(Type[].class); } catch (NoSuchMethodException e) { throw new RuntimeException(e); } }) .collect(Collectors.toMap(Function.identity(), annotationClass -> { try { - return annotationClass.getMethod(RegisterValidator.MAX_VERSION_METHOD_NAME).getReturnType(); + return annotationClass.getMethod(RegisterValidator.APPLY_UNTIL_METHOD_NAME).getReturnType(); } catch (NoSuchMethodException e) { throw new RuntimeException(e); } @@ -502,21 +502,21 @@ private & Version> String annotationOf(RequestProcessingPhase } private & Version> String annotationOf( - RequestProcessingPhase phase, Type reqType, Class annotationClass, V maxVersion) { - return annotationOf(phase.name(), reqType, annotationClass, maxVersion); + RequestProcessingPhase phase, Type reqType, Class annotationClass, V applyUntilVersion) { + return annotationOf(phase.name(), reqType, annotationClass, applyUntilVersion); } private & Version> String annotationOf( String phase, Type reqType, Class annotationClass, - V maxVersion) { + V applyUntilVersion) { StringBuilder annotation = new StringBuilder(); annotation.append("@" + annotationClass.getName() + "("); annotation.append("processingPhase = ").append(phase); annotation.append(", requestType = ").append(reqType.name()); - if (maxVersion != null) { - annotation.append(", maxVersion = ").append(maxVersion.name()); + if (applyUntilVersion != null) { + annotation.append(", applyUntil = ").append(applyUntilVersion.name()); } annotation.append(" )"); return annotation.toString(); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestRequestValidations.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestRequestValidations.java index 47fba0d83248..efc5e71b60ab 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestRequestValidations.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestRequestValidations.java @@ -192,7 +192,7 @@ public void testOldClientConditionIsRecognizedAndPreValidatorsApplied() validationListener.assertNumOfEvents(1); validationListener.assertExactListOfValidatorsCalled( - "oldClientPreProcessCreateKeyValidator"); + "preProcessCreateKeyBucketLayoutClientValidator"); } @Test @@ -206,8 +206,8 @@ public void testOldClientConditionIsRecognizedAndPostValidatorsApplied() validationListener.assertNumOfEvents(2); validationListener.assertExactListOfValidatorsCalled( - "oldClientPostProcessCreateKeyValidator", - "oldClientPostProcessCreateKeyValidator2"); + "postProcessCreateKeyBucketLayoutClientValidator", + "postProcessCreateKeyECReplicaIndexRequiredClientValidator"); } @Test @@ -220,8 +220,8 @@ public void testPreFinalizedWithOldClientConditionPreProcValidatorsApplied() validationListener.assertNumOfEvents(2); validationListener.assertExactListOfValidatorsCalled( - "preFinalizePreProcessCreateKeyValidator", - "oldClientPreProcessCreateKeyValidator"); + "preProcessCreateKeyQuotaLayoutValidator", + "preProcessCreateKeyBucketLayoutClientValidator"); } @Test @@ -235,9 +235,9 @@ public void testPreFinalizedWithOldClientConditionPostProcValidatorsApplied() validationListener.assertNumOfEvents(3); validationListener.assertExactListOfValidatorsCalled( - "preFinalizePostProcessCreateKeyValidator", - "oldClientPostProcessCreateKeyValidator", - "oldClientPostProcessCreateKeyValidator2"); + "postProcessCreateKeyQuotaLayoutValidator", + "postProcessCreateKeyBucketLayoutClientValidator", + "postProcessCreateKeyECReplicaIndexRequiredClientValidator"); } /** diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestValidatorRegistry.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestValidatorRegistry.java index cc191f446534..4e2df3d8f1d0 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestValidatorRegistry.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestValidatorRegistry.java @@ -99,7 +99,7 @@ public void testRegistryHasThePreFinalizePreProcessCreateKeyValidator() { OMLayoutFeature.FILESYSTEM_SNAPSHOT)); assertEquals(1, validators.size()); - String expectedMethodName = "preFinalizePreProcessCreateKeyValidator"; + String expectedMethodName = "preProcessCreateKeyQuotaLayoutValidator"; assertEquals(expectedMethodName, validators.get(0).getName()); } @@ -113,7 +113,7 @@ public void testRegistryHasThePreFinalizePostProcessCreateKeyValidator() { asList(ClientVersion.CURRENT, OMLayoutFeature.BUCKET_LAYOUT_SUPPORT)); assertEquals(1, validators.size()); - String expectedMethodName = "preFinalizePostProcessCreateKeyValidator"; + String expectedMethodName = "postProcessCreateKeyQuotaLayoutValidator"; assertEquals(expectedMethodName, validators.get(0).getName()); } @@ -129,8 +129,8 @@ public void testRegistryHasTheOldClientPreProcessCreateKeyValidator() { assertEquals(2, validators.size()); List methodNames = validators.stream().map(Method::getName).collect(Collectors.toList()); - assertThat(methodNames).contains("oldClientPreProcessCreateKeyValidator"); - assertThat(methodNames).contains("oldClientPreProcessCreateKeyValidator2"); + assertEquals(Arrays.asList("preProcessCreateKeyBucketLayoutClientValidator", + "preProcessCreateKeyBucketLayoutClientValidator"), methodNames); } @Test @@ -145,8 +145,8 @@ public void testRegistryHasTheOldClientPostProcessCreateKeyValidator() { assertEquals(2, validators.size()); List methodNames = validators.stream().map(Method::getName).collect(Collectors.toList()); - assertThat(methodNames).contains("oldClientPostProcessCreateKeyValidator"); - assertThat(methodNames).contains("oldClientPostProcessCreateKeyValidator2"); + assertThat(methodNames).contains("postProcessCreateKeyBucketLayoutClientValidator"); + assertThat(methodNames).contains("postProcessCreateKeyECReplicaIndexRequiredClientValidator"); } @Test @@ -162,7 +162,7 @@ public void testRegistryHasTheMultiPurposePreProcessCreateVolumeValidator() { assertEquals(1, preFinalizeValidators.size()); assertEquals(1, newClientValidators.size()); - String expectedMethodName = "multiPurposePreProcessCreateVolumeValidator"; + String expectedMethodName = "multiPurposePreProcessCreateVolumeBucketLayoutCLientQuotaLayoutValidator"; assertEquals(expectedMethodName, preFinalizeValidators.get(0).getName()); assertEquals(expectedMethodName, newClientValidators.get(0).getName()); } @@ -180,7 +180,7 @@ public void testRegistryHasTheMultiPurposePostProcessCreateVolumeValidator() { assertEquals(1, preFinalizeValidators.size()); assertEquals(1, oldClientValidators.size()); - String expectedMethodName = "multiPurposePostProcessCreateVolumeValidator"; + String expectedMethodName = "multiPurposePostProcessCreateVolumeBucketLayoutCLientQuotaLayoutValidator"; assertEquals(expectedMethodName, preFinalizeValidators.get(0).getName()); assertEquals(expectedMethodName, oldClientValidators.get(0).getName()); } @@ -197,9 +197,9 @@ public void testValidatorsAreReturnedForMultiCondition() { assertEquals(3, validators.size()); List methodNames = validators.stream().map(Method::getName).collect(Collectors.toList()); - assertThat(methodNames).contains("preFinalizePostProcessCreateKeyValidator"); - assertThat(methodNames).contains("oldClientPostProcessCreateKeyValidator"); - assertThat(methodNames).contains("oldClientPostProcessCreateKeyValidator2"); + assertThat(methodNames).contains("postProcessCreateKeyQuotaLayoutValidator"); + assertThat(methodNames).contains("postProcessCreateKeyBucketLayoutClientValidator"); + assertThat(methodNames).contains("postProcessCreateKeyECReplicaIndexRequiredClientValidator"); } @Test @@ -240,4 +240,19 @@ public void testNoDefinedValidationForRequestReturnsEmptyList() { assertTrue(registry.validationsFor(CreateDirectory, null, ClientVersion.ERASURE_CODING_SUPPORT).isEmpty()); } + @Test + public void testFutureVersionForRequestReturnsOnlyFutureVersionValidators() { + ValidatorRegistry registry = new ValidatorRegistry<>( + OzoneManagerProtocolProtos.Type.class, PACKAGE, + Arrays.stream(VersionExtractor.values()).map(VersionExtractor::getVersionClass).collect(Collectors.toSet()), + REQUEST_PROCESSING_PHASES); + + List validators = registry.validationsFor(CreateKey, PRE_PROCESS, ClientVersion.FUTURE_VERSION); + + assertEquals(1, validators.size()); + List methodNames = + validators.stream().map(Method::getName).collect(Collectors.toList()); + assertThat(methodNames).contains("preProcessCreateKeyFutureClientValidator"); + } + } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset1/GeneralValidatorsForTesting.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset1/GeneralValidatorsForTesting.java index 50c825ca6c13..c5d287a72076 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset1/GeneralValidatorsForTesting.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset1/GeneralValidatorsForTesting.java @@ -90,87 +90,97 @@ private static void fireValidationEvent(String calledMethodName) { } @OMLayoutVersionValidator( - maxVersion = OMLayoutFeature.QUOTA, + applyUntil = OMLayoutFeature.QUOTA, processingPhase = PRE_PROCESS, requestType = CreateKey) - public static OMRequest preFinalizePreProcessCreateKeyValidator( + public static OMRequest preProcessCreateKeyQuotaLayoutValidator( OMRequest req, ValidationContext ctx) { - fireValidationEvent("preFinalizePreProcessCreateKeyValidator"); + fireValidationEvent("preProcessCreateKeyQuotaLayoutValidator"); + return req; + } + + @OMClientVersionValidator( + applyUntil = ClientVersion.FUTURE_VERSION, + processingPhase = PRE_PROCESS, + requestType = CreateKey) + public static OMRequest preProcessCreateKeyFutureClientValidator( + OMRequest req, ValidationContext ctx) { + fireValidationEvent("preProcessCreateKeyFutureClientValidator"); return req; } @OMLayoutVersionValidator( - maxVersion = OMLayoutFeature.QUOTA, + applyUntil = OMLayoutFeature.QUOTA, processingPhase = POST_PROCESS, requestType = CreateKey) - public static OMResponse preFinalizePostProcessCreateKeyValidator( + public static OMResponse postProcessCreateKeyQuotaLayoutValidator( OMRequest req, OMResponse resp, ValidationContext ctx) { - fireValidationEvent("preFinalizePostProcessCreateKeyValidator"); + fireValidationEvent("postProcessCreateKeyQuotaLayoutValidator"); return resp; } @OMClientVersionValidator( processingPhase = PRE_PROCESS, requestType = CreateKey, - maxVersion = ClientVersion.BUCKET_LAYOUT_SUPPORT) - public static OMRequest oldClientPreProcessCreateKeyValidator( + applyUntil = ClientVersion.BUCKET_LAYOUT_SUPPORT) + public static OMRequest preProcessCreateKeyBucketLayoutClientValidator( OMRequest req, ValidationContext ctx) { - fireValidationEvent("oldClientPreProcessCreateKeyValidator"); + fireValidationEvent("preProcessCreateKeyBucketLayoutClientValidator"); return req; } @OMClientVersionValidator( processingPhase = POST_PROCESS, requestType = CreateKey, - maxVersion = ClientVersion.BUCKET_LAYOUT_SUPPORT) - public static OMResponse oldClientPostProcessCreateKeyValidator( + applyUntil = ClientVersion.BUCKET_LAYOUT_SUPPORT) + public static OMResponse postProcessCreateKeyBucketLayoutClientValidator( OMRequest req, OMResponse resp, ValidationContext ctx) { - fireValidationEvent("oldClientPostProcessCreateKeyValidator"); + fireValidationEvent("postProcessCreateKeyBucketLayoutClientValidator"); return resp; } @OMClientVersionValidator( processingPhase = PRE_PROCESS, requestType = CreateVolume, - maxVersion = ClientVersion.BUCKET_LAYOUT_SUPPORT) + applyUntil = ClientVersion.BUCKET_LAYOUT_SUPPORT) @OMLayoutVersionValidator(processingPhase = PRE_PROCESS, requestType = CreateVolume, - maxVersion = OMLayoutFeature.QUOTA) - public static OMRequest multiPurposePreProcessCreateVolumeValidator( + applyUntil = OMLayoutFeature.QUOTA) + public static OMRequest multiPurposePreProcessCreateVolumeBucketLayoutCLientQuotaLayoutValidator( OMRequest req, ValidationContext ctx) { - fireValidationEvent("multiPurposePreProcessCreateVolumeValidator"); + fireValidationEvent("multiPurposePreProcessCreateVolumeBucketLayoutCLientQuotaLayoutValidator"); return req; } @OMClientVersionValidator( processingPhase = POST_PROCESS, requestType = CreateVolume, - maxVersion = ClientVersion.BUCKET_LAYOUT_SUPPORT) + applyUntil = ClientVersion.BUCKET_LAYOUT_SUPPORT) @OMLayoutVersionValidator(processingPhase = POST_PROCESS, requestType = CreateVolume, - maxVersion = OMLayoutFeature.QUOTA) - public static OMResponse multiPurposePostProcessCreateVolumeValidator( + applyUntil = OMLayoutFeature.QUOTA) + public static OMResponse multiPurposePostProcessCreateVolumeBucketLayoutCLientQuotaLayoutValidator( OMRequest req, OMResponse resp, ValidationContext ctx) { - fireValidationEvent("multiPurposePostProcessCreateVolumeValidator"); + fireValidationEvent("multiPurposePostProcessCreateVolumeBucketLayoutCLientQuotaLayoutValidator"); return resp; } @OMClientVersionValidator( processingPhase = POST_PROCESS, requestType = CreateKey, - maxVersion = ClientVersion.EC_REPLICA_INDEX_REQUIRED_IN_BLOCK_REQUEST) - public static OMResponse oldClientPostProcessCreateKeyValidator2( + applyUntil = ClientVersion.EC_REPLICA_INDEX_REQUIRED_IN_BLOCK_REQUEST) + public static OMResponse postProcessCreateKeyECReplicaIndexRequiredClientValidator( OMRequest req, OMResponse resp, ValidationContext ctx) { - fireValidationEvent("oldClientPostProcessCreateKeyValidator2"); + fireValidationEvent("postProcessCreateKeyECReplicaIndexRequiredClientValidator"); return resp; } @OMClientVersionValidator( processingPhase = PRE_PROCESS, requestType = DeleteKeys, - maxVersion = ClientVersion.BUCKET_LAYOUT_SUPPORT + applyUntil = ClientVersion.BUCKET_LAYOUT_SUPPORT ) - public static OMRequest throwingPreProcessValidator( + public static OMRequest throwingPreProcessDeleteKeyBucketLayoutClientValidator( OMRequest req, ValidationContext ctx) throws IOException { fireValidationEvent("throwingPreProcessValidator"); if (validatorTestsRunning) { @@ -182,9 +192,9 @@ public static OMRequest throwingPreProcessValidator( @OMClientVersionValidator( processingPhase = POST_PROCESS, requestType = DeleteKeys, - maxVersion = ClientVersion.BUCKET_LAYOUT_SUPPORT + applyUntil = ClientVersion.BUCKET_LAYOUT_SUPPORT ) - public static OMResponse throwingPostProcessValidator( + public static OMResponse throwingPostProcessDeleteKeyBucketLayoutClientValidator( OMRequest req, OMResponse resp, ValidationContext ctx) throws IOException { fireValidationEvent("throwingPostProcessValidator"); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset2/ValidatorsForOnlyOldClientValidations.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset2/ValidatorsForOnlyOldClientValidations.java index 3b8c1a870853..4bd966772980 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset2/ValidatorsForOnlyOldClientValidations.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset2/ValidatorsForOnlyOldClientValidations.java @@ -35,8 +35,8 @@ private ValidatorsForOnlyOldClientValidations() { } @OMClientVersionValidator( processingPhase = PRE_PROCESS, requestType = CreateKey, - maxVersion = ClientVersion.BUCKET_LAYOUT_SUPPORT) - public static OMRequest oldClientPreProcessCreateKeyValidator2( + applyUntil = ClientVersion.BUCKET_LAYOUT_SUPPORT) + public static OMRequest preProcessCreateKeyBucketLayoutClientValidator( OMRequest req, ValidationContext ctx) { return req; } From 66027c2222cfceb5ed790a9f35bc0ae2b0f26ab6 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Tue, 17 Dec 2024 14:48:02 -0800 Subject: [PATCH 11/28] HDDS-11149. Change method name Change-Id: I2d19e9052abe0702f6fe391d32737a53601852b1 --- .../RegisterValidatorProcessor.java | 12 +++++----- .../request/validation/RegisterValidator.java | 4 ++-- .../request/validation/ValidatorRegistry.java | 20 ++++++++-------- .../request/validation/package-info.java | 2 +- .../request/bucket/OMBucketCreateRequest.java | 6 ++--- .../request/bucket/OMBucketDeleteRequest.java | 2 +- .../bucket/OMBucketSetPropertyRequest.java | 2 +- .../file/OMDirectoryCreateRequest.java | 4 ++-- .../om/request/file/OMFileCreateRequest.java | 4 ++-- .../request/key/OMAllocateBlockRequest.java | 4 ++-- .../om/request/key/OMKeyCommitRequest.java | 6 ++--- .../om/request/key/OMKeyCreateRequest.java | 4 ++-- .../om/request/key/OMKeyDeleteRequest.java | 2 +- .../om/request/key/OMKeyRenameRequest.java | 2 +- .../om/request/key/OMKeysDeleteRequest.java | 2 +- .../om/request/key/OMKeysRenameRequest.java | 2 +- .../request/key/acl/OMKeyAddAclRequest.java | 2 +- .../key/acl/OMKeyRemoveAclRequest.java | 2 +- .../request/key/acl/OMKeySetAclRequest.java | 2 +- .../S3InitiateMultipartUploadRequest.java | 4 ++-- .../S3MultipartUploadAbortRequest.java | 4 ++-- .../S3MultipartUploadCommitPartRequest.java | 4 ++-- .../S3MultipartUploadCompleteRequest.java | 4 ++-- .../validation/OMClientVersionValidator.java | 2 +- .../validation/OMLayoutVersionValidator.java | 2 +- .../OzoneManagerRequestHandler.java | 24 +++++++++---------- .../validation/TestOMValidatorProcessor.java | 12 +++++----- .../GeneralValidatorsForTesting.java | 24 +++++++++---------- ...ValidatorsForOnlyOldClientValidations.java | 2 +- 29 files changed, 83 insertions(+), 83 deletions(-) diff --git a/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/RegisterValidatorProcessor.java b/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/RegisterValidatorProcessor.java index ccc83438a133..8089c330e4a1 100644 --- a/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/RegisterValidatorProcessor.java +++ b/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/RegisterValidatorProcessor.java @@ -52,11 +52,11 @@ public class RegisterValidatorProcessor extends AbstractProcessor { public static final String VERSION_CLASS_NAME = "org.apache.hadoop.ozone.Version"; public static final String REQUEST_PROCESSING_PHASE_CLASS_NAME = "org.apache.hadoop.ozone.om.request.validation" + ".RequestProcessingPhase"; - public static final String APPLY_UNTIL_METHOD_NAME = "applyUntil"; + public static final String APPLY_BEFORE_METHOD_NAME = "applyBefore"; public static final String REQUEST_TYPE_METHOD_NAME = "requestType"; public static final String PROCESSING_PHASE_METHOD_NAME = "processingPhase"; - public static final String MAX_VERSION_NOT_FOUND_ERROR_MESSAGE = "Method " + APPLY_UNTIL_METHOD_NAME + + public static final String MAX_VERSION_NOT_FOUND_ERROR_MESSAGE = "Method " + APPLY_BEFORE_METHOD_NAME + " returning an enum implementing " + VERSION_CLASS_NAME + " not found"; public static final String REQUEST_TYPE_NOT_FOUND_ERROR_MESSAGE = "Method " + REQUEST_TYPE_METHOD_NAME + " returning an enum not found"; @@ -104,22 +104,22 @@ private boolean validateMethod(ExecutableElement method, String expectedMethodNa private void processElements(Set annotatedElements) { for (Element element : annotatedElements) { if (element.getKind().equals(ElementKind.ANNOTATION_TYPE)) { - boolean hasApplyUntilMethod = false; + boolean hasApplyBeforeMethod = false; boolean hasRequestType = false; boolean hasRequestProcessPhase = false; for (Element enclosedElement : element.getEnclosedElements()) { // Check if the annotation has a method called "validatorName" returning a String if (enclosedElement instanceof ExecutableElement) { ExecutableElement method = (ExecutableElement) enclosedElement; - hasApplyUntilMethod = hasApplyUntilMethod || validateMethod(method, APPLY_UNTIL_METHOD_NAME, ElementKind.ENUM, - VERSION_CLASS_NAME); + hasApplyBeforeMethod = hasApplyBeforeMethod || validateMethod(method, APPLY_BEFORE_METHOD_NAME, + ElementKind.ENUM, VERSION_CLASS_NAME); hasRequestType = hasRequestType || validateArrayMethod(method, REQUEST_TYPE_METHOD_NAME, ElementKind.ENUM, null); hasRequestProcessPhase = hasRequestProcessPhase || validateMethod(method, PROCESSING_PHASE_METHOD_NAME, ElementKind.ENUM, REQUEST_PROCESSING_PHASE_CLASS_NAME); } } - if (!hasApplyUntilMethod) { + if (!hasApplyBeforeMethod) { emitErrorMsg(MAX_VERSION_NOT_FOUND_ERROR_MESSAGE + " for " + element.getSimpleName().toString()); } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/RegisterValidator.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/RegisterValidator.java index eb5457ab5734..1df192ac2e86 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/RegisterValidator.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/RegisterValidator.java @@ -28,14 +28,14 @@ /** * Annotations to register a validator. {@link org.apache.ozone.annotations.RegisterValidatorProcessor} * enforces other annotation to have the following methods: - * applyUntil : Returns an enum which implement {@link Version} + * applyBefore : Returns an enum which implement {@link Version} * requestType: Returns an Enum value. * processingPhase: Returns {@link RequestProcessingPhase} */ @Retention(RetentionPolicy.RUNTIME) @Target(ElementType.ANNOTATION_TYPE) public @interface RegisterValidator { - String APPLY_UNTIL_METHOD_NAME = "applyUntil"; + String APPLY_BEFORE_METHOD_NAME = "applyBefore"; String REQUEST_TYPE_METHOD_NAME = "requestType"; String PROCESSING_PHASE_METHOD_NAME = "processingPhase"; } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/ValidatorRegistry.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/ValidatorRegistry.java index e9e974f31a37..c79dd20f4fde 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/ValidatorRegistry.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/ValidatorRegistry.java @@ -99,7 +99,7 @@ public ValidatorRegistry(Class requestType, .equals(requestArrayClass)) .filter(annotationClass -> allowedVersionTypes.contains(getReturnTypeOfAnnotationMethod( (Class) annotationClass, - RegisterValidator.APPLY_UNTIL_METHOD_NAME))) + RegisterValidator.APPLY_BEFORE_METHOD_NAME))) .map(annotationClass -> (Class) annotationClass) .collect(Collectors.toSet()); this.indexedValidatorMap = allowedVersionTypes.stream().collect(ImmutableMap.toImmutableMap(Function.identity(), @@ -170,8 +170,8 @@ private ReturnValue callAnnotationMe } } - private Version getApplyUntilVersion(Validator validator) { - return callAnnotationMethod(validator, RegisterValidator.APPLY_UNTIL_METHOD_NAME, Version.class); + private Version getApplyBeforeVersion(Validator validator) { + return callAnnotationMethod(validator, RegisterValidator.APPLY_BEFORE_METHOD_NAME, Version.class); } private RequestProcessingPhase getRequestPhase(Validator validator) { @@ -216,18 +216,18 @@ private void registerValidator(Class requestType, Reflections reflections) { Collection methods = reflections.getMethodsAnnotatedWith(validatorToBeRegistered); Class versionClass = (Class) - this.getReturnTypeOfAnnotationMethod(validatorToBeRegistered, RegisterValidator.APPLY_UNTIL_METHOD_NAME); - List> sortedMethodsByApplyUntilVersion = methods.stream() + this.getReturnTypeOfAnnotationMethod(validatorToBeRegistered, RegisterValidator.APPLY_BEFORE_METHOD_NAME); + List> sortedMethodsByApplyBeforeVersion = methods.stream() .map(method -> Pair.of(method.getAnnotation(validatorToBeRegistered), method)) .sorted((validatorMethodPair1, validatorMethodPair2) -> Integer.compare( - this.getApplyUntilVersion(validatorMethodPair1.getKey()).version(), - this.getApplyUntilVersion(validatorMethodPair2.getKey()).version())) + this.getApplyBeforeVersion(validatorMethodPair1.getKey()).version(), + this.getApplyBeforeVersion(validatorMethodPair2.getKey()).version())) .collect(Collectors.toList()); - for (Pair validatorMethodPair : sortedMethodsByApplyUntilVersion) { + for (Pair validatorMethodPair : sortedMethodsByApplyBeforeVersion) { Annotation validator = validatorMethodPair.getKey(); Method method = validatorMethodPair.getValue(); - Version applyUntilVersion = this.getApplyUntilVersion(validator); + Version applyBeforeVersion = this.getApplyBeforeVersion(validator); RequestProcessingPhase phase = this.getRequestPhase(validator); checkAllowedAnnotationValues(allowedPhases, phase, RegisterValidator.PROCESSING_PHASE_METHOD_NAME, method.getName()); @@ -238,7 +238,7 @@ private void registerValidator(Class requestType, this.indexedValidatorMap.get(versionClass); EnumMap> phaseMap = this.getAndInitialize(type, () -> new EnumMap<>(RequestProcessingPhase.class), requestMap); - this.getAndInitialize(phase, IndexedItems::new, phaseMap).add(method, applyUntilVersion.version()); + this.getAndInitialize(phase, IndexedItems::new, phaseMap).add(method, applyBeforeVersion.version()); } } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/package-info.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/package-info.java index cf7ad12d5471..2666db0b6066 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/package-info.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/package-info.java @@ -23,7 +23,7 @@ * code for any server. * {@link org.apache.hadoop.ozone.request.validation.RegisterValidator} * is used to register any validator which has the following methods: - * - applyUntil : Returns an enum which implement {@link org.apache.hadoop.ozone.Version} + * - applyBefore : Returns an enum which implement {@link org.apache.hadoop.ozone.Version} * - requestType: Returns an Enum value. * - processingPhase: Returns {@link org.apache.hadoop.ozone.request.validation.RequestProcessingPhase} * diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java index a1cb80d5c16f..0b7dc4968d62 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java @@ -398,7 +398,7 @@ public boolean checkQuotaBytesValid(OMMetadataManager metadataManager, @OMLayoutVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CreateBucket, - applyUntil = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT + applyBefore = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT ) public static OMRequest disallowCreateBucketWithECReplicationConfig( OMRequest req, ValidationContext ctx) throws OMException { @@ -421,7 +421,7 @@ public static OMRequest disallowCreateBucketWithECReplicationConfig( @OMLayoutVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CreateBucket, - applyUntil = OMLayoutFeature.BUCKET_LAYOUT_SUPPORT + applyBefore = OMLayoutFeature.BUCKET_LAYOUT_SUPPORT ) public static OMRequest handleCreateBucketWithBucketLayoutDuringPreFinalize( OMRequest req, ValidationContext ctx) throws OMException { @@ -462,7 +462,7 @@ public static OMRequest handleCreateBucketWithBucketLayoutDuringPreFinalize( @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CreateBucket, - applyUntil = ClientVersion.ERASURE_CODING_SUPPORT + applyBefore = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest setDefaultBucketLayoutForOlderClients(OMRequest req, ValidationContext ctx) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketDeleteRequest.java index 10ba3abc2808..6246bb9b5f23 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketDeleteRequest.java @@ -283,7 +283,7 @@ private boolean bucketContainsSnapshotInCache( @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.DeleteBucket, - applyUntil = ClientVersion.ERASURE_CODING_SUPPORT + applyBefore = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockBucketDeleteWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketSetPropertyRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketSetPropertyRequest.java index 7336b4a0158a..5211b1762cda 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketSetPropertyRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketSetPropertyRequest.java @@ -359,7 +359,7 @@ public boolean checkQuotaNamespaceValid(OmVolumeArgs omVolumeArgs, @OMLayoutVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.SetBucketProperty, - applyUntil = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT + applyBefore = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT ) public static OMRequest disallowSetBucketPropertyWithECReplicationConfig( OMRequest req, ValidationContext ctx) throws OMException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java index 47aa53ee5cf7..f79e80afcbe7 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java @@ -402,7 +402,7 @@ static long getMaxNumOfRecursiveDirs() { @OMLayoutVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CreateDirectory, - applyUntil = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT + applyBefore = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT ) public static OMRequest disallowCreateDirectoryWithECReplicationConfig( OMRequest req, ValidationContext ctx) throws OMException { @@ -435,7 +435,7 @@ public static OMRequest disallowCreateDirectoryWithECReplicationConfig( @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CreateDirectory, - applyUntil = ClientVersion.ERASURE_CODING_SUPPORT + applyBefore = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockCreateDirectoryWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java index aeaeb4da40c9..9669ede2a23b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java @@ -396,7 +396,7 @@ protected void checkAllParentsExist(KeyArgs keyArgs, @OMLayoutVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = CreateFile, - applyUntil = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT + applyBefore = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT ) public static OMRequest disallowCreateFileWithECReplicationConfig( OMRequest req, ValidationContext ctx) throws OMException { @@ -428,7 +428,7 @@ public static OMRequest disallowCreateFileWithECReplicationConfig( @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CreateFile, - applyUntil = ClientVersion.ERASURE_CODING_SUPPORT + applyBefore = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockCreateFileWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java index 2cd958f0fafd..c15e58bd9d95 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java @@ -271,7 +271,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn @OMLayoutVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.AllocateBlock, - applyUntil = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT + applyBefore = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT ) public static OMRequest disallowAllocateBlockWithECReplicationConfig( OMRequest req, ValidationContext ctx) throws OMException { @@ -301,7 +301,7 @@ public static OMRequest disallowAllocateBlockWithECReplicationConfig( @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.AllocateBlock, - applyUntil = ClientVersion.ERASURE_CODING_SUPPORT + applyBefore = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockAllocateBlockWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java index d7104c35b0a5..231a30eadee3 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java @@ -436,7 +436,7 @@ protected void processResult(CommitKeyRequest commitKeyRequest, @OMLayoutVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CommitKey, - applyUntil = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT + applyBefore = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT ) public static OMRequest disallowCommitKeyWithECReplicationConfig( OMRequest req, ValidationContext ctx) throws OMException { @@ -466,7 +466,7 @@ public static OMRequest disallowCommitKeyWithECReplicationConfig( @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CommitKey, - applyUntil = ClientVersion.ERASURE_CODING_SUPPORT + applyBefore = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockCommitKeyWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { @@ -485,7 +485,7 @@ public static OMRequest blockCommitKeyWithBucketLayoutFromOldClient( @OMLayoutVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CommitKey, - applyUntil = OMLayoutFeature.HSYNC + applyBefore = OMLayoutFeature.HSYNC ) public static OMRequest disallowHsync( OMRequest req, ValidationContext ctx) throws OMException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java index 4de22ba819d8..3dbcd4f9c6c2 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java @@ -397,7 +397,7 @@ protected void logResult(CreateKeyRequest createKeyRequest, @OMLayoutVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CreateKey, - applyUntil = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT + applyBefore = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT ) public static OMRequest disallowCreateKeyWithECReplicationConfig( OMRequest req, ValidationContext ctx) throws OMException { @@ -427,7 +427,7 @@ public static OMRequest disallowCreateKeyWithECReplicationConfig( @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CreateKey, - applyUntil = ClientVersion.ERASURE_CODING_SUPPORT + applyBefore = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockCreateKeyWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java index 8f66296c9b13..ae259aee3e09 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java @@ -242,7 +242,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.DeleteKey, - applyUntil = ClientVersion.ERASURE_CODING_SUPPORT + applyBefore = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockDeleteKeyWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java index 0c463392e429..13065e882c9a 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java @@ -272,7 +272,7 @@ private Map buildAuditMap( @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.RenameKey, - applyUntil = ClientVersion.ERASURE_CODING_SUPPORT + applyBefore = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockRenameKeyWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java index d98143713ed8..fb3d72255633 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java @@ -363,7 +363,7 @@ protected static void addDeletedKeys(Map auditMap, @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.DeleteKeys, - applyUntil = ClientVersion.ERASURE_CODING_SUPPORT + applyBefore = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockDeleteKeysWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysRenameRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysRenameRequest.java index 2d56dac212dd..bf7c5505e4aa 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysRenameRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysRenameRequest.java @@ -299,7 +299,7 @@ private Map buildAuditMap(Map auditMap, @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.RenameKeys, - applyUntil = ClientVersion.ERASURE_CODING_SUPPORT + applyBefore = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockRenameKeysWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAddAclRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAddAclRequest.java index 725f83ba59d1..4789c48cabc9 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAddAclRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAddAclRequest.java @@ -167,7 +167,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.AddAcl, - applyUntil = ClientVersion.ERASURE_CODING_SUPPORT + applyBefore = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockAddAclWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyRemoveAclRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyRemoveAclRequest.java index 212a70422b15..e14d03169a8b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyRemoveAclRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyRemoveAclRequest.java @@ -168,7 +168,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.RemoveAcl, - applyUntil = ClientVersion.ERASURE_CODING_SUPPORT + applyBefore = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockRemoveAclWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeySetAclRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeySetAclRequest.java index f01fb8561d28..a76bdcfb5ff4 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeySetAclRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeySetAclRequest.java @@ -164,7 +164,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.SetAcl, - applyUntil = ClientVersion.BUCKET_LAYOUT_SUPPORT + applyBefore = ClientVersion.BUCKET_LAYOUT_SUPPORT ) public static OMRequest blockSetAclWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java index 620e6e076753..d24694c10767 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java @@ -290,7 +290,7 @@ protected void logResult(OzoneManager ozoneManager, @OMLayoutVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.InitiateMultiPartUpload, - applyUntil = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT + applyBefore = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT ) public static OMRequest disallowInitiateMultiPartUploadWithECReplicationConfig( @@ -323,7 +323,7 @@ protected void logResult(OzoneManager ozoneManager, @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.InitiateMultiPartUpload, - applyUntil = ClientVersion.ERASURE_CODING_SUPPORT + applyBefore = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockInitiateMPUWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java index 2bb909884fb2..27c032d49bc9 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java @@ -261,7 +261,7 @@ protected String getMultipartOpenKey(String multipartUploadID, @OMLayoutVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.AbortMultiPartUpload, - applyUntil = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT + applyBefore = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT ) public static OMRequest disallowAbortMultiPartUploadWithECReplicationConfig( OMRequest req, ValidationContext ctx) throws OMException { @@ -292,7 +292,7 @@ public static OMRequest disallowAbortMultiPartUploadWithECReplicationConfig( @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.AbortMultiPartUpload, - applyUntil = ClientVersion.ERASURE_CODING_SUPPORT + applyBefore = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockMPUAbortWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java index 79581b62b963..6696839eeb5f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java @@ -352,7 +352,7 @@ private String getMultipartKey(String volumeName, String bucketName, @OMLayoutVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CommitMultiPartUpload, - applyUntil = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT + applyBefore = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT ) public static OMRequest disallowCommitMultiPartUploadWithECReplicationConfig( OMRequest req, ValidationContext ctx) throws OMException { @@ -383,7 +383,7 @@ public static OMRequest disallowCommitMultiPartUploadWithECReplicationConfig( @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CommitMultiPartUpload, - applyUntil = ClientVersion.ERASURE_CODING_SUPPORT + applyBefore = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockMPUCommitWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java index 11b217805de8..d11262112485 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java @@ -713,7 +713,7 @@ private void updateCache(OMMetadataManager omMetadataManager, @OMLayoutVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CompleteMultiPartUpload, - applyUntil = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT + applyBefore = OMLayoutFeature.ERASURE_CODED_STORAGE_SUPPORT ) public static OMRequest disallowCompleteMultiPartUploadWithECReplicationConfig( @@ -745,7 +745,7 @@ private void updateCache(OMMetadataManager omMetadataManager, @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CompleteMultiPartUpload, - applyUntil = ClientVersion.ERASURE_CODING_SUPPORT + applyBefore = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMRequest blockMPUCompleteWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/OMClientVersionValidator.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/OMClientVersionValidator.java index 26f27b8803cd..2ec6c70401d1 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/OMClientVersionValidator.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/OMClientVersionValidator.java @@ -97,6 +97,6 @@ * where the version is older than the excluding of the specified version. * @returns the max client version until which the validator runs excluding the specified version itself. */ - ClientVersion applyUntil(); + ClientVersion applyBefore(); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/OMLayoutVersionValidator.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/OMLayoutVersionValidator.java index 1ad1df607045..c3396e633895 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/OMLayoutVersionValidator.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/OMLayoutVersionValidator.java @@ -101,6 +101,6 @@ * where the version is older than the excluding of the specified version. * @returns the max layout version until which the validator runs excluding the specified version itself. */ - OMLayoutFeature applyUntil(); + OMLayoutFeature applyBefore(); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java index 6eaf2697579c..cd577363c880 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java @@ -631,7 +631,7 @@ private GetKeyInfoResponse getKeyInfo(GetKeyInfoRequest request, @OMClientVersionValidator( processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.LookupKey, - applyUntil = ClientVersion.VERSION_HANDLES_UNKNOWN_DN_PORTS + applyBefore = ClientVersion.VERSION_HANDLES_UNKNOWN_DN_PORTS ) public static OMResponse disallowLookupKeyResponseWithECReplicationConfig( OMRequest req, OMResponse resp, ValidationContext ctx) @@ -658,7 +658,7 @@ public static OMResponse disallowLookupKeyResponseWithECReplicationConfig( @OMClientVersionValidator( processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.LookupKey, - applyUntil = ClientVersion.ERASURE_CODING_SUPPORT + applyBefore = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMResponse disallowLookupKeyWithBucketLayout( OMRequest req, OMResponse resp, ValidationContext ctx) @@ -745,7 +745,7 @@ private ListKeysLightResponse listKeysLight(ListKeysRequest request) @OMClientVersionValidator( processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.ListKeys, - applyUntil = ClientVersion.VERSION_HANDLES_UNKNOWN_DN_PORTS + applyBefore = ClientVersion.VERSION_HANDLES_UNKNOWN_DN_PORTS ) public static OMResponse disallowListKeysResponseWithECReplicationConfig( OMRequest req, OMResponse resp, ValidationContext ctx) @@ -772,7 +772,7 @@ public static OMResponse disallowListKeysResponseWithECReplicationConfig( @OMClientVersionValidator( processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.ListKeys, - applyUntil = ClientVersion.ERASURE_CODING_SUPPORT + applyBefore = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMResponse disallowListKeysWithBucketLayout( OMRequest req, OMResponse resp, ValidationContext ctx) @@ -833,7 +833,7 @@ private ListTrashResponse listTrash(ListTrashRequest request, @OMClientVersionValidator( processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.ListTrash, - applyUntil = ClientVersion.VERSION_HANDLES_UNKNOWN_DN_PORTS + applyBefore = ClientVersion.VERSION_HANDLES_UNKNOWN_DN_PORTS ) public static OMResponse disallowListTrashWithECReplicationConfig( OMRequest req, OMResponse resp, ValidationContext ctx) @@ -863,7 +863,7 @@ public static OMResponse disallowListTrashWithECReplicationConfig( @OMClientVersionValidator( processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.ListTrash, - applyUntil = ClientVersion.ERASURE_CODING_SUPPORT + applyBefore = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMResponse disallowListTrashWithBucketLayout( OMRequest req, OMResponse resp, ValidationContext ctx) @@ -1050,7 +1050,7 @@ private RefetchSecretKeyResponse refetchSecretKey() { @OMClientVersionValidator( processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.GetFileStatus, - applyUntil = ClientVersion.VERSION_HANDLES_UNKNOWN_DN_PORTS + applyBefore = ClientVersion.VERSION_HANDLES_UNKNOWN_DN_PORTS ) public static OMResponse disallowGetFileStatusWithECReplicationConfig( OMRequest req, OMResponse resp, ValidationContext ctx) @@ -1080,7 +1080,7 @@ public static OMResponse disallowGetFileStatusWithECReplicationConfig( @OMClientVersionValidator( processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.GetFileStatus, - applyUntil = ClientVersion.ERASURE_CODING_SUPPORT + applyBefore = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMResponse disallowGetFileStatusWithBucketLayout( OMRequest req, OMResponse resp, ValidationContext ctx) @@ -1129,7 +1129,7 @@ private LookupFileResponse lookupFile(LookupFileRequest request, @OMClientVersionValidator( processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.LookupFile, - applyUntil = ClientVersion.VERSION_HANDLES_UNKNOWN_DN_PORTS + applyBefore = ClientVersion.VERSION_HANDLES_UNKNOWN_DN_PORTS ) public static OMResponse disallowLookupFileWithECReplicationConfig( OMRequest req, OMResponse resp, ValidationContext ctx) @@ -1157,7 +1157,7 @@ public static OMResponse disallowLookupFileWithECReplicationConfig( @OMClientVersionValidator( processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.LookupFile, - applyUntil = ClientVersion.ERASURE_CODING_SUPPORT + applyBefore = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMResponse disallowLookupFileWithBucketLayout( OMRequest req, OMResponse resp, ValidationContext ctx) @@ -1239,7 +1239,7 @@ private ListStatusLightResponse listStatusLight( @OMClientVersionValidator( processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.ListStatus, - applyUntil = ClientVersion.VERSION_HANDLES_UNKNOWN_DN_PORTS + applyBefore = ClientVersion.VERSION_HANDLES_UNKNOWN_DN_PORTS ) public static OMResponse disallowListStatusResponseWithECReplicationConfig( OMRequest req, OMResponse resp, ValidationContext ctx) @@ -1267,7 +1267,7 @@ public static OMResponse disallowListStatusResponseWithECReplicationConfig( @OMClientVersionValidator( processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.ListStatus, - applyUntil = ClientVersion.ERASURE_CODING_SUPPORT + applyBefore = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMResponse disallowListStatusResponseWithBucketLayout( OMRequest req, OMResponse resp, ValidationContext ctx) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestOMValidatorProcessor.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestOMValidatorProcessor.java index 7aeead51f5b6..2f4dc1d5bada 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestOMValidatorProcessor.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestOMValidatorProcessor.java @@ -76,7 +76,7 @@ public class TestOMValidatorProcessor { }) .collect(Collectors.toMap(Function.identity(), annotationClass -> { try { - return annotationClass.getMethod(RegisterValidator.APPLY_UNTIL_METHOD_NAME).getReturnType(); + return annotationClass.getMethod(RegisterValidator.APPLY_BEFORE_METHOD_NAME).getReturnType(); } catch (NoSuchMethodException e) { throw new RuntimeException(e); } @@ -502,21 +502,21 @@ private & Version> String annotationOf(RequestProcessingPhase } private & Version> String annotationOf( - RequestProcessingPhase phase, Type reqType, Class annotationClass, V applyUntilVersion) { - return annotationOf(phase.name(), reqType, annotationClass, applyUntilVersion); + RequestProcessingPhase phase, Type reqType, Class annotationClass, V applyBeforeVersion) { + return annotationOf(phase.name(), reqType, annotationClass, applyBeforeVersion); } private & Version> String annotationOf( String phase, Type reqType, Class annotationClass, - V applyUntilVersion) { + V applyBeforeVersion) { StringBuilder annotation = new StringBuilder(); annotation.append("@" + annotationClass.getName() + "("); annotation.append("processingPhase = ").append(phase); annotation.append(", requestType = ").append(reqType.name()); - if (applyUntilVersion != null) { - annotation.append(", applyUntil = ").append(applyUntilVersion.name()); + if (applyBeforeVersion != null) { + annotation.append(", applyBefore = ").append(applyBeforeVersion.name()); } annotation.append(" )"); return annotation.toString(); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset1/GeneralValidatorsForTesting.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset1/GeneralValidatorsForTesting.java index c5d287a72076..bc60dbf21b29 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset1/GeneralValidatorsForTesting.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset1/GeneralValidatorsForTesting.java @@ -90,7 +90,7 @@ private static void fireValidationEvent(String calledMethodName) { } @OMLayoutVersionValidator( - applyUntil = OMLayoutFeature.QUOTA, + applyBefore = OMLayoutFeature.QUOTA, processingPhase = PRE_PROCESS, requestType = CreateKey) public static OMRequest preProcessCreateKeyQuotaLayoutValidator( @@ -100,7 +100,7 @@ public static OMRequest preProcessCreateKeyQuotaLayoutValidator( } @OMClientVersionValidator( - applyUntil = ClientVersion.FUTURE_VERSION, + applyBefore = ClientVersion.FUTURE_VERSION, processingPhase = PRE_PROCESS, requestType = CreateKey) public static OMRequest preProcessCreateKeyFutureClientValidator( @@ -110,7 +110,7 @@ public static OMRequest preProcessCreateKeyFutureClientValidator( } @OMLayoutVersionValidator( - applyUntil = OMLayoutFeature.QUOTA, + applyBefore = OMLayoutFeature.QUOTA, processingPhase = POST_PROCESS, requestType = CreateKey) public static OMResponse postProcessCreateKeyQuotaLayoutValidator( @@ -122,7 +122,7 @@ public static OMResponse postProcessCreateKeyQuotaLayoutValidator( @OMClientVersionValidator( processingPhase = PRE_PROCESS, requestType = CreateKey, - applyUntil = ClientVersion.BUCKET_LAYOUT_SUPPORT) + applyBefore = ClientVersion.BUCKET_LAYOUT_SUPPORT) public static OMRequest preProcessCreateKeyBucketLayoutClientValidator( OMRequest req, ValidationContext ctx) { fireValidationEvent("preProcessCreateKeyBucketLayoutClientValidator"); @@ -132,7 +132,7 @@ public static OMRequest preProcessCreateKeyBucketLayoutClientValidator( @OMClientVersionValidator( processingPhase = POST_PROCESS, requestType = CreateKey, - applyUntil = ClientVersion.BUCKET_LAYOUT_SUPPORT) + applyBefore = ClientVersion.BUCKET_LAYOUT_SUPPORT) public static OMResponse postProcessCreateKeyBucketLayoutClientValidator( OMRequest req, OMResponse resp, ValidationContext ctx) { fireValidationEvent("postProcessCreateKeyBucketLayoutClientValidator"); @@ -142,10 +142,10 @@ public static OMResponse postProcessCreateKeyBucketLayoutClientValidator( @OMClientVersionValidator( processingPhase = PRE_PROCESS, requestType = CreateVolume, - applyUntil = ClientVersion.BUCKET_LAYOUT_SUPPORT) + applyBefore = ClientVersion.BUCKET_LAYOUT_SUPPORT) @OMLayoutVersionValidator(processingPhase = PRE_PROCESS, requestType = CreateVolume, - applyUntil = OMLayoutFeature.QUOTA) + applyBefore = OMLayoutFeature.QUOTA) public static OMRequest multiPurposePreProcessCreateVolumeBucketLayoutCLientQuotaLayoutValidator( OMRequest req, ValidationContext ctx) { fireValidationEvent("multiPurposePreProcessCreateVolumeBucketLayoutCLientQuotaLayoutValidator"); @@ -155,10 +155,10 @@ public static OMRequest multiPurposePreProcessCreateVolumeBucketLayoutCLientQuot @OMClientVersionValidator( processingPhase = POST_PROCESS, requestType = CreateVolume, - applyUntil = ClientVersion.BUCKET_LAYOUT_SUPPORT) + applyBefore = ClientVersion.BUCKET_LAYOUT_SUPPORT) @OMLayoutVersionValidator(processingPhase = POST_PROCESS, requestType = CreateVolume, - applyUntil = OMLayoutFeature.QUOTA) + applyBefore = OMLayoutFeature.QUOTA) public static OMResponse multiPurposePostProcessCreateVolumeBucketLayoutCLientQuotaLayoutValidator( OMRequest req, OMResponse resp, ValidationContext ctx) { fireValidationEvent("multiPurposePostProcessCreateVolumeBucketLayoutCLientQuotaLayoutValidator"); @@ -168,7 +168,7 @@ public static OMResponse multiPurposePostProcessCreateVolumeBucketLayoutCLientQu @OMClientVersionValidator( processingPhase = POST_PROCESS, requestType = CreateKey, - applyUntil = ClientVersion.EC_REPLICA_INDEX_REQUIRED_IN_BLOCK_REQUEST) + applyBefore = ClientVersion.EC_REPLICA_INDEX_REQUIRED_IN_BLOCK_REQUEST) public static OMResponse postProcessCreateKeyECReplicaIndexRequiredClientValidator( OMRequest req, OMResponse resp, ValidationContext ctx) { fireValidationEvent("postProcessCreateKeyECReplicaIndexRequiredClientValidator"); @@ -178,7 +178,7 @@ public static OMResponse postProcessCreateKeyECReplicaIndexRequiredClientValidat @OMClientVersionValidator( processingPhase = PRE_PROCESS, requestType = DeleteKeys, - applyUntil = ClientVersion.BUCKET_LAYOUT_SUPPORT + applyBefore = ClientVersion.BUCKET_LAYOUT_SUPPORT ) public static OMRequest throwingPreProcessDeleteKeyBucketLayoutClientValidator( OMRequest req, ValidationContext ctx) throws IOException { @@ -192,7 +192,7 @@ public static OMRequest throwingPreProcessDeleteKeyBucketLayoutClientValidator( @OMClientVersionValidator( processingPhase = POST_PROCESS, requestType = DeleteKeys, - applyUntil = ClientVersion.BUCKET_LAYOUT_SUPPORT + applyBefore = ClientVersion.BUCKET_LAYOUT_SUPPORT ) public static OMResponse throwingPostProcessDeleteKeyBucketLayoutClientValidator( OMRequest req, OMResponse resp, ValidationContext ctx) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset2/ValidatorsForOnlyOldClientValidations.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset2/ValidatorsForOnlyOldClientValidations.java index 4bd966772980..061a5b9c429d 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset2/ValidatorsForOnlyOldClientValidations.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset2/ValidatorsForOnlyOldClientValidations.java @@ -35,7 +35,7 @@ private ValidatorsForOnlyOldClientValidations() { } @OMClientVersionValidator( processingPhase = PRE_PROCESS, requestType = CreateKey, - applyUntil = ClientVersion.BUCKET_LAYOUT_SUPPORT) + applyBefore = ClientVersion.BUCKET_LAYOUT_SUPPORT) public static OMRequest preProcessCreateKeyBucketLayoutClientValidator( OMRequest req, ValidationContext ctx) { return req; From c33f78ee204acfb493f109e583d3fd662181c84f Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Tue, 17 Dec 2024 17:08:52 -0800 Subject: [PATCH 12/28] Revert "HDDS-11132. Revert client version bump done as part of HDDS-10983 (#7348)" This reverts commit e7bf1547 Change-Id: Ie8d5b1ec85844b890039eb0026fad90fa4d45601 --- .../org/apache/hadoop/ozone/ClientVersion.java | 4 ++++ .../container/keyvalue/KeyValueHandler.java | 18 ++++++++++++++++-- .../container/keyvalue/helpers/BlockUtils.java | 4 +--- ...tKeyValueHandlerWithUnhealthyContainer.java | 6 ++++-- .../scm/storage/TestContainerCommandsEC.java | 2 +- .../om/request/key/OMKeyCommitRequest.java | 4 ++-- 6 files changed, 28 insertions(+), 10 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java index 08e293563435..c5627ec30d89 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java @@ -43,6 +43,10 @@ public enum ClientVersion implements ComponentVersion { "This client version has support for Object Store and File " + "System Optimized Bucket Layouts."), + EC_REPLICA_INDEX_REQUIRED_IN_BLOCK_REQUEST(4, + "This client version enforces replica index is set for fixing read corruption that could occur when " + + "replicaIndex parameter is not validated before EC block reads."), + FUTURE_VERSION(-1, "Used internally when the server side is older and an" + " unknown client version has arrived from the client."); diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java index 06a4543bd794..d20fda555b60 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java @@ -122,6 +122,7 @@ import static org.apache.hadoop.hdds.scm.protocolPB.ContainerCommandResponseBuilders.putBlockResponseSuccess; import static org.apache.hadoop.hdds.scm.protocolPB.ContainerCommandResponseBuilders.unsupportedRequest; import static org.apache.hadoop.hdds.scm.utils.ClientCommandsUtils.getReadChunkVersion; +import static org.apache.hadoop.ozone.ClientVersion.EC_REPLICA_INDEX_REQUIRED_IN_BLOCK_REQUEST; import static org.apache.hadoop.ozone.OzoneConsts.INCREMENTAL_CHUNK_LIST; import static org.apache.hadoop.ozone.container.common.interfaces.Container.ScanResult; @@ -665,6 +666,15 @@ ContainerCommandResponseProto handleEcho( return getEchoResponse(request); } + /** + * Checks if a replicaIndex needs to be checked based on the client version for a request. + * @param request ContainerCommandRequest object. + * @return true if the validation is required for the client version else false. + */ + private boolean replicaIndexCheckRequired(ContainerCommandRequestProto request) { + return request.hasVersion() && request.getVersion() >= EC_REPLICA_INDEX_REQUIRED_IN_BLOCK_REQUEST.toProtoValue(); + } + /** * Handle Get Block operation. Calls BlockManager to process the request. */ @@ -683,7 +693,9 @@ ContainerCommandResponseProto handleGetBlock( try { BlockID blockID = BlockID.getFromProtobuf( request.getGetBlock().getBlockID()); - BlockUtils.verifyReplicaIdx(kvContainer, blockID); + if (replicaIndexCheckRequired(request)) { + BlockUtils.verifyReplicaIdx(kvContainer, blockID); + } responseData = blockManager.getBlock(kvContainer, blockID).getProtoBufMessage(); final long numBytes = responseData.getSerializedSize(); metrics.incContainerBytesStats(Type.GetBlock, numBytes); @@ -806,7 +818,9 @@ ContainerCommandResponseProto handleReadChunk( ChunkInfo chunkInfo = ChunkInfo.getFromProtoBuf(request.getReadChunk() .getChunkData()); Preconditions.checkNotNull(chunkInfo); - BlockUtils.verifyReplicaIdx(kvContainer, blockID); + if (replicaIndexCheckRequired(request)) { + BlockUtils.verifyReplicaIdx(kvContainer, blockID); + } BlockUtils.verifyBCSId(kvContainer, blockID); if (dispatcherContext == null) { diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/BlockUtils.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/BlockUtils.java index 8bbc2478004d..945efbcf6ea9 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/BlockUtils.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/BlockUtils.java @@ -247,9 +247,7 @@ public static void verifyBCSId(Container container, BlockID blockID) public static void verifyReplicaIdx(Container container, BlockID blockID) throws IOException { Integer containerReplicaIndex = container.getContainerData().getReplicaIndex(); - Integer blockReplicaIndex = blockID.getReplicaIndex(); - if (containerReplicaIndex > 0 && blockReplicaIndex != null && blockReplicaIndex != 0 && - !containerReplicaIndex.equals(blockReplicaIndex)) { + if (containerReplicaIndex > 0 && !containerReplicaIndex.equals(blockID.getReplicaIndex())) { throw new StorageContainerException( "Unable to find the Container with replicaIdx " + blockID.getReplicaIndex() + ". Container " + container.getContainerData().getContainerID() + " replicaIdx is " diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandlerWithUnhealthyContainer.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandlerWithUnhealthyContainer.java index 395e943c7684..1db2d7ff53eb 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandlerWithUnhealthyContainer.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandlerWithUnhealthyContainer.java @@ -124,7 +124,8 @@ public void testGetBlockWithReplicaIndexMismatch(ClientVersion clientVersion, in handler.handleGetBlock( getDummyCommandRequestProto(clientVersion, ContainerProtos.Type.GetBlock, rid), container); - assertEquals((replicaIndex > 0 && rid != 0 && rid != replicaIndex) ? + assertEquals((replicaIndex > 0 && rid != replicaIndex && clientVersion.toProtoValue() >= + ClientVersion.EC_REPLICA_INDEX_REQUIRED_IN_BLOCK_REQUEST.toProtoValue()) ? ContainerProtos.Result.CONTAINER_NOT_FOUND : UNKNOWN_BCSID, response.getResult()); } @@ -166,7 +167,8 @@ public void testReadChunkWithReplicaIndexMismatch(ClientVersion clientVersion, i ContainerProtos.ContainerCommandResponseProto response = handler.handleReadChunk(getDummyCommandRequestProto(clientVersion, ContainerProtos.Type.ReadChunk, rid), container, null); - assertEquals((replicaIndex > 0 && rid != 0 && rid != replicaIndex) ? + assertEquals((replicaIndex > 0 && rid != replicaIndex && + clientVersion.toProtoValue() >= ClientVersion.EC_REPLICA_INDEX_REQUIRED_IN_BLOCK_REQUEST.toProtoValue()) ? ContainerProtos.Result.CONTAINER_NOT_FOUND : UNKNOWN_BCSID, response.getResult()); } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestContainerCommandsEC.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestContainerCommandsEC.java index 1b7eb837cf89..6f79839cd021 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestContainerCommandsEC.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestContainerCommandsEC.java @@ -460,7 +460,7 @@ public void testCreateRecoveryContainer() throws Exception { int replicaIndex = 4; XceiverClientSpi dnClient = xceiverClientManager.acquireClient( createSingleNodePipeline(newPipeline, newPipeline.getNodes().get(0), - 2)); + replicaIndex)); try { // To create the actual situation, container would have been in closed // state at SCM. diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java index 84e907a20338..ef374658cb43 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java @@ -575,8 +575,8 @@ public static OMRequest disallowHsync( * @return the validated request * @throws OMException if the request is invalid */ - @RequestFeatureValidator( - conditions = ValidationCondition.CLUSTER_NEEDS_FINALIZATION, + @OMLayoutVersionValidator( + applyBefore = OMLayoutFeature.HBASE_SUPPORT, processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CommitKey ) From a39467057846bcf669380289f5674bb6eed9e752 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Tue, 17 Dec 2024 18:38:33 -0800 Subject: [PATCH 13/28] HDDS-11149. Address review comments Change-Id: I9f6854136551102dfe94086a8afaf1c820331533 --- .../request/validation/ValidatorRegistry.java | 44 +++++++++++++------ .../file/OMDirectoryCreateRequest.java | 3 -- 2 files changed, 31 insertions(+), 16 deletions(-) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/ValidatorRegistry.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/ValidatorRegistry.java index c79dd20f4fde..b53964a730f6 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/ValidatorRegistry.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/ValidatorRegistry.java @@ -49,6 +49,16 @@ */ public class ValidatorRegistry> { + /** + * A validator registered should have the following parameters: + * applyBeforeVersion: Enum extending Version + * RequestType: Enum signifying the type of request. + * RequestProcessingPhase: Signifying if the validator is supposed to run pre or post submitting the request. + * Based on the afforementioned parameters a complete map is built which stores the validators in a sorted order of + * the applyBeforeVersion value of the validator method. + * Thus when a request comes with a certain version value, all validators containing `applyBeforeVersion` parameter + * greater than the request versions get triggered. {@link #validationsFor(Enum, RequestProcessingPhase, Version)} + */ private final Map, EnumMap>>> indexedValidatorMap; @@ -58,31 +68,32 @@ public class ValidatorRegistry> { * A validation method is recognized by all the annotations classes which * are annotated by {@link RegisterValidator} annotation that contains * important information about how and when to use the validator. + * @param requestType class of request type enum. * @param validatorPackage the main package inside which validatiors should * be discovered. + * @param allowedVersionTypes a set containing the various types of version allowed to be registered. + * @param allowedProcessingPhases set of request processing phases which would be allowed to be registered to + * registry. + * */ public ValidatorRegistry(Class requestType, - String validatorPackage, - Set> allowedVersionTypes, - Set allowedProcessingPhases) { + String validatorPackage, + Set> allowedVersionTypes, + Set allowedProcessingPhases) { this(requestType, ClasspathHelper.forPackage(validatorPackage), allowedVersionTypes, allowedProcessingPhases); } - private Class getReturnTypeOfAnnotationMethod(Class clzz, String methodName) { - try { - return clzz.getMethod(methodName).getReturnType(); - } catch (NoSuchMethodException e) { - throw new IllegalArgumentException("Method " + methodName + " not found in class:" + clzz.getCanonicalName()); - } - } - /** * Creates a {@link ValidatorRegistry} instance that discovers validation * methods under the provided URL. - * A validation method is recognized by all annotations having the annotated by {@link RegisterValidator} + * A validation method is recognized by all annotations annotated by the {@link RegisterValidator} * annotation that contains important information about how and when to use * the validator. + * @param requestType class of request type enum. * @param searchUrls the path in which the annotated methods are searched. + * @param allowedVersionTypes a set containing the various types of version allowed to be registered. + * @param allowedProcessingPhases set of request processing phases which would be allowed to be registered to + * registry. */ public ValidatorRegistry(Class requestType, Collection searchUrls, @@ -170,6 +181,14 @@ private ReturnValue callAnnotationMe } } + private Class getReturnTypeOfAnnotationMethod(Class clzz, String methodName) { + try { + return clzz.getMethod(methodName).getReturnType(); + } catch (NoSuchMethodException e) { + throw new IllegalArgumentException("Method " + methodName + " not found in class:" + clzz.getCanonicalName()); + } + } + private Version getApplyBeforeVersion(Validator validator) { return callAnnotationMethod(validator, RegisterValidator.APPLY_BEFORE_METHOD_NAME, Version.class); } @@ -242,7 +261,6 @@ private void registerValidator(Class requestType, } } - } private V getAndInitialize(K key, Supplier defaultSupplier, Map from) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java index 5ae4b68a8c8f..92a15baef3f9 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java @@ -26,9 +26,6 @@ import java.util.Map; import com.google.common.base.Preconditions; -import org.apache.hadoop.hdds.client.ECReplicationConfig; -import org.apache.hadoop.hdds.client.ReplicationConfig; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.ozone.ClientVersion; import org.apache.hadoop.ozone.om.request.validation.OMLayoutVersionValidator; import org.apache.ratis.server.protocol.TermIndex; From d024eb96a7ca68fdb94fa05696cf93be78a9d5fd Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Tue, 17 Dec 2024 19:07:02 -0800 Subject: [PATCH 14/28] HDDS-11149. Address review comments Change-Id: Ia0d97078d6c9fc61f2f79bc322f966e11ab4a3c4 --- .../request/validation/ValidatorRegistry.java | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/ValidatorRegistry.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/ValidatorRegistry.java index b53964a730f6..f3e35eeb8d59 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/ValidatorRegistry.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/ValidatorRegistry.java @@ -156,7 +156,8 @@ public List validationsFor(RequestType requestType, V requestVersion) { return Optional.ofNullable(this.indexedValidatorMap.get(requestVersion.getClass())) - .map(requestTypeMap -> requestTypeMap.get(requestType)).map(phaseMap -> phaseMap.get(phase)) + .map(requestTypeMap -> requestTypeMap.get(requestType)) + .map(phaseMap -> phaseMap.get(phase)) .map(indexedMethods -> requestVersion.version() < 0 ? indexedMethods.getItemsEqualToIdx(requestVersion.version()) : indexedMethods.getItemsGreaterThanIdx(requestVersion.version())) @@ -256,18 +257,20 @@ private void registerValidator(Class requestType, EnumMap>> requestMap = this.indexedValidatorMap.get(versionClass); EnumMap> phaseMap = - this.getAndInitialize(type, () -> new EnumMap<>(RequestProcessingPhase.class), requestMap); - this.getAndInitialize(phase, IndexedItems::new, phaseMap).add(method, applyBeforeVersion.version()); + requestMap.computeIfAbsent(type, k -> new EnumMap<>(RequestProcessingPhase.class)); + phaseMap.computeIfAbsent(phase, k -> new IndexedItems<>()).add(method, applyBeforeVersion.version()); } - } } - private V getAndInitialize(K key, Supplier defaultSupplier, Map from) { - return from.computeIfAbsent(key, k -> defaultSupplier.get()); - } - - static final class IndexedItems> { + /** + * Class responsible for maintaining indexs of items. Here each item should have an index corresponding to it. + * The class implements functions for efficiently fetching range gets on the items added to the data structure. + * @param Refers to the Type of the item in the data structure + * @param Type of the index of an item added in the data structure. It is important that the index is + * comparable to each other. + */ + private static final class IndexedItems> { private final List items; private final TreeMap indexMap; @@ -288,7 +291,6 @@ public void add(T item, IDX idx) { } /** - * * @param indexValue Given index value. * @return All the items which has an index value greater than given index value. */ @@ -299,7 +301,6 @@ public List getItemsGreaterThanIdx(IDX indexValue) { } /** - * * @param indexValue Given index value. * @return All the items which has an index value greater than given index value. */ From 889e0812cea72c9fc6625f8e797e08dedf86b507 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Tue, 17 Dec 2024 21:09:00 -0800 Subject: [PATCH 15/28] HDDS-11149. Address review comments Change-Id: I63b013066618c0568e13c2323b8403d1ca6d0426 --- .../hadoop/ozone/request/validation/ValidatorRegistry.java | 1 - .../ozone/om/request/validation/OMClientVersionValidator.java | 2 +- .../ozone/om/request/validation/OMLayoutVersionValidator.java | 2 +- .../hadoop/ozone/om/request/validation/RequestValidations.java | 2 +- 4 files changed, 3 insertions(+), 4 deletions(-) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/ValidatorRegistry.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/ValidatorRegistry.java index f3e35eeb8d59..98722f369e3d 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/ValidatorRegistry.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/ValidatorRegistry.java @@ -40,7 +40,6 @@ import java.util.Set; import java.util.TreeMap; import java.util.function.Function; -import java.util.function.Supplier; import java.util.stream.Collectors; /** diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/OMClientVersionValidator.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/OMClientVersionValidator.java index 2ec6c70401d1..fe3e735c4753 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/OMClientVersionValidator.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/OMClientVersionValidator.java @@ -88,7 +88,7 @@ /** * The type of the request handled by this validator method. - * @return the requestType to whihc the validator shoudl be applied + * @return the requestType to whihc the validator should be applied */ Type[] requestType(); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/OMLayoutVersionValidator.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/OMLayoutVersionValidator.java index c3396e633895..be7e62829534 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/OMLayoutVersionValidator.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/OMLayoutVersionValidator.java @@ -92,7 +92,7 @@ /** * The type of the request handled by this validator method. - * @return the requestType to whihc the validator shoudl be applied + * @return the requestType to whihc the validator should be applied */ Type[] requestType(); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestValidations.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestValidations.java index a41411270781..2abc5dd31924 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestValidations.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestValidations.java @@ -77,7 +77,7 @@ public OMRequest validateRequest(OMRequest request) List validations = registry.validationsFor(request.getCmdType(), PRE_PROCESS, this.getVersions(request)); - OMRequest validatedRequest = request.toBuilder().build(); + OMRequest validatedRequest = request; try { for (Method m : validations) { LOG.debug("Running the {} request pre-process validation from {}.{}", From 436dfbd5da5f7b6d286e60360f0025bafbccbdac Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Wed, 18 Dec 2024 08:33:51 -0800 Subject: [PATCH 16/28] HDDS-11149. Fix findbugs Change-Id: I7701ad25c3fb4cde9f78daef3053c822b41fe4a0 --- .../OmRequestFeatureValidatorProcessor.java | 53 ------------------- .../validation/TestOMValidatorProcessor.java | 10 +--- 2 files changed, 1 insertion(+), 62 deletions(-) diff --git a/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/OmRequestFeatureValidatorProcessor.java b/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/OmRequestFeatureValidatorProcessor.java index f11977262f41..03e6a0272511 100644 --- a/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/OmRequestFeatureValidatorProcessor.java +++ b/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/OmRequestFeatureValidatorProcessor.java @@ -91,18 +91,14 @@ public class OmRequestFeatureValidatorProcessor extends AbstractProcessor { private static final List ANNOTATION_SIMPLE_NAMES = Arrays.asList("OMClientVersionValidator", "OMLayoutVersionValidator"); - public static final String ANNOTATION_CONDITIONS_PROPERTY_NAME = "conditions"; public static final String ANNOTATION_PROCESSING_PHASE_PROPERTY_NAME = "processingPhase"; - public static final String ANNOTATION_MAX_CLIENT_VERSION_PROPERTY_NAME = "maxClientVersion"; public static final String PROCESSING_PHASE_PRE_PROCESS = "PRE_PROCESS"; public static final String PROCESSING_PHASE_POST_PROCESS = "POST_PROCESS"; public static final String ERROR_NO_PROCESSING_PHASE_DEFINED = "RequestFeatureValidator has an invalid ProcessingPhase defined."; - public static final String MAX_CLIENT_VERSION_FUTURE_VERSION = "FUTURE_VERSION"; - @Override public boolean process(Set annotations, RoundEnvironment roundEnv) { @@ -224,30 +220,11 @@ private boolean evaluateProcessingPhase( return false; } - private boolean hasValidMaxClientVersion( - Entry entry) { - if (isPropertyNamedAs(entry, ANNOTATION_MAX_CLIENT_VERSION_PROPERTY_NAME)) { - Name maxClientVersion = visit(entry, new MaxClientVersionVisitor()); - return !maxClientVersion.contentEquals(MAX_CLIENT_VERSION_FUTURE_VERSION); - } - return false; - } - private boolean isProcessingPhaseValue( Entry entry) { return isPropertyNamedAs(entry, ANNOTATION_PROCESSING_PHASE_PROPERTY_NAME); } - private boolean isMaxClientVersionValue( - Entry entry) { - return isPropertyNamedAs(entry, ANNOTATION_MAX_CLIENT_VERSION_PROPERTY_NAME); - } - - private boolean hasValidValidationCondition( - Entry entry) { - return isPropertyNamedAs(entry, ANNOTATION_CONDITIONS_PROPERTY_NAME) - && visit(entry, new ConditionValidator()); - } private boolean isPropertyNamedAs( Entry entry, @@ -261,36 +238,6 @@ private T visit( return entry.getValue().accept(visitor, null); } - private static class ConditionValidator - extends SimpleAnnotationValueVisitor8 { - - ConditionValidator() { - super(Boolean.FALSE); - } - - @Override - public Boolean visitArray(List vals, - Void unused) { - if (vals.isEmpty()) { - return Boolean.FALSE; - } - return Boolean.TRUE; - } - - } - - private static class MaxClientVersionVisitor - extends SimpleAnnotationValueVisitor8 { - - MaxClientVersionVisitor() { - } - - @Override - public Name visitEnumConstant(VariableElement c, Void unused) { - return c.getSimpleName(); - } - } - private static class ProcessingPhaseVisitor extends SimpleAnnotationValueVisitor8 { diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestOMValidatorProcessor.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestOMValidatorProcessor.java index 2f4dc1d5bada..b2615851630f 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestOMValidatorProcessor.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestOMValidatorProcessor.java @@ -489,18 +489,10 @@ private List generateSourceOfValidatorMethodWith( lines.add(" }"); lines.add("}"); lines.add(""); - lines.stream() -// .filter(s -> !s.startsWith("import")) - .forEach(System.out::println); + lines.stream().forEach(System.out::println); return lines; } - private & Version> String annotationOf(RequestProcessingPhase phase, - Type reqType, Class annotationClass) { - return annotationOf(phase.name(), reqType, annotationClass, - (V)ANNOTATION_VERSION_CLASS_MAP.get(annotationClass).getEnumConstants()[0]); - } - private & Version> String annotationOf( RequestProcessingPhase phase, Type reqType, Class annotationClass, V applyBeforeVersion) { return annotationOf(phase.name(), reqType, annotationClass, applyBeforeVersion); From c1b690d17dfee0eeb1ab4a44e83369a1c3cc9084 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Wed, 18 Dec 2024 08:38:51 -0800 Subject: [PATCH 17/28] HDDS-11149. Fix annotation processor Change-Id: I855a19c67256a27504b7e3d134b2f500a94bf330 --- .../ozone/annotations/OmRequestFeatureValidatorProcessor.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/OmRequestFeatureValidatorProcessor.java b/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/OmRequestFeatureValidatorProcessor.java index 03e6a0272511..50ee15e0d08b 100644 --- a/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/OmRequestFeatureValidatorProcessor.java +++ b/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/OmRequestFeatureValidatorProcessor.java @@ -200,12 +200,10 @@ private boolean checkAndEvaluateAnnotation( boolean isPreprocessor = false; for (Entry entry : methodAnnotation.getElementValues().entrySet()) { - if (isProcessingPhaseValue(entry)) { isPreprocessor = evaluateProcessingPhase(entry); } } - return isPreprocessor; } From b9ef7d0c476b8f7f78ae4022d9184c280bc7bb5f Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Wed, 18 Dec 2024 12:45:06 -0800 Subject: [PATCH 18/28] HDDS-11149. Fix annotation processor Change-Id: I6a5e84b8053a9401ce4210849495af8254561f09 --- .../ozone/annotations/OmRequestFeatureValidatorProcessor.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/OmRequestFeatureValidatorProcessor.java b/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/OmRequestFeatureValidatorProcessor.java index 50ee15e0d08b..0c93caba7e6b 100644 --- a/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/OmRequestFeatureValidatorProcessor.java +++ b/hadoop-hdds/annotations/src/main/java/org/apache/ozone/annotations/OmRequestFeatureValidatorProcessor.java @@ -28,7 +28,6 @@ import javax.lang.model.element.ElementKind; import javax.lang.model.element.ExecutableElement; import javax.lang.model.element.Modifier; -import javax.lang.model.element.Name; import javax.lang.model.element.TypeElement; import javax.lang.model.element.VariableElement; import javax.lang.model.type.ExecutableType; From ba133d2df074fe062c86e9fa63e3fc52dac3f85f Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Wed, 18 Dec 2024 15:59:55 -0800 Subject: [PATCH 19/28] HDDS-11149. Fix package scanner prefix Change-Id: I13bc9e28c5db3b706265abb8cecaf71c5201151c --- .../hadoop/ozone/request/validation/ValidatorRegistry.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/ValidatorRegistry.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/ValidatorRegistry.java index 98722f369e3d..bbb10d455c8b 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/ValidatorRegistry.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/ValidatorRegistry.java @@ -101,7 +101,7 @@ public ValidatorRegistry(Class requestType, Class requestArrayClass = (Class) Array.newInstance(requestType, 0) .getClass(); Set> validatorsToBeRegistered = - new Reflections(new ConfigurationBuilder().setUrls(ClasspathHelper.forPackage("")) + new Reflections(new ConfigurationBuilder().setUrls(ClasspathHelper.forPackage("org.apache")) .setScanners(Scanners.TypesAnnotated) .setParallel(true)).getTypesAnnotatedWith(RegisterValidator.class).stream() .filter(annotationClass -> getReturnTypeOfAnnotationMethod((Class) annotationClass, From df698f86d44d86932dca342f409321e8bad19753 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Wed, 18 Dec 2024 16:13:22 -0800 Subject: [PATCH 20/28] HDDS-11149. Fix package scanner prefix Change-Id: I8302abcfcaa70b07a85ef9505c7dabdc312955d4 --- .../hadoop/ozone/request/validation/ValidatorRegistry.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/ValidatorRegistry.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/ValidatorRegistry.java index bbb10d455c8b..d66e463f1c10 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/ValidatorRegistry.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/ValidatorRegistry.java @@ -101,7 +101,7 @@ public ValidatorRegistry(Class requestType, Class requestArrayClass = (Class) Array.newInstance(requestType, 0) .getClass(); Set> validatorsToBeRegistered = - new Reflections(new ConfigurationBuilder().setUrls(ClasspathHelper.forPackage("org.apache")) + new Reflections(new ConfigurationBuilder().setUrls(ClasspathHelper.forPackage("org.apache.hadoop")) .setScanners(Scanners.TypesAnnotated) .setParallel(true)).getTypesAnnotatedWith(RegisterValidator.class).stream() .filter(annotationClass -> getReturnTypeOfAnnotationMethod((Class) annotationClass, From 0b5ceb5aae4db4a90732e2b8c8fbe694e3f58ca2 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Wed, 18 Dec 2024 19:25:28 -0800 Subject: [PATCH 21/28] HDDS-11149. Fix version values Change-Id: I9b30d241a216733266754086b2add6fd623f27dd --- .../request/bucket/OMBucketCreateRequest.java | 2 +- .../request/bucket/OMBucketDeleteRequest.java | 2 +- .../file/OMDirectoryCreateRequest.java | 2 +- .../om/request/file/OMFileCreateRequest.java | 2 +- .../request/key/OMAllocateBlockRequest.java | 2 +- .../om/request/key/OMKeyCommitRequest.java | 4 ++-- .../om/request/key/OMKeyCreateRequest.java | 2 +- .../om/request/key/OMKeyDeleteRequest.java | 2 +- .../om/request/key/OMKeyRenameRequest.java | 2 +- .../om/request/key/OMKeysDeleteRequest.java | 2 +- .../om/request/key/OMKeysRenameRequest.java | 2 +- .../request/key/acl/OMKeyAddAclRequest.java | 2 +- .../key/acl/OMKeyRemoveAclRequest.java | 2 +- .../S3InitiateMultipartUploadRequest.java | 2 +- .../S3MultipartUploadAbortRequest.java | 2 +- .../S3MultipartUploadCommitPartRequest.java | 2 +- .../S3MultipartUploadCompleteRequest.java | 2 +- .../OzoneManagerRequestHandler.java | 24 +++++++++---------- 18 files changed, 30 insertions(+), 30 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java index be108d509240..29e4f2be61dc 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java @@ -467,7 +467,7 @@ public static OMRequest handleCreateBucketWithBucketLayoutDuringPreFinalize( @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CreateBucket, - applyBefore = ClientVersion.ERASURE_CODING_SUPPORT + applyBefore = ClientVersion.BUCKET_LAYOUT_SUPPORT ) public static OMRequest setDefaultBucketLayoutForOlderClients(OMRequest req, ValidationContext ctx) { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketDeleteRequest.java index f5d66c416064..8c13d6ac4ef8 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketDeleteRequest.java @@ -283,7 +283,7 @@ private boolean bucketContainsSnapshotInCache( @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.DeleteBucket, - applyBefore = ClientVersion.ERASURE_CODING_SUPPORT + applyBefore = ClientVersion.BUCKET_LAYOUT_SUPPORT ) public static OMRequest blockBucketDeleteWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java index 92a15baef3f9..b4afe8d614d1 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java @@ -308,7 +308,7 @@ public static OMRequest disallowCreateDirectoryWithECReplicationConfig( @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CreateDirectory, - applyBefore = ClientVersion.ERASURE_CODING_SUPPORT + applyBefore = ClientVersion.BUCKET_LAYOUT_SUPPORT ) public static OMRequest blockCreateDirectoryWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java index 0f39db2d695e..e81eb27bf0d1 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileCreateRequest.java @@ -420,7 +420,7 @@ public static OMRequest disallowCreateFileWithECReplicationConfig( @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CreateFile, - applyBefore = ClientVersion.ERASURE_CODING_SUPPORT + applyBefore = ClientVersion.BUCKET_LAYOUT_SUPPORT ) public static OMRequest blockCreateFileWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java index 794a285a8da9..d1576c11adcc 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java @@ -311,7 +311,7 @@ public static OMRequest disallowAllocateBlockWithECReplicationConfig( @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.AllocateBlock, - applyBefore = ClientVersion.ERASURE_CODING_SUPPORT + applyBefore = ClientVersion.BUCKET_LAYOUT_SUPPORT ) public static OMRequest blockAllocateBlockWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java index ef374658cb43..91efd57073de 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java @@ -526,7 +526,7 @@ public static OMRequest disallowCommitKeyWithECReplicationConfig( @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CommitKey, - applyBefore = ClientVersion.ERASURE_CODING_SUPPORT + applyBefore = ClientVersion.BUCKET_LAYOUT_SUPPORT ) public static OMRequest blockCommitKeyWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { @@ -545,7 +545,7 @@ public static OMRequest blockCommitKeyWithBucketLayoutFromOldClient( @OMLayoutVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CommitKey, - applyBefore = OMLayoutFeature.HSYNC + applyBefore = OMLayoutFeature.HBASE_SUPPORT ) public static OMRequest disallowHsync( OMRequest req, ValidationContext ctx) throws OMException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java index fbab19ecaf70..5d8e9f379a06 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java @@ -424,7 +424,7 @@ public static OMRequest disallowCreateKeyWithECReplicationConfig( @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CreateKey, - applyBefore = ClientVersion.ERASURE_CODING_SUPPORT + applyBefore = ClientVersion.BUCKET_LAYOUT_SUPPORT ) public static OMRequest blockCreateKeyWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java index 1afe93391e31..7b26513726e6 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyDeleteRequest.java @@ -245,7 +245,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.DeleteKey, - applyBefore = ClientVersion.ERASURE_CODING_SUPPORT + applyBefore = ClientVersion.BUCKET_LAYOUT_SUPPORT ) public static OMRequest blockDeleteKeyWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java index 2d7cca7c72e2..dc968d2844af 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequest.java @@ -265,7 +265,7 @@ private Map buildAuditMap( @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.RenameKey, - applyBefore = ClientVersion.ERASURE_CODING_SUPPORT + applyBefore = ClientVersion.BUCKET_LAYOUT_SUPPORT ) public static OMRequest blockRenameKeyWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java index 9ce42873c17d..d7792f8765d2 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java @@ -364,7 +364,7 @@ protected static void addDeletedKeys(Map auditMap, @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.DeleteKeys, - applyBefore = ClientVersion.ERASURE_CODING_SUPPORT + applyBefore = ClientVersion.BUCKET_LAYOUT_SUPPORT ) public static OMRequest blockDeleteKeysWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysRenameRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysRenameRequest.java index 599acd6f2d37..40146116e66e 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysRenameRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysRenameRequest.java @@ -299,7 +299,7 @@ private Map buildAuditMap(Map auditMap, @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.RenameKeys, - applyBefore = ClientVersion.ERASURE_CODING_SUPPORT + applyBefore = ClientVersion.BUCKET_LAYOUT_SUPPORT ) public static OMRequest blockRenameKeysWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAddAclRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAddAclRequest.java index 4c38d6f522d2..934eaf8fdfc2 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAddAclRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAddAclRequest.java @@ -167,7 +167,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.AddAcl, - applyBefore = ClientVersion.ERASURE_CODING_SUPPORT + applyBefore = ClientVersion.BUCKET_LAYOUT_SUPPORT ) public static OMRequest blockAddAclWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyRemoveAclRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyRemoveAclRequest.java index dec769f832a2..aa17720d1a7a 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyRemoveAclRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyRemoveAclRequest.java @@ -168,7 +168,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.RemoveAcl, - applyBefore = ClientVersion.ERASURE_CODING_SUPPORT + applyBefore = ClientVersion.BUCKET_LAYOUT_SUPPORT ) public static OMRequest blockRemoveAclWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java index 8eae50bef5ff..bee24223648b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java @@ -323,7 +323,7 @@ protected void logResult(OzoneManager ozoneManager, @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.InitiateMultiPartUpload, - applyBefore = ClientVersion.ERASURE_CODING_SUPPORT + applyBefore = ClientVersion.BUCKET_LAYOUT_SUPPORT ) public static OMRequest blockInitiateMPUWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java index cc477f5c8423..2ab8e89b4c86 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadAbortRequest.java @@ -292,7 +292,7 @@ public static OMRequest disallowAbortMultiPartUploadWithECReplicationConfig( @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.AbortMultiPartUpload, - applyBefore = ClientVersion.ERASURE_CODING_SUPPORT + applyBefore = ClientVersion.BUCKET_LAYOUT_SUPPORT ) public static OMRequest blockMPUAbortWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java index fc081e025ff1..779362e82fca 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCommitPartRequest.java @@ -383,7 +383,7 @@ public static OMRequest disallowCommitMultiPartUploadWithECReplicationConfig( @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CommitMultiPartUpload, - applyBefore = ClientVersion.ERASURE_CODING_SUPPORT + applyBefore = ClientVersion.BUCKET_LAYOUT_SUPPORT ) public static OMRequest blockMPUCommitWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java index 17a2e0e9048a..5fa2b92484a7 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3MultipartUploadCompleteRequest.java @@ -743,7 +743,7 @@ private void updateCache(OMMetadataManager omMetadataManager, @OMClientVersionValidator( processingPhase = RequestProcessingPhase.PRE_PROCESS, requestType = Type.CompleteMultiPartUpload, - applyBefore = ClientVersion.ERASURE_CODING_SUPPORT + applyBefore = ClientVersion.BUCKET_LAYOUT_SUPPORT ) public static OMRequest blockMPUCompleteWithBucketLayoutFromOldClient( OMRequest req, ValidationContext ctx) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java index 8891f11c2302..7cb8d482e2eb 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java @@ -669,7 +669,7 @@ private GetKeyInfoResponse getKeyInfo(GetKeyInfoRequest request, @OMClientVersionValidator( processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.LookupKey, - applyBefore = ClientVersion.VERSION_HANDLES_UNKNOWN_DN_PORTS + applyBefore = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMResponse disallowLookupKeyResponseWithECReplicationConfig( OMRequest req, OMResponse resp, ValidationContext ctx) @@ -696,7 +696,7 @@ public static OMResponse disallowLookupKeyResponseWithECReplicationConfig( @OMClientVersionValidator( processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.LookupKey, - applyBefore = ClientVersion.ERASURE_CODING_SUPPORT + applyBefore = ClientVersion.BUCKET_LAYOUT_SUPPORT ) public static OMResponse disallowLookupKeyWithBucketLayout( OMRequest req, OMResponse resp, ValidationContext ctx) @@ -783,7 +783,7 @@ private ListKeysLightResponse listKeysLight(ListKeysRequest request) @OMClientVersionValidator( processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.ListKeys, - applyBefore = ClientVersion.VERSION_HANDLES_UNKNOWN_DN_PORTS + applyBefore = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMResponse disallowListKeysResponseWithECReplicationConfig( OMRequest req, OMResponse resp, ValidationContext ctx) @@ -810,7 +810,7 @@ public static OMResponse disallowListKeysResponseWithECReplicationConfig( @OMClientVersionValidator( processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.ListKeys, - applyBefore = ClientVersion.ERASURE_CODING_SUPPORT + applyBefore = ClientVersion.BUCKET_LAYOUT_SUPPORT ) public static OMResponse disallowListKeysWithBucketLayout( OMRequest req, OMResponse resp, ValidationContext ctx) @@ -851,7 +851,7 @@ public static OMResponse disallowListKeysWithBucketLayout( @OMClientVersionValidator( processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.ListTrash, - applyBefore = ClientVersion.VERSION_HANDLES_UNKNOWN_DN_PORTS + applyBefore = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMResponse disallowListTrashWithECReplicationConfig( OMRequest req, OMResponse resp, ValidationContext ctx) @@ -881,7 +881,7 @@ public static OMResponse disallowListTrashWithECReplicationConfig( @OMClientVersionValidator( processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.ListTrash, - applyBefore = ClientVersion.ERASURE_CODING_SUPPORT + applyBefore = ClientVersion.BUCKET_LAYOUT_SUPPORT ) public static OMResponse disallowListTrashWithBucketLayout( OMRequest req, OMResponse resp, ValidationContext ctx) @@ -1094,7 +1094,7 @@ private RefetchSecretKeyResponse refetchSecretKey() { @OMClientVersionValidator( processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.GetFileStatus, - applyBefore = ClientVersion.VERSION_HANDLES_UNKNOWN_DN_PORTS + applyBefore = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMResponse disallowGetFileStatusWithECReplicationConfig( OMRequest req, OMResponse resp, ValidationContext ctx) @@ -1124,7 +1124,7 @@ public static OMResponse disallowGetFileStatusWithECReplicationConfig( @OMClientVersionValidator( processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.GetFileStatus, - applyBefore = ClientVersion.ERASURE_CODING_SUPPORT + applyBefore = ClientVersion.BUCKET_LAYOUT_SUPPORT ) public static OMResponse disallowGetFileStatusWithBucketLayout( OMRequest req, OMResponse resp, ValidationContext ctx) @@ -1173,7 +1173,7 @@ private LookupFileResponse lookupFile(LookupFileRequest request, @OMClientVersionValidator( processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.LookupFile, - applyBefore = ClientVersion.VERSION_HANDLES_UNKNOWN_DN_PORTS + applyBefore = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMResponse disallowLookupFileWithECReplicationConfig( OMRequest req, OMResponse resp, ValidationContext ctx) @@ -1201,7 +1201,7 @@ public static OMResponse disallowLookupFileWithECReplicationConfig( @OMClientVersionValidator( processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.LookupFile, - applyBefore = ClientVersion.ERASURE_CODING_SUPPORT + applyBefore = ClientVersion.BUCKET_LAYOUT_SUPPORT ) public static OMResponse disallowLookupFileWithBucketLayout( OMRequest req, OMResponse resp, ValidationContext ctx) @@ -1283,7 +1283,7 @@ private ListStatusLightResponse listStatusLight( @OMClientVersionValidator( processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.ListStatus, - applyBefore = ClientVersion.VERSION_HANDLES_UNKNOWN_DN_PORTS + applyBefore = ClientVersion.ERASURE_CODING_SUPPORT ) public static OMResponse disallowListStatusResponseWithECReplicationConfig( OMRequest req, OMResponse resp, ValidationContext ctx) @@ -1311,7 +1311,7 @@ public static OMResponse disallowListStatusResponseWithECReplicationConfig( @OMClientVersionValidator( processingPhase = RequestProcessingPhase.POST_PROCESS, requestType = Type.ListStatus, - applyBefore = ClientVersion.ERASURE_CODING_SUPPORT + applyBefore = ClientVersion.BUCKET_LAYOUT_SUPPORT ) public static OMResponse disallowListStatusResponseWithBucketLayout( OMRequest req, OMResponse resp, ValidationContext ctx) From 49efb2031d51e0eb9c2187e5e1d3948c8a92af7d Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Wed, 18 Dec 2024 21:33:47 -0800 Subject: [PATCH 22/28] HDDS-11963. Unify client version and layout version under one interface to accomodate in the request validator framework Change-Id: I4c1844a1dfb6127a73b46a92933db33b09c6b06e --- .../apache/hadoop/hdds/ComponentVersion.java | 9 ++- .../apache/hadoop/ozone/ClientVersion.java | 5 +- .../java/org/apache/hadoop/ozone/Version.java | 26 ++++++++ .../hadoop/ozone/upgrade/LayoutFeature.java | 9 ++- .../request/validation/VersionExtractor.java | 63 +++++++++++++++++++ .../validation/TestVersionExtractor.java | 58 +++++++++++++++++ 6 files changed, 166 insertions(+), 4 deletions(-) create mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/Version.java create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/VersionExtractor.java create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestVersionExtractor.java diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ComponentVersion.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ComponentVersion.java index 9545869e1636..a86d07fa5869 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ComponentVersion.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ComponentVersion.java @@ -17,10 +17,12 @@ */ package org.apache.hadoop.hdds; +import org.apache.hadoop.ozone.Version; + /** * Base type for component version enums. */ -public interface ComponentVersion { +public interface ComponentVersion extends Version { /** * Returns the description of the version enum value. @@ -34,4 +36,9 @@ public interface ComponentVersion { * @return the version associated with the enum value. */ int toProtoValue(); + + @Override + default int version() { + return toProtoValue(); + } } diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java index cc6695dc7d68..08e293563435 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/ClientVersion.java @@ -20,6 +20,7 @@ import org.apache.hadoop.hdds.ComponentVersion; import java.util.Arrays; +import java.util.Comparator; import java.util.Map; import static java.util.function.Function.identity; @@ -75,8 +76,8 @@ public static ClientVersion fromProtoValue(int value) { } private static ClientVersion latest() { - ClientVersion[] versions = ClientVersion.values(); - return versions[versions.length - 2]; + return Arrays.stream(ClientVersion.values()) + .max(Comparator.comparingInt(ComponentVersion::toProtoValue)).orElse(null); } } diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/Version.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/Version.java new file mode 100644 index 000000000000..b6dc636e248c --- /dev/null +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/Version.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone; + + +/** + * Base class defining the version in the entire system. + */ +public interface Version { + int version(); +} diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/LayoutFeature.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/LayoutFeature.java index 92dd706f4bb4..93c5ef0a0b73 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/LayoutFeature.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/LayoutFeature.java @@ -18,12 +18,14 @@ package org.apache.hadoop.ozone.upgrade; +import org.apache.hadoop.ozone.Version; + import java.util.Optional; /** * Generic Layout feature interface for Ozone. */ -public interface LayoutFeature { +public interface LayoutFeature extends Version { String name(); int layoutVersion(); @@ -48,6 +50,11 @@ default String name() { void execute(T arg) throws Exception; } + @Override + default int version() { + return this.layoutVersion(); + } + /** * Phase of execution for this action. */ diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/VersionExtractor.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/VersionExtractor.java new file mode 100644 index 000000000000..94f833be456c --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/VersionExtractor.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.ozone.om.request.validation; + +import org.apache.hadoop.ozone.ClientVersion; +import org.apache.hadoop.ozone.Version; +import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; +import org.apache.hadoop.ozone.upgrade.LayoutVersionManager; + +/** + * Class to extract version out of OM request. + */ +public enum VersionExtractor { + /** + * Extracts current metadata layout version. + */ + LAYOUT_VERSION_EXTRACTOR { + @Override + public Version extractVersion(OMRequest req, ValidationContext ctx) { + LayoutVersionManager layoutVersionManager = ctx.versionManager(); + return ctx.versionManager().getFeature(layoutVersionManager.getMetadataLayoutVersion()); + } + + @Override + public Class getVersionClass() { + return OMLayoutFeature.class; + } + }, + + /** + * Extracts client version from the OMRequests. + */ + CLIENT_VERSION_EXTRACTOR { + @Override + public Version extractVersion(OMRequest req, ValidationContext ctx) { + return req.getVersion() > ClientVersion.CURRENT_VERSION ? + ClientVersion.FUTURE_VERSION : ClientVersion.fromProtoValue(req.getVersion()); + } + + @Override + public Class getVersionClass() { + return ClientVersion.class; + } + }; + + public abstract Version extractVersion(OMRequest req, ValidationContext ctx); + public abstract Class getVersionClass(); +} diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestVersionExtractor.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestVersionExtractor.java new file mode 100644 index 000000000000..832cb751fd1b --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestVersionExtractor.java @@ -0,0 +1,58 @@ +package org.apache.hadoop.ozone.om.request.validation; + +import org.apache.hadoop.ozone.ClientVersion; +import org.apache.hadoop.ozone.Version; +import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature; +import org.apache.hadoop.ozone.om.upgrade.OMLayoutVersionManager; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; +import org.apache.hadoop.ozone.upgrade.LayoutVersionManager; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.util.Arrays; +import java.util.function.Function; +import java.util.stream.IntStream; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class TestVersionExtractor { + + private static Stream layoutValues() { + return Arrays.stream(OMLayoutFeature.values()).map(Arguments::of); + } + + private static Stream clientVersionValues() { + return Stream.of( + Arrays.stream(ClientVersion.values()) + .map(clientVersion -> Arguments.of(clientVersion, clientVersion.version())), + IntStream.range(1, 10) + .mapToObj(delta -> Arguments.of(ClientVersion.FUTURE_VERSION, ClientVersion.CURRENT_VERSION + delta)) + ).flatMap(Function.identity()); + } + + @ParameterizedTest + @MethodSource("layoutValues") + void testLayoutVersionExtractor(OMLayoutFeature layoutVersionValue) throws OMException { + ValidationContext context = mock(ValidationContext.class); + LayoutVersionManager layoutVersionManager = new OMLayoutVersionManager(layoutVersionValue.version()); + when(context.versionManager()).thenReturn(layoutVersionManager); + Version version = VersionExtractor.LAYOUT_VERSION_EXTRACTOR.extractVersion(null, context); + assertEquals(layoutVersionValue, version); + assertEquals(OMLayoutFeature.class, VersionExtractor.LAYOUT_VERSION_EXTRACTOR.getVersionClass()); + } + + @ParameterizedTest + @MethodSource("clientVersionValues") + void testClientVersionExtractor(ClientVersion expectedClientVersion, int clientVersion) { + OMRequest request = mock(OMRequest.class); + when(request.getVersion()).thenReturn(clientVersion); + Version version = VersionExtractor.CLIENT_VERSION_EXTRACTOR.extractVersion(request, null); + assertEquals(expectedClientVersion, version); + assertEquals(ClientVersion.class, VersionExtractor.CLIENT_VERSION_EXTRACTOR.getVersionClass()); + } +} From b819b63bb7bae3eaa20304c8f247475692a71e4c Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Wed, 18 Dec 2024 21:38:04 -0800 Subject: [PATCH 23/28] HDDS-11963. Fix rat failure Change-Id: I63cddb955ea6178cd489c2a83922576e8d2ea5eb --- .../request/validation/TestVersionExtractor.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestVersionExtractor.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestVersionExtractor.java index 832cb751fd1b..0a2344840cfd 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestVersionExtractor.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestVersionExtractor.java @@ -1,3 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ package org.apache.hadoop.ozone.om.request.validation; import org.apache.hadoop.ozone.ClientVersion; From bfa7618271d151611ce9214bceb4b4b1c57e6533 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Wed, 18 Dec 2024 22:25:10 -0800 Subject: [PATCH 24/28] HDDS-11963. Address review comments Change-Id: I11bff280f429d2eb64f8828887f7349e169e938f --- .../apache/hadoop/hdds/ComponentVersion.java | 4 +- .../ozone/{Version.java => Versioned.java} | 2 +- .../hadoop/ozone/upgrade/LayoutFeature.java | 4 +- .../request/validation/VersionExtractor.java | 10 ++--- .../validation/TestVersionExtractor.java | 40 +++++++++---------- 5 files changed, 30 insertions(+), 30 deletions(-) rename hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/{Version.java => Versioned.java} (96%) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ComponentVersion.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ComponentVersion.java index a86d07fa5869..7f65010e2c03 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ComponentVersion.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ComponentVersion.java @@ -17,12 +17,12 @@ */ package org.apache.hadoop.hdds; -import org.apache.hadoop.ozone.Version; +import org.apache.hadoop.ozone.Versioned; /** * Base type for component version enums. */ -public interface ComponentVersion extends Version { +public interface ComponentVersion extends Versioned { /** * Returns the description of the version enum value. diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/Version.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/Versioned.java similarity index 96% rename from hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/Version.java rename to hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/Versioned.java index b6dc636e248c..7f89b403b34b 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/Version.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/Versioned.java @@ -21,6 +21,6 @@ /** * Base class defining the version in the entire system. */ -public interface Version { +public interface Versioned { int version(); } diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/LayoutFeature.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/LayoutFeature.java index 93c5ef0a0b73..9ec9b4cb589a 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/LayoutFeature.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/LayoutFeature.java @@ -18,14 +18,14 @@ package org.apache.hadoop.ozone.upgrade; -import org.apache.hadoop.ozone.Version; +import org.apache.hadoop.ozone.Versioned; import java.util.Optional; /** * Generic Layout feature interface for Ozone. */ -public interface LayoutFeature extends Version { +public interface LayoutFeature extends Versioned { String name(); int layoutVersion(); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/VersionExtractor.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/VersionExtractor.java index 94f833be456c..e7acef45bfe2 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/VersionExtractor.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/VersionExtractor.java @@ -17,7 +17,7 @@ package org.apache.hadoop.ozone.om.request.validation; import org.apache.hadoop.ozone.ClientVersion; -import org.apache.hadoop.ozone.Version; +import org.apache.hadoop.ozone.Versioned; import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.ozone.upgrade.LayoutVersionManager; @@ -31,7 +31,7 @@ public enum VersionExtractor { */ LAYOUT_VERSION_EXTRACTOR { @Override - public Version extractVersion(OMRequest req, ValidationContext ctx) { + public Versioned extractVersion(OMRequest req, ValidationContext ctx) { LayoutVersionManager layoutVersionManager = ctx.versionManager(); return ctx.versionManager().getFeature(layoutVersionManager.getMetadataLayoutVersion()); } @@ -47,7 +47,7 @@ public Class getVersionClass() { */ CLIENT_VERSION_EXTRACTOR { @Override - public Version extractVersion(OMRequest req, ValidationContext ctx) { + public Versioned extractVersion(OMRequest req, ValidationContext ctx) { return req.getVersion() > ClientVersion.CURRENT_VERSION ? ClientVersion.FUTURE_VERSION : ClientVersion.fromProtoValue(req.getVersion()); } @@ -58,6 +58,6 @@ public Class getVersionClass() { } }; - public abstract Version extractVersion(OMRequest req, ValidationContext ctx); - public abstract Class getVersionClass(); + public abstract Versioned extractVersion(OMRequest req, ValidationContext ctx); + public abstract Class getVersionClass(); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestVersionExtractor.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestVersionExtractor.java index 0a2344840cfd..c1eaf3b7c21b 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestVersionExtractor.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestVersionExtractor.java @@ -17,7 +17,7 @@ package org.apache.hadoop.ozone.om.request.validation; import org.apache.hadoop.ozone.ClientVersion; -import org.apache.hadoop.ozone.Version; +import org.apache.hadoop.ozone.Versioned; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature; import org.apache.hadoop.ozone.om.upgrade.OMLayoutVersionManager; @@ -25,10 +25,9 @@ import org.apache.hadoop.ozone.upgrade.LayoutVersionManager; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.EnumSource; import org.junit.jupiter.params.provider.MethodSource; -import java.util.Arrays; -import java.util.function.Function; import java.util.stream.IntStream; import java.util.stream.Stream; @@ -38,37 +37,38 @@ class TestVersionExtractor { - private static Stream layoutValues() { - return Arrays.stream(OMLayoutFeature.values()).map(Arguments::of); - } - - private static Stream clientVersionValues() { - return Stream.of( - Arrays.stream(ClientVersion.values()) - .map(clientVersion -> Arguments.of(clientVersion, clientVersion.version())), - IntStream.range(1, 10) - .mapToObj(delta -> Arguments.of(ClientVersion.FUTURE_VERSION, ClientVersion.CURRENT_VERSION + delta)) - ).flatMap(Function.identity()); + private static Stream futureClientVersionValues() { + return IntStream.range(1, 10).mapToObj(delta -> Arguments.of(ClientVersion.CURRENT_VERSION + delta)); } @ParameterizedTest - @MethodSource("layoutValues") + @EnumSource(OMLayoutFeature.class) void testLayoutVersionExtractor(OMLayoutFeature layoutVersionValue) throws OMException { ValidationContext context = mock(ValidationContext.class); LayoutVersionManager layoutVersionManager = new OMLayoutVersionManager(layoutVersionValue.version()); when(context.versionManager()).thenReturn(layoutVersionManager); - Version version = VersionExtractor.LAYOUT_VERSION_EXTRACTOR.extractVersion(null, context); + Versioned version = VersionExtractor.LAYOUT_VERSION_EXTRACTOR.extractVersion(null, context); assertEquals(layoutVersionValue, version); assertEquals(OMLayoutFeature.class, VersionExtractor.LAYOUT_VERSION_EXTRACTOR.getVersionClass()); } @ParameterizedTest - @MethodSource("clientVersionValues") - void testClientVersionExtractor(ClientVersion expectedClientVersion, int clientVersion) { + @EnumSource(ClientVersion.class) + void testClientVersionExtractor(ClientVersion expectedClientVersion) { OMRequest request = mock(OMRequest.class); - when(request.getVersion()).thenReturn(clientVersion); - Version version = VersionExtractor.CLIENT_VERSION_EXTRACTOR.extractVersion(request, null); + when(request.getVersion()).thenReturn(expectedClientVersion.version()); + Versioned version = VersionExtractor.CLIENT_VERSION_EXTRACTOR.extractVersion(request, null); assertEquals(expectedClientVersion, version); assertEquals(ClientVersion.class, VersionExtractor.CLIENT_VERSION_EXTRACTOR.getVersionClass()); } + + @ParameterizedTest + @MethodSource("futureClientVersionValues") + void testClientVersionExtractorForFutureValues(int clientVersion) { + OMRequest request = mock(OMRequest.class); + when(request.getVersion()).thenReturn(clientVersion); + Versioned version = VersionExtractor.CLIENT_VERSION_EXTRACTOR.extractVersion(request, null); + assertEquals(ClientVersion.FUTURE_VERSION, version); + assertEquals(ClientVersion.class, VersionExtractor.CLIENT_VERSION_EXTRACTOR.getVersionClass()); + } } From ded3450ade16e1414047b81e305cc39f85f56d83 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Wed, 18 Dec 2024 22:27:46 -0800 Subject: [PATCH 25/28] HDDS-11963. Address review comments Change-Id: I9ebc447eb9b60caa69e140b06fe899c58dd712a3 --- .../request/validation/TestVersionExtractor.java | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestVersionExtractor.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestVersionExtractor.java index c1eaf3b7c21b..a3c9c76e70fe 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestVersionExtractor.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestVersionExtractor.java @@ -24,12 +24,8 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.ozone.upgrade.LayoutVersionManager; import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.EnumSource; -import org.junit.jupiter.params.provider.MethodSource; - -import java.util.stream.IntStream; -import java.util.stream.Stream; +import org.junit.jupiter.params.provider.ValueSource; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.mock; @@ -37,10 +33,6 @@ class TestVersionExtractor { - private static Stream futureClientVersionValues() { - return IntStream.range(1, 10).mapToObj(delta -> Arguments.of(ClientVersion.CURRENT_VERSION + delta)); - } - @ParameterizedTest @EnumSource(OMLayoutFeature.class) void testLayoutVersionExtractor(OMLayoutFeature layoutVersionValue) throws OMException { @@ -63,10 +55,10 @@ void testClientVersionExtractor(ClientVersion expectedClientVersion) { } @ParameterizedTest - @MethodSource("futureClientVersionValues") - void testClientVersionExtractorForFutureValues(int clientVersion) { + @ValueSource(ints = {1, 2, 5, 10, 1000, 10000}) + void testClientVersionExtractorForFutureValues(int futureVersion) { OMRequest request = mock(OMRequest.class); - when(request.getVersion()).thenReturn(clientVersion); + when(request.getVersion()).thenReturn(ClientVersion.CURRENT_VERSION + futureVersion); Versioned version = VersionExtractor.CLIENT_VERSION_EXTRACTOR.extractVersion(request, null); assertEquals(ClientVersion.FUTURE_VERSION, version); assertEquals(ClientVersion.class, VersionExtractor.CLIENT_VERSION_EXTRACTOR.getVersionClass()); From 2124f0307b166eb95d613e2000a561a5240812d0 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Thu, 19 Dec 2024 06:24:28 -0800 Subject: [PATCH 26/28] HDDS-11149. Merge with master Change-Id: I55280a1b2fdeb7b6ed202098cc0ca412ee7761cf --- .../java/org/apache/hadoop/ozone/Version.java | 26 ---------- .../request/validation/RegisterValidator.java | 4 +- .../request/validation/ValidatorRegistry.java | 22 ++++---- .../validation/RequestValidations.java | 4 +- .../validation/TestOMValidatorProcessor.java | 52 +++++++++---------- 5 files changed, 41 insertions(+), 67 deletions(-) delete mode 100644 hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/Version.java diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/Version.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/Version.java deleted file mode 100644 index b6dc636e248c..000000000000 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/Version.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.ozone; - - -/** - * Base class defining the version in the entire system. - */ -public interface Version { - int version(); -} diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/RegisterValidator.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/RegisterValidator.java index 1df192ac2e86..d2b3d7b37738 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/RegisterValidator.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/RegisterValidator.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.ozone.request.validation; -import org.apache.hadoop.ozone.Version; +import org.apache.hadoop.ozone.Versioned; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; @@ -28,7 +28,7 @@ /** * Annotations to register a validator. {@link org.apache.ozone.annotations.RegisterValidatorProcessor} * enforces other annotation to have the following methods: - * applyBefore : Returns an enum which implement {@link Version} + * applyBefore : Returns an enum which implement {@link Versioned} * requestType: Returns an Enum value. * processingPhase: Returns {@link RequestProcessingPhase} */ diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/ValidatorRegistry.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/ValidatorRegistry.java index d66e463f1c10..bc4becbd1878 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/ValidatorRegistry.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/ValidatorRegistry.java @@ -19,7 +19,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Sets; import org.apache.commons.lang3.tuple.Pair; -import org.apache.hadoop.ozone.Version; +import org.apache.hadoop.ozone.Versioned; import org.reflections.Reflections; import org.reflections.scanners.Scanners; import org.reflections.util.ClasspathHelper; @@ -56,9 +56,9 @@ public class ValidatorRegistry> { * Based on the afforementioned parameters a complete map is built which stores the validators in a sorted order of * the applyBeforeVersion value of the validator method. * Thus when a request comes with a certain version value, all validators containing `applyBeforeVersion` parameter - * greater than the request versions get triggered. {@link #validationsFor(Enum, RequestProcessingPhase, Version)} + * greater than the request versions get triggered. {@link #validationsFor(Enum, RequestProcessingPhase, Versioned)} */ - private final Map, EnumMap, EnumMap>>> indexedValidatorMap; /** @@ -77,7 +77,7 @@ public class ValidatorRegistry> { */ public ValidatorRegistry(Class requestType, String validatorPackage, - Set> allowedVersionTypes, + Set> allowedVersionTypes, Set allowedProcessingPhases) { this(requestType, ClasspathHelper.forPackage(validatorPackage), allowedVersionTypes, allowedProcessingPhases); } @@ -96,7 +96,7 @@ public ValidatorRegistry(Class requestType, */ public ValidatorRegistry(Class requestType, Collection searchUrls, - Set> allowedVersionTypes, + Set> allowedVersionTypes, Set allowedProcessingPhases) { Class requestArrayClass = (Class) Array.newInstance(requestType, 0) .getClass(); @@ -134,7 +134,7 @@ public ValidatorRegistry(Class requestType, */ public List validationsFor(RequestType requestType, RequestProcessingPhase phase, - List requestVersions) { + List requestVersions) { return requestVersions.stream() .flatMap(requestVersion -> this.validationsFor(requestType, phase, requestVersion).stream()) .distinct().collect(Collectors.toList()); @@ -150,7 +150,7 @@ public List validationsFor(RequestType requestType, * @param requestVersion version extracted corresponding to the request. * @return the list of validation methods that has to run. */ - public List validationsFor(RequestType requestType, + public List validationsFor(RequestType requestType, RequestProcessingPhase phase, V requestVersion) { @@ -189,8 +189,8 @@ private Class getReturnTypeOfAnnotationMethod(Class clz } } - private Version getApplyBeforeVersion(Validator validator) { - return callAnnotationMethod(validator, RegisterValidator.APPLY_BEFORE_METHOD_NAME, Version.class); + private Versioned getApplyBeforeVersion(Validator validator) { + return callAnnotationMethod(validator, RegisterValidator.APPLY_BEFORE_METHOD_NAME, Versioned.class); } private RequestProcessingPhase getRequestPhase(Validator validator) { @@ -234,7 +234,7 @@ private void registerValidator(Class requestType, Class validatorToBeRegistered, Reflections reflections) { Collection methods = reflections.getMethodsAnnotatedWith(validatorToBeRegistered); - Class versionClass = (Class) + Class versionClass = (Class) this.getReturnTypeOfAnnotationMethod(validatorToBeRegistered, RegisterValidator.APPLY_BEFORE_METHOD_NAME); List> sortedMethodsByApplyBeforeVersion = methods.stream() .map(method -> Pair.of(method.getAnnotation(validatorToBeRegistered), method)) @@ -246,7 +246,7 @@ private void registerValidator(Class requestType, for (Pair validatorMethodPair : sortedMethodsByApplyBeforeVersion) { Annotation validator = validatorMethodPair.getKey(); Method method = validatorMethodPair.getValue(); - Version applyBeforeVersion = this.getApplyBeforeVersion(validator); + Versioned applyBeforeVersion = this.getApplyBeforeVersion(validator); RequestProcessingPhase phase = this.getRequestPhase(validator); checkAllowedAnnotationValues(allowedPhases, phase, RegisterValidator.PROCESSING_PHASE_METHOD_NAME, method.getName()); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestValidations.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestValidations.java index 2abc5dd31924..f48b2f7deb16 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestValidations.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestValidations.java @@ -18,7 +18,7 @@ import com.google.common.collect.Sets; import com.google.protobuf.ServiceException; -import org.apache.hadoop.ozone.Version; +import org.apache.hadoop.ozone.Versioned; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; @@ -117,7 +117,7 @@ public OMResponse validateResponse(OMRequest request, OMResponse response) return validatedResponse; } - private List getVersions(OMRequest request) { + private List getVersions(OMRequest request) { return Arrays.stream(VersionExtractor.values()) .map(versionExtractor -> versionExtractor.extractVersion(request, context)) .filter(Objects::nonNull) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestOMValidatorProcessor.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestOMValidatorProcessor.java index b2615851630f..7441c74998c9 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestOMValidatorProcessor.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestOMValidatorProcessor.java @@ -18,7 +18,7 @@ import com.google.testing.compile.Compilation; import com.google.testing.compile.JavaFileObjects; -import org.apache.hadoop.ozone.Version; +import org.apache.hadoop.ozone.Versioned; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; import org.apache.hadoop.ozone.request.validation.RegisterValidator; import org.apache.hadoop.ozone.request.validation.RequestProcessingPhase; @@ -89,7 +89,7 @@ private static Stream annotatedClasses() { @ParameterizedTest @MethodSource("annotatedClasses") - public & Version> void testAnnotationCanOnlyBeAppliedOnMethods(Class annotationClass, + public & Versioned> void testAnnotationCanOnlyBeAppliedOnMethods(Class annotationClass, V version) { for (Annotation a : annotationClass.getAnnotations()) { if (a instanceof Target) { @@ -101,7 +101,7 @@ public & Version> void testAnnotationCanOnlyBeAppliedOnMethod @ParameterizedTest @MethodSource("annotatedClasses") - public & Version> void testACorrectAnnotationSetupForPreProcessCompiles(Class annotationClass, + public & Versioned> void testACorrectAnnotationSetupForPreProcessCompiles(Class annotationClass, V version) { List source = generateSourceOfValidatorMethodWith( annotationOf(preProcess(), aReqType(), annotationClass, version), @@ -115,7 +115,7 @@ public & Version> void testACorrectAnnotationSetupForPreProce @ParameterizedTest @MethodSource("annotatedClasses") - public & Version> void testACorrectAnnotationSetupForPostProcessCompiles(Class annotationClass, + public & Versioned> void testACorrectAnnotationSetupForPostProcessCompiles(Class annotationClass, V version) { List source = generateSourceOfValidatorMethodWith( annotationOf(postProcess(), aReqType(), annotationClass, version), @@ -129,7 +129,7 @@ public & Version> void testACorrectAnnotationSetupForPostProc @ParameterizedTest @MethodSource("annotatedClasses") - public & Version> void testValidatorDoesNotNecessarilyThrowsExceptions(Class annotationClass, + public & Versioned> void testValidatorDoesNotNecessarilyThrowsExceptions(Class annotationClass, V version) { List source = generateSourceOfValidatorMethodWith( annotationOf(preProcess(), aReqType(), annotationClass, version), @@ -143,7 +143,7 @@ public & Version> void testValidatorDoesNotNecessarilyThrowsE @ParameterizedTest @MethodSource("annotatedClasses") - public & Version> void testNonStaticValidatorDoesNotCompile(Class annotationClass, + public & Versioned> void testNonStaticValidatorDoesNotCompile(Class annotationClass, V version) { List source = generateSourceOfValidatorMethodWith( annotationOf(preProcess(), aReqType(), annotationClass, version), @@ -158,7 +158,7 @@ public & Version> void testNonStaticValidatorDoesNotCompile(C @ParameterizedTest @MethodSource("annotatedClasses") - public & Version> void testValidatorMethodCanBeFinal(Class annotationClass, + public & Versioned> void testValidatorMethodCanBeFinal(Class annotationClass, V version) { List source = generateSourceOfValidatorMethodWith( annotationOf(preProcess(), aReqType(), annotationClass, version), @@ -172,7 +172,7 @@ public & Version> void testValidatorMethodCanBeFinal(Class @ParameterizedTest @MethodSource("annotatedClasses") - public & Version> void testValidatorMethodCanBePrivate(Class annotationClass, + public & Versioned> void testValidatorMethodCanBePrivate(Class annotationClass, V version) { List source = generateSourceOfValidatorMethodWith( annotationOf(preProcess(), aReqType(), annotationClass, version), @@ -186,7 +186,7 @@ public & Version> void testValidatorMethodCanBePrivate(Class< @ParameterizedTest @MethodSource("annotatedClasses") - public & Version> void testValidatorMethodCanBeDefaultVisible(Class annotationClass, + public & Versioned> void testValidatorMethodCanBeDefaultVisible(Class annotationClass, V version) { List source = generateSourceOfValidatorMethodWith( annotationOf(preProcess(), aReqType(), annotationClass, version), @@ -200,7 +200,7 @@ public & Version> void testValidatorMethodCanBeDefaultVisible @ParameterizedTest @MethodSource("annotatedClasses") - public & Version> void testValidatorMethodCanBeProtected(Class annotationClass, + public & Versioned> void testValidatorMethodCanBeProtected(Class annotationClass, V version) { List source = generateSourceOfValidatorMethodWith( annotationOf(preProcess(), aReqType(), annotationClass, version), @@ -214,7 +214,7 @@ public & Version> void testValidatorMethodCanBeProtected(Clas @ParameterizedTest @MethodSource("annotatedClasses") - public & Version> void testNotEnoughParametersForPreProcess(Class annotationClass, + public & Versioned> void testNotEnoughParametersForPreProcess(Class annotationClass, V version) { List source = generateSourceOfValidatorMethodWith( annotationOf(preProcess(), aReqType(), annotationClass, version), @@ -230,7 +230,7 @@ public & Version> void testNotEnoughParametersForPreProcess(C @ParameterizedTest @MethodSource("annotatedClasses") - public & Version> void testTooManyParametersForPreProcess(Class annotationClass, + public & Versioned> void testTooManyParametersForPreProcess(Class annotationClass, V version) { List source = generateSourceOfValidatorMethodWith( annotationOf(preProcess(), aReqType(), annotationClass, version), @@ -246,7 +246,7 @@ public & Version> void testTooManyParametersForPreProcess(Cla @ParameterizedTest @MethodSource("annotatedClasses") - public & Version> void testNotEnoughParametersForPostProcess(Class annotationClass, + public & Versioned> void testNotEnoughParametersForPostProcess(Class annotationClass, V version) { List source = generateSourceOfValidatorMethodWith( annotationOf(postProcess(), aReqType(), annotationClass, version), @@ -262,7 +262,7 @@ public & Version> void testNotEnoughParametersForPostProcess( @ParameterizedTest @MethodSource("annotatedClasses") - public & Version> void testTooManyParametersForPostProcess(Class annotationClass, + public & Versioned> void testTooManyParametersForPostProcess(Class annotationClass, V version) { List source = generateSourceOfValidatorMethodWith( annotationOf(postProcess(), aReqType(), annotationClass, version), @@ -279,7 +279,7 @@ public & Version> void testTooManyParametersForPostProcess(Cl @ParameterizedTest @MethodSource("annotatedClasses") - public & Version> void testWrongReturnValueForPreProcess(Class annotationClass, + public & Versioned> void testWrongReturnValueForPreProcess(Class annotationClass, V version) { List source = generateSourceOfValidatorMethodWith( annotationOf(preProcess(), aReqType(), annotationClass, version), @@ -294,7 +294,7 @@ public & Version> void testWrongReturnValueForPreProcess(Clas @ParameterizedTest @MethodSource("annotatedClasses") - public & Version> void testWrongReturnValueForPostProcess(Class annotationClass, + public & Versioned> void testWrongReturnValueForPostProcess(Class annotationClass, V version) { List source = generateSourceOfValidatorMethodWith( annotationOf(postProcess(), aReqType(), annotationClass, version), @@ -309,7 +309,7 @@ public & Version> void testWrongReturnValueForPostProcess(Cla @ParameterizedTest @MethodSource("annotatedClasses") - public & Version> void testWrongFirstArgumentForPreProcess(Class annotationClass, + public & Versioned> void testWrongFirstArgumentForPreProcess(Class annotationClass, V version) { List source = generateSourceOfValidatorMethodWith( annotationOf(preProcess(), aReqType(), annotationClass, version), @@ -324,7 +324,7 @@ public & Version> void testWrongFirstArgumentForPreProcess(Cl @ParameterizedTest @MethodSource("annotatedClasses") - public & Version> void testWrongFirstArgumentForPostProcess(Class annotationClass, + public & Versioned> void testWrongFirstArgumentForPostProcess(Class annotationClass, V version) { List source = generateSourceOfValidatorMethodWith( annotationOf(postProcess(), aReqType(), annotationClass, version), @@ -339,7 +339,7 @@ public & Version> void testWrongFirstArgumentForPostProcess(C @ParameterizedTest @MethodSource("annotatedClasses") - public & Version> void testWrongSecondArgumentForPreProcess(Class annotationClass, + public & Versioned> void testWrongSecondArgumentForPreProcess(Class annotationClass, V version) { List source = generateSourceOfValidatorMethodWith( annotationOf(preProcess(), aReqType(), annotationClass, version), @@ -354,7 +354,7 @@ public & Version> void testWrongSecondArgumentForPreProcess(C @ParameterizedTest @MethodSource("annotatedClasses") - public & Version> void testWrongSecondArgumentForPostProcess(Class annotationClass, + public & Versioned> void testWrongSecondArgumentForPostProcess(Class annotationClass, V version) { List source = generateSourceOfValidatorMethodWith( annotationOf(postProcess(), aReqType(), annotationClass, version), @@ -369,7 +369,7 @@ public & Version> void testWrongSecondArgumentForPostProcess( @ParameterizedTest @MethodSource("annotatedClasses") - public & Version> void testWrongThirdArgumentForPostProcess(Class annotationClass, + public & Versioned> void testWrongThirdArgumentForPostProcess(Class annotationClass, V version) { List source = generateSourceOfValidatorMethodWith( annotationOf(postProcess(), aReqType(), annotationClass, version), @@ -384,7 +384,7 @@ public & Version> void testWrongThirdArgumentForPostProcess(C @ParameterizedTest @MethodSource("annotatedClasses") - public & Version> void testInvalidProcessingPhase(Class annotationClass, + public & Versioned> void testInvalidProcessingPhase(Class annotationClass, V version) { List source = generateSourceOfValidatorMethodWith( annotationOf("INVALID", aReqType(), annotationClass, version), @@ -398,7 +398,7 @@ public & Version> void testInvalidProcessingPhase(Class an @ParameterizedTest @MethodSource("annotatedClasses") - public & Version> void testInvalidClientVersion(Class annotationClass, + public & Versioned> void testInvalidClientVersion(Class annotationClass, V version) { List source = generateSourceOfValidatorMethodWith( annotationOf(RequestProcessingPhase.PRE_PROCESS, aReqType(), annotationClass, null), @@ -412,7 +412,7 @@ public & Version> void testInvalidClientVersion(Class anno @ParameterizedTest @MethodSource("annotatedClasses") - public & Version> void testMultipleErrorMessages(Class annotationClass, + public & Versioned> void testMultipleErrorMessages(Class annotationClass, V version) { List source = generateSourceOfValidatorMethodWith( annotationOf(postProcess(), aReqType(), annotationClass, version), @@ -493,12 +493,12 @@ private List generateSourceOfValidatorMethodWith( return lines; } - private & Version> String annotationOf( + private & Versioned> String annotationOf( RequestProcessingPhase phase, Type reqType, Class annotationClass, V applyBeforeVersion) { return annotationOf(phase.name(), reqType, annotationClass, applyBeforeVersion); } - private & Version> String annotationOf( + private & Versioned> String annotationOf( String phase, Type reqType, Class annotationClass, From 40046632410c05942185ebd02e7e0b5d8fe32fd7 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Fri, 20 Dec 2024 15:03:23 -0800 Subject: [PATCH 27/28] HDDS-11149. Change validator key Change-Id: I33d392ec1f55f26336186551dd4bb81ee1617cff --- .../request/validation/ValidatorRegistry.java | 35 +++++---- .../validation/RequestValidations.java | 14 ++-- .../request/validation/VersionExtractor.java | 13 ++-- .../validation/TestOMValidatorProcessor.java | 4 +- .../validation/TestValidatorRegistry.java | 75 +++++++++++-------- .../validation/TestVersionExtractor.java | 6 +- 6 files changed, 83 insertions(+), 64 deletions(-) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/ValidatorRegistry.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/ValidatorRegistry.java index bc4becbd1878..6d1263bf372f 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/ValidatorRegistry.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/request/validation/ValidatorRegistry.java @@ -56,9 +56,10 @@ public class ValidatorRegistry> { * Based on the afforementioned parameters a complete map is built which stores the validators in a sorted order of * the applyBeforeVersion value of the validator method. * Thus when a request comes with a certain version value, all validators containing `applyBeforeVersion` parameter - * greater than the request versions get triggered. {@link #validationsFor(Enum, RequestProcessingPhase, Versioned)} + * greater than the request versions get triggered. + * {@link #validationsFor(Enum, RequestProcessingPhase, Class, Versioned)} */ - private final Map, EnumMap, EnumMap>>> indexedValidatorMap; /** @@ -70,16 +71,16 @@ public class ValidatorRegistry> { * @param requestType class of request type enum. * @param validatorPackage the main package inside which validatiors should * be discovered. - * @param allowedVersionTypes a set containing the various types of version allowed to be registered. + * @param allowedValidators a set containing the various types of version allowed to be registered. * @param allowedProcessingPhases set of request processing phases which would be allowed to be registered to * registry. * */ public ValidatorRegistry(Class requestType, String validatorPackage, - Set> allowedVersionTypes, + Set> allowedValidators, Set allowedProcessingPhases) { - this(requestType, ClasspathHelper.forPackage(validatorPackage), allowedVersionTypes, allowedProcessingPhases); + this(requestType, ClasspathHelper.forPackage(validatorPackage), allowedValidators, allowedProcessingPhases); } /** @@ -90,13 +91,13 @@ public ValidatorRegistry(Class requestType, * the validator. * @param requestType class of request type enum. * @param searchUrls the path in which the annotated methods are searched. - * @param allowedVersionTypes a set containing the various types of version allowed to be registered. + * @param allowedValidators a set containing the various types of validator annotation allowed to be registered. * @param allowedProcessingPhases set of request processing phases which would be allowed to be registered to * registry. */ public ValidatorRegistry(Class requestType, Collection searchUrls, - Set> allowedVersionTypes, + Set> allowedValidators, Set allowedProcessingPhases) { Class requestArrayClass = (Class) Array.newInstance(requestType, 0) .getClass(); @@ -104,16 +105,14 @@ public ValidatorRegistry(Class requestType, new Reflections(new ConfigurationBuilder().setUrls(ClasspathHelper.forPackage("org.apache.hadoop")) .setScanners(Scanners.TypesAnnotated) .setParallel(true)).getTypesAnnotatedWith(RegisterValidator.class).stream() + .filter(allowedValidators::contains) .filter(annotationClass -> getReturnTypeOfAnnotationMethod((Class) annotationClass, RegisterValidator.REQUEST_TYPE_METHOD_NAME) .equals(requestArrayClass)) - .filter(annotationClass -> allowedVersionTypes.contains(getReturnTypeOfAnnotationMethod( - (Class) annotationClass, - RegisterValidator.APPLY_BEFORE_METHOD_NAME))) .map(annotationClass -> (Class) annotationClass) .collect(Collectors.toSet()); - this.indexedValidatorMap = allowedVersionTypes.stream().collect(ImmutableMap.toImmutableMap(Function.identity(), - versionClass -> new EnumMap<>(requestType))); + this.indexedValidatorMap = allowedValidators.stream().collect(ImmutableMap.toImmutableMap(Function.identity(), + validatorClass -> new EnumMap<>(requestType))); Reflections reflections = new Reflections(new ConfigurationBuilder() .setUrls(searchUrls) .setScanners(Scanners.MethodsAnnotated) @@ -134,9 +133,10 @@ public ValidatorRegistry(Class requestType, */ public List validationsFor(RequestType requestType, RequestProcessingPhase phase, - List requestVersions) { - return requestVersions.stream() - .flatMap(requestVersion -> this.validationsFor(requestType, phase, requestVersion).stream()) + Map, ? extends Versioned> requestVersions) { + return requestVersions.entrySet().stream() + .flatMap(requestVersion -> this.validationsFor(requestType, phase, requestVersion.getKey(), + requestVersion.getValue()).stream()) .distinct().collect(Collectors.toList()); } @@ -152,6 +152,7 @@ public List validationsFor(RequestType requestType, */ public List validationsFor(RequestType requestType, RequestProcessingPhase phase, + Class validatorClass, V requestVersion) { return Optional.ofNullable(this.indexedValidatorMap.get(requestVersion.getClass())) @@ -234,8 +235,6 @@ private void registerValidator(Class requestType, Class validatorToBeRegistered, Reflections reflections) { Collection methods = reflections.getMethodsAnnotatedWith(validatorToBeRegistered); - Class versionClass = (Class) - this.getReturnTypeOfAnnotationMethod(validatorToBeRegistered, RegisterValidator.APPLY_BEFORE_METHOD_NAME); List> sortedMethodsByApplyBeforeVersion = methods.stream() .map(method -> Pair.of(method.getAnnotation(validatorToBeRegistered), method)) .sorted((validatorMethodPair1, validatorMethodPair2) -> @@ -254,7 +253,7 @@ private void registerValidator(Class requestType, method.setAccessible(true); for (RequestType type : types) { EnumMap>> requestMap = - this.indexedValidatorMap.get(versionClass); + this.indexedValidatorMap.get(validatorToBeRegistered); EnumMap> phaseMap = requestMap.computeIfAbsent(type, k -> new EnumMap<>(RequestProcessingPhase.class)); phaseMap.computeIfAbsent(phase, k -> new IndexedItems<>()).add(method, applyBeforeVersion.version()); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestValidations.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestValidations.java index f48b2f7deb16..56a29f27fd2f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestValidations.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestValidations.java @@ -25,13 +25,16 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; import org.apache.hadoop.ozone.request.validation.RequestProcessingPhase; import org.apache.hadoop.ozone.request.validation.ValidatorRegistry; +import org.jgrapht.alg.util.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.lang.annotation.Annotation; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.util.Arrays; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; @@ -67,7 +70,7 @@ public RequestValidations withinContext(ValidationContext validationContext) { public synchronized RequestValidations load() { registry = new ValidatorRegistry<>(Type.class, validationsPackageName, - Arrays.stream(VersionExtractor.values()).map(VersionExtractor::getVersionClass).collect(Collectors.toSet()), + Arrays.stream(VersionExtractor.values()).map(VersionExtractor::getValidatorClass).collect(Collectors.toSet()), ALLOWED_REQUEST_PROCESSING_PHASES); return this; } @@ -117,10 +120,11 @@ public OMResponse validateResponse(OMRequest request, OMResponse response) return validatedResponse; } - private List getVersions(OMRequest request) { + private Map, Versioned> getVersions(OMRequest request) { return Arrays.stream(VersionExtractor.values()) - .map(versionExtractor -> versionExtractor.extractVersion(request, context)) - .filter(Objects::nonNull) - .collect(Collectors.toList()); + .map(versionExtractor -> Pair.of(versionExtractor.getValidatorClass(), + versionExtractor.extractVersion(request, context))) + .filter(pair -> Objects.nonNull(pair.getSecond())) + .collect(Collectors.toMap(pair -> pair.getFirst(), pair -> pair.getSecond())); } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/VersionExtractor.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/VersionExtractor.java index e7acef45bfe2..9c214d3122c3 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/VersionExtractor.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/VersionExtractor.java @@ -18,10 +18,11 @@ import org.apache.hadoop.ozone.ClientVersion; import org.apache.hadoop.ozone.Versioned; -import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.ozone.upgrade.LayoutVersionManager; +import java.lang.annotation.Annotation; + /** * Class to extract version out of OM request. */ @@ -37,8 +38,8 @@ public Versioned extractVersion(OMRequest req, ValidationContext ctx) { } @Override - public Class getVersionClass() { - return OMLayoutFeature.class; + public Class getValidatorClass() { + return OMLayoutVersionValidator.class; } }, @@ -53,11 +54,11 @@ public Versioned extractVersion(OMRequest req, ValidationContext ctx) { } @Override - public Class getVersionClass() { - return ClientVersion.class; + public Class getValidatorClass() { + return OMClientVersionValidator.class; } }; public abstract Versioned extractVersion(OMRequest req, ValidationContext ctx); - public abstract Class getVersionClass(); + public abstract Class getValidatorClass(); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestOMValidatorProcessor.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestOMValidatorProcessor.java index 7441c74998c9..2dd96e9d7d3f 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestOMValidatorProcessor.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestOMValidatorProcessor.java @@ -115,8 +115,8 @@ public & Versioned> void testACorrectAnnotationSetupForPrePro @ParameterizedTest @MethodSource("annotatedClasses") - public & Versioned> void testACorrectAnnotationSetupForPostProcessCompiles(Class annotationClass, - V version) { + public & Versioned> void testACorrectAnnotationSetupForPostProcessCompiles( + Class annotationClass, V version) { List source = generateSourceOfValidatorMethodWith( annotationOf(postProcess(), aReqType(), annotationClass, version), modifiers("public", "static"), diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestValidatorRegistry.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestValidatorRegistry.java index 4e2df3d8f1d0..434322af86df 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestValidatorRegistry.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestValidatorRegistry.java @@ -16,6 +16,7 @@ */ package org.apache.hadoop.ozone.om.request.validation; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Sets; import org.apache.hadoop.ozone.ClientVersion; import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature; @@ -37,7 +38,8 @@ import java.util.Set; import java.util.stream.Collectors; -import static java.util.Arrays.asList; +import static org.apache.hadoop.ozone.om.request.validation.VersionExtractor.CLIENT_VERSION_EXTRACTOR; +import static org.apache.hadoop.ozone.om.request.validation.VersionExtractor.LAYOUT_VERSION_EXTRACTOR; import static org.apache.hadoop.ozone.request.validation.RequestProcessingPhase.POST_PROCESS; import static org.apache.hadoop.ozone.request.validation.RequestProcessingPhase.PRE_PROCESS; import static org.apache.hadoop.ozone.om.request.validation.testvalidatorset1.GeneralValidatorsForTesting.startValidatorTest; @@ -81,9 +83,10 @@ public void tearDown() { public void testNoValidatorsReturnedForEmptyConditionList() { ValidatorRegistry registry = new ValidatorRegistry<>(OzoneManagerProtocolProtos.Type.class, PACKAGE, - Arrays.stream(VersionExtractor.values()).map(VersionExtractor::getVersionClass).collect(Collectors.toSet()), - REQUEST_PROCESSING_PHASES); - List validators = registry.validationsFor(CreateKey, PRE_PROCESS, ClientVersion.CURRENT); + Arrays.stream(VersionExtractor.values()).map(VersionExtractor::getValidatorClass) + .collect(Collectors.toSet()), REQUEST_PROCESSING_PHASES); + List validators = registry.validationsFor(CreateKey, PRE_PROCESS, + CLIENT_VERSION_EXTRACTOR.getValidatorClass(), ClientVersion.CURRENT); assertTrue(validators.isEmpty()); } @@ -92,11 +95,12 @@ public void testNoValidatorsReturnedForEmptyConditionList() { public void testRegistryHasThePreFinalizePreProcessCreateKeyValidator() { ValidatorRegistry registry = new ValidatorRegistry<>( OzoneManagerProtocolProtos.Type.class, PACKAGE, - Arrays.stream(VersionExtractor.values()).map(VersionExtractor::getVersionClass).collect(Collectors.toSet()), + Arrays.stream(VersionExtractor.values()).map(VersionExtractor::getValidatorClass).collect(Collectors.toSet()), REQUEST_PROCESSING_PHASES); List validators = - registry.validationsFor(CreateKey, PRE_PROCESS, asList(ClientVersion.CURRENT, - OMLayoutFeature.FILESYSTEM_SNAPSHOT)); + registry.validationsFor(CreateKey, PRE_PROCESS, + ImmutableMap.of(CLIENT_VERSION_EXTRACTOR.getValidatorClass(), ClientVersion.CURRENT, + LAYOUT_VERSION_EXTRACTOR.getValidatorClass(), OMLayoutFeature.FILESYSTEM_SNAPSHOT)); assertEquals(1, validators.size()); String expectedMethodName = "preProcessCreateKeyQuotaLayoutValidator"; @@ -107,10 +111,11 @@ public void testRegistryHasThePreFinalizePreProcessCreateKeyValidator() { public void testRegistryHasThePreFinalizePostProcessCreateKeyValidator() { ValidatorRegistry registry = new ValidatorRegistry<>(OzoneManagerProtocolProtos.Type.class, PACKAGE, - Arrays.stream(VersionExtractor.values()).map(VersionExtractor::getVersionClass).collect(Collectors.toSet()), + Arrays.stream(VersionExtractor.values()).map(VersionExtractor::getValidatorClass).collect(Collectors.toSet()), REQUEST_PROCESSING_PHASES); List validators = registry.validationsFor(CreateKey, POST_PROCESS, - asList(ClientVersion.CURRENT, OMLayoutFeature.BUCKET_LAYOUT_SUPPORT)); + ImmutableMap.of(CLIENT_VERSION_EXTRACTOR.getValidatorClass(), ClientVersion.CURRENT, + LAYOUT_VERSION_EXTRACTOR.getValidatorClass(), OMLayoutFeature.BUCKET_LAYOUT_SUPPORT)); assertEquals(1, validators.size()); String expectedMethodName = "postProcessCreateKeyQuotaLayoutValidator"; @@ -121,10 +126,11 @@ public void testRegistryHasThePreFinalizePostProcessCreateKeyValidator() { public void testRegistryHasTheOldClientPreProcessCreateKeyValidator() { ValidatorRegistry registry = new ValidatorRegistry<>(OzoneManagerProtocolProtos.Type.class, PACKAGE, - Arrays.stream(VersionExtractor.values()).map(VersionExtractor::getVersionClass).collect(Collectors.toSet()), + Arrays.stream(VersionExtractor.values()).map(VersionExtractor::getValidatorClass).collect(Collectors.toSet()), REQUEST_PROCESSING_PHASES); List validators = - registry.validationsFor(CreateKey, PRE_PROCESS, ClientVersion.ERASURE_CODING_SUPPORT); + registry.validationsFor(CreateKey, PRE_PROCESS, CLIENT_VERSION_EXTRACTOR.getValidatorClass(), + ClientVersion.ERASURE_CODING_SUPPORT); assertEquals(2, validators.size()); List methodNames = @@ -137,10 +143,10 @@ public void testRegistryHasTheOldClientPreProcessCreateKeyValidator() { public void testRegistryHasTheOldClientPostProcessCreateKeyValidator() { ValidatorRegistry registry = new ValidatorRegistry<>(OzoneManagerProtocolProtos.Type.class, PACKAGE, - Arrays.stream(VersionExtractor.values()).map(VersionExtractor::getVersionClass).collect(Collectors.toSet()), + Arrays.stream(VersionExtractor.values()).map(VersionExtractor::getValidatorClass).collect(Collectors.toSet()), REQUEST_PROCESSING_PHASES); List validators = registry.validationsFor(CreateKey, POST_PROCESS, - ClientVersion.ERASURE_CODING_SUPPORT); + CLIENT_VERSION_EXTRACTOR.getValidatorClass(), ClientVersion.ERASURE_CODING_SUPPORT); assertEquals(2, validators.size()); List methodNames = @@ -153,12 +159,14 @@ public void testRegistryHasTheOldClientPostProcessCreateKeyValidator() { public void testRegistryHasTheMultiPurposePreProcessCreateVolumeValidator() { ValidatorRegistry registry = new ValidatorRegistry<>(OzoneManagerProtocolProtos.Type.class, PACKAGE, - Arrays.stream(VersionExtractor.values()).map(VersionExtractor::getVersionClass).collect(Collectors.toSet()), + Arrays.stream(VersionExtractor.values()).map(VersionExtractor::getValidatorClass).collect(Collectors.toSet()), REQUEST_PROCESSING_PHASES); List preFinalizeValidators = - registry.validationsFor(CreateVolume, PRE_PROCESS, OMLayoutFeature.HSYNC); + registry.validationsFor(CreateVolume, PRE_PROCESS, LAYOUT_VERSION_EXTRACTOR.getValidatorClass(), + OMLayoutFeature.HSYNC); List newClientValidators = - registry.validationsFor(CreateVolume, PRE_PROCESS, ClientVersion.ERASURE_CODING_SUPPORT); + registry.validationsFor(CreateVolume, PRE_PROCESS, CLIENT_VERSION_EXTRACTOR.getValidatorClass(), + ClientVersion.ERASURE_CODING_SUPPORT); assertEquals(1, preFinalizeValidators.size()); assertEquals(1, newClientValidators.size()); @@ -171,12 +179,14 @@ public void testRegistryHasTheMultiPurposePreProcessCreateVolumeValidator() { public void testRegistryHasTheMultiPurposePostProcessCreateVolumeValidator() { ValidatorRegistry registry = new ValidatorRegistry<>( OzoneManagerProtocolProtos.Type.class, PACKAGE, - Arrays.stream(VersionExtractor.values()).map(VersionExtractor::getVersionClass).collect(Collectors.toSet()), + Arrays.stream(VersionExtractor.values()).map(VersionExtractor::getValidatorClass).collect(Collectors.toSet()), REQUEST_PROCESSING_PHASES); List preFinalizeValidators = - registry.validationsFor(CreateVolume, POST_PROCESS, OMLayoutFeature.HSYNC); + registry.validationsFor(CreateVolume, POST_PROCESS, LAYOUT_VERSION_EXTRACTOR.getValidatorClass(), + OMLayoutFeature.HSYNC); List oldClientValidators = - registry.validationsFor(CreateVolume, POST_PROCESS, ClientVersion.ERASURE_CODING_SUPPORT); + registry.validationsFor(CreateVolume, POST_PROCESS, CLIENT_VERSION_EXTRACTOR.getValidatorClass(), + ClientVersion.ERASURE_CODING_SUPPORT); assertEquals(1, preFinalizeValidators.size()); assertEquals(1, oldClientValidators.size()); @@ -189,10 +199,11 @@ public void testRegistryHasTheMultiPurposePostProcessCreateVolumeValidator() { public void testValidatorsAreReturnedForMultiCondition() { ValidatorRegistry registry = new ValidatorRegistry<>( OzoneManagerProtocolProtos.Type.class, PACKAGE, - Arrays.stream(VersionExtractor.values()).map(VersionExtractor::getVersionClass).collect(Collectors.toSet()), + Arrays.stream(VersionExtractor.values()).map(VersionExtractor::getValidatorClass).collect(Collectors.toSet()), REQUEST_PROCESSING_PHASES); List validators = registry.validationsFor(CreateKey, POST_PROCESS, - Arrays.asList(ClientVersion.ERASURE_CODING_SUPPORT, OMLayoutFeature.HSYNC)); + ImmutableMap.of(CLIENT_VERSION_EXTRACTOR.getValidatorClass(), ClientVersion.ERASURE_CODING_SUPPORT, + LAYOUT_VERSION_EXTRACTOR.getValidatorClass(), OMLayoutFeature.HSYNC)); assertEquals(3, validators.size()); List methodNames = @@ -207,10 +218,11 @@ public void testNoValidatorForRequestsAtAllReturnsEmptyList() { ValidatorRegistry registry = new ValidatorRegistry<>( OzoneManagerProtocolProtos.Type.class, PACKAGE_WO_VALIDATORS, - Arrays.stream(VersionExtractor.values()).map(VersionExtractor::getVersionClass).collect(Collectors.toSet()), + Arrays.stream(VersionExtractor.values()).map(VersionExtractor::getValidatorClass).collect(Collectors.toSet()), REQUEST_PROCESSING_PHASES); assertTrue(registry.validationsFor(CreateKey, PRE_PROCESS, - asList(ClientVersion.ERASURE_CODING_SUPPORT, OMLayoutFeature.HSYNC)).isEmpty()); + ImmutableMap.of(CLIENT_VERSION_EXTRACTOR.getValidatorClass(), ClientVersion.ERASURE_CODING_SUPPORT, + LAYOUT_VERSION_EXTRACTOR.getValidatorClass(), OMLayoutFeature.HSYNC)).isEmpty()); } @Test @@ -223,31 +235,34 @@ public void testNoValidatorForConditionReturnsEmptyList() } ValidatorRegistry registry = new ValidatorRegistry<>( OzoneManagerProtocolProtos.Type.class, urlsToUse, - Arrays.stream(VersionExtractor.values()).map(VersionExtractor::getVersionClass).collect(Collectors.toSet()), + Arrays.stream(VersionExtractor.values()).map(VersionExtractor::getValidatorClass).collect(Collectors.toSet()), REQUEST_PROCESSING_PHASES); - assertTrue(registry.validationsFor(CreateKey, PRE_PROCESS, asList(ClientVersion.CURRENT, - OMLayoutFeature.BUCKET_LAYOUT_SUPPORT)).isEmpty()); + assertTrue(registry.validationsFor(CreateKey, PRE_PROCESS, + ImmutableMap.of(CLIENT_VERSION_EXTRACTOR.getValidatorClass(), ClientVersion.CURRENT, + LAYOUT_VERSION_EXTRACTOR.getValidatorClass(), OMLayoutFeature.BUCKET_LAYOUT_SUPPORT)).isEmpty()); } @Test public void testNoDefinedValidationForRequestReturnsEmptyList() { ValidatorRegistry registry = new ValidatorRegistry<>( OzoneManagerProtocolProtos.Type.class, PACKAGE, - Arrays.stream(VersionExtractor.values()).map(VersionExtractor::getVersionClass).collect(Collectors.toSet()), + Arrays.stream(VersionExtractor.values()).map(VersionExtractor::getValidatorClass).collect(Collectors.toSet()), REQUEST_PROCESSING_PHASES); - assertTrue(registry.validationsFor(CreateDirectory, null, ClientVersion.ERASURE_CODING_SUPPORT).isEmpty()); + assertTrue(registry.validationsFor(CreateDirectory, null, CLIENT_VERSION_EXTRACTOR.getValidatorClass(), + ClientVersion.ERASURE_CODING_SUPPORT).isEmpty()); } @Test public void testFutureVersionForRequestReturnsOnlyFutureVersionValidators() { ValidatorRegistry registry = new ValidatorRegistry<>( OzoneManagerProtocolProtos.Type.class, PACKAGE, - Arrays.stream(VersionExtractor.values()).map(VersionExtractor::getVersionClass).collect(Collectors.toSet()), + Arrays.stream(VersionExtractor.values()).map(VersionExtractor::getValidatorClass).collect(Collectors.toSet()), REQUEST_PROCESSING_PHASES); - List validators = registry.validationsFor(CreateKey, PRE_PROCESS, ClientVersion.FUTURE_VERSION); + List validators = registry.validationsFor(CreateKey, PRE_PROCESS, + CLIENT_VERSION_EXTRACTOR.getValidatorClass(), ClientVersion.FUTURE_VERSION); assertEquals(1, validators.size()); List methodNames = diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestVersionExtractor.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestVersionExtractor.java index a3c9c76e70fe..07ed411c1daf 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestVersionExtractor.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestVersionExtractor.java @@ -41,7 +41,7 @@ void testLayoutVersionExtractor(OMLayoutFeature layoutVersionValue) throws OMExc when(context.versionManager()).thenReturn(layoutVersionManager); Versioned version = VersionExtractor.LAYOUT_VERSION_EXTRACTOR.extractVersion(null, context); assertEquals(layoutVersionValue, version); - assertEquals(OMLayoutFeature.class, VersionExtractor.LAYOUT_VERSION_EXTRACTOR.getVersionClass()); + assertEquals(OMLayoutFeature.class, VersionExtractor.LAYOUT_VERSION_EXTRACTOR.getValidatorClass()); } @ParameterizedTest @@ -51,7 +51,7 @@ void testClientVersionExtractor(ClientVersion expectedClientVersion) { when(request.getVersion()).thenReturn(expectedClientVersion.version()); Versioned version = VersionExtractor.CLIENT_VERSION_EXTRACTOR.extractVersion(request, null); assertEquals(expectedClientVersion, version); - assertEquals(ClientVersion.class, VersionExtractor.CLIENT_VERSION_EXTRACTOR.getVersionClass()); + assertEquals(ClientVersion.class, VersionExtractor.CLIENT_VERSION_EXTRACTOR.getValidatorClass()); } @ParameterizedTest @@ -61,6 +61,6 @@ void testClientVersionExtractorForFutureValues(int futureVersion) { when(request.getVersion()).thenReturn(ClientVersion.CURRENT_VERSION + futureVersion); Versioned version = VersionExtractor.CLIENT_VERSION_EXTRACTOR.extractVersion(request, null); assertEquals(ClientVersion.FUTURE_VERSION, version); - assertEquals(ClientVersion.class, VersionExtractor.CLIENT_VERSION_EXTRACTOR.getVersionClass()); + assertEquals(ClientVersion.class, VersionExtractor.CLIENT_VERSION_EXTRACTOR.getValidatorClass()); } } From 56e95eb267fd903f97801fed037f623ecd5456c4 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Fri, 20 Dec 2024 15:28:27 -0800 Subject: [PATCH 28/28] HDDS-11149. Fix test case Change-Id: I866f87f947db6306e0c145b83180ece7e458ee07 --- .../validation/TestVersionExtractor.java | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestVersionExtractor.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestVersionExtractor.java index 479e8e072d82..5b3c0660a7cf 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestVersionExtractor.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestVersionExtractor.java @@ -16,6 +16,7 @@ */ package org.apache.hadoop.ozone.om.request.validation; +import com.google.common.collect.ImmutableMap; import org.apache.hadoop.ozone.ClientVersion; import org.apache.hadoop.ozone.Versioned; import org.apache.hadoop.ozone.om.exceptions.OMException; @@ -23,10 +24,14 @@ import org.apache.hadoop.ozone.om.upgrade.OMLayoutVersionManager; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.ozone.upgrade.LayoutVersionManager; +import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; import org.junit.jupiter.params.provider.ValueSource; +import java.lang.annotation.Annotation; +import java.util.Map; + import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -41,7 +46,6 @@ void testLayoutVersionExtractor(OMLayoutFeature layoutVersionValue) throws OMExc when(context.versionManager()).thenReturn(layoutVersionManager); Versioned version = VersionExtractor.LAYOUT_VERSION_EXTRACTOR.extractVersion(null, context); assertEquals(layoutVersionValue, version); - assertEquals(OMLayoutVersionValidator.class, VersionExtractor.LAYOUT_VERSION_EXTRACTOR.getValidatorClass()); } @ParameterizedTest @@ -51,7 +55,6 @@ void testClientVersionExtractor(ClientVersion expectedClientVersion) { when(request.getVersion()).thenReturn(expectedClientVersion.version()); Versioned version = VersionExtractor.CLIENT_VERSION_EXTRACTOR.extractVersion(request, null); assertEquals(expectedClientVersion, version); - assertEquals(OMClientVersionValidator.class, VersionExtractor.CLIENT_VERSION_EXTRACTOR.getValidatorClass()); } @ParameterizedTest @@ -61,6 +64,15 @@ void testClientVersionExtractorForFutureValues(int futureVersion) { when(request.getVersion()).thenReturn(ClientVersion.CURRENT_VERSION + futureVersion); Versioned version = VersionExtractor.CLIENT_VERSION_EXTRACTOR.extractVersion(request, null); assertEquals(ClientVersion.FUTURE_VERSION, version); - assertEquals(OMClientVersionValidator.class, VersionExtractor.CLIENT_VERSION_EXTRACTOR.getValidatorClass()); + } + + @Test + void testGetValidatorClass() { + Map> expectedValidatorClasses = + ImmutableMap.of(VersionExtractor.CLIENT_VERSION_EXTRACTOR, OMClientVersionValidator.class, + VersionExtractor.LAYOUT_VERSION_EXTRACTOR, OMLayoutVersionValidator.class); + for (VersionExtractor extractor : VersionExtractor.values()) { + assertEquals(expectedValidatorClasses.get(extractor), extractor.getValidatorClass()); + } } }