diff --git a/dev-support/annotations/pom.xml b/dev-support/annotations/pom.xml new file mode 100644 index 000000000000..e65400285b3c --- /dev/null +++ b/dev-support/annotations/pom.xml @@ -0,0 +1,114 @@ + + + + 4.0.0 + + org.apache.ozone + ozone-annotation-processing + 1.3.0-SNAPSHOT + Apache Ozone annotation processing tools for validating custom + annotations at compile time. + + Apache Ozone Annotation Processing + jar + + + UTF-8 + UTF-8 + 3.1.12 + 3.1.0 + 8.29 + + + + + com.github.spotbugs + spotbugs + ${spotbugs.version} + provided + + + + + + + + com.github.spotbugs + spotbugs-maven-plugin + ${spotbugs.version} + + 1024 + true + true + + + + org.apache.maven.plugins + maven-checkstyle-plugin + ${maven-checkstyle-plugin.version} + + + com.puppycrawl.tools + checkstyle + ${checkstyle.version} + + + + ../../hadoop-hdds/dev-support/checkstyle/checkstyle.xml + ../../hadoop-hdds/dev-support/checkstyle/suppressions.xml + true + false + ${project.build.directory}/test/checkstyle-errors.xml + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.9.0 + + 1.8 + 1.8 + + + + default-compile + + -proc:none + + org/apache/ozone/annotations/** + + + + + + + + + \ No newline at end of file diff --git a/dev-support/annotations/src/main/java/org/apache/ozone/annotations/RequestFeatureValidatorProcessor.java b/dev-support/annotations/src/main/java/org/apache/ozone/annotations/RequestFeatureValidatorProcessor.java new file mode 100644 index 000000000000..830706e9a540 --- /dev/null +++ b/dev-support/annotations/src/main/java/org/apache/ozone/annotations/RequestFeatureValidatorProcessor.java @@ -0,0 +1,289 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.ozone.annotations; + +import javax.annotation.processing.AbstractProcessor; +import javax.annotation.processing.RoundEnvironment; +import javax.annotation.processing.SupportedAnnotationTypes; +import javax.annotation.processing.SupportedSourceVersion; +import javax.lang.model.SourceVersion; +import javax.lang.model.element.AnnotationMirror; +import javax.lang.model.element.AnnotationValue; +import javax.lang.model.element.AnnotationValueVisitor; +import javax.lang.model.element.Element; +import javax.lang.model.element.ElementKind; +import javax.lang.model.element.ExecutableElement; +import javax.lang.model.element.Modifier; +import javax.lang.model.element.TypeElement; +import javax.lang.model.element.VariableElement; +import javax.lang.model.type.ExecutableType; +import javax.lang.model.type.TypeMirror; +import javax.lang.model.util.SimpleAnnotationValueVisitor8; +import javax.tools.Diagnostic; +import java.util.List; +import java.util.Map.Entry; +import java.util.Set; + +/** + * This class is an annotation processor that is hooked into the java compiler + * and is used to validate the RequestFeatureValidator annotations in the + * codebase, to ensure that the annotated methods have the proper signature and + * return type. + * + * The module is compiled in a different execution via Maven before anything + * else is compiled, and then javac picks this class up as an annotation + * processor from the classpath via a ServiceLoader, based on the + * META-INF/services/javax.annotation.processing.Processor file in the module's + * resources folder. + */ +@SupportedAnnotationTypes( + "org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator") +@SupportedSourceVersion(SourceVersion.RELEASE_8) +public class RequestFeatureValidatorProcessor extends AbstractProcessor { + + public static final String ERROR_CONDITION_IS_EMPTY = + "RequestFeatureValidator has an empty condition list. Please define the" + + " ValidationCondition in which the validator has to be applied."; + public static final String ERROR_ANNOTATED_ELEMENT_IS_NOT_A_METHOD = + "RequestFeatureValidator annotation is not applied to a method."; + public static final String ERROR_VALIDATOR_METHOD_HAS_TO_BE_STATIC = + "Only static methods can be annotated with the RequestFeatureValidator" + + " annotation."; + public static final String ERROR_UNEXPECTED_PARAMETER_COUNT = + "Unexpected parameter count. Expected: %d; found: %d."; + public static final String ERROR_VALIDATOR_METHOD_HAS_TO_RETURN_OMREQUEST = + "Pre-processing validator methods annotated with RequestFeatureValidator" + + " annotation has to return an OMRequest object."; + public static final String ERROR_VALIDATOR_METHOD_HAS_TO_RETURN_OMRESPONSE = + "Post-processing validator methods annotated with RequestFeatureValidator" + + " annotation has to return an OMResponse object."; + public static final String ERROR_FIRST_PARAM_HAS_TO_BE_OMREQUEST = + "First parameter of a RequestFeatureValidator method has to be an" + + " OMRequest object."; + public static final String ERROR_LAST_PARAM_HAS_TO_BE_VALIDATION_CONTEXT = + "Last parameter of a RequestFeatureValidator method has to be" + + " ValidationContext object."; + public static final String ERROR_SECOND_PARAM_HAS_TO_BE_OMRESPONSE = + "Second parameter of a RequestFeatureValidator method has to be an" + + " OMResponse object."; + + public static final String OM_REQUEST_CLASS_NAME = + "org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos" + + ".OMRequest"; + public static final String OM_RESPONSE_CLASS_NAME = + "org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos" + + ".OMResponse"; + public static final String VALIDATION_CONTEXT_CLASS_NAME = + "org.apache.hadoop.ozone.om.request.validation.ValidationContext"; + + public static final String ANNOTATION_SIMPLE_NAME = "RequestFeatureValidator"; + public static final String ANNOTATION_CONDITIONS_PROPERTY_NAME = "conditions"; + public static final String ANNOTATION_PROCESSING_PHASE_PROPERTY_NAME = + "processingPhase"; + + public static final String PROCESSING_PHASE_PRE_PROCESS = "PRE_PROCESS"; + public static final String PROCESSING_PHASE_POST_PROCESS = "POST_PROCESS"; + public static final String ERROR_NO_PROCESSING_PHASE_DEFINED = + "RequestFeatureValidator has an invalid ProcessingPhase defined."; + + @Override + public boolean process(Set annotations, + RoundEnvironment roundEnv) { + for (TypeElement annotation : annotations) { + if (!annotation.getSimpleName().contentEquals(ANNOTATION_SIMPLE_NAME)) { + continue; + } + processElements(roundEnv.getElementsAnnotatedWith(annotation)); + } + return false; + } + + private void processElements(Set annotatedElements) { + for (Element elem : annotatedElements) { + for (AnnotationMirror methodAnnotation : elem.getAnnotationMirrors()) { + validateAnnotatedMethod(elem, methodAnnotation); + } + } + } + + private void validateAnnotatedMethod( + Element elem, AnnotationMirror methodAnnotation) { + boolean isPreprocessor = checkAndEvaluateAnnotation(methodAnnotation); + + checkMethodIsAnnotated(elem); + ensureAnnotatedMethodIsStatic(elem); + ensurePreProcessorReturnsOMReqest((ExecutableElement) elem, isPreprocessor); + ensurePostProcessorReturnsOMResponse( + (ExecutableElement) elem, isPreprocessor); + ensureMethodParameters(elem, isPreprocessor); + } + + private void ensureMethodParameters(Element elem, boolean isPreprocessor) { + List paramTypes = + ((ExecutableType) elem.asType()).getParameterTypes(); + ensureParameterCount(isPreprocessor, paramTypes); + ensureParameterRequirements(paramTypes, 0, OM_REQUEST_CLASS_NAME, + ERROR_FIRST_PARAM_HAS_TO_BE_OMREQUEST); + if (!isPreprocessor) { + ensureParameterRequirements(paramTypes, 1, OM_RESPONSE_CLASS_NAME, + ERROR_SECOND_PARAM_HAS_TO_BE_OMRESPONSE); + } + int contextOrder = isPreprocessor ? 1 : 2; + ensureParameterRequirements(paramTypes, contextOrder, + VALIDATION_CONTEXT_CLASS_NAME, + ERROR_LAST_PARAM_HAS_TO_BE_VALIDATION_CONTEXT); + } + + private void ensureParameterCount(boolean isPreprocessor, + List paramTypes) { + int realParamCount = paramTypes.size(); + int expectedParamCount = isPreprocessor ? 2 : 3; + if (realParamCount != expectedParamCount) { + emitErrorMsg(String.format(ERROR_UNEXPECTED_PARAMETER_COUNT, + expectedParamCount, realParamCount)); + } + } + + private void ensureParameterRequirements( + List paramTypes, + int index, String validationContextClassName, + String errorLastParamHasToBeValidationContext) { + if (paramTypes.size() >= index + 1 && + !paramTypes.get(index).toString().equals(validationContextClassName)) { + emitErrorMsg(errorLastParamHasToBeValidationContext); + } + } + + private void ensurePostProcessorReturnsOMResponse( + ExecutableElement elem, boolean isPreprocessor) { + if (!isPreprocessor && !elem.getReturnType().toString() + .equals(OM_RESPONSE_CLASS_NAME)) { + emitErrorMsg(ERROR_VALIDATOR_METHOD_HAS_TO_RETURN_OMRESPONSE); + } + } + + private void ensurePreProcessorReturnsOMReqest( + ExecutableElement elem, boolean isPreprocessor) { + if (isPreprocessor && !elem.getReturnType().toString() + .equals(OM_REQUEST_CLASS_NAME)) { + emitErrorMsg(ERROR_VALIDATOR_METHOD_HAS_TO_RETURN_OMREQUEST); + } + } + + private void ensureAnnotatedMethodIsStatic(Element elem) { + if (!elem.getModifiers().contains(Modifier.STATIC)) { + emitErrorMsg(ERROR_VALIDATOR_METHOD_HAS_TO_BE_STATIC); + } + } + + private void checkMethodIsAnnotated(Element elem) { + if (elem.getKind() != ElementKind.METHOD) { + emitErrorMsg(ERROR_ANNOTATED_ELEMENT_IS_NOT_A_METHOD); + } + } + + private boolean checkAndEvaluateAnnotation( + AnnotationMirror methodAnnotation) { + boolean isPreprocessor = false; + for (Entry + entry : methodAnnotation.getElementValues().entrySet()) { + + if (hasInvalidValidationCondition(entry)) { + emitErrorMsg(ERROR_CONDITION_IS_EMPTY); + } + if (isProcessingPhaseValue(entry)) { + isPreprocessor = evaluateProcessingPhase(entry); + } + } + return isPreprocessor; + } + + private boolean evaluateProcessingPhase( + Entry entry) { + String procPhase = visit(entry, new ProcessingPhaseVisitor()); + if (procPhase.equals(PROCESSING_PHASE_PRE_PROCESS)) { + return true; + } else if (procPhase.equals(PROCESSING_PHASE_POST_PROCESS)) { + return false; + } + return false; + } + + private boolean isProcessingPhaseValue( + Entry entry) { + return isPropertyNamedAs(entry, ANNOTATION_PROCESSING_PHASE_PROPERTY_NAME); + } + + private boolean hasInvalidValidationCondition( + Entry entry) { + return isPropertyNamedAs(entry, ANNOTATION_CONDITIONS_PROPERTY_NAME) + && !visit(entry, new ConditionValidator()); + } + + private boolean isPropertyNamedAs( + Entry entry, + String simpleName) { + return entry.getKey().getSimpleName().contentEquals(simpleName); + } + + private T visit( + Entry entry, + AnnotationValueVisitor visitor) { + return entry.getValue().accept(visitor, null); + } + + private static class ConditionValidator + extends SimpleAnnotationValueVisitor8 { + + ConditionValidator() { + super(Boolean.TRUE); + } + + @Override + public Boolean visitArray(List vals, + Void unused) { + if (vals.isEmpty()) { + return Boolean.FALSE; + } + return Boolean.TRUE; + } + + } + + private static class ProcessingPhaseVisitor + extends SimpleAnnotationValueVisitor8 { + + ProcessingPhaseVisitor() { + super("UNKNOWN"); + } + + @Override + public String visitEnumConstant(VariableElement c, Void unused) { + if (c.getSimpleName().contentEquals(PROCESSING_PHASE_PRE_PROCESS)) { + return PROCESSING_PHASE_PRE_PROCESS; + } + if (c.getSimpleName().contentEquals(PROCESSING_PHASE_POST_PROCESS)) { + return PROCESSING_PHASE_POST_PROCESS; + } + throw new IllegalStateException(ERROR_NO_PROCESSING_PHASE_DEFINED); + } + } + + private void emitErrorMsg(String s) { + processingEnv.getMessager().printMessage(Diagnostic.Kind.ERROR, s); + } +} diff --git a/dev-support/annotations/src/main/java/org/apache/ozone/annotations/package-info.java b/dev-support/annotations/src/main/java/org/apache/ozone/annotations/package-info.java new file mode 100644 index 000000000000..b13720d5600f --- /dev/null +++ b/dev-support/annotations/src/main/java/org/apache/ozone/annotations/package-info.java @@ -0,0 +1,5 @@ +/** + * Annotation processors used at compile time by the Ozone project to validate + * internal annotations and related code as needed, if needed. + */ +package org.apache.ozone.annotations; \ No newline at end of file diff --git a/dev-support/annotations/src/main/resources/META-INF/services/javax.annotation.processing.Processor b/dev-support/annotations/src/main/resources/META-INF/services/javax.annotation.processing.Processor new file mode 100644 index 000000000000..35b0c1e2f11e --- /dev/null +++ b/dev-support/annotations/src/main/resources/META-INF/services/javax.annotation.processing.Processor @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +org.apache.ozone.annotations.RequestFeatureValidatorProcessor \ No newline at end of file diff --git a/hadoop-hdds/interface-client/pom.xml b/hadoop-hdds/interface-client/pom.xml index 4b97b4539c26..eed1807e6e1e 100644 --- a/hadoop-hdds/interface-client/pom.xml +++ b/hadoop-hdds/interface-client/pom.xml @@ -47,6 +47,11 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> javax.annotation javax.annotation-api + + com.google.code.findbugs + jsr305 + compile + diff --git a/hadoop-ozone/dev-support/checks/rat.sh b/hadoop-ozone/dev-support/checks/rat.sh index 464d636f9381..b67c594cd62d 100755 --- a/hadoop-ozone/dev-support/checks/rat.sh +++ b/hadoop-ozone/dev-support/checks/rat.sh @@ -21,7 +21,9 @@ mkdir -p "$REPORT_DIR" REPORT_FILE="$REPORT_DIR/summary.txt" -cd hadoop-hdds || exit 1 +cd dev-support/annotations || exit 1 +mvn -B -fn org.apache.rat:apache-rat-plugin:0.13:check +cd ../../hadoop-hdds || exit 1 mvn -B -fn org.apache.rat:apache-rat-plugin:0.13:check cd ../hadoop-ozone || exit 1 mvn -B -fn org.apache.rat:apache-rat-plugin:0.13:check diff --git a/hadoop-ozone/dist/src/main/license/jar-report.txt b/hadoop-ozone/dist/src/main/license/jar-report.txt index ec949fdf33a4..da8350d4b77a 100644 --- a/hadoop-ozone/dist/src/main/license/jar-report.txt +++ b/hadoop-ozone/dist/src/main/license/jar-report.txt @@ -183,6 +183,7 @@ share/ozone/lib/opentracing-noop.jar share/ozone/lib/opentracing-tracerresolver.jar share/ozone/lib/opentracing-util.jar share/ozone/lib/osgi-resource-locator.jar +share/ozone/lib/ozone-annotation-processing.jar share/ozone/lib/ozone-client.jar share/ozone/lib/ozone-common.jar share/ozone/lib/ozone-csi.jar diff --git a/hadoop-ozone/ozone-manager/pom.xml b/hadoop-ozone/ozone-manager/pom.xml index cea140b122c3..5e42414ed701 100644 --- a/hadoop-ozone/ozone-manager/pom.xml +++ b/hadoop-ozone/ozone-manager/pom.xml @@ -133,6 +133,11 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> test + + com.google.testing.compile + compile-testing + + org.reflections reflections diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestFeatureValidator.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestFeatureValidator.java new file mode 100644 index 000000000000..5f1f08266b85 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestFeatureValidator.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.ozone.om.request.validation; + +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * An annotation to mark methods that do certain request validations. + * + * The methods annotated with this annotation are collected by the + * {@link ValidatorRegistry} class during the initialization of the server. + * + * The conditions specify the specific use case in which the validator should be + * applied to the request. See {@link ValidationCondition} for more details + * on the specific conditions. + * The validator method should be applied to just one specific request type + * to help keep these methods simple and straightforward. If you want to use + * the same validation for different request types, use inheritance, and + * annotate the override method that just calls super. + * Note that the aim is to have these validators together with the request + * processing code, so the handling of these specific situations are easy to + * find. + * + * The annotated methods have to have a fixed signature. + * A {@link RequestProcessingPhase#PRE_PROCESS} phase method is running before + * the request is processed by the regular code. + * Its signature has to be the following: + * - it has to be static and idempotent + * - it has to have two parameters + * - the first parameter it is an + * {@link + * org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest} + * - the second parameter of type {@link ValidationContext} + * - the method has to return the modified request, or throw a ServiceException + * in case the request is considered to be invalid + * - the method does not need to care about preserving the request it gets, + * the original request is captured and saved by the calling environment. + * + * A {@link RequestProcessingPhase#POST_PROCESS} phase method is running once + * the + * {@link + * org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse} + * is calculated for a given request. + * Its signature has to be the following: + * - it has to be static and idempotent + * - it has three parameters + * - similalry to the pre-processing validators, first parameter is the + * OMRequest, the second parameter is the OMResponse, and the third + * parameter is a ValidationContext. + * - the method has to return the modified OMResponse or throw a + * ServiceException if the request is considered invalid based on response. + * - the method gets the request object that was supplied for the general + * request processing code, not the original request, while it gets a copy + * of the original response object provided by the general request processing + * code. + */ +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.METHOD) +public @interface RequestFeatureValidator { + + /** + * Runtime conditions in which a validator should run. + * @return a list of conditions when the validator should be applied + */ + ValidationCondition[] conditions(); + + /** + * Defines if the validation has to run before or after the general request + * processing. + * @return if this is a pre or post processing validator + */ + RequestProcessingPhase processingPhase(); + + /** + * The type of the request handled by this validator method. + * @return the requestType to whihc the validator shoudl be applied + */ + Type requestType(); + +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestProcessingPhase.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestProcessingPhase.java new file mode 100644 index 000000000000..672156842914 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestProcessingPhase.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.ozone.om.request.validation; + +/** + * Processing phase defines when a request validator should run. + * + * There are two hooking point at the moment, before and after the generic + * request processing code. + */ +public enum RequestProcessingPhase { + PRE_PROCESS, + POST_PROCESS +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestValidations.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestValidations.java new file mode 100644 index 000000000000..fe34b746094f --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestValidations.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.ozone.om.request.validation; + +import com.google.protobuf.ServiceException; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import static org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase.POST_PROCESS; +import static org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase.PRE_PROCESS; + +/** + * Main class to configure and set up and access the request/response + * validation framework. + */ +public class RequestValidations { + + static final Logger LOG = LoggerFactory.getLogger(RequestValidations.class); + private static final String DEFAULT_PACKAGE = "org.apache.hadoop.ozone"; + + private String validationsPackageName = DEFAULT_PACKAGE; + private ValidationContext context = null; + private ValidatorRegistry registry = null; + + public synchronized RequestValidations fromPackage(String packageName) { + validationsPackageName = packageName; + return this; + } + + public RequestValidations withinContext(ValidationContext validationContext) { + this.context = validationContext; + return this; + } + + public synchronized RequestValidations load() { + registry = new ValidatorRegistry(validationsPackageName); + return this; + } + + public OMRequest validateRequest(OMRequest request) throws ServiceException { + List validations = registry.validationsFor( + conditions(request), request.getCmdType(), PRE_PROCESS); + + OMRequest validatedRequest = request.toBuilder().build(); + try { + for (Method m : validations) { + validatedRequest = + (OMRequest) m.invoke(null, validatedRequest, context); + LOG.debug("Running the {} request pre-process validation from {}.{}", + m.getName(), m.getDeclaringClass().getPackage().getName(), + m.getDeclaringClass().getSimpleName()); + } + } catch (IllegalAccessException | InvocationTargetException e) { + throw new ServiceException(e); + } + return validatedRequest; + } + + public OMResponse validateResponse(OMRequest request, OMResponse response) + throws ServiceException { + List validations = registry.validationsFor( + conditions(request), request.getCmdType(), POST_PROCESS); + + OMResponse validatedResponse = response.toBuilder().build(); + try { + for (Method m : validations) { + validatedResponse = + (OMResponse) m.invoke(null, request, response, context); + LOG.debug("Running the {} request post-process validation from {}.{}", + m.getName(), m.getDeclaringClass().getPackage().getName(), + m.getDeclaringClass().getSimpleName()); + } + } catch (InvocationTargetException | IllegalAccessException e) { + throw new ServiceException(e); + } + return validatedResponse; + } + + private List conditions(OMRequest request) { + return Arrays.stream(ValidationCondition.values()) + .filter(c -> c.shouldApply(request, context)) + .collect(Collectors.toList()); + } + +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidationCondition.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidationCondition.java new file mode 100644 index 000000000000..9630500cbd53 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidationCondition.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.ozone.om.request.validation; + +import org.apache.hadoop.ozone.ClientVersion; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; + +/** + * Defines conditions for which validators can be assigned to. + * + * These conditions describe a situation where special request handling might + * be necessary. In these cases we do not override the actual request handling + * code, but based on certain request properties we might reject a request + * early, or we might modify the request, or the response received/sent from/to + * the client. + */ +public enum ValidationCondition { + /** + * Classifies validations that has to run after an upgrade until the cluster + * is in a pre-finalized state. + */ + CLUSTER_NEEDS_FINALIZATION { + @Override + public boolean shouldApply(OMRequest req, ValidationContext ctx) { + return ctx.versionManager().needsFinalization(); + } + }, + + /** + * Classifies validations that has to run, when the client uses an older + * protocol version than the server. + */ + OLDER_CLIENT_REQUESTS { + @Override + public boolean shouldApply(OMRequest req, ValidationContext ctx) { + return req.getVersion() < ClientVersion.CURRENT_VERSION; + } + }; + + public abstract boolean shouldApply(OMRequest req, ValidationContext ctx); +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidationContext.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidationContext.java new file mode 100644 index 000000000000..510d4336983a --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidationContext.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.ozone.om.request.validation; + +import org.apache.hadoop.hdds.annotation.InterfaceStability; +import org.apache.hadoop.ozone.upgrade.LayoutVersionManager; + +/** + * A context that contains useful information for request validator instances. + */ +@InterfaceStability.Evolving +public interface ValidationContext { + + /** + * Gets the {@link LayoutVersionManager} of the service, so that a pre + * finalization validation can check if the layout version it belongs to + * is finalized already or not. + * + * @return the {@link LayoutVersionManager} of the service + */ + LayoutVersionManager versionManager(); + + /** + * Creates a context object based on the given parameters. + * + * @param versionManager the {@link LayoutVersionManager} of the service + * @return the {@link ValidationContext} specified by the parameters. + */ + static ValidationContext of(LayoutVersionManager versionManager) { + + return new ValidationContext() { + @Override + public LayoutVersionManager versionManager() { + return versionManager; + } + }; + } +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidatorRegistry.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidatorRegistry.java new file mode 100644 index 000000000000..72bd0bbfc631 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidatorRegistry.java @@ -0,0 +1,201 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.ozone.om.request.validation; + +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; +import org.reflections.Reflections; +import org.reflections.scanners.MethodAnnotationsScanner; +import org.reflections.util.ClasspathHelper; +import org.reflections.util.ConfigurationBuilder; + +import java.lang.reflect.Method; +import java.net.URL; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.EnumMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase.POST_PROCESS; +import static org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase.PRE_PROCESS; + +/** + * Registry that loads and stores the request validators to be applied by + * a service. + */ +public class ValidatorRegistry { + + private final EnumMap>>> + validators = new EnumMap<>(ValidationCondition.class); + + /** + * Creates a {@link ValidatorRegistry} instance that discovers validation + * methods in the provided package and the packages in the same resource. + * A validation method is recognized by the {@link RequestFeatureValidator} + * annotation that contains important information about how and when to use + * the validator. + * @param validatorPackage the main package inside which validatiors should + * be discovered. + */ + ValidatorRegistry(String validatorPackage) { + this(ClasspathHelper.forPackage(validatorPackage)); + } + + /** + * Creates a {@link ValidatorRegistry} instance that discovers validation + * methods under the provided URL. + * A validation method is recognized by the {@link RequestFeatureValidator} + * annotation that contains important information about how and when to use + * the validator. + * @param searchUrls the path in which the annotated methods are searched. + */ + ValidatorRegistry(Collection searchUrls) { + Reflections reflections = new Reflections(new ConfigurationBuilder() + .setUrls(searchUrls) + .setScanners(new MethodAnnotationsScanner()) + .useParallelExecutor() + ); + + Set describedValidators = + reflections.getMethodsAnnotatedWith(RequestFeatureValidator.class); + initMaps(describedValidators); + } + + /** + * Get the validators that has to be run in the given list of + * {@link ValidationCondition}s, for the given requestType and + * {@link RequestProcessingPhase}. + * + * @param conditions conditions that are present for the request + * @param requestType the type of the protocol message + * @param phase the request processing phase + * @return the list of validation methods that has to run. + */ + List validationsFor( + List conditions, + Type requestType, + RequestProcessingPhase phase) { + + if (conditions.isEmpty() || validators.isEmpty()) { + return Collections.emptyList(); + } + + Set returnValue = + new HashSet<>(validationsFor(conditions.get(0), requestType, phase)); + + for (int i = 1; i < conditions.size(); i++) { + returnValue.addAll(validationsFor(conditions.get(i), requestType, phase)); + } + return new ArrayList<>(returnValue); + } + + /** + * Grabs validations for one particular condition. + * + * @param condition conditions that are present for the request + * @param requestType the type of the protocol message + * @param phase the request processing phase + * @return the list of validation methods that has to run. + */ + private List validationsFor( + ValidationCondition condition, + Type requestType, + RequestProcessingPhase phase) { + + EnumMap>> + requestTypeMap = validators.get(condition); + if (requestTypeMap == null || requestTypeMap.isEmpty()) { + return Collections.emptyList(); + } + + EnumMap> phases = + requestTypeMap.get(requestType); + if (phases == null) { + return Collections.emptyList(); + } + + List validatorsForPhase = phases.get(phase); + if (validatorsForPhase == null) { + return Collections.emptyList(); + } + return validatorsForPhase; + } + + /** + * Initializes the internal request validator store. + * The requests are stored in the following structure: + * - An EnumMap with the {@link ValidationCondition} as the key, and in which + * - values are an EnumMap with the request type as the key, and in which + * - values are Pair of lists, in which + * - left side is the pre-processing validations list + * - right side is the post-processing validations list + * @param describedValidators collection of the annotated methods to process. + */ + void initMaps(Collection describedValidators) { + for (Method m : describedValidators) { + RequestFeatureValidator descriptor = + m.getAnnotation(RequestFeatureValidator.class); + m.setAccessible(true); + + for (ValidationCondition condition : descriptor.conditions()) { + EnumMap>> + requestTypeMap = getAndInitialize( + condition, newTypeMap(), validators); + EnumMap> phases = getAndInitialize( + descriptor.requestType(), newPhaseMap(), requestTypeMap); + if (isPreProcessValidator(descriptor)) { + getAndInitialize(PRE_PROCESS, new ArrayList<>(), phases).add(m); + } else if (isPostProcessValidator(descriptor)) { + getAndInitialize(POST_PROCESS, new ArrayList<>(), phases).add(m); + } + } + } + } + + private EnumMap>> newTypeMap() { + return new EnumMap<>(Type.class); + } + + private EnumMap> newPhaseMap() { + return new EnumMap<>(RequestProcessingPhase.class); + } + + private V getAndInitialize(K key, V defaultValue, Map from) { + V inMapValue = from.get(key); + if (inMapValue == null || !from.containsKey(key)) { + from.put(key, defaultValue); + return defaultValue; + } + return inMapValue; + } + + private boolean isPreProcessValidator(RequestFeatureValidator descriptor) { + return descriptor.processingPhase() + .equals(PRE_PROCESS); + } + + private boolean isPostProcessValidator(RequestFeatureValidator descriptor) { + return descriptor.processingPhase() + .equals(POST_PROCESS); + } + +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/package-info.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/package-info.java new file mode 100644 index 000000000000..e82bab7307a0 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/package-info.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +/** + * Request's feature validation handling. + * + * This package holds facilities to add new situation specific behaviour to + * request handling without cluttering the basic logic of the request handler + * code. + * + * Typical use case scenarios, that we had in mind during the design: + * - during an upgrade, in the pre-finalized state certain request types are + * to be rejected based on provided properties of the request not based on the + * request type + * - a client connects to the server but uses an older version of the protocol + * - a client connects to the server but uses a newer version of the protocol + * - the code can handle certain checks that have to run all the time, but at + * first we do not see a general use case that we would pull in immediately. + * These are the current + * {@link org.apache.hadoop.ozone.om.request.validation.ValidationCondition}s + * but this list might be extended later on if we see other use cases. + * + * The system uses a reflection based discovery to find methods that are + * annotated with the + * {@link org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator} + * annotation. + * This annotation is used to specify the condition in which a certain validator + * has to be used, the request type to which the validation should be applied, + * and the request processing phase in which we apply the validation. + * + * One validator can be applied in multiple + * {@link org.apache.hadoop.ozone.om.request.validation.ValidationCondition} + * but a validator has to handle strictly just one + * {@link org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type + * }. + * The main reason to avoid validating multiple request types with the same + * validator, is that these validators have to be simple methods without state + * any complex validation has to happen in the reql request handling. + * In these validators we need to ensure that in the given condition the request + * is rejected with a proper message, or rewritten to the proper format if for + * example we want to handle an old request with a new server, but we need some + * additional values set to something default, while in the meantime we want to + * add meaning to a null value from newer clients. + * + * In general, it is a good practice to have the request handling code, and the + * validations tied together in one class. + */ +package org.apache.hadoop.ozone.om.request.validation; \ No newline at end of file diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java index ef8206937ec8..06aa6c546a3b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java @@ -38,6 +38,8 @@ import org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer.RaftServerStatus; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; import org.apache.hadoop.ozone.om.request.OMClientRequest; +import org.apache.hadoop.ozone.om.request.validation.RequestValidations; +import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; @@ -60,6 +62,9 @@ public class OzoneManagerProtocolServerSideTranslatorPB implements OzoneManagerProtocolPB { private static final Logger LOG = LoggerFactory .getLogger(OzoneManagerProtocolServerSideTranslatorPB.class); + private static final String OM_REQUESTS_PACKAGE = + "org.apache.hadoop.ozone"; + private final OzoneManagerRatisServer omRatisServer; private final RequestHandler handler; private final boolean isRatisEnabled; @@ -68,6 +73,7 @@ public class OzoneManagerProtocolServerSideTranslatorPB implements private final AtomicLong transactionIndex; private final OzoneProtocolMessageDispatcher dispatcher; + private final RequestValidations requestValidations; /** * Constructs an instance of the server handler. @@ -110,19 +116,28 @@ public OzoneManagerProtocolServerSideTranslatorPB( this.omRatisServer = ratisServer; dispatcher = new OzoneProtocolMessageDispatcher<>("OzoneProtocol", metrics, LOG, OMPBHelper::processForDebug, OMPBHelper::processForDebug); + // TODO: make this injectable for testing... + requestValidations = + new RequestValidations() + .fromPackage(OM_REQUESTS_PACKAGE) + .withinContext( + ValidationContext.of(ozoneManager.getVersionManager())) + .load(); } /** - * Submit requests to Ratis server for OM HA implementation. - * TODO: Once HA is implemented fully, we should have only one server side - * translator for OM protocol. + * Submit mutating requests to Ratis server in OM, and process read requests. */ @Override public OMResponse submitRequest(RpcController controller, OMRequest request) throws ServiceException { + OMRequest validatedRequest = requestValidations.validateRequest(request); - return dispatcher.processRequest(request, this::processRequest, + OMResponse response = + dispatcher.processRequest(validatedRequest, this::processRequest, request.getCmdType(), request.getTraceID()); + + return requestValidations.validateResponse(request, response); } private OMResponse processRequest(OMRequest request) throws diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestRequestFeatureValidatorProcessor.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestRequestFeatureValidatorProcessor.java new file mode 100644 index 000000000000..57b369d71bb1 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestRequestFeatureValidatorProcessor.java @@ -0,0 +1,524 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.ozone.om.request.validation; + +import com.google.testing.compile.Compilation; +import com.google.testing.compile.JavaFileObjects; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; +import org.apache.ozone.annotations.RequestFeatureValidatorProcessor; +import org.junit.Test; + +import java.lang.annotation.Annotation; +import java.lang.annotation.ElementType; +import java.lang.annotation.Target; +import java.util.ArrayList; +import java.util.List; + +import static com.google.testing.compile.CompilationSubject.assertThat; +import static com.google.testing.compile.Compiler.javac; +import static org.apache.ozone.annotations.RequestFeatureValidatorProcessor.ERROR_CONDITION_IS_EMPTY; +import static org.apache.ozone.annotations.RequestFeatureValidatorProcessor.ERROR_FIRST_PARAM_HAS_TO_BE_OMREQUEST; +import static org.apache.ozone.annotations.RequestFeatureValidatorProcessor.ERROR_LAST_PARAM_HAS_TO_BE_VALIDATION_CONTEXT; +import static org.apache.ozone.annotations.RequestFeatureValidatorProcessor.ERROR_SECOND_PARAM_HAS_TO_BE_OMRESPONSE; +import static org.apache.ozone.annotations.RequestFeatureValidatorProcessor.ERROR_UNEXPECTED_PARAMETER_COUNT; +import static org.apache.ozone.annotations.RequestFeatureValidatorProcessor.ERROR_VALIDATOR_METHOD_HAS_TO_BE_STATIC; +import static org.apache.ozone.annotations.RequestFeatureValidatorProcessor.ERROR_VALIDATOR_METHOD_HAS_TO_RETURN_OMREQUEST; +import static org.apache.ozone.annotations.RequestFeatureValidatorProcessor.ERROR_VALIDATOR_METHOD_HAS_TO_RETURN_OMRESPONSE; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; + +/** + * Compile tests against the annotation processor for the + * {@link RequestFeatureValidator} annotation. + * + * The processor should ensure the method signatures and return values, based + * on annotation arguments provided. + */ +public class TestRequestFeatureValidatorProcessor { + + private static final String CLASSNAME = "Validation"; + + @Test + public void testAnnotationCanOnlyBeAppliedOnMethods() { + Class c = RequestFeatureValidator.class; + for (Annotation a : c.getAnnotations()) { + if (a instanceof Target) { + assertEquals(1, ((Target) a).value().length); + assertSame(((Target) a).value()[0], ElementType.METHOD); + } + } + } + + @Test + public void testACorrectAnnotationSetupForPreProcessCompiles() { + List source = generateSourceOfValidatorMethodWith( + annotationOf(someConditions(), preProcess(), aReqType()), + modifiers("public", "static"), + returnValue("OMRequest"), + parameters("OMRequest rq", "ValidationContext ctx"), + exceptions("ServiceException")); + + assertThat(compile(source)).succeeded(); + } + + @Test + public void testACorrectAnnotationSetupForPostProcessCompiles() { + List source = generateSourceOfValidatorMethodWith( + annotationOf(someConditions(), postProcess(), aReqType()), + modifiers("public", "static"), + returnValue("OMResponse"), + parameters("OMRequest rq", "OMResponse rp", "ValidationContext ctx"), + exceptions("ServiceException")); + + assertThat(compile(source)).succeeded(); + } + + @Test + public void testValidatorDoesNotNecessarilyThrowsExceptions() { + List source = generateSourceOfValidatorMethodWith( + annotationOf(someConditions(), preProcess(), aReqType()), + modifiers("public", "static"), + returnValue("OMRequest"), + parameters("OMRequest rq", "ValidationContext ctx"), + exceptions()); + + assertThat(compile(source)).succeeded(); + } + + @Test + public void testNonStaticValidatorDoesNotCompile() { + List source = generateSourceOfValidatorMethodWith( + annotationOf(someConditions(), preProcess(), aReqType()), + modifiers("public"), + returnValue("OMRequest"), + parameters("OMRequest rq", "ValidationContext ctx"), + exceptions()); + + assertThat(compile(source)) + .hadErrorContaining(ERROR_VALIDATOR_METHOD_HAS_TO_BE_STATIC); + } + + @Test + public void testValidatorMethodCanBeFinal() { + List source = generateSourceOfValidatorMethodWith( + annotationOf(someConditions(), preProcess(), aReqType()), + modifiers("public", "static", "final"), + returnValue("OMRequest"), + parameters("OMRequest rq", "ValidationContext ctx"), + exceptions()); + + assertThat(compile(source)).succeeded(); + } + + @Test + public void testValidatorMethodCanBePrivate() { + List source = generateSourceOfValidatorMethodWith( + annotationOf(someConditions(), preProcess(), aReqType()), + modifiers("private", "static"), + returnValue("OMRequest"), + parameters("OMRequest rq", "ValidationContext ctx"), + exceptions()); + + assertThat(compile(source)).succeeded(); + } + + @Test + public void testValidatorMethodCanBeDefaultVisible() { + List source = generateSourceOfValidatorMethodWith( + annotationOf(someConditions(), preProcess(), aReqType()), + modifiers("static"), + returnValue("OMRequest"), + parameters("OMRequest rq", "ValidationContext ctx"), + exceptions()); + + assertThat(compile(source)).succeeded(); + } + + @Test + public void testValidatorMethodCanBeProtected() { + List source = generateSourceOfValidatorMethodWith( + annotationOf(someConditions(), preProcess(), aReqType()), + modifiers("protected", "static"), + returnValue("OMRequest"), + parameters("OMRequest rq", "ValidationContext ctx"), + exceptions()); + + assertThat(compile(source)).succeeded(); + } + + @Test + public void testEmptyValidationConditionListDoesNotCompile() { + List source = generateSourceOfValidatorMethodWith( + annotationOf(emptyConditions(), preProcess(), aReqType()), + modifiers("public", "static"), + returnValue("OMRequest"), + parameters("OMRequest rq", "ValidationContext ctx"), + exceptions()); + + assertThat(compile(source)).hadErrorContaining(ERROR_CONDITION_IS_EMPTY); + } + + @Test + public void testNotEnoughParametersForPreProcess() { + List source = generateSourceOfValidatorMethodWith( + annotationOf(someConditions(), preProcess(), aReqType()), + modifiers("public", "static"), + returnValue("OMRequest"), + parameters("OMRequest rq"), + exceptions()); + + assertThat(compile(source)) + .hadErrorContaining( + String.format(ERROR_UNEXPECTED_PARAMETER_COUNT, 2, 1)); + } + + @Test + public void testTooManyParametersForPreProcess() { + List source = generateSourceOfValidatorMethodWith( + annotationOf(someConditions(), preProcess(), aReqType()), + modifiers("public", "static"), + returnValue("OMRequest"), + parameters("OMRequest rq", "OMResponse rp", "ValidationContext ctx"), + exceptions()); + + assertThat(compile(source)) + .hadErrorContaining( + String.format(ERROR_UNEXPECTED_PARAMETER_COUNT, 2, 3)); + } + + @Test + public void testNotEnoughParametersForPostProcess() { + List source = generateSourceOfValidatorMethodWith( + annotationOf(someConditions(), postProcess(), aReqType()), + modifiers("public", "static"), + returnValue("OMResponse"), + parameters("OMRequest rq", "OMResponse rp"), + exceptions()); + + assertThat(compile(source)) + .hadErrorContaining( + String.format(ERROR_UNEXPECTED_PARAMETER_COUNT, 3, 2)); + } + + @Test + public void testTooManyParametersForPostProcess() { + List source = generateSourceOfValidatorMethodWith( + annotationOf(someConditions(), postProcess(), aReqType()), + modifiers("public", "static"), + returnValue("OMResponse"), + parameters("OMRequest rq", "OMResponse rp", "ValidationContext ctx", + "String name"), + exceptions()); + + assertThat(compile(source)) + .hadErrorContaining( + String.format(ERROR_UNEXPECTED_PARAMETER_COUNT, 3, 4)); + } + + @Test + public void testWrongReturnValueForPreProcess() { + List source = generateSourceOfValidatorMethodWith( + annotationOf(someConditions(), preProcess(), aReqType()), + modifiers("public", "static"), + returnValue("String"), + parameters("OMRequest rq", "ValidationContext ctx"), + exceptions()); + + assertThat(compile(source)) + .hadErrorContaining(ERROR_VALIDATOR_METHOD_HAS_TO_RETURN_OMREQUEST); + } + + @Test + public void testWrongReturnValueForPostProcess() { + List source = generateSourceOfValidatorMethodWith( + annotationOf(someConditions(), postProcess(), aReqType()), + modifiers("public", "static"), + returnValue("String"), + parameters("OMRequest rq", "OMResponse rp", "ValidationContext ctx"), + exceptions()); + + assertThat(compile(source)) + .hadErrorContaining(ERROR_VALIDATOR_METHOD_HAS_TO_RETURN_OMRESPONSE); + } + + @Test + public void testWrongFirstArgumentForPreProcess() { + List source = generateSourceOfValidatorMethodWith( + annotationOf(someConditions(), preProcess(), aReqType()), + modifiers("public", "static"), + returnValue("OMRequest"), + parameters("String rq", "ValidationContext ctx"), + exceptions()); + + assertThat(compile(source)) + .hadErrorContaining(ERROR_FIRST_PARAM_HAS_TO_BE_OMREQUEST); + } + + @Test + public void testWrongFirstArgumentForPostProcess() { + List source = generateSourceOfValidatorMethodWith( + annotationOf(someConditions(), postProcess(), aReqType()), + modifiers("public", "static"), + returnValue("OMResponse"), + parameters("String rq", "OMResponse rp", "ValidationContext ctx"), + exceptions()); + + assertThat(compile(source)) + .hadErrorContaining(ERROR_FIRST_PARAM_HAS_TO_BE_OMREQUEST); + } + + @Test + public void testWrongSecondArgumentForPreProcess() { + List source = generateSourceOfValidatorMethodWith( + annotationOf(someConditions(), preProcess(), aReqType()), + modifiers("public", "static"), + returnValue("OMRequest"), + parameters("OMRequest rq", "String ctx"), + exceptions()); + + assertThat(compile(source)) + .hadErrorContaining(ERROR_LAST_PARAM_HAS_TO_BE_VALIDATION_CONTEXT); + } + + @Test + public void testWrongSecondArgumentForPostProcess() { + List source = generateSourceOfValidatorMethodWith( + annotationOf(someConditions(), postProcess(), aReqType()), + modifiers("public", "static"), + returnValue("OMResponse"), + parameters("OMRequest rq", "String rp", "ValidationContext ctx"), + exceptions()); + + assertThat(compile(source)) + .hadErrorContaining(ERROR_SECOND_PARAM_HAS_TO_BE_OMRESPONSE); + } + + @Test + public void testWrongThirdArgumentForPostProcess() { + List source = generateSourceOfValidatorMethodWith( + annotationOf(someConditions(), postProcess(), aReqType()), + modifiers("public", "static"), + returnValue("OMResponse"), + parameters("OMRequest rq", "OMResponse rp", "String ctx"), + exceptions()); + + assertThat(compile(source)) + .hadErrorContaining(ERROR_LAST_PARAM_HAS_TO_BE_VALIDATION_CONTEXT); + } + + @Test + public void testInvalidProcessingPhase() { + List source = generateSourceOfValidatorMethodWith( + annotationOf(someConditions(), "INVALID", aReqType()), + modifiers("public", "static"), + returnValue("OMResponse"), + parameters("OMRequest rq", "OMResponse rp", "ValidationContext ctx"), + exceptions("ServiceException")); + + assertThat(compile(source)).failed(); + } + + @Test + public void testMultipleErrorMessages() { + List source = generateSourceOfValidatorMethodWith( + annotationOf(emptyConditions(), postProcess(), aReqType()), + modifiers(), + returnValue("String"), + parameters("String rq", "int rp", "String ctx"), + exceptions()); + + Compilation compilation = compile(source); + assertThat(compilation).hadErrorContaining(ERROR_CONDITION_IS_EMPTY); + assertThat(compilation) + .hadErrorContaining(ERROR_VALIDATOR_METHOD_HAS_TO_BE_STATIC); + assertThat(compilation) + .hadErrorContaining(ERROR_VALIDATOR_METHOD_HAS_TO_RETURN_OMRESPONSE); + assertThat(compilation) + .hadErrorContaining(ERROR_FIRST_PARAM_HAS_TO_BE_OMREQUEST); + assertThat(compilation) + .hadErrorContaining(ERROR_SECOND_PARAM_HAS_TO_BE_OMRESPONSE); + assertThat(compilation) + .hadErrorContaining(ERROR_LAST_PARAM_HAS_TO_BE_VALIDATION_CONTEXT); + } + + private Compilation compile(List source) { + Compilation c = javac() + .withProcessors(new RequestFeatureValidatorProcessor()) + .compile(JavaFileObjects.forSourceLines(CLASSNAME, source)); + c.diagnostics().forEach(System.out::println); + return c; + } + + private ValidationCondition[] someConditions() { + return + new ValidationCondition[] {ValidationCondition.OLDER_CLIENT_REQUESTS}; + } + + private ValidationCondition[] emptyConditions() { + return new ValidationCondition[] {}; + } + + private RequestProcessingPhase preProcess() { + return RequestProcessingPhase.PRE_PROCESS; + } + + private RequestProcessingPhase postProcess() { + return RequestProcessingPhase.POST_PROCESS; + } + + private Type aReqType() { + return Type.CreateVolume; + } + + private String returnValue(String retVal) { + return retVal; + } + + private String[] parameters(String... params) { + return params; + } + + private String[] modifiers(String... modifiers) { + return modifiers; + } + + private String[] exceptions(String... exceptions) { + return exceptions; + } + + private List generateSourceOfValidatorMethodWith( + String annotation, + String[] modifiers, + String returnType, + String[] paramspecs, + String[] exceptions) { + List lines = new ArrayList<>(allImports()); + lines.add(""); + lines.add("public class " + CLASSNAME + " {"); + lines.add(""); + lines.add(" " + annotation); + StringBuilder signature = + buildMethodSignature(modifiers, returnType, paramspecs, exceptions); + lines.add(signature.toString()); + lines.add(" return null;"); + lines.add(" }"); + lines.add("}"); + lines.add(""); + lines.stream() + .filter(s -> !s.startsWith("import")) + .forEach(System.out::println); + return lines; + } + + private String annotationOf( + ValidationCondition[] conditions, + RequestProcessingPhase phase, + Type reqType) { + return annotationOf(conditions, phase.name(), reqType); + } + + private String annotationOf( + ValidationCondition[] conditions, + String phase, + Type reqType) { + StringBuilder annotation = new StringBuilder(); + annotation.append("@RequestFeatureValidator("); + StringBuilder conditionsArray = new StringBuilder(); + conditionsArray.append("conditions = { "); + if (conditions.length > 0) { + for (ValidationCondition condition : conditions) { + conditionsArray.append(condition.name()).append(", "); + } + annotation + .append(conditionsArray.substring(0, conditionsArray.length() - 2)); + } else { + annotation.append(conditionsArray); + } + annotation.append(" }"); + annotation.append(", processingPhase = ").append(phase); + annotation.append(", requestType = ").append(reqType.name()); + annotation.append(" )"); + return annotation.toString(); + } + + private List allImports() { + List imports = new ArrayList<>(); + imports.add("import org.apache.hadoop.ozone.om.request.validation" + + ".RequestFeatureValidator;"); + imports.add("import org.apache.hadoop.ozone.protocol.proto" + + ".OzoneManagerProtocolProtos.OMRequest;"); + imports.add("import org.apache.hadoop.ozone.protocol.proto" + + ".OzoneManagerProtocolProtos.OMResponse;"); + imports.add("import org.apache.hadoop.ozone.om.request.validation" + + ".ValidationContext;"); + imports.add("import com.google.protobuf.ServiceException;"); + for (ValidationCondition condition : ValidationCondition.values()) { + imports.add("import static org.apache.hadoop.ozone.om.request.validation" + + ".ValidationCondition." + condition.name() + ";"); + } + for (RequestProcessingPhase phase : RequestProcessingPhase.values()) { + imports.add("import static org.apache.hadoop.ozone.om.request.validation" + + ".RequestProcessingPhase." + phase.name() + ";"); + } + for (Type reqType : Type.values()) { + imports.add("import static org.apache.hadoop.ozone.protocol.proto" + + ".OzoneManagerProtocolProtos.Type." + reqType.name() + ";"); + } + return imports; + } + + private StringBuilder buildMethodSignature( + String[] modifiers, String returnType, + String[] paramspecs, String[] exceptions) { + StringBuilder signature = new StringBuilder(); + signature.append(" "); + for (String modifier : modifiers) { + signature.append(modifier).append(" "); + } + signature.append(returnType).append(" "); + signature.append("validatorMethod("); + signature.append(createParameterList(paramspecs)); + signature.append(") "); + signature.append(createThrowsClause(exceptions)); + return signature.append(" {"); + } + + private String createParameterList(String[] paramSpecs) { + if (paramSpecs == null || paramSpecs.length == 0) { + return ""; + } + StringBuilder parameters = new StringBuilder(); + for (String paramSpec : paramSpecs) { + parameters.append(paramSpec).append(", "); + } + return parameters.substring(0, parameters.length() - 2); + } + + private String createThrowsClause(String[] exceptions) { + StringBuilder throwsClause = new StringBuilder(); + if (exceptions != null && exceptions.length > 0) { + throwsClause.append(" throws "); + StringBuilder exceptionList = new StringBuilder(); + for (String exception : exceptions) { + exceptionList.append(exception).append(", "); + } + throwsClause + .append(exceptionList.substring(0, exceptionList.length() - 2)); + } + return throwsClause.toString(); + } +} diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestRequestValidations.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestRequestValidations.java new file mode 100644 index 000000000000..4508eafd0459 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestRequestValidations.java @@ -0,0 +1,349 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.ozone.om.request.validation; + +import com.google.protobuf.ServiceException; +import org.apache.hadoop.ozone.ClientVersion; +import org.apache.hadoop.ozone.om.request.validation.testvalidatorset1.GeneralValidatorsForTesting; +import org.apache.hadoop.ozone.om.request.validation.testvalidatorset1.GeneralValidatorsForTesting.ValidationListener; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; +import org.apache.hadoop.ozone.upgrade.LayoutVersionManager; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.apache.hadoop.ozone.om.request.validation.ValidationContext.of; +import static org.apache.hadoop.ozone.om.request.validation.testvalidatorset1.GeneralValidatorsForTesting.startValidatorTest; +import static org.apache.hadoop.ozone.om.request.validation.testvalidatorset1.GeneralValidatorsForTesting.finishValidatorTest; +import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.OK; +import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.CreateKey; +import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.DeleteKeys; +import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.RenameKey; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Testing the RequestValidations class that is used to run the validation for + * any given request that arrives to OzoneManager. + */ +public class TestRequestValidations { + private static final String PACKAGE = + "org.apache.hadoop.ozone.om.request.validation.testvalidatorset1"; + + private static final String PACKAGE_WO_VALIDATORS = + "org.apache.hadoop.hdds.annotation"; + + private final ValidationListenerImpl validationListener = + new ValidationListenerImpl(); + + @Before + public void setup() { + startValidatorTest(); + validationListener.attach(); + } + + @After + public void tearDown() { + validationListener.detach(); + finishValidatorTest(); + } + + @Test(expected = NullPointerException.class) + public void testUsingRegistryWithoutLoading() throws ServiceException { + new RequestValidations() + .fromPackage(PACKAGE) + .withinContext(of(aFinalizedVersionManager())) + .validateRequest(aCreateKeyRequest(currentClientVersion())); + } + + @Test(expected = NullPointerException.class) + public void testUsingRegistryWithoutContext() throws ServiceException { + new RequestValidations() + .fromPackage(PACKAGE) + .load() + .validateRequest(aCreateKeyRequest(currentClientVersion())); + } + + @Test + public void testUsingRegistryWithoutPackage() throws ServiceException { + new RequestValidations() + .withinContext(of(aFinalizedVersionManager())) + .load() + .validateRequest(aCreateKeyRequest(currentClientVersion())); + + validationListener.assertNumOfEvents(0); + } + + @Test + public void testNoPreValidationsWithoutValidationMethods() + throws ServiceException { + int omVersion = 0; + ValidationContext ctx = of(aFinalizedVersionManager()); + RequestValidations validations = loadEmptyValidations(ctx); + + validations.validateRequest(aCreateKeyRequest(omVersion)); + + validationListener.assertNumOfEvents(0); + } + + @Test + public void testNoPostValidationsWithoutValidationMethods() + throws ServiceException { + ValidationContext ctx = of(aFinalizedVersionManager()); + RequestValidations validations = loadEmptyValidations(ctx); + + validations.validateResponse( + aCreateKeyRequest(currentClientVersion()), aCreateKeyResponse()); + + validationListener.assertNumOfEvents(0); + } + + @Test + public void testNoPreValidationsRunningForRequestTypeWithoutValidators() + throws ServiceException { + ValidationContext ctx = of(aFinalizedVersionManager()); + RequestValidations validations = loadValidations(ctx); + + validations.validateRequest(aRenameKeyRequest(currentClientVersion())); + + validationListener.assertNumOfEvents(0); + } + + @Test + public void testNoPostValidationsAreRunningForRequestTypeWithoutValidators() + throws ServiceException { + ValidationContext ctx = of(aFinalizedVersionManager()); + RequestValidations validations = loadValidations(ctx); + + validations.validateResponse( + aRenameKeyRequest(currentClientVersion()), aRenameKeyResponse()); + + validationListener.assertNumOfEvents(0); + } + + @Test + public void testPreProcessorExceptionHandling() { + ValidationContext ctx = of(aFinalizedVersionManager()); + RequestValidations validations = loadValidations(ctx); + + try { + validations.validateRequest(aDeleteKeysRequest(olderClientVersion())); + fail("ServiceException was expected but was not thrown."); + } catch (ServiceException ignored) { } + + validationListener.assertNumOfEvents(1); + validationListener.assertExactListOfValidatorsCalled( + "throwingPreProcessValidator"); + } + + @Test + public void testPostProcessorExceptionHandling() { + ValidationContext ctx = of(aFinalizedVersionManager()); + RequestValidations validations = loadValidations(ctx); + + try { + validations.validateResponse( + aDeleteKeysRequest(olderClientVersion()), aDeleteKeysResponse()); + fail("ServiceException was expected but was not thrown."); + } catch (ServiceException ignored) { } + + validationListener.assertNumOfEvents(1); + validationListener.assertExactListOfValidatorsCalled( + "throwingPostProcessValidator"); + } + + @Test + public void testOldClientConditionIsRecognizedAndPreValidatorsApplied() + throws ServiceException { + ValidationContext ctx = of(aFinalizedVersionManager()); + RequestValidations validations = loadValidations(ctx); + + validations.validateRequest(aCreateKeyRequest(olderClientVersion())); + + validationListener.assertNumOfEvents(1); + validationListener.assertExactListOfValidatorsCalled( + "oldClientPreProcessCreateKeyValidator"); + } + + @Test + public void testOldClientConditionIsRecognizedAndPostValidatorsApplied() + throws ServiceException { + ValidationContext ctx = of(aFinalizedVersionManager()); + RequestValidations validations = loadValidations(ctx); + + validations.validateResponse( + aCreateKeyRequest(olderClientVersion()), aCreateKeyResponse()); + + validationListener.assertNumOfEvents(2); + validationListener.assertExactListOfValidatorsCalled( + "oldClientPostProcessCreateKeyValidator", + "oldClientPostProcessCreateKeyValidator2"); + } + + @Test + public void testPreFinalizedWithOldClientConditionPreProcValidatorsApplied() + throws ServiceException { + ValidationContext ctx = of(anUnfinalizedVersionManager()); + RequestValidations validations = loadValidations(ctx); + + validations.validateRequest(aCreateKeyRequest(olderClientVersion())); + + validationListener.assertNumOfEvents(2); + validationListener.assertExactListOfValidatorsCalled( + "preFinalizePreProcessCreateKeyValidator", + "oldClientPreProcessCreateKeyValidator"); + } + + @Test + public void testPreFinalizedWithOldClientConditionPostProcValidatorsApplied() + throws ServiceException { + ValidationContext ctx = of(anUnfinalizedVersionManager()); + RequestValidations validations = loadValidations(ctx); + + validations.validateResponse( + aCreateKeyRequest(olderClientVersion()), aCreateKeyResponse()); + + validationListener.assertNumOfEvents(3); + validationListener.assertExactListOfValidatorsCalled( + "preFinalizePostProcessCreateKeyValidator", + "oldClientPostProcessCreateKeyValidator", + "oldClientPostProcessCreateKeyValidator2"); + } + + private RequestValidations loadValidations(ValidationContext ctx) { + return new RequestValidations() + .fromPackage(PACKAGE) + .withinContext(ctx) + .load(); + } + + private RequestValidations loadEmptyValidations(ValidationContext ctx) { + return new RequestValidations() + .fromPackage(PACKAGE_WO_VALIDATORS) + .withinContext(ctx) + .load(); + } + + private int olderClientVersion() { + return ClientVersion.CURRENT_VERSION - 1; + } + + private int currentClientVersion() { + return ClientVersion.CURRENT_VERSION; + } + + private OMRequest aCreateKeyRequest(int clientVersion) { + return aRequest(CreateKey, clientVersion); + } + + private OMRequest aDeleteKeysRequest(int clientVersion) { + return aRequest(DeleteKeys, clientVersion); + } + + private OMRequest aRenameKeyRequest(int clientVersion) { + return aRequest(RenameKey, clientVersion); + } + + private OMRequest aRequest(Type type, int clientVersion) { + return OMRequest.newBuilder() + .setVersion(clientVersion) + .setCmdType(type) + .setClientId("TestClient") + .build(); + } + + private OMResponse aCreateKeyResponse() { + return aResponse(CreateKey); + } + + private OMResponse aDeleteKeysResponse() { + return aResponse(DeleteKeys); + } + + private OMResponse aRenameKeyResponse() { + return aResponse(RenameKey); + } + + private OMResponse aResponse(Type type) { + return OMResponse.newBuilder() + .setCmdType(type) + .setStatus(OK) + .build(); + } + + private LayoutVersionManager aFinalizedVersionManager() { + LayoutVersionManager vm = mock(LayoutVersionManager.class); + when(vm.needsFinalization()).thenReturn(false); + return vm; + } + + private LayoutVersionManager anUnfinalizedVersionManager() { + LayoutVersionManager vm = mock(LayoutVersionManager.class); + when(vm.needsFinalization()).thenReturn(true); + return vm; + } + + private static class ValidationListenerImpl implements ValidationListener { + private List calledMethods = new ArrayList<>(); + + @Override + public void validationCalled(String calledMethodName) { + calledMethods.add(calledMethodName); + } + + public void attach() { + GeneralValidatorsForTesting.addListener(this); + } + + public void detach() { + GeneralValidatorsForTesting.removeListener(this); + reset(); + } + + public void reset() { + calledMethods = new ArrayList<>(); + } + + public void assertExactListOfValidatorsCalled(String... methodNames) { + List calls = new ArrayList<>(calledMethods); + for (String methodName : methodNames) { + if (!calls.remove(methodName)) { + fail("Expected method call for " + methodName + " did not happened."); + } + } + if (!calls.isEmpty()) { + fail("Some of the methods were not called." + + "Missing calls for: " + calls); + } + } + + public void assertNumOfEvents(int count) { + if (calledMethods.size() != count) { + fail("Unexpected validation call count." + + " Expected: " + count + "; Happened: " + calledMethods.size()); + } + } + } +} + + diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestValidatorRegistry.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestValidatorRegistry.java new file mode 100644 index 000000000000..e11fa5160d5c --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestValidatorRegistry.java @@ -0,0 +1,215 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.ozone.om.request.validation; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.reflections.util.ClasspathHelper; + +import java.lang.reflect.Method; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; + +import static java.util.Arrays.asList; +import static java.util.Collections.emptyList; +import static org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase.POST_PROCESS; +import static org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase.PRE_PROCESS; +import static org.apache.hadoop.ozone.om.request.validation.ValidationCondition.CLUSTER_NEEDS_FINALIZATION; +import static org.apache.hadoop.ozone.om.request.validation.ValidationCondition.OLDER_CLIENT_REQUESTS; +import static org.apache.hadoop.ozone.om.request.validation.testvalidatorset1.GeneralValidatorsForTesting.startValidatorTest; +import static org.apache.hadoop.ozone.om.request.validation.testvalidatorset1.GeneralValidatorsForTesting.finishValidatorTest; +import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.CreateDirectory; +import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.CreateKey; +import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.CreateVolume; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Validator registry tests. + * For validator method declarations see the GeneralValidatorsForTesting + * and ValidatorsForOnlyNewClientValidations (in ../avalidation2) classes. + */ +public class TestValidatorRegistry { + private static final String PACKAGE = + "org.apache.hadoop.ozone.om.request.validation.testvalidatorset1"; + + private static final String PACKAGE2 = + "org.apache.hadoop.ozone.om.request.validation.testvalidatorset2"; + + private static final String PACKAGE_WO_VALIDATORS = + "org.apache.hadoop.hdds.annotation"; + + @Before + public void setup() { + startValidatorTest(); + } + + @After + public void tearDown() { + finishValidatorTest(); + } + + @Test + public void testNoValidatorsReturnedForEmptyConditionList() { + ValidatorRegistry registry = new ValidatorRegistry(PACKAGE); + List validators = + registry.validationsFor(emptyList(), CreateKey, PRE_PROCESS); + + assertTrue(validators.isEmpty()); + } + + @Test + public void testRegistryHasThePreFinalizePreProcessCreateKeyValidator() { + ValidatorRegistry registry = new ValidatorRegistry(PACKAGE); + List validators = + registry.validationsFor( + asList(CLUSTER_NEEDS_FINALIZATION), CreateKey, PRE_PROCESS); + + assertEquals(1, validators.size()); + String expectedMethodName = "preFinalizePreProcessCreateKeyValidator"; + assertEquals(expectedMethodName, validators.get(0).getName()); + } + + @Test + public void testRegistryHasThePreFinalizePostProcessCreateKeyValidator() { + ValidatorRegistry registry = new ValidatorRegistry(PACKAGE); + List validators = + registry.validationsFor( + asList(CLUSTER_NEEDS_FINALIZATION), CreateKey, POST_PROCESS); + + assertEquals(1, validators.size()); + String expectedMethodName = "preFinalizePostProcessCreateKeyValidator"; + assertEquals(expectedMethodName, validators.get(0).getName()); + } + + @Test + public void testRegistryHasTheOldClientPreProcessCreateKeyValidator() { + ValidatorRegistry registry = new ValidatorRegistry(PACKAGE); + List validators = + registry.validationsFor( + asList(OLDER_CLIENT_REQUESTS), CreateKey, PRE_PROCESS); + + assertEquals(2, validators.size()); + List methodNames = + validators.stream().map(Method::getName).collect(Collectors.toList()); + assertTrue(methodNames.contains("oldClientPreProcessCreateKeyValidator")); + assertTrue(methodNames.contains("oldClientPreProcessCreateKeyValidator2")); + } + + @Test + public void testRegistryHasTheOldClientPostProcessCreateKeyValidator() { + ValidatorRegistry registry = new ValidatorRegistry(PACKAGE); + List validators = + registry.validationsFor( + asList(OLDER_CLIENT_REQUESTS), CreateKey, POST_PROCESS); + + assertEquals(2, validators.size()); + List methodNames = + validators.stream().map(Method::getName).collect(Collectors.toList()); + assertTrue(methodNames.contains("oldClientPostProcessCreateKeyValidator")); + assertTrue(methodNames.contains("oldClientPostProcessCreateKeyValidator2")); + } + + @Test + public void testRegistryHasTheMultiPurposePreProcessCreateVolumeValidator() { + ValidatorRegistry registry = new ValidatorRegistry(PACKAGE); + List preFinalizeValidators = + registry.validationsFor( + asList(CLUSTER_NEEDS_FINALIZATION), CreateVolume, PRE_PROCESS); + List newClientValidators = + registry.validationsFor( + asList(OLDER_CLIENT_REQUESTS), CreateVolume, PRE_PROCESS); + + assertEquals(1, preFinalizeValidators.size()); + assertEquals(1, newClientValidators.size()); + String expectedMethodName = "multiPurposePreProcessCreateVolumeValidator"; + assertEquals(expectedMethodName, preFinalizeValidators.get(0).getName()); + assertEquals(expectedMethodName, newClientValidators.get(0).getName()); + } + + @Test + public void testRegistryHasTheMultiPurposePostProcessCreateVolumeValidator() { + ValidatorRegistry registry = new ValidatorRegistry(PACKAGE); + List preFinalizeValidators = + registry.validationsFor( + asList(CLUSTER_NEEDS_FINALIZATION), CreateVolume, POST_PROCESS); + List oldClientValidators = + registry.validationsFor( + asList(OLDER_CLIENT_REQUESTS), CreateVolume, POST_PROCESS); + + assertEquals(1, preFinalizeValidators.size()); + assertEquals(1, oldClientValidators.size()); + String expectedMethodName = "multiPurposePostProcessCreateVolumeValidator"; + assertEquals(expectedMethodName, preFinalizeValidators.get(0).getName()); + assertEquals(expectedMethodName, oldClientValidators.get(0).getName()); + } + + @Test + public void testValidatorsAreReturnedForMultiCondition() { + ValidatorRegistry registry = new ValidatorRegistry(PACKAGE); + List validators = + registry.validationsFor( + asList(CLUSTER_NEEDS_FINALIZATION, OLDER_CLIENT_REQUESTS), + CreateKey, POST_PROCESS); + + assertEquals(3, validators.size()); + List methodNames = + validators.stream().map(Method::getName).collect(Collectors.toList()); + assertTrue( + methodNames.contains("preFinalizePostProcessCreateKeyValidator")); + assertTrue( + methodNames.contains("oldClientPostProcessCreateKeyValidator")); + assertTrue( + methodNames.contains("oldClientPostProcessCreateKeyValidator2")); + } + + @Test + public void testNoValidatorForRequestsAtAllReturnsEmptyList() { + ValidatorRegistry registry = new ValidatorRegistry(PACKAGE_WO_VALIDATORS); + + assertTrue(registry.validationsFor( + asList(OLDER_CLIENT_REQUESTS), CreateKey, PRE_PROCESS).isEmpty()); + } + + @Test + public void testNoValidatorForConditionReturnsEmptyList() + throws MalformedURLException { + Collection urls = ClasspathHelper.forPackage(PACKAGE2); + Collection urlsToUse = new ArrayList<>(); + for (URL url : urls) { + urlsToUse.add(new URL(url, PACKAGE2.replaceAll("\\.", "/"))); + } + ValidatorRegistry registry = new ValidatorRegistry(urlsToUse); + + assertTrue(registry.validationsFor( + asList(CLUSTER_NEEDS_FINALIZATION), CreateKey, PRE_PROCESS).isEmpty()); + } + + @Test + public void testNoDefinedValidationForRequestReturnsEmptyList() { + ValidatorRegistry registry = new ValidatorRegistry(PACKAGE); + + assertTrue(registry.validationsFor( + asList(OLDER_CLIENT_REQUESTS), CreateDirectory, null).isEmpty()); + } + +} diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset1/GeneralValidatorsForTesting.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset1/GeneralValidatorsForTesting.java new file mode 100644 index 000000000000..35c3afa4cf96 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset1/GeneralValidatorsForTesting.java @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.ozone.om.request.validation.testvalidatorset1; + +import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; +import org.apache.hadoop.ozone.om.request.validation.TestRequestValidations; +import org.apache.hadoop.ozone.om.request.validation.ValidationContext; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase.POST_PROCESS; +import static org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase.PRE_PROCESS; +import static org.apache.hadoop.ozone.om.request.validation.ValidationCondition.CLUSTER_NEEDS_FINALIZATION; +import static org.apache.hadoop.ozone.om.request.validation.ValidationCondition.OLDER_CLIENT_REQUESTS; +import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.CreateKey; +import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.CreateVolume; +import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.DeleteKeys; + +/** + * Some annotated request validator method, and facilities to help check if + * validations were properly called from tests where applicable. + */ +public final class GeneralValidatorsForTesting { + + /** + * As the validators written here does not override any request or response + * but throw exceptions for specific tests, a test that wants to directly + * use a validator here, has to turn on this boolean, and the method that + * the test relies on has to check for this value. + * + * This is necessary to do not affect other tests that are testing requests + * processing, as for some of those tests this package is on the classpath + * and therefore the annotated validations are loadede for them. + */ + private static boolean validatorTestsRunning = false; + + public static void startValidatorTest() { + validatorTestsRunning = true; + } + + public static void finishValidatorTest() { + validatorTestsRunning = false; + } + + private GeneralValidatorsForTesting() { } + + /** + * Interface to easily add listeners that get notified if a certain validator + * method defined in this class was called. + * + * @see TestRequestValidations for more details on how this intercace is + * being used. + */ + @FunctionalInterface + public interface ValidationListener { + void validationCalled(String calledMethodName); + } + + private static List listeners = new ArrayList<>(); + + public static void addListener(ValidationListener listener) { + listeners.add(listener); + } + + public static void removeListener(ValidationListener listener) { + listeners.remove(listener); + } + + private static void fireValidationEvent(String calledMethodName) { + listeners.forEach(l -> l.validationCalled(calledMethodName)); + } + + @RequestFeatureValidator( + conditions = { CLUSTER_NEEDS_FINALIZATION }, + processingPhase = PRE_PROCESS, + requestType = CreateKey) + public static OMRequest preFinalizePreProcessCreateKeyValidator( + OMRequest req, ValidationContext ctx) { + fireValidationEvent("preFinalizePreProcessCreateKeyValidator"); + return req; + } + + @RequestFeatureValidator( + conditions = { CLUSTER_NEEDS_FINALIZATION }, + processingPhase = POST_PROCESS, + requestType = CreateKey) + public static OMResponse preFinalizePostProcessCreateKeyValidator( + OMRequest req, OMResponse resp, ValidationContext ctx) { + fireValidationEvent("preFinalizePostProcessCreateKeyValidator"); + return resp; + } + + @RequestFeatureValidator( + conditions = { OLDER_CLIENT_REQUESTS }, + processingPhase = PRE_PROCESS, + requestType = CreateKey) + public static OMRequest oldClientPreProcessCreateKeyValidator( + OMRequest req, ValidationContext ctx) { + fireValidationEvent("oldClientPreProcessCreateKeyValidator"); + return req; + } + + @RequestFeatureValidator( + conditions = { OLDER_CLIENT_REQUESTS }, + processingPhase = POST_PROCESS, + requestType = CreateKey) + public static OMResponse oldClientPostProcessCreateKeyValidator( + OMRequest req, OMResponse resp, ValidationContext ctx) { + fireValidationEvent("oldClientPostProcessCreateKeyValidator"); + return resp; + } + + @RequestFeatureValidator( + conditions = { CLUSTER_NEEDS_FINALIZATION, OLDER_CLIENT_REQUESTS }, + processingPhase = PRE_PROCESS, + requestType = CreateVolume) + public static OMRequest multiPurposePreProcessCreateVolumeValidator( + OMRequest req, ValidationContext ctx) { + fireValidationEvent("multiPurposePreProcessCreateVolumeValidator"); + return req; + } + + @RequestFeatureValidator( + conditions = { OLDER_CLIENT_REQUESTS, CLUSTER_NEEDS_FINALIZATION }, + processingPhase = POST_PROCESS, + requestType = CreateVolume) + public static OMResponse multiPurposePostProcessCreateVolumeValidator( + OMRequest req, OMResponse resp, ValidationContext ctx) { + fireValidationEvent("multiPurposePostProcessCreateVolumeValidator"); + return resp; + } + + @RequestFeatureValidator( + conditions = { OLDER_CLIENT_REQUESTS }, + processingPhase = POST_PROCESS, + requestType = CreateKey) + public static OMResponse oldClientPostProcessCreateKeyValidator2( + OMRequest req, OMResponse resp, ValidationContext ctx) { + fireValidationEvent("oldClientPostProcessCreateKeyValidator2"); + return resp; + } + + @RequestFeatureValidator( + conditions = {OLDER_CLIENT_REQUESTS}, + processingPhase = PRE_PROCESS, + requestType = DeleteKeys + ) + public static OMRequest throwingPreProcessValidator( + OMRequest req, ValidationContext ctx) throws IOException { + fireValidationEvent("throwingPreProcessValidator"); + if (validatorTestsRunning) { + throw new IOException("IOException: fail for testing..."); + } + return req; + } + + @RequestFeatureValidator( + conditions = {OLDER_CLIENT_REQUESTS}, + processingPhase = POST_PROCESS, + requestType = DeleteKeys + ) + public static OMResponse throwingPostProcessValidator( + OMRequest req, OMResponse resp, ValidationContext ctx) + throws IOException { + fireValidationEvent("throwingPostProcessValidator"); + if (validatorTestsRunning) { + throw new IOException("IOException: fail for testing..."); + } + return resp; + } + +} diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset2/ValidatorsForOnlyOldClientValidations.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset2/ValidatorsForOnlyOldClientValidations.java new file mode 100644 index 000000000000..447b9ab0260b --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset2/ValidatorsForOnlyOldClientValidations.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.ozone.om.request.validation.testvalidatorset2; + +import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; +import org.apache.hadoop.ozone.om.request.validation.ValidationContext; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; + +import static org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase.PRE_PROCESS; +import static org.apache.hadoop.ozone.om.request.validation.ValidationCondition.OLDER_CLIENT_REQUESTS; +import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.CreateKey; + +/** + * Separate validator methods for a few specific tests that covers cases where + * there are almost no validators added. + */ +public final class ValidatorsForOnlyOldClientValidations { + + private ValidatorsForOnlyOldClientValidations() { } + + @RequestFeatureValidator( + conditions = { OLDER_CLIENT_REQUESTS }, + processingPhase = PRE_PROCESS, + requestType = CreateKey) + public static OMRequest oldClientPreProcessCreateKeyValidator2( + OMRequest req, ValidationContext ctx) { + return req; + } +} diff --git a/pom.xml b/pom.xml index d8a6252e4f77..2d1ef2c52d4a 100644 --- a/pom.xml +++ b/pom.xml @@ -24,6 +24,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs pom + dev-support/annotations hadoop-hdds hadoop-ozone @@ -163,6 +164,10 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs 1.1.1 3.0.0 + 3.1.12 2.1.7 @@ -232,8 +237,16 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs 1.3.1 1.0-beta-1 1.0-alpha-8 + 3.1.2 3.9.1 + 9.3 1200 1.12.124 @@ -1286,6 +1299,12 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs ${mockito2.version} test + + com.google.testing.compile + compile-testing + 0.19 + test + org.objenesis objenesis @@ -1620,6 +1639,13 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs + + + org.apache.ozone + ozone-annotation-processing + ${ozone.version} + +