diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java index 450da59168c7..6c3f09b46bb8 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java @@ -28,6 +28,7 @@ import java.io.IOException; import java.net.URL; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Enumeration; import java.util.HashMap; @@ -45,6 +46,7 @@ import com.google.common.base.Preconditions; import org.apache.ratis.server.RaftServerConfigKeys; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_TAGS_CUSTOM; import static org.apache.hadoop.hdds.ratis.RatisHelper.HDDS_DATANODE_RATIS_PREFIX_KEY; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CONTAINER_COPY_WORKDIR; @@ -54,6 +56,9 @@ @InterfaceAudience.Private public class OzoneConfiguration extends Configuration implements MutableConfigurationSource { + public static final String OZONE_TAGS_SYSTEM_KEY = + "ozone.tags.system"; + static { addDeprecatedKeys(); @@ -303,6 +308,20 @@ public Map getPropsMatchPrefixAndTrimPrefix( return configMap; } + @Override + public void addTags(Properties prop) { + if (!prop.containsKey(OZONE_TAGS_SYSTEM_KEY)) { + String ozoneTags = Arrays.stream(ConfigTag.values()) + .map(Enum::name) + .sorted() + .collect(Collectors.joining(",")); + prop.setProperty(HADOOP_TAGS_CUSTOM, ozoneTags); + prop.setProperty(OZONE_TAGS_SYSTEM_KEY, ozoneTags); + } + + super.addTags(prop); + } + private static void addDeprecatedKeys() { Configuration.addDeprecations(new DeprecationDelta[]{ new DeprecationDelta("ozone.datanode.pipeline.limit", diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java index 5e924be7f9f7..b4e08e885f12 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java @@ -36,8 +36,6 @@ @InterfaceAudience.Public @InterfaceStability.Unstable public final class OzoneConfigKeys { - public static final String OZONE_TAGS_SYSTEM_KEY = - "ozone.tags.system"; public static final String DFS_CONTAINER_IPC_PORT = "dfs.container.ipc"; public static final int DFS_CONTAINER_IPC_PORT_DEFAULT = 9859; diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index f525478e8f27..f093bf7303bf 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -20,9 +20,7 @@ - - - + @@ -1339,17 +1337,6 @@ - - hadoop.tags.custom - OZONE,MANAGEMENT,SECURITY,PERFORMANCE,DEBUG,CLIENT,SERVER,OM,SCM,CRITICAL,RATIS,CONTAINER,REQUIRED,REST,STORAGE,PIPELINE,STANDALONE,S3GATEWAY,RECON - - - - ozone.tags.system - OZONE,MANAGEMENT,SECURITY,PERFORMANCE,DEBUG,CLIENT,SERVER,OM,SCM,CRITICAL,RATIS,CONTAINER,REQUIRED,REST,STORAGE,PIPELINE,STANDALONE,S3GATEWAY,TOKEN,TLS,RECON - - - hdds.rest.rest-csrf.enabled false @@ -2728,7 +2715,7 @@ ozone.s3g.http.auth.type simple - S3G, SECURITY, KERBEROS + S3GATEWAY, SECURITY, KERBEROS simple or kerberos. If kerberos is set, Kerberos SPNEOGO will be used for http authentication. @@ -3416,7 +3403,7 @@ ozone.audit.log.debug.cmd.list.dnaudit - DN + DATANODE A comma separated list of Datanode commands that are written to the DN audit logs only if the audit log level is debug. Ex: "CREATE_CONTAINER,READ_CONTAINER,UPDATE_CONTAINER". diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/TestOzoneConfiguration.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/TestOzoneConfiguration.java index fe83cf170549..3f580a1a1bc0 100644 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/TestOzoneConfiguration.java +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/TestOzoneConfiguration.java @@ -33,6 +33,8 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; /** * Test class for OzoneConfiguration. @@ -257,6 +259,20 @@ public void postConstructValidation() { () -> ozoneConfiguration.getObject(SimpleConfiguration.class)); } + @ParameterizedTest + @EnumSource + void tagIsRecognized(ConfigTag tag) { + OzoneConfiguration subject = new OzoneConfiguration(); + Assertions.assertTrue(subject.isPropertyTag(tag.name()), + () -> tag + " should be recognized as config tag"); + } + + @Test + void unknownTag() { + OzoneConfiguration subject = new OzoneConfiguration(); + Assertions.assertFalse(subject.isPropertyTag("not-a-tag")); + } + private void appendProperty(BufferedWriter out, String name, String val) throws IOException { this.appendProperty(out, name, val, false); diff --git a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigTag.java b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigTag.java index 3728a0b1f590..24feb69389b1 100644 --- a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigTag.java +++ b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigTag.java @@ -18,34 +18,39 @@ package org.apache.hadoop.hdds.conf; /** - * Available config tags. - *

- * Note: the values are defined in ozone-default.xml by hadoop.tags.custom. + * The definitive list of configuration tags. Used as a source to define the + * value of {@code hadoop.tags.custom} and {@code ozone.tags.system}. */ public enum ConfigTag { - OZONE, - MANAGEMENT, - SECURITY, - PERFORMANCE, - DEBUG, + ACL, + BALANCER, CLIENT, - SERVER, - OM, - SCM, - CRITICAL, - RATIS, CONTAINER, - REQUIRED, - REST, - STORAGE, - PIPELINE, - STANDALONE, - S3GATEWAY, DATANODE, - RECON, + DATASTREAM, + DEBUG, DELETION, + DEPRECATED, + FREON, HA, - BALANCER, + HDDS, + KERBEROS, + MANAGEMENT, + OM, + OPERATION, + OZONE, + OZONEFS, + PERFORMANCE, + PIPELINE, + RATIS, + RECON, + REQUIRED, + S3GATEWAY, + SCM, + SECURITY, + STORAGE, + TLS, + TOKEN, UPGRADE, - DATASTREAM + X509 } diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/HddsConfServlet.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/HddsConfServlet.java index 368197a27642..ca966911e868 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/HddsConfServlet.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/HddsConfServlet.java @@ -34,7 +34,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.gson.Gson; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_TAGS_SYSTEM_KEY; +import static org.apache.hadoop.hdds.conf.OzoneConfiguration.OZONE_TAGS_SYSTEM_KEY; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java index fbd33d0d6dd1..8129bf0c03a1 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java @@ -57,7 +57,6 @@ public void initializeMemberVariables() { }; errorIfMissingConfigProps = true; errorIfMissingXmlProps = true; - xmlPropsToSkipCompare.add("hadoop.tags.custom"); xmlPropsToSkipCompare.add("ozone.om.nodes.EXAMPLEOMSERVICEID"); xmlPropsToSkipCompare.add("ozone.om.decommissioned.nodes" + ".EXAMPLEOMSERVICEID");