From 76da80361d4691b042bc7da50fa4f4d5c7b67677 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Tue, 7 Mar 2023 09:49:00 +0100 Subject: [PATCH 1/7] HDDS-8091. Generate list of config tags from ConfigTag enum --- .../hadoop/hdds/conf/OzoneConfiguration.java | 17 +++++++++++++++++ .../common/src/main/resources/ozone-default.xml | 11 ----------- .../hdds/conf/TestOzoneConfiguration.java | 16 ++++++++++++++++ 3 files changed, 33 insertions(+), 11 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java index 450da59168c7..d549430a07c1 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java @@ -28,6 +28,7 @@ import java.io.IOException; import java.net.URL; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Enumeration; import java.util.HashMap; @@ -45,8 +46,10 @@ import com.google.common.base.Preconditions; import org.apache.ratis.server.RaftServerConfigKeys; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_TAGS_CUSTOM; import static org.apache.hadoop.hdds.ratis.RatisHelper.HDDS_DATANODE_RATIS_PREFIX_KEY; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CONTAINER_COPY_WORKDIR; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_TAGS_SYSTEM_KEY; /** * Configuration for ozone. @@ -303,6 +306,20 @@ public Map getPropsMatchPrefixAndTrimPrefix( return configMap; } + @Override + public void addTags(Properties prop) { + if (!prop.containsKey(OZONE_TAGS_SYSTEM_KEY)) { + String ozoneTags = Arrays.stream(ConfigTag.values()) + .map(Enum::name) + .sorted() + .collect(Collectors.joining(",")); + prop.setProperty(HADOOP_TAGS_CUSTOM, ozoneTags); + prop.setProperty(OZONE_TAGS_SYSTEM_KEY, ozoneTags); + } + + super.addTags(prop); + } + private static void addDeprecatedKeys() { Configuration.addDeprecations(new DeprecationDelta[]{ new DeprecationDelta("ozone.datanode.pipeline.limit", diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index f525478e8f27..f8f928497a78 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -1339,17 +1339,6 @@ - - hadoop.tags.custom - OZONE,MANAGEMENT,SECURITY,PERFORMANCE,DEBUG,CLIENT,SERVER,OM,SCM,CRITICAL,RATIS,CONTAINER,REQUIRED,REST,STORAGE,PIPELINE,STANDALONE,S3GATEWAY,RECON - - - - ozone.tags.system - OZONE,MANAGEMENT,SECURITY,PERFORMANCE,DEBUG,CLIENT,SERVER,OM,SCM,CRITICAL,RATIS,CONTAINER,REQUIRED,REST,STORAGE,PIPELINE,STANDALONE,S3GATEWAY,TOKEN,TLS,RECON - - - hdds.rest.rest-csrf.enabled false diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/TestOzoneConfiguration.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/TestOzoneConfiguration.java index fe83cf170549..3f580a1a1bc0 100644 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/TestOzoneConfiguration.java +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/TestOzoneConfiguration.java @@ -33,6 +33,8 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; /** * Test class for OzoneConfiguration. @@ -257,6 +259,20 @@ public void postConstructValidation() { () -> ozoneConfiguration.getObject(SimpleConfiguration.class)); } + @ParameterizedTest + @EnumSource + void tagIsRecognized(ConfigTag tag) { + OzoneConfiguration subject = new OzoneConfiguration(); + Assertions.assertTrue(subject.isPropertyTag(tag.name()), + () -> tag + " should be recognized as config tag"); + } + + @Test + void unknownTag() { + OzoneConfiguration subject = new OzoneConfiguration(); + Assertions.assertFalse(subject.isPropertyTag("not-a-tag")); + } + private void appendProperty(BufferedWriter out, String name, String val) throws IOException { this.appendProperty(out, name, val, false); From e460320891d045f832815a14f0db8ad036147537 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Tue, 7 Mar 2023 09:52:08 +0100 Subject: [PATCH 2/7] Update class comment in ConfigTag --- .../src/main/java/org/apache/hadoop/hdds/conf/ConfigTag.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigTag.java b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigTag.java index 3728a0b1f590..40fcc239a8f7 100644 --- a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigTag.java +++ b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigTag.java @@ -18,9 +18,8 @@ package org.apache.hadoop.hdds.conf; /** - * Available config tags. - *

- * Note: the values are defined in ozone-default.xml by hadoop.tags.custom. + * The definitive list of configuration tags. Used as a source to define the + * value of {@code hadoop.tags.custom} and {@code ozone.tags.system}. */ public enum ConfigTag { OZONE, From 67dc41445b5c73127a2b401a1a5419f67d2a23eb Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Tue, 7 Mar 2023 10:01:50 +0100 Subject: [PATCH 3/7] Fix TestOzoneConfigurationFields --- .../java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java | 4 +++- .../main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java | 2 -- .../java/org/apache/hadoop/hdds/conf/HddsConfServlet.java | 2 +- .../org/apache/hadoop/ozone/TestOzoneConfigurationFields.java | 1 - 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java index d549430a07c1..6c3f09b46bb8 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java @@ -49,7 +49,6 @@ import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_TAGS_CUSTOM; import static org.apache.hadoop.hdds.ratis.RatisHelper.HDDS_DATANODE_RATIS_PREFIX_KEY; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CONTAINER_COPY_WORKDIR; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_TAGS_SYSTEM_KEY; /** * Configuration for ozone. @@ -57,6 +56,9 @@ @InterfaceAudience.Private public class OzoneConfiguration extends Configuration implements MutableConfigurationSource { + public static final String OZONE_TAGS_SYSTEM_KEY = + "ozone.tags.system"; + static { addDeprecatedKeys(); diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java index 5e924be7f9f7..b4e08e885f12 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java @@ -36,8 +36,6 @@ @InterfaceAudience.Public @InterfaceStability.Unstable public final class OzoneConfigKeys { - public static final String OZONE_TAGS_SYSTEM_KEY = - "ozone.tags.system"; public static final String DFS_CONTAINER_IPC_PORT = "dfs.container.ipc"; public static final int DFS_CONTAINER_IPC_PORT_DEFAULT = 9859; diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/HddsConfServlet.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/HddsConfServlet.java index 368197a27642..ca966911e868 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/HddsConfServlet.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/HddsConfServlet.java @@ -34,7 +34,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.gson.Gson; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_TAGS_SYSTEM_KEY; +import static org.apache.hadoop.hdds.conf.OzoneConfiguration.OZONE_TAGS_SYSTEM_KEY; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java index fbd33d0d6dd1..8129bf0c03a1 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java @@ -57,7 +57,6 @@ public void initializeMemberVariables() { }; errorIfMissingConfigProps = true; errorIfMissingXmlProps = true; - xmlPropsToSkipCompare.add("hadoop.tags.custom"); xmlPropsToSkipCompare.add("ozone.om.nodes.EXAMPLEOMSERVICEID"); xmlPropsToSkipCompare.add("ozone.om.decommissioned.nodes" + ".EXAMPLEOMSERVICEID"); From f8ffa13ac9e91b1455b9ec9b685ca5ae63f047d1 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Tue, 7 Mar 2023 10:13:04 +0100 Subject: [PATCH 4/7] Refer to ConfigTag instead of obsolete list of tags in ozone-default.xml --- hadoop-hdds/common/src/main/resources/ozone-default.xml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index f8f928497a78..5a0bc998df9d 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -20,9 +20,7 @@ - - - + From 773efae575efed11a639b1e06331345214e77bfe Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Tue, 7 Mar 2023 10:15:48 +0100 Subject: [PATCH 5/7] Sort tags alphabetically in ConfigTag --- .../apache/hadoop/hdds/conf/ConfigTag.java | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigTag.java b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigTag.java index 40fcc239a8f7..9c3f0ff3560c 100644 --- a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigTag.java +++ b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigTag.java @@ -22,29 +22,29 @@ * value of {@code hadoop.tags.custom} and {@code ozone.tags.system}. */ public enum ConfigTag { - OZONE, - MANAGEMENT, - SECURITY, - PERFORMANCE, - DEBUG, + BALANCER, CLIENT, - SERVER, - OM, - SCM, + CONTAINER, CRITICAL, + DATANODE, + DATASTREAM, + DEBUG, + DELETION, + HA, + MANAGEMENT, + OM, + OZONE, + PERFORMANCE, + PIPELINE, RATIS, - CONTAINER, + RECON, REQUIRED, REST, - STORAGE, - PIPELINE, - STANDALONE, S3GATEWAY, - DATANODE, - RECON, - DELETION, - HA, - BALANCER, - UPGRADE, - DATASTREAM + SCM, + SECURITY, + SERVER, + STANDALONE, + STORAGE, + UPGRADE } From d3772d8b77e27ec4f584a66de51abff5dd431a15 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Tue, 7 Mar 2023 10:18:46 +0100 Subject: [PATCH 6/7] Update ConfigTag with tags in actual use --- .../org/apache/hadoop/hdds/conf/ConfigTag.java | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigTag.java b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigTag.java index 9c3f0ff3560c..ef3be28beaee 100644 --- a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigTag.java +++ b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigTag.java @@ -22,29 +22,37 @@ * value of {@code hadoop.tags.custom} and {@code ozone.tags.system}. */ public enum ConfigTag { + ACL, BALANCER, CLIENT, CONTAINER, - CRITICAL, DATANODE, DATASTREAM, DEBUG, DELETION, + DEPRECATED, + DN, + FREON, HA, + HDDS, + KERBEROS, MANAGEMENT, OM, + OPERATION, OZONE, + OZONEFS, PERFORMANCE, PIPELINE, RATIS, RECON, REQUIRED, - REST, + S3G, S3GATEWAY, SCM, SECURITY, - SERVER, - STANDALONE, STORAGE, - UPGRADE + TLS, + TOKEN, + UPGRADE, + X509 } From 7e49d321c78c8857c9ac7c6e0b838de4b1005d10 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Tue, 7 Mar 2023 10:20:40 +0100 Subject: [PATCH 7/7] Replace tags DN and S3G with DATANODE and S3GATEWAY --- hadoop-hdds/common/src/main/resources/ozone-default.xml | 4 ++-- .../src/main/java/org/apache/hadoop/hdds/conf/ConfigTag.java | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index 5a0bc998df9d..f093bf7303bf 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -2715,7 +2715,7 @@ ozone.s3g.http.auth.type simple - S3G, SECURITY, KERBEROS + S3GATEWAY, SECURITY, KERBEROS simple or kerberos. If kerberos is set, Kerberos SPNEOGO will be used for http authentication. @@ -3403,7 +3403,7 @@ ozone.audit.log.debug.cmd.list.dnaudit - DN + DATANODE A comma separated list of Datanode commands that are written to the DN audit logs only if the audit log level is debug. Ex: "CREATE_CONTAINER,READ_CONTAINER,UPDATE_CONTAINER". diff --git a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigTag.java b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigTag.java index ef3be28beaee..24feb69389b1 100644 --- a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigTag.java +++ b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigTag.java @@ -31,7 +31,6 @@ public enum ConfigTag { DEBUG, DELETION, DEPRECATED, - DN, FREON, HA, HDDS, @@ -46,7 +45,6 @@ public enum ConfigTag { RATIS, RECON, REQUIRED, - S3G, S3GATEWAY, SCM, SECURITY,