Skip to content

HDDS-8091. Generate list of config tags from ConfigTag enum #4359

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
adoroszlai merged 7 commits into from
Mar 9, 2023
Merged

HDDS-8091. Generate list of config tags from ConfigTag enum #4359

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
76da803
HDDS-8091. Generate list of config tags from ConfigTag enum
adoroszlai Mar 7, 2023
e460320
Update class comment in ConfigTag
adoroszlai Mar 7, 2023
67dc414
Fix TestOzoneConfigurationFields
adoroszlai Mar 7, 2023
f8ffa13
Refer to ConfigTag instead of obsolete list of tags in ozone-default.xml
adoroszlai Mar 7, 2023
773efae
Sort tags alphabetically in ConfigTag
adoroszlai Mar 7, 2023
d3772d8
Update ConfigTag with tags in actual use
adoroszlai Mar 7, 2023
7e49d32
Replace tags DN and S3G with DATANODE and S3GATEWAY
adoroszlai Mar 7, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Enumeration;
import java.util.HashMap;
Expand All @@ -45,6 +46,7 @@
import com.google.common.base.Preconditions;
import org.apache.ratis.server.RaftServerConfigKeys;

import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_TAGS_CUSTOM;
import static org.apache.hadoop.hdds.ratis.RatisHelper.HDDS_DATANODE_RATIS_PREFIX_KEY;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CONTAINER_COPY_WORKDIR;

Expand All @@ -54,6 +56,9 @@
@InterfaceAudience.Private
public class OzoneConfiguration extends Configuration
implements MutableConfigurationSource {
public static final String OZONE_TAGS_SYSTEM_KEY =
"ozone.tags.system";

static {
addDeprecatedKeys();

Expand Down Expand Up @@ -303,6 +308,20 @@ public Map<String, String> getPropsMatchPrefixAndTrimPrefix(
return configMap;
}

@Override
public void addTags(Properties prop) {
if (!prop.containsKey(OZONE_TAGS_SYSTEM_KEY)) {
String ozoneTags = Arrays.stream(ConfigTag.values())
.map(Enum::name)
.sorted()
.collect(Collectors.joining(","));
prop.setProperty(HADOOP_TAGS_CUSTOM, ozoneTags);
prop.setProperty(OZONE_TAGS_SYSTEM_KEY, ozoneTags);
}

super.addTags(prop);
}

private static void addDeprecatedKeys() {
Configuration.addDeprecations(new DeprecationDelta[]{
new DeprecationDelta("ozone.datanode.pipeline.limit",
Expand Down
2 changes: 0 additions & 2 deletions hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,6 @@
@InterfaceAudience.Public
@InterfaceStability.Unstable
public final class OzoneConfigKeys {
public static final String OZONE_TAGS_SYSTEM_KEY =
"ozone.tags.system";
public static final String DFS_CONTAINER_IPC_PORT =
Copy link
Contributor

@SaketaChalamchala SaketaChalamchala Mar 8, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For my understanding, Is there a reason why this config needed to be taken out of OzoneConfigKeys and added in OzoneConfiguration?

Copy link
Contributor Author

@adoroszlai adoroszlai Mar 8, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks @SaketaChalamchala for the review.

TestOzoneConfigurationFields verifies that all keys defined in OzoneConfigKeys (and some others) are also present in ozone-default.xml (and vice versa). It would have been enough to add as an exception in TestOzoneConfigurationFields.

"dfs.container.ipc";
public static final int DFS_CONTAINER_IPC_PORT_DEFAULT = 9859;
Expand Down
19 changes: 3 additions & 16 deletions hadoop-hdds/common/src/main/resources/ozone-default.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,7 @@
<!-- wish to modify from this file into ozone-site.xml and change them -->
<!-- there. If ozone-site.xml does not already exist, create it. -->

<!--Tags supported are OZONE, CBLOCK, MANAGEMENT, SECURITY, PERFORMANCE, -->
<!--DEBUG, CLIENT, SERVER, OM, SCM, CRITICAL, RATIS, CONTAINER, REQUIRED, -->
<!--REST, STORAGE, PIPELINE, STANDALONE -->
<!--Tags supported are defined in org.apache.hadoop.hdds.conf.ConfigTag-->

<configuration>

Expand Down Expand Up @@ -1339,17 +1337,6 @@
</description>
</property>

<property>
<name>hadoop.tags.custom</name>
<value>OZONE,MANAGEMENT,SECURITY,PERFORMANCE,DEBUG,CLIENT,SERVER,OM,SCM,CRITICAL,RATIS,CONTAINER,REQUIRED,REST,STORAGE,PIPELINE,STANDALONE,S3GATEWAY,RECON</value>
</property>

<property>
<name>ozone.tags.system</name>
<value>OZONE,MANAGEMENT,SECURITY,PERFORMANCE,DEBUG,CLIENT,SERVER,OM,SCM,CRITICAL,RATIS,CONTAINER,REQUIRED,REST,STORAGE,PIPELINE,STANDALONE,S3GATEWAY,TOKEN,TLS,RECON</value>
</property>


<property>
<name>hdds.rest.rest-csrf.enabled</name>
<value>false</value>
Expand Down Expand Up @@ -2728,7 +2715,7 @@
<property>
<name>ozone.s3g.http.auth.type</name>
<value>simple</value>
<tag>S3G, SECURITY, KERBEROS</tag>
<tag>S3GATEWAY, SECURITY, KERBEROS</tag>
<description> simple or kerberos. If kerberos is set, Kerberos SPNEOGO
will be used for http authentication.
</description>
Expand Down Expand Up @@ -3416,7 +3403,7 @@
<property>
<name>ozone.audit.log.debug.cmd.list.dnaudit</name>
<value></value>
<tag>DN</tag>
<tag>DATANODE</tag>
<description>
A comma separated list of Datanode commands that are written to the DN audit logs only if the audit
log level is debug. Ex: "CREATE_CONTAINER,READ_CONTAINER,UPDATE_CONTAINER".
Expand Down
16 changes: 16 additions & 0 deletions hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/TestOzoneConfiguration.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.EnumSource;

/**
* Test class for OzoneConfiguration.
Expand Down Expand Up @@ -257,6 +259,20 @@ public void postConstructValidation() {
() -> ozoneConfiguration.getObject(SimpleConfiguration.class));
}

@ParameterizedTest
@EnumSource
void tagIsRecognized(ConfigTag tag) {
OzoneConfiguration subject = new OzoneConfiguration();
Assertions.assertTrue(subject.isPropertyTag(tag.name()),
() -> tag + " should be recognized as config tag");
}

@Test
void unknownTag() {
OzoneConfiguration subject = new OzoneConfiguration();
Assertions.assertFalse(subject.isPropertyTag("not-a-tag"));
}

private void appendProperty(BufferedWriter out, String name, String val)
throws IOException {
this.appendProperty(out, name, val, false);
Expand Down
49 changes: 27 additions & 22 deletions hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigTag.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,34 +18,39 @@
package org.apache.hadoop.hdds.conf;

/**
* Available config tags.
* <p>
* Note: the values are defined in ozone-default.xml by hadoop.tags.custom.
* The definitive list of configuration tags. Used as a source to define the
* value of {@code hadoop.tags.custom} and {@code ozone.tags.system}.
*/
public enum ConfigTag {
OZONE,
MANAGEMENT,
SECURITY,
PERFORMANCE,
DEBUG,
ACL,
BALANCER,
CLIENT,
SERVER,
OM,
SCM,
CRITICAL,
RATIS,
CONTAINER,
REQUIRED,
REST,
STORAGE,
PIPELINE,
STANDALONE,
S3GATEWAY,
DATANODE,
RECON,
DATASTREAM,
DEBUG,
DELETION,
DEPRECATED,
FREON,
HA,
BALANCER,
HDDS,
KERBEROS,
MANAGEMENT,
OM,
OPERATION,
OZONE,
OZONEFS,
PERFORMANCE,
PIPELINE,
RATIS,
RECON,
REQUIRED,
S3GATEWAY,
SCM,
SECURITY,
STORAGE,
TLS,
TOKEN,
UPGRADE,
DATASTREAM
X509
}
2 changes: 1 addition & 1 deletion hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/conf/HddsConfServlet.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@

import com.google.common.annotations.VisibleForTesting;
import com.google.gson.Gson;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_TAGS_SYSTEM_KEY;
import static org.apache.hadoop.hdds.conf.OzoneConfiguration.OZONE_TAGS_SYSTEM_KEY;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down
1 change: 0 additions & 1 deletion .../integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,6 @@ public void initializeMemberVariables() {
};
errorIfMissingConfigProps = true;
errorIfMissingXmlProps = true;
xmlPropsToSkipCompare.add("hadoop.tags.custom");
xmlPropsToSkipCompare.add("ozone.om.nodes.EXAMPLEOMSERVICEID");
xmlPropsToSkipCompare.add("ozone.om.decommissioned.nodes" +
".EXAMPLEOMSERVICEID");
Expand Down