diff --git a/hadoop-hdds/client/pom.xml b/hadoop-hdds/client/pom.xml index 917d90f88e8..bbcf366bf6b 100644 --- a/hadoop-hdds/client/pom.xml +++ b/hadoop-hdds/client/pom.xml @@ -110,6 +110,16 @@ + + com.coderplus.maven.plugins + copy-rename-maven-plugin + + + rename-generated-config + process-classes + + + com.github.spotbugs spotbugs-maven-plugin diff --git a/hadoop-hdds/common/pom.xml b/hadoop-hdds/common/pom.xml index b0ed7c479c8..6218076fb8c 100644 --- a/hadoop-hdds/common/pom.xml +++ b/hadoop-hdds/common/pom.xml @@ -268,6 +268,20 @@ + + com.coderplus.maven.plugins + copy-rename-maven-plugin + + + rename-generated-config + process-classes + + + rename-generated-test-config + process-test-classes + + + com.github.spotbugs spotbugs-maven-plugin diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java index 6959641c84f..f863f3a303e 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java @@ -23,7 +23,6 @@ import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CONTAINER_COPY_WORKDIR; import com.google.common.base.Preconditions; -import java.io.IOException; import java.net.URL; import java.util.ArrayList; import java.util.Arrays; @@ -105,42 +104,10 @@ public static T newInstanceOf(Class configurationClass) { } public OzoneConfiguration() { - OzoneConfiguration.activate(); - loadDefaults(); } public OzoneConfiguration(Configuration conf) { super(conf); - //load the configuration from the classloader of the original conf. - setClassLoader(conf.getClassLoader()); - if (!(conf instanceof OzoneConfiguration)) { - loadDefaults(); - addResource(conf); - } - } - - private void loadDefaults() { - try { - //there could be multiple ozone-default-generated.xml files on the - // classpath, which are generated by the annotation processor. - // Here we add all of them to the list of the available configuration. - Enumeration generatedDefaults = - OzoneConfiguration.class.getClassLoader().getResources( - "ozone-default-generated.xml"); - while (generatedDefaults.hasMoreElements()) { - addResource(generatedDefaults.nextElement()); - } - } catch (IOException e) { - e.printStackTrace(); - } - addResource("ozone-default.xml"); - // Adding core-site here because properties from core-site are - // distributed to executors by spark driver. Ozone properties which are - // added to core-site, will be overridden by properties from adding Resource - // ozone-default.xml. So, adding core-site again will help to resolve - // this override issue. - addResource("core-site.xml"); - addResource("ozone-site.xml"); } public List readPropertyFromXml(URL url) throws JAXBException { @@ -242,10 +209,31 @@ public boolean equals(Object obj) { } } + /** Add default resources. */ public static void activate() { - // adds the default resources - Configuration.addDefaultResource("hdfs-default.xml"); - Configuration.addDefaultResource("hdfs-site.xml"); + // core-default and core-site are added by parent class + addDefaultResource("hdfs-default.xml"); + addDefaultResource("hdfs-site.xml"); + + // Modules with @Config annotations. If new one is introduced, add it to this list. + String[] modules = new String[] { + "hdds-common", + "hdds-client", + "hdds-container-service", + "hdds-server-framework", + "hdds-server-scm", + "ozone-common", + "ozone-csi", + "ozone-manager", + "ozone-recon", + }; + for (String module : modules) { + addDefaultResource(module + "-default.xml"); + } + + // Non-generated configs + addDefaultResource("ozone-default.xml"); + addDefaultResource("ozone-site.xml"); } /** diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/TestGeneratedConfigurationOverwrite.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/TestGeneratedConfigurationOverwrite.java index d74f0d8631e..eb897e02d08 100644 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/TestGeneratedConfigurationOverwrite.java +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/TestGeneratedConfigurationOverwrite.java @@ -35,9 +35,9 @@ public class TestGeneratedConfigurationOverwrite { private final Path generatedConfigurationPath = - Paths.get("target/test-classes/ozone-default-generated.xml"); + Paths.get("target/test-classes/hdds-common-default.xml"); private final Path generatedConfigurationPathBak = - Paths.get("target/test-classes/ozone-default-generated.xml.bak"); + Paths.get("target/test-classes/hdds-common-default.xml.bak"); private OzoneConfiguration conf; diff --git a/hadoop-hdds/container-service/pom.xml b/hadoop-hdds/container-service/pom.xml index 774b72fa931..53e2528a7f2 100644 --- a/hadoop-hdds/container-service/pom.xml +++ b/hadoop-hdds/container-service/pom.xml @@ -261,6 +261,16 @@ + + com.coderplus.maven.plugins + copy-rename-maven-plugin + + + rename-generated-config + process-classes + + + com.github.spotbugs spotbugs-maven-plugin diff --git a/hadoop-hdds/framework/pom.xml b/hadoop-hdds/framework/pom.xml index 585764280f1..921a1d1a6a0 100644 --- a/hadoop-hdds/framework/pom.xml +++ b/hadoop-hdds/framework/pom.xml @@ -334,6 +334,20 @@ + + com.coderplus.maven.plugins + copy-rename-maven-plugin + + + rename-generated-config + process-classes + + + rename-generated-test-config + process-test-classes + + + com.github.spotbugs spotbugs-maven-plugin diff --git a/hadoop-hdds/server-scm/pom.xml b/hadoop-hdds/server-scm/pom.xml index 50bf14c6a95..f67144deebd 100644 --- a/hadoop-hdds/server-scm/pom.xml +++ b/hadoop-hdds/server-scm/pom.xml @@ -238,6 +238,16 @@ + + com.coderplus.maven.plugins + copy-rename-maven-plugin + + + rename-generated-config + process-classes + + + org.apache.maven.plugins maven-compiler-plugin diff --git a/hadoop-ozone/client/pom.xml b/hadoop-ozone/client/pom.xml index d8a2a77dd8e..c2b8211a5f8 100644 --- a/hadoop-ozone/client/pom.xml +++ b/hadoop-ozone/client/pom.xml @@ -130,40 +130,9 @@ org.apache.maven.plugins maven-compiler-plugin - - - org.apache.ozone - hdds-config - ${hdds.version} - - - - org.apache.hadoop.hdds.conf.ConfigFileGenerator - + none - - org.apache.maven.plugins - maven-enforcer-plugin - - - ban-annotations - - - - - Only selected annotation processors are enabled, see configuration of maven-compiler-plugin. - - org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator - org.apache.hadoop.hdds.scm.metadata.Replicate - org.kohsuke.MetaInfServices - - - - - - - diff --git a/hadoop-ozone/common/pom.xml b/hadoop-ozone/common/pom.xml index 377dade8730..2e383b9c3fb 100644 --- a/hadoop-ozone/common/pom.xml +++ b/hadoop-ozone/common/pom.xml @@ -185,6 +185,16 @@ + + com.coderplus.maven.plugins + copy-rename-maven-plugin + + + rename-generated-config + process-classes + + + org.apache.hadoop hadoop-maven-plugins diff --git a/hadoop-ozone/csi/pom.xml b/hadoop-ozone/csi/pom.xml index 6100ed16013..5304edec8a8 100644 --- a/hadoop-ozone/csi/pom.xml +++ b/hadoop-ozone/csi/pom.xml @@ -195,6 +195,16 @@ + + com.coderplus.maven.plugins + copy-rename-maven-plugin + + + rename-generated-config + process-classes + + + com.salesforce.servicelibs proto-backwards-compatibility diff --git a/hadoop-ozone/dist/src/main/smoketest/httpfs/operations_tests.robot b/hadoop-ozone/dist/src/main/smoketest/httpfs/operations_tests.robot index 603162622e8..1fb34839a57 100644 --- a/hadoop-ozone/dist/src/main/smoketest/httpfs/operations_tests.robot +++ b/hadoop-ozone/dist/src/main/smoketest/httpfs/operations_tests.robot @@ -23,6 +23,7 @@ Resource operations.robot Resource ../lib/os.robot Resource ../commonlib.robot Suite Setup Generate volume +Test Timeout 2 minutes *** Variables *** ${volume} generated diff --git a/hadoop-ozone/httpfsgateway/pom.xml b/hadoop-ozone/httpfsgateway/pom.xml index 89f35feefed..704d0e6a4bf 100644 --- a/hadoop-ozone/httpfsgateway/pom.xml +++ b/hadoop-ozone/httpfsgateway/pom.xml @@ -100,6 +100,10 @@ org.apache.ozone hdds-server-framework + + org.apache.ozone + ozone-common + org.apache.ozone ozone-filesystem-common diff --git a/hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/fs/http/server/HttpFSServerWebServer.java b/hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/fs/http/server/HttpFSServerWebServer.java index 0b6cf06dbdc..1523c60cdb7 100644 --- a/hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/fs/http/server/HttpFSServerWebServer.java +++ b/hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/fs/http/server/HttpFSServerWebServer.java @@ -17,7 +17,8 @@ package org.apache.ozone.fs.http.server; -import static org.apache.hadoop.util.StringUtils.startupShutdownMessage; +import static org.apache.hadoop.hdds.utils.HddsServerUtil.startupShutdownMessage; +import static org.apache.hadoop.ozone.util.OzoneVersionInfo.OZONE_VERSION_INFO; import java.io.IOException; import java.net.InetSocketAddress; @@ -177,8 +178,8 @@ public URL getUrl() { } public static void main(String[] args) throws Exception { - startupShutdownMessage(HttpFSServerWebServer.class, args, LOG); OzoneConfiguration conf = new OzoneConfiguration(); + startupShutdownMessage(OZONE_VERSION_INFO, HttpFSServerWebServer.class, args, LOG, conf); Configuration sslConf = SSLFactory.readSSLConfiguration(conf, SSLFactory.Mode.SERVER); HttpFSServerWebServer webServer = diff --git a/hadoop-ozone/insight/pom.xml b/hadoop-ozone/insight/pom.xml index 49a50603fb9..87839121965 100644 --- a/hadoop-ozone/insight/pom.xml +++ b/hadoop-ozone/insight/pom.xml @@ -130,16 +130,8 @@ org.apache.maven.plugins maven-compiler-plugin - - - org.apache.ozone - hdds-config - ${hdds.version} - - - - org.apache.hadoop.hdds.conf.ConfigFileGenerator - + + none diff --git a/hadoop-ozone/insight/src/test/java/org/apache/hadoop/ozone/insight/TestConfigurationSubCommand.java b/hadoop-ozone/insight/src/test/java/org/apache/hadoop/ozone/insight/TestConfigurationSubCommand.java index 8ef58dca61a..ee7eed68500 100644 --- a/hadoop-ozone/insight/src/test/java/org/apache/hadoop/ozone/insight/TestConfigurationSubCommand.java +++ b/hadoop-ozone/insight/src/test/java/org/apache/hadoop/ozone/insight/TestConfigurationSubCommand.java @@ -23,10 +23,8 @@ import java.io.PrintStream; import java.io.UnsupportedEncodingException; import java.nio.charset.StandardCharsets; -import org.apache.hadoop.hdds.conf.Config; -import org.apache.hadoop.hdds.conf.ConfigGroup; -import org.apache.hadoop.hdds.conf.ConfigTag; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.ozone.om.OmConfig; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -53,49 +51,18 @@ public void reset() { @Test public void testPrintConfig() throws UnsupportedEncodingException { OzoneConfiguration conf = new OzoneConfiguration(); - conf.set("ozone.scm.client.address", "omclient"); + long customValue = OmConfig.Defaults.SERVER_LIST_MAX_SIZE + 50; + conf.setLong(OmConfig.Keys.SERVER_LIST_MAX_SIZE, customValue); ConfigurationSubCommand subCommand = new ConfigurationSubCommand(); - subCommand.printConfig(CustomConfig.class, conf); + subCommand.printConfig(OmConfig.class, conf); final String output = out.toString(StandardCharsets.UTF_8.name()); - assertThat(output).contains(">>> ozone.scm.client.address"); - assertThat(output).contains("default: localhost"); - assertThat(output).contains("current: omclient"); - assertThat(output).contains(">>> ozone.scm.client.secure"); - assertThat(output).contains("default: true"); - assertThat(output).contains("current: true"); - } - - /** - * Example configuration parent. - */ - public static class ParentConfig { - @Config(key = "secure", defaultValue = "true", description = "Make " - + "everything secure.", tags = ConfigTag.MANAGEMENT) - private boolean secure = true; - - public boolean isSecure() { - return secure; - } - } - - /** - * Example configuration. - */ - @ConfigGroup(prefix = "ozone.scm.client") - public static class CustomConfig extends ParentConfig { - - @Config(key = "address", defaultValue = "localhost", description = "Client " - + "address (To test string injection).", tags = ConfigTag.MANAGEMENT) - private String clientAddress; - - public String getClientAddress() { - return clientAddress; - } - - public void setClientAddress(String clientAddress) { - this.clientAddress = clientAddress; - } + assertThat(output).contains(">>> " + OmConfig.Keys.SERVER_LIST_MAX_SIZE); + assertThat(output).contains("default: " + OmConfig.Defaults.SERVER_LIST_MAX_SIZE); + assertThat(output).contains("current: " + customValue); + assertThat(output).contains(">>> " + OmConfig.Keys.ENABLE_FILESYSTEM_PATHS); + assertThat(output).contains("default: " + OmConfig.Defaults.ENABLE_FILESYSTEM_PATHS); + assertThat(output).contains("current: " + OmConfig.Defaults.ENABLE_FILESYSTEM_PATHS); } } diff --git a/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/ReconService.java b/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/ReconService.java index 99f0faa35fd..bedfb70e457 100644 --- a/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/ReconService.java +++ b/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/recon/ReconService.java @@ -31,7 +31,6 @@ import java.util.Objects; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.MiniOzoneCluster; -import org.apache.ozone.recon.schema.ReconSqlDbConfig; import org.apache.ratis.util.Preconditions; /** Recon for {@link MiniOzoneCluster}. */ diff --git a/hadoop-ozone/interface-storage/pom.xml b/hadoop-ozone/interface-storage/pom.xml index 77961757ae2..5d1b4cc137e 100644 --- a/hadoop-ozone/interface-storage/pom.xml +++ b/hadoop-ozone/interface-storage/pom.xml @@ -87,40 +87,9 @@ org.apache.maven.plugins maven-compiler-plugin - - - org.apache.ozone - hdds-config - ${hdds.version} - - - - org.apache.hadoop.hdds.conf.ConfigFileGenerator - + none - - org.apache.maven.plugins - maven-enforcer-plugin - - - ban-annotations - - - - - Only selected annotation processors are enabled, see configuration of maven-compiler-plugin. - - org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator - org.apache.hadoop.hdds.scm.metadata.Replicate - org.kohsuke.MetaInfServices - - - - - - - org.xolstice.maven.plugins protobuf-maven-plugin diff --git a/hadoop-ozone/ozone-manager/pom.xml b/hadoop-ozone/ozone-manager/pom.xml index 186a26ffa7c..d303fc88bd9 100644 --- a/hadoop-ozone/ozone-manager/pom.xml +++ b/hadoop-ozone/ozone-manager/pom.xml @@ -381,6 +381,16 @@ + + com.coderplus.maven.plugins + copy-rename-maven-plugin + + + rename-generated-config + process-classes + + + org.apache.maven.plugins maven-compiler-plugin diff --git a/hadoop-ozone/ozonefs-hadoop3-client/pom.xml b/hadoop-ozone/ozonefs-hadoop3-client/pom.xml index 3acd28bed14..249b8c2cc4b 100644 --- a/hadoop-ozone/ozonefs-hadoop3-client/pom.xml +++ b/hadoop-ozone/ozonefs-hadoop3-client/pom.xml @@ -96,9 +96,6 @@ - - ozone-default-generated.xml - diff --git a/hadoop-ozone/ozonefs-shaded/pom.xml b/hadoop-ozone/ozonefs-shaded/pom.xml index 8ade37ce205..c5e5d4f0ea1 100644 --- a/hadoop-ozone/ozonefs-shaded/pom.xml +++ b/hadoop-ozone/ozonefs-shaded/pom.xml @@ -189,9 +189,6 @@ - - ozone-default-generated.xml - diff --git a/hadoop-ozone/recon-codegen/pom.xml b/hadoop-ozone/recon-codegen/pom.xml index ba7261aedf7..9582f435774 100644 --- a/hadoop-ozone/recon-codegen/pom.xml +++ b/hadoop-ozone/recon-codegen/pom.xml @@ -83,40 +83,9 @@ org.apache.maven.plugins maven-compiler-plugin - - - org.apache.ozone - hdds-config - ${hdds.version} - - - - org.apache.hadoop.hdds.conf.ConfigFileGenerator - + none - - org.apache.maven.plugins - maven-enforcer-plugin - - - ban-annotations - - - - - Only selected annotation processors are enabled, see configuration of maven-compiler-plugin. - - org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator - org.apache.hadoop.hdds.scm.metadata.Replicate - org.kohsuke.MetaInfServices - - - - - - - diff --git a/hadoop-ozone/recon/pom.xml b/hadoop-ozone/recon/pom.xml index 38cb0f0fd9b..3fe3b9b8b83 100644 --- a/hadoop-ozone/recon/pom.xml +++ b/hadoop-ozone/recon/pom.xml @@ -320,6 +320,16 @@ + + com.coderplus.maven.plugins + copy-rename-maven-plugin + + + rename-generated-config + process-classes + + + org.apache.maven.plugins maven-compiler-plugin diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconControllerModule.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconControllerModule.java index ad2d25a29d4..e5879a3986d 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconControllerModule.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconControllerModule.java @@ -69,7 +69,6 @@ import org.apache.hadoop.ozone.recon.tasks.ReconTaskControllerImpl; import org.apache.hadoop.ozone.recon.tasks.updater.ReconTaskStatusUpdaterManager; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.ozone.recon.schema.ReconSqlDbConfig; import org.apache.ozone.recon.schema.generated.tables.daos.ClusterGrowthDailyDao; import org.apache.ozone.recon.schema.generated.tables.daos.ContainerCountBySizeDao; import org.apache.ozone.recon.schema.generated.tables.daos.FileCountBySizeDao; diff --git a/hadoop-ozone/recon-codegen/src/main/java/org/apache/ozone/recon/schema/ReconSqlDbConfig.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconSqlDbConfig.java similarity index 99% rename from hadoop-ozone/recon-codegen/src/main/java/org/apache/ozone/recon/schema/ReconSqlDbConfig.java rename to hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconSqlDbConfig.java index 7acd9349037..283ec01fff9 100644 --- a/hadoop-ozone/recon-codegen/src/main/java/org/apache/ozone/recon/schema/ReconSqlDbConfig.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconSqlDbConfig.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.ozone.recon.schema; +package org.apache.hadoop.ozone.recon; import static java.util.concurrent.TimeUnit.SECONDS; diff --git a/hadoop-ozone/s3gateway/pom.xml b/hadoop-ozone/s3gateway/pom.xml index 365010e7b70..7b6ca80b4ec 100644 --- a/hadoop-ozone/s3gateway/pom.xml +++ b/hadoop-ozone/s3gateway/pom.xml @@ -238,40 +238,9 @@ org.apache.maven.plugins maven-compiler-plugin - - - org.apache.ozone - hdds-config - ${hdds.version} - - - - org.apache.hadoop.hdds.conf.ConfigFileGenerator - + none - - org.apache.maven.plugins - maven-enforcer-plugin - - - ban-annotations - - - - - Only selected annotation processors are enabled, see configuration of maven-compiler-plugin. - - org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator - org.apache.hadoop.hdds.scm.metadata.Replicate - org.kohsuke.MetaInfServices - - - - - - - org.apache.maven.plugins maven-dependency-plugin diff --git a/pom.xml b/pom.xml index fc406ac3885..ee419d4bc09 100644 --- a/pom.xml +++ b/pom.xml @@ -2174,6 +2174,48 @@ com.coderplus.maven.plugins copy-rename-maven-plugin ${copy-rename-maven-plugin.version} + + + + rename-generated-config + + rename + + + none + + + + ${project.build.outputDirectory}/ozone-default-generated.xml + ${project.build.outputDirectory}/${project.artifactId}-default.xml + + + + + + rename-generated-test-config + + rename + + + none + + + + ${project.build.testOutputDirectory}/ozone-default-generated.xml + ${project.build.testOutputDirectory}/${project.artifactId}-default.xml + + + + +