From f270d33c90ab1f4fbde5c4eb936ef8e8574f973a Mon Sep 17 00:00:00 2001 From: Symious Date: Sun, 28 Mar 2021 17:31:50 +0800 Subject: [PATCH 1/8] HDDS-5035. Use default config values to solve generated config file conflict --- .../conf/ConfigurationReflectionUtil.java | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigurationReflectionUtil.java b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigurationReflectionUtil.java index 229390e2d84b..62319df27d75 100644 --- a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigurationReflectionUtil.java +++ b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigurationReflectionUtil.java @@ -68,6 +68,8 @@ public static void injectConfigurationToObject(ConfigurationSource from, String key = prefix + "." + configAnnotation.key(); + String defaultValue = configAnnotation.defaultValue(); + ConfigType type = configAnnotation.type(); if (type == ConfigType.AUTO) { @@ -79,24 +81,29 @@ public static void injectConfigurationToObject(ConfigurationSource from, try { switch (type) { case STRING: - forcedFieldSet(field, configuration, from.get(key)); + forcedFieldSet(field, configuration, from.get(key, defaultValue)); break; case INT: - forcedFieldSet(field, configuration, from.getInt(key, 0)); + forcedFieldSet(field, configuration, + from.getInt(key, Integer.parseInt(defaultValue))); break; case BOOLEAN: - forcedFieldSet(field, configuration, from.getBoolean(key, false)); + forcedFieldSet(field, configuration, + from.getBoolean(key, Boolean.parseBoolean(defaultValue))); break; case LONG: - forcedFieldSet(field, configuration, from.getLong(key, 0)); + forcedFieldSet(field, configuration, + from.getLong(key, Long.parseLong(defaultValue))); break; case TIME: forcedFieldSet(field, configuration, - from.getTimeDuration(key, "0s", configAnnotation.timeUnit())); + from.getTimeDuration(key, defaultValue, + configAnnotation.timeUnit())); break; case SIZE: final long value = - Math.round(from.getStorageSize(key, "0b", StorageUnit.BYTES)); + Math.round(from.getStorageSize(key, + defaultValue, StorageUnit.BYTES)); if (field.getType() == int.class) { forcedFieldSet(field, configuration, (int) value); } else { @@ -106,13 +113,13 @@ public static void injectConfigurationToObject(ConfigurationSource from, break; case CLASS: forcedFieldSet(field, configuration, - from.getClass(key, Object.class)); + from.getClass(key, Class.forName(defaultValue))); break; default: throw new ConfigurationException( "Unsupported ConfigType " + type + " on " + fieldLocation); } - } catch (IllegalAccessException e) { + } catch (IllegalAccessException | ClassNotFoundException e) { throw new ConfigurationException( "Can't inject configuration to " + fieldLocation, e); } From 37f7a10dd03b155c1b2a8d357623473317caa48f Mon Sep 17 00:00:00 2001 From: Symious Date: Sun, 28 Mar 2021 19:47:54 +0800 Subject: [PATCH 2/8] trigger new CI check From ccda50f7341f740694813bf3c6a17f0654979947 Mon Sep 17 00:00:00 2001 From: Symious Date: Tue, 30 Mar 2021 07:43:37 +0800 Subject: [PATCH 3/8] HDDS_5035. Remove notes --- .../apache/hadoop/hdds/conf/ConfigurationReflectionUtil.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigurationReflectionUtil.java b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigurationReflectionUtil.java index 62319df27d75..5b6d4301184b 100644 --- a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigurationReflectionUtil.java +++ b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigurationReflectionUtil.java @@ -76,8 +76,6 @@ public static void injectConfigurationToObject(ConfigurationSource from, type = detectConfigType(field.getType(), fieldLocation); } - //Note: default value is handled by ozone-default.xml. Here we can - //use any default. try { switch (type) { case STRING: From 1981b42e9b0ab91f5a93e8cfca8c14c55c903ac5 Mon Sep 17 00:00:00 2001 From: Symious Date: Tue, 30 Mar 2021 09:40:40 +0800 Subject: [PATCH 4/8] trigger new CI check From fa593d4b552bb477d01b367cd1a796427d967acc Mon Sep 17 00:00:00 2001 From: Symious Date: Wed, 31 Mar 2021 17:25:15 +0800 Subject: [PATCH 5/8] HDDS-5035. Add unit test to simulate the overwrite case --- .../TestGeneratedConfigurationOverwrite.java | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/TestGeneratedConfigurationOverwrite.java diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/TestGeneratedConfigurationOverwrite.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/TestGeneratedConfigurationOverwrite.java new file mode 100644 index 000000000000..099cb54a2f13 --- /dev/null +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/TestGeneratedConfigurationOverwrite.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdds.conf; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +/** + * In HDDS-5035, we met the case that ozone-default-generated.xml got + * overwritten in an assemble jar, here we are trying to simulate this case + * by rename the generated config file. + */ +public class TestGeneratedConfigurationOverwrite { + + private final Path generatedConfigurationPath = + Paths.get("target/test-classes/ozone-default-generated.xml"); + private final Path generatedConfigurationPathBak = + Paths.get("target/test-classes/ozone-default-generated.xml.bak"); + + private OzoneConfiguration conf; + + @Rule + public TemporaryFolder tempConfigs = new TemporaryFolder(); + + @Before + public void overwriteConfigFile() throws Exception { + Files.move(generatedConfigurationPath, generatedConfigurationPathBak); + conf = new OzoneConfiguration(); + } + + @After + public void recoverConfigFile() throws Exception { + Files.move(generatedConfigurationPathBak, generatedConfigurationPath); + } + + @Test + public void getConfigurationObject() { + // Check Config Type of String + Assert.assertNotNull( + conf.getObject(SimpleConfiguration.class).getBindHost()); + // Check Config Type of Int + Assert.assertNotEquals( + conf.getObject(SimpleConfiguration.class).getPort(), 0); + // Check Config Type of Time + Assert.assertNotEquals( + conf.getObject(SimpleConfiguration.class).getWaitTime(), 0); + } +} \ No newline at end of file From 606533ce92677130eb03b30bf54085d69461a183 Mon Sep 17 00:00:00 2001 From: Symious Date: Wed, 31 Mar 2021 20:28:36 +0800 Subject: [PATCH 6/8] HDDS-5035. Remove unused variables --- .../hadoop/hdds/conf/TestGeneratedConfigurationOverwrite.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/TestGeneratedConfigurationOverwrite.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/TestGeneratedConfigurationOverwrite.java index 099cb54a2f13..fb8392f2fd01 100644 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/TestGeneratedConfigurationOverwrite.java +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/TestGeneratedConfigurationOverwrite.java @@ -42,9 +42,6 @@ public class TestGeneratedConfigurationOverwrite { private OzoneConfiguration conf; - @Rule - public TemporaryFolder tempConfigs = new TemporaryFolder(); - @Before public void overwriteConfigFile() throws Exception { Files.move(generatedConfigurationPath, generatedConfigurationPathBak); From 92811120a6bd73e01393c94b3f378cc22468da3a Mon Sep 17 00:00:00 2001 From: Symious Date: Wed, 31 Mar 2021 22:19:45 +0800 Subject: [PATCH 7/8] HDDS-5035. Remove unused import --- .../hadoop/hdds/conf/TestGeneratedConfigurationOverwrite.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/TestGeneratedConfigurationOverwrite.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/TestGeneratedConfigurationOverwrite.java index fb8392f2fd01..f00107b0149d 100644 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/TestGeneratedConfigurationOverwrite.java +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/conf/TestGeneratedConfigurationOverwrite.java @@ -20,9 +20,7 @@ import org.junit.After; import org.junit.Assert; import org.junit.Before; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.TemporaryFolder; import java.nio.file.Files; import java.nio.file.Path; From b8d62e804f25bbd88f845d685353ecb703400cfa Mon Sep 17 00:00:00 2001 From: Symious Date: Thu, 1 Apr 2021 06:05:56 +0800 Subject: [PATCH 8/8] trigger new CI check