From 3b01c567bec607fe8f356f8200655521774f50ac Mon Sep 17 00:00:00 2001 From: daniilchik Date: Thu, 17 Oct 2024 14:32:58 +0300 Subject: [PATCH 1/7] HDDS-11580. Validate 'hdds.datanode.dir.du.reserved' property --- .../hadoop/ozone/container/common/volume/VolumeUsage.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeUsage.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeUsage.java index 733dc7964f1c..aab84b2d714e 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeUsage.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeUsage.java @@ -19,6 +19,7 @@ package org.apache.hadoop.ozone.container.common.volume; import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.hdds.conf.ConfigurationException; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.conf.StorageSize; import org.apache.hadoop.hdds.conf.StorageUnit; @@ -222,7 +223,7 @@ private static long getReserved(ConfigurationSource conf, String rootDir, if (words.length < 2) { LOG.error("Reserved space should be configured in a pair, but current value is {}", reserve); - continue; + throw new ConfigurationException("Reserved space should be configured in a pair"); } try { From 56df4b24fd33fdb9af72cc7607e767b31815ee24 Mon Sep 17 00:00:00 2001 From: daniilchik Date: Fri, 18 Oct 2024 13:05:35 +0300 Subject: [PATCH 2/7] HDDS-11580. Validate 'hdds.datanode.dir.du.reserved' property --- .../common/volume/TestReservedVolumeSpace.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestReservedVolumeSpace.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestReservedVolumeSpace.java index 5e2dd0c75c9b..97f1685e87c2 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestReservedVolumeSpace.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestReservedVolumeSpace.java @@ -20,6 +20,7 @@ import org.apache.hadoop.conf.StorageUnit; import org.apache.hadoop.hdds.HddsConfigKeys; +import org.apache.hadoop.hdds.conf.ConfigurationException; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.fs.MockSpaceUsageCheckFactory; import org.apache.hadoop.hdds.scm.ScmConfigKeys; @@ -36,6 +37,7 @@ import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DATANODE_VOLUME_MIN_FREE_SPACE_PERCENT; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_DU_RESERVED_PERCENT; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_DU_RESERVED_PERCENT_DEFAULT; +import static org.apache.ozone.test.GenericTestUtils.assertThrows; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotEquals; @@ -164,6 +166,15 @@ public void testInvalidConfig() throws Exception { long reservedFromVolume2 = hddsVolume2.getVolumeInfo().get() .getReservedInBytes(); assertEquals(getExpectedDefaultReserved(hddsVolume2), reservedFromVolume2); + + OzoneConfiguration conf3 = new OzoneConfiguration(); + + conf3.set(ScmConfigKeys.HDDS_DATANODE_DIR_DU_RESERVED, "15GB"); + + assertThrows(ConfigurationException.class, () -> { + HddsVolume hddsVolume3 = volumeBuilder.conf(conf3).build(); + return null; + }); } @Test From e21e1622bc1b13504ca7faa74faf1c6d6d8c455a Mon Sep 17 00:00:00 2001 From: daniilchik Date: Fri, 18 Oct 2024 14:25:16 +0300 Subject: [PATCH 3/7] HDDS-11580. Validate 'hdds.datanode.dir.du.reserved' property --- .../ozone/container/common/volume/VolumeUsage.java | 2 -- .../common/volume/TestReservedVolumeSpace.java | 13 ++++++++----- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeUsage.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeUsage.java index aab84b2d714e..368125d66036 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeUsage.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeUsage.java @@ -221,8 +221,6 @@ private static long getReserved(ConfigurationSource conf, String rootDir, for (String reserve : reserveList) { String[] words = reserve.split(":"); if (words.length < 2) { - LOG.error("Reserved space should be configured in a pair, but current value is {}", - reserve); throw new ConfigurationException("Reserved space should be configured in a pair"); } diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestReservedVolumeSpace.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestReservedVolumeSpace.java index 97f1685e87c2..1a76fcd0ca96 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestReservedVolumeSpace.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestReservedVolumeSpace.java @@ -166,14 +166,17 @@ public void testInvalidConfig() throws Exception { long reservedFromVolume2 = hddsVolume2.getVolumeInfo().get() .getReservedInBytes(); assertEquals(getExpectedDefaultReserved(hddsVolume2), reservedFromVolume2); + } - OzoneConfiguration conf3 = new OzoneConfiguration(); + @Test + public void testInvalidConfigThrowsException() { + OzoneConfiguration conf = new OzoneConfiguration(); - conf3.set(ScmConfigKeys.HDDS_DATANODE_DIR_DU_RESERVED, "15GB"); + conf.set(ScmConfigKeys.HDDS_DATANODE_DIR_DU_RESERVED, "15GB"); - assertThrows(ConfigurationException.class, () -> { - HddsVolume hddsVolume3 = volumeBuilder.conf(conf3).build(); - return null; + ConfigurationException thrown = assertThrows(ConfigurationException.class, () -> { + HddsVolume hddsVolume = volumeBuilder.conf(conf).build(); + return null; }); } From 0f51afe676fef30885e1fa5b968c0e73bc21db6c Mon Sep 17 00:00:00 2001 From: daniilchik Date: Fri, 18 Oct 2024 14:42:17 +0300 Subject: [PATCH 4/7] HDDS-11580. Validate 'hdds.datanode.dir.du.reserved' property --- .../ozone/container/common/volume/TestReservedVolumeSpace.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestReservedVolumeSpace.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestReservedVolumeSpace.java index 1a76fcd0ca96..71e9d1f19efb 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestReservedVolumeSpace.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestReservedVolumeSpace.java @@ -176,7 +176,7 @@ public void testInvalidConfigThrowsException() { ConfigurationException thrown = assertThrows(ConfigurationException.class, () -> { HddsVolume hddsVolume = volumeBuilder.conf(conf).build(); - return null; + return null; }); } From aad72a8b4c7fe2155838dce27241b9ce51c30f4a Mon Sep 17 00:00:00 2001 From: daniilchik Date: Fri, 18 Oct 2024 15:22:25 +0300 Subject: [PATCH 5/7] HDDS-11580. Validate 'hdds.datanode.dir.du.reserved' property --- .../container/common/volume/TestReservedVolumeSpace.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestReservedVolumeSpace.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestReservedVolumeSpace.java index 71e9d1f19efb..055faf6dbc65 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestReservedVolumeSpace.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestReservedVolumeSpace.java @@ -168,13 +168,12 @@ public void testInvalidConfig() throws Exception { assertEquals(getExpectedDefaultReserved(hddsVolume2), reservedFromVolume2); } - @Test + @Test() public void testInvalidConfigThrowsException() { OzoneConfiguration conf = new OzoneConfiguration(); - conf.set(ScmConfigKeys.HDDS_DATANODE_DIR_DU_RESERVED, "15GB"); - ConfigurationException thrown = assertThrows(ConfigurationException.class, () -> { + assertThrows(ConfigurationException.class, () -> { HddsVolume hddsVolume = volumeBuilder.conf(conf).build(); return null; }); From 208fb9605ca56fe1af7a8db0d8cd4d1a0f669a3e Mon Sep 17 00:00:00 2001 From: daniilchik Date: Fri, 18 Oct 2024 17:02:57 +0300 Subject: [PATCH 6/7] HDDS-11580. Validate 'hdds.datanode.dir.du.reserved' property --- .../hadoop/ozone/container/common/volume/VolumeUsage.java | 3 ++- .../container/common/volume/TestReservedVolumeSpace.java | 8 +++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeUsage.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeUsage.java index 368125d66036..34ba66c91bb2 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeUsage.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/volume/VolumeUsage.java @@ -221,7 +221,8 @@ private static long getReserved(ConfigurationSource conf, String rootDir, for (String reserve : reserveList) { String[] words = reserve.split(":"); if (words.length < 2) { - throw new ConfigurationException("Reserved space should be configured in a pair"); + throw new ConfigurationException("hdds.datanode.dir.du.reserved - " + + "Reserved space should be configured in a pair, but current value is " + reserve); } try { diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestReservedVolumeSpace.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestReservedVolumeSpace.java index 055faf6dbc65..7eff7df55874 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestReservedVolumeSpace.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestReservedVolumeSpace.java @@ -38,6 +38,7 @@ import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_DU_RESERVED_PERCENT; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_DU_RESERVED_PERCENT_DEFAULT; import static org.apache.ozone.test.GenericTestUtils.assertThrows; +import static org.apache.ratis.util.Preconditions.assertTrue; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotEquals; @@ -168,15 +169,16 @@ public void testInvalidConfig() throws Exception { assertEquals(getExpectedDefaultReserved(hddsVolume2), reservedFromVolume2); } - @Test() + @Test public void testInvalidConfigThrowsException() { OzoneConfiguration conf = new OzoneConfiguration(); conf.set(ScmConfigKeys.HDDS_DATANODE_DIR_DU_RESERVED, "15GB"); - assertThrows(ConfigurationException.class, () -> { - HddsVolume hddsVolume = volumeBuilder.conf(conf).build(); + ConfigurationException exception = assertThrows(ConfigurationException.class, () -> { + volumeBuilder.conf(conf).build(); return null; }); + assertTrue(exception.getMessage().contains("Reserved space should be configured in a pair")); } @Test From 417e8c960d6e3275dcd63c96132e34f95e88b81d Mon Sep 17 00:00:00 2001 From: daniilchik Date: Mon, 21 Oct 2024 11:35:31 +0300 Subject: [PATCH 7/7] HDDS-11580. Validate 'hdds.datanode.dir.du.reserved' property --- .../common/volume/TestReservedVolumeSpace.java | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestReservedVolumeSpace.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestReservedVolumeSpace.java index 7eff7df55874..5e0a31944f78 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestReservedVolumeSpace.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestReservedVolumeSpace.java @@ -37,9 +37,8 @@ import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DATANODE_VOLUME_MIN_FREE_SPACE_PERCENT; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_DU_RESERVED_PERCENT; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_DU_RESERVED_PERCENT_DEFAULT; -import static org.apache.ozone.test.GenericTestUtils.assertThrows; -import static org.apache.ratis.util.Preconditions.assertTrue; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertNotEquals; /** @@ -174,11 +173,9 @@ public void testInvalidConfigThrowsException() { OzoneConfiguration conf = new OzoneConfiguration(); conf.set(ScmConfigKeys.HDDS_DATANODE_DIR_DU_RESERVED, "15GB"); - ConfigurationException exception = assertThrows(ConfigurationException.class, () -> { - volumeBuilder.conf(conf).build(); - return null; - }); - assertTrue(exception.getMessage().contains("Reserved space should be configured in a pair")); + assertThrows(ConfigurationException.class, + () -> volumeBuilder.conf(conf).build(), + "Reserved space should be configured in a pair"); } @Test