From dcafe00b7e62e7de9afcfca22cdbad6823a3a6d7 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Sun, 4 Aug 2024 10:13:39 +0200 Subject: [PATCH 1/2] HDDS-11274. Replace Hadoop annotations/configs with Ozone-specific ones --- .../org/apache/hadoop/hdds/ratis/RatisHelper.java | 6 +++--- .../java/org/apache/hadoop/util/MutableMinMax.java | 4 ++-- .../hdds/protocol/SecretKeyProtocolDatanode.java | 2 +- .../hdds/protocolPB/SecretKeyProtocolDatanodePB.java | 2 +- .../hadoop/hdds/scm/node/NodeDecommissionManager.java | 6 +++--- .../org/apache/hadoop/fs/ozone/OzoneFsDelete.java | 4 ++-- pom.xml | 11 ++++------- 7 files changed, 16 insertions(+), 19 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisHelper.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisHelper.java index 5288c0bf50bb..d7afe8ed0238 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisHelper.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisHelper.java @@ -32,6 +32,7 @@ import java.util.stream.Collectors; import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.hdds.DFSConfigKeysLegacy; import org.apache.hadoop.hdds.StringUtils; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.conf.OzoneConfiguration; @@ -43,7 +44,6 @@ import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.security.SecurityConfig; -import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.ratis.RaftConfigKeys; import org.apache.ratis.client.RaftClient; import org.apache.ratis.client.RaftClientConfigKeys; @@ -450,8 +450,8 @@ public static Long getMinReplicatedIndex( private static boolean datanodeUseHostName() { return CONF.getBoolean( - DFSConfigKeys.DFS_DATANODE_USE_DN_HOSTNAME, - DFSConfigKeys.DFS_DATANODE_USE_DN_HOSTNAME_DEFAULT); + DFSConfigKeysLegacy.DFS_DATANODE_USE_DN_HOSTNAME, + DFSConfigKeysLegacy.DFS_DATANODE_USE_DN_HOSTNAME_DEFAULT); } private static Class getClass(String name, diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/util/MutableMinMax.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/util/MutableMinMax.java index 1f056ed742b3..0c5802ec25c1 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/util/MutableMinMax.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/util/MutableMinMax.java @@ -17,8 +17,8 @@ */ package org.apache.hadoop.util; -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hdds.annotation.InterfaceAudience; +import org.apache.hadoop.hdds.annotation.InterfaceStability; import org.apache.hadoop.metrics2.MetricsInfo; import org.apache.hadoop.metrics2.MetricsRecordBuilder; import org.apache.hadoop.metrics2.lib.MetricsRegistry; diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/protocol/SecretKeyProtocolDatanode.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/protocol/SecretKeyProtocolDatanode.java index 433c27bd8a8a..269c12ed149c 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/protocol/SecretKeyProtocolDatanode.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/protocol/SecretKeyProtocolDatanode.java @@ -20,7 +20,7 @@ import org.apache.hadoop.security.KerberosInfo; import static org.apache.hadoop.hdds.scm.ScmConfig.ConfigStrings.HDDS_SCM_KERBEROS_PRINCIPAL_KEY; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_KERBEROS_PRINCIPAL_KEY; +import static org.apache.hadoop.hdds.DFSConfigKeysLegacy.DFS_DATANODE_KERBEROS_PRINCIPAL_KEY; /** * The client protocol to access secret key from Datanode. diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/protocolPB/SecretKeyProtocolDatanodePB.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/protocolPB/SecretKeyProtocolDatanodePB.java index 57f53df87925..ff118afff135 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/protocolPB/SecretKeyProtocolDatanodePB.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/protocolPB/SecretKeyProtocolDatanodePB.java @@ -21,7 +21,7 @@ import org.apache.hadoop.security.KerberosInfo; import static org.apache.hadoop.hdds.scm.ScmConfig.ConfigStrings.HDDS_SCM_KERBEROS_PRINCIPAL_KEY; -import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_KERBEROS_PRINCIPAL_KEY; +import static org.apache.hadoop.hdds.DFSConfigKeysLegacy.DFS_DATANODE_KERBEROS_PRINCIPAL_KEY; /** * Protocol for secret key related operations, to be used by datanode diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeDecommissionManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeDecommissionManager.java index a593062bcdb8..d6f0e89c96d2 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeDecommissionManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeDecommissionManager.java @@ -19,6 +19,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.hdds.DFSConfigKeysLegacy; import org.apache.hadoop.hdds.client.ECReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; @@ -34,7 +35,6 @@ import org.apache.hadoop.hdds.scm.ha.SCMContext; import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException; import org.apache.hadoop.hdds.server.events.EventPublisher; -import org.apache.hadoop.hdfs.DFSConfigKeys; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -273,8 +273,8 @@ public NodeDecommissionManager(OzoneConfiguration config, NodeManager nm, Contai ); useHostnames = config.getBoolean( - DFSConfigKeys.DFS_DATANODE_USE_DN_HOSTNAME, - DFSConfigKeys.DFS_DATANODE_USE_DN_HOSTNAME_DEFAULT); + DFSConfigKeysLegacy.DFS_DATANODE_USE_DN_HOSTNAME, + DFSConfigKeysLegacy.DFS_DATANODE_USE_DN_HOSTNAME_DEFAULT); long monitorInterval = config.getTimeDuration( ScmConfigKeys.OZONE_SCM_DATANODE_ADMIN_MONITOR_INTERVAL, diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneFsDelete.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneFsDelete.java index 672d82203013..bc4d1480c55e 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneFsDelete.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneFsDelete.java @@ -23,8 +23,6 @@ import java.util.LinkedList; import java.util.List; -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathIOException; @@ -35,6 +33,8 @@ import org.apache.hadoop.fs.shell.CommandFormat; import org.apache.hadoop.fs.shell.FsCommand; import org.apache.hadoop.fs.shell.PathData; +import org.apache.hadoop.hdds.annotation.InterfaceAudience; +import org.apache.hadoop.hdds.annotation.InterfaceStability; import org.apache.hadoop.util.ToolRunner; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SHELL_SAFELY_DELETE_LIMIT_NUM_FILES; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SHELL_SAFELY_DELETE_LIMIT_NUM_FILES_DEFAULT; diff --git a/pom.xml b/pom.xml index 79b565597b0a..92d310571524 100644 --- a/pom.xml +++ b/pom.xml @@ -1486,24 +1486,21 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs org.apache.hadoop.test.GenericTestUtils org.apache.hadoop.test.LambdaTestUtils + org.apache.hadoop.test.MetricsAssert + org.apache.hadoop.classification.InterfaceAudience + org.apache.hadoop.classification.InterfaceStability org.apache.hadoop.fs.contract.* org.apache.hadoop.tools.contract.* - - true - Use Ozone's version of the same class - - org.apache.hadoop.test.MetricsAssert - - true Use Ozone's similar class org.apache.hadoop.hdfs.MiniDFSCluster + org.apache.hadoop.hdfs.DFSConfigKeys From 38c7d1fba5555bc07c4bf3b3ff5795118499c790 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Sun, 4 Aug 2024 15:05:09 +0200 Subject: [PATCH 2/2] avoid using hadoop.util.Preconditions --- .../org/apache/ozone/compaction/log/CompactionFileInfo.java | 2 +- .../org/apache/ozone/compaction/log/CompactionLogEntry.java | 2 +- pom.xml | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/compaction/log/CompactionFileInfo.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/compaction/log/CompactionFileInfo.java index 5a633bcc6de0..fa0d1f5491d5 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/compaction/log/CompactionFileInfo.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/compaction/log/CompactionFileInfo.java @@ -19,8 +19,8 @@ package org.apache.ozone.compaction.log; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.util.Preconditions; import java.util.Objects; diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/compaction/log/CompactionLogEntry.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/compaction/log/CompactionLogEntry.java index bbd24d1fc326..c27763b97880 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/compaction/log/CompactionLogEntry.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/compaction/log/CompactionLogEntry.java @@ -19,12 +19,12 @@ package org.apache.ozone.compaction.log; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.CompactionLogEntryProto; import org.apache.hadoop.hdds.utils.db.Codec; import org.apache.hadoop.hdds.utils.db.CopyObject; import org.apache.hadoop.hdds.utils.db.DelegatedCodec; import org.apache.hadoop.hdds.utils.db.Proto2Codec; -import org.apache.hadoop.util.Preconditions; import java.util.List; import java.util.Objects; diff --git a/pom.xml b/pom.xml index 92d310571524..5bd100efeb0f 100644 --- a/pom.xml +++ b/pom.xml @@ -1471,6 +1471,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs org.apache.ratis.thirdparty.com.google.common.** org.apache.hadoop.thirdparty.com.google.common.** + org.apache.hadoop.util.Preconditions