From 0036645677fedb450d81925ef9568a285b3296de Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Thu, 3 Apr 2025 11:34:36 -0700 Subject: [PATCH 1/3] HDDS-12761. Add missing network port configuration properties into ozone-default.xml. Change-Id: I27af73d4239fffffed0669dadd9bc64aaedd1786 --- .../apache/hadoop/hdds/scm/ScmConfigKeys.java | 4 -- .../src/main/resources/ozone-default.xml | 44 ++++++++++++++----- .../hadoop/ozone/UniformDatanodesFactory.java | 2 - 3 files changed, 34 insertions(+), 16 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java index 34848a9d5f73..7140b688fa7b 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java @@ -225,10 +225,6 @@ public final class ScmConfigKeys { public static final String OZONE_SCM_HTTP_BIND_HOST_DEFAULT = "0.0.0.0"; public static final int OZONE_SCM_HTTP_BIND_PORT_DEFAULT = 9876; public static final int OZONE_SCM_HTTPS_BIND_PORT_DEFAULT = 9877; - - public static final String HDDS_REST_HTTP_ADDRESS_KEY = - "hdds.rest.http-address"; - public static final String HDDS_REST_HTTP_ADDRESS_DEFAULT = "0.0.0.0:9880"; public static final String HDDS_DATANODE_DIR_KEY = "hdds.datanode.dir"; public static final String HDDS_DATANODE_DIR_DU_RESERVED = "hdds.datanode.dir.du.reserved"; diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index 3bc86c370c21..a68e4c4b0517 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -1527,12 +1527,6 @@ - - hdds.rest.http-address - 0.0.0.0:9880 - The http address of Object Store REST server inside the - datanode. - hdds.datanode.plugins @@ -1853,10 +1847,11 @@ ozone.s3g.https-address - + 0.0.0.0:9879 OZONE, S3GATEWAY - Ozone S3Gateway serverHTTPS server address and port - . + + Ozone S3Gateway serverHTTPS server address and port. + @@ -1878,7 +1873,7 @@ ozone.s3g.webadmin.https-address - + 0.0.0.0:19879 OZONE, S3GATEWAY Ozone S3Gateway content server's HTTPS address and port. @@ -3049,6 +3044,15 @@ Property to determine the transport protocol for the client to Ozone Manager channel. + + + ozone.recon.address + 0.0.0.0:9891 + RECON, MANAGEMENT + + RPC address of Recon Server. + + ozone.recon.http.enabled true @@ -3959,6 +3963,26 @@ OM/SCM/DN/S3GATEWAY Server connection timeout in milliseconds. + + + ozone.om.s3.grpc.server_enabled + true + OZONE, OM, S3GATEWAY + + Property to enable or disable Ozone Manager gRPC endpoint for clients. Right now, it is used by S3 Gateway only. + + + + + + ozone.om.grpc.port + + OZONE, OM, S3GATEWAY + + gRPC endpoint for clients to connect to Ozone Manager. + + + ozone.om.grpc.maximum.response.length 134217728 diff --git a/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/UniformDatanodesFactory.java b/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/UniformDatanodesFactory.java index fed26e641389..771d2b190ce3 100644 --- a/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/UniformDatanodesFactory.java +++ b/hadoop-ozone/mini-cluster/src/main/java/org/apache/hadoop/ozone/UniformDatanodesFactory.java @@ -24,7 +24,6 @@ import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_DU_RESERVED; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_KEY; -import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_REST_HTTP_ADDRESS_KEY; import static org.apache.hadoop.ozone.HddsDatanodeService.TESTING_DATANODE_VERSION_CURRENT; import static org.apache.hadoop.ozone.HddsDatanodeService.TESTING_DATANODE_VERSION_INITIAL; import static org.apache.hadoop.ozone.OzoneConfigKeys.HDDS_CONTAINER_IPC_PORT; @@ -124,7 +123,6 @@ public OzoneConfiguration apply(OzoneConfiguration conf) throws IOException { } private void configureDatanodePorts(ConfigurationTarget conf) { - conf.set(HDDS_REST_HTTP_ADDRESS_KEY, anyHostWithFreePort()); conf.set(HDDS_DATANODE_HTTP_ADDRESS_KEY, anyHostWithFreePort()); conf.set(HDDS_DATANODE_CLIENT_ADDRESS_KEY, anyHostWithFreePort()); conf.setInt(HDDS_CONTAINER_IPC_PORT, getFreePort()); From d2da8a09cfddc761118d73d44af9cab57ec4934d Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Tue, 8 Apr 2025 19:27:58 -0700 Subject: [PATCH 2/3] Remove default recon server value Change-Id: I3ea4f985bb27fb96bf1ce719a88a9c62f586d131 --- hadoop-hdds/common/src/main/resources/ozone-default.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index a68e4c4b0517..20486cd02b92 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -3047,10 +3047,10 @@ ozone.recon.address - 0.0.0.0:9891 + RECON, MANAGEMENT - RPC address of Recon Server. + RPC address of Recon Server. If not set, datanodes will not configure Recon Server. From 1e6d5d6aead8e8941be051b773d18b5698529f88 Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Wed, 9 Apr 2025 14:20:18 -0700 Subject: [PATCH 3/3] Fix tests Change-Id: I064eb36a45e69bc62ad37c32acccf095db02b9b4 --- .../common/src/main/resources/ozone-default.xml | 12 +----------- .../hadoop/ozone/TestOzoneConfigurationFields.java | 2 -- 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index 20486cd02b92..21107e145ba2 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -1850,7 +1850,7 @@ 0.0.0.0:9879 OZONE, S3GATEWAY - Ozone S3Gateway serverHTTPS server address and port. + Ozone S3Gateway HTTPS server address and port. @@ -3973,16 +3973,6 @@ - - - ozone.om.grpc.port - - OZONE, OM, S3GATEWAY - - gRPC endpoint for clients to connect to Ozone Manager. - - - ozone.om.grpc.maximum.response.length 134217728 diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java index 45dc992980d8..01dae381572a 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java @@ -91,7 +91,6 @@ private void addPropertiesNotInXml() { ScmConfigKeys.OZONE_CHUNK_READ_NETTY_CHUNKED_NIO_FILE_KEY, OMConfigKeys.OZONE_FS_TRASH_INTERVAL_KEY, OMConfigKeys.OZONE_FS_TRASH_CHECKPOINT_INTERVAL_KEY, - OMConfigKeys.OZONE_OM_S3_GPRC_SERVER_ENABLED, OMConfigKeys.OZONE_OM_FEATURES_DISABLED, OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS, OzoneConfigKeys.OZONE_ACL_AUTHORIZER_CLASS_NATIVE, @@ -101,7 +100,6 @@ private void addPropertiesNotInXml() { OzoneConfigKeys.OZONE_RECOVERING_CONTAINER_TIMEOUT, OzoneConfigKeys.OZONE_GPRC_METRICS_PERCENTILES_INTERVALS_KEY, ReconConfigKeys.RECON_SCM_CONFIG_PREFIX, - ReconConfigKeys.OZONE_RECON_ADDRESS_KEY, ReconConfigKeys.OZONE_RECON_DATANODE_ADDRESS_KEY, ReconConfigKeys.OZONE_RECON_DATANODE_BIND_HOST_KEY, ReconConfigKeys.OZONE_RECON_PROMETHEUS_HTTP_ENDPOINT,