Skip to content

HDDS-12164. Rename DFSConfigKeysLegacy config keys prefixed with 'dfs' #7803

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
adoroszlai merged 3 commits into from
Feb 18, 2025
Merged

HDDS-12164. Rename DFSConfigKeysLegacy config keys prefixed with 'dfs' #7803

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
647fa5f
HDDS-12164. Rename DFSConfigKeysLegacy config keys prefixed with 'dfs'
sarvekshayr Jan 31, 2025
c5b683a
Addressed comments
sarvekshayr Feb 13, 2025
3822902
Removed dfs.web.authentication config
sarvekshayr Feb 17, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 16 additions & 16 deletions hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/DFSConfigKeysLegacy.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,58 +29,58 @@ private DFSConfigKeysLegacy() {
}

public static final String DFS_DATANODE_DNS_INTERFACE_KEY =
"dfs.datanode.dns.interface";
"hdds.datanode.dns.interface";
public static final String DFS_DATANODE_DNS_NAMESERVER_KEY =
"dfs.datanode.dns.nameserver";
"hdds.datanode.dns.nameserver";

public static final String DFS_DATANODE_HOST_NAME_KEY =
"dfs.datanode.hostname";
"hdds.datanode.hostname";

public static final String DFS_DATANODE_DATA_DIR_KEY =
"dfs.datanode.data.dir";
"hdds.datanode.data.dir";

public static final String DFS_DATANODE_USE_DN_HOSTNAME =
"dfs.datanode.use.datanode.hostname";
"hdds.datanode.use.datanode.hostname";

public static final boolean DFS_DATANODE_USE_DN_HOSTNAME_DEFAULT = false;

public static final String DFS_XFRAME_OPTION_ENABLED = "dfs.xframe.enabled";
public static final String DFS_XFRAME_OPTION_ENABLED = "hdds.xframe.enabled";

public static final boolean DFS_XFRAME_OPTION_ENABLED_DEFAULT = true;

public static final String DFS_XFRAME_OPTION_VALUE = "dfs.xframe.value";
public static final String DFS_XFRAME_OPTION_VALUE = "hdds.xframe.value";

public static final String DFS_XFRAME_OPTION_VALUE_DEFAULT = "SAMEORIGIN";

public static final String DFS_METRICS_SESSION_ID_KEY =
"dfs.metrics.session-id";
"hdds.metrics.session-id";

public static final String NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY =
"net.topology.node.switch.mapping.impl";

public static final String DFS_CLIENT_HTTPS_KEYSTORE_RESOURCE_KEY =
"dfs.client.https.keystore.resource";
"hdds.client.https.keystore.resource";

public static final String DFS_SERVER_HTTPS_KEYSTORE_RESOURCE_KEY =
"dfs.https.server.keystore.resource";
"hdds.https.server.keystore.resource";

public static final String DFS_HTTP_POLICY_KEY = "dfs.http.policy";
public static final String DFS_HTTP_POLICY_KEY = "hdds.http.policy";

public static final String DFS_DATANODE_KERBEROS_PRINCIPAL_KEY =
"dfs.datanode.kerberos.principal";
"hdds.datanode.kerberos.principal";

@Deprecated
public static final String DFS_DATANODE_KEYTAB_FILE_KEY =
"dfs.datanode.keytab.file";
"hdds.datanode.keytab.file";

public static final String DFS_DATANODE_KERBEROS_KEYTAB_FILE_KEY =
"dfs.datanode.kerberos.keytab.file";
"hdds.datanode.kerberos.keytab.file";

public static final String DFS_METRICS_PERCENTILES_INTERVALS_KEY =
"dfs.metrics.percentiles.intervals";
"hdds.metrics.percentiles.intervals";

public static final String DFS_WEB_AUTHENTICATION_KERBEROS_KEYTAB_KEY =
"dfs.web.authentication.kerberos.keytab";
"hdds.web.authentication.kerberos.keytab";

}

Expand Down
2 changes: 1 addition & 1 deletion hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -360,7 +360,7 @@ public static InetSocketAddress getReconAddresses(
* @param conf Configuration
*
* @return the hostname (NB: may not be a FQDN)
* @throws UnknownHostException if the dfs.datanode.dns.interface
* @throws UnknownHostException if the hdds.datanode.dns.interface
* option is used and the hostname can not be determined
*/
public static String getHostName(ConfigurationSource conf)
Expand Down
32 changes: 31 additions & 1 deletion hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -382,7 +382,37 @@ private static void addDeprecatedKeys() {
new DeprecationDelta("dfs.ratis.server.retry-cache.timeout.duration",
ScmConfigKeys.HDDS_RATIS_SERVER_RETRY_CACHE_TIMEOUT_DURATION_KEY),
new DeprecationDelta("dfs.ratis.snapshot.threshold",
ScmConfigKeys.HDDS_RATIS_SNAPSHOT_THRESHOLD_KEY)
ScmConfigKeys.HDDS_RATIS_SNAPSHOT_THRESHOLD_KEY),
new DeprecationDelta("dfs.datanode.dns.interface",
DFSConfigKeysLegacy.DFS_DATANODE_DNS_INTERFACE_KEY),
new DeprecationDelta("dfs.datanode.dns.nameserver",
DFSConfigKeysLegacy.DFS_DATANODE_DNS_NAMESERVER_KEY),
new DeprecationDelta("dfs.datanode.hostname",
DFSConfigKeysLegacy.DFS_DATANODE_HOST_NAME_KEY),
new DeprecationDelta("dfs.datanode.data.dir",
DFSConfigKeysLegacy.DFS_DATANODE_DATA_DIR_KEY),
new DeprecationDelta("dfs.datanode.use.datanode.hostname",
DFSConfigKeysLegacy.DFS_DATANODE_USE_DN_HOSTNAME),
new DeprecationDelta("dfs.xframe.enabled",
DFSConfigKeysLegacy.DFS_XFRAME_OPTION_ENABLED),
new DeprecationDelta("dfs.xframe.value",
DFSConfigKeysLegacy.DFS_XFRAME_OPTION_VALUE),
new DeprecationDelta("dfs.metrics.session-id",
DFSConfigKeysLegacy.DFS_METRICS_SESSION_ID_KEY),
new DeprecationDelta("dfs.client.https.keystore.resource",
DFSConfigKeysLegacy.DFS_CLIENT_HTTPS_KEYSTORE_RESOURCE_KEY),
new DeprecationDelta("dfs.https.server.keystore.resource",
DFSConfigKeysLegacy.DFS_SERVER_HTTPS_KEYSTORE_RESOURCE_KEY),
new DeprecationDelta("dfs.http.policy",
DFSConfigKeysLegacy.DFS_HTTP_POLICY_KEY),
new DeprecationDelta("dfs.datanode.kerberos.principal",
DFSConfigKeysLegacy.DFS_DATANODE_KERBEROS_PRINCIPAL_KEY),
new DeprecationDelta("dfs.datanode.kerberos.keytab.file",
DFSConfigKeysLegacy.DFS_DATANODE_KERBEROS_KEYTAB_FILE_KEY),
new DeprecationDelta("dfs.metrics.percentiles.intervals",
DFSConfigKeysLegacy.DFS_METRICS_PERCENTILES_INTERVALS_KEY),
new DeprecationDelta("dfs.web.authentication.kerberos.keytab",
DFSConfigKeysLegacy.DFS_WEB_AUTHENTICATION_KERBEROS_KEYTAB_KEY)
});
}

Expand Down
2 changes: 1 addition & 1 deletion hadoop-hdds/common/src/main/resources/ozone-default.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -172,7 +172,7 @@
<value/>
<tag>OZONE, CONTAINER, STORAGE, MANAGEMENT</tag>
<description>Determines where on the local filesystem HDDS data will be
stored. Defaults to dfs.datanode.data.dir if not specified.
stored. Defaults to hdds.datanode.data.dir if not specified.
The directories should be tagged with corresponding storage types
([SSD]/[DISK]/[ARCHIVE]/[RAM_DISK]) for storage policies. The default
storage type will be DISK if the directory does not have a storage type
Expand Down
4 changes: 2 additions & 2 deletions hadoop-hdds/docs/content/security/SecuringDatanodes.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,8 @@ that is setup in hdfs-site.xml.

Property|Description
--------|--------------
dfs.datanode.kerberos.principal|The datanode service principal. <br/> e.g. dn/[email protected]
dfs.datanode.kerberos.keytab.file| The keytab file used by datanode daemon to login as its service principal.
hdds.datanode.kerberos.principal|The datanode service principal. <br/> e.g. dn/[email protected]
hdds.datanode.kerberos.keytab.file| The keytab file used by datanode daemon to login as its service principal.
hdds.datanode.http.auth.kerberos.principal| Datanode http server service principal.
hdds.datanode.http.auth.kerberos.keytab| The keytab file used by datanode http server to login as its service principal.

Expand Down
4 changes: 2 additions & 2 deletions hadoop-hdds/docs/content/security/SecuringDatanodes.zh.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,8 @@ Hadoop 中 datanode 的安全机制是通过给每个节点创建 Keytab 文件

参数名|描述
--------|--------------
dfs.datanode.kerberos.principal| datanode 的服务主体名 <br/> 比如：dn/[email protected]
dfs.datanode.kerberos.keytab.file| datanode 进程所使用的 keytab 文件
hdds.datanode.kerberos.principal| datanode 的服务主体名 <br/> 比如：dn/[email protected]
hdds.datanode.kerberos.keytab.file| datanode 进程所使用的 keytab 文件
hdds.datanode.http.auth.kerberos.principal| datanode http 服务器的服务主体名
hdds.datanode.http.auth.kerberos.keytab| datanode http 服务器的服务主体登录所使用的 keytab 文件

Expand Down
6 changes: 3 additions & 3 deletions hadoop-ozone/dist/src/main/compose/common/security.conf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,10 +36,10 @@ OZONE-SITE.XML_ozone.s3g.kerberos.principal=s3g/[email protected]
OZONE-SITE.XML_ozone.httpfs.kerberos.keytab.file=/etc/security/keytabs/httpfs.keytab
OZONE-SITE.XML_ozone.httpfs.kerberos.principal=httpfs/[email protected]

HDFS-SITE.XML_dfs.datanode.kerberos.principal=dn/[email protected]
HDFS-SITE.XML_dfs.datanode.kerberos.keytab.file=/etc/security/keytabs/dn.keytab
HDFS-SITE.XML_hdds.datanode.kerberos.principal=dn/[email protected]
HDFS-SITE.XML_hdds.datanode.kerberos.keytab.file=/etc/security/keytabs/dn.keytab
HDFS-SITE.XML_dfs.web.authentication.kerberos.principal=HTTP/[email protected]
HDFS-SITE.XML_dfs.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
HDFS-SITE.XML_hdds.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab

OZONE-SITE.XML_hdds.block.token.enabled=true
OZONE-SITE.XML_hdds.container.token.enabled=true
Expand Down
6 changes: 3 additions & 3 deletions hadoop-ozone/dist/src/main/compose/ozonesecure-ha/docker-config
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,10 +92,10 @@ OZONE-SITE.XML_ozone.s3g.kerberos.principal=s3g/[email protected]
OZONE-SITE.XML_ozone.httpfs.kerberos.keytab.file=/etc/security/keytabs/httpfs.keytab
OZONE-SITE.XML_ozone.httpfs.kerberos.principal=httpfs/[email protected]

HDFS-SITE.XML_dfs.datanode.kerberos.principal=dn/[email protected]
HDFS-SITE.XML_dfs.datanode.kerberos.keytab.file=/etc/security/keytabs/dn.keytab
HDFS-SITE.XML_hdds.datanode.kerberos.principal=dn/[email protected]
HDFS-SITE.XML_hdds.datanode.kerberos.keytab.file=/etc/security/keytabs/dn.keytab
HDFS-SITE.XML_dfs.web.authentication.kerberos.principal=HTTP/[email protected]
HDFS-SITE.XML_dfs.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
HDFS-SITE.XML_hdds.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab


OZONE-SITE.XML_ozone.security.http.kerberos.enabled=true
Expand Down
6 changes: 3 additions & 3 deletions hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,10 +56,10 @@ OZONE-SITE.XML_hdds.grpc.tls.enabled=true
OZONE-SITE.XML_ozone.s3g.kerberos.keytab.file=/etc/security/keytabs/s3g.keytab
OZONE-SITE.XML_ozone.s3g.kerberos.principal=s3g/[email protected]

HDFS-SITE.XML_dfs.datanode.kerberos.principal=dn/[email protected]
HDFS-SITE.XML_dfs.datanode.kerberos.keytab.file=/etc/security/keytabs/dn.keytab
HDFS-SITE.XML_hdds.datanode.kerberos.principal=dn/[email protected]
HDFS-SITE.XML_hdds.datanode.kerberos.keytab.file=/etc/security/keytabs/dn.keytab
HDFS-SITE.XML_dfs.web.authentication.kerberos.principal=HTTP/[email protected]
HDFS-SITE.XML_dfs.web.authentication.kerberos.keytab=/etc/security/keytabs/dn.keytab
HDFS-SITE.XML_hdds.web.authentication.kerberos.keytab=/etc/security/keytabs/dn.keytab

OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
OZONE-SITE.XML_hdds.datanode.volume.min.free.space=100MB
Expand Down
6 changes: 3 additions & 3 deletions hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,10 +88,10 @@ OZONE-SITE.XML_ozone.scm.close.container.wait.duration=5s
# Ratis streaming is disabled to ensure coverage for both cases
OZONE-SITE.XML_hdds.container.ratis.datastream.enabled=false

HDFS-SITE.XML_dfs.datanode.kerberos.principal=dn/[email protected]
HDFS-SITE.XML_dfs.datanode.kerberos.keytab.file=/etc/security/keytabs/dn.keytab
HDFS-SITE.XML_hdds.datanode.kerberos.principal=dn/[email protected]
HDFS-SITE.XML_hdds.datanode.kerberos.keytab.file=/etc/security/keytabs/dn.keytab
HDFS-SITE.XML_dfs.web.authentication.kerberos.principal=HTTP/[email protected]
HDFS-SITE.XML_dfs.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
HDFS-SITE.XML_hdds.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab


OZONE-SITE.XML_ozone.security.http.kerberos.enabled=true
Expand Down
6 changes: 3 additions & 3 deletions hadoop-ozone/dist/src/main/compose/xcompat/docker-config
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,11 +74,11 @@ OZONE-SITE.XML_ozone.s3g.kerberos.principal=s3g/[email protected]
OZONE-SITE.XML_ozone.httpfs.kerberos.keytab.file=/etc/security/keytabs/httpfs.keytab
OZONE-SITE.XML_ozone.httpfs.kerberos.principal=httpfs/[email protected]

HDFS-SITE.XML_dfs.datanode.kerberos.principal=dn/[email protected]
HDFS-SITE.XML_hdds.datanode.kerberos.principal=dn/[email protected]
HDFS-SITE.XML_dfs.datanode.keytab.file=/etc/security/keytabs/dn.keytab
HDFS-SITE.XML_dfs.datanode.kerberos.keytab.file=/etc/security/keytabs/dn.keytab
HDFS-SITE.XML_hdds.datanode.kerberos.keytab.file=/etc/security/keytabs/dn.keytab
HDFS-SITE.XML_dfs.web.authentication.kerberos.principal=HTTP/[email protected]
HDFS-SITE.XML_dfs.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab
HDFS-SITE.XML_hdds.web.authentication.kerberos.keytab=/etc/security/keytabs/HTTP.keytab

OZONE-SITE.XML_ozone.security.http.kerberos.enabled=true
OZONE-SITE.XML_ozone.s3g.secret.http.enabled=true
Expand Down
2 changes: 1 addition & 1 deletion hadoop-ozone/dist/src/main/k8s/definitions/ozone/config.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ data:
HTTPFS-SITE.XML_httpfs.hadoop.config.dir: /opt/hadoop/etc/config
HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.groups: "*"
HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.hosts: "*"
OZONE-SITE.XML_dfs.datanode.use.datanode.hostname: "true"
OZONE-SITE.XML_hdds.datanode.use.datanode.hostname: "true"
OZONE-SITE.XML_hdds.datanode.dir: "/data/storage"
OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3"
OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1"
Expand Down
2 changes: 1 addition & 1 deletion hadoop-ozone/dist/src/main/k8s/examples/getting-started/config-configmap.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ data:
HTTPFS-SITE.XML_httpfs.hadoop.config.dir: /opt/hadoop/etc/config
HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.groups: '*'
HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.hosts: '*'
OZONE-SITE.XML_dfs.datanode.use.datanode.hostname: "true"
OZONE-SITE.XML_hdds.datanode.use.datanode.hostname: "true"
OZONE-SITE.XML_hdds.datanode.dir: /data/storage
OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3"
OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1"
Expand Down
2 changes: 1 addition & 1 deletion hadoop-ozone/dist/src/main/k8s/examples/minikube/config-configmap.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ data:
HTTPFS-SITE.XML_httpfs.hadoop.config.dir: /opt/hadoop/etc/config
HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.groups: '*'
HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.hosts: '*'
OZONE-SITE.XML_dfs.datanode.use.datanode.hostname: "true"
OZONE-SITE.XML_hdds.datanode.use.datanode.hostname: "true"
OZONE-SITE.XML_hdds.datanode.dir: /data/storage
OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3"
OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1"
Expand Down
2 changes: 1 addition & 1 deletion hadoop-ozone/dist/src/main/k8s/examples/ozone-dev/config-configmap.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ data:
HTTPFS-SITE.XML_httpfs.hadoop.config.dir: /opt/hadoop/etc/config
HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.groups: '*'
HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.hosts: '*'
OZONE-SITE.XML_dfs.datanode.use.datanode.hostname: "true"
OZONE-SITE.XML_hdds.datanode.use.datanode.hostname: "true"
OZONE-SITE.XML_hdds.datanode.dir: /data/storage
OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3"
OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1"
Expand Down
2 changes: 1 addition & 1 deletion hadoop-ozone/dist/src/main/k8s/examples/ozone-ha/config-configmap.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ data:
HTTPFS-SITE.XML_httpfs.hadoop.config.dir: /opt/hadoop/etc/config
HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.groups: '*'
HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.hosts: '*'
OZONE-SITE.XML_dfs.datanode.use.datanode.hostname: "true"
OZONE-SITE.XML_hdds.datanode.use.datanode.hostname: "true"
OZONE-SITE.XML_hdds.datanode.dir: /data/storage
OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3"
OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1"
Expand Down
2 changes: 1 addition & 1 deletion hadoop-ozone/dist/src/main/k8s/examples/ozone/config-configmap.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ data:
HTTPFS-SITE.XML_httpfs.hadoop.config.dir: /opt/hadoop/etc/config
HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.groups: '*'
HTTPFS-SITE.XML_httpfs.proxyuser.hadoop.hosts: '*'
OZONE-SITE.XML_dfs.datanode.use.datanode.hostname: "true"
OZONE-SITE.XML_hdds.datanode.use.datanode.hostname: "true"
OZONE-SITE.XML_hdds.datanode.dir: /data/storage
OZONE-SITE.XML_hdds.scm.safemode.min.datanode: "3"
OZONE-SITE.XML_ozone.datanode.pipeline.limit: "1"
Expand Down
2 changes: 1 addition & 1 deletion ...pfsgateway/src/main/java/org/apache/ozone/fs/http/server/metrics/HttpFSServerMetrics.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@
public class HttpFSServerMetrics {

private static final String DFS_METRICS_SESSION_ID_KEY
= "dfs.metrics.session-id";
= "hdds.metrics.session-id";
Comment on lines 50 to +51
Copy link
Contributor

@adoroszlai adoroszlai Feb 5, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This can be removed in HDDS-12172, and use global constant instead.

private @Metric MutableCounterLong bytesWritten;
private @Metric MutableCounterLong bytesRead;

Expand Down