Skip to content

HDDS-815. Rename Ozone/HDDS config keys prefixed with 'dfs' #6187

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
sarvekshayr wants to merge 8 commits into from
Closed

HDDS-815. Rename Ozone/HDDS config keys prefixed with 'dfs' #6187

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
2d95bb5
HDDS-815. Rename Ozone/HDDS config keys prefixed with 'dfs'
sarvekshayr Feb 7, 2024
312b36c
Merge remote-tracking branch 'origin' into HDDS-815
sarvekshayr Feb 7, 2024
3804fdf
Addressed the TestContainerStateMachineStream changes
sarvekshayr Feb 7, 2024
13000fa
Corrected the indentation
sarvekshayr Feb 7, 2024
7c879b6
Added backward compatibility for all the keys
sarvekshayr Feb 8, 2024
2cbabcc
Fixed a config key
sarvekshayr Feb 8, 2024
9527771
Merge remote-tracking branch 'origin' into HDDS-815
sarvekshayr Feb 9, 2024
3a77d13
Revert "Merge remote-tracking branch 'origin' into HDDS-815"
sarvekshayr Feb 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,7 @@ private synchronized void connectToDatanode(DatanodeDetails dn)
// port.
int port = dn.getPort(DatanodeDetails.Port.Name.STANDALONE).getValue();
if (port == 0) {
port = config.getInt(OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
port = config.getInt(OzoneConfigKeys.OZONE_CONTAINER_IPC_PORT,
OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT);
}

Expand Down
2 changes: 1 addition & 1 deletion hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ public static XceiverClientRatis newXceiverClientRatis(
org.apache.hadoop.hdds.scm.pipeline.Pipeline pipeline,
ConfigurationSource ozoneConf, ClientTrustManager trustManager) {
final String rpcType = ozoneConf
.get(ScmConfigKeys.DFS_CONTAINER_RATIS_RPC_TYPE_KEY,
.get(ScmConfigKeys.OZONE_CONTAINER_RATIS_RPC_TYPE_KEY,
ScmConfigKeys.DFS_CONTAINER_RATIS_RPC_TYPE_DEFAULT);
final RetryPolicy retryPolicy = RatisHelper.createRetryPolicy(ozoneConf);
final GrpcTlsConfig tlsConfig = RatisHelper.createTlsClientConfig(new
Expand Down
31 changes: 30 additions & 1 deletion hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@
import org.apache.hadoop.hdds.utils.LegacyHadoopConfigurationSource;

import com.google.common.base.Preconditions;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.ratis.server.RaftServerConfigKeys;

import static java.util.Collections.unmodifiableSortedSet;
Expand Down Expand Up @@ -323,7 +324,35 @@ private static void addDeprecatedKeys() {
new DeprecationDelta("ozone.scm.chunk.layout",
ScmConfigKeys.OZONE_SCM_CONTAINER_LAYOUT_KEY),
new DeprecationDelta("hdds.datanode.replication.work.dir",
OZONE_CONTAINER_COPY_WORKDIR)
OZONE_CONTAINER_COPY_WORKDIR),
new DeprecationDelta("dfs.container.ratis.enabled",
ScmConfigKeys.OZONE_CONTAINER_RATIS_ENABLED_KEY),
new DeprecationDelta("dfs.container.ratis.rpc.type",
ScmConfigKeys.OZONE_CONTAINER_RATIS_RPC_TYPE_KEY),
new DeprecationDelta("dfs.container.ratis.replication.level",
ScmConfigKeys.OZONE_CONTAINER_RATIS_REPLICATION_LEVEL_KEY),
new DeprecationDelta("dfs.container.ratis.num.container.op.executors",
ScmConfigKeys.OZONE_CONTAINER_RATIS_NUM_CONTAINER_OP_EXECUTORS_KEY),
new DeprecationDelta("dfs.container.ratis.segment.size",
ScmConfigKeys.OZONE_CONTAINER_RATIS_SEGMENT_SIZE_KEY),
new DeprecationDelta("dfs.container.ratis.segment.preallocated.size",
ScmConfigKeys.OZONE_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY),
new DeprecationDelta("dfs.container.ratis.statemachinedata.sync.timeout",
ScmConfigKeys.OZONE_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT),
new DeprecationDelta("dfs.ratis.leader.election.minimum.timeout.duration",
ScmConfigKeys.OZONE_RATIS_LEADER_ELECTION_MINIMUM_TIMEOUT_DURATION_KEY),
new DeprecationDelta("dfs.ratis.snapshot.threshold",
ScmConfigKeys.OZONE_RATIS_SNAPSHOT_THRESHOLD_KEY),
new DeprecationDelta("dfs.container.ipc",
OzoneConfigKeys.OZONE_CONTAINER_IPC_PORT),
new DeprecationDelta("dfs.container.ipc.random.port",
OzoneConfigKeys.OZONE_CONTAINER_IPC_RANDOM_PORT),
new DeprecationDelta("dfs.container.ratis.ipc",
OzoneConfigKeys.OZONE_CONTAINER_RATIS_IPC_PORT),
new DeprecationDelta("dfs.container.ratis.ipc.random.port",
OzoneConfigKeys.OZONE_CONTAINER_RATIS_IPC_RANDOM_PORT),
new DeprecationDelta("dfs.container.ratis.datanode.storage.dir",
OzoneConfigKeys.OZONE_CONTAINER_RATIS_DATANODE_STORAGE_DIR)
});
}

Expand Down
2 changes: 1 addition & 1 deletion hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisHelper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -234,7 +234,7 @@ public static RaftClient newRaftClient(RpcType rpcType, Pipeline pipeline,

private static RpcType getRpcType(ConfigurationSource conf) {
return SupportedRpcType.valueOfIgnoreCase(conf.get(
ScmConfigKeys.DFS_CONTAINER_RATIS_RPC_TYPE_KEY,
ScmConfigKeys.OZONE_CONTAINER_RATIS_RPC_TYPE_KEY,
ScmConfigKeys.DFS_CONTAINER_RATIS_RPC_TYPE_DEFAULT));
}

Expand Down
36 changes: 18 additions & 18 deletions hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,12 +41,12 @@ public final class ScmConfigKeys {
public static final String OZONE_SCM_DB_DIRS_PERMISSIONS =
"ozone.scm.db.dirs.permissions";

public static final String DFS_CONTAINER_RATIS_ENABLED_KEY
= "dfs.container.ratis.enabled";
public static final String OZONE_CONTAINER_RATIS_ENABLED_KEY
= "ozone.container.ratis.enabled";
public static final boolean DFS_CONTAINER_RATIS_ENABLED_DEFAULT
= false;
public static final String DFS_CONTAINER_RATIS_RPC_TYPE_KEY
= "dfs.container.ratis.rpc.type";
public static final String OZONE_CONTAINER_RATIS_RPC_TYPE_KEY
= "ozone.container.ratis.rpc.type";
public static final String DFS_CONTAINER_RATIS_RPC_TYPE_DEFAULT
= "GRPC";
public static final String
Expand All @@ -55,25 +55,25 @@ public final class ScmConfigKeys {
public static final int
DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_PER_VOLUME_DEFAULT
= 10;
public static final String DFS_CONTAINER_RATIS_REPLICATION_LEVEL_KEY
= "dfs.container.ratis.replication.level";
public static final String OZONE_CONTAINER_RATIS_REPLICATION_LEVEL_KEY
= "ozone.container.ratis.replication.level";
public static final ReplicationLevel
DFS_CONTAINER_RATIS_REPLICATION_LEVEL_DEFAULT = ReplicationLevel.MAJORITY;
public static final String DFS_CONTAINER_RATIS_NUM_CONTAINER_OP_EXECUTORS_KEY
= "dfs.container.ratis.num.container.op.executors";
public static final String OZONE_CONTAINER_RATIS_NUM_CONTAINER_OP_EXECUTORS_KEY
= "ozone.container.ratis.num.container.op.executors";
public static final int DFS_CONTAINER_RATIS_NUM_CONTAINER_OP_EXECUTORS_DEFAULT
= 10;
public static final String DFS_CONTAINER_RATIS_SEGMENT_SIZE_KEY =
"dfs.container.ratis.segment.size";
public static final String OZONE_CONTAINER_RATIS_SEGMENT_SIZE_KEY =
"ozone.container.ratis.segment.size";
public static final String DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT =
"64MB";
public static final String DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY =
"dfs.container.ratis.segment.preallocated.size";
public static final String OZONE_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY =
"ozone.container.ratis.segment.preallocated.size";
public static final String
DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT = "4MB";
public static final String
DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT =
"dfs.container.ratis.statemachinedata.sync.timeout";
OZONE_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT =
"ozone.container.ratis.statemachinedata.sync.timeout";
public static final TimeDuration
DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT_DEFAULT =
TimeDuration.valueOf(10, TimeUnit.SECONDS);
Expand Down Expand Up @@ -121,14 +121,14 @@ public final class ScmConfigKeys {
DFS_RATIS_SERVER_RETRY_CACHE_TIMEOUT_DURATION_DEFAULT =
TimeDuration.valueOf(600000, TimeUnit.MILLISECONDS);
public static final String
DFS_RATIS_LEADER_ELECTION_MINIMUM_TIMEOUT_DURATION_KEY =
"dfs.ratis.leader.election.minimum.timeout.duration";
OZONE_RATIS_LEADER_ELECTION_MINIMUM_TIMEOUT_DURATION_KEY =
"ozone.ratis.leader.election.minimum.timeout.duration";
public static final TimeDuration
DFS_RATIS_LEADER_ELECTION_MINIMUM_TIMEOUT_DURATION_DEFAULT =
TimeDuration.valueOf(5, TimeUnit.SECONDS);

public static final String DFS_RATIS_SNAPSHOT_THRESHOLD_KEY =
"dfs.ratis.snapshot.threshold";
public static final String OZONE_RATIS_SNAPSHOT_THRESHOLD_KEY =
"ozone.ratis.snapshot.threshold";
public static final long DFS_RATIS_SNAPSHOT_THRESHOLD_DEFAULT = 100000;

// TODO : this is copied from OzoneConsts, may need to move to a better place
Expand Down
58 changes: 29 additions & 29 deletions hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,8 @@
@InterfaceAudience.Public
@InterfaceStability.Unstable
public final class OzoneConfigKeys {
public static final String DFS_CONTAINER_IPC_PORT =
"dfs.container.ipc";
public static final String OZONE_CONTAINER_IPC_PORT =
"ozone.container.ipc.port";
public static final int DFS_CONTAINER_IPC_PORT_DEFAULT = 9859;

public static final String OZONE_METADATA_DIRS = "ozone.metadata.dirs";
Expand All @@ -56,11 +56,11 @@ public final class OzoneConfigKeys {
* so that a mini cluster is able to launch multiple containers on a node.
*
* When set to false (default), the container port will be specified as
* {@link #DFS_CONTAINER_IPC_PORT} and the default value will be specified
* {@link #OZONE_CONTAINER_IPC_PORT} and the default value will be specified
* as {@link #DFS_CONTAINER_IPC_PORT_DEFAULT}.
*/
public static final String DFS_CONTAINER_IPC_RANDOM_PORT =
"dfs.container.ipc.random.port";
public static final String OZONE_CONTAINER_IPC_RANDOM_PORT =
"ozone.container.ipc.random.port";
public static final boolean DFS_CONTAINER_IPC_RANDOM_PORT_DEFAULT =
false;

Expand All @@ -76,8 +76,8 @@ public final class OzoneConfigKeys {
/**
* Ratis Port where containers listen to.
*/
public static final String DFS_CONTAINER_RATIS_IPC_PORT =
"dfs.container.ratis.ipc";
public static final String OZONE_CONTAINER_RATIS_IPC_PORT =
"ozone.container.ratis.ipc";
public static final int DFS_CONTAINER_RATIS_IPC_PORT_DEFAULT = 9858;
/**
* Ratis Port where containers listen to admin requests.
Expand Down Expand Up @@ -133,8 +133,8 @@ public final class OzoneConfigKeys {
* When set to true, allocate a random free port for ozone container, so that
* a mini cluster is able to launch multiple containers on a node.
*/
public static final String DFS_CONTAINER_RATIS_IPC_RANDOM_PORT =
"dfs.container.ratis.ipc.random.port";
public static final String OZONE_CONTAINER_RATIS_IPC_RANDOM_PORT =
"ozone.container.ratis.ipc.random.port";
public static final boolean DFS_CONTAINER_RATIS_IPC_RANDOM_PORT_DEFAULT =
false;

Expand Down Expand Up @@ -325,12 +325,12 @@ public final class OzoneConfigKeys {
public static final int
OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL_DEFAULT = 10;

public static final String DFS_CONTAINER_RATIS_ENABLED_KEY
= ScmConfigKeys.DFS_CONTAINER_RATIS_ENABLED_KEY;
public static final String OZONE_CONTAINER_RATIS_ENABLED_KEY
= ScmConfigKeys.OZONE_CONTAINER_RATIS_ENABLED_KEY;
public static final boolean DFS_CONTAINER_RATIS_ENABLED_DEFAULT
= ScmConfigKeys.DFS_CONTAINER_RATIS_ENABLED_DEFAULT;
public static final String DFS_CONTAINER_RATIS_RPC_TYPE_KEY
= ScmConfigKeys.DFS_CONTAINER_RATIS_RPC_TYPE_KEY;
public static final String OZONE_CONTAINER_RATIS_RPC_TYPE_KEY
= ScmConfigKeys.OZONE_CONTAINER_RATIS_RPC_TYPE_KEY;
public static final String DFS_CONTAINER_RATIS_RPC_TYPE_DEFAULT
= ScmConfigKeys.DFS_CONTAINER_RATIS_RPC_TYPE_DEFAULT;
public static final String
Expand All @@ -340,35 +340,35 @@ public final class OzoneConfigKeys {
DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_PER_VOLUME_DEFAULT
= ScmConfigKeys.
DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_PER_VOLUME_DEFAULT;
public static final String DFS_CONTAINER_RATIS_REPLICATION_LEVEL_KEY
= ScmConfigKeys.DFS_CONTAINER_RATIS_REPLICATION_LEVEL_KEY;
public static final String OZONE_CONTAINER_RATIS_REPLICATION_LEVEL_KEY
= ScmConfigKeys.OZONE_CONTAINER_RATIS_REPLICATION_LEVEL_KEY;
public static final ReplicationLevel
DFS_CONTAINER_RATIS_REPLICATION_LEVEL_DEFAULT
= ScmConfigKeys.DFS_CONTAINER_RATIS_REPLICATION_LEVEL_DEFAULT;
public static final String DFS_CONTAINER_RATIS_NUM_CONTAINER_OP_EXECUTORS_KEY
= ScmConfigKeys.DFS_CONTAINER_RATIS_NUM_CONTAINER_OP_EXECUTORS_KEY;
public static final String OZONE_CONTAINER_RATIS_NUM_CONTAINER_OP_EXECUTORS_KEY
= ScmConfigKeys.OZONE_CONTAINER_RATIS_NUM_CONTAINER_OP_EXECUTORS_KEY;
public static final int DFS_CONTAINER_RATIS_NUM_CONTAINER_OP_EXECUTORS_DEFAULT
= ScmConfigKeys.DFS_CONTAINER_RATIS_NUM_CONTAINER_OP_EXECUTORS_DEFAULT;
public static final String DFS_CONTAINER_RATIS_SEGMENT_SIZE_KEY
= ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_SIZE_KEY;
public static final String OZONE_CONTAINER_RATIS_SEGMENT_SIZE_KEY
= ScmConfigKeys.OZONE_CONTAINER_RATIS_SEGMENT_SIZE_KEY;
public static final String DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT
= ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT;
public static final String DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY
= ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY;
public static final String OZONE_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY
= ScmConfigKeys.OZONE_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY;
public static final String
DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT =
ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT;

// config settings to enable stateMachineData write timeout
public static final String
DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT =
ScmConfigKeys.DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT;
OZONE_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT =
ScmConfigKeys.OZONE_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT;
public static final TimeDuration
DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT_DEFAULT =
ScmConfigKeys.DFS_CONTAINER_RATIS_STATEMACHINEDATA_SYNC_TIMEOUT_DEFAULT;

public static final String DFS_CONTAINER_RATIS_DATANODE_STORAGE_DIR =
"dfs.container.ratis.datanode.storage.dir";
public static final String OZONE_CONTAINER_RATIS_DATANODE_STORAGE_DIR =
"ozone.container.ratis.datanode.storage.dir";

public static final String DFS_RATIS_SERVER_RETRY_CACHE_TIMEOUT_DURATION_KEY =
ScmConfigKeys.DFS_RATIS_SERVER_RETRY_CACHE_TIMEOUT_DURATION_KEY;
Expand Down Expand Up @@ -407,13 +407,13 @@ public final class OzoneConfigKeys {
DFS_CONTAINER_RATIS_LEADER_PENDING_BYTES_LIMIT_DEFAULT =
ScmConfigKeys.DFS_CONTAINER_RATIS_LEADER_PENDING_BYTES_LIMIT_DEFAULT;
public static final String
DFS_RATIS_LEADER_ELECTION_MINIMUM_TIMEOUT_DURATION_KEY =
ScmConfigKeys.DFS_RATIS_LEADER_ELECTION_MINIMUM_TIMEOUT_DURATION_KEY;
OZONE_RATIS_LEADER_ELECTION_MINIMUM_TIMEOUT_DURATION_KEY =
ScmConfigKeys.OZONE_RATIS_LEADER_ELECTION_MINIMUM_TIMEOUT_DURATION_KEY;
public static final TimeDuration
DFS_RATIS_LEADER_ELECTION_MINIMUM_TIMEOUT_DURATION_DEFAULT =
ScmConfigKeys.DFS_RATIS_LEADER_ELECTION_MINIMUM_TIMEOUT_DURATION_DEFAULT;
public static final String DFS_RATIS_SNAPSHOT_THRESHOLD_KEY =
ScmConfigKeys.DFS_RATIS_SNAPSHOT_THRESHOLD_KEY;
public static final String OZONE_RATIS_SNAPSHOT_THRESHOLD_KEY =
ScmConfigKeys.OZONE_RATIS_SNAPSHOT_THRESHOLD_KEY;
public static final long DFS_RATIS_SNAPSHOT_THRESHOLD_DEFAULT =
ScmConfigKeys.DFS_RATIS_SNAPSHOT_THRESHOLD_DEFAULT;

Expand Down
Loading