Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -128,17 +128,6 @@ public class ScmConfig extends ReconfigurableConfig {
)
private Duration blockDeletionInterval = Duration.ofSeconds(60);

@Config(key = "init.default.layout.version",
defaultValue = "-1",
type = ConfigType.INT,
tags = { ConfigTag.SCM, ConfigTag.UPGRADE },
description =
" Default Layout Version to init the SCM with. Intended to be used " +
"in tests to finalize from an older version of SCM to the " +
"latest. By default, SCM init uses the highest layout version."
)
private int defaultLayoutVersionOnInit = -1;

public Duration getBlockDeletionInterval() {
return blockDeletionInterval;
}
Expand Down Expand Up @@ -195,10 +184,6 @@ public int getBlockDeletionLimit() {
return blockDeletionLimit;
}

public int getScmDefaultLayoutVersionOnInit() {
return defaultLayoutVersionOnInit;
}

/**
* Configuration strings class.
* required for SCMSecurityProtocol where the KerberosInfo references
Expand All @@ -209,6 +194,5 @@ public int getScmDefaultLayoutVersionOnInit() {
public static class ConfigStrings {
public static final String HDDS_SCM_KERBEROS_PRINCIPAL_KEY = "hdds.scm.kerberos.principal";
public static final String HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY = "hdds.scm.kerberos.keytab.file";
public static final String HDDS_SCM_INIT_DEFAULT_LAYOUT_VERSION = "hdds.scm.init.default.layout.version";
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@

package org.apache.hadoop.hdds.scm.server;

import static org.apache.hadoop.hdds.scm.ScmConfig.ConfigStrings.HDDS_SCM_INIT_DEFAULT_LAYOUT_VERSION;
import static org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager.maxLayoutVersion;
import static org.apache.hadoop.ozone.OzoneConsts.PRIMARY_SCM_NODE_ID;
import static org.apache.hadoop.ozone.OzoneConsts.SCM_CERT_SERIAL_ID;
Expand All @@ -41,13 +40,15 @@
*/
public class SCMStorageConfig extends Storage {

public static final String TESTING_INIT_LAYOUT_VERSION_KEY = "testing.hdds.scm.init.layout.version";

/**
* Construct SCMStorageConfig.
* @throws IOException if any directories are inaccessible.
*/
public SCMStorageConfig(OzoneConfiguration conf) throws IOException {
super(NodeType.SCM, ServerUtils.getScmDbDir(conf), STORAGE_DIR,
getInitLayoutVersion(conf, HDDS_SCM_INIT_DEFAULT_LAYOUT_VERSION,
getInitLayoutVersion(conf, TESTING_INIT_LAYOUT_VERSION_KEY,
HDDSLayoutVersionManager::maxLayoutVersion));
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,37 +42,11 @@ public class OmUpgradeConfig {
private long ratisBasedFinalizationTimeout =
Duration.ofSeconds(30).getSeconds();

@Config(key = "init.default.layout.version",
defaultValue = "-1",
type = ConfigType.INT,
tags = {ConfigTag.OM, ConfigTag.UPGRADE},
description = "Default Layout Version to init the OM with. Intended to "
+ "be used in tests to finalize from an older version of OM to the "
+ "latest. By default, OM init uses the highest layout version."
)
private int defaultLayoutVersionOnInit = -1;

/**
* Config key class.
*/
public static class ConfigStrings {
public static final String OZONE_OM_INIT_DEFAULT_LAYOUT_VERSION =
"ozone.om.init.default.layout.version";
}

public long getRatisBasedFinalizationTimeout() {
return ratisBasedFinalizationTimeout;
}

public void setRatisBasedFinalizationTimeout(long timeout) {
this.ratisBasedFinalizationTimeout = timeout;
}

public int getDefaultLayoutVersionOnInit() {
return defaultLayoutVersionOnInit;
}

public void setDefaultLayoutVersionOnInit(int defaultLayoutVersionOnInit) {
this.defaultLayoutVersionOnInit = defaultLayoutVersionOnInit;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_LEASE_HARD_LIMIT;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_OPEN_KEY_CLEANUP_SERVICE_INTERVAL;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_OPEN_KEY_EXPIRE_THRESHOLD;
import static org.apache.hadoop.ozone.om.OmUpgradeConfig.ConfigStrings.OZONE_OM_INIT_DEFAULT_LAYOUT_VERSION;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NOT_SUPPORTED_OPERATION_PRIOR_FINALIZATION;
import static org.apache.hadoop.ozone.upgrade.UpgradeFinalization.isDone;
import static org.apache.hadoop.ozone.upgrade.UpgradeFinalization.isStarting;
Expand Down Expand Up @@ -60,6 +59,7 @@
import org.apache.hadoop.ozone.container.keyvalue.KeyValueHandler;
import org.apache.hadoop.ozone.container.keyvalue.impl.BlockManagerImpl;
import org.apache.hadoop.ozone.container.metadata.AbstractDatanodeStore;
import org.apache.hadoop.ozone.om.OMStorage;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
Expand Down Expand Up @@ -117,7 +117,7 @@ public void init() throws Exception {
conf.setTimeDuration(OZONE_OM_LEASE_HARD_LIMIT,
EXPIRE_THRESHOLD_MS, TimeUnit.MILLISECONDS);
conf.set(OzoneConfigKeys.OZONE_OM_LEASE_SOFT_LIMIT, "0s");
conf.setInt(OZONE_OM_INIT_DEFAULT_LAYOUT_VERSION, OMLayoutFeature.MULTITENANCY_SCHEMA.layoutVersion());
conf.setInt(OMStorage.TESTING_INIT_LAYOUT_VERSION_KEY, OMLayoutFeature.MULTITENANCY_SCHEMA.layoutVersion());

ClientConfigForTesting.newBuilder(StorageUnit.BYTES)
.setBlockSize(BLOCK_SIZE)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,10 @@
import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State.CLOSED;
import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.HEALTHY;
import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.HEALTHY_READONLY;
import static org.apache.hadoop.hdds.scm.ScmConfig.ConfigStrings.HDDS_SCM_INIT_DEFAULT_LAYOUT_VERSION;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_DATANODE_PIPELINE_LIMIT;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_RATIS_PIPELINE_LIMIT;
import static org.apache.hadoop.hdds.scm.pipeline.Pipeline.PipelineState.OPEN;
import static org.apache.hadoop.ozone.om.OmUpgradeConfig.ConfigStrings.OZONE_OM_INIT_DEFAULT_LAYOUT_VERSION;
import static org.apache.hadoop.ozone.upgrade.InjectedUpgradeFinalizationExecutor.UpgradeTestInjectionPoints.AFTER_COMPLETE_FINALIZATION;
import static org.apache.hadoop.ozone.upgrade.InjectedUpgradeFinalizationExecutor.UpgradeTestInjectionPoints.AFTER_POST_FINALIZE_UPGRADE;
import static org.apache.hadoop.ozone.upgrade.InjectedUpgradeFinalizationExecutor.UpgradeTestInjectionPoints.AFTER_PRE_FINALIZE_UPGRADE;
Expand Down Expand Up @@ -67,6 +65,7 @@
import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
import org.apache.hadoop.hdds.scm.pipeline.PipelineManager;
import org.apache.hadoop.hdds.scm.server.SCMConfigurator;
import org.apache.hadoop.hdds.scm.server.SCMStorageConfig;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
import org.apache.hadoop.hdds.scm.server.upgrade.SCMUpgradeFinalizationContext;
import org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls;
Expand All @@ -80,6 +79,7 @@
import org.apache.hadoop.ozone.client.OzoneClientFactory;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
import org.apache.hadoop.ozone.om.OMStorage;
import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature;
import org.apache.hadoop.ozone.upgrade.BasicUpgradeFinalizer;
import org.apache.hadoop.ozone.upgrade.InjectedUpgradeFinalizationExecutor;
Expand Down Expand Up @@ -153,8 +153,8 @@ public static void initClass() {
conf.setInt(OZONE_DATANODE_PIPELINE_LIMIT, 1);
// allow only one FACTOR THREE pipeline.
conf.setInt(OZONE_SCM_RATIS_PIPELINE_LIMIT, NUM_DATA_NODES + 1);
conf.setInt(HDDS_SCM_INIT_DEFAULT_LAYOUT_VERSION, HDDSLayoutFeature.INITIAL_VERSION.layoutVersion());
conf.setInt(OZONE_OM_INIT_DEFAULT_LAYOUT_VERSION, OMLayoutFeature.INITIAL_VERSION.layoutVersion());
conf.setInt(SCMStorageConfig.TESTING_INIT_LAYOUT_VERSION_KEY, HDDSLayoutFeature.INITIAL_VERSION.layoutVersion());
conf.setInt(OMStorage.TESTING_INIT_LAYOUT_VERSION_KEY, OMLayoutFeature.INITIAL_VERSION.layoutVersion());
conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 500, TimeUnit.MILLISECONDS);
conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, 500, TimeUnit.MILLISECONDS);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
package org.apache.hadoop.hdds.upgrade;

import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State.CLOSED;
import static org.apache.hadoop.hdds.scm.ScmConfig.ConfigStrings.HDDS_SCM_INIT_DEFAULT_LAYOUT_VERSION;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
Expand All @@ -41,6 +40,7 @@
import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol;
import org.apache.hadoop.hdds.scm.server.SCMConfigurator;
import org.apache.hadoop.hdds.scm.server.SCMStorageConfig;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
import org.apache.hadoop.hdds.scm.server.upgrade.FinalizationCheckpoint;
import org.apache.hadoop.hdds.scm.server.upgrade.FinalizationStateManagerImpl;
Expand Down Expand Up @@ -88,7 +88,7 @@ public void init(OzoneConfiguration conf,
SCMConfigurator configurator = new SCMConfigurator();
configurator.setUpgradeFinalizationExecutor(executor);

conf.setInt(HDDS_SCM_INIT_DEFAULT_LAYOUT_VERSION, HDDSLayoutFeature.INITIAL_VERSION.layoutVersion());
conf.setInt(SCMStorageConfig.TESTING_INIT_LAYOUT_VERSION_KEY, HDDSLayoutFeature.INITIAL_VERSION.layoutVersion());
conf.set(ScmConfigKeys.OZONE_SCM_HA_RATIS_SERVER_RPC_FIRST_ELECTION_TIMEOUT, "5s");

MiniOzoneHAClusterImpl.Builder clusterBuilder = MiniOzoneCluster.newHABuilder(conf);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,8 +59,6 @@ public void initializeMemberVariables() {
xmlPrefixToSkipCompare.add("ipc.client.rpc-timeout.ms");
xmlPropsToSkipCompare.add("ozone.om.leader.election.minimum.timeout" +
".duration"); // Deprecated config
configurationPropsToSkipCompare
.add(ScmConfig.ConfigStrings.HDDS_SCM_INIT_DEFAULT_LAYOUT_VERSION);
// Currently replication and type configs moved to server side.
configurationPropsToSkipCompare
.add(OzoneConfigKeys.OZONE_REPLICATION);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@

import static org.apache.hadoop.ozone.OzoneConsts.LAYOUT_VERSION_KEY;
import static org.apache.hadoop.ozone.om.OMUpgradeTestUtils.waitForFinalization;
import static org.apache.hadoop.ozone.om.OmUpgradeConfig.ConfigStrings.OZONE_OM_INIT_DEFAULT_LAYOUT_VERSION;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NOT_SUPPORTED_OPERATION_PRIOR_FINALIZATION;
import static org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature.INITIAL_VERSION;
import static org.apache.hadoop.ozone.om.upgrade.OMLayoutVersionManager.maxLayoutVersion;
Expand Down Expand Up @@ -88,7 +87,7 @@ class TestOMBucketLayoutUpgrade {
@BeforeAll
void setup() throws Exception {
OzoneConfiguration conf = new OzoneConfiguration();
conf.setInt(OZONE_OM_INIT_DEFAULT_LAYOUT_VERSION, fromLayoutVersion);
conf.setInt(OMStorage.TESTING_INIT_LAYOUT_VERSION_KEY, fromLayoutVersion);
String omServiceId = UUID.randomUUID().toString();
MiniOzoneHAClusterImpl.Builder builder = MiniOzoneCluster.newHABuilder(conf);
builder.setOMServiceId(omServiceId)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
import static org.apache.hadoop.ozone.OzoneConsts.LAYOUT_VERSION_KEY;
import static org.apache.hadoop.ozone.om.OMUpgradeTestUtils.assertClusterPrepared;
import static org.apache.hadoop.ozone.om.OMUpgradeTestUtils.waitForFinalization;
import static org.apache.hadoop.ozone.om.OmUpgradeConfig.ConfigStrings.OZONE_OM_INIT_DEFAULT_LAYOUT_VERSION;
import static org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature.INITIAL_VERSION;
import static org.apache.hadoop.ozone.om.upgrade.OMLayoutVersionManager.maxLayoutVersion;
import static org.apache.ozone.test.GenericTestUtils.waitFor;
Expand Down Expand Up @@ -123,7 +122,7 @@ void testOMUpgradeFinalizationWithOneOMDown() throws Exception {

private static MiniOzoneHAClusterImpl newCluster(OzoneConfiguration conf)
throws IOException {
conf.setInt(OZONE_OM_INIT_DEFAULT_LAYOUT_VERSION, INITIAL_VERSION.layoutVersion());
conf.setInt(OMStorage.TESTING_INIT_LAYOUT_VERSION_KEY, INITIAL_VERSION.layoutVersion());
MiniOzoneHAClusterImpl.Builder builder = MiniOzoneCluster.newHABuilder(conf);
builder.setOMServiceId(UUID.randomUUID().toString())
.setNumOfOzoneManagers(3)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
package org.apache.hadoop.ozone.om.multitenant;

import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_MULTITENANCY_ENABLED;
import static org.apache.hadoop.ozone.om.OmUpgradeConfig.ConfigStrings.OZONE_OM_INIT_DEFAULT_LAYOUT_VERSION;
import static org.apache.hadoop.ozone.upgrade.UpgradeFinalization.isDone;
import static org.apache.hadoop.ozone.upgrade.UpgradeFinalization.isStarting;
import static org.assertj.core.api.Assertions.assertThat;
Expand All @@ -43,6 +42,7 @@
import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.ozone.client.rpc.RpcClient;
import org.apache.hadoop.ozone.om.OMMultiTenantManagerImpl;
import org.apache.hadoop.ozone.om.OMStorage;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.helpers.S3SecretValue;
import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
Expand Down Expand Up @@ -76,7 +76,7 @@ public static void initClusterProvider() throws Exception {
conf.setBoolean(
OMMultiTenantManagerImpl.OZONE_OM_TENANT_DEV_SKIP_RANGER, true);
conf.setBoolean(OZONE_OM_MULTITENANCY_ENABLED, true);
conf.setInt(OZONE_OM_INIT_DEFAULT_LAYOUT_VERSION, OMLayoutFeature.INITIAL_VERSION.layoutVersion());
conf.setInt(OMStorage.TESTING_INIT_LAYOUT_VERSION_KEY, OMLayoutFeature.INITIAL_VERSION.layoutVersion());
MiniOzoneCluster.Builder builder = MiniOzoneCluster.newBuilder(conf)
.withoutDatanodes();
cluster = builder.build();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_FORCE_FULL_DIFF;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_SST_FILTERING_SERVICE_INTERVAL;
import static org.apache.hadoop.ozone.om.OmSnapshotManager.DELIMITER;
import static org.apache.hadoop.ozone.om.OmUpgradeConfig.ConfigStrings.OZONE_OM_INIT_DEFAULT_LAYOUT_VERSION;
import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.FILE_TABLE;
import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.BUCKET_NOT_FOUND;
Expand Down Expand Up @@ -118,6 +117,7 @@
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
import org.apache.hadoop.ozone.om.KeyManagerImpl;
import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.ozone.om.OMStorage;
import org.apache.hadoop.ozone.om.OmSnapshot;
import org.apache.hadoop.ozone.om.OmSnapshotManager;
import org.apache.hadoop.ozone.om.OzoneManager;
Expand Down Expand Up @@ -215,7 +215,7 @@ private void init() throws Exception {
conf.setEnum(HDDS_DB_PROFILE, DBProfile.TEST);
// Enable filesystem snapshot feature for the test regardless of the default
conf.setBoolean(OMConfigKeys.OZONE_FILESYSTEM_SNAPSHOT_ENABLED_KEY, true);
conf.setInt(OZONE_OM_INIT_DEFAULT_LAYOUT_VERSION, OMLayoutFeature.BUCKET_LAYOUT_SUPPORT.layoutVersion());
conf.setInt(OMStorage.TESTING_INIT_LAYOUT_VERSION_KEY, OMLayoutFeature.BUCKET_LAYOUT_SUPPORT.layoutVersion());
conf.setTimeDuration(OZONE_SNAPSHOT_DELETING_SERVICE_INTERVAL, 1, TimeUnit.SECONDS);
conf.setInt(OZONE_SNAPSHOT_SST_FILTERING_SERVICE_INTERVAL, KeyManagerImpl.DISABLE_VALUE);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,6 @@

package org.apache.hadoop.ozone.om;

import static org.apache.hadoop.ozone.om.OmUpgradeConfig.ConfigStrings.OZONE_OM_INIT_DEFAULT_LAYOUT_VERSION;

import java.io.File;
import java.io.IOException;
import java.util.Properties;
Expand Down Expand Up @@ -49,6 +47,8 @@
*/
public class OMStorage extends Storage {

public static final String TESTING_INIT_LAYOUT_VERSION_KEY = "testing.ozone.om.init.layout.version";

static final String ERROR_OM_IS_ALREADY_INITIALIZED =
"OM is already initialized.";
static final String ERROR_UNEXPECTED_OM_NODE_ID_TEMPLATE =
Expand Down Expand Up @@ -91,7 +91,7 @@ public class OMStorage extends Storage {
*/
public OMStorage(OzoneConfiguration conf) throws IOException {
super(NodeType.OM, getOmDbDir(conf), STORAGE_DIR,
getInitLayoutVersion(conf, OZONE_OM_INIT_DEFAULT_LAYOUT_VERSION,
getInitLayoutVersion(conf, TESTING_INIT_LAYOUT_VERSION_KEY,
OMLayoutVersionManager::maxLayoutVersion));
}

Expand Down