Skip to content

HDDS-10289. Remove unused MiniOzoneCluster.Builder properties #6161

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
fapifta merged 7 commits into from
Feb 5, 2024
Merged

HDDS-10289. Remove unused MiniOzoneCluster.Builder properties #6161

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
3d2879a
HDDS-10289. Remove unused MiniOzoneCluster.Builder properties
adoroszlai Feb 4, 2024
b99cc00
Remove useless setClusterId, setOmId, setScmId
adoroszlai Feb 4, 2024
a5c64b3
fix findbugs
adoroszlai Feb 4, 2024
89673f2
remove useless OMStorage creation
adoroszlai Feb 4, 2024
f4b482f
eagerly initialize omId, scmId; make non-Optional
adoroszlai Feb 4, 2024
aced7dd
restore setOMServiceId call in TestOMUpgradeFinalization
adoroszlai Feb 4, 2024
eccf8ad
fix bind() error in TestContainerServer
adoroszlai Feb 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 4 additions & 19 deletions ...op-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,6 @@
import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.ozone.ha.ConfUtils;
import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.ozone.om.OMStorage;
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.ozone.test.GenericTestUtils;
Expand All @@ -46,12 +45,13 @@
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.Timeout;
import org.junit.jupiter.api.io.TempDir;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.util.Optional;
import java.util.OptionalInt;
import java.util.UUID;

import static org.apache.hadoop.hdds.HddsUtils.getHostName;
import static org.apache.hadoop.hdds.HddsUtils.getHostPort;
Expand All @@ -73,10 +73,7 @@ public class TestOzoneFsHAURLs {

private OzoneConfiguration conf;
private static MiniOzoneCluster cluster;
private static String omId;
private static String omServiceId;
private static String clusterId;
private static String scmId;
private static OzoneManager om;
private static int numOfOMs;

Expand All @@ -98,33 +95,21 @@ public class TestOzoneFsHAURLs {


@BeforeAll
public static void initClass() throws Exception {
static void initClass(@TempDir File tempDir) throws Exception {
OzoneConfiguration conf = new OzoneConfiguration();
omId = UUID.randomUUID().toString();
omServiceId = "om-service-test1";
numOfOMs = 3;
clusterId = UUID.randomUUID().toString();
scmId = UUID.randomUUID().toString();
final String path = GenericTestUtils.getTempPath(omId);
java.nio.file.Path metaDirPath = java.nio.file.Paths.get(path, "om-meta");
conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, metaDirPath.toString());
conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, tempDir.getAbsolutePath());
conf.setInt(ScmConfigKeys.OZONE_DATANODE_PIPELINE_LIMIT, 3);

conf.set(OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT,
BucketLayout.LEGACY.name());
conf.setBoolean(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS, true);

OMStorage omStore = new OMStorage(conf);
omStore.setClusterId(clusterId);
// writes the version file properties
omStore.initialize();

// Start the cluster
cluster = MiniOzoneCluster.newOMHABuilder(conf)
.setNumDatanodes(5)
.setTotalPipelineNumLimit(3)
.setClusterId(clusterId)
.setScmId(scmId)
.setOMServiceId(omServiceId)
.setNumOfOzoneManagers(numOfOMs)
.build();
Expand Down
3 changes: 0 additions & 3 deletions ...-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFsSnapshot.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.UUID;
import java.util.stream.Stream;

import org.apache.commons.io.FileUtils;
Expand Down Expand Up @@ -88,8 +87,6 @@ static void initClass() throws Exception {

// Start the cluster
cluster = MiniOzoneCluster.newOMHABuilder(conf)
.setClusterId(UUID.randomUUID().toString())
.setScmId(UUID.randomUUID().toString())
.setOMServiceId(OM_SERVICE_ID)
.setNumOfOzoneManagers(1)
.build();
Expand Down
7 changes: 1 addition & 6 deletions ...zone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestFailoverWithSCMHA.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,6 @@

import java.io.IOException;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.TimeoutException;

import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ContainerBalancerConfigurationProto;
Expand All @@ -65,8 +64,6 @@
public class TestFailoverWithSCMHA {
private MiniOzoneHAClusterImpl cluster = null;
private OzoneConfiguration conf;
private String clusterId;
private String scmId;
private String omServiceId;
private String scmServiceId;
private int numOfOMs = 1;
Expand All @@ -82,15 +79,13 @@ public class TestFailoverWithSCMHA {
@BeforeEach
public void init() throws Exception {
conf = new OzoneConfiguration();
clusterId = UUID.randomUUID().toString();
scmId = UUID.randomUUID().toString();
omServiceId = "om-service-test1";
scmServiceId = "scm-service-test1";
conf.setLong(ScmConfigKeys.OZONE_SCM_HA_RATIS_SNAPSHOT_THRESHOLD,
SNAPSHOT_THRESHOLD);

cluster = (MiniOzoneHAClusterImpl) MiniOzoneCluster.newHABuilder(conf)
.setClusterId(clusterId).setScmId(scmId).setOMServiceId(omServiceId)
.setOMServiceId(omServiceId)
.setSCMServiceId(scmServiceId).setNumOfOzoneManagers(numOfOMs)
.setNumOfStorageContainerManagers(numOfSCMs).setNumOfActiveSCMs(3)
.build();
Expand Down
10 changes: 0 additions & 10 deletions ...integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMDbCheckpointServlet.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,6 @@
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.UUID;
import java.util.stream.Stream;

import org.apache.hadoop.hdds.conf.OzoneConfiguration;
Expand Down Expand Up @@ -81,9 +80,6 @@ public class TestSCMDbCheckpointServlet {
private StorageContainerManager scm;
private SCMMetrics scmMetrics;
private OzoneConfiguration conf;
private String clusterId;
private String scmId;
private String omId;
private HttpServletRequest requestMock;
private HttpServletResponse responseMock;
private String method;
Expand All @@ -100,14 +96,8 @@ public class TestSCMDbCheckpointServlet {
@BeforeEach
public void init() throws Exception {
conf = new OzoneConfiguration();
clusterId = UUID.randomUUID().toString();
scmId = UUID.randomUUID().toString();
omId = UUID.randomUUID().toString();
conf.setBoolean(OZONE_ACL_ENABLED, true);
cluster = MiniOzoneCluster.newBuilder(conf)
.setClusterId(clusterId)
.setScmId(scmId)
.setOmId(omId)
.build();
cluster.waitForClusterToBeReady();
scm = cluster.getStorageContainerManager();
Expand Down
7 changes: 0 additions & 7 deletions ...tegration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMInstallSnapshotWithHA.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.TimeoutException;

import org.apache.commons.io.FileUtils;
Expand Down Expand Up @@ -71,8 +70,6 @@ public class TestSCMInstallSnapshotWithHA {

private MiniOzoneHAClusterImpl cluster = null;
private OzoneConfiguration conf;
private String clusterId;
private String scmId;
private String omServiceId;
private String scmServiceId;
private int numOfOMs = 1;
Expand All @@ -89,8 +86,6 @@ public class TestSCMInstallSnapshotWithHA {
@BeforeEach
public void init() throws Exception {
conf = new OzoneConfiguration();
clusterId = UUID.randomUUID().toString();
scmId = UUID.randomUUID().toString();
omServiceId = "om-service-test1";
scmServiceId = "scm-service-test1";

Expand All @@ -100,8 +95,6 @@ public void init() throws Exception {
SNAPSHOT_THRESHOLD);

cluster = (MiniOzoneHAClusterImpl) MiniOzoneCluster.newHABuilder(conf)
.setClusterId(clusterId)
.setScmId(scmId)
.setOMServiceId(omServiceId)
.setSCMServiceId(scmServiceId)
.setNumOfOzoneManagers(numOfOMs)
Expand Down
3 changes: 0 additions & 3 deletions hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMSnapshot.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,6 @@
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;

import java.util.UUID;

import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.ONE;
import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.THREE;
import static org.assertj.core.api.Assertions.assertThat;
Expand All @@ -52,7 +50,6 @@ public static void setup() throws Exception {
cluster = MiniOzoneCluster
.newBuilder(conf)
.setNumDatanodes(3)
.setScmId(UUID.randomUUID().toString())
.build();
cluster.waitForClusterToBeReady();
}
Expand Down
8 changes: 0 additions & 8 deletions ...zone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSecretKeySnapshot.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,6 @@
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import java.util.UUID;
import java.util.concurrent.TimeoutException;

import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION;
Expand Down Expand Up @@ -96,8 +95,6 @@ public final class TestSecretKeySnapshot {
private File ozoneKeytab;
private File spnegoKeytab;
private String host;
private String clusterId;
private String scmId;
private MiniOzoneHAClusterImpl cluster;

@BeforeEach
Expand All @@ -107,9 +104,6 @@ public void init() throws Exception {

ExitUtils.disableSystemExit();

clusterId = UUID.randomUUID().toString();
scmId = UUID.randomUUID().toString();

startMiniKdc();
setSecureConfig();
createCredentialsInKDC();
Expand All @@ -125,9 +119,7 @@ public void init() throws Exception {
conf.set(HDDS_SECRET_KEY_EXPIRY_DURATION, EXPIRY_DURATION_MS + "ms");

MiniOzoneCluster.Builder builder = MiniOzoneCluster.newHABuilder(conf)
.setClusterId(clusterId)
.setSCMServiceId("TestSecretKeySnapshot")
.setScmId(scmId)
.setSCMServiceId("SCMServiceId")
.setNumDatanodes(1)
.setNumOfStorageContainerManagers(3)
Expand Down
7 changes: 0 additions & 7 deletions ...op-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSecretKeysApi.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,8 +99,6 @@ public final class TestSecretKeysApi {
private File testUserKeytab;
private String testUserPrincipal;
private String ozonePrincipal;
private String clusterId;
private String scmId;
private MiniOzoneHAClusterImpl cluster;

@BeforeEach
Expand All @@ -111,9 +109,6 @@ public void init() throws Exception {
ExitUtils.disableSystemExit();
ExitUtil.disableSystemExit();

clusterId = UUID.randomUUID().toString();
scmId = UUID.randomUUID().toString();

startMiniKdc();
setSecureConfig();
createCredentialsInKDC();
Expand Down Expand Up @@ -332,9 +327,7 @@ private void startCluster(int numSCMs)
throws IOException, TimeoutException, InterruptedException {
OzoneManager.setTestSecureOmFlag(true);
MiniOzoneCluster.Builder builder = MiniOzoneCluster.newHABuilder(conf)
.setClusterId(clusterId)
.setSCMServiceId("TestSecretKey")
.setScmId(scmId)
.setNumDatanodes(3)
.setNumOfStorageContainerManagers(numSCMs)
.setNumOfOzoneManagers(1);
Expand Down
9 changes: 2 additions & 7 deletions ...ntegration-test/src/test/java/org/apache/hadoop/hdds/scm/TestStorageContainerManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -374,7 +374,7 @@ public void testOldDNRegistersToReInitialisedSCM() throws Exception {
MiniOzoneCluster cluster =
MiniOzoneCluster.newBuilder(conf).setHbInterval(1000)
.setHbProcessorInterval(3000).setNumDatanodes(1)
.setClusterId(UUID.randomUUID().toString()).build();
.build();
cluster.waitForClusterToBeReady();

try {
Expand Down Expand Up @@ -776,7 +776,6 @@ public void testScmInfo() throws Exception {
@Test
public void testScmProcessDatanodeHeartbeat() throws Exception {
OzoneConfiguration conf = new OzoneConfiguration();
String scmId = UUID.randomUUID().toString();
conf.setClass(NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
StaticMapping.class, DNSToSwitchMapping.class);
StaticMapping.addNodeToRack(NetUtils.normalizeHostNames(
Expand All @@ -786,7 +785,6 @@ public void testScmProcessDatanodeHeartbeat() throws Exception {
final int datanodeNum = 3;
MiniOzoneCluster cluster = MiniOzoneCluster.newBuilder(conf)
.setNumDatanodes(datanodeNum)
.setScmId(scmId)
.build();
cluster.waitForClusterToBeReady();
StorageContainerManager scm = cluster.getStorageContainerManager();
Expand Down Expand Up @@ -1062,10 +1060,7 @@ public void testNonRatisToRatis()
throws IOException, AuthenticationException, InterruptedException,
TimeoutException {
final OzoneConfiguration conf = new OzoneConfiguration();
final String clusterID = UUID.randomUUID().toString();
try (MiniOzoneCluster cluster = MiniOzoneCluster.newBuilder(conf)
.setClusterId(clusterID)
.setScmId(UUID.randomUUID().toString())
.setNumDatanodes(3)
.build()) {
final StorageContainerManager nonRatisSCM = cluster
Expand All @@ -1077,7 +1072,7 @@ public void testNonRatisToRatis()

DefaultConfigManager.clearDefaultConfigs();
conf.setBoolean(ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY, true);
StorageContainerManager.scmInit(conf, clusterID);
StorageContainerManager.scmInit(conf, cluster.getClusterId());
cluster.restartStorageContainerManager(false);

final StorageContainerManager ratisSCM = cluster
Expand Down
6 changes: 0 additions & 6 deletions ...egration-test/src/test/java/org/apache/hadoop/hdds/scm/TestStorageContainerManagerHA.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,8 +74,6 @@ public class TestStorageContainerManagerHA {

private MiniOzoneHAClusterImpl cluster = null;
private OzoneConfiguration conf;
private String clusterId;
private String scmId;
private String omServiceId;
private static int numOfOMs = 3;
private String scmServiceId;
Expand All @@ -95,13 +93,9 @@ public void init() throws Exception {
conf.set(ScmConfigKeys.OZONE_SCM_HA_DBTRANSACTIONBUFFER_FLUSH_INTERVAL,
"5s");
conf.set(ScmConfigKeys.OZONE_SCM_HA_RATIS_SNAPSHOT_GAP, "1");
clusterId = UUID.randomUUID().toString();
scmId = UUID.randomUUID().toString();
omServiceId = "om-service-test1";
scmServiceId = "scm-service-test1";
cluster = (MiniOzoneHAClusterImpl) MiniOzoneCluster.newHABuilder(conf)
.setClusterId(clusterId)
.setScmId(scmId)
.setOMServiceId(omServiceId)
.setSCMServiceId(scmServiceId)
.setNumOfStorageContainerManagers(numOfSCMs)
Expand Down
3 changes: 0 additions & 3 deletions ...ration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestContainerCommandsEC.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,8 +130,6 @@ public class TestContainerCommandsEC {
private static ObjectStore store;
private static StorageContainerLocationProtocolClientSideTranslatorPB
storageContainerLocationClient;
private static final String SCM_ID = UUID.randomUUID().toString();
private static final String CLUSTER_ID = UUID.randomUUID().toString();
private static final int EC_DATA = 3;
private static final int EC_PARITY = 2;
private static final EcCodec EC_CODEC = EcCodec.RS;
Expand Down Expand Up @@ -926,7 +924,6 @@ public static void startCluster(OzoneConfiguration conf) throws Exception {
secretKeyClient = new SecretKeyTestClient();

cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(NUM_DN)
.setScmId(SCM_ID).setClusterId(CLUSTER_ID)
.setCertificateClient(certClient)
.setSecretKeyClient(secretKeyClient)
.build();
Expand Down
5 changes: 0 additions & 5 deletions ...st/src/test/java/org/apache/hadoop/hdds/utils/db/managed/TestRocksObjectLeakDetector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@
import org.junit.jupiter.api.Test;

import java.io.IOException;
import java.util.UUID;
import java.util.concurrent.TimeoutException;
import java.util.function.Supplier;

Expand All @@ -51,12 +50,8 @@ static void setUp() throws IOException, InterruptedException,
TimeoutException {
OzoneConfiguration conf = new OzoneConfiguration();
conf.set(OZONE_METADATA_STORE_ROCKSDB_STATISTICS, "ALL");
String clusterId = UUID.randomUUID().toString();
String scmId = UUID.randomUUID().toString();
String omServiceId = "omServiceId1";
cluster = MiniOzoneCluster.newBuilder(conf)
.setClusterId(clusterId)
.setScmId(scmId)
.setOMServiceId(omServiceId)
.setNumOfOzoneManagers(1)
.build();
Expand Down
Loading