Skip to content

Commit a2ad1e3

Browse files
szetszwoszetszwo
authored
HDDS-12536. Move InMemoryTestTable to test (#8043)
1 parent 7164c76 commit a2ad1e3

File tree

30 files changed

+140
-77
lines changed

30 files changed

+140
-77
lines changed

‎hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerSet.java‎

Lines changed: 15 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -25,12 +25,12 @@
2525
import com.google.protobuf.Message;
2626
import java.io.IOException;
2727
import java.time.Clock;
28-
import java.time.ZoneOffset;
2928
import java.util.ArrayList;
3029
import java.util.Collections;
3130
import java.util.Iterator;
3231
import java.util.List;
3332
import java.util.Map;
33+
import java.util.Objects;
3434
import java.util.Set;
3535
import java.util.concurrent.ConcurrentNavigableMap;
3636
import java.util.concurrent.ConcurrentSkipListMap;
@@ -41,7 +41,6 @@
4141
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State;
4242
import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsProto;
4343
import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
44-
import org.apache.hadoop.hdds.utils.db.InMemoryTestTable;
4544
import org.apache.hadoop.hdds.utils.db.Table;
4645
import org.apache.hadoop.ozone.container.common.interfaces.Container;
4746
import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
@@ -57,43 +56,39 @@ public class ContainerSet implements Iterable<Container<?>> {
5756

5857
private static final Logger LOG = LoggerFactory.getLogger(ContainerSet.class);
5958

59+
public static ContainerSet newReadOnlyContainerSet(long recoveringTimeout) {
60+
return new ContainerSet(null, recoveringTimeout);
61+
}
62+
63+
public static ContainerSet newRwContainerSet(Table<Long, String> containerIdsTable, long recoveringTimeout) {
64+
Objects.requireNonNull(containerIdsTable, "containerIdsTable == null");
65+
return new ContainerSet(containerIdsTable, recoveringTimeout);
66+
}
67+
6068
private final ConcurrentSkipListMap<Long, Container<?>> containerMap = new
6169
ConcurrentSkipListMap<>();
6270
private final ConcurrentSkipListSet<Long> missingContainerSet =
6371
new ConcurrentSkipListSet<>();
6472
private final ConcurrentSkipListMap<Long, Long> recoveringContainerMap =
6573
new ConcurrentSkipListMap<>();
66-
private Clock clock;
74+
private final Clock clock;
6775
private long recoveringTimeout;
6876
private final Table<Long, String> containerIdsTable;
6977

70-
@VisibleForTesting
71-
public ContainerSet(long recoveringTimeout) {
72-
this(new InMemoryTestTable<>(), recoveringTimeout);
78+
private ContainerSet(Table<Long, String> continerIdsTable, long recoveringTimeout) {
79+
this(continerIdsTable, recoveringTimeout, null);
7380
}
7481

75-
public ContainerSet(Table<Long, String> continerIdsTable, long recoveringTimeout) {
76-
this(continerIdsTable, recoveringTimeout, false);
77-
}
78-
79-
public ContainerSet(Table<Long, String> continerIdsTable, long recoveringTimeout, boolean readOnly) {
80-
this.clock = Clock.system(ZoneOffset.UTC);
82+
ContainerSet(Table<Long, String> continerIdsTable, long recoveringTimeout, Clock clock) {
83+
this.clock = clock != null ? clock : Clock.systemUTC();
8184
this.containerIdsTable = continerIdsTable;
8285
this.recoveringTimeout = recoveringTimeout;
83-
if (!readOnly && containerIdsTable == null) {
84-
throw new IllegalArgumentException("Container table cannot be null when container set is not read only");
85-
}
8686
}
8787

8888
public long getCurrentTime() {
8989
return clock.millis();
9090
}
9191

92-
@VisibleForTesting
93-
public void setClock(Clock clock) {
94-
this.clock = clock;
95-
}
96-
9792
@VisibleForTesting
9893
public void setRecoveringTimeout(long recoveringTimeout) {
9994
this.recoveringTimeout = recoveringTimeout;

‎hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -188,7 +188,8 @@ public OzoneContainer(HddsDatanodeService hddsDatanodeService,
188188
OZONE_RECOVERING_CONTAINER_TIMEOUT,
189189
OZONE_RECOVERING_CONTAINER_TIMEOUT_DEFAULT, TimeUnit.MILLISECONDS);
190190
this.witnessedContainerMetadataStore = WitnessedContainerMetadataStoreImpl.get(conf);
191-
containerSet = new ContainerSet(witnessedContainerMetadataStore.getContainerIdsTable(), recoveringContainerTimeout);
191+
containerSet = ContainerSet.newRwContainerSet(witnessedContainerMetadataStore.getContainerIdsTable(),
192+
recoveringContainerTimeout);
192193
metadataScanner = null;
193194

194195
metrics = ContainerMetrics.create(conf);

‎hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/ContainerTestUtils.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,8 +49,8 @@
4949
import org.apache.hadoop.ozone.OzoneConfigKeys;
5050
import org.apache.hadoop.ozone.container.ContainerTestHelper;
5151
import org.apache.hadoop.ozone.container.common.impl.ContainerData;
52+
import org.apache.hadoop.ozone.container.common.impl.ContainerImplTestUtils;
5253
import org.apache.hadoop.ozone.container.common.impl.ContainerLayoutVersion;
53-
import org.apache.hadoop.ozone.container.common.impl.ContainerSet;
5454
import org.apache.hadoop.ozone.container.common.interfaces.Container;
5555
import org.apache.hadoop.ozone.container.common.interfaces.Container.ScanResult;
5656
import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
@@ -335,7 +335,7 @@ public static ContainerDispatcher getNoopContainerDispatcher() {
335335
}
336336

337337
private static final ContainerController EMPTY_CONTAINER_CONTROLLER
338-
= new ContainerController(new ContainerSet(1000), Collections.emptyMap());
338+
= new ContainerController(ContainerImplTestUtils.newContainerSet(), Collections.emptyMap());
339339

340340
public static ContainerController getEmptyContainerController() {
341341
return EMPTY_CONTAINER_CONTROLLER;

‎hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java‎

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -17,15 +17,14 @@
1717

1818
package org.apache.hadoop.ozone.container.common;
1919

20-
import static java.nio.charset.StandardCharsets.UTF_8;
21-
import static org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric;
2220
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL;
2321
import static org.apache.hadoop.ozone.OzoneConsts.SCHEMA_V1;
2422
import static org.apache.hadoop.ozone.OzoneConsts.SCHEMA_V2;
2523
import static org.apache.hadoop.ozone.OzoneConsts.SCHEMA_V3;
2624
import static org.apache.hadoop.ozone.container.common.ContainerTestUtils.COMMIT_STAGE;
2725
import static org.apache.hadoop.ozone.container.common.ContainerTestUtils.WRITE_STAGE;
2826
import static org.apache.hadoop.ozone.container.common.ContainerTestUtils.createDbInstancesForTestIfNeeded;
27+
import static org.apache.hadoop.ozone.container.common.impl.ContainerImplTestUtils.newContainerSet;
2928
import static org.apache.hadoop.ozone.container.common.impl.ContainerLayoutVersion.FILE_PER_BLOCK;
3029
import static org.apache.hadoop.ozone.container.common.states.endpoint.VersionEndpointTask.LOG;
3130
import static org.apache.hadoop.ozone.container.keyvalue.helpers.KeyValueContainerUtil.isSameSchemaVersion;
@@ -50,6 +49,7 @@
5049
import java.util.Map;
5150
import java.util.Set;
5251
import java.util.UUID;
52+
import java.util.concurrent.ThreadLocalRandom;
5353
import java.util.concurrent.TimeUnit;
5454
import java.util.concurrent.TimeoutException;
5555
import org.apache.commons.lang3.StringUtils;
@@ -168,7 +168,8 @@ private KeyValueContainerData createToDeleteBlocks(ContainerSet containerSet,
168168
} else {
169169
chunkManager = new FilePerChunkStrategy(true, null);
170170
}
171-
byte[] arr = randomAlphanumeric(1048576).getBytes(UTF_8);
171+
byte[] arr = new byte[1048576];
172+
ThreadLocalRandom.current().nextBytes(arr);
172173
ChunkBuffer buffer = ChunkBuffer.wrap(ByteBuffer.wrap(arr));
173174
int txnID = 0;
174175
long containerID = ContainerTestHelper.getTestContainerID();
@@ -425,7 +426,7 @@ public void testPendingDeleteBlockReset(ContainerTestVersionInfo versionInfo)
425426
dnConf.setBlockDeletionLimit(blockDeleteLimit);
426427
this.blockLimitPerInterval = dnConf.getBlockDeletionLimit();
427428
conf.setFromObject(dnConf);
428-
ContainerSet containerSet = new ContainerSet(1000);
429+
ContainerSet containerSet = newContainerSet();
429430

430431
// Create one container with no actual pending delete blocks, but an
431432
// incorrect metadata value indicating it has enough pending deletes to
@@ -533,7 +534,7 @@ public void testBlockDeletion(ContainerTestVersionInfo versionInfo)
533534
dnConf.setBlockDeletionLimit(2);
534535
this.blockLimitPerInterval = dnConf.getBlockDeletionLimit();
535536
conf.setFromObject(dnConf);
536-
ContainerSet containerSet = new ContainerSet(1000);
537+
ContainerSet containerSet = newContainerSet();
537538
createToDeleteBlocks(containerSet, 1, 3, 1);
538539
ContainerMetrics metrics = ContainerMetrics.create(conf);
539540
KeyValueHandler keyValueHandler =
@@ -659,7 +660,7 @@ public void testWithUnrecordedBlocks(ContainerTestVersionInfo versionInfo)
659660
dnConf.setBlockDeletionLimit(2);
660661
this.blockLimitPerInterval = dnConf.getBlockDeletionLimit();
661662
conf.setFromObject(dnConf);
662-
ContainerSet containerSet = new ContainerSet(1000);
663+
ContainerSet containerSet = newContainerSet();
663664

664665
createToDeleteBlocks(containerSet, numOfContainers, numOfBlocksPerContainer,
665666
numOfChunksPerBlock);
@@ -767,7 +768,7 @@ public void testShutdownService(ContainerTestVersionInfo versionInfo)
767768
conf.setTimeDuration(OZONE_BLOCK_DELETING_SERVICE_INTERVAL, 500,
768769
TimeUnit.MILLISECONDS);
769770

770-
ContainerSet containerSet = new ContainerSet(1000);
771+
ContainerSet containerSet = newContainerSet();
771772
// Create 1 container with 100 blocks
772773
createToDeleteBlocks(containerSet, 1, 100, 1);
773774
ContainerMetrics metrics = ContainerMetrics.create(conf);
@@ -798,7 +799,7 @@ public void testBlockDeletionTimeout(ContainerTestVersionInfo versionInfo)
798799
blockLimitPerInterval = dnConf.getBlockDeletionLimit();
799800
conf.setFromObject(dnConf);
800801

801-
ContainerSet containerSet = new ContainerSet(1000);
802+
ContainerSet containerSet = newContainerSet();
802803
createToDeleteBlocks(containerSet, 1, 3, 1);
803804
ContainerMetrics metrics = ContainerMetrics.create(conf);
804805
KeyValueHandler keyValueHandler =
@@ -900,7 +901,7 @@ public void testContainerThrottle(ContainerTestVersionInfo versionInfo)
900901
dnConf.setBlockDeletionLimit(1);
901902
this.blockLimitPerInterval = dnConf.getBlockDeletionLimit();
902903
conf.setFromObject(dnConf);
903-
ContainerSet containerSet = new ContainerSet(1000);
904+
ContainerSet containerSet = newContainerSet();
904905

905906
int containerCount = 2;
906907
int chunksPerBlock = 10;
@@ -960,7 +961,7 @@ public void testContainerMaxLockHoldingTime(
960961
dnConf.setBlockDeletingMaxLockHoldingTime(Duration.ofMillis(-1));
961962
dnConf.setBlockDeletionLimit(3);
962963
conf.setFromObject(dnConf);
963-
ContainerSet containerSet = new ContainerSet(1000);
964+
ContainerSet containerSet = newContainerSet();
964965

965966
int containerCount = 1;
966967
int chunksPerBlock = 10;
@@ -1024,7 +1025,7 @@ public void testBlockThrottle(ContainerTestVersionInfo versionInfo)
10241025
dnConf.setBlockDeletionLimit(10);
10251026
this.blockLimitPerInterval = dnConf.getBlockDeletionLimit();
10261027
conf.setFromObject(dnConf);
1027-
ContainerSet containerSet = new ContainerSet(1000);
1028+
ContainerSet containerSet = newContainerSet();
10281029
ContainerMetrics metrics = ContainerMetrics.create(conf);
10291030
KeyValueHandler keyValueHandler =
10301031
new KeyValueHandler(conf, datanodeUuid, containerSet, volumeSet,

‎hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestSchemaOneBackwardsCompatibility.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
package org.apache.hadoop.ozone.container.common;
1919

20+
import static org.apache.hadoop.ozone.container.common.impl.ContainerImplTestUtils.newContainerSet;
2021
import static org.assertj.core.api.Assertions.assertThat;
2122
import static org.junit.jupiter.api.Assertions.assertEquals;
2223
import static org.junit.jupiter.api.Assertions.assertNull;
@@ -564,7 +565,7 @@ private void runBlockDeletingService(KeyValueHandler keyValueHandler)
564565
}
565566

566567
private ContainerSet makeContainerSet() throws Exception {
567-
ContainerSet containerSet = new ContainerSet(1000);
568+
ContainerSet containerSet = newContainerSet();
568569
KeyValueContainer container = new KeyValueContainer(newKvData(), conf);
569570
containerSet.addContainer(container);
570571

‎hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestSchemaTwoBackwardsCompatibility.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import static org.apache.hadoop.ozone.OzoneConsts.PENDING_DELETE_BLOCK_COUNT;
2525
import static org.apache.hadoop.ozone.container.common.ContainerTestUtils.COMMIT_STAGE;
2626
import static org.apache.hadoop.ozone.container.common.ContainerTestUtils.WRITE_STAGE;
27+
import static org.apache.hadoop.ozone.container.common.impl.ContainerImplTestUtils.newContainerSet;
2728
import static org.junit.jupiter.api.Assertions.assertEquals;
2829
import static org.mockito.Mockito.any;
2930
import static org.mockito.Mockito.mock;
@@ -131,7 +132,7 @@ public void setup() throws Exception {
131132
blockManager = new BlockManagerImpl(conf);
132133
chunkManager = new FilePerBlockStrategy(true, blockManager);
133134

134-
containerSet = new ContainerSet(1000);
135+
containerSet = newContainerSet();
135136
keyValueHandler = new KeyValueHandler(conf, datanodeUuid,
136137
containerSet, volumeSet, ContainerMetrics.create(conf), c -> { });
137138
ozoneContainer = mock(OzoneContainer.class);

‎hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestStaleRecoveringContainerScrubbingService.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State.CLOSED;
2121
import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State.RECOVERING;
2222
import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State.UNHEALTHY;
23+
import static org.apache.hadoop.ozone.container.common.impl.ContainerImplTestUtils.newContainerSet;
2324
import static org.junit.jupiter.api.Assertions.assertEquals;
2425
import static org.mockito.Mockito.anyList;
2526
import static org.mockito.Mockito.anyLong;
@@ -145,8 +146,7 @@ private List<Long> createTestContainers(
145146
public void testScrubbingStaleRecoveringContainers(
146147
ContainerTestVersionInfo versionInfo) throws Exception {
147148
initVersionInfo(versionInfo);
148-
ContainerSet containerSet = new ContainerSet(10);
149-
containerSet.setClock(testClock);
149+
ContainerSet containerSet = newContainerSet(10, testClock);
150150
StaleRecoveringContainerScrubbingService srcss =
151151
new StaleRecoveringContainerScrubbingService(
152152
50, TimeUnit.MILLISECONDS, 10,

‎hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/ContainerImplTestUtils.java‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.hadoop.ozone.container.common.impl;
19+
20+
import java.time.Clock;
21+
import org.apache.hadoop.hdds.utils.db.InMemoryTestTable;
22+
23+
/**
24+
* Helper utility to test container impl.
25+
*/
26+
public final class ContainerImplTestUtils {
27+
28+
private ContainerImplTestUtils() {
29+
}
30+
31+
public static ContainerSet newContainerSet() {
32+
return newContainerSet(1000);
33+
}
34+
35+
public static ContainerSet newContainerSet(long recoveringTimeout) {
36+
return ContainerSet.newRwContainerSet(new InMemoryTestTable<>(), recoveringTimeout);
37+
}
38+
39+
public static ContainerSet newContainerSet(long recoveringTimeout, Clock clock) {
40+
return new ContainerSet(new InMemoryTestTable<>(), recoveringTimeout, clock);
41+
}
42+
}

‎hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDeletionChoosingPolicy.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
package org.apache.hadoop.ozone.container.common.impl;
1919

20+
import static org.apache.hadoop.ozone.container.common.impl.ContainerImplTestUtils.newContainerSet;
2021
import static org.assertj.core.api.Assertions.assertThat;
2122
import static org.junit.jupiter.api.Assertions.assertEquals;
2223
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -81,7 +82,7 @@ public void testRandomChoosingPolicy(ContainerLayoutVersion layout)
8182
conf.set(
8283
ScmConfigKeys.OZONE_SCM_KEY_VALUE_CONTAINER_DELETION_CHOOSING_POLICY,
8384
RandomContainerDeletionChoosingPolicy.class.getName());
84-
containerSet = new ContainerSet(1000);
85+
containerSet = newContainerSet();
8586

8687
int numContainers = 10;
8788
for (int i = 0; i < numContainers; i++) {
@@ -142,7 +143,7 @@ public void testTopNOrderedChoosingPolicy(ContainerLayoutVersion layout)
142143
conf.set(
143144
ScmConfigKeys.OZONE_SCM_KEY_VALUE_CONTAINER_DELETION_CHOOSING_POLICY,
144145
TopNOrderedContainerDeletionChoosingPolicy.class.getName());
145-
containerSet = new ContainerSet(1000);
146+
containerSet = newContainerSet();
146147

147148
int numContainers = 10;
148149
Random random = new Random();

‎hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import static org.apache.hadoop.ozone.container.ContainerTestHelper.getChunk;
2323
import static org.apache.hadoop.ozone.container.ContainerTestHelper.getData;
2424
import static org.apache.hadoop.ozone.container.ContainerTestHelper.setDataChecksum;
25+
import static org.apache.hadoop.ozone.container.common.impl.ContainerImplTestUtils.newContainerSet;
2526
import static org.apache.hadoop.ozone.container.keyvalue.helpers.KeyValueContainerUtil.isSameSchemaVersion;
2627
import static org.assertj.core.api.Assertions.assertThat;
2728
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -151,7 +152,7 @@ public static void shutdown() throws IOException {
151152

152153
@BeforeEach
153154
public void setupPaths() throws IOException {
154-
containerSet = new ContainerSet(1000);
155+
containerSet = newContainerSet();
155156
volumeSet = new MutableVolumeSet(DATANODE_UUID, conf, null,
156157
StorageVolume.VolumeType.DATA_VOLUME, null);
157158
// Initialize volume directories.

0 commit comments

Comments
 (0)