diff --git a/hadoop-hdds/test-utils/src/main/java/org/apache/ozone/test/OzoneTestBase.java b/hadoop-hdds/test-utils/src/main/java/org/apache/ozone/test/OzoneTestBase.java new file mode 100644 index 000000000000..bb675bddafda --- /dev/null +++ b/hadoop-hdds/test-utils/src/main/java/org/apache/ozone/test/OzoneTestBase.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ozone.test; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.TestInfo; + +import java.lang.reflect.Method; + +/** + * Base class for Ozone JUnit tests. + * Provides test method name, which can be used to create unique items. + */ +public abstract class OzoneTestBase { + + private TestInfo info; + + @BeforeEach + void storeTestInfo(TestInfo testInfo) { + this.info = testInfo; + } + + protected String getTestName() { + return info.getTestMethod() + .map(Method::getName) + .orElse("unknown"); + } + +} diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/OmTestManagers.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/OmTestManagers.java index 3db3263ec57c..43d29c1608a8 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/OmTestManagers.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/OmTestManagers.java @@ -17,7 +17,9 @@ package org.apache.hadoop.ozone.om; +import static org.apache.ozone.test.GenericTestUtils.waitFor; import static org.mockito.Mockito.mock; + import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.scm.HddsWhiteboxTestUtils; @@ -28,9 +30,11 @@ import org.apache.hadoop.ozone.client.OzoneClientFactory; import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol; import org.apache.hadoop.hdds.security.token.OzoneBlockTokenSecretManager; +import org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer.RaftServerStatus; import org.apache.hadoop.security.authentication.client.AuthenticationException; import java.io.IOException; +import java.util.concurrent.TimeoutException; /** * Test utility for creating a dummy OM, the associated @@ -38,15 +42,15 @@ */ public final class OmTestManagers { - private OzoneManagerProtocol writeClient; - private OzoneManager om; - private KeyManager keyManager; - private OMMetadataManager metadataManager; + private final OzoneManagerProtocol writeClient; + private final OzoneManager om; + private final KeyManager keyManager; + private final OMMetadataManager metadataManager; private KeyProviderCryptoExtension kmsProvider; - private VolumeManager volumeManager; - private BucketManager bucketManager; - private PrefixManager prefixManager; - private ScmBlockLocationProtocol scmBlockClient; + private final VolumeManager volumeManager; + private final BucketManager bucketManager; + private final PrefixManager prefixManager; + private final ScmBlockLocationProtocol scmBlockClient; public OzoneManager getOzoneManager() { return om; @@ -74,14 +78,14 @@ public ScmBlockLocationProtocol getScmBlockClient() { } public OmTestManagers(OzoneConfiguration conf) - throws AuthenticationException, IOException { + throws AuthenticationException, IOException, InterruptedException, TimeoutException { this(conf, null, null); } public OmTestManagers(OzoneConfiguration conf, ScmBlockLocationProtocol blockClient, StorageContainerLocationProtocol containerClient) - throws AuthenticationException, IOException { + throws AuthenticationException, IOException, InterruptedException, TimeoutException { if (containerClient == null) { containerClient = mock(StorageContainerLocationProtocol.class); } @@ -109,6 +113,9 @@ public OmTestManagers(OzoneConfiguration conf, "secretManager", mock(OzoneBlockTokenSecretManager.class)); om.start(); + waitFor(() -> om.getOmRatisServer().checkLeaderStatus() == RaftServerStatus.LEADER_AND_READY, + 10, 10_000); + writeClient = OzoneClientFactory.getRpcClient(conf) .getObjectStore().getClientProxy().getOzoneManagerClient(); metadataManager = (OmMetadataManagerImpl) HddsWhiteboxTestUtils diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestBucketManagerImpl.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestBucketManagerImpl.java index 7919b013b126..3844fb45b38f 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestBucketManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestBucketManagerImpl.java @@ -18,7 +18,6 @@ import java.io.File; import java.io.IOException; -import java.nio.file.Path; import java.util.ArrayList; import java.util.Collections; @@ -43,9 +42,11 @@ import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; -import org.apache.hadoop.security.authentication.client.AuthenticationException; -import org.junit.jupiter.api.AfterEach; +import org.apache.ozone.test.OzoneTestBase; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.io.TempDir; @@ -66,40 +67,36 @@ * This class tests the Bucket Manager Implementation using Mockito. */ @ExtendWith(MockitoExtension.class) -public class TestBucketManagerImpl { - - @TempDir - private Path folder; +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class TestBucketManagerImpl extends OzoneTestBase { private OmTestManagers omTestManagers; private OzoneManagerProtocol writeClient; - @AfterEach - public void cleanup() throws Exception { - OzoneManager om = omTestManagers.getOzoneManager(); - om.stop(); - } - - private OzoneConfiguration createNewTestPath() throws IOException { + @BeforeAll + void setup(@TempDir File folder) throws Exception { OzoneConfiguration conf = new OzoneConfiguration(); - File newFolder = folder.toFile(); - if (!newFolder.exists()) { - assertTrue(newFolder.mkdirs()); - } - ServerUtils.setOzoneMetaDirPath(conf, newFolder.toString()); - return conf; - } + ServerUtils.setOzoneMetaDirPath(conf, folder.toString()); - private void createSampleVol() throws IOException, AuthenticationException { - OzoneConfiguration conf = createNewTestPath(); omTestManagers = new OmTestManagers(conf); writeClient = omTestManagers.getWriteClient(); + } + @AfterAll + void cleanup() throws Exception { + omTestManagers.getOzoneManager().stop(); + } + + public String volumeName() { + return getTestName().toLowerCase(); + } + + private void createSampleVol(String volume) throws IOException { // This is a simple hack for testing, we just test if the volume via a // null check, do not parse the value part. So just write some dummy value. OmVolumeArgs args = OmVolumeArgs.newBuilder() - .setVolume("sample-vol") + .setVolume(volume) .setAdminName("bilbo") .setOwnerName("bilbo") .build(); @@ -107,25 +104,20 @@ private void createSampleVol() throws IOException, AuthenticationException { } @Test - public void testCreateBucketWithoutVolume() throws Exception { - OzoneConfiguration conf = createNewTestPath(); - omTestManagers = new OmTestManagers(conf); - OMException omEx = assertThrows(OMException.class, () -> { - writeClient = omTestManagers.getWriteClient(); - - OmBucketInfo bucketInfo = OmBucketInfo.newBuilder() - .setVolumeName("sample-vol") - .setBucketName("bucket-one") - .build(); - writeClient.createBucket(bucketInfo); - }); + void testCreateBucketWithoutVolume() { + OmBucketInfo bucketInfo = OmBucketInfo.newBuilder() + .setVolumeName(volumeName()) + .setBucketName("bucket-one") + .build(); + OMException omEx = assertThrows(OMException.class, () -> writeClient.createBucket(bucketInfo)); assertEquals(ResultCodes.VOLUME_NOT_FOUND, omEx.getResult()); assertEquals("Volume doesn't exist", omEx.getMessage()); } @Test - public void testCreateEncryptedBucket() throws Exception { - createSampleVol(); + void testCreateEncryptedBucket() throws Exception { + String volume = volumeName(); + createSampleVol(volume); KeyProviderCryptoExtension kmsProvider = omTestManagers.kmsProviderInit(); String testBekName = "key1"; @@ -138,17 +130,16 @@ public void testCreateEncryptedBucket() throws Exception { BucketManager bucketManager = omTestManagers.getBucketManager(); OmBucketInfo bucketInfo = OmBucketInfo.newBuilder() - .setVolumeName("sample-vol") + .setVolumeName(volume) .setBucketName("bucket-one") .setBucketEncryptionKey(new BucketEncryptionKeyInfo.Builder().setKeyName("key1").build()) .build(); writeClient.createBucket(bucketInfo); - assertNotNull(bucketManager.getBucketInfo("sample-vol", - "bucket-one")); + assertNotNull(bucketManager.getBucketInfo(volume, "bucket-one")); OmBucketInfo bucketInfoRead = - bucketManager.getBucketInfo("sample-vol", "bucket-one"); + bucketManager.getBucketInfo(volume, "bucket-one"); assertEquals(bucketInfoRead.getEncryptionKeyInfo().getKeyName(), bucketInfo.getEncryptionKeyInfo().getKeyName()); @@ -157,67 +148,62 @@ public void testCreateEncryptedBucket() throws Exception { @Test public void testCreateBucket() throws Exception { - createSampleVol(); + String volume = volumeName(); + createSampleVol(volume); BucketManager bucketManager = omTestManagers.getBucketManager(); OmBucketInfo bucketInfo = OmBucketInfo.newBuilder() - .setVolumeName("sample-vol") + .setVolumeName(volume) .setBucketName("bucket-one") .build(); writeClient.createBucket(bucketInfo); - assertNotNull(bucketManager.getBucketInfo("sample-vol", - "bucket-one")); + assertNotNull(bucketManager.getBucketInfo(volume, "bucket-one")); } @Test public void testCreateAlreadyExistingBucket() throws Exception { - createSampleVol(); + String volume = volumeName(); + createSampleVol(volume); - OMException omEx = assertThrows(OMException.class, () -> { - OmBucketInfo bucketInfo = OmBucketInfo.newBuilder() - .setVolumeName("sample-vol") - .setBucketName("bucket-one") - .build(); - writeClient.createBucket(bucketInfo); - writeClient.createBucket(bucketInfo); - }); - assertEquals(ResultCodes.BUCKET_ALREADY_EXISTS, - omEx.getResult()); + OmBucketInfo bucketInfo = OmBucketInfo.newBuilder() + .setVolumeName(volume) + .setBucketName("bucket-one") + .build(); + writeClient.createBucket(bucketInfo); + + OMException omEx = assertThrows(OMException.class, + () -> writeClient.createBucket(bucketInfo)); + assertEquals(ResultCodes.BUCKET_ALREADY_EXISTS, omEx.getResult()); assertEquals("Bucket already exist", omEx.getMessage()); } @Test public void testGetBucketInfoForInvalidBucket() throws Exception { - createSampleVol(); - OMException exception = assertThrows(OMException.class, () -> { - BucketManager bucketManager = omTestManagers.getBucketManager(); - bucketManager.getBucketInfo("sample-vol", "bucket-one"); - }); + String volume = volumeName(); + createSampleVol(volume); + + BucketManager bucketManager = omTestManagers.getBucketManager(); + + OMException exception = assertThrows(OMException.class, + () -> bucketManager.getBucketInfo(volume, "bucket-one")); assertThat(exception.getMessage()).contains("Bucket not found"); - assertEquals(ResultCodes.BUCKET_NOT_FOUND, - exception.getResult()); + assertEquals(ResultCodes.BUCKET_NOT_FOUND, exception.getResult()); } @Test - public void testGetBucketInfo() throws Exception { - final String volumeName = "sample-vol"; + void testGetBucketInfo() throws Exception { + final String volumeName = volumeName(); final String bucketName = "bucket-one"; - OzoneConfiguration conf = createNewTestPath(); - omTestManagers = new OmTestManagers(conf); - writeClient = omTestManagers.getWriteClient(); - OMMetadataManager metaMgr = omTestManagers.getMetadataManager(); BucketManager bucketManager = omTestManagers.getBucketManager(); // Check exception thrown when volume does not exist - try { - bucketManager.getBucketInfo(volumeName, bucketName); - fail("Should have thrown OMException"); - } catch (OMException omEx) { - assertEquals(ResultCodes.VOLUME_NOT_FOUND, omEx.getResult(), - "getBucketInfo() should have thrown " + - "VOLUME_NOT_FOUND as the parent volume is not created!"); - } + OMException omEx = assertThrows(OMException.class, + () -> bucketManager.getBucketInfo(volumeName, bucketName)); + assertEquals(ResultCodes.VOLUME_NOT_FOUND, omEx.getResult(), + "getBucketInfo() should have thrown " + + "VOLUME_NOT_FOUND as the parent volume is not created!"); + OmBucketInfo bucketInfo = OmBucketInfo.newBuilder() .setVolumeName(volumeName) .setBucketName(bucketName) @@ -235,16 +221,12 @@ public void testGetBucketInfo() throws Exception { writeClient.createVolume(args); // Create bucket createBucket(metaMgr, bucketInfo); + // Check exception thrown when bucket does not exist - try { - bucketManager.getBucketInfo(volumeName, "bucketNotExist"); - fail("Should have thrown OMException"); - } catch (OMException omEx) { - assertEquals( - ResultCodes.BUCKET_NOT_FOUND, omEx.getResult(), - "getBucketInfo() should have thrown BUCKET_NOT_FOUND " + - "as the parent volume exists but bucket doesn't!"); - } + OMException e2 = assertThrows(OMException.class, + () -> bucketManager.getBucketInfo(volumeName, "bucketNotExist")); + assertEquals(ResultCodes.BUCKET_NOT_FOUND, e2.getResult()); + OmBucketInfo result = bucketManager.getBucketInfo(volumeName, bucketName); assertEquals(volumeName, result.getVolumeName()); assertEquals(bucketName, result.getBucketName()); @@ -259,64 +241,68 @@ private void createBucket(OMMetadataManager metadataManager, @Test public void testSetBucketPropertyChangeStorageType() throws Exception { + String volume = volumeName(); + createSampleVol(volume); - createSampleVol(); OMMetadataManager metaMgr = omTestManagers.getMetadataManager(); BucketManager bucketManager = omTestManagers.getBucketManager(); OmBucketInfo bucketInfo = OmBucketInfo.newBuilder() - .setVolumeName("sample-vol") + .setVolumeName(volume) .setBucketName("bucket-one") .setStorageType(StorageType.DISK) .build(); createBucket(metaMgr, bucketInfo); OmBucketInfo result = bucketManager.getBucketInfo( - "sample-vol", "bucket-one"); + volume, "bucket-one"); assertEquals(StorageType.DISK, result.getStorageType()); OmBucketArgs bucketArgs = OmBucketArgs.newBuilder() - .setVolumeName("sample-vol") + .setVolumeName(volume) .setBucketName("bucket-one") .setStorageType(StorageType.SSD) .build(); writeClient.setBucketProperty(bucketArgs); OmBucketInfo updatedResult = bucketManager.getBucketInfo( - "sample-vol", "bucket-one"); + volume, "bucket-one"); assertEquals(StorageType.SSD, updatedResult.getStorageType()); } @Test public void testSetBucketPropertyChangeVersioning() throws Exception { - createSampleVol(); + String volume = volumeName(); + createSampleVol(volume); BucketManager bucketManager = omTestManagers.getBucketManager(); OmBucketInfo bucketInfo = OmBucketInfo.newBuilder() - .setVolumeName("sample-vol") + .setVolumeName(volume) .setBucketName("bucket-one") .setIsVersionEnabled(false) .build(); writeClient.createBucket(bucketInfo); OmBucketInfo result = bucketManager.getBucketInfo( - "sample-vol", "bucket-one"); + volume, "bucket-one"); assertFalse(result.getIsVersionEnabled()); OmBucketArgs bucketArgs = OmBucketArgs.newBuilder() - .setVolumeName("sample-vol") + .setVolumeName(volume) .setBucketName("bucket-one") .setIsVersionEnabled(true) .build(); writeClient.setBucketProperty(bucketArgs); OmBucketInfo updatedResult = bucketManager.getBucketInfo( - "sample-vol", "bucket-one"); + volume, "bucket-one"); assertTrue(updatedResult.getIsVersionEnabled()); } @Test public void testDeleteBucket() throws Exception { - createSampleVol(); + String volume = volumeName(); + createSampleVol(volume); + BucketManager bucketManager = omTestManagers.getBucketManager(); for (int i = 0; i < 5; i++) { OmBucketInfo bucketInfo = OmBucketInfo.newBuilder() - .setVolumeName("sample-vol") + .setVolumeName(volume) .setBucketName("bucket-" + i) .build(); writeClient.createBucket(bucketInfo); @@ -324,17 +310,17 @@ public void testDeleteBucket() throws Exception { for (int i = 0; i < 5; i++) { assertEquals("bucket-" + i, bucketManager.getBucketInfo( - "sample-vol", "bucket-" + i).getBucketName()); + volume, "bucket-" + i).getBucketName()); } try { - writeClient.deleteBucket("sample-vol", "bucket-1"); + writeClient.deleteBucket(volume, "bucket-1"); assertNotNull(bucketManager.getBucketInfo( - "sample-vol", "bucket-2")); + volume, "bucket-2")); } catch (IOException ex) { fail(ex.getMessage()); } OMException omEx = assertThrows(OMException.class, () -> { - bucketManager.getBucketInfo("sample-vol", "bucket-1"); + bucketManager.getBucketInfo(volume, "bucket-1"); }); assertEquals(ResultCodes.BUCKET_NOT_FOUND, omEx.getResult()); @@ -343,15 +329,17 @@ public void testDeleteBucket() throws Exception { @Test public void testDeleteNonEmptyBucket() throws Exception { - createSampleVol(); + String volume = volumeName(); + createSampleVol(volume); + OmBucketInfo bucketInfo = OmBucketInfo.newBuilder() - .setVolumeName("sample-vol") + .setVolumeName(volume) .setBucketName("bucket-one") .build(); writeClient.createBucket(bucketInfo); //Create keys in bucket OmKeyArgs args1 = new OmKeyArgs.Builder() - .setVolumeName("sample-vol") + .setVolumeName(volume) .setBucketName("bucket-one") .setKeyName("key-one") .setAcls(Collections.emptyList()) @@ -364,7 +352,7 @@ public void testDeleteNonEmptyBucket() throws Exception { writeClient.commitKey(args1, session1.getId()); OmKeyArgs args2 = new OmKeyArgs.Builder() - .setVolumeName("sample-vol") + .setVolumeName(volume) .setBucketName("bucket-one") .setKeyName("key-two") .setAcls(Collections.emptyList()) @@ -376,7 +364,7 @@ public void testDeleteNonEmptyBucket() throws Exception { OpenKeySession session2 = writeClient.openKey(args2); writeClient.commitKey(args2, session2.getId()); OMException omEx = assertThrows(OMException.class, () -> { - writeClient.deleteBucket("sample-vol", "bucket-one"); + writeClient.deleteBucket(volume, "bucket-one"); }); assertEquals(ResultCodes.BUCKET_NOT_EMPTY, omEx.getResult()); @@ -385,10 +373,12 @@ public void testDeleteNonEmptyBucket() throws Exception { @Test public void testLinkedBucketResolution() throws Exception { - createSampleVol(); + String volume = volumeName(); + createSampleVol(volume); + ECReplicationConfig ecConfig = new ECReplicationConfig(3, 2); OmBucketInfo bucketInfo = OmBucketInfo.newBuilder() - .setVolumeName("sample-vol") + .setVolumeName(volume) .setBucketName("bucket-one") .setDefaultReplicationConfig( new DefaultReplicationConfig( @@ -405,23 +395,23 @@ public void testLinkedBucketResolution() throws Exception { writeClient.createBucket(bucketInfo); OmBucketInfo bucketLinkInfo = OmBucketInfo.newBuilder() - .setVolumeName("sample-vol") + .setVolumeName(volume) .setBucketName("link-one") - .setSourceVolume("sample-vol") + .setSourceVolume(volume) .setSourceBucket("bucket-one") .build(); writeClient.createBucket(bucketLinkInfo); OmBucketInfo bucketLink2 = OmBucketInfo.newBuilder() - .setVolumeName("sample-vol") + .setVolumeName(volume) .setBucketName("link-two") - .setSourceVolume("sample-vol") + .setSourceVolume(volume) .setSourceBucket("link-one") .build(); writeClient.createBucket(bucketLink2); OmBucketInfo storedLinkBucket = - writeClient.getBucketInfo("sample-vol", "link-two"); + writeClient.getBucketInfo(volume, "link-two"); assertNotNull(storedLinkBucket.getDefaultReplicationConfig(), "Replication config is not set"); assertEquals(ecConfig, @@ -432,12 +422,12 @@ public void testLinkedBucketResolution() throws Exception { assertEquals( "link-two", storedLinkBucket.getBucketName()); assertEquals( - "sample-vol", storedLinkBucket.getVolumeName()); + volume, storedLinkBucket.getVolumeName()); assertEquals( "link-one", storedLinkBucket.getSourceBucket()); assertEquals( - "sample-vol", storedLinkBucket.getSourceVolume()); + volume, storedLinkBucket.getSourceVolume()); assertEquals( bucketInfo.getBucketLayout(), diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java index 3fa8462de1b1..0a515f6ede69 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerUnit.java @@ -31,6 +31,7 @@ import java.util.Map; import java.util.Set; import java.util.UUID; +import java.util.concurrent.atomic.AtomicLong; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.RandomUtils; @@ -71,11 +72,13 @@ import org.apache.ozone.test.GenericTestUtils; import org.apache.hadoop.util.Time; +import org.apache.ozone.test.OzoneTestBase; import org.apache.ratis.util.ExitUtils; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; @@ -90,6 +93,7 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.mockito.Mockito.anySet; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.reset; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -97,7 +101,10 @@ /** * Unit test key manager. */ -public class TestKeyManagerUnit { +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class TestKeyManagerUnit extends OzoneTestBase { + + private static final AtomicLong CONTAINER_ID = new AtomicLong(); private OzoneConfiguration configuration; private OMMetadataManager metadataManager; @@ -112,12 +119,9 @@ public class TestKeyManagerUnit { private OzoneManager om; @BeforeAll - public static void setup() { + void setup() throws Exception { ExitUtils.disableSystemExit(); - } - - @BeforeEach - public void init() throws Exception { + configuration = new OzoneConfiguration(); testDir = GenericTestUtils.getRandomizedTestDir(); configuration.set(HddsConfigKeys.OZONE_METADATA_DIRS, @@ -131,10 +135,15 @@ public void init() throws Exception { metadataManager = omTestManagers.getMetadataManager(); keyManager = (KeyManagerImpl)omTestManagers.getKeyManager(); writeClient = omTestManagers.getWriteClient(); + } + + @BeforeEach + void init() { + reset(blockClient, containerClient); startDate = Instant.ofEpochMilli(Time.now()); } - @AfterEach + @AfterAll public void cleanup() throws Exception { om.stop(); FileUtils.deleteDirectory(testDir); @@ -143,34 +152,40 @@ public void cleanup() throws Exception { @Test public void listMultipartUploadPartsWithZeroUpload() throws IOException { //GIVEN - createBucket(metadataManager, "vol1", "bucket1"); + final String volume = volumeName(); + createBucket(metadataManager, volume, "bucket1"); OmMultipartInfo omMultipartInfo = - initMultipartUpload(writeClient, "vol1", "bucket1", "dir/key1"); + initMultipartUpload(writeClient, volume, "bucket1", "dir/key1"); //WHEN OmMultipartUploadListParts omMultipartUploadListParts = keyManager - .listParts("vol1", "bucket1", "dir/key1", omMultipartInfo.getUploadID(), + .listParts(volume, "bucket1", "dir/key1", omMultipartInfo.getUploadID(), 0, 10); assertEquals(0, omMultipartUploadListParts.getPartInfoList().size()); } + private String volumeName() { + return getTestName(); + } + @Test public void listMultipartUploads() throws IOException { //GIVEN - createBucket(metadataManager, "vol1", "bucket1"); - createBucket(metadataManager, "vol1", "bucket2"); + final String volume = volumeName(); + createBucket(metadataManager, volume, "bucket1"); + createBucket(metadataManager, volume, "bucket2"); - initMultipartUpload(writeClient, "vol1", "bucket1", "dir/key1"); - initMultipartUpload(writeClient, "vol1", "bucket1", "dir/key2"); - initMultipartUpload(writeClient, "vol1", "bucket2", "dir/key1"); + initMultipartUpload(writeClient, volume, "bucket1", "dir/key1"); + initMultipartUpload(writeClient, volume, "bucket1", "dir/key2"); + initMultipartUpload(writeClient, volume, "bucket2", "dir/key1"); //WHEN OmMultipartUploadList omMultipartUploadList = - keyManager.listMultipartUploads("vol1", "bucket1", ""); + keyManager.listMultipartUploads(volume, "bucket1", ""); //THEN List uploads = omMultipartUploadList.getUploads(); @@ -188,8 +203,8 @@ public void listMultipartUploads() throws IOException { @Test public void listMultipartUploadsWithFewEntriesInCache() throws IOException { - String volume = UUID.randomUUID().toString(); - String bucket = UUID.randomUUID().toString(); + String volume = volumeName(); + String bucket = "bucket"; //GIVEN createBucket(metadataManager, volume, bucket); @@ -276,20 +291,21 @@ public void listMultipartUploadsWithFewEntriesInCache() throws IOException { public void listMultipartUploadsWithPrefix() throws IOException { //GIVEN - createBucket(metadataManager, "vol1", "bucket1"); - createBucket(metadataManager, "vol1", "bucket2"); + final String volumeName = volumeName(); + createBucket(metadataManager, volumeName, "bucket1"); + createBucket(metadataManager, volumeName, "bucket2"); - initMultipartUpload(writeClient, "vol1", "bucket1", "dip/key1"); + initMultipartUpload(writeClient, volumeName, "bucket1", "dip/key1"); - initMultipartUpload(writeClient, "vol1", "bucket1", "dir/key1"); - initMultipartUpload(writeClient, "vol1", "bucket1", "dir/key2"); - initMultipartUpload(writeClient, "vol1", "bucket1", "key3"); + initMultipartUpload(writeClient, volumeName, "bucket1", "dir/key1"); + initMultipartUpload(writeClient, volumeName, "bucket1", "dir/key2"); + initMultipartUpload(writeClient, volumeName, "bucket1", "key3"); - initMultipartUpload(writeClient, "vol1", "bucket2", "dir/key1"); + initMultipartUpload(writeClient, volumeName, "bucket2", "dir/key1"); //WHEN OmMultipartUploadList omMultipartUploadList = - keyManager.listMultipartUploads("vol1", "bucket1", "dir"); + keyManager.listMultipartUploads(volumeName, "bucket1", "dir"); //THEN List uploads = omMultipartUploadList.getUploads(); @@ -357,7 +373,7 @@ public void testGetKeyInfo() throws IOException { final DatanodeDetails dn2 = MockDatanodeDetails.randomDatanodeDetails(); final DatanodeDetails dn3 = MockDatanodeDetails.randomDatanodeDetails(); final DatanodeDetails dn4 = MockDatanodeDetails.randomDatanodeDetails(); - final long containerID = 1L; + final long containerID = CONTAINER_ID.incrementAndGet(); Set containerIDs = newHashSet(containerID); final Pipeline pipeline1 = Pipeline.newBuilder() @@ -388,18 +404,19 @@ public void testGetKeyInfo() throws IOException { singletonList(new ContainerWithPipeline(ci, pipeline1)), singletonList(new ContainerWithPipeline(ci, pipeline2))); - insertVolume("volumeOne"); + final String volume = volumeName(); + insertVolume(volume); - insertBucket("volumeOne", "bucketOne"); + insertBucket(volume, "bucketOne"); BlockID blockID1 = new BlockID(containerID, 1L); - insertKey(null, "volumeOne", "bucketOne", "keyOne", blockID1); + insertKey(null, volume, "bucketOne", "keyOne", blockID1); BlockID blockID2 = new BlockID(containerID, 2L); - insertKey(null, "volumeOne", "bucketOne", "keyTwo", blockID2); + insertKey(null, volume, "bucketOne", "keyTwo", blockID2); // 1st call to get key1. OmKeyArgs keyArgs = new Builder() - .setVolumeName("volumeOne") + .setVolumeName(volume) .setBucketName("bucketOne") .setKeyName("keyOne") .build(); @@ -415,7 +432,7 @@ public void testGetKeyInfo() throws IOException { // subsequent call to key2 in same container sound result no scm calls. keyArgs = new Builder() - .setVolumeName("volumeOne") + .setVolumeName(volume) .setBucketName("bucketOne") .setKeyName("keyTwo") .build(); @@ -431,7 +448,7 @@ public void testGetKeyInfo() throws IOException { // Yet, another call with forceCacheUpdate should trigger a call to SCM. keyArgs = new Builder() - .setVolumeName("volumeOne") + .setVolumeName(volume) .setBucketName("bucketOne") .setKeyName("keyTwo") .setForceUpdateContainerCacheFromSCM(true) @@ -492,15 +509,16 @@ public void testLookupFileWithDnFailure() throws IOException { when(containerClient.getContainerWithPipelineBatch(containerIDs)) .thenReturn(cps); - insertVolume("volumeOne"); + final String volume = volumeName(); + insertVolume(volume); - insertBucket("volumeOne", "bucketOne"); + insertBucket(volume, "bucketOne"); - insertKey(pipelineOne, "volumeOne", "bucketOne", "keyOne", + insertKey(pipelineOne, volume, "bucketOne", "keyOne", new BlockID(1L, 1L)); final OmKeyArgs.Builder keyArgs = new OmKeyArgs.Builder() - .setVolumeName("volumeOne") + .setVolumeName(volume) .setBucketName("bucketOne") .setKeyName("keyOne"); @@ -566,7 +584,7 @@ private void insertVolume(String volumeName) throws IOException { @Test public void listStatus() throws Exception { - String volume = "vol"; + String volume = volumeName(); String bucket = "bucket"; String keyPrefix = "key"; String client = "client.host"; @@ -584,19 +602,20 @@ public void listStatus() throws Exception { Set containerIDs = new HashSet<>(); List containersWithPipeline = new ArrayList<>(); for (long i = 1; i <= 10; i++) { + final long containerID = CONTAINER_ID.incrementAndGet(); final OmKeyLocationInfo keyLocationInfo = new OmKeyLocationInfo.Builder() - .setBlockID(new BlockID(i, 1L)) + .setBlockID(new BlockID(containerID, 1L)) .setPipeline(pipeline) .setOffset(0) .setLength(256000) .build(); ContainerInfo containerInfo = new ContainerInfo.Builder() - .setContainerID(i) + .setContainerID(containerID) .build(); containersWithPipeline.add( new ContainerWithPipeline(containerInfo, pipeline)); - containerIDs.add(i); + containerIDs.add(containerID); OmKeyInfo keyInfo = new OmKeyInfo.Builder() .setVolumeName(volume) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java index 79e201b4299b..e1ae8f57d15e 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotManager.java @@ -19,7 +19,6 @@ package org.apache.hadoop.ozone.om; -import org.apache.commons.io.FileUtils; import org.apache.hadoop.hdds.HddsConfigKeys; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.scm.HddsWhiteboxTestUtils; @@ -31,9 +30,12 @@ import org.apache.hadoop.ozone.om.snapshot.OmSnapshotUtils; import org.apache.hadoop.util.Time; import org.apache.ozone.test.GenericTestUtils; -import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.io.TempDir; import java.io.File; import java.io.IOException; @@ -79,10 +81,10 @@ /** * Unit test ozone snapshot manager. */ -public class TestOmSnapshotManager { +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class TestOmSnapshotManager { private OzoneManager om; - private File testDir; private static final String CANDIDATE_DIR_NAME = OM_DB_NAME + SNAPSHOT_CANDIDATE_DIR; private File leaderDir; @@ -94,12 +96,10 @@ public class TestOmSnapshotManager { private File s1File; private File f1File; - @BeforeEach - public void init() throws Exception { + @BeforeAll + void init(@TempDir File tempDir) throws Exception { OzoneConfiguration configuration = new OzoneConfiguration(); - testDir = GenericTestUtils.getRandomizedTestDir(); - configuration.set(HddsConfigKeys.OZONE_METADATA_DIRS, - testDir.toString()); + configuration.set(HddsConfigKeys.OZONE_METADATA_DIRS, tempDir.toString()); // Enable filesystem snapshot feature for the test regardless of the default configuration.setBoolean(OMConfigKeys.OZONE_FILESYSTEM_SNAPSHOT_ENABLED_KEY, true); @@ -110,13 +110,11 @@ public void init() throws Exception { OmTestManagers omTestManagers = new OmTestManagers(configuration); om = omTestManagers.getOzoneManager(); - setupData(); } - @AfterEach - public void cleanup() throws Exception { + @AfterAll + void stop() { om.stop(); - FileUtils.deleteDirectory(testDir); } @Test @@ -206,7 +204,8 @@ public void testCloseOnEviction() throws IOException { verify(firstSnapshotStore, timeout(3000).times(1)).close(); } - private void setupData() throws IOException { + @BeforeEach + void setupData(@TempDir File testDir) throws IOException { // Set up the leader with the following files: // leader/db.checkpoints/checkpoint1/f1.sst // leader/db.snapshots/checkpointState/snap1/s1.sst @@ -230,8 +229,7 @@ private void setupData() throws IOException { byte[] dummyData = {0}; // Create dummy leader files to calculate links. - leaderDir = new File(testDir.toString(), - "leader"); + leaderDir = new File(testDir, "leader"); assertTrue(leaderDir.mkdirs()); String pathSnap1 = OM_SNAPSHOT_CHECKPOINT_DIR + OM_KEY_PREFIX + "snap1"; String pathSnap2 = OM_SNAPSHOT_CHECKPOINT_DIR + OM_KEY_PREFIX + "snap2"; @@ -245,8 +243,7 @@ private void setupData() throws IOException { Files.write(Paths.get(leaderSnapDir2.toString(), "nonSstFile"), dummyData); // Also create the follower files. - candidateDir = new File(testDir.toString(), - CANDIDATE_DIR_NAME); + candidateDir = new File(testDir, CANDIDATE_DIR_NAME); File followerSnapDir1 = new File(candidateDir.toString(), pathSnap1); followerSnapDir2 = new File(candidateDir.toString(), pathSnap2); copyDirectory(leaderDir.toPath(), candidateDir.toPath()); @@ -359,9 +356,9 @@ public void testExcludeUtilities() throws IOException { * This test always passes in a null dest dir. */ @Test - public void testProcessFileWithNullDestDirParameter() throws IOException { - assertTrue(new File(testDir.toString(), "snap1").mkdirs()); - assertTrue(new File(testDir.toString(), "snap2").mkdirs()); + void testProcessFileWithNullDestDirParameter(@TempDir File testDir) throws IOException { + assertTrue(new File(testDir, "snap1").mkdirs()); + assertTrue(new File(testDir, "snap2").mkdirs()); Path copyFile = Paths.get(testDir.toString(), "snap1/copyfile.sst"); Files.write(copyFile, @@ -450,10 +447,10 @@ public void testProcessFileWithNullDestDirParameter() throws IOException { * This test always passes in a non-null dest dir. */ @Test - public void testProcessFileWithDestDirParameter() throws IOException { - assertTrue(new File(testDir.toString(), "snap1").mkdirs()); - assertTrue(new File(testDir.toString(), "snap2").mkdirs()); - assertTrue(new File(testDir.toString(), "snap3").mkdirs()); + void testProcessFileWithDestDirParameter(@TempDir File testDir) throws IOException { + assertTrue(new File(testDir, "snap1").mkdirs()); + assertTrue(new File(testDir, "snap2").mkdirs()); + assertTrue(new File(testDir, "snap3").mkdirs()); Path destDir = Paths.get(testDir.toString(), "destDir"); assertTrue(new File(destDir.toString()).mkdirs()); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestSstFilteringService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestSstFilteringService.java index edc228a5d313..a8b026af05b5 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestSstFilteringService.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestSstFilteringService.java @@ -36,7 +36,6 @@ import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted; import org.apache.hadoop.ozone.om.snapshot.SnapshotCache; -import org.apache.hadoop.security.authentication.client.AuthenticationException; import org.apache.ratis.util.ExitUtils; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; @@ -87,7 +86,7 @@ public static void setup() { } @BeforeEach - public void init() throws AuthenticationException, IOException { + void init() throws Exception { conf = new OzoneConfiguration(); conf.set(OZONE_METADATA_DIRS, folder.getAbsolutePath()); conf.setTimeDuration(HDDS_CONTAINER_REPORT_INTERVAL, 200, diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestTrashService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestTrashService.java index 9cdd965068d3..b7c83956085f 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestTrashService.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestTrashService.java @@ -32,7 +32,6 @@ import org.apache.hadoop.ozone.om.helpers.OpenKeySession; import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; -import org.apache.hadoop.security.authentication.client.AuthenticationException; import org.apache.ratis.util.ExitUtils; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -66,7 +65,7 @@ public class TestTrashService { private String bucketName; @BeforeEach - public void setup() throws IOException, AuthenticationException { + void setup() throws Exception { ExitUtils.disableSystemExit(); OzoneConfiguration configuration = new OzoneConfiguration(); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestMultipartUploadCleanupService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestMultipartUploadCleanupService.java index 272b7b72db19..762d8740565f 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestMultipartUploadCleanupService.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestMultipartUploadCleanupService.java @@ -36,23 +36,23 @@ import org.apache.hadoop.ozone.om.helpers.OpenKeySession; import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; -import org.apache.ozone.test.GenericTestUtils; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ExpiredMultipartUploadsBucket; import org.apache.ratis.util.ExitUtils; -import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.Timeout; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; +import java.io.UncheckedIOException; import java.nio.file.Path; import java.time.Duration; import java.util.ArrayList; import java.util.Collections; +import java.util.List; import java.util.UUID; import java.util.concurrent.TimeUnit; @@ -60,31 +60,27 @@ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_MPU_CLEANUP_SERVICE_INTERVAL; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_MPU_EXPIRE_THRESHOLD; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_MPU_PARTS_CLEANUP_LIMIT_PER_TASK; +import static org.apache.ozone.test.GenericTestUtils.waitFor; import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; /** * Test Multipart Upload Cleanup Service. */ -public class TestMultipartUploadCleanupService { +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@Timeout(300) +class TestMultipartUploadCleanupService { private OzoneManagerProtocol writeClient; private OzoneManager om; - private static final Logger LOG = - LoggerFactory.getLogger(TestMultipartUploadCleanupService.class); - private static final Duration SERVICE_INTERVAL = Duration.ofMillis(500); - private static final Duration EXPIRE_THRESHOLD = Duration.ofMillis(1000); + private static final Duration SERVICE_INTERVAL = Duration.ofMillis(100); + private static final Duration EXPIRE_THRESHOLD = Duration.ofMillis(200); private KeyManager keyManager; private OMMetadataManager omMetadataManager; @BeforeAll - public static void setup() { + void setup(@TempDir Path tempDir) throws Exception { ExitUtils.disableSystemExit(); - } - @BeforeEach - public void createConfAndInitValues(@TempDir Path tempDir) throws Exception { OzoneConfiguration conf = new OzoneConfiguration(); System.setProperty(DBConfigFromFile.CONFIG_DIR, "/"); ServerUtils.setOzoneMetaDirPath(conf, tempDir.toString()); @@ -101,8 +97,8 @@ public void createConfAndInitValues(@TempDir Path tempDir) throws Exception { om = omTestManagers.getOzoneManager(); } - @AfterEach - public void cleanup() throws Exception { + @AfterAll + void cleanup() { om.stop(); } @@ -110,7 +106,6 @@ public void cleanup() throws Exception { * Create a bunch incomplete/inflight multipart upload info. Then we start * the MultipartUploadCleanupService. We make sure that all the multipart * upload info is picked up and aborted by OzoneManager. - * @throws Exception */ @ParameterizedTest @CsvSource({ @@ -118,9 +113,7 @@ public void cleanup() throws Exception { "0, 88", "66, 77" }) - @Timeout(300) - public void checkIfCleanupServiceIsDeletingExpiredMultipartUpload( - int numDEFKeys, int numFSOKeys) throws Exception { + void deletesExpiredUpload(int numDEFKeys, int numFSOKeys) throws Exception { MultipartUploadCleanupService multipartUploadCleanupService = (MultipartUploadCleanupService) @@ -140,24 +133,27 @@ public void checkIfCleanupServiceIsDeletingExpiredMultipartUpload( // wait for MPU info to expire Thread.sleep(EXPIRE_THRESHOLD.toMillis()); - assertFalse(keyManager.getExpiredMultipartUploads(EXPIRE_THRESHOLD, - 10000).isEmpty()); + assertThat(getExpiredMultipartUploads()).isNotEmpty(); multipartUploadCleanupService.resume(); - GenericTestUtils.waitFor(() -> multipartUploadCleanupService - .getRunCount() > oldRunCount, - (int) SERVICE_INTERVAL.toMillis(), - 5 * (int) SERVICE_INTERVAL.toMillis()); - // wait for requests to complete - Thread.sleep(10 * SERVICE_INTERVAL.toMillis()); + waitFor(() -> getExpiredMultipartUploads().isEmpty(), + (int) SERVICE_INTERVAL.toMillis(), + 15 * (int) SERVICE_INTERVAL.toMillis()); + assertThat(multipartUploadCleanupService.getRunCount()) + .isGreaterThan(oldRunCount); assertThat(multipartUploadCleanupService.getSubmittedMpuInfoCount()) .isGreaterThanOrEqualTo(oldMpuInfoCount + numDEFKeys + numFSOKeys); - assertTrue(keyManager.getExpiredMultipartUploads(EXPIRE_THRESHOLD, - 10000).isEmpty()); + } + private List getExpiredMultipartUploads() { + try { + return keyManager.getExpiredMultipartUploads(EXPIRE_THRESHOLD, 10000); + } catch (IOException e) { + throw new UncheckedIOException(e); + } } private void createIncompleteMPUKeys(int mpuKeyCount, @@ -202,10 +198,6 @@ private void createVolumeAndBucket(String volumeName, String bucketName, /** * Create inflight multipart upload that are not completed / aborted yet. - * @param volumeName - * @param bucketName - * @param keyName - * @throws IOException */ private void createIncompleteMPUKey(String volumeName, String bucketName, String keyName, int numParts) throws IOException { diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestOpenKeyCleanupService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestOpenKeyCleanupService.java index 2f4016d0e967..418608e8559c 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestOpenKeyCleanupService.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestOpenKeyCleanupService.java @@ -43,9 +43,9 @@ import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; import org.apache.ozone.test.GenericTestUtils; import org.apache.ratis.util.ExitUtils; -import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.Timeout; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; @@ -67,6 +67,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; +@TestInstance(TestInstance.Lifecycle.PER_CLASS) class TestOpenKeyCleanupService { private OzoneManagerProtocol writeClient; private OzoneManager om; @@ -83,12 +84,9 @@ class TestOpenKeyCleanupService { private OMMetadataManager omMetadataManager; @BeforeAll - public static void setup() { + void setup(@TempDir Path tempDir) throws Exception { ExitUtils.disableSystemExit(); - } - @BeforeEach - public void createConfAndInitValues(@TempDir Path tempDir) throws Exception { OzoneConfiguration conf = new OzoneConfiguration(); System.setProperty(DBConfigFromFile.CONFIG_DIR, "/"); ServerUtils.setOzoneMetaDirPath(conf, tempDir.toString()); @@ -105,8 +103,8 @@ public void createConfAndInitValues(@TempDir Path tempDir) throws Exception { om = omTestManagers.getOzoneManager(); } - @AfterEach - public void cleanup() throws Exception { + @AfterAll + void cleanup() { if (om.stop()) { om.join(); } @@ -144,12 +142,11 @@ public void testCleanupExpiredOpenKeys( final long oldkeyCount = openKeyCleanupService.getSubmittedOpenKeyCount(); final long oldrunCount = openKeyCleanupService.getRunCount(); LOG.info("oldkeyCount={}, oldrunCount={}", oldkeyCount, oldrunCount); - assertEquals(0, oldkeyCount); final OMMetrics metrics = om.getMetrics(); - assertEquals(0, metrics.getNumKeyHSyncs()); - assertEquals(0, metrics.getNumOpenKeysCleaned()); - assertEquals(0, metrics.getNumOpenKeysHSyncCleaned()); + long numKeyHSyncs = metrics.getNumKeyHSyncs(); + long numOpenKeysCleaned = metrics.getNumOpenKeysCleaned(); + long numOpenKeysHSyncCleaned = metrics.getNumOpenKeysHSyncCleaned(); final int keyCount = numDEFKeys + numFSOKeys; createOpenKeys(numDEFKeys, false, BucketLayout.DEFAULT); createOpenKeys(numFSOKeys, hsync, BucketLayout.FILE_SYSTEM_OPTIMIZED); @@ -164,7 +161,7 @@ public void testCleanupExpiredOpenKeys( openKeyCleanupService.resume(); GenericTestUtils.waitFor( - () -> openKeyCleanupService.getSubmittedOpenKeyCount() >= keyCount, + () -> openKeyCleanupService.getSubmittedOpenKeyCount() >= oldkeyCount + keyCount, SERVICE_INTERVAL, WAIT_TIME); GenericTestUtils.waitFor( () -> openKeyCleanupService.getRunCount() >= oldrunCount + 2, @@ -174,13 +171,13 @@ public void testCleanupExpiredOpenKeys( waitForOpenKeyCleanup(hsync, BucketLayout.FILE_SYSTEM_OPTIMIZED); if (hsync) { - assertAtLeast(numDEFKeys, metrics.getNumOpenKeysCleaned()); - assertAtLeast(numFSOKeys, metrics.getNumOpenKeysHSyncCleaned()); - assertEquals(numFSOKeys, metrics.getNumKeyHSyncs()); + assertAtLeast(numOpenKeysCleaned + numDEFKeys, metrics.getNumOpenKeysCleaned()); + assertAtLeast(numOpenKeysHSyncCleaned + numFSOKeys, metrics.getNumOpenKeysHSyncCleaned()); + assertEquals(numKeyHSyncs + numFSOKeys, metrics.getNumKeyHSyncs()); } else { - assertAtLeast(keyCount, metrics.getNumOpenKeysCleaned()); - assertEquals(0, metrics.getNumOpenKeysHSyncCleaned()); - assertEquals(0, metrics.getNumKeyHSyncs()); + assertAtLeast(numOpenKeysCleaned + keyCount, metrics.getNumOpenKeysCleaned()); + assertEquals(numOpenKeysHSyncCleaned, metrics.getNumOpenKeysHSyncCleaned()); + assertEquals(numKeyHSyncs, metrics.getNumKeyHSyncs()); } } @@ -211,12 +208,11 @@ public void testExcludeMPUOpenKeys( final long oldkeyCount = openKeyCleanupService.getSubmittedOpenKeyCount(); final long oldrunCount = openKeyCleanupService.getRunCount(); LOG.info("oldMpuKeyCount={}, oldMpuRunCount={}", oldkeyCount, oldrunCount); - assertEquals(0, oldkeyCount); final OMMetrics metrics = om.getMetrics(); - assertEquals(0, metrics.getNumKeyHSyncs()); - assertEquals(0, metrics.getNumOpenKeysCleaned()); - assertEquals(0, metrics.getNumOpenKeysHSyncCleaned()); + long numKeyHSyncs = metrics.getNumKeyHSyncs(); + long numOpenKeysCleaned = metrics.getNumOpenKeysCleaned(); + long numOpenKeysHSyncCleaned = metrics.getNumOpenKeysHSyncCleaned(); createIncompleteMPUKeys(numDEFKeys, BucketLayout.DEFAULT, NUM_MPU_PARTS, true); createIncompleteMPUKeys(numFSOKeys, BucketLayout.FILE_SYSTEM_OPTIMIZED, @@ -245,7 +241,9 @@ public void testExcludeMPUOpenKeys( assertExpiredOpenKeys(true, false, BucketLayout.FILE_SYSTEM_OPTIMIZED); - assertEquals(0, metrics.getNumOpenKeysCleaned()); + assertEquals(numKeyHSyncs, metrics.getNumKeyHSyncs()); + assertEquals(numOpenKeysCleaned, metrics.getNumOpenKeysCleaned()); + assertEquals(numOpenKeysHSyncCleaned, metrics.getNumOpenKeysHSyncCleaned()); } /** @@ -275,12 +273,9 @@ public void testCleanupExpiredOpenMPUPartKeys( final long oldkeyCount = openKeyCleanupService.getSubmittedOpenKeyCount(); final long oldrunCount = openKeyCleanupService.getRunCount(); LOG.info("oldMpuKeyCount={}, oldMpuRunCount={}", oldkeyCount, oldrunCount); - assertEquals(0, oldkeyCount); final OMMetrics metrics = om.getMetrics(); - assertEquals(0, metrics.getNumKeyHSyncs()); - assertEquals(0, metrics.getNumOpenKeysCleaned()); - assertEquals(0, metrics.getNumOpenKeysHSyncCleaned()); + long numOpenKeysCleaned = metrics.getNumOpenKeysCleaned(); final int keyCount = numDEFKeys + numFSOKeys; final int partCount = NUM_MPU_PARTS * keyCount; createIncompleteMPUKeys(numDEFKeys, BucketLayout.DEFAULT, NUM_MPU_PARTS, @@ -300,7 +295,7 @@ public void testCleanupExpiredOpenMPUPartKeys( openKeyCleanupService.resume(); GenericTestUtils.waitFor( - () -> openKeyCleanupService.getSubmittedOpenKeyCount() >= partCount, + () -> openKeyCleanupService.getSubmittedOpenKeyCount() >= oldkeyCount + partCount, SERVICE_INTERVAL, WAIT_TIME); GenericTestUtils.waitFor( () -> openKeyCleanupService.getRunCount() >= oldrunCount + 2, @@ -309,7 +304,7 @@ public void testCleanupExpiredOpenMPUPartKeys( // No expired MPU parts fetched waitForOpenKeyCleanup(false, BucketLayout.DEFAULT); waitForOpenKeyCleanup(false, BucketLayout.FILE_SYSTEM_OPTIMIZED); - assertAtLeast(partCount, metrics.getNumOpenKeysCleaned()); + assertAtLeast(numOpenKeysCleaned + partCount, metrics.getNumOpenKeysCleaned()); } private static void assertAtLeast(long expectedMinimum, long actual) { diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/acl/TestParentAcl.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/acl/TestParentAcl.java index e9249b1ab793..c73752904e79 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/acl/TestParentAcl.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/acl/TestParentAcl.java @@ -44,7 +44,6 @@ import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.authentication.client.AuthenticationException; import org.apache.ozone.test.GenericTestUtils; import org.apache.ozone.test.tag.Unhealthy; import org.junit.jupiter.api.AfterAll; @@ -95,7 +94,7 @@ public class TestParentAcl { private static File testDir; @BeforeAll - public static void setup() throws IOException, AuthenticationException { + static void setup() throws Exception { ozConfig = new OzoneConfiguration(); ozConfig.set(OZONE_ACL_AUTHORIZER_CLASS, OZONE_ACL_AUTHORIZER_CLASS_NATIVE); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/acl/TestVolumeOwner.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/acl/TestVolumeOwner.java index 7a2d22b338b4..6670dc644d05 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/acl/TestVolumeOwner.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/acl/TestVolumeOwner.java @@ -37,7 +37,6 @@ import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol; import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.authentication.client.AuthenticationException; import org.apache.ozone.test.GenericTestUtils; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -77,7 +76,7 @@ public class TestVolumeOwner { private static File testDir; @BeforeAll - public static void setup() throws IOException, AuthenticationException { + static void setup() throws Exception { ozoneConfig = new OzoneConfiguration(); ozoneConfig.set(OZONE_ACL_AUTHORIZER_CLASS, OZONE_ACL_AUTHORIZER_CLASS_NATIVE);