diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyDeletingService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyDeletingService.java index 143d483bb191..77bf15ed76b1 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyDeletingService.java @@ -21,18 +21,18 @@ import java.io.File; import java.io.IOException; -import java.nio.file.Path; +import java.io.UncheckedIOException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import java.util.stream.Collectors; +import java.util.concurrent.atomic.AtomicInteger; +import com.google.common.collect.ImmutableMap; import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.TableIterator; @@ -53,8 +53,13 @@ import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted; import org.apache.hadoop.ozone.om.snapshot.SnapshotCache; +import org.apache.ozone.test.OzoneTestBase; import org.apache.ratis.util.ExitUtils; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.io.TempDir; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -63,30 +68,27 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList; -import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol; import org.apache.hadoop.hdds.server.ServerUtils; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.OpenKeySession; import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol; -import org.apache.hadoop.security.authentication.client.AuthenticationException; import org.apache.ozone.test.GenericTestUtils; import org.apache.hadoop.hdds.utils.db.DBConfigFromFile; -import org.apache.commons.lang3.RandomStringUtils; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_REPORT_INTERVAL; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.THREE; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SNAPSHOT_DELETING_SERVICE_INTERVAL; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_SST_FILTERING_SERVICE_INTERVAL; import static org.apache.hadoop.ozone.om.OmSnapshotManager.getSnapshotPrefix; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.fail; -import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; @@ -99,607 +101,533 @@ * Metadata Manager. 3. Waits for a while for the KeyDeleting Service to pick up * and call into SCM. 4. Confirms that calls have been successful. */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) @Timeout(300) -public class TestKeyDeletingService { - @TempDir - private Path folder; - private OzoneManagerProtocol writeClient; - private OzoneManager om; +class TestKeyDeletingService extends OzoneTestBase { private static final Logger LOG = LoggerFactory.getLogger(TestKeyDeletingService.class); + private static final AtomicInteger OBJECT_COUNTER = new AtomicInteger(); + + private OzoneConfiguration conf; + private OzoneManagerProtocol writeClient; + private OzoneManager om; + private KeyManager keyManager; + private OMMetadataManager metadataManager; + private KeyDeletingService keyDeletingService; + private ScmBlockLocationTestingClient scmBlockTestingClient; @BeforeAll - public static void setup() { + void setup() { ExitUtils.disableSystemExit(); } - private OzoneConfiguration createConfAndInitValues() throws IOException { - OzoneConfiguration conf = new OzoneConfiguration(); - File newFolder = folder.toFile(); - if (!newFolder.exists()) { - assertTrue(newFolder.mkdirs()); - } + private void createConfig(File testDir) { + conf = new OzoneConfiguration(); System.setProperty(DBConfigFromFile.CONFIG_DIR, "/"); - ServerUtils.setOzoneMetaDirPath(conf, newFolder.toString()); - conf.setTimeDuration(OZONE_BLOCK_DELETING_SERVICE_INTERVAL, 1000, - TimeUnit.MILLISECONDS); + ServerUtils.setOzoneMetaDirPath(conf, testDir.toString()); + conf.setTimeDuration(OZONE_BLOCK_DELETING_SERVICE_INTERVAL, + 100, TimeUnit.MILLISECONDS); conf.setTimeDuration(OZONE_SNAPSHOT_DELETING_SERVICE_INTERVAL, 100, TimeUnit.MILLISECONDS); - conf.setTimeDuration(HDDS_CONTAINER_REPORT_INTERVAL, 200, - TimeUnit.MILLISECONDS); + conf.setTimeDuration(OZONE_SNAPSHOT_SST_FILTERING_SERVICE_INTERVAL, + 1, TimeUnit.SECONDS); + conf.setTimeDuration(HDDS_CONTAINER_REPORT_INTERVAL, + 200, TimeUnit.MILLISECONDS); conf.setQuietMode(false); - - return conf; } - @AfterEach - public void cleanup() throws Exception { - om.stop(); + private void createSubject() throws Exception { + OmTestManagers omTestManagers = new OmTestManagers(conf, scmBlockTestingClient, null); + keyManager = omTestManagers.getKeyManager(); + keyDeletingService = keyManager.getDeletingService(); + writeClient = omTestManagers.getWriteClient(); + om = omTestManagers.getOzoneManager(); + metadataManager = omTestManagers.getMetadataManager(); } /** - * In this test, we create a bunch of keys and delete them. Then we start the - * KeyDeletingService and pass a SCMClient which does not fail. We make sure - * that all the keys that we deleted is picked up and deleted by - * OzoneManager. - * - * @throws IOException - on Failure. + * Tests happy path. */ + @Nested + @TestInstance(TestInstance.Lifecycle.PER_CLASS) + class Normal { - @Test - public void checkIfDeleteServiceIsDeletingKeys() - throws IOException, TimeoutException, InterruptedException, - AuthenticationException { - OzoneConfiguration conf = createConfAndInitValues(); - OmTestManagers omTestManagers - = new OmTestManagers(conf); - KeyManager keyManager = omTestManagers.getKeyManager(); - writeClient = omTestManagers.getWriteClient(); - om = omTestManagers.getOzoneManager(); + @BeforeAll + void setup(@TempDir File testDir) throws Exception { + // failCallsFrequency = 0 means all calls succeed + scmBlockTestingClient = new ScmBlockLocationTestingClient(null, null, 0); - final int keyCount = 100; - createAndDeleteKeys(keyManager, keyCount, 1); - KeyDeletingService keyDeletingService = - (KeyDeletingService) keyManager.getDeletingService(); - GenericTestUtils.waitFor( - () -> keyDeletingService.getDeletedKeyCount().get() >= keyCount, - 1000, 10000); - assertThat(keyDeletingService.getRunCount().get()).isGreaterThanOrEqualTo(1); - assertEquals(0, keyManager.getPendingDeletionKeys( - Integer.MAX_VALUE).getKeyBlocksList().size()); - } + createConfig(testDir); + createSubject(); + } - @Test - public void checkIfDeleteServiceWithFailingSCM() - throws IOException, TimeoutException, InterruptedException, - AuthenticationException { - OzoneConfiguration conf = createConfAndInitValues(); - ScmBlockLocationProtocol blockClient = - //failCallsFrequency = 1 , means all calls fail. - new ScmBlockLocationTestingClient(null, null, 1); - OmTestManagers omTestManagers - = new OmTestManagers(conf, blockClient, null); - KeyManager keyManager = omTestManagers.getKeyManager(); - writeClient = omTestManagers.getWriteClient(); - om = omTestManagers.getOzoneManager(); + @AfterEach + void resume() { + keyDeletingService.resume(); + } - final int keyCount = 100; - createAndDeleteKeys(keyManager, keyCount, 1); - KeyDeletingService keyDeletingService = - (KeyDeletingService) keyManager.getDeletingService(); - GenericTestUtils.waitFor( - () -> { - try { - int numPendingDeletionKeys = - keyManager.getPendingDeletionKeys(Integer.MAX_VALUE) - .getKeyBlocksList().size(); - if (numPendingDeletionKeys != keyCount) { - LOG.info("Expected {} keys to be pending deletion, but got {}", - keyCount, numPendingDeletionKeys); - return false; - } - return true; - } catch (IOException e) { - LOG.error("Error while getting pending deletion keys.", e); - return false; - } - }, 100, 2000); - // Make sure that we have run the background thread 5 times more - GenericTestUtils.waitFor( - () -> keyDeletingService.getRunCount().get() >= 5, - 100, 10000); - // Since SCM calls are failing, deletedKeyCount should be zero. - assertEquals(0, keyDeletingService.getDeletedKeyCount().get()); - assertEquals(keyCount, keyManager - .getPendingDeletionKeys(Integer.MAX_VALUE).getKeyBlocksList().size()); - } + @AfterAll + void cleanup() { + if (om.stop()) { + om.join(); + } + } - @Test - public void checkDeletionForEmptyKey() - throws IOException, TimeoutException, InterruptedException, - AuthenticationException { - OzoneConfiguration conf = createConfAndInitValues(); - ScmBlockLocationProtocol blockClient = - //failCallsFrequency = 1 , means all calls fail. - new ScmBlockLocationTestingClient(null, null, 1); - OmTestManagers omTestManagers - = new OmTestManagers(conf, blockClient, null); - KeyManager keyManager = omTestManagers.getKeyManager(); - writeClient = omTestManagers.getWriteClient(); - om = omTestManagers.getOzoneManager(); + /** + * In this test, we create a bunch of keys and delete them. Then we start the + * KeyDeletingService and pass a SCMClient which does not fail. We make sure + * that all the keys that we deleted is picked up and deleted by + * OzoneManager. + */ + @Test + void checkIfDeleteServiceIsDeletingKeys() + throws IOException, TimeoutException, InterruptedException { + final long initialDeletedCount = getDeletedKeyCount(); + final long initialRunCount = getRunCount(); + + final int keyCount = 100; + createAndDeleteKeys(keyCount, 1); + + GenericTestUtils.waitFor( + () -> getDeletedKeyCount() >= initialDeletedCount + keyCount, + 100, 10000); + assertThat(getRunCount()).isGreaterThan(initialRunCount); + assertThat(keyManager.getPendingDeletionKeys(Integer.MAX_VALUE).getKeyBlocksList()) + .isEmpty(); + } - final int keyCount = 100; - createAndDeleteKeys(keyManager, keyCount, 0); - KeyDeletingService keyDeletingService = - (KeyDeletingService) keyManager.getDeletingService(); - - // the pre-allocated blocks are not committed, hence they will be deleted. - GenericTestUtils.waitFor( - () -> { - try { - int numPendingDeletionKeys = - keyManager.getPendingDeletionKeys(Integer.MAX_VALUE) - .getKeyBlocksList().size(); - if (numPendingDeletionKeys != keyCount) { - LOG.info("Expected {} keys to be pending deletion, but got {}", - keyCount, numPendingDeletionKeys); - return false; - } - return true; - } catch (IOException e) { - LOG.error("Error while getting pending deletion keys.", e); - return false; - } - }, 100, 2000); - - // Make sure that we have run the background thread 2 times or more - GenericTestUtils.waitFor( - () -> keyDeletingService.getRunCount().get() >= 2, - 100, 1000); - // the blockClient is set to fail the deletion of key blocks, hence no keys - // will be deleted - assertEquals(0, keyDeletingService.getDeletedKeyCount().get()); - } + @Test + void checkDeletionForKeysWithMultipleVersions() throws Exception { + final long initialDeletedCount = getDeletedKeyCount(); + final long initialRunCount = getRunCount(); + final int initialDeletedBlockCount = scmBlockTestingClient.getNumberOfDeletedBlocks(); - @Test - public void checkDeletionForPartiallyCommitKey() - throws IOException, TimeoutException, InterruptedException, - AuthenticationException { - OzoneConfiguration conf = createConfAndInitValues(); - ScmBlockLocationProtocol blockClient = - //failCallsFrequency = 1 , means all calls fail. - new ScmBlockLocationTestingClient(null, null, 1); - OmTestManagers omTestManagers - = new OmTestManagers(conf, blockClient, null); - KeyManager keyManager = omTestManagers.getKeyManager(); - writeClient = omTestManagers.getWriteClient(); - om = omTestManagers.getOzoneManager(); + final String volumeName = getTestName(); + final String bucketName = uniqueObjectName("bucket"); - String volumeName = String.format("volume%s", - RandomStringUtils.randomAlphanumeric(5)); - String bucketName = String.format("bucket%s", - RandomStringUtils.randomAlphanumeric(5)); - String keyName = String.format("key%s", - RandomStringUtils.randomAlphanumeric(5)); - - // Create Volume and Bucket - createVolumeAndBucket(keyManager, volumeName, bucketName, false); - - OmKeyArgs keyArg = createAndCommitKey(keyManager, volumeName, bucketName, - keyName, 3, 1); - - // Only the uncommitted block should be pending to be deleted. - GenericTestUtils.waitFor( - () -> { - try { - return keyManager.getPendingDeletionKeys(Integer.MAX_VALUE) - .getKeyBlocksList() - .stream() - .map(BlockGroup::getBlockIDList) - .flatMap(Collection::stream) - .collect(Collectors.toList()).size() == 1; - } catch (IOException e) { - e.printStackTrace(); - } - return false; - }, - 500, 3000); - - // Delete the key - writeClient.deleteKey(keyArg); + // Create Volume and Bucket with versioning enabled + createVolumeAndBucket(volumeName, bucketName, true); - KeyDeletingService keyDeletingService = - (KeyDeletingService) keyManager.getDeletingService(); - - // All blocks should be pending to be deleted. - GenericTestUtils.waitFor( - () -> { - try { - return keyManager.getPendingDeletionKeys(Integer.MAX_VALUE) - .getKeyBlocksList() - .stream() - .map(BlockGroup::getBlockIDList) - .flatMap(Collection::stream) - .collect(Collectors.toList()).size() == 3; - } catch (IOException e) { - e.printStackTrace(); - } - return false; - }, - 500, 3000); - - // the blockClient is set to fail the deletion of key blocks, hence no keys - // will be deleted - assertEquals(0, keyDeletingService.getDeletedKeyCount().get()); - } + // Create 2 versions of the same key + final String keyName = uniqueObjectName("key"); + OmKeyArgs keyArgs = createAndCommitKey(volumeName, bucketName, keyName, 1); + createAndCommitKey(volumeName, bucketName, keyName, 2); - @Test - public void checkDeletionForKeysWithMultipleVersions() - throws IOException, TimeoutException, InterruptedException, - AuthenticationException { - OzoneConfiguration conf = createConfAndInitValues(); - OmTestManagers omTestManagers = new OmTestManagers(conf); - KeyManager keyManager = omTestManagers.getKeyManager(); - writeClient = omTestManagers.getWriteClient(); - om = omTestManagers.getOzoneManager(); + // Delete the key + writeClient.deleteKey(keyArgs); + + GenericTestUtils.waitFor( + () -> getDeletedKeyCount() >= initialDeletedCount + 1, + 1000, 10000); + assertThat(getRunCount()) + .isGreaterThan(initialRunCount); + assertThat(keyManager.getPendingDeletionKeys(Integer.MAX_VALUE).getKeyBlocksList()) + .isEmpty(); + + // The 1st version of the key has 1 block and the 2nd version has 2 + // blocks. Hence, the ScmBlockClient should have received at least 3 + // blocks for deletion from the KeyDeletionService + assertThat(scmBlockTestingClient.getNumberOfDeletedBlocks()) + .isGreaterThanOrEqualTo(initialDeletedBlockCount + 3); + } - String volumeName = String.format("volume%s", - RandomStringUtils.randomAlphanumeric(5)); - String bucketName = String.format("bucket%s", - RandomStringUtils.randomAlphanumeric(5)); - - // Create Volume and Bucket with versioning enabled - createVolumeAndBucket(keyManager, volumeName, bucketName, true); - - // Create 2 versions of the same key - String keyName = String.format("key%s", - RandomStringUtils.randomAlphanumeric(5)); - OmKeyArgs keyArgs = createAndCommitKey(keyManager, volumeName, bucketName, - keyName, 1); - createAndCommitKey(keyManager, volumeName, bucketName, keyName, 2); - - // Delete the key - writeClient.deleteKey(keyArgs); - - KeyDeletingService keyDeletingService = - (KeyDeletingService) keyManager.getDeletingService(); - GenericTestUtils.waitFor( - () -> keyDeletingService.getDeletedKeyCount().get() >= 1, - 1000, 10000); - assertThat(keyDeletingService.getRunCount().get()).isGreaterThanOrEqualTo(1); - assertEquals(0, keyManager.getPendingDeletionKeys( - Integer.MAX_VALUE).getKeyBlocksList().size()); - - // The 1st version of the key has 1 block and the 2nd version has 2 - // blocks. Hence, the ScmBlockClient should have received atleast 3 - // blocks for deletion from the KeyDeletionService - ScmBlockLocationTestingClient scmBlockTestingClient = - (ScmBlockLocationTestingClient) omTestManagers.getScmBlockClient(); - assertThat(scmBlockTestingClient.getNumberOfDeletedBlocks()).isGreaterThanOrEqualTo(3); - } + @Test + void checkDeletedTableCleanUpForSnapshot() throws Exception { + final String volumeName = getTestName(); + final String bucketName1 = uniqueObjectName("bucket"); + final String bucketName2 = uniqueObjectName("bucket"); + final String keyName = uniqueObjectName("key"); - private void createAndDeleteKeys(KeyManager keyManager, int keyCount, - int numBlocks) throws IOException { - for (int x = 0; x < keyCount; x++) { - String volumeName = String.format("volume%s", - RandomStringUtils.randomAlphanumeric(5)); - String bucketName = String.format("bucket%s", - RandomStringUtils.randomAlphanumeric(5)); - String keyName = String.format("key%s", - RandomStringUtils.randomAlphanumeric(5)); + final long initialDeletedCount = getDeletedKeyCount(); + final long initialRunCount = getRunCount(); - // Create Volume and Bucket - createVolumeAndBucket(keyManager, volumeName, bucketName, false); + // Create Volume and Buckets + createVolumeAndBucket(volumeName, bucketName1, false); + createVolumeAndBucket(volumeName, bucketName2, false); - // Create the key - OmKeyArgs keyArg = createAndCommitKey(keyManager, volumeName, bucketName, - keyName, numBlocks); + // Create the keys + OmKeyArgs key1 = createAndCommitKey(volumeName, bucketName1, keyName, 3); + OmKeyArgs key2 = createAndCommitKey(volumeName, bucketName2, keyName, 3); + + // Create snapshot + String snapName = uniqueObjectName("snap"); + writeClient.createSnapshot(volumeName, bucketName1, snapName); // Delete the key - writeClient.deleteKey(keyArg); + writeClient.deleteKey(key1); + writeClient.deleteKey(key2); + + // Run KeyDeletingService + GenericTestUtils.waitFor( + () -> getDeletedKeyCount() >= initialDeletedCount + 1, + 1000, 10000); + assertThat(getRunCount()) + .isGreaterThan(initialRunCount); + assertThat(keyManager.getPendingDeletionKeys(Integer.MAX_VALUE).getKeyBlocksList()) + .isEmpty(); + + // deletedTable should have deleted key of the snapshot bucket + assertFalse(metadataManager.getDeletedTable().isEmpty()); + String ozoneKey1 = + metadataManager.getOzoneKey(volumeName, bucketName1, keyName); + String ozoneKey2 = + metadataManager.getOzoneKey(volumeName, bucketName2, keyName); + + // key1 belongs to snapshot, so it should not be deleted when + // KeyDeletingService runs. But key2 can be reclaimed as it doesn't + // belong to any snapshot scope. + List> rangeKVs + = metadataManager.getDeletedTable().getRangeKVs( + null, 100, ozoneKey1); + assertThat(rangeKVs.size()).isGreaterThan(0); + rangeKVs + = metadataManager.getDeletedTable().getRangeKVs( + null, 100, ozoneKey2); + assertEquals(0, rangeKVs.size()); } - } - @Test - public void checkDeletedTableCleanUpForSnapshot() - throws Exception { - OzoneConfiguration conf = createConfAndInitValues(); - OmTestManagers omTestManagers - = new OmTestManagers(conf); - KeyManager keyManager = omTestManagers.getKeyManager(); - writeClient = omTestManagers.getWriteClient(); - om = omTestManagers.getOzoneManager(); - OMMetadataManager metadataManager = omTestManagers.getMetadataManager(); - - String volumeName = String.format("volume%s", - RandomStringUtils.randomAlphanumeric(5)); - String bucketName1 = String.format("bucket%s", - RandomStringUtils.randomAlphanumeric(5)); - String bucketName2 = String.format("bucket%s", - RandomStringUtils.randomAlphanumeric(5)); - String keyName = String.format("key%s", - RandomStringUtils.randomAlphanumeric(5)); - - // Create Volume and Buckets - createVolumeAndBucket(keyManager, volumeName, bucketName1, false); - createVolumeAndBucket(keyManager, volumeName, bucketName2, false); - - // Create the keys - OmKeyArgs key1 = createAndCommitKey(keyManager, volumeName, bucketName1, - keyName, 3); - OmKeyArgs key2 = createAndCommitKey(keyManager, volumeName, bucketName2, - keyName, 3); - - // Create snapshot - String snapName = "snap1"; - writeClient.createSnapshot(volumeName, bucketName1, snapName); - - // Delete the key - writeClient.deleteKey(key1); - writeClient.deleteKey(key2); - - // Run KeyDeletingService - KeyDeletingService keyDeletingService = - (KeyDeletingService) keyManager.getDeletingService(); - GenericTestUtils.waitFor( - () -> keyDeletingService.getDeletedKeyCount().get() >= 1, - 1000, 10000); - assertThat(keyDeletingService.getRunCount().get()).isGreaterThanOrEqualTo(1); - assertEquals(0, keyManager - .getPendingDeletionKeys(Integer.MAX_VALUE).getKeyBlocksList().size()); - - // deletedTable should have deleted key of the snapshot bucket - assertFalse(metadataManager.getDeletedTable().isEmpty()); - String ozoneKey1 = - metadataManager.getOzoneKey(volumeName, bucketName1, keyName); - String ozoneKey2 = - metadataManager.getOzoneKey(volumeName, bucketName2, keyName); - - // key1 belongs to snapshot, so it should not be deleted when - // KeyDeletingService runs. But key2 can be reclaimed as it doesn't - // belong to any snapshot scope. - List> rangeKVs - = metadataManager.getDeletedTable().getRangeKVs( - null, 100, ozoneKey1); - assertThat(rangeKVs.size()).isGreaterThan(0); - rangeKVs - = metadataManager.getDeletedTable().getRangeKVs( - null, 100, ozoneKey2); - assertEquals(0, rangeKVs.size()); - } + /* + * Create Snap1 + * Create 10 keys + * Create Snap2 + * Delete 10 keys + * Create 5 keys + * Delete 5 keys -> but stop KeyDeletingService so + that keys won't be reclaimed. + * Create snap3 + * Now wait for snap3 to be deepCleaned -> Deleted 5 + keys should be deep cleaned. + * Now delete snap2 -> Wait for snap3 to be deep cleaned so deletedTable + of Snap3 should be empty. + */ + @Test + void testSnapshotDeepClean() throws Exception { + Table snapshotInfoTable = + om.getMetadataManager().getSnapshotInfoTable(); + Table deletedTable = + om.getMetadataManager().getDeletedTable(); + Table keyTable = + om.getMetadataManager().getKeyTable(BucketLayout.DEFAULT); + + // Suspend KeyDeletingService + keyDeletingService.suspend(); + + final long initialSnapshotCount = metadataManager.countRowsInTable(snapshotInfoTable); + final long initialKeyCount = metadataManager.countRowsInTable(keyTable); + final long initialDeletedCount = metadataManager.countRowsInTable(deletedTable); + + final String volumeName = getTestName(); + final String bucketName = uniqueObjectName("bucket"); + + // Create Volume and Buckets + createVolumeAndBucket(volumeName, bucketName, false); + + writeClient.createSnapshot(volumeName, bucketName, uniqueObjectName("snap")); + assertTableRowCount(snapshotInfoTable, initialSnapshotCount + 1, metadataManager); + + List createdKeys = new ArrayList<>(); + for (int i = 1; i <= 10; i++) { + OmKeyArgs args = createAndCommitKey(volumeName, bucketName, + uniqueObjectName("key"), 3); + createdKeys.add(args); + } + assertTableRowCount(keyTable, initialKeyCount + 10, metadataManager); - /* - * Create Snap1 - * Create 10 keys - * Create Snap2 - * Delete 10 keys - * Create 5 keys - * Delete 5 keys -> but stop KeyDeletingService so - that keys won't be reclaimed. - * Create snap3 -t - * Now wait for snap3 to be deepCleaned -> Deleted 5 - keys should be deep cleaned. - * Now delete snap2 -> Wait for snap3 to be deep cleaned so deletedTable - of Snap3 should be empty. - */ - @Test - public void testSnapshotDeepClean() throws Exception { - OzoneConfiguration conf = createConfAndInitValues(); - OmTestManagers omTestManagers - = new OmTestManagers(conf); - KeyManager keyManager = omTestManagers.getKeyManager(); - writeClient = omTestManagers.getWriteClient(); - om = omTestManagers.getOzoneManager(); - OMMetadataManager metadataManager = omTestManagers.getMetadataManager(); - Table snapshotInfoTable = - om.getMetadataManager().getSnapshotInfoTable(); - Table deletedTable = - om.getMetadataManager().getDeletedTable(); - Table keyTable = - om.getMetadataManager().getKeyTable(BucketLayout.DEFAULT); - - KeyDeletingService keyDeletingService = keyManager.getDeletingService(); - // Suspend KeyDeletingService - keyDeletingService.suspend(); - - String volumeName = String.format("volume%s", - RandomStringUtils.randomAlphanumeric(5)); - String bucketName = String.format("bucket%s", - RandomStringUtils.randomAlphanumeric(5)); - String keyName = String.format("key%s", - RandomStringUtils.randomAlphanumeric(5)); - - // Create Volume and Buckets - createVolumeAndBucket(keyManager, volumeName, bucketName, false); - - writeClient.createSnapshot(volumeName, bucketName, "snap1"); - assertTableRowCount(snapshotInfoTable, 1, metadataManager); - - List createdKeys = new ArrayList<>(); - for (int i = 1; i <= 10; i++) { - OmKeyArgs args = createAndCommitKey(keyManager, volumeName, bucketName, - keyName + i, 3); - createdKeys.add(args); - } - assertTableRowCount(keyTable, 10, metadataManager); + String snap2 = uniqueObjectName("snap"); + writeClient.createSnapshot(volumeName, bucketName, snap2); + assertTableRowCount(snapshotInfoTable, initialSnapshotCount + 2, metadataManager); + + // Create 5 Keys + for (int i = 11; i <= 15; i++) { + OmKeyArgs args = createAndCommitKey(volumeName, bucketName, + uniqueObjectName("key"), 3); + createdKeys.add(args); + } - writeClient.createSnapshot(volumeName, bucketName, "snap2"); - assertTableRowCount(snapshotInfoTable, 2, metadataManager); + // Delete all 15 keys. + for (int i = 0; i < 15; i++) { + writeClient.deleteKey(createdKeys.get(i)); + } - // Create 5 Keys - for (int i = 11; i <= 15; i++) { - OmKeyArgs args = createAndCommitKey(keyManager, volumeName, bucketName, - keyName + i, 3); - createdKeys.add(args); - } + assertTableRowCount(deletedTable, initialDeletedCount + 15, metadataManager); - // Delete all 15 keys. - for (int i = 0; i < 15; i++) { - writeClient.deleteKey(createdKeys.get(i)); - } + // Create Snap3, traps all the deleted keys. + String snap3 = uniqueObjectName("snap"); + writeClient.createSnapshot(volumeName, bucketName, snap3); + assertTableRowCount(snapshotInfoTable, initialSnapshotCount + 3, metadataManager); + checkSnapDeepCleanStatus(snapshotInfoTable, volumeName, false); - assertTableRowCount(deletedTable, 15, metadataManager); + keyDeletingService.resume(); - // Create Snap3, traps all the deleted keys. - writeClient.createSnapshot(volumeName, bucketName, "snap3"); - assertTableRowCount(snapshotInfoTable, 3, metadataManager); - checkSnapDeepCleanStatus(snapshotInfoTable, false); + try (ReferenceCounted rcOmSnapshot = + om.getOmSnapshotManager().checkForSnapshot( + volumeName, bucketName, getSnapshotPrefix(snap3), true)) { + OmSnapshot snapshot3 = (OmSnapshot) rcOmSnapshot.get(); - keyDeletingService.resume(); + Table snap3deletedTable = + snapshot3.getMetadataManager().getDeletedTable(); - try (ReferenceCounted rcOmSnapshot = - om.getOmSnapshotManager().checkForSnapshot( - volumeName, bucketName, getSnapshotPrefix("snap3"), true)) { - OmSnapshot snap3 = (OmSnapshot) rcOmSnapshot.get(); + // 5 keys can be deep cleaned as it was stuck previously + assertTableRowCount(snap3deletedTable, initialDeletedCount + 10, metadataManager); - Table snap3deletedTable = - snap3.getMetadataManager().getDeletedTable(); + writeClient.deleteSnapshot(volumeName, bucketName, snap2); + assertTableRowCount(snapshotInfoTable, initialSnapshotCount + 2, metadataManager); - // 5 keys can be deep cleaned as it was stuck previously - assertTableRowCount(snap3deletedTable, 10, metadataManager); + assertTableRowCount(snap3deletedTable, initialDeletedCount, metadataManager); + assertTableRowCount(deletedTable, initialDeletedCount, metadataManager); + checkSnapDeepCleanStatus(snapshotInfoTable, volumeName, true); + } + } - writeClient.deleteSnapshot(volumeName, bucketName, "snap2"); - assertTableRowCount(snapshotInfoTable, 2, metadataManager); + @Test + void testSnapshotExclusiveSize() throws Exception { + Table snapshotInfoTable = + om.getMetadataManager().getSnapshotInfoTable(); + Table deletedTable = + om.getMetadataManager().getDeletedTable(); + Table renamedTable = + om.getMetadataManager().getSnapshotRenamedTable(); + Table keyTable = + om.getMetadataManager().getKeyTable(BucketLayout.DEFAULT); + + // Supspend KDS + keyDeletingService.suspend(); + + final long initialSnapshotCount = metadataManager.countRowsInTable(snapshotInfoTable); + final long initialKeyCount = metadataManager.countRowsInTable(keyTable); + final long initialDeletedCount = metadataManager.countRowsInTable(deletedTable); + final long initialRenamedCount = metadataManager.countRowsInTable(renamedTable); + + final String volumeName = getTestName(); + final String bucketName = uniqueObjectName("bucket"); + final String keyName = uniqueObjectName("key"); + + // Create Volume and Buckets + createVolumeAndBucket(volumeName, bucketName, false); + + // Create 3 keys + for (int i = 1; i <= 3; i++) { + createAndCommitKey(volumeName, bucketName, keyName + i, 3); + } + assertTableRowCount(keyTable, initialKeyCount + 3, metadataManager); + + // Create Snapshot1 + String snap1 = uniqueObjectName("snap"); + writeClient.createSnapshot(volumeName, bucketName, snap1); + assertTableRowCount(snapshotInfoTable, initialSnapshotCount + 1, metadataManager); + assertTableRowCount(deletedTable, initialDeletedCount, metadataManager); + + // Create 2 keys + for (int i = 4; i <= 5; i++) { + createAndCommitKey(volumeName, bucketName, keyName + i, 3); + } + // Delete a key, rename 2 keys. We will be using this to test + // how we handle renamed key for exclusive size calculation. + renameKey(volumeName, bucketName, keyName + 1, "renamedKey1"); + renameKey(volumeName, bucketName, keyName + 2, "renamedKey2"); + deleteKey(volumeName, bucketName, keyName + 3); + assertTableRowCount(deletedTable, initialDeletedCount + 1, metadataManager); + assertTableRowCount(renamedTable, initialRenamedCount + 2, metadataManager); + + // Create Snapshot2 + String snap2 = uniqueObjectName("snap"); + writeClient.createSnapshot(volumeName, bucketName, snap2); + assertTableRowCount(snapshotInfoTable, initialSnapshotCount + 2, metadataManager); + assertTableRowCount(deletedTable, initialDeletedCount, metadataManager); + + // Create 2 keys + for (int i = 6; i <= 7; i++) { + createAndCommitKey(volumeName, bucketName, keyName + i, 3); + } + + deleteKey(volumeName, bucketName, "renamedKey1"); + deleteKey(volumeName, bucketName, keyName + 4); + // Do a second rename of already renamedKey2 + renameKey(volumeName, bucketName, "renamedKey2", "renamedKey22"); + assertTableRowCount(deletedTable, initialDeletedCount + 2, metadataManager); + assertTableRowCount(renamedTable, initialRenamedCount + 1, metadataManager); + + // Create Snapshot3 + String snap3 = uniqueObjectName("snap"); + writeClient.createSnapshot(volumeName, bucketName, snap3); + // Delete 4 keys + deleteKey(volumeName, bucketName, "renamedKey22"); + for (int i = 5; i <= 7; i++) { + deleteKey(volumeName, bucketName, keyName + i); + } - assertTableRowCount(snap3deletedTable, 0, metadataManager); - assertTableRowCount(deletedTable, 0, metadataManager); - checkSnapDeepCleanStatus(snapshotInfoTable, true); + // Create Snapshot4 + String snap4 = uniqueObjectName("snap"); + writeClient.createSnapshot(volumeName, bucketName, snap4); + createAndCommitKey(volumeName, bucketName, uniqueObjectName("key"), 3); + + long prevKdsRunCount = getRunCount(); + keyDeletingService.resume(); + + Map expectedSize = new ImmutableMap.Builder() + .put(snap1, 1000L) + .put(snap2, 1000L) + .put(snap3, 2000L) + .put(snap4, 0L) + .build(); + + // Let KeyDeletingService to run for some iterations + GenericTestUtils.waitFor( + () -> (getRunCount() > prevKdsRunCount + 5), + 100, 10000); + + // Check if the exclusive size is set. + try (TableIterator> + iterator = snapshotInfoTable.iterator()) { + while (iterator.hasNext()) { + Table.KeyValue snapshotEntry = iterator.next(); + String snapshotName = snapshotEntry.getValue().getName(); + Long expected = expectedSize.getOrDefault(snapshotName, 0L); + assertNotNull(expected); + assertEquals(expected, snapshotEntry.getValue().getExclusiveSize()); + // Since for the test we are using RATIS/THREE + assertEquals(expected * 3, snapshotEntry.getValue().getExclusiveReplicatedSize()); + } + } } } - @Test - public void testSnapshotExclusiveSize() throws Exception { - OzoneConfiguration conf = createConfAndInitValues(); - OmTestManagers omTestManagers - = new OmTestManagers(conf); - KeyManager keyManager = omTestManagers.getKeyManager(); - writeClient = omTestManagers.getWriteClient(); - om = omTestManagers.getOzoneManager(); - OMMetadataManager metadataManager = omTestManagers.getMetadataManager(); - Table snapshotInfoTable = - om.getMetadataManager().getSnapshotInfoTable(); - Table deletedTable = - om.getMetadataManager().getDeletedTable(); - Table renamedTable = - om.getMetadataManager().getSnapshotRenamedTable(); - Table keyTable = - om.getMetadataManager().getKeyTable(BucketLayout.DEFAULT); - - KeyDeletingService keyDeletingService = keyManager.getDeletingService(); - // Supspend KDS - keyDeletingService.suspend(); - - String volumeName = "volume1"; - String bucketName = "bucket1"; - String keyName = "key"; - - // Create Volume and Buckets - createVolumeAndBucket(keyManager, volumeName, bucketName, false); - - // Create 3 keys - for (int i = 1; i <= 3; i++) { - createAndCommitKey(keyManager, volumeName, bucketName, keyName + i, 3); + /** + * Tests failure scenarios. + */ + @Nested + @TestInstance(TestInstance.Lifecycle.PER_CLASS) + class Failing { + + @BeforeAll + void setup(@TempDir File testDir) throws Exception { + // failCallsFrequency = 1 means all calls fail + scmBlockTestingClient = new ScmBlockLocationTestingClient(null, null, 1); + createConfig(testDir); + createSubject(); } - assertTableRowCount(keyTable, 3, metadataManager); - // Create Snapshot1 - writeClient.createSnapshot(volumeName, bucketName, "snap1"); - assertTableRowCount(snapshotInfoTable, 1, metadataManager); - assertTableRowCount(deletedTable, 0, metadataManager); + @AfterEach + void resume() { + keyDeletingService.resume(); + } - // Create 2 keys - for (int i = 4; i <= 5; i++) { - createAndCommitKey(keyManager, volumeName, bucketName, keyName + i, 3); + @AfterAll + void cleanup() { + if (om.stop()) { + om.join(); + } } - // Delete a key, rename 2 keys. We will be using this to test - // how we handle renamed key for exclusive size calculation. - renameKey(volumeName, bucketName, keyName + 1, "renamedKey1"); - renameKey(volumeName, bucketName, keyName + 2, "renamedKey2"); - deleteKey(volumeName, bucketName, keyName + 3); - assertTableRowCount(deletedTable, 1, metadataManager); - assertTableRowCount(renamedTable, 2, metadataManager); - - // Create Snapshot2 - writeClient.createSnapshot(volumeName, bucketName, "snap2"); - assertTableRowCount(snapshotInfoTable, 2, metadataManager); - assertTableRowCount(deletedTable, 0, metadataManager); - - // Create 2 keys - for (int i = 6; i <= 7; i++) { - createAndCommitKey(keyManager, volumeName, bucketName, keyName + i, 3); + + @Test + void checkIfDeleteServiceWithFailingSCM() throws Exception { + final int initialCount = countKeysPendingDeletion(); + final long initialRunCount = getRunCount(); + final int keyCount = 100; + + createAndDeleteKeys(keyCount, 1); + + GenericTestUtils.waitFor( + () -> countKeysPendingDeletion() == initialCount + keyCount, + 100, 2000); + // Make sure that we have run the background thread 5 times more + GenericTestUtils.waitFor( + () -> getRunCount() >= initialRunCount + 5, + 100, 10000); + // Since SCM calls are failing, deletedKeyCount should be zero. + assertEquals(0, getDeletedKeyCount()); + assertEquals(initialCount + keyCount, countKeysPendingDeletion()); } - deleteKey(volumeName, bucketName, "renamedKey1"); - deleteKey(volumeName, bucketName, "key4"); - // Do a second rename of already renamedKey2 - renameKey(volumeName, bucketName, "renamedKey2", "renamedKey22"); - assertTableRowCount(deletedTable, 2, metadataManager); - assertTableRowCount(renamedTable, 1, metadataManager); - - // Create Snapshot3 - writeClient.createSnapshot(volumeName, bucketName, "snap3"); - // Delete 4 keys - deleteKey(volumeName, bucketName, "renamedKey22"); - for (int i = 5; i <= 7; i++) { - deleteKey(volumeName, bucketName, keyName + i); + @Test + void checkDeletionForEmptyKey() throws Exception { + final int initialCount = countKeysPendingDeletion(); + final long initialRunCount = getRunCount(); + final int keyCount = 100; + + createAndDeleteKeys(keyCount, 0); + + // the pre-allocated blocks are not committed, hence they will be deleted. + GenericTestUtils.waitFor( + () -> countKeysPendingDeletion() == initialCount + keyCount, + 100, 2000); + // Make sure that we have run the background thread 2 times or more + GenericTestUtils.waitFor( + () -> getRunCount() >= initialRunCount + 2, + 100, 1000); + // the blockClient is set to fail the deletion of key blocks, hence no keys + // will be deleted + assertEquals(0, getDeletedKeyCount()); } - // Create Snapshot4 - writeClient.createSnapshot(volumeName, bucketName, "snap4"); - createAndCommitKey(keyManager, volumeName, bucketName, "key8", 3); - keyDeletingService.resume(); + @Test + void checkDeletionForPartiallyCommitKey() throws Exception { + final String volumeName = getTestName(); + final String bucketName = uniqueObjectName("bucket"); + final String keyName = uniqueObjectName("key"); + final long initialCount = countBlocksPendingDeletion(); + createVolumeAndBucket(volumeName, bucketName, false); - Map expectedSize = new HashMap() {{ - put("snap1", 1000L); - put("snap2", 1000L); - put("snap3", 2000L); - put("snap4", 0L); - }}; + OmKeyArgs keyArg = createAndCommitKey(volumeName, bucketName, keyName, 3, 1); - long prevKdsRunCount = keyDeletingService.getRunCount().get(); + // Only the uncommitted block should be pending to be deleted. + GenericTestUtils.waitFor( + () -> countBlocksPendingDeletion() == initialCount + 1, + 500, 3000); - // Let KeyDeletingService to run for some iterations - GenericTestUtils.waitFor( - () -> (keyDeletingService.getRunCount().get() > prevKdsRunCount + 5), - 100, 10000); + writeClient.deleteKey(keyArg); - // Check if the exclusive size is set. - try (TableIterator> - iterator = snapshotInfoTable.iterator()) { - while (iterator.hasNext()) { - Table.KeyValue snapshotEntry = iterator.next(); - String snapshotName = snapshotEntry.getValue().getName(); - assertEquals(expectedSize.get(snapshotName), - snapshotEntry.getValue(). - getExclusiveSize()); - // Since for the test we are using RATIS/THREE - assertEquals(expectedSize.get(snapshotName) * 3, - snapshotEntry.getValue().getExclusiveReplicatedSize()); - } + // All blocks should be pending to be deleted. + GenericTestUtils.waitFor( + () -> countBlocksPendingDeletion() == initialCount + 3, + 500, 3000); + + // the blockClient is set to fail the deletion of key blocks, hence no keys + // will be deleted + assertEquals(0, getDeletedKeyCount()); } } - private void checkSnapDeepCleanStatus(Table - snapshotInfoTable, boolean deepClean) throws IOException { + private void createAndDeleteKeys(int keyCount, int numBlocks) throws IOException { + for (int x = 0; x < keyCount; x++) { + final String volumeName = getTestName(); + final String bucketName = uniqueObjectName("bucket"); + final String keyName = uniqueObjectName("key"); - try (TableIterator> - iterator = snapshotInfoTable.iterator()) { + // Create Volume and Bucket + createVolumeAndBucket(volumeName, bucketName, false); + + // Create the key + OmKeyArgs keyArg = createAndCommitKey(volumeName, bucketName, + keyName, numBlocks); + + // Delete the key + writeClient.deleteKey(keyArg); + } + } + + private static void checkSnapDeepCleanStatus(Table table, String volumeName, boolean deepClean) + throws IOException { + try (TableIterator> iterator = table.iterator()) { while (iterator.hasNext()) { SnapshotInfo snapInfo = iterator.next().getValue(); - assertEquals(snapInfo.getDeepClean(), deepClean); + if (volumeName.equals(snapInfo.getVolumeName())) { + assertThat(snapInfo.getDeepClean()) + .as(snapInfo.toAuditMap().toString()) + .isEqualTo(deepClean); + } } } } - private void assertTableRowCount(Table table, - int count, OMMetadataManager metadataManager) + private static void assertTableRowCount(Table table, + long count, OMMetadataManager metadataManager) throws TimeoutException, InterruptedException { GenericTestUtils.waitFor(() -> assertTableRowCount(count, table, metadataManager), 1000, 120000); // 2 minutes } - private boolean assertTableRowCount(int expectedCount, + private static boolean assertTableRowCount(long expectedCount, Table table, OMMetadataManager metadataManager) { long count = 0L; @@ -713,7 +641,7 @@ private boolean assertTableRowCount(int expectedCount, return count == expectedCount; } - private void createVolumeAndBucket(KeyManager keyManager, String volumeName, + private void createVolumeAndBucket(String volumeName, String bucketName, boolean isVersioningEnabled) throws IOException { // cheat here, just create a volume and bucket entry so that we can // create the keys, we put the same data for key and value since the @@ -763,13 +691,13 @@ private void renameKey(String volumeName, writeClient.renameKey(keyArg, toKeyName); } - private OmKeyArgs createAndCommitKey(KeyManager keyManager, String volumeName, + private OmKeyArgs createAndCommitKey(String volumeName, String bucketName, String keyName, int numBlocks) throws IOException { - return createAndCommitKey(keyManager, volumeName, bucketName, keyName, + return createAndCommitKey(volumeName, bucketName, keyName, numBlocks, 0); } - private OmKeyArgs createAndCommitKey(KeyManager keyManager, String volumeName, + private OmKeyArgs createAndCommitKey(String volumeName, String bucketName, String keyName, int numBlocks, int numUncommitted) throws IOException { // Even if no key size is appointed, there will be at least one @@ -818,4 +746,44 @@ private OmKeyArgs createAndCommitKey(KeyManager keyManager, String volumeName, writeClient.commitKey(keyArg, session.getId()); return keyArg; } + + private long getDeletedKeyCount() { + final long count = keyDeletingService.getDeletedKeyCount().get(); + LOG.debug("KeyDeletingService deleted keys: {}", count); + return count; + } + + private long getRunCount() { + final long count = keyDeletingService.getRunCount().get(); + LOG.debug("KeyDeletingService run count: {}", count); + return count; + } + + private int countKeysPendingDeletion() { + try { + final int count = keyManager.getPendingDeletionKeys(Integer.MAX_VALUE) + .getKeyBlocksList().size(); + LOG.debug("KeyManager keys pending deletion: {}", count); + return count; + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private long countBlocksPendingDeletion() { + try { + return keyManager.getPendingDeletionKeys(Integer.MAX_VALUE) + .getKeyBlocksList() + .stream() + .map(BlockGroup::getBlockIDList) + .mapToLong(Collection::size) + .sum(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private static String uniqueObjectName(String prefix) { + return prefix + OBJECT_COUNTER.getAndIncrement(); + } }