Skip to content

HDDS-7328. Improve Deletion of FSO Paths #3844

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
JacksonYao287 merged 4 commits into from
Oct 27, 2022
Merged

HDDS-7328. Improve Deletion of FSO Paths #3844

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
a9c80cc
HDDS-7328. Improve Deletion of FSO Paths
Xushaohong Sep 29, 2022
c237baf
debug log the result
Xushaohong Oct 25, 2022
143f2db
omit non deletion result
Xushaohong Oct 26, 2022
db42806
escape using Optional as parameter
Xushaohong Oct 26, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
107 changes: 61 additions & 46 deletions ...on-test/src/test/java/org/apache/hadoop/fs/ozone/TestDirectoryDeletingServiceWithFSO.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,6 @@
import org.apache.ozone.test.GenericTestUtils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.AfterAll;
import org.junit.Assert;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.Timeout;
Expand All @@ -54,10 +53,12 @@
import java.util.concurrent.TimeoutException;
import java.util.function.LongSupplier;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_ENABLED;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_ITERATE_BATCH_SIZE;
import static org.junit.Assert.fail;

/**
* Directory deletion service test cases.
Expand All @@ -78,7 +79,7 @@ public class TestDirectoryDeletingServiceWithFSO {
@BeforeAll
public static void init() throws Exception {
OzoneConfiguration conf = new OzoneConfiguration();
conf.setInt(OMConfigKeys.OZONE_DIR_DELETING_SERVICE_INTERVAL, 1);
conf.setInt(OMConfigKeys.OZONE_DIR_DELETING_SERVICE_INTERVAL, 2000);
conf.setInt(OMConfigKeys.OZONE_PATH_DELETING_LIMIT_PER_TASK, 5);
conf.setTimeDuration(OZONE_BLOCK_DELETING_SERVICE_INTERVAL, 100,
TimeUnit.MILLISECONDS);
Expand Down Expand Up @@ -147,6 +148,7 @@ public void testDeleteEmptyDirectory() throws Exception {
assertTableRowCount(dirTable, 2);

assertSubPathsCount(dirDeletingService::getDeletedDirsCount, 0);
assertSubPathsCount(dirDeletingService::getMovedDirsCount, 0);
assertSubPathsCount(dirDeletingService::getMovedFilesCount, 0);

// Delete the appRoot, empty dir
Expand All @@ -159,13 +161,14 @@ public void testDeleteEmptyDirectory() throws Exception {
assertTableRowCount(dirTable, 1);

assertSubPathsCount(dirDeletingService::getDeletedDirsCount, 1);
assertSubPathsCount(dirDeletingService::getMovedDirsCount, 0);
assertSubPathsCount(dirDeletingService::getMovedFilesCount, 0);

Assert.assertTrue(dirTable.iterator().hasNext());
Assert.assertEquals(root.getName(),
assertTrue(dirTable.iterator().hasNext());
assertEquals(root.getName(),
dirTable.iterator().next().getValue().getName());

Assert.assertTrue(dirDeletingService.getRunCount() > 1);
assertTrue(dirDeletingService.getRunCount() > 1);
}

/**
Expand Down Expand Up @@ -211,8 +214,11 @@ public void testDeleteWithLargeSubPathsThanBatchSize() throws Exception {
assertTableRowCount(dirTable, 20);

assertSubPathsCount(dirDeletingService::getMovedFilesCount, 0);
assertSubPathsCount(dirDeletingService::getMovedDirsCount, 0);
assertSubPathsCount(dirDeletingService::getDeletedDirsCount, 0);

long preRunCount = dirDeletingService.getRunCount();

// Delete the appRoot
fs.delete(appRoot, true);

Expand All @@ -224,9 +230,14 @@ public void testDeleteWithLargeSubPathsThanBatchSize() throws Exception {
assertTableRowCount(dirTable, 1);

assertSubPathsCount(dirDeletingService::getMovedFilesCount, 15);
// 15 subDir + 3 parentDir
assertSubPathsCount(dirDeletingService::getMovedDirsCount, 18);
assertSubPathsCount(dirDeletingService::getDeletedDirsCount, 19);

Assert.assertTrue(dirDeletingService.getRunCount() > 1);
long elapsedRunCount = dirDeletingService.getRunCount() - preRunCount;
assertTrue(dirDeletingService.getRunCount() > 1);
// Ensure dir deleting speed, here provide a backup value for safe CI
assertTrue(elapsedRunCount == 8 || elapsedRunCount == 9);
}

@Test
Expand Down Expand Up @@ -258,6 +269,7 @@ public void testDeleteWithMultiLevels() throws Exception {
assertTableRowCount(keyTable, 3);

assertSubPathsCount(dirDeletingService::getMovedFilesCount, 0);
assertSubPathsCount(dirDeletingService::getMovedDirsCount, 0);
assertSubPathsCount(dirDeletingService::getDeletedDirsCount, 0);

// Delete the rootDir, which should delete all keys.
Expand All @@ -271,45 +283,10 @@ public void testDeleteWithMultiLevels() throws Exception {
assertTableRowCount(dirTable, 0);

assertSubPathsCount(dirDeletingService::getMovedFilesCount, 3);
assertSubPathsCount(dirDeletingService::getMovedDirsCount, 4);
assertSubPathsCount(dirDeletingService::getDeletedDirsCount, 5);

Assert.assertTrue(dirDeletingService.getRunCount() > 1);
}

static void assertSubPathsCount(LongSupplier pathCount, long expectedCount)
throws TimeoutException, InterruptedException {
GenericTestUtils.waitFor(() -> pathCount.getAsLong() >= expectedCount,
1000, 120000);
}

private void assertTableRowCount(Table<String, ?> table, int count)
throws TimeoutException, InterruptedException {
GenericTestUtils.waitFor(() -> assertTableRowCount(count, table), 1000,
120000); // 2 minutes
}

private boolean assertTableRowCount(int expectedCount,
Table<String, ?> table) {
long count = 0L;
try {
count = cluster.getOzoneManager().getMetadataManager()
.countRowsInTable(table);
LOG.info("{} actual row count={}, expectedCount={}", table.getName(),
count, expectedCount);
} catch (IOException ex) {
fail("testDoubleBuffer failed with: " + ex);
}
return count == expectedCount;
}

private void checkPath(Path path) {
try {
fs.getFileStatus(path);
fail("testRecursiveDelete failed");
} catch (IOException ex) {
Assert.assertTrue(ex instanceof FileNotFoundException);
Assert.assertTrue(ex.getMessage().contains("No such file or directory"));
}
assertTrue(dirDeletingService.getRunCount() > 1);
}

@Test
Expand Down Expand Up @@ -367,10 +344,11 @@ public void testDeleteFilesAndSubFiles() throws Exception {
assertTableRowCount(deletedKeyTable, 0);

assertSubPathsCount(dirDeletingService::getMovedFilesCount, 0);
assertSubPathsCount(dirDeletingService::getMovedDirsCount, 0);
assertSubPathsCount(dirDeletingService::getDeletedDirsCount, 0);
// verify whether KeyDeletingService has purged the keys
long currentDeletedKeyCount = keyDeletingService.getDeletedKeyCount().get();
Assert.assertEquals(prevDeletedKeyCount + 3, currentDeletedKeyCount);
assertEquals(prevDeletedKeyCount + 3, currentDeletedKeyCount);


// Case-2) Delete dir, this will cleanup sub-files under the deleted dir.
Expand All @@ -385,10 +363,47 @@ public void testDeleteFilesAndSubFiles() throws Exception {
assertTableRowCount(deletedKeyTable, 0);

assertSubPathsCount(dirDeletingService::getMovedFilesCount, 2);
assertSubPathsCount(dirDeletingService::getMovedDirsCount, 0);
assertSubPathsCount(dirDeletingService::getDeletedDirsCount, 1);
// verify whether KeyDeletingService has purged the keys
currentDeletedKeyCount = keyDeletingService.getDeletedKeyCount().get();
Assert.assertEquals(prevDeletedKeyCount + 5, currentDeletedKeyCount);
assertEquals(prevDeletedKeyCount + 5, currentDeletedKeyCount);
}

static void assertSubPathsCount(LongSupplier pathCount, long expectedCount)
throws TimeoutException, InterruptedException {
GenericTestUtils.waitFor(() -> pathCount.getAsLong() >= expectedCount,
1000, 120000);
}

private void assertTableRowCount(Table<String, ?> table, int count)
throws TimeoutException, InterruptedException {
GenericTestUtils.waitFor(() -> assertTableRowCount(count, table), 1000,
120000); // 2 minutes
}

private boolean assertTableRowCount(int expectedCount,
Table<String, ?> table) {
long count = 0L;
try {
count = cluster.getOzoneManager().getMetadataManager()
.countRowsInTable(table);
LOG.info("{} actual row count={}, expectedCount={}", table.getName(),
count, expectedCount);
} catch (IOException ex) {
fail("testDoubleBuffer failed with: " + ex);
}
return count == expectedCount;
}

private void checkPath(Path path) {
try {
fs.getFileStatus(path);
fail("testRecursiveDelete failed");
} catch (IOException ex) {
assertTrue(ex instanceof FileNotFoundException);
assertTrue(ex.getMessage().contains("No such file or directory"));
}
}

private static BucketLayout getFSOBucketLayout() {
Expand Down
Loading