Skip to content

HDDS-9312. Expand all the entries deletedDirTable for Snapshot. #5339

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
aswinshakil wants to merge 12 commits into from
Closed

HDDS-9312. Expand all the entries deletedDirTable for Snapshot. #5339

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
56b2ab1
HDDS-9312. Expand all the entries deletedDirTable for Snapshot.
aswinshakil Sep 20, 2023
5f4bbf3
HDDS-9312. Fix Findbugs.
aswinshakil Sep 20, 2023
61e9376
Fix TestOmSnapshotInfo unit test
aswinshakil Sep 21, 2023
3f74baa
Address review comments.
aswinshakil Sep 27, 2023
382962d
Merge master
aswinshakil Sep 27, 2023
da3f654
Fix deletedIterator logic.
aswinshakil Sep 28, 2023
d647921
Fix checkstyle.
aswinshakil Sep 28, 2023
cd82700
Merge branch 'master' of https://github.com/apache/ozone into HDDS-93…
aswinshakil Oct 4, 2023
8fc3a25
Fix testSnapshotWithFSO and remove move directories code.
aswinshakil Oct 9, 2023
75f3f5e
Merge branch 'master' of https://github.com/apache/ozone into HDDS-93…
aswinshakil Oct 9, 2023
9baa010
Fix findbugs
aswinshakil Oct 9, 2023
fb4aca2
Modify KDT and DDT
aswinshakil Oct 19, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 36 additions & 7 deletions hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,7 @@ public static SnapshotStatus valueOf(SnapshotStatusProto status) {
private long referencedReplicatedSize;
private long exclusiveSize;
private long exclusiveReplicatedSize;
private boolean expandedDeletedDir;

/**
* Private constructor, constructed via builder.
Expand All @@ -149,6 +150,7 @@ public static SnapshotStatus valueOf(SnapshotStatusProto status) {
* @param referencedReplicatedSize - Snapshot referenced size w/ replication.
* @param exclusiveSize - Snapshot exclusive size.
* @param exclusiveReplicatedSize - Snapshot exclusive size w/ replication.
* @param expandedDeletedDir - Indicates if the deleted dir is expanded.
*/
@SuppressWarnings("checkstyle:ParameterNumber")
private SnapshotInfo(UUID snapshotId,
Expand All @@ -168,7 +170,8 @@ private SnapshotInfo(UUID snapshotId,
long referencedSize,
long referencedReplicatedSize,
long exclusiveSize,
long exclusiveReplicatedSize) {
long exclusiveReplicatedSize,
boolean expandedDeletedDir) {
this.snapshotId = snapshotId;
this.name = name;
this.volumeName = volumeName;
Expand All @@ -187,6 +190,7 @@ private SnapshotInfo(UUID snapshotId,
this.referencedReplicatedSize = referencedReplicatedSize;
this.exclusiveSize = exclusiveSize;
this.exclusiveReplicatedSize = exclusiveReplicatedSize;
this.expandedDeletedDir = expandedDeletedDir;
}

public void setName(String name) {
Expand Down Expand Up @@ -285,6 +289,14 @@ public void setSstFiltered(boolean sstFiltered) {
this.sstFiltered = sstFiltered;
}

public boolean getExpandedDeletedDir() {
return expandedDeletedDir;
}

public void setExpandedDeletedDir(boolean expandedDeletedDir) {
this.expandedDeletedDir = expandedDeletedDir;
}

public static org.apache.hadoop.ozone.om.helpers.SnapshotInfo.Builder
newBuilder() {
return new org.apache.hadoop.ozone.om.helpers.SnapshotInfo.Builder();
Expand All @@ -308,7 +320,8 @@ public SnapshotInfo.Builder toBuilder() {
.setReferencedSize(referencedSize)
.setReferencedReplicatedSize(referencedReplicatedSize)
.setExclusiveSize(exclusiveSize)
.setExclusiveReplicatedSize(exclusiveReplicatedSize);
.setExclusiveReplicatedSize(exclusiveReplicatedSize)
.setExpandedDeletedDir(expandedDeletedDir);
}

/**
Expand All @@ -333,6 +346,7 @@ public static class Builder {
private long referencedReplicatedSize;
private long exclusiveSize;
private long exclusiveReplicatedSize;
private boolean expandedDeletedDir;

public Builder() {
// default values
Expand Down Expand Up @@ -428,6 +442,11 @@ public Builder setExclusiveReplicatedSize(long exclusiveReplicatedSize) {
this.exclusiveReplicatedSize = exclusiveReplicatedSize;
return this;
}
public Builder setExpandedDeletedDir(boolean expandedDeletedDir) {
this.expandedDeletedDir = expandedDeletedDir;
return this;
}


public SnapshotInfo build() {
Preconditions.checkNotNull(name);
Expand All @@ -449,7 +468,8 @@ public SnapshotInfo build() {
referencedSize,
referencedReplicatedSize,
exclusiveSize,
exclusiveReplicatedSize
exclusiveReplicatedSize,
expandedDeletedDir
);
}
}
Expand All @@ -471,7 +491,8 @@ public OzoneManagerProtocolProtos.SnapshotInfo getProtobuf() {
.setReferencedSize(referencedSize)
.setReferencedReplicatedSize(referencedReplicatedSize)
.setExclusiveSize(exclusiveSize)
.setExclusiveReplicatedSize(exclusiveReplicatedSize);
.setExclusiveReplicatedSize(exclusiveReplicatedSize)
.setExpandedDeletedDir(expandedDeletedDir);

if (pathPreviousSnapshotId != null) {
sib.setPathPreviousSnapshotID(toProtobuf(pathPreviousSnapshotId));
Expand Down Expand Up @@ -544,6 +565,11 @@ public static SnapshotInfo getFromProtobuf(
snapshotInfoProto.getExclusiveReplicatedSize());
}

if (snapshotInfoProto.hasExpandedDeletedDir()) {
osib.setExpandedDeletedDir(
snapshotInfoProto.getExpandedDeletedDir());
}

osib.setSnapshotPath(snapshotInfoProto.getSnapshotPath())
.setCheckpointDir(snapshotInfoProto.getCheckpointDir())
.setDbTxSequenceNumber(snapshotInfoProto.getDbTxSequenceNumber());
Expand Down Expand Up @@ -661,7 +687,8 @@ public static SnapshotInfo newInstance(String volumeName,
.setSnapshotPath(volumeName + OM_KEY_PREFIX + bucketName)
.setVolumeName(volumeName)
.setBucketName(bucketName)
.setDeepClean(true);
.setDeepClean(true)
.setExpandedDeletedDir(false);

if (snapshotId != null) {
builder.setCheckpointDir(getCheckpointDirName(snapshotId));
Expand Down Expand Up @@ -694,7 +721,8 @@ public boolean equals(Object o) {
referencedSize == that.referencedSize &&
referencedReplicatedSize == that.referencedReplicatedSize &&
exclusiveSize == that.exclusiveSize &&
exclusiveReplicatedSize == that.exclusiveReplicatedSize;
exclusiveReplicatedSize == that.exclusiveReplicatedSize &&
expandedDeletedDir == that.expandedDeletedDir;
}

@Override
Expand All @@ -705,7 +733,7 @@ public int hashCode() {
globalPreviousSnapshotId, snapshotPath, checkpointDir,
deepClean, sstFiltered,
referencedSize, referencedReplicatedSize,
exclusiveSize, exclusiveReplicatedSize);
exclusiveSize, exclusiveReplicatedSize, expandedDeletedDir);
}

/**
Expand All @@ -732,6 +760,7 @@ public SnapshotInfo copyObject() {
.setReferencedReplicatedSize(referencedReplicatedSize)
.setExclusiveSize(exclusiveSize)
.setExclusiveReplicatedSize(exclusiveReplicatedSize)
.setExpandedDeletedDir(expandedDeletedDir)
.build();
}
}
7 changes: 6 additions & 1 deletion hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmSnapshotInfo.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@ private SnapshotInfo createSnapshotInfo() {
.setReferencedReplicatedSize(6000L)
.setExclusiveSize(1000L)
.setExclusiveReplicatedSize(3000L)
.setExpandedDeletedDir(false)
.build();
}

Expand All @@ -95,6 +96,7 @@ private OzoneManagerProtocolProtos.SnapshotInfo createSnapshotInfoProto() {
.setReferencedReplicatedSize(6000L)
.setExclusiveSize(1000L)
.setExclusiveReplicatedSize(3000L)
.setExpandedDeletedDir(false)
.build();
}

Expand Down Expand Up @@ -138,7 +140,8 @@ public void testSnapshotInfoToProto() {
snapshotInfoEntryActual.getExclusiveSize());
Assert.assertEquals(snapshotInfoEntryExpected.getExclusiveReplicatedSize(),
snapshotInfoEntryActual.getExclusiveReplicatedSize());

Assert.assertEquals(snapshotInfoEntryExpected.getExpandedDeletedDir(),
snapshotInfoEntryActual.getExpandedDeletedDir());
Assert.assertEquals(snapshotInfoEntryExpected, snapshotInfoEntryActual);
}

Expand Down Expand Up @@ -174,6 +177,8 @@ public void testSnapshotInfoProtoToSnapshotInfo() {
snapshotInfoActual.getExclusiveSize());
Assert.assertEquals(snapshotInfoExpected.getExclusiveReplicatedSize(),
snapshotInfoActual.getExclusiveReplicatedSize());
Assert.assertEquals(snapshotInfoExpected.getExpandedDeletedDir(),
Copy link
Contributor

@hemantk-12 hemantk-12 Sep 27, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we just assertEquals on snapshotInfoExpected and snapshotInfoActual?

Copy link
Member Author

@aswinshakil aswinshakil Sep 27, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes we can, but this is just to check each field individually.

Copy link
Contributor

@hemantk-12 hemantk-12 Oct 2, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

assertEquals() will do the same thing as long as equals method is overridden.
https://github.com/junit-team/junit4/blob/main/src/main/java/org/junit/Assert.java#L145

snapshotInfoActual.getExpandedDeletedDir());

Assert.assertEquals(snapshotInfoExpected, snapshotInfoActual);
}
Expand Down
122 changes: 116 additions & 6 deletions ...on-test/src/test/java/org/apache/hadoop/fs/ozone/TestDirectoryDeletingServiceWithFSO.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,9 @@
import org.apache.hadoop.ozone.TestDataUtil;
import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.om.IOmMetadataReader;
import org.apache.hadoop.ozone.om.OMMetadataManager;
import org.apache.hadoop.ozone.om.OmSnapshot;
import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
import org.apache.hadoop.ozone.om.service.DirectoryDeletingService;
import org.apache.hadoop.ozone.om.service.KeyDeletingService;
Expand All @@ -42,6 +44,8 @@
import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted;
import org.apache.hadoop.ozone.om.snapshot.SnapshotCache;
import org.apache.ozone.test.GenericTestUtils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.AfterAll;
Expand All @@ -57,6 +61,8 @@
import java.util.concurrent.TimeoutException;
import java.util.function.LongSupplier;

import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SNAPSHOT_DELETING_SERVICE_INTERVAL;
import static org.apache.hadoop.ozone.om.OmSnapshotManager.getSnapshotPrefix;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
Expand Down Expand Up @@ -84,10 +90,12 @@ public class TestDirectoryDeletingServiceWithFSO {
@BeforeAll
public static void init() throws Exception {
OzoneConfiguration conf = new OzoneConfiguration();
conf.setInt(OMConfigKeys.OZONE_DIR_DELETING_SERVICE_INTERVAL, 2000);
conf.setInt(OMConfigKeys.OZONE_PATH_DELETING_LIMIT_PER_TASK, 5);
conf.setInt(OMConfigKeys.OZONE_DIR_DELETING_SERVICE_INTERVAL, 100);
conf.setInt(OMConfigKeys.OZONE_PATH_DELETING_LIMIT_PER_TASK, 10);
conf.setTimeDuration(OZONE_BLOCK_DELETING_SERVICE_INTERVAL, 100,
TimeUnit.MILLISECONDS);
conf.setTimeDuration(OZONE_SNAPSHOT_DELETING_SERVICE_INTERVAL, 100,
TimeUnit.MILLISECONDS);
conf.setBoolean(OMConfigKeys.OZONE_OM_RATIS_ENABLE_KEY, omRatisEnabled);
conf.setBoolean(OZONE_ACL_ENABLED, true);
cluster = MiniOzoneCluster.newBuilder(conf)
Expand Down Expand Up @@ -323,9 +331,8 @@ public void testDeleteFilesAndSubFiles() throws Exception {
}
}

KeyDeletingService keyDeletingService =
(KeyDeletingService) cluster.getOzoneManager().getKeyManager()
.getDeletingService();
KeyDeletingService keyDeletingService = cluster.getOzoneManager()
.getKeyManager().getDeletingService();

// Before delete
assertTableRowCount(deletedDirTable, 0);
Expand Down Expand Up @@ -493,7 +500,103 @@ public void testDirDeletedTableCleanUpForSnapshot() throws Exception {
cleanupTables();
}

private void cleanupTables() throws IOException {
@Test
public void testExpandDirDeletedTableForAllSnapshot() throws Exception {
Table<String, SnapshotInfo> snapshotInfoTable =
cluster.getOzoneManager().getMetadataManager().getSnapshotInfoTable();
DirectoryDeletingService dirDeletingService =
(DirectoryDeletingService) cluster.getOzoneManager().getKeyManager()
.getDirDeletingService();
dirDeletingService.suspend();
/* For each snapshot
DirTable KeyTable
/v/b/snapDir /v/b/snapDir/testKey0 - testKey5
/v/b/snapDir/root/ /v/b/snapDir/root/parentDir0/file1 - file10
/v/b/snapDir/root/parentDir0/ /v/b/snapDir/root/parentDir1/file1 - file10
/v/b/snapDir/root/parentDir1/ /v/b/snapDir/root/parentDir2/file1 - file10
/v/b/snapDir/root/parentDir2/
*/

for (int i = 1; i <= 5; i++) {
createSnapshotForDirectoryDeletingServiceTest("snap" + i, i, 11 - i);
}

dirDeletingService.resume();
long prevDDSRunCount = dirDeletingService.getRunCount().get();

GenericTestUtils.waitFor(() -> dirDeletingService.getRunCount().get() >
prevDDSRunCount + 25, 100, 10000);

try (TableIterator<String, ? extends Table.KeyValue
<String, SnapshotInfo>> iterator = snapshotInfoTable.iterator()) {
int fileCountMultiplier = 10;
while (iterator.hasNext()) {
SnapshotInfo currSnapInfo = iterator.next().getValue();
assertTrue(currSnapInfo.getExpandedDeletedDir());
try (ReferenceCounted<IOmMetadataReader, SnapshotCache>
rcCurrOmSnapshot = cluster.getOzoneManager()
.getOmSnapshotManager().checkForSnapshot(
currSnapInfo.getVolumeName(),
currSnapInfo.getBucketName(),
getSnapshotPrefix(currSnapInfo.getName()),
true)) {

OmSnapshot currOmSnapshot = (OmSnapshot) rcCurrOmSnapshot.get();
Table<String, OmKeyInfo> snapDeletedDirTable =
currOmSnapshot.getMetadataManager().getDeletedDirTable();
Table<String, RepeatedOmKeyInfo> deletedTable =
currOmSnapshot.getMetadataManager().getDeletedTable();
assertTrue(snapDeletedDirTable.isEmpty());
// Created 5 files. And two subdirectories with fileCountMultiplier.
assertTableRowCount(deletedTable, fileCountMultiplier * 2 + 5);
fileCountMultiplier--;
}
}
}
cleanupTables();
}

private void createSnapshotForDirectoryDeletingServiceTest(
String rootPath, int count, int fileCount) throws Exception {

Table<String, OmKeyInfo> deletedDirTable =
cluster.getOzoneManager().getMetadataManager().getDeletedDirTable();
Table<String, SnapshotInfo> snapshotInfoTable =
cluster.getOzoneManager().getMetadataManager().getSnapshotInfoTable();
Path root = new Path("/" + rootPath);
Path appRoot = new Path(root, "root");
// Create parent dir from root.
fs.mkdirs(root);

// Added 5 sub files inside root dir
for (int i = 0; i < 5; i++) {
Path path = new Path(root, "testKey" + i);
try (FSDataOutputStream stream = fs.create(path)) {
stream.write(1);
}
}

// Add 2*5 more sub files in different level
for (int i = 0; i < 2; i++) {
Path parent = new Path(appRoot, "parentDir" + i);
for (int j = 1; j <= fileCount; j++) {
Path child = new Path(parent, "file" + j);
ContractTestUtils.touch(fs, child);
}
}

// Delete dir
fs.delete(root, true);
// For 1st snapshot deletedDirTable will be expanded, as DeletedDirService
// runs every 100ms for this test
if (count != 1) {
assertTableRowCount(deletedDirTable, 1);
}
client.getObjectStore().createSnapshot(volumeName, bucketName, rootPath);
assertTableRowCount(snapshotInfoTable, count);
}

private void cleanupTables() throws Exception {
OMMetadataManager metadataManager =
cluster.getOzoneManager().getMetadataManager();

Expand All @@ -508,6 +611,13 @@ private void cleanupTables() throws IOException {
.iterator()) {
removeAllFromDB(it);
}

try (TableIterator<?, ?> it = metadataManager.getSnapshotInfoTable()
.iterator()) {
removeAllFromDB(it);
cluster.getOzoneManager().getOmSnapshotManager()
.getSnapshotCache().invalidateAll();
}
}

private static void removeAllFromDB(TableIterator<?, ?> iterator)
Expand Down
Loading