Skip to content

HDDS-11215. Quota count can go wrong when double buffer flush takes time #6978

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
sadanand48 merged 1 commit into from
Jul 24, 2024
Merged

HDDS-11215. Quota count can go wrong when double buffer flush takes time #6978

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
80 changes: 80 additions & 0 deletions ...on-test/src/test/java/org/apache/hadoop/fs/ozone/TestDirectoryDeletingServiceWithFSO.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

package org.apache.hadoop.fs.ozone;

import java.util.concurrent.CompletableFuture;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
Expand All @@ -33,8 +34,13 @@
import org.apache.hadoop.ozone.TestDataUtil;
import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.ozone.om.OMMetadataManager;
import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
import org.apache.hadoop.ozone.om.ratis.OzoneManagerDoubleBuffer;
import org.apache.hadoop.ozone.om.ratis.OzoneManagerStateMachine;
import org.apache.hadoop.ozone.om.request.file.OMFileRequest;
import org.apache.hadoop.ozone.om.service.DirectoryDeletingService;
import org.apache.hadoop.ozone.om.service.KeyDeletingService;
import org.apache.hadoop.ozone.om.OMConfigKeys;
Expand Down Expand Up @@ -299,6 +305,80 @@ public void testDeleteWithMultiLevels() throws Exception {
assertThat(dirDeletingService.getRunCount().get()).isGreaterThan(1);
}

@Test
public void testDeleteWithMultiLevelsBlockDoubleBuffer() throws Exception {
Path root = new Path("/rootDirdd");
Path appRoot = new Path(root, "appRoot");

for (int i = 1; i <= 3; i++) {
Path parent = new Path(appRoot, "parentDir" + i);
Path child = new Path(parent, "childFile");
ContractTestUtils.touch(fs, child);
}

OMMetadataManager metadataManager = cluster.getOzoneManager().getMetadataManager();
Table<String, OmKeyInfo> deletedDirTable = metadataManager.getDeletedDirTable();
Table<String, OmKeyInfo> keyTable = metadataManager.getKeyTable(getFSOBucketLayout());
Table<String, OmDirectoryInfo> dirTable = metadataManager.getDirectoryTable();

DirectoryDeletingService dirDeletingService = (DirectoryDeletingService) cluster.getOzoneManager().getKeyManager()
.getDirDeletingService();

// Before delete
assertTableRowCount(deletedDirTable, 0);
assertTableRowCount(dirTable, 5);
assertTableRowCount(keyTable, 3);

// stop daemon to simulate blocked double buffer
OzoneManagerStateMachine omStateMachine = cluster.getOzoneManager().getOmRatisServer().getOmStateMachine();
OzoneManagerDoubleBuffer omDoubleBuffer = omStateMachine.getOzoneManagerDoubleBuffer();
omDoubleBuffer.awaitFlush();
omDoubleBuffer.stopDaemon();

OzoneVolume volume = client.getObjectStore().getVolume(volumeName);
OzoneBucket bucket = volume.getBucket(bucketName); long volumeId = metadataManager.getVolumeId(volumeName);

// manually delete dir and add to deleted table. namespace count occupied "1" as manual deletion do not reduce
long bucketId = metadataManager.getBucketId(volumeName, bucketName);
OzoneFileStatus keyStatus = OMFileRequest.getOMKeyInfoIfExists(
metadataManager, volumeName, bucketName, "rootDirdd", 0,
cluster.getOzoneManager().getDefaultReplicationConfig());
String ozoneDbKey = metadataManager.getOzonePathKey(volumeId, bucketId, keyStatus.getKeyInfo().getParentObjectID(),
keyStatus.getKeyInfo().getFileName());
deletedDirTable.put(ozoneDbKey, keyStatus.getKeyInfo());
dirTable.delete(ozoneDbKey);
assertTableRowCount(deletedDirTable, 1);
assertTableRowCount(dirTable, 4);

System.out.println("starting count: " + bucket.getUsedNamespace());

// wait for running 2 more iteration
long currentCount = dirDeletingService.getRunCount().get();
GenericTestUtils.waitFor(() -> dirDeletingService.getRunCount().get() > currentCount + 2, 1000,
10000);

// verify bucket used namespace is not going -ve
assertTrue(volume.getBucket(bucketName).getUsedNamespace() >= 0);

// re-init double buffer in state machine to resume further processing
omStateMachine.pause();
omStateMachine.unpause(omStateMachine.getLastAppliedTermIndex().getIndex(),
omStateMachine.getLatestSnapshot().getIndex());
// flush previous pending transaction manually
omDoubleBuffer.resume();
CompletableFuture.supplyAsync(() -> {
omDoubleBuffer.flushTransactions();
return null;
});
omDoubleBuffer.awaitFlush();
omDoubleBuffer.stop();
// verify deletion progress completion
assertTableRowCount(deletedDirTable, 0);
assertTableRowCount(keyTable, 0);
assertTrue(volume.getBucket(bucketName).getUsedNamespace() >= 0);
assertTrue(volume.getBucket(bucketName).getUsedBytes() == 0);
}

@Test
public void testDeleteFilesAndSubFiles() throws Exception {

Expand Down
6 changes: 6 additions & 0 deletions hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2031,6 +2031,9 @@ private List<OmKeyInfo> gatherSubDirsWithIterator(OmKeyInfo parentInfo,
parentInfo.getObjectID())) {
break;
}
if (!metadataManager.getDirectoryTable().isExist(entry.getKey())) {
continue;
}
String dirName = OMFileRequest.getAbsolutePath(parentInfo.getKeyName(),
dirInfo.getName());
OmKeyInfo omKeyInfo = OMFileRequest.getOmKeyInfo(
Expand Down Expand Up @@ -2065,6 +2068,9 @@ public List<OmKeyInfo> getPendingDeletionSubFiles(long volumeId,
parentInfo.getObjectID())) {
break;
}
if (!metadataManager.getFileTable().isExist(entry.getKey())) {
continue;
}
fileInfo.setFileName(fileInfo.getKeyName());
String fullKeyPath = OMFileRequest.getAbsolutePath(
parentInfo.getKeyName(), fileInfo.getKeyName());
Expand Down
4 changes: 2 additions & 2 deletions ...zone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerDoubleBuffer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -290,7 +290,7 @@ private void addToBatchTransactionInfoWithTrace(String parentName,
* and commit to DB.
*/
@VisibleForTesting
void flushTransactions() {
public void flushTransactions() {
while (isRunning.get() && canFlush()) {
flushCurrentBuffer();
}
Expand Down Expand Up @@ -617,7 +617,7 @@ int getReadyBufferSize() {
}

@VisibleForTesting
void resume() {
public void resume() {
isRunning.set(true);
}

Expand Down
10 changes: 10 additions & 0 deletions ...rc/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@
import java.util.Map;
import java.util.Set;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
import org.apache.hadoop.hdds.utils.db.cache.CacheValue;
import org.apache.hadoop.ozone.om.OMMetrics;
import org.apache.ratis.server.protocol.TermIndex;
import org.apache.hadoop.ozone.om.OMMetadataManager;
Expand Down Expand Up @@ -95,6 +97,10 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn
if (null != omBucketInfo
&& omBucketInfo.getObjectID() == path.getBucketId()) {
omBucketInfo.incrUsedNamespace(-1L);
String ozoneDbKey = omMetadataManager.getOzonePathKey(path.getVolumeId(),
path.getBucketId(), keyInfo.getParentObjectID(), keyInfo.getFileName());
omMetadataManager.getDirectoryTable().addCacheEntry(new CacheKey<>(ozoneDbKey),
CacheValue.get(termIndex.getIndex()));
volBucketInfoMap.putIfAbsent(volBucketPair, omBucketInfo);
}
}
Expand All @@ -119,6 +125,10 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, TermIn
&& omBucketInfo.getObjectID() == path.getBucketId()) {
omBucketInfo.incrUsedBytes(-sumBlockLengths(keyInfo));
omBucketInfo.incrUsedNamespace(-1L);
String ozoneDbKey = omMetadataManager.getOzonePathKey(path.getVolumeId(),
path.getBucketId(), keyInfo.getParentObjectID(), keyInfo.getFileName());
omMetadataManager.getFileTable().addCacheEntry(new CacheKey<>(ozoneDbKey),
CacheValue.get(termIndex.getIndex()));
volBucketInfoMap.putIfAbsent(volBucketPair, omBucketInfo);
}
}
Expand Down