Skip to content

HDDS-11808. Key deletion service should support multiple threads #7486

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
aryangupta1998 wants to merge 9 commits into from
Closed

HDDS-11808. Key deletion service should support multiple threads #7486

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
5b19ae0
HDDS-11808. Key deletion service should support multiple threads
Nov 26, 2024
e01fbbb
Made minor changes.
Dec 3, 2024
bb6248e
Closed Iterator.
Dec 3, 2024
3a7c3b0
Fixed TestSnapshotDirectoryCleaningService.
Dec 3, 2024
de85f77
Made minor changes II.
Dec 4, 2024
63da515
Fixed snapshot test cases.
Jan 6, 2025
15f7796
Fixed TestSnapshotDirectoryCleaningService.
Jan 13, 2025
cb54d63
Fixed TestKeyDeletingService.
Jan 23, 2025
0d5d0fd
Merge remote-tracking branch 'origin/master' into HDDS-11808
adoroszlai Feb 18, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -414,6 +414,11 @@ private OMConfigKeys() {

public static final int OZONE_THREAD_NUMBER_DIR_DELETION_DEFAULT = 10;

public static final String OZONE_THREAD_NUMBER_KEY_DELETION =
"ozone.thread.number.key.deletion";

public static final int OZONE_THREAD_NUMBER_KEY_DELETION_DEFAULT = 10;

public static final String SNAPSHOT_SST_DELETING_LIMIT_PER_TASK =
"ozone.snapshot.filtering.limit.per.task";
public static final int SNAPSHOT_SST_DELETING_LIMIT_PER_TASK_DEFAULT = 2;
Expand Down
1 change: 1 addition & 0 deletions .../integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,7 @@ private void addPropertiesNotInXml() {
OMConfigKeys.OZONE_OM_RANGER_HTTPS_ADMIN_API_USER,
OMConfigKeys.OZONE_OM_RANGER_HTTPS_ADMIN_API_PASSWD,
OMConfigKeys.OZONE_THREAD_NUMBER_DIR_DELETION,
OMConfigKeys.OZONE_THREAD_NUMBER_KEY_DELETION,
ScmConfigKeys.OZONE_SCM_PIPELINE_PLACEMENT_IMPL_KEY,
ScmConfigKeys.OZONE_SCM_HA_PREFIX,
S3GatewayConfigKeys.OZONE_S3G_FSO_DIRECTORY_CREATION_ENABLED,
Expand Down
2 changes: 1 addition & 1 deletion hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyPurging.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,7 @@ public void testKeysPurgingByKeyDeletingService() throws Exception {
GenericTestUtils.waitFor(
() -> {
try {
return keyManager.getPendingDeletionKeys(Integer.MAX_VALUE)
return keyManager.getPendingDeletionKeys(Integer.MAX_VALUE, keyDeletingService.getDeletedKeySupplier())
Copy link
Contributor

@ashishkumar50 ashishkumar50 Jan 3, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Tests will fail because now same iterator is used by background service and test code. Need to make sure only one is running at a time to get correct result.

.getKeyBlocksList().size() == 0;
} catch (IOException e) {
return false;
Expand Down
5 changes: 3 additions & 2 deletions .../java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDeletingServiceIntegrationTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_DEEP_CLEANING_ENABLED;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyString;
Expand Down Expand Up @@ -504,11 +505,11 @@ private KeyDeletingService getMockedKeyDeletingService(AtomicBoolean keyDeletion
when(ozoneManager.getKeyManager()).thenReturn(keyManager);
KeyDeletingService keyDeletingService = Mockito.spy(new KeyDeletingService(ozoneManager,
ozoneManager.getScmClient().getBlockClient(), keyManager, 10000,
100000, cluster.getConf(), false));
100000, cluster.getConf(), false, 1));
keyDeletingService.shutdown();
GenericTestUtils.waitFor(() -> keyDeletingService.getThreadCount() == 0, 1000,
100000);
when(keyManager.getPendingDeletionKeys(anyInt())).thenAnswer(i -> {
when(keyManager.getPendingDeletionKeys(anyInt(), any())).thenAnswer(i -> {
// wait for SDS to reach the KDS wait block before processing any key.
GenericTestUtils.waitFor(keyDeletionWaitStarted::get, 1000, 100000);
keyDeletionStarted.set(true);
Expand Down
2 changes: 2 additions & 0 deletions ...c/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDirectoryCleaningService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_ITERATE_BATCH_SIZE;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_DEEP_CLEANING_ENABLED;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_THREAD_NUMBER_KEY_DELETION;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;

Expand Down Expand Up @@ -83,6 +84,7 @@ public static void init() throws Exception {
conf.setTimeDuration(OZONE_BLOCK_DELETING_SERVICE_INTERVAL, 2500,
TimeUnit.MILLISECONDS);
conf.setBoolean(OZONE_ACL_ENABLED, true);
conf.setInt(OZONE_THREAD_NUMBER_KEY_DELETION, 1);
cluster = MiniOzoneCluster.newBuilder(conf)
.setNumDatanodes(3)
.build();
Expand Down
5 changes: 4 additions & 1 deletion hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,11 +116,14 @@ ListKeysResult listKeys(String volumeName, String bucketName, String startKey,
* Second is a Mapping of Key-Value pair which is updated in the deletedTable.
*
* @param count max number of keys to return.
* @param deletedKeySupplier DeletedKeySupplier object.
* @return a Pair of list of {@link BlockGroup} representing keys and blocks,
* and a hashmap for key-value pair to be updated in the deletedTable.
* @throws IOException
*/
PendingKeysDeletion getPendingDeletionKeys(int count) throws IOException;
PendingKeysDeletion getPendingDeletionKeys(int count,
KeyDeletingService.DeletedKeySupplier deletedKeySupplier)
throws IOException;

/**
* Returns a list rename entries from the snapshotRenamedTable.
Expand Down
19 changes: 13 additions & 6 deletions hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,8 @@
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SNAPSHOT_SST_FILTERING_SERVICE_INTERVAL_DEFAULT;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_THREAD_NUMBER_DIR_DELETION;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_THREAD_NUMBER_DIR_DELETION_DEFAULT;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_THREAD_NUMBER_KEY_DELETION;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_THREAD_NUMBER_KEY_DELETION_DEFAULT;
import static org.apache.hadoop.ozone.om.OzoneManagerUtils.getBucketLayout;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.BUCKET_NOT_FOUND;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.FILE_NOT_FOUND;
Expand Down Expand Up @@ -238,9 +240,13 @@ public void start(OzoneConfiguration configuration) {
OZONE_BLOCK_DELETING_SERVICE_TIMEOUT,
OZONE_BLOCK_DELETING_SERVICE_TIMEOUT_DEFAULT,
TimeUnit.MILLISECONDS);
keyDeletingService = new KeyDeletingService(ozoneManager,
scmClient.getBlockClient(), this, blockDeleteInterval,
serviceTimeout, configuration, isSnapshotDeepCleaningEnabled);
int keyDeletingCorePoolSize =
configuration.getInt(OZONE_THREAD_NUMBER_KEY_DELETION,
OZONE_THREAD_NUMBER_KEY_DELETION_DEFAULT);
keyDeletingService =
new KeyDeletingService(ozoneManager, scmClient.getBlockClient(), this,
blockDeleteInterval, serviceTimeout, configuration,
isSnapshotDeepCleaningEnabled, keyDeletingCorePoolSize);
keyDeletingService.start();
}

Expand Down Expand Up @@ -679,12 +685,13 @@ public ListKeysResult listKeys(String volumeName, String bucketName,
}

@Override
public PendingKeysDeletion getPendingDeletionKeys(final int count)
public PendingKeysDeletion getPendingDeletionKeys(final int count,
KeyDeletingService.DeletedKeySupplier deletedKeySupplier)
throws IOException {
OmMetadataManagerImpl omMetadataManager =
(OmMetadataManagerImpl) metadataManager;
return omMetadataManager
.getPendingDeletionKeys(count, ozoneManager.getOmSnapshotManager());
return omMetadataManager.getPendingDeletionKeys(count,
ozoneManager.getOmSnapshotManager(), deletedKeySupplier);
}

private <V, R> List<Table.KeyValue<String, R>> getTableEntries(String startKey,
Expand Down
Loading