From 8c626c65dbcccb45da7146adde7b5d0b2818fa20 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Sun, 11 May 2025 20:12:28 -0400 Subject: [PATCH 1/2] HDDS-13021. AbstractKeyDeletingService should have unique call id for each and every ratis request Change-Id: I5c29c572df9d2240b1d58fbc88826eb5ed8ad881 --- .../om/service/AbstractKeyDeletingService.java | 14 ++++++++++---- .../ozone/om/service/SnapshotDeletingService.java | 4 ++-- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java index 0d36da711703..7e3df6337e59 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java @@ -81,6 +81,7 @@ public abstract class AbstractKeyDeletingService extends BackgroundService private final AtomicLong movedDirsCount; private final AtomicLong movedFilesCount; private final AtomicLong runCount; + private final AtomicLong callId; private final BootstrapStateHandler.Lock lock = new BootstrapStateHandler.Lock(); @@ -97,6 +98,7 @@ public AbstractKeyDeletingService(String serviceName, long interval, this.runCount = new AtomicLong(0); this.metrics = ozoneManager.getDeletionMetrics(); this.perfMetrics = ozoneManager.getPerfMetrics(); + this.callId = new AtomicLong(0); } protected int processKeyDeletes(List keyBlocksList, @@ -220,7 +222,7 @@ private int submitPurgeKeysRequest(List results, // Submit PurgeKeys request to OM try { - OzoneManagerRatisUtils.submitRequest(ozoneManager, omRequest, clientId, runCount.get()); + OzoneManagerRatisUtils.submitRequest(ozoneManager, omRequest, clientId, callId.incrementAndGet()); } catch (ServiceException e) { LOG.error("PurgeKey request failed. Will retry at next run.", e); return 0; @@ -250,7 +252,7 @@ private void addToMap(Map, List> map, String object protected void submitPurgePaths(List requests, String snapTableKey, - UUID expectedPreviousSnapshotId, long rnCnt) { + UUID expectedPreviousSnapshotId) { OzoneManagerProtocolProtos.PurgeDirectoriesRequest.Builder purgeDirRequest = OzoneManagerProtocolProtos.PurgeDirectoriesRequest.newBuilder(); @@ -275,7 +277,7 @@ protected void submitPurgePaths(List requests, // Submit Purge paths request to OM try { - OzoneManagerRatisUtils.submitRequest(ozoneManager, omRequest, clientId, rnCnt); + OzoneManagerRatisUtils.submitRequest(ozoneManager, omRequest, clientId, callId.incrementAndGet()); } catch (ServiceException e) { LOG.error("PurgePaths request failed. Will retry at next run.", e); } @@ -403,7 +405,7 @@ public void optimizeDirDeletesAndSubmitRequest( } if (!purgePathRequestList.isEmpty()) { - submitPurgePaths(purgePathRequestList, snapTableKey, expectedPreviousSnapshotId, rnCnt); + submitPurgePaths(purgePathRequestList, snapTableKey, expectedPreviousSnapshotId); } if (dirNum != 0 || subDirNum != 0 || subFileNum != 0) { @@ -613,6 +615,10 @@ public AtomicLong getRunCount() { return runCount; } + public AtomicLong getCallId() { + return callId; + } + /** * Returns the number of dirs deleted by the background service. * diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java index 5c2b16a604b3..44bb7ab088a7 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java @@ -304,8 +304,8 @@ private void submitSnapshotMoveDeletedKeys(SnapshotInfo snapInfo, private void submitRequest(OMRequest omRequest) { try { - Status status = - OzoneManagerRatisUtils.submitRequest(ozoneManager, omRequest, clientId, getRunCount().get()).getStatus(); + Status status = OzoneManagerRatisUtils.submitRequest(ozoneManager, omRequest, clientId, + getCallId().incrementAndGet()).getStatus(); if (!Objects.equals(status, Status.OK)) { LOG.error("Request: {} failed with an status: {}. Will retry in the next run.", omRequest, status); } From 6c3ccdf1fb73456ded0236260b05d857ad09fe8e Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Mon, 12 May 2025 13:44:55 -0400 Subject: [PATCH 2/2] HDDS-13021. Create a protected function and submit om request Change-Id: Ie13cabdb966e947b5eb63c75c8cb634d938ab561 --- .../ozone/om/service/AbstractKeyDeletingService.java | 12 ++++++------ .../ozone/om/service/SnapshotDeletingService.java | 10 ++++------ 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java index 7e3df6337e59..dc03b25a99d5 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java @@ -222,7 +222,7 @@ private int submitPurgeKeysRequest(List results, // Submit PurgeKeys request to OM try { - OzoneManagerRatisUtils.submitRequest(ozoneManager, omRequest, clientId, callId.incrementAndGet()); + submitRequest(omRequest); } catch (ServiceException e) { LOG.error("PurgeKey request failed. Will retry at next run.", e); return 0; @@ -231,6 +231,10 @@ private int submitPurgeKeysRequest(List results, return deletedCount; } + protected OzoneManagerProtocolProtos.OMResponse submitRequest(OMRequest omRequest) throws ServiceException { + return OzoneManagerRatisUtils.submitRequest(ozoneManager, omRequest, clientId, callId.incrementAndGet()); + } + /** * Parse Volume and Bucket Name from ObjectKey and add it to given map of * keys to be purged per bucket. @@ -277,7 +281,7 @@ protected void submitPurgePaths(List requests, // Submit Purge paths request to OM try { - OzoneManagerRatisUtils.submitRequest(ozoneManager, omRequest, clientId, callId.incrementAndGet()); + submitRequest(omRequest); } catch (ServiceException e) { LOG.error("PurgePaths request failed. Will retry at next run.", e); } @@ -615,10 +619,6 @@ public AtomicLong getRunCount() { return runCount; } - public AtomicLong getCallId() { - return callId; - } - /** * Returns the number of dirs deleted by the background service. * diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java index 44bb7ab088a7..a570a452c224 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/SnapshotDeletingService.java @@ -53,7 +53,6 @@ import org.apache.hadoop.ozone.om.SnapshotChainManager; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; -import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; import org.apache.hadoop.ozone.om.snapshot.ReferenceCounted; import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; @@ -260,7 +259,7 @@ private void submitSnapshotPurgeRequest(List purgeSnapshotKeys) throws I .build(); try (BootstrapStateHandler.Lock lock = getBootstrapStateLock().lock()) { - submitRequest(omRequest); + submitOMRequest(omRequest); } } } @@ -298,14 +297,13 @@ private void submitSnapshotMoveDeletedKeys(SnapshotInfo snapInfo, .setClientId(clientId.toString()) .build(); try (BootstrapStateHandler.Lock lock = getBootstrapStateLock().lock()) { - submitRequest(omRequest); + submitOMRequest(omRequest); } } - private void submitRequest(OMRequest omRequest) { + private void submitOMRequest(OMRequest omRequest) { try { - Status status = OzoneManagerRatisUtils.submitRequest(ozoneManager, omRequest, clientId, - getCallId().incrementAndGet()).getStatus(); + Status status = submitRequest(omRequest).getStatus(); if (!Objects.equals(status, Status.OK)) { LOG.error("Request: {} failed with an status: {}. Will retry in the next run.", omRequest, status); }