Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,10 @@ public enum FlatResource implements Resource {
// Lock acquired on a Snapshot's RocksDB Handle.
SNAPSHOT_DB_LOCK("SNAPSHOT_DB_LOCK"),
// Lock acquired on a Snapshot's Local Data.
SNAPSHOT_LOCAL_DATA_LOCK("SNAPSHOT_LOCAL_DATA_LOCK");
SNAPSHOT_LOCAL_DATA_LOCK("SNAPSHOT_LOCAL_DATA_LOCK"),
// Lock acquired on a Snapshot's RocksDB contents.
SNAPSHOT_DB_CONTENT_LOCK("SNAPSHOT_DB_CONTENT_LOCK");


private String name;
private IOzoneManagerLock.ResourceManager resourceManager;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ public SnapshotDefragService(long interval, TimeUnit unit, long serviceTimeout,
snapshotsDefraggedCount = new AtomicLong(0);
running = new AtomicBoolean(false);
IOzoneManagerLock omLock = ozoneManager.getMetadataManager().getLock();
this.snapshotIdLocks = new MultiSnapshotLocks(omLock, SNAPSHOT_GC_LOCK, true);
this.snapshotIdLocks = new MultiSnapshotLocks(omLock, SNAPSHOT_GC_LOCK, true, 1);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,14 @@
import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.DIRECTORY_TABLE;
import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.FILE_TABLE;
import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.SNAPSHOT_INFO_TABLE;
import static org.apache.hadoop.ozone.om.lock.FlatResource.SNAPSHOT_DB_CONTENT_LOCK;

import com.google.common.annotations.VisibleForTesting;
import jakarta.annotation.Nonnull;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.hdds.utils.db.BatchOperation;
import org.apache.hadoop.hdds.utils.db.DBStore;
Expand All @@ -36,11 +38,14 @@
import org.apache.hadoop.ozone.om.OmMetadataManagerImpl;
import org.apache.hadoop.ozone.om.OmSnapshot;
import org.apache.hadoop.ozone.om.OmSnapshotManager;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
import org.apache.hadoop.ozone.om.lock.IOzoneManagerLock;
import org.apache.hadoop.ozone.om.lock.OMLockDetails;
import org.apache.hadoop.ozone.om.request.key.OMDirectoriesPurgeRequestWithFSO;
import org.apache.hadoop.ozone.om.response.CleanupTableInfo;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
Expand Down Expand Up @@ -86,9 +91,15 @@ public void addToDBBatch(OMMetadataManager metadataManager,
OmSnapshotManager omSnapshotManager =
((OmMetadataManagerImpl) metadataManager)
.getOzoneManager().getOmSnapshotManager();

IOzoneManagerLock lock = metadataManager.getLock();
UUID fromSnapshotId = fromSnapshotInfo.getSnapshotId();
OMLockDetails lockDetails = lock.acquireReadLock(SNAPSHOT_DB_CONTENT_LOCK, fromSnapshotId.toString());
if (!lockDetails.isLockAcquired()) {
throw new OMException("Unable to acquire read lock on " + SNAPSHOT_DB_CONTENT_LOCK + " for snapshot: " +
fromSnapshotId, OMException.ResultCodes.INTERNAL_ERROR);
}
try (UncheckedAutoCloseableSupplier<OmSnapshot>
rcFromSnapshotInfo = omSnapshotManager.getSnapshot(fromSnapshotInfo.getSnapshotId())) {
rcFromSnapshotInfo = omSnapshotManager.getSnapshot(fromSnapshotId)) {
OmSnapshot fromSnapshot = rcFromSnapshotInfo.get();
DBStore fromSnapshotStore = fromSnapshot.getMetadataManager()
.getStore();
Expand All @@ -98,6 +109,8 @@ public void addToDBBatch(OMMetadataManager metadataManager,
processPaths(metadataManager, fromSnapshot.getMetadataManager(), batchOp, writeBatch);
fromSnapshotStore.commitBatchOperation(writeBatch);
}
} finally {
lock.releaseReadLock(SNAPSHOT_DB_CONTENT_LOCK, fromSnapshotId.toString());
}
metadataManager.getSnapshotInfoTable().putWithBatch(batchOp, fromSnapshotInfo.getTableKey(), fromSnapshotInfo);
} else {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,21 +19,26 @@

import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.DELETED_TABLE;
import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.SNAPSHOT_INFO_TABLE;
import static org.apache.hadoop.ozone.om.lock.FlatResource.SNAPSHOT_DB_CONTENT_LOCK;
import static org.apache.hadoop.ozone.om.response.snapshot.OMSnapshotMoveDeletedKeysResponse.createRepeatedOmKeyInfo;

import jakarta.annotation.Nonnull;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.UUID;
import org.apache.hadoop.hdds.utils.db.BatchOperation;
import org.apache.hadoop.hdds.utils.db.DBStore;
import org.apache.hadoop.ozone.om.OMMetadataManager;
import org.apache.hadoop.ozone.om.OmMetadataManagerImpl;
import org.apache.hadoop.ozone.om.OmSnapshot;
import org.apache.hadoop.ozone.om.OmSnapshotManager;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
import org.apache.hadoop.ozone.om.lock.IOzoneManagerLock;
import org.apache.hadoop.ozone.om.lock.OMLockDetails;
import org.apache.hadoop.ozone.om.request.key.OMKeyPurgeRequest;
import org.apache.hadoop.ozone.om.response.CleanupTableInfo;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyInfo;
Expand Down Expand Up @@ -82,10 +87,15 @@ public void addToDBBatch(OMMetadataManager omMetadataManager,
if (fromSnapshot != null) {
OmSnapshotManager omSnapshotManager =
((OmMetadataManagerImpl) omMetadataManager).getOzoneManager().getOmSnapshotManager();

IOzoneManagerLock lock = omMetadataManager.getLock();
UUID fromSnapshotId = fromSnapshot.getSnapshotId();
OMLockDetails lockDetails = lock.acquireReadLock(SNAPSHOT_DB_CONTENT_LOCK, fromSnapshotId.toString());
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

do we only use read lock on SNAPSHOT_DB_CONTENT_LOCK?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We would acquire a write lock when we want to do a defrag

Copy link
Contributor Author

@swamirishi swamirishi Nov 1, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Acquire write GC_LOCK
Snapshot is flushed

   take a checkpoint on tmp path
   compact if first snapshot(kforce Compaction)
   update keyTable, fileTable, directory()
   Acquire write SNAPSHOT_DB_CONTENT_LOCK
    truncate and ingest other tables(dumpToSst and ingestSstFile)
    delete if new prodPath if exists and mv tmp to production version snapshotDB path
    bump version snapshotLocalDataYaml -> new versionPath
    SNapshot cache lock on snapshotId(previosu rocksdb instance)
        delete of old version of snapshot path
    release snapshot cache lock on snapshotId
   releaseWrite content Lock
Release gc lock

if (!lockDetails.isLockAcquired()) {
throw new OMException("Unable to acquire read lock on " + SNAPSHOT_DB_CONTENT_LOCK + " for snapshot: " +
fromSnapshotId, OMException.ResultCodes.INTERNAL_ERROR);
}
try (UncheckedAutoCloseableSupplier<OmSnapshot> rcOmFromSnapshot =
omSnapshotManager.getSnapshot(fromSnapshot.getSnapshotId())) {

omSnapshotManager.getSnapshot(fromSnapshotId)) {
OmSnapshot fromOmSnapshot = rcOmFromSnapshot.get();
DBStore fromSnapshotStore = fromOmSnapshot.getMetadataManager().getStore();
// Init Batch Operation for snapshot db.
Expand All @@ -95,6 +105,8 @@ public void addToDBBatch(OMMetadataManager omMetadataManager,
processKeysToUpdate(writeBatch, fromOmSnapshot.getMetadataManager());
fromSnapshotStore.commitBatchOperation(writeBatch);
}
} finally {
lock.releaseReadLock(SNAPSHOT_DB_CONTENT_LOCK, fromSnapshotId.toString());
}
omMetadataManager.getSnapshotInfoTable().putWithBatch(batchOperation, fromSnapshot.getTableKey(), fromSnapshot);
} else {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,10 @@
package org.apache.hadoop.ozone.om.response.snapshot;

import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.SNAPSHOT_INFO_TABLE;
import static org.apache.hadoop.ozone.om.lock.FlatResource.SNAPSHOT_DB_CONTENT_LOCK;
import static org.apache.hadoop.ozone.om.snapshot.SnapshotUtils.createMergedRepeatedOmKeyInfoFromDeletedTableEntry;

import com.google.common.collect.Lists;
import jakarta.annotation.Nonnull;
import java.io.IOException;
import java.util.List;
Expand All @@ -30,9 +32,12 @@
import org.apache.hadoop.ozone.om.OmMetadataManagerImpl;
import org.apache.hadoop.ozone.om.OmSnapshot;
import org.apache.hadoop.ozone.om.OmSnapshotManager;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
import org.apache.hadoop.ozone.om.lock.IOzoneManagerLock;
import org.apache.hadoop.ozone.om.lock.OMLockDetails;
import org.apache.hadoop.ozone.om.response.CleanupTableInfo;
import org.apache.hadoop.ozone.om.response.OMClientResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
Expand Down Expand Up @@ -80,7 +85,15 @@ public OMSnapshotMoveTableKeysResponse(@Nonnull OMResponse omResponse) {
protected void addToDBBatch(OMMetadataManager omMetadataManager, BatchOperation batchOperation) throws IOException {
OmSnapshotManager omSnapshotManager = ((OmMetadataManagerImpl) omMetadataManager)
.getOzoneManager().getOmSnapshotManager();

IOzoneManagerLock lock = omMetadataManager.getLock();
String[] fromSnapshotId = new String[] {fromSnapshot.getSnapshotId().toString()};
String[] nextSnapshotId = nextSnapshot == null ? null : new String[] {nextSnapshot.getSnapshotId().toString()};
List<String[]> snapshotIds = Lists.newArrayList(fromSnapshotId, nextSnapshotId);
OMLockDetails lockDetails = lock.acquireReadLocks(SNAPSHOT_DB_CONTENT_LOCK, snapshotIds);
if (!lockDetails.isLockAcquired()) {
throw new OMException("Unable to acquire read lock on " + SNAPSHOT_DB_CONTENT_LOCK + " for snapshot: " +
snapshotIds, OMException.ResultCodes.INTERNAL_ERROR);
}
try (UncheckedAutoCloseableSupplier<OmSnapshot> rcOmFromSnapshot =
omSnapshotManager.getSnapshot(fromSnapshot.getSnapshotId())) {

Expand Down Expand Up @@ -113,6 +126,8 @@ protected void addToDBBatch(OMMetadataManager omMetadataManager, BatchOperation
fromSnapshotStore.getDb().flushWal(true);
fromSnapshotStore.getDb().flush();
}
} finally {
lock.releaseReadLocks(SNAPSHOT_DB_CONTENT_LOCK, snapshotIds);
}

// Flush snapshot info to rocksDB.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@ public SnapshotDeletingService(long interval, long serviceTimeout,
OZONE_SNAPSHOT_KEY_DELETING_LIMIT_PER_TASK,
OZONE_SNAPSHOT_KEY_DELETING_LIMIT_PER_TASK_DEFAULT);
IOzoneManagerLock lock = getOzoneManager().getMetadataManager().getLock();
this.snapshotIdLocks = new MultiSnapshotLocks(lock, SNAPSHOT_GC_LOCK, true);
this.snapshotIdLocks = new MultiSnapshotLocks(lock, SNAPSHOT_GC_LOCK, true, 2);
this.lockIds = new ArrayList<>(2);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

package org.apache.hadoop.ozone.om.snapshot;

import com.google.common.annotations.VisibleForTesting;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
Expand All @@ -39,11 +40,16 @@ public class MultiSnapshotLocks {
private final boolean writeLock;
private OMLockDetails lockDetails;

@VisibleForTesting
public MultiSnapshotLocks(IOzoneManagerLock lock, Resource resource, boolean writeLock) {
this(lock, resource, writeLock, 0);
}

public MultiSnapshotLocks(IOzoneManagerLock lock, Resource resource, boolean writeLock, int maxNumberOfLocks) {
this.writeLock = writeLock;
this.resource = resource;
this.lock = lock;
this.objectLocks = new ArrayList<>();
this.objectLocks = new ArrayList<>(maxNumberOfLocks);
this.lockDetails = OMLockDetails.EMPTY_DETAILS_LOCK_NOT_ACQUIRED;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,8 @@ public ReclaimableFilter(
this.omSnapshotManager = omSnapshotManager;
this.currentSnapshotInfo = currentSnapshotInfo;
this.snapshotChainManager = snapshotChainManager;
this.snapshotIdLocks = new MultiSnapshotLocks(lock, SNAPSHOT_GC_LOCK, false);
this.snapshotIdLocks = new MultiSnapshotLocks(lock, SNAPSHOT_GC_LOCK, false,
numberOfPreviousSnapshotsFromChain + 1);
this.keyManager = keyManager;
this.numberOfPreviousSnapshotsFromChain = numberOfPreviousSnapshotsFromChain;
this.previousOmSnapshots = new ArrayList<>(numberOfPreviousSnapshotsFromChain);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
package org.apache.hadoop.ozone.om.request.key;

import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.ONE;
import static org.apache.hadoop.ozone.om.lock.FlatResource.SNAPSHOT_DB_CONTENT_LOCK;
import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.LeveledResource.BUCKET_LOCK;
import static org.apache.hadoop.ozone.om.request.file.OMFileRequest.getOmKeyInfo;
import static org.junit.jupiter.api.Assertions.assertEquals;
Expand All @@ -32,6 +33,7 @@
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;

import com.google.common.collect.Lists;
import jakarta.annotation.Nonnull;
import java.io.IOException;
import java.util.ArrayList;
Expand Down Expand Up @@ -433,7 +435,24 @@ public void testDirectoryPurge(boolean fromSnapshot, boolean purgeDirectory, int
OMDirectoriesPurgeRequestWithFSO omKeyPurgeRequest = new OMDirectoriesPurgeRequestWithFSO(preExecutedRequest);
OMDirectoriesPurgeResponseWithFSO omClientResponse = (OMDirectoriesPurgeResponseWithFSO) omKeyPurgeRequest
.validateAndUpdateCache(ozoneManager, 100L);

IOzoneManagerLock lock = spy(omMetadataManager.getLock());
when(omMetadataManager.getLock()).thenReturn(lock);
List<String> locks = Lists.newArrayList();
doAnswer(i -> {
locks.add(i.getArgument(1));
return i.callRealMethod();
}).when(lock).acquireReadLock(eq(SNAPSHOT_DB_CONTENT_LOCK), anyString());

List<String> snapshotIds;
if (fromSnapshot) {
snapshotIds = Collections.singletonList(snapshotInfo.getSnapshotId().toString());
} else {
snapshotIds = Collections.emptyList();
}

performBatchOperationCommit(omClientResponse);
assertEquals(snapshotIds, locks);
OmBucketInfo updatedBucketInfo = purgeDirectory || numberOfSubEntries > 0 ?
omMetadataManager.getBucketTable().getSkipCache(bucketKey) : omMetadataManager.getBucketTable().get(bucketKey);
long currentSnapshotUsedNamespace = updatedBucketInfo.getSnapshotUsedNamespace();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,21 @@

package org.apache.hadoop.ozone.om.request.key;

import static org.apache.hadoop.ozone.om.lock.FlatResource.SNAPSHOT_DB_CONTENT_LOCK;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;

import com.google.common.collect.Lists;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.UUID;
import org.apache.commons.lang3.tuple.Pair;
Expand All @@ -35,6 +42,7 @@
import org.apache.hadoop.ozone.om.OmSnapshot;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
import org.apache.hadoop.ozone.om.lock.IOzoneManagerLock;
import org.apache.hadoop.ozone.om.request.OMRequestTestUtils;
import org.apache.hadoop.ozone.om.response.key.OMKeyPurgeResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeletedKeys;
Expand Down Expand Up @@ -186,7 +194,6 @@ public void testKeyPurgeInSnapshot() throws Exception {
.thenReturn(RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE));
// Create and Delete keys. The keys should be moved to DeletedKeys table
Pair<List<String>, List<String>> deleteKeysAndRenamedEntry = createAndDeleteKeysAndRenamedEntry(1, null);

SnapshotInfo snapInfo = createSnapshot("snap1");
assertEquals(snapInfo.getLastTransactionInfo(),
TransactionInfo.valueOf(TransactionInfo.getTermIndex(1L)).toByteString());
Expand Down Expand Up @@ -235,6 +242,14 @@ public void testKeyPurgeInSnapshot() throws Exception {
.setStatus(Status.OK)
.build();

IOzoneManagerLock lock = spy(omMetadataManager.getLock());
when(omMetadataManager.getLock()).thenReturn(lock);
List<String> locks = Lists.newArrayList();
doAnswer(i -> {
locks.add(i.getArgument(1));
return i.callRealMethod();
}).when(lock).acquireReadLock(eq(SNAPSHOT_DB_CONTENT_LOCK), anyString());
List<String> snapshotIds = Collections.singletonList(snapInfo.getSnapshotId().toString());
try (BatchOperation batchOperation =
omMetadataManager.getStore().initBatchOperation()) {

Expand All @@ -245,6 +260,7 @@ public void testKeyPurgeInSnapshot() throws Exception {
// Do manual commit and see whether addToBatch is successful or not.
omMetadataManager.getStore().commitBatchOperation(batchOperation);
}
assertEquals(snapshotIds, locks);
snapshotInfoOnDisk = omMetadataManager.getSnapshotInfoTable().getSkipCache(snapInfo.getTableKey());
assertEquals(snapshotInfoOnDisk, snapInfo);
// The keys should not exist in the DeletedKeys table
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -349,5 +349,4 @@ protected SnapshotInfo createSnapshot(String volume, String bucket, String snaps
assertNotNull(snapshotInfo);
return snapshotInfo;
}

}
Loading
Loading