From 5a6266819ca8ea833f9877d078d949667db07686 Mon Sep 17 00:00:00 2001 From: Sadanand Shenoy Date: Fri, 9 May 2025 14:08:23 +0530 Subject: [PATCH 1/9] HDDS-12985. Exclude same SST files while calculating snapdiff. --- .../apache/ozone/rocksdb/util/RdbUtil.java | 20 ++++++++++++++++ .../om/snapshot/SnapshotDiffManager.java | 24 +++++++++++++++---- 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/RdbUtil.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/RdbUtil.java index 48a8ee696c85..151f97736ed7 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/RdbUtil.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/RdbUtil.java @@ -19,8 +19,12 @@ import com.google.common.collect.Sets; import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.attribute.BasicFileAttributes; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.stream.Collectors; import org.apache.hadoop.hdds.StringUtils; @@ -49,4 +53,20 @@ public static Set getSSTFilesForComparison( .map(lfm -> new File(lfm.path(), lfm.fileName()).getPath()) .collect(Collectors.toCollection(HashSet::new)); } + + public static Map getSSTFilesWithInodesForComparison( + final ManagedRocksDB rocksDB, List cfs){ + return getLiveSSTFilesForCFs(rocksDB, cfs).stream() + .collect(Collectors.toMap( + lfm -> { + try { + return Files.readAttributes(new File(lfm.path(), lfm.fileName()).toPath(), + BasicFileAttributes.class).fileKey(); + } catch (IOException e) { + throw new RuntimeException(e); + } + }, + lfm -> new File(lfm.path(), lfm.fileName()).getPath() + )); + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java index 17199debbd47..b84357ce8e4f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java @@ -90,6 +90,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import java.util.function.BiFunction; +import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.commons.io.file.PathUtils; import org.apache.commons.lang3.tuple.Pair; @@ -374,6 +375,14 @@ protected Set getSSTFileListForSnapshot(OmSnapshot snapshot, tablesToLookUp); } + @VisibleForTesting + protected Map getSSTFileMapForSnapshot(OmSnapshot snapshot, + List tablesToLookUp) { + return RdbUtil.getSSTFilesWithInodesForComparison(((RDBStore)snapshot + .getMetadataManager().getStore()).getDb().getManagedRocksDb(), + tablesToLookUp); + } + /** * Gets the report key for a particular index of snapshot diff job. */ @@ -1187,11 +1196,7 @@ Set getDeltaFiles(OmSnapshot fromSnapshot, .getDb().getManagedRocksDb(); ManagedRocksDB toDB = ((RDBStore)toSnapshot.getMetadataManager().getStore()) .getDb().getManagedRocksDb(); - Set fromSnapshotFiles = getSSTFileListForSnapshot(fromSnapshot, tablesToLookUp); - Set toSnapshotFiles = getSSTFileListForSnapshot(toSnapshot, tablesToLookUp); - Set diffFiles = new HashSet<>(); - diffFiles.addAll(fromSnapshotFiles); - diffFiles.addAll(toSnapshotFiles); + Set diffFiles = getDiffFiles(fromSnapshot, toSnapshot, tablesToLookUp); RocksDiffUtils.filterRelevantSstFiles(diffFiles, tablePrefixes, fromDB, toDB); deltaFiles = Optional.of(diffFiles); } @@ -1201,6 +1206,15 @@ Set getDeltaFiles(OmSnapshot fromSnapshot, toSnapshot.getSnapshotTableKey())); } + private Set getDiffFiles(OmSnapshot fromSnapshot, OmSnapshot toSnapshot, List tablesToLookUp) { + Map fromSnapshotFiles = getSSTFileMapForSnapshot(fromSnapshot, tablesToLookUp); + Map toSnapshotFiles = getSSTFileMapForSnapshot(toSnapshot, tablesToLookUp); + return Stream.concat(fromSnapshotFiles.entrySet().stream(), + toSnapshotFiles.entrySet().stream().filter(e -> !fromSnapshotFiles.containsKey(e.getKey()))) + .map(Map.Entry::getValue) + .collect(Collectors.toSet()); + } + private void validateEstimatedKeyChangesAreInLimits( SstFileSetReader sstFileReader ) throws RocksDBException, IOException { From 72006e3d2a08ce7cb1253b26312a8daa81bf7586 Mon Sep 17 00:00:00 2001 From: Sadanand Shenoy Date: Fri, 9 May 2025 14:31:48 +0530 Subject: [PATCH 2/9] add test --- .../om/snapshot/TestSnapshotDiffManager.java | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java index 793d87296f15..f539fa3120e5 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java @@ -1533,9 +1533,8 @@ private void setupMocksForRunningASnapDiff( String bucketKey = omMetadataManager.getBucketKey(volumeName, bucketName); when(bucketInfoTable.get(bucketKey)).thenReturn(bucketInfo); } - @Test - public void testGetDeltaFilesWithFullDiff() throws IOException { + public void testGetDeltaFilesWithFullDiff() throws IOException { SnapshotDiffManager spy = spy(snapshotDiffManager); UUID snap1 = UUID.randomUUID(); OmSnapshot fromSnapshot = getMockedOmSnapshot(snap1); @@ -1544,20 +1543,28 @@ public void testGetDeltaFilesWithFullDiff() throws IOException { Mockito.doAnswer(invocation -> { OmSnapshot snapshot = invocation.getArgument(0); if (snapshot == fromSnapshot) { - return Sets.newHashSet("1", "2", "3"); + Map inodeToFileMap = new HashMap<>(); + inodeToFileMap.put(1, "1.sst"); + inodeToFileMap.put(2, "2.sst"); + inodeToFileMap.put(3, "3.sst"); + return inodeToFileMap; } if (snapshot == toSnapshot) { - return Sets.newHashSet("3", "4", "5"); + Map inodeToFileMap = new HashMap<>(); + inodeToFileMap.put(1, "10.sst"); + inodeToFileMap.put(2, "20.sst"); + inodeToFileMap.put(4, "4.sst"); + return inodeToFileMap; } - return Sets.newHashSet("6", "7", "8"); - }).when(spy).getSSTFileListForSnapshot(Mockito.any(OmSnapshot.class), + return null; + }).when(spy).getSSTFileMapForSnapshot(Mockito.any(OmSnapshot.class), Mockito.anyList()); doNothing().when(spy).recordActivity(any(), any()); doNothing().when(spy).updateProgress(anyString(), anyDouble()); String diffJobKey = snap1 + DELIMITER + snap2; Set deltaFiles = spy.getDeltaFiles(fromSnapshot, toSnapshot, Collections.emptyList(), snapshotInfo, snapshotInfo, true, Collections.emptyMap(), null, diffJobKey); - Assertions.assertEquals(Sets.newHashSet("1", "2", "3", "4", "5"), deltaFiles); + Assertions.assertEquals(Sets.newHashSet("1.sst", "2.sst", "3.sst", "4.sst"), deltaFiles); } @Test From 7ccc478cccf70239d17ba05d1905d346614c3811 Mon Sep 17 00:00:00 2001 From: Sadanand Shenoy Date: Fri, 9 May 2025 15:05:12 +0530 Subject: [PATCH 3/9] fix checkstyle --- .../src/main/java/org/apache/ozone/rocksdb/util/RdbUtil.java | 3 +-- .../apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/RdbUtil.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/RdbUtil.java index 151f97736ed7..60d52e1fb5e6 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/RdbUtil.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/RdbUtil.java @@ -54,8 +54,7 @@ public static Set getSSTFilesForComparison( .collect(Collectors.toCollection(HashSet::new)); } - public static Map getSSTFilesWithInodesForComparison( - final ManagedRocksDB rocksDB, List cfs){ + public static Map getSSTFilesWithInodesForComparison(final ManagedRocksDB rocksDB, List cfs) { return getLiveSSTFilesForCFs(rocksDB, cfs).stream() .collect(Collectors.toMap( lfm -> { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java index b84357ce8e4f..74d3ef01c574 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java @@ -376,7 +376,7 @@ protected Set getSSTFileListForSnapshot(OmSnapshot snapshot, } @VisibleForTesting - protected Map getSSTFileMapForSnapshot(OmSnapshot snapshot, + protected Map getSSTFileMapForSnapshot(OmSnapshot snapshot, List tablesToLookUp) { return RdbUtil.getSSTFilesWithInodesForComparison(((RDBStore)snapshot .getMetadataManager().getStore()).getDb().getManagedRocksDb(), From e3a3427d116503d0c2e6528150d93727617b5195 Mon Sep 17 00:00:00 2001 From: Sadanand Shenoy Date: Sun, 11 May 2025 21:51:22 +0530 Subject: [PATCH 4/9] fix checkstyle --- .../apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java index e61d51120c06..122a58a41600 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java @@ -1534,6 +1534,7 @@ private void setupMocksForRunningASnapDiff( String bucketKey = omMetadataManager.getBucketKey(volumeName, bucketName); when(bucketInfoTable.get(bucketKey)).thenReturn(bucketInfo); } + @Test public void testGetDeltaFilesWithFullDiff() throws IOException { SnapshotDiffManager spy = spy(snapshotDiffManager); From d92fed08f10b9efdc56224be1b8ffc7a5be6fe6e Mon Sep 17 00:00:00 2001 From: Sadanand Shenoy Date: Mon, 12 May 2025 13:03:16 +0530 Subject: [PATCH 5/9] handle error case --- .../apache/ozone/rocksdb/util/RdbUtil.java | 3 ++- .../om/snapshot/SnapshotDiffManager.java | 21 +++++++++++++------ 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/RdbUtil.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/RdbUtil.java index 60d52e1fb5e6..e8aa1bc79543 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/RdbUtil.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/RdbUtil.java @@ -20,6 +20,7 @@ import com.google.common.collect.Sets; import java.io.File; import java.io.IOException; +import java.io.UncheckedIOException; import java.nio.file.Files; import java.nio.file.attribute.BasicFileAttributes; import java.util.HashSet; @@ -62,7 +63,7 @@ public static Map getSSTFilesWithInodesForComparison(final Manag return Files.readAttributes(new File(lfm.path(), lfm.fileName()).toPath(), BasicFileAttributes.class).fileKey(); } catch (IOException e) { - throw new RuntimeException(e); + throw new UncheckedIOException("Failed to read inode for " + lfm.path(), e); } }, lfm -> new File(lfm.path(), lfm.fileName()).getPath() diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java index e89e4b5b8985..9b909fbf158e 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java @@ -70,6 +70,7 @@ import java.io.BufferedWriter; import java.io.File; import java.io.IOException; +import java.io.UncheckedIOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -1223,12 +1224,20 @@ Set getDeltaFiles(OmSnapshot fromSnapshot, } private Set getDiffFiles(OmSnapshot fromSnapshot, OmSnapshot toSnapshot, List tablesToLookUp) { - Map fromSnapshotFiles = getSSTFileMapForSnapshot(fromSnapshot, tablesToLookUp); - Map toSnapshotFiles = getSSTFileMapForSnapshot(toSnapshot, tablesToLookUp); - return Stream.concat(fromSnapshotFiles.entrySet().stream(), - toSnapshotFiles.entrySet().stream().filter(e -> !fromSnapshotFiles.containsKey(e.getKey()))) - .map(Map.Entry::getValue) - .collect(Collectors.toSet()); + Set diffFiles; + try { + Map fromSnapshotFiles = getSSTFileMapForSnapshot(fromSnapshot, tablesToLookUp); + Map toSnapshotFiles = getSSTFileMapForSnapshot(toSnapshot, tablesToLookUp); + diffFiles = Stream.concat(fromSnapshotFiles.entrySet().stream(), + toSnapshotFiles.entrySet().stream().filter(e -> !fromSnapshotFiles.containsKey(e.getKey()))) + .map(Map.Entry::getValue).collect(Collectors.toSet()); + } catch (UncheckedIOException e) { + // In case of exception during inode read use all files + diffFiles = new HashSet<>(); + diffFiles.addAll(getSSTFileListForSnapshot(fromSnapshot, tablesToLookUp)); + diffFiles.addAll(getSSTFileListForSnapshot(toSnapshot, tablesToLookUp)); + } + return diffFiles; } private void validateEstimatedKeyChangesAreInLimits( From daeaa682a11ad9b436be99b434a3230c6c585759 Mon Sep 17 00:00:00 2001 From: Sadanand Shenoy Date: Mon, 12 May 2025 16:38:36 +0530 Subject: [PATCH 6/9] address comments --- .../apache/ozone/rocksdb/util/RdbUtil.java | 21 +++++++++---------- .../om/snapshot/SnapshotDiffManager.java | 10 ++++++--- .../om/snapshot/TestSnapshotDiffManager.java | 2 +- 3 files changed, 18 insertions(+), 15 deletions(-) diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/RdbUtil.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/RdbUtil.java index e8aa1bc79543..b846642a8e9c 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/RdbUtil.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/RdbUtil.java @@ -23,6 +23,7 @@ import java.io.UncheckedIOException; import java.nio.file.Files; import java.nio.file.attribute.BasicFileAttributes; +import java.util.AbstractMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -57,16 +58,14 @@ public static Set getSSTFilesForComparison( public static Map getSSTFilesWithInodesForComparison(final ManagedRocksDB rocksDB, List cfs) { return getLiveSSTFilesForCFs(rocksDB, cfs).stream() - .collect(Collectors.toMap( - lfm -> { - try { - return Files.readAttributes(new File(lfm.path(), lfm.fileName()).toPath(), - BasicFileAttributes.class).fileKey(); - } catch (IOException e) { - throw new UncheckedIOException("Failed to read inode for " + lfm.path(), e); - } - }, - lfm -> new File(lfm.path(), lfm.fileName()).getPath() - )); + .map(lfm -> { + File sstFile = new File(lfm.path(), lfm.fileName()); + try { + Object inode = Files.readAttributes(sstFile.toPath(), BasicFileAttributes.class).fileKey(); + return new AbstractMap.SimpleEntry<>(inode, sstFile.getPath()); + } catch (IOException e) { + throw new UncheckedIOException("Failed to read inode for " + sstFile.getPath(), e); + } + }).collect(Collectors.toMap(Map.Entry::getKey,Map.Entry::getValue)); } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java index 9b909fbf158e..ab23dda94c90 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java @@ -1228,9 +1228,13 @@ private Set getDiffFiles(OmSnapshot fromSnapshot, OmSnapshot toSnapshot, try { Map fromSnapshotFiles = getSSTFileMapForSnapshot(fromSnapshot, tablesToLookUp); Map toSnapshotFiles = getSSTFileMapForSnapshot(toSnapshot, tablesToLookUp); - diffFiles = Stream.concat(fromSnapshotFiles.entrySet().stream(), - toSnapshotFiles.entrySet().stream().filter(e -> !fromSnapshotFiles.containsKey(e.getKey()))) - .map(Map.Entry::getValue).collect(Collectors.toSet()); + diffFiles = Stream.concat( + fromSnapshotFiles.entrySet().stream() + .filter(e -> !toSnapshotFiles.containsKey(e.getKey())), + toSnapshotFiles.entrySet().stream() + .filter(e -> !fromSnapshotFiles.containsKey(e.getKey()))) + .map(Map.Entry::getValue) + .collect(Collectors.toSet()); } catch (UncheckedIOException e) { // In case of exception during inode read use all files diffFiles = new HashSet<>(); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java index 122a58a41600..b941000d9421 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestSnapshotDiffManager.java @@ -1566,7 +1566,7 @@ public void testGetDeltaFilesWithFullDiff() throws IOException { String diffJobKey = snap1 + DELIMITER + snap2; Set deltaFiles = spy.getDeltaFiles(fromSnapshot, toSnapshot, Collections.emptyList(), snapshotInfo, snapshotInfo, true, Collections.emptyMap(), null, diffJobKey); - Assertions.assertEquals(Sets.newHashSet("1.sst", "2.sst", "3.sst", "4.sst"), deltaFiles); + Assertions.assertEquals(Sets.newHashSet("3.sst", "4.sst"), deltaFiles); } @Test From dd2311c8b6156465e7f0543e364b1b73646f4f3f Mon Sep 17 00:00:00 2001 From: Sadanand Shenoy Date: Mon, 12 May 2025 16:44:43 +0530 Subject: [PATCH 7/9] fix checkstyle --- .../src/main/java/org/apache/ozone/rocksdb/util/RdbUtil.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/RdbUtil.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/RdbUtil.java index b846642a8e9c..c5b1b422833d 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/RdbUtil.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/RdbUtil.java @@ -66,6 +66,6 @@ public static Map getSSTFilesWithInodesForComparison(final Manag } catch (IOException e) { throw new UncheckedIOException("Failed to read inode for " + sstFile.getPath(), e); } - }).collect(Collectors.toMap(Map.Entry::getKey,Map.Entry::getValue)); + }).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); } } From b08118bcaf3cc3dc2249d86ae9f725c3e6806d14 Mon Sep 17 00:00:00 2001 From: Sadanand Shenoy Date: Mon, 12 May 2025 17:21:17 +0530 Subject: [PATCH 8/9] address comments --- .../apache/ozone/rocksdb/util/RdbUtil.java | 24 +++++++++---------- .../om/snapshot/SnapshotDiffManager.java | 6 ++--- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/RdbUtil.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/RdbUtil.java index c5b1b422833d..977928760b92 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/RdbUtil.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/RdbUtil.java @@ -20,10 +20,9 @@ import com.google.common.collect.Sets; import java.io.File; import java.io.IOException; -import java.io.UncheckedIOException; import java.nio.file.Files; import java.nio.file.attribute.BasicFileAttributes; -import java.util.AbstractMap; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -56,16 +55,15 @@ public static Set getSSTFilesForComparison( .collect(Collectors.toCollection(HashSet::new)); } - public static Map getSSTFilesWithInodesForComparison(final ManagedRocksDB rocksDB, List cfs) { - return getLiveSSTFilesForCFs(rocksDB, cfs).stream() - .map(lfm -> { - File sstFile = new File(lfm.path(), lfm.fileName()); - try { - Object inode = Files.readAttributes(sstFile.toPath(), BasicFileAttributes.class).fileKey(); - return new AbstractMap.SimpleEntry<>(inode, sstFile.getPath()); - } catch (IOException e) { - throw new UncheckedIOException("Failed to read inode for " + sstFile.getPath(), e); - } - }).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + public static Map getSSTFilesWithInodesForComparison(final ManagedRocksDB rocksDB, List cfs) + throws IOException { + List liveSSTFilesForCFs = getLiveSSTFilesForCFs(rocksDB, cfs); + Map inodeToSstMap = new HashMap<>(); + for (LiveFileMetaData lfm : liveSSTFilesForCFs) { + File sstFile = new File(lfm.path(), lfm.fileName()); + Object inode = Files.readAttributes(sstFile.toPath(), BasicFileAttributes.class).fileKey(); + inodeToSstMap.put(inode, sstFile.getPath()); + } + return inodeToSstMap; } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java index ab23dda94c90..871a90c6ee4c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java @@ -70,7 +70,6 @@ import java.io.BufferedWriter; import java.io.File; import java.io.IOException; -import java.io.UncheckedIOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -379,7 +378,7 @@ protected Set getSSTFileListForSnapshot(OmSnapshot snapshot, @VisibleForTesting protected Map getSSTFileMapForSnapshot(OmSnapshot snapshot, - List tablesToLookUp) { + List tablesToLookUp) throws IOException { return RdbUtil.getSSTFilesWithInodesForComparison(((RDBStore)snapshot .getMetadataManager().getStore()).getDb().getManagedRocksDb(), tablesToLookUp); @@ -1235,8 +1234,9 @@ private Set getDiffFiles(OmSnapshot fromSnapshot, OmSnapshot toSnapshot, .filter(e -> !fromSnapshotFiles.containsKey(e.getKey()))) .map(Map.Entry::getValue) .collect(Collectors.toSet()); - } catch (UncheckedIOException e) { + } catch (IOException e) { // In case of exception during inode read use all files + LOG.error("Exception occurred while populating delta files for snapDiff", e); diffFiles = new HashSet<>(); diffFiles.addAll(getSSTFileListForSnapshot(fromSnapshot, tablesToLookUp)); diffFiles.addAll(getSSTFileListForSnapshot(toSnapshot, tablesToLookUp)); From d33b3ff7085f901401e3c94a40bf109ce1a6c5ca Mon Sep 17 00:00:00 2001 From: Sadanand Shenoy Date: Tue, 13 May 2025 09:45:12 +0530 Subject: [PATCH 9/9] address comments --- .../main/java/org/apache/ozone/rocksdb/util/RdbUtil.java | 8 +++++--- .../hadoop/ozone/om/snapshot/SnapshotDiffManager.java | 1 + 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/RdbUtil.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/RdbUtil.java index 977928760b92..97eaa945fdce 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/RdbUtil.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdb/util/RdbUtil.java @@ -21,6 +21,8 @@ import java.io.File; import java.io.IOException; import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.nio.file.attribute.BasicFileAttributes; import java.util.HashMap; import java.util.HashSet; @@ -60,9 +62,9 @@ public static Map getSSTFilesWithInodesForComparison(final Manag List liveSSTFilesForCFs = getLiveSSTFilesForCFs(rocksDB, cfs); Map inodeToSstMap = new HashMap<>(); for (LiveFileMetaData lfm : liveSSTFilesForCFs) { - File sstFile = new File(lfm.path(), lfm.fileName()); - Object inode = Files.readAttributes(sstFile.toPath(), BasicFileAttributes.class).fileKey(); - inodeToSstMap.put(inode, sstFile.getPath()); + Path sstFilePath = Paths.get(lfm.path(), lfm.fileName()); + Object inode = Files.readAttributes(sstFilePath, BasicFileAttributes.class).fileKey(); + inodeToSstMap.put(inode, sstFilePath.toString()); } return inodeToSstMap; } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java index 871a90c6ee4c..75b4196dd86b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java @@ -1237,6 +1237,7 @@ private Set getDiffFiles(OmSnapshot fromSnapshot, OmSnapshot toSnapshot, } catch (IOException e) { // In case of exception during inode read use all files LOG.error("Exception occurred while populating delta files for snapDiff", e); + LOG.warn("Falling back to full file list comparison, inode-based optimization skipped."); diffFiles = new HashSet<>(); diffFiles.addAll(getSSTFileListForSnapshot(fromSnapshot, tablesToLookUp)); diffFiles.addAll(getSSTFileListForSnapshot(toSnapshot, tablesToLookUp));