From c8256419096f81279a0ff46815e6b8e9c8a04431 Mon Sep 17 00:00:00 2001 From: Hemant Kumar Date: Tue, 11 Jul 2023 15:25:36 -0700 Subject: [PATCH 1/3] HDDS-8652. [Snapshot] Added sequence number to the compaction log entry --- .../rocksdiff/RocksDBCheckpointDiffer.java | 33 ++++++++--- .../TestRocksDBCheckpointDiffer.java | 56 +++++++++---------- 2 files changed, 52 insertions(+), 37 deletions(-) diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java index 4bac3978d125..f26609f7bbee 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java @@ -28,6 +28,8 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; + +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator; @@ -543,6 +545,8 @@ public void onCompactionCompleted(RocksDB db, // Mark the beginning of a compaction log sb.append(COMPACTION_LOG_ENTRY_LINE_PREFIX); + sb.append(db.getLatestSequenceNumber()); + sb.append(SPACE_DELIMITER); // Trim DB path, only keep the SST file name final int filenameOffset = @@ -749,12 +753,17 @@ void processCompactionLogLine(String line) { reconstructionSnapshotGeneration = snapshotLogInfo.snapshotGenerationId; reconstructionLastSnapshotID = snapshotLogInfo.snapshotId; } else if (line.startsWith(COMPACTION_LOG_ENTRY_LINE_PREFIX)) { - // Read compaction log entry + // Compaction log entry is like following: + // C sequence_number input_files:output_files + // where input_files and output_files are joined by ','. + String[] lineSpilt = line.split(SPACE_DELIMITER); + if (lineSpilt.length != 3) { + LOG.error("Invalid line in compaction log: {}", line); + return; + } - // Trim the beginning - line = line.substring(COMPACTION_LOG_ENTRY_LINE_PREFIX.length()); - String[] io = - line.split(COMPACTION_LOG_ENTRY_INPUT_OUTPUT_FILES_DELIMITER); + String[] io = lineSpilt[2] + .split(COMPACTION_LOG_ENTRY_INPUT_OUTPUT_FILES_DELIMITER); if (io.length != 2) { if (line.endsWith(":")) { @@ -1162,8 +1171,11 @@ public void pruneOlderSnapshotsWithCompactionHistory() { Set sstFileNodesRemoved = pruneSstFileNodesFromDag(lastCompactionSstFiles); - LOG.info("Removing SST files: {} as part of compaction DAG pruning.", - sstFileNodesRemoved); + if (CollectionUtils.isNotEmpty(sstFileNodesRemoved)) { + LOG.info("Removing SST files: {} as part of compaction DAG pruning.", + sstFileNodesRemoved); + } + try (BootstrapStateHandler.Lock lock = getBootstrapStateLock().lock()) { removeSstFiles(sstFileNodesRemoved); deleteOlderSnapshotsCompactionFiles(olderSnapshotsLogFilePaths); @@ -1471,8 +1483,11 @@ public synchronized void pruneSstFiles() { .map(node -> node.getFileName()) .collect(Collectors.toSet()); - LOG.info("Removing SST files: {} as part of SST file pruning.", - nonLeafSstFiles); + if (CollectionUtils.isNotEmpty(nonLeafSstFiles)) { + LOG.info("Removing SST files: {} as part of SST file pruning.", + nonLeafSstFiles); + } + try (BootstrapStateHandler.Lock lock = getBootstrapStateLock().lock()) { removeSstFiles(nonLeafSstFiles); } catch (InterruptedException e) { diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java index 69ceaa231c15..6001d1ae7865 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java @@ -282,22 +282,22 @@ public void testGetSSTDiffListWithoutDB(String description, // Snapshot 0 + "S 1000 df6410c7-151b-4e90-870e-5ef12875acd5 " + createdTime + " \n" // Additional "compaction" to trigger and test early exit condition - + "C 000001,000002:000062\n" + + "C 1 000001,000002:000062\n" // Snapshot 1 + "S 3008 ef6410c7-151b-4e90-870e-5ef12875acd5 " + createdTime + " \n" // Regular compaction - + "C 000068,000062:000069\n" + + "C 2 000068,000062:000069\n" // Trivial move - + "C 000071,000064,000060,000052:000071,000064,000060,000052\n" - + "C 000073,000066:000074\n" - + "C 000082,000076,000069:000083\n" - + "C 000087,000080,000074:000088\n" + + "C 3 000071,000064,000060,000052:000071,000064,000060,000052\n" + + "C 4 000073,000066:000074\n" + + "C 5 000082,000076,000069:000083\n" + + "C 6 000087,000080,000074:000088\n" // Deletion? - + "C 000093,000090,000083:\n" + + "C 7 000093,000090,000083:\n" // Snapshot 2 + "S 14980 e7ad72f8-52df-4430-93f6-0ee91d4a47fd " + createdTime + "\n" - + "C 000098,000096,000085,000078,000071,000064,000060,000052:000099\n" - + "C 000105,000095,000088:000107\n" + + "C 8 000098,000096,000085,000078,000071,000064,000060,000052:000099\n" + + "C 9 000105,000095,000088:000107\n" // Snapshot 3 + "S 17975 4f084f6e-ed3d-4780-8362-f832303309ea " + createdTime + "\n"; @@ -921,38 +921,38 @@ private static Stream compactionDagPruningScenarios() { String compactionLogFile0 = "S 1000 snapshotId0 " + (currentTimeMillis - MINUTES.toMillis(30)) + " \n"; - String compactionLogFile1 = "C 000015,000013,000011,000009:000018,000016," + - "000017\n" + String compactionLogFile1 = "C 1 000015,000013,000011,000009:000018," + + "000016,000017\n" + "S 2000 snapshotId1 " + (currentTimeMillis - MINUTES.toMillis(24)) + " \n"; - String compactionLogFile2 = "C 000018,000016,000017,000026,000024,000022," + - "000020:000027,000030,000028,000031,000029\n" + String compactionLogFile2 = "C 2 000018,000016,000017,000026,000024," + + "000022,000020:000027,000030,000028,000031,000029\n" + "S 3000 snapshotId2 " + (currentTimeMillis - MINUTES.toMillis(18)) + " \n"; - String compactionLogFile3 = "C 000027,000030,000028,000031,000029,000039," + - "000037,000035,000033:000040,000044,000042,000043,000046,000041," + - "000045\n" + String compactionLogFile3 = "C 3 000027,000030,000028,000031,000029," + + "000039,000037,000035,000033:000040,000044,000042,000043,000046," + + "000041,000045\n" + "S 3000 snapshotId3 " + (currentTimeMillis - MINUTES.toMillis(12)) + " \n"; - String compactionLogFile4 = "C 000040,000044,000042,000043,000046,000041," + - "000045,000054,000052,000050,000048:000059,000055,000056,000060," + - "000057,000058\n" + String compactionLogFile4 = "C 4 000040,000044,000042,000043,000046," + + "000041,000045,000054,000052,000050,000048:000059,000055,000056," + + "000060,000057,000058\n" + "S 3000 snapshotId4 " + (currentTimeMillis - MINUTES.toMillis(6)) + " \n"; - String compactionLogFileWithoutSnapshot1 = "C 000015,000013,000011," + + String compactionLogFileWithoutSnapshot1 = "C 1 000015,000013,000011," + "000009:000018,000016,000017\n" + - "C 000018,000016,000017,000026,000024,000022,000020:000027,000030," + + "C 2 000018,000016,000017,000026,000024,000022,000020:000027,000030," + "000028,000031,000029\n"; - String compactionLogFileWithoutSnapshot2 = "C 000027,000030,000028," + + String compactionLogFileWithoutSnapshot2 = "C 3 000027,000030,000028," + "000031,000029,000039,000037,000035,000033:000040,000044,000042," + "000043,000046,000041,000045\n"; - String compactionLogFileWithoutSnapshot3 = "C 000040,000044,000042," + + String compactionLogFileWithoutSnapshot3 = "C 4 000040,000044,000042," + "000043,000046,000041,000045,000054,000052,000050,000048:000059," + "000055,000056,000060,000057,000058\n"; @@ -1156,19 +1156,19 @@ private static Stream sstFilePruningScenarios() { Arrays.asList("000015", "000013", "000011", "000009") ), Arguments.of("Case 2: One level compaction.", - "C 000015,000013,000011,000009:000018,000016,000017\n", + "C 1 000015,000013,000011,000009:000018,000016,000017\n", Arrays.asList("000015", "000013", "000011", "000009", "000018", "000016", "000017", "000026", "000024", "000022", "000020"), Arrays.asList("000015", "000013", "000011", "000009", "000026", "000024", "000022", "000020") ), Arguments.of("Case 3: Multi-level compaction.", - "C 000015,000013,000011,000009:000018,000016,000017\n" + - "C 000018,000016,000017,000026,000024,000022,000020:000027," + + "C 1 000015,000013,000011,000009:000018,000016,000017\n" + + "C 2 000018,000016,000017,000026,000024,000022,000020:000027," + "000030,000028,000031,000029\n" + - "C 000027,000030,000028,000031,000029,000039,000037,000035," + + "C 3 000027,000030,000028,000031,000029,000039,000037,000035," + "000033:000040,000044,000042,000043,000046,000041,000045\n" + - "C 000040,000044,000042,000043,000046,000041,000045,000054," + + "C 4 000040,000044,000042,000043,000046,000041,000045,000054," + "000052,000050,000048:000059,000055,000056,000060,000057," + "000058\n", Arrays.asList("000015", "000013", "000011", "000009", "000018", From 5d3150a034ad4eed8cd023d6b6a3e3a71a7e75c7 Mon Sep 17 00:00:00 2001 From: Hemant Kumar Date: Tue, 11 Jul 2023 21:26:07 -0700 Subject: [PATCH 2/3] Addressed code review comments --- .../TestRocksDBCheckpointDiffer.java | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java index 6001d1ae7865..3df1da9e4118 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java @@ -282,22 +282,22 @@ public void testGetSSTDiffListWithoutDB(String description, // Snapshot 0 + "S 1000 df6410c7-151b-4e90-870e-5ef12875acd5 " + createdTime + " \n" // Additional "compaction" to trigger and test early exit condition - + "C 1 000001,000002:000062\n" + + "C 1291 000001,000002:000062\n" // Snapshot 1 + "S 3008 ef6410c7-151b-4e90-870e-5ef12875acd5 " + createdTime + " \n" // Regular compaction - + "C 2 000068,000062:000069\n" + + "C 4023 000068,000062:000069\n" // Trivial move - + "C 3 000071,000064,000060,000052:000071,000064,000060,000052\n" - + "C 4 000073,000066:000074\n" - + "C 5 000082,000076,000069:000083\n" - + "C 6 000087,000080,000074:000088\n" + + "C 5647 000071,000064,000060,000052:000071,000064,000060,000052\n" + + "C 7658 000073,000066:000074\n" + + "C 7872 000082,000076,000069:000083\n" + + "C 9001 000087,000080,000074:000088\n" // Deletion? - + "C 7 000093,000090,000083:\n" + + "C 12755 000093,000090,000083:\n" // Snapshot 2 + "S 14980 e7ad72f8-52df-4430-93f6-0ee91d4a47fd " + createdTime + "\n" - + "C 8 000098,000096,000085,000078,000071,000064,000060,000052:000099\n" - + "C 9 000105,000095,000088:000107\n" + + "C 16192 000098,000096,000085,000078,000071,000064,000060,000052:000099\n" + + "C 16762 000105,000095,000088:000107\n" // Snapshot 3 + "S 17975 4f084f6e-ed3d-4780-8362-f832303309ea " + createdTime + "\n"; @@ -921,38 +921,38 @@ private static Stream compactionDagPruningScenarios() { String compactionLogFile0 = "S 1000 snapshotId0 " + (currentTimeMillis - MINUTES.toMillis(30)) + " \n"; - String compactionLogFile1 = "C 1 000015,000013,000011,000009:000018," + + String compactionLogFile1 = "C 1000 000015,000013,000011,000009:000018," + "000016,000017\n" + "S 2000 snapshotId1 " + (currentTimeMillis - MINUTES.toMillis(24)) + " \n"; - String compactionLogFile2 = "C 2 000018,000016,000017,000026,000024," + + String compactionLogFile2 = "C 1000 000018,000016,000017,000026,000024," + "000022,000020:000027,000030,000028,000031,000029\n" + "S 3000 snapshotId2 " + (currentTimeMillis - MINUTES.toMillis(18)) + " \n"; - String compactionLogFile3 = "C 3 000027,000030,000028,000031,000029," + + String compactionLogFile3 = "C 1000 000027,000030,000028,000031,000029," + "000039,000037,000035,000033:000040,000044,000042,000043,000046," + "000041,000045\n" + "S 3000 snapshotId3 " + (currentTimeMillis - MINUTES.toMillis(12)) + " \n"; - String compactionLogFile4 = "C 4 000040,000044,000042,000043,000046," + + String compactionLogFile4 = "C 1000 000040,000044,000042,000043,000046," + "000041,000045,000054,000052,000050,000048:000059,000055,000056," + "000060,000057,000058\n" + "S 3000 snapshotId4 " + (currentTimeMillis - MINUTES.toMillis(6)) + " \n"; - String compactionLogFileWithoutSnapshot1 = "C 1 000015,000013,000011," + + String compactionLogFileWithoutSnapshot1 = "C 1000 000015,000013,000011," + "000009:000018,000016,000017\n" + - "C 2 000018,000016,000017,000026,000024,000022,000020:000027,000030," + + "C 2000 000018,000016,000017,000026,000024,000022,000020:000027,000030," + "000028,000031,000029\n"; - String compactionLogFileWithoutSnapshot2 = "C 3 000027,000030,000028," + + String compactionLogFileWithoutSnapshot2 = "C 3000 000027,000030,000028," + "000031,000029,000039,000037,000035,000033:000040,000044,000042," + "000043,000046,000041,000045\n"; - String compactionLogFileWithoutSnapshot3 = "C 4 000040,000044,000042," + + String compactionLogFileWithoutSnapshot3 = "C 4000 000040,000044,000042," + "000043,000046,000041,000045,000054,000052,000050,000048:000059," + "000055,000056,000060,000057,000058\n"; From 963a5158985c1759d49d8e71ce005e18f6464f43 Mon Sep 17 00:00:00 2001 From: Siyao Meng <50227127+smengcl@users.noreply.github.com> Date: Wed, 12 Jul 2023 11:15:39 -0700 Subject: [PATCH 3/3] checkstyle --- .../ozone/rocksdiff/TestRocksDBCheckpointDiffer.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java index 3df1da9e4118..100aef0a8d3a 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java @@ -296,7 +296,8 @@ public void testGetSSTDiffListWithoutDB(String description, + "C 12755 000093,000090,000083:\n" // Snapshot 2 + "S 14980 e7ad72f8-52df-4430-93f6-0ee91d4a47fd " + createdTime + "\n" - + "C 16192 000098,000096,000085,000078,000071,000064,000060,000052:000099\n" + + "C 16192 000098,000096,000085,000078,000071,000064,000060,000052" + + ":000099\n" + "C 16762 000105,000095,000088:000107\n" // Snapshot 3 + "S 17975 4f084f6e-ed3d-4780-8362-f832303309ea " + createdTime + "\n"; @@ -945,8 +946,8 @@ private static Stream compactionDagPruningScenarios() { String compactionLogFileWithoutSnapshot1 = "C 1000 000015,000013,000011," + "000009:000018,000016,000017\n" + - "C 2000 000018,000016,000017,000026,000024,000022,000020:000027,000030," + - "000028,000031,000029\n"; + "C 2000 000018,000016,000017,000026,000024,000022,000020" + + ":000027,000030,000028,000031,000029\n"; String compactionLogFileWithoutSnapshot2 = "C 3000 000027,000030,000028," + "000031,000029,000039,000037,000035,000033:000040,000044,000042," +