Skip to content

HDDS-8652. [Snapshot] Added DB sequence number to the compaction log entry #5046

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
smengcl merged 3 commits into from
Jul 12, 2023
Merged

HDDS-8652. [Snapshot] Added DB sequence number to the compaction log entry #5046

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c825641
HDDS-8652. [Snapshot] Added sequence number to the compaction log entry
hemantk-12 Jul 11, 2023
5d3150a
Addressed code review comments
hemantk-12 Jul 12, 2023
963a515
checkstyle
smengcl Jul 12, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 24 additions & 9 deletions ...b-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;

import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksIterator;
Expand Down Expand Up @@ -543,6 +545,8 @@ public void onCompactionCompleted(RocksDB db,

// Mark the beginning of a compaction log
sb.append(COMPACTION_LOG_ENTRY_LINE_PREFIX);
sb.append(db.getLatestSequenceNumber());
sb.append(SPACE_DELIMITER);

// Trim DB path, only keep the SST file name
final int filenameOffset =
Expand Down Expand Up @@ -749,12 +753,17 @@ void processCompactionLogLine(String line) {
reconstructionSnapshotGeneration = snapshotLogInfo.snapshotGenerationId;
reconstructionLastSnapshotID = snapshotLogInfo.snapshotId;
} else if (line.startsWith(COMPACTION_LOG_ENTRY_LINE_PREFIX)) {
// Read compaction log entry
// Compaction log entry is like following:
// C sequence_number input_files:output_files
// where input_files and output_files are joined by ','.
String[] lineSpilt = line.split(SPACE_DELIMITER);
if (lineSpilt.length != 3) {
LOG.error("Invalid line in compaction log: {}", line);
return;
}

// Trim the beginning
line = line.substring(COMPACTION_LOG_ENTRY_LINE_PREFIX.length());
String[] io =
line.split(COMPACTION_LOG_ENTRY_INPUT_OUTPUT_FILES_DELIMITER);
String[] io = lineSpilt[2]
.split(COMPACTION_LOG_ENTRY_INPUT_OUTPUT_FILES_DELIMITER);

if (io.length != 2) {
if (line.endsWith(":")) {
Expand Down Expand Up @@ -1162,8 +1171,11 @@ public void pruneOlderSnapshotsWithCompactionHistory() {
Set<String> sstFileNodesRemoved =
pruneSstFileNodesFromDag(lastCompactionSstFiles);

LOG.info("Removing SST files: {} as part of compaction DAG pruning.",
sstFileNodesRemoved);
if (CollectionUtils.isNotEmpty(sstFileNodesRemoved)) {
LOG.info("Removing SST files: {} as part of compaction DAG pruning.",
sstFileNodesRemoved);
}

try (BootstrapStateHandler.Lock lock = getBootstrapStateLock().lock()) {
removeSstFiles(sstFileNodesRemoved);
deleteOlderSnapshotsCompactionFiles(olderSnapshotsLogFilePaths);
Expand Down Expand Up @@ -1471,8 +1483,11 @@ public synchronized void pruneSstFiles() {
.map(node -> node.getFileName())
.collect(Collectors.toSet());

LOG.info("Removing SST files: {} as part of SST file pruning.",
nonLeafSstFiles);
if (CollectionUtils.isNotEmpty(nonLeafSstFiles)) {
LOG.info("Removing SST files: {} as part of SST file pruning.",
nonLeafSstFiles);
}

try (BootstrapStateHandler.Lock lock = getBootstrapStateLock().lock()) {
removeSstFiles(nonLeafSstFiles);
} catch (InterruptedException e) {
Expand Down
59 changes: 30 additions & 29 deletions ...eckpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -282,22 +282,23 @@ public void testGetSSTDiffListWithoutDB(String description,
// Snapshot 0
+ "S 1000 df6410c7-151b-4e90-870e-5ef12875acd5 " + createdTime + " \n"
// Additional "compaction" to trigger and test early exit condition
+ "C 000001,000002:000062\n"
+ "C 1291 000001,000002:000062\n"
// Snapshot 1
+ "S 3008 ef6410c7-151b-4e90-870e-5ef12875acd5 " + createdTime + " \n"
// Regular compaction
+ "C 000068,000062:000069\n"
+ "C 4023 000068,000062:000069\n"
// Trivial move
+ "C 000071,000064,000060,000052:000071,000064,000060,000052\n"
+ "C 000073,000066:000074\n"
+ "C 000082,000076,000069:000083\n"
+ "C 000087,000080,000074:000088\n"
+ "C 5647 000071,000064,000060,000052:000071,000064,000060,000052\n"
+ "C 7658 000073,000066:000074\n"
+ "C 7872 000082,000076,000069:000083\n"
+ "C 9001 000087,000080,000074:000088\n"
// Deletion?
+ "C 000093,000090,000083:\n"
+ "C 12755 000093,000090,000083:\n"
// Snapshot 2
+ "S 14980 e7ad72f8-52df-4430-93f6-0ee91d4a47fd " + createdTime + "\n"
+ "C 000098,000096,000085,000078,000071,000064,000060,000052:000099\n"
+ "C 000105,000095,000088:000107\n"
+ "C 16192 000098,000096,000085,000078,000071,000064,000060,000052"
+ ":000099\n"
+ "C 16762 000105,000095,000088:000107\n"
// Snapshot 3
+ "S 17975 4f084f6e-ed3d-4780-8362-f832303309ea " + createdTime + "\n";

Expand Down Expand Up @@ -921,38 +922,38 @@ private static Stream<Arguments> compactionDagPruningScenarios() {

String compactionLogFile0 = "S 1000 snapshotId0 " +
(currentTimeMillis - MINUTES.toMillis(30)) + " \n";
String compactionLogFile1 = "C 000015,000013,000011,000009:000018,000016," +
"000017\n"
String compactionLogFile1 = "C 1000 000015,000013,000011,000009:000018," +
"000016,000017\n"
+ "S 2000 snapshotId1 " +
(currentTimeMillis - MINUTES.toMillis(24)) + " \n";

String compactionLogFile2 = "C 000018,000016,000017,000026,000024,000022," +
"000020:000027,000030,000028,000031,000029\n"
String compactionLogFile2 = "C 1000 000018,000016,000017,000026,000024," +
"000022,000020:000027,000030,000028,000031,000029\n"
+ "S 3000 snapshotId2 " +
(currentTimeMillis - MINUTES.toMillis(18)) + " \n";

String compactionLogFile3 = "C 000027,000030,000028,000031,000029,000039," +
"000037,000035,000033:000040,000044,000042,000043,000046,000041," +
"000045\n"
String compactionLogFile3 = "C 1000 000027,000030,000028,000031,000029," +
"000039,000037,000035,000033:000040,000044,000042,000043,000046," +
"000041,000045\n"
+ "S 3000 snapshotId3 " +
(currentTimeMillis - MINUTES.toMillis(12)) + " \n";

String compactionLogFile4 = "C 000040,000044,000042,000043,000046,000041," +
"000045,000054,000052,000050,000048:000059,000055,000056,000060," +
"000057,000058\n"
String compactionLogFile4 = "C 1000 000040,000044,000042,000043,000046," +
"000041,000045,000054,000052,000050,000048:000059,000055,000056," +
"000060,000057,000058\n"
+ "S 3000 snapshotId4 " +
(currentTimeMillis - MINUTES.toMillis(6)) + " \n";

String compactionLogFileWithoutSnapshot1 = "C 000015,000013,000011," +
String compactionLogFileWithoutSnapshot1 = "C 1000 000015,000013,000011," +
"000009:000018,000016,000017\n" +
"C 000018,000016,000017,000026,000024,000022,000020:000027,000030," +
"000028,000031,000029\n";
"C 2000 000018,000016,000017,000026,000024,000022,000020" +
":000027,000030,000028,000031,000029\n";

String compactionLogFileWithoutSnapshot2 = "C 000027,000030,000028," +
String compactionLogFileWithoutSnapshot2 = "C 3000 000027,000030,000028," +
"000031,000029,000039,000037,000035,000033:000040,000044,000042," +
"000043,000046,000041,000045\n";

String compactionLogFileWithoutSnapshot3 = "C 000040,000044,000042," +
String compactionLogFileWithoutSnapshot3 = "C 4000 000040,000044,000042," +
"000043,000046,000041,000045,000054,000052,000050,000048:000059," +
"000055,000056,000060,000057,000058\n";

Expand Down Expand Up @@ -1156,19 +1157,19 @@ private static Stream<Arguments> sstFilePruningScenarios() {
Arrays.asList("000015", "000013", "000011", "000009")
),
Arguments.of("Case 2: One level compaction.",
"C 000015,000013,000011,000009:000018,000016,000017\n",
"C 1 000015,000013,000011,000009:000018,000016,000017\n",
Arrays.asList("000015", "000013", "000011", "000009", "000018",
"000016", "000017", "000026", "000024", "000022", "000020"),
Arrays.asList("000015", "000013", "000011", "000009", "000026",
"000024", "000022", "000020")
),
Arguments.of("Case 3: Multi-level compaction.",
"C 000015,000013,000011,000009:000018,000016,000017\n" +
"C 000018,000016,000017,000026,000024,000022,000020:000027," +
"C 1 000015,000013,000011,000009:000018,000016,000017\n" +
"C 2 000018,000016,000017,000026,000024,000022,000020:000027," +
"000030,000028,000031,000029\n" +
"C 000027,000030,000028,000031,000029,000039,000037,000035," +
"C 3 000027,000030,000028,000031,000029,000039,000037,000035," +
"000033:000040,000044,000042,000043,000046,000041,000045\n" +
"C 000040,000044,000042,000043,000046,000041,000045,000054," +
"C 4 000040,000044,000042,000043,000046,000041,000045,000054," +
"000052,000050,000048:000059,000055,000056,000060,000057," +
"000058\n",
Arrays.asList("000015", "000013", "000011", "000009", "000018",
Expand Down