Skip to content

Commit c4bccea

Browse files
dnhatndnhatn
committed
Use local checkpoint of safe commit to calculate committed stats
1 parent 6572ed4 commit c4bccea

File tree

7 files changed

+10
-47
lines changed

7 files changed

+10
-47
lines changed

‎server/src/main/java/org/elasticsearch/index/engine/CombinedDeletionPolicy.java‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -123,9 +123,7 @@ private void updateRetentionPolicy() throws IOException {
123123
assert safeCommit.isDeleted() == false : "The safe commit must not be deleted";
124124
assert lastCommit.isDeleted() == false : "The last commit must not be deleted";
125125
final long localCheckpointOfSafeCommit = Long.parseLong(safeCommit.getUserData().get(SequenceNumbers.LOCAL_CHECKPOINT_KEY));
126-
final long localCheckpointOfLastCommit = Long.parseLong(lastCommit.getUserData().get(SequenceNumbers.LOCAL_CHECKPOINT_KEY));
127126
softDeletesPolicy.setLocalCheckpointOfSafeCommit(localCheckpointOfSafeCommit);
128-
translogDeletionPolicy.setLocalCheckpointOfLastCommit(localCheckpointOfLastCommit);
129127
translogDeletionPolicy.setLocalCheckpointOfSafeCommit(localCheckpointOfSafeCommit);
130128
}
131129

‎server/src/main/java/org/elasticsearch/index/engine/NoOpEngine.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -150,7 +150,6 @@ public void trimUnreferencedTranslogFiles() {
150150
final long localCheckpoint = Long.parseLong(commitUserData.get(SequenceNumbers.LOCAL_CHECKPOINT_KEY));
151151
final TranslogDeletionPolicy translogDeletionPolicy = new TranslogDeletionPolicy();
152152
translogDeletionPolicy.setLocalCheckpointOfSafeCommit(localCheckpoint);
153-
translogDeletionPolicy.setLocalCheckpointOfLastCommit(localCheckpoint);
154153
try (Translog translog = new Translog(translogConfig, translogUuid, translogDeletionPolicy,
155154
engineConfig.getGlobalCheckpointSupplier(), engineConfig.getPrimaryTermSupplier(), seqNo -> {})) {
156155
translog.trimUnreferencedReaders();

‎server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -223,7 +223,7 @@ private static TranslogStats translogStats(final EngineConfig config, final Segm
223223
final TranslogConfig translogConfig = config.getTranslogConfig();
224224
final TranslogDeletionPolicy translogDeletionPolicy = new TranslogDeletionPolicy();
225225
final long localCheckpoint = Long.parseLong(infos.getUserData().get(SequenceNumbers.LOCAL_CHECKPOINT_KEY));
226-
translogDeletionPolicy.setLocalCheckpointOfLastCommit(localCheckpoint);
226+
translogDeletionPolicy.setLocalCheckpointOfSafeCommit(localCheckpoint);
227227
try (Translog translog = new Translog(translogConfig, translogUuid, translogDeletionPolicy, config.getGlobalCheckpointSupplier(),
228228
config.getPrimaryTermSupplier(), seqNo -> {})
229229
) {

‎server/src/main/java/org/elasticsearch/index/translog/Translog.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -843,7 +843,7 @@ protected void closeOnTragicEvent(final Exception ex) {
843843
public TranslogStats stats() {
844844
// acquire lock to make the two numbers roughly consistent (no file change half way)
845845
try (ReleasableLock lock = readLock.acquire()) {
846-
final long uncommittedGen = minGenerationForSeqNo(deletionPolicy.getLocalCheckpointOfLastCommit() + 1, current, readers);
846+
final long uncommittedGen = minGenerationForSeqNo(deletionPolicy.getLocalCheckpointOfSafeCommit() + 1, current, readers);
847847
return new TranslogStats(totalOperations(), sizeInBytes(), totalOperationsByMinGen(uncommittedGen),
848848
sizeInBytesByMinGen(uncommittedGen), earliestLastModifiedAge());
849849
}

‎server/src/main/java/org/elasticsearch/index/translog/TranslogDeletionPolicy.java‎

Lines changed: 0 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,6 @@ public void assertNoOpenTranslogRefs() {
4747
*/
4848
private final Map<Long, Counter> translogRefCounts = new HashMap<>();
4949
private long localCheckpointOfSafeCommit = SequenceNumbers.NO_OPS_PERFORMED;
50-
private long localCheckpointOfLastCommit = SequenceNumbers.NO_OPS_PERFORMED;
5150

5251

5352
public TranslogDeletionPolicy() {
@@ -66,14 +65,6 @@ public synchronized void setLocalCheckpointOfSafeCommit(long newCheckpoint) {
6665
this.localCheckpointOfSafeCommit = newCheckpoint;
6766
}
6867

69-
public synchronized void setLocalCheckpointOfLastCommit(long newCheckpoint) {
70-
if (newCheckpoint < this.localCheckpointOfLastCommit) {
71-
throw new IllegalArgumentException("local checkpoint of the last commit can't go backwards: " +
72-
"current [" + this.localCheckpointOfLastCommit + "] new [" + newCheckpoint + "]");
73-
}
74-
this.localCheckpointOfLastCommit = newCheckpoint;
75-
}
76-
7768
/**
7869
* acquires the basis generation for a new snapshot. Any translog generation above, and including, the returned generation
7970
* will not be deleted until the returned {@link Releasable} is closed.
@@ -134,12 +125,6 @@ public synchronized long getLocalCheckpointOfSafeCommit() {
134125
return localCheckpointOfSafeCommit;
135126
}
136127

137-
/**
138-
* Returns the local checkpoint of the last commit. This value is used to calculate the translog stats.
139-
*/
140-
public synchronized long getLocalCheckpointOfLastCommit() {
141-
return localCheckpointOfLastCommit;
142-
}
143128

144129
synchronized long getTranslogRefCount(long gen) {
145130
final Counter counter = translogRefCounts.get(gen);

‎server/src/test/java/org/elasticsearch/index/engine/CombinedDeletionPolicyTests.java‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,6 @@ public void testKeepCommitsAfterGlobalCheckpoint() throws Exception {
8484
}
8585
}
8686
assertThat(translogPolicy.getLocalCheckpointOfSafeCommit(), equalTo(getLocalCheckpoint(commitList.get(keptIndex))));
87-
assertThat(translogPolicy.getLocalCheckpointOfLastCommit(), equalTo(getLocalCheckpoint(commitList.get(commitList.size() - 1))));
8887
assertThat(softDeletesPolicy.getMinRetainedSeqNo(),
8988
equalTo(Math.max(NO_OPS_PERFORMED,
9089
Math.min(getLocalCheckpoint(commitList.get(keptIndex)) + 1, globalCheckpoint.get() + 1 - extraRetainedOps))));
@@ -149,7 +148,6 @@ public void testAcquireIndexCommit() throws Exception {
149148
snapshottingCommits.forEach(snapshot -> assertThat(snapshot.isDeleted(), equalTo(false)));
150149
// We don't need to retain translog for snapshotting commits.
151150
assertThat(translogPolicy.getLocalCheckpointOfSafeCommit(), equalTo(getLocalCheckpoint(commitList.get(safeIndex))));
152-
assertThat(translogPolicy.getLocalCheckpointOfLastCommit(), equalTo(getLocalCheckpoint(commitList.get(commitList.size() - 1))));
153151
assertThat(softDeletesPolicy.getMinRetainedSeqNo(), equalTo(
154152
Math.max(NO_OPS_PERFORMED,
155153
Math.min(getLocalCheckpoint(commitList.get(safeIndex)) + 1, globalCheckpoint.get() + 1 - extraRetainedOps))));
@@ -163,7 +161,6 @@ public void testAcquireIndexCommit() throws Exception {
163161
}
164162
assertThat(commitList.get(commitList.size() - 1).isDeleted(), equalTo(false));
165163
assertThat(translogPolicy.getLocalCheckpointOfSafeCommit(), equalTo(getLocalCheckpoint(commitList.get(commitList.size() - 1))));
166-
assertThat(translogPolicy.getLocalCheckpointOfLastCommit(), equalTo(getLocalCheckpoint(commitList.get(commitList.size() - 1))));
167164
IndexCommit safeCommit = CombinedDeletionPolicy.findSafeCommitPoint(commitList, globalCheckpoint.get());
168165
assertThat(softDeletesPolicy.getMinRetainedSeqNo(), equalTo(
169166
Math.max(NO_OPS_PERFORMED, Math.min(getLocalCheckpoint(safeCommit) + 1, globalCheckpoint.get() + 1 - extraRetainedOps))));
@@ -224,7 +221,6 @@ public void testCheckUnreferencedCommits() throws Exception {
224221
if (safeCommitIndex == commitList.size() - 1) {
225222
// Safe commit is the last commit - no need to clean up
226223
assertThat(translogPolicy.getLocalCheckpointOfSafeCommit(), equalTo(getLocalCheckpoint(commitList.get(commitList.size() - 1))));
227-
assertThat(translogPolicy.getLocalCheckpointOfLastCommit(), equalTo(getLocalCheckpoint(commitList.get(commitList.size() - 1))));
228224
assertThat(indexPolicy.hasUnreferencedCommits(), equalTo(false));
229225
} else {
230226
// Advanced but not enough for any commit after the safe commit becomes safe
@@ -242,7 +238,6 @@ public void testCheckUnreferencedCommits() throws Exception {
242238
indexPolicy.onCommit(commitList);
243239
// Safe commit is the last commit - no need to clean up
244240
assertThat(translogPolicy.getLocalCheckpointOfSafeCommit(), equalTo(getLocalCheckpoint(commitList.get(commitList.size() - 1))));
245-
assertThat(translogPolicy.getLocalCheckpointOfLastCommit(), equalTo(getLocalCheckpoint(commitList.get(commitList.size() - 1))));
246241
assertThat(indexPolicy.hasUnreferencedCommits(), equalTo(false));
247242
}
248243
}

‎server/src/test/java/org/elasticsearch/index/translog/TranslogTests.java‎

Lines changed: 8 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -165,7 +165,6 @@ protected void afterIfSuccessful() throws Exception {
165165

166166
if (translog.isOpen()) {
167167
if (translog.currentFileGeneration() > 1) {
168-
translog.getDeletionPolicy().setLocalCheckpointOfLastCommit(Long.MAX_VALUE);
169168
translog.getDeletionPolicy().setLocalCheckpointOfSafeCommit(Long.MAX_VALUE);
170169
translog.trimUnreferencedReaders();
171170
assertFileDeleted(translog, translog.currentFileGeneration() - 1);
@@ -479,7 +478,6 @@ public void testStats() throws IOException {
479478
}
480479
}
481480
translog.getDeletionPolicy().setLocalCheckpointOfSafeCommit(randomLongBetween(3, Long.MAX_VALUE));
482-
translog.getDeletionPolicy().setLocalCheckpointOfLastCommit(randomLongBetween(3, Long.MAX_VALUE));
483481
translog.trimUnreferencedReaders();
484482
{
485483
final TranslogStats stats = stats();
@@ -506,8 +504,7 @@ public void testUncommittedOperations() throws Exception {
506504
}
507505
assertThat(translog.stats().getUncommittedOperations(), equalTo(uncommittedOps));
508506
if (frequently()) {
509-
deletionPolicy.setLocalCheckpointOfSafeCommit(randomLongBetween(deletionPolicy.getLocalCheckpointOfSafeCommit(), i));
510-
deletionPolicy.setLocalCheckpointOfLastCommit(i);
507+
deletionPolicy.setLocalCheckpointOfSafeCommit(i);
511508
assertThat(translog.stats().getUncommittedOperations(), equalTo(operationsInLastGen));
512509
uncommittedOps = operationsInLastGen;
513510
}
@@ -625,7 +622,7 @@ public void testSnapshotWithNewTranslog() throws IOException {
625622

626623
Translog.Snapshot snapshot2 = translog.newSnapshot();
627624
toClose.add(snapshot2);
628-
translog.getDeletionPolicy().setLocalCheckpointOfLastCommit(2);
625+
translog.getDeletionPolicy().setLocalCheckpointOfSafeCommit(2);
629626
assertThat(snapshot2, containsOperationsInAnyOrder(ops));
630627
assertThat(snapshot2.totalOperations(), equalTo(ops.size()));
631628
} finally {
@@ -972,7 +969,6 @@ public void doRun() throws BrokenBarrierException, InterruptedException, IOExcep
972969
translog.rollGeneration();
973970
// expose the new checkpoint (simulating a commit), before we trim the translog
974971
lastCommittedLocalCheckpoint.set(localCheckpoint);
975-
deletionPolicy.setLocalCheckpointOfLastCommit(localCheckpoint);
976972
deletionPolicy.setLocalCheckpointOfSafeCommit(localCheckpoint);
977973
translog.trimUnreferencedReaders();
978974
}
@@ -1360,7 +1356,7 @@ public void testBasicRecovery() throws IOException {
13601356
Integer.toString(op).getBytes(Charset.forName("UTF-8")))));
13611357
final boolean commit = commitOften ? frequently() : rarely();
13621358
if (commit && op < translogOperations - 1) {
1363-
translog.getDeletionPolicy().setLocalCheckpointOfLastCommit(op);
1359+
translog.getDeletionPolicy().setLocalCheckpointOfSafeCommit(op);
13641360
minUncommittedOp = op + 1;
13651361
translogGeneration = translog.getGeneration();
13661362
}
@@ -1816,7 +1812,6 @@ public void testOpenForeignTranslog() throws IOException {
18161812
if (randomBoolean()) {
18171813
translog.rollGeneration();
18181814
translog.getDeletionPolicy().setLocalCheckpointOfSafeCommit(op);
1819-
translog.getDeletionPolicy().setLocalCheckpointOfLastCommit(op);
18201815
translog.trimUnreferencedReaders();
18211816
firstUncommitted = op + 1;
18221817
}
@@ -2219,7 +2214,6 @@ public void testRecoveryFromAFutureGenerationCleansUp() throws IOException {
22192214
TranslogConfig config = translog.getConfig();
22202215
final TranslogDeletionPolicy deletionPolicy = new TranslogDeletionPolicy();
22212216
deletionPolicy.setLocalCheckpointOfSafeCommit(localCheckpoint);
2222-
deletionPolicy.setLocalCheckpointOfLastCommit(randomLongBetween(localCheckpoint, Long.MAX_VALUE));
22232217
translog = new Translog(config, translog.getTranslogUUID(), deletionPolicy,
22242218
() -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, seqNo -> {});
22252219
assertThat(translog.getMinFileGeneration(), equalTo(1L));
@@ -2268,7 +2262,6 @@ public void testRecoveryFromFailureOnTrimming() throws IOException {
22682262
}
22692263
}
22702264
deletionPolicy.setLocalCheckpointOfSafeCommit(localCheckpoint);
2271-
deletionPolicy.setLocalCheckpointOfLastCommit(randomLongBetween(localCheckpoint, op));
22722265
minGenForRecovery = translog.getMinGenerationForSeqNo(localCheckpoint + 1).translogFileGeneration;
22732266
fail.failRandomly();
22742267
try {
@@ -2278,7 +2271,6 @@ public void testRecoveryFromFailureOnTrimming() throws IOException {
22782271
}
22792272
}
22802273
final TranslogDeletionPolicy deletionPolicy = new TranslogDeletionPolicy();
2281-
deletionPolicy.setLocalCheckpointOfLastCommit(randomLongBetween(localCheckpoint, Long.MAX_VALUE));
22822274
deletionPolicy.setLocalCheckpointOfSafeCommit(localCheckpoint);
22832275
try (Translog translog = new Translog(config, translogUUID, deletionPolicy,
22842276
() -> SequenceNumbers.NO_OPS_PERFORMED, primaryTerm::get, seqNo -> {})) {
@@ -2626,7 +2618,6 @@ public void testWithRandomException() throws IOException {
26262618
unsynced.clear();
26272619
failableTLog.rollGeneration();
26282620
committing = true;
2629-
failableTLog.getDeletionPolicy().setLocalCheckpointOfLastCommit(opsAdded);
26302621
failableTLog.getDeletionPolicy().setLocalCheckpointOfSafeCommit(opsAdded);
26312622
syncedDocs.clear();
26322623
failableTLog.trimUnreferencedReaders();
@@ -2671,7 +2662,6 @@ public void testWithRandomException() throws IOException {
26712662
if (randomBoolean()) {
26722663
try {
26732664
TranslogDeletionPolicy deletionPolicy = new TranslogDeletionPolicy();
2674-
deletionPolicy.setLocalCheckpointOfLastCommit(localCheckpointOfSafeCommit);
26752665
deletionPolicy.setLocalCheckpointOfSafeCommit(localCheckpointOfSafeCommit);
26762666
IOUtils.close(getFailableTranslog(fail, config, randomBoolean(), false, generationUUID, deletionPolicy));
26772667
} catch (TranslogException | MockDirectoryWrapper.FakeIOException ex) {
@@ -2684,7 +2674,6 @@ public void testWithRandomException() throws IOException {
26842674
fail.failNever(); // we don't wanna fail here but we might since we write a new checkpoint and create a new tlog file
26852675
TranslogDeletionPolicy deletionPolicy = new TranslogDeletionPolicy();
26862676
deletionPolicy.setLocalCheckpointOfSafeCommit(localCheckpointOfSafeCommit);
2687-
deletionPolicy.setLocalCheckpointOfLastCommit(localCheckpointOfSafeCommit);
26882677
if (generationUUID == null) {
26892678
// we never managed to successfully create a translog, make it
26902679
generationUUID = Translog.createEmptyTranslog(config.getTranslogPath(),
@@ -2868,12 +2857,15 @@ public void testRollGeneration() throws Exception {
28682857
Stream.concat(translog.getReaders().stream(), Stream.of(translog.getCurrent()))
28692858
.filter(r -> r.getCheckpoint().minSeqNo >= 0)
28702859
.collect(Collectors.toList()));
2860+
int retainedOps = Stream.concat(translog.getReaders().stream(), Stream.of(translog.getCurrent()))
2861+
.filter(r -> r.getCheckpoint().generation >= minRetainedReader.generation)
2862+
.mapToInt(r -> r.getCheckpoint().numOps)
2863+
.sum();
28712864
deletionPolicy.setLocalCheckpointOfSafeCommit(
28722865
randomLongBetween(minRetainedReader.getCheckpoint().minSeqNo, minRetainedReader.getCheckpoint().maxSeqNo) - 1);
2873-
deletionPolicy.setLocalCheckpointOfLastCommit(seqNo);
28742866
translog.trimUnreferencedReaders();
28752867
assertThat(translog.currentFileGeneration(), equalTo(generation + rolls));
2876-
assertThat(translog.stats().getUncommittedOperations(), equalTo(0));
2868+
assertThat(translog.stats().getUncommittedOperations(), equalTo(retainedOps));
28772869
// immediate cleanup
28782870
for (long i = 0; i < minRetainedReader.generation; i++) {
28792871
assertFileDeleted(translog, i);
@@ -2967,7 +2959,6 @@ public void testSimpleCommit() throws IOException {
29672959
}
29682960
}
29692961
translog.getDeletionPolicy().setLocalCheckpointOfSafeCommit(randomLongBetween(0, operations));
2970-
translog.getDeletionPolicy().setLocalCheckpointOfLastCommit(randomLongBetween(operations, Long.MAX_VALUE));
29712962
}
29722963

29732964
public void testAcquiredLockIsPassedToDeletionPolicy() throws IOException {
@@ -2981,8 +2972,6 @@ public void testAcquiredLockIsPassedToDeletionPolicy() throws IOException {
29812972
if (rarely()) {
29822973
translog.getDeletionPolicy().setLocalCheckpointOfSafeCommit(
29832974
randomLongBetween(deletionPolicy.getLocalCheckpointOfSafeCommit(), i));
2984-
translog.getDeletionPolicy().setLocalCheckpointOfLastCommit(
2985-
randomLongBetween(deletionPolicy.getLocalCheckpointOfLastCommit(), Long.MAX_VALUE));
29862975
}
29872976
if (frequently()) {
29882977
long minGen;
@@ -3136,9 +3125,6 @@ public void testMaxSeqNo() throws Exception {
31363125
translog.sync();
31373126
assertThat(translog.getMaxSeqNo(),
31383127
equalTo(maxSeqNoPerGeneration.isEmpty() ? SequenceNumbers.NO_OPS_PERFORMED : Collections.max(maxSeqNoPerGeneration.values())));
3139-
if (randomBoolean()) {
3140-
translog.getDeletionPolicy().setLocalCheckpointOfLastCommit(randomFrom(maxSeqNoPerGeneration.values()));
3141-
}
31423128
long expectedMaxSeqNo = maxSeqNoPerGeneration.entrySet().stream()
31433129
.filter(e -> e.getKey() >= translog.getMinFileGeneration()).mapToLong(e -> e.getValue())
31443130
.max().orElse(SequenceNumbers.NO_OPS_PERFORMED);

0 commit comments

Comments
 (0)