Skip to content

Commit 352e59c

Browse files
dnhatndnhatn
authored
Fix doc_stats and segment_stats of ReadOnlyEngine (#53345)
We can't always have the same segment stats and doc stats between InternalEngine and ReadOnlyEngine if there are some fully deleted segments. ReadOnlyEngine always filters out them. InternalEngine, however, will keep them if peer recovery retention leases exist or the number of the retaining operations is non-zero. This change reverts the fix in #51331 and uses the wrapped reader to calculate the segment stats and doc stats. For the test, we need to disable the extra retaining soft-deletes operations. Closes #51303
1 parent 2b96a6b commit 352e59c

File tree

4 files changed

+38
-48
lines changed

4 files changed

+38
-48
lines changed

‎server/src/main/java/org/elasticsearch/index/engine/NoOpEngine.java‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import org.elasticsearch.common.lucene.Lucene;
3030
import org.elasticsearch.common.util.concurrent.ReleasableLock;
3131
import org.elasticsearch.index.seqno.SequenceNumbers;
32+
import org.elasticsearch.index.shard.DocsStats;
3233
import org.elasticsearch.index.store.Store;
3334
import org.elasticsearch.index.translog.Translog;
3435
import org.elasticsearch.index.translog.TranslogConfig;
@@ -49,18 +50,19 @@
4950
*/
5051
public final class NoOpEngine extends ReadOnlyEngine {
5152

52-
private final SegmentsStats stats;
53+
private final SegmentsStats segmentsStats;
54+
private final DocsStats docsStats;
5355

5456
public NoOpEngine(EngineConfig config) {
5557
super(config, null, null, true, Function.identity());
56-
this.stats = new SegmentsStats();
58+
this.segmentsStats = new SegmentsStats();
5759
Directory directory = store.directory();
58-
// Do not wrap soft-deletes reader when calculating segment stats as the wrapper might filter out fully deleted segments.
59-
try (DirectoryReader reader = openDirectory(directory, false)) {
60+
try (DirectoryReader reader = openDirectory(directory)) {
6061
for (LeafReaderContext ctx : reader.getContext().leaves()) {
6162
SegmentReader segmentReader = Lucene.segmentReader(ctx.reader());
62-
fillSegmentStats(segmentReader, true, stats);
63+
fillSegmentStats(segmentReader, true, segmentsStats);
6364
}
65+
this.docsStats = docsStats(reader);
6466
} catch (IOException e) {
6567
throw new UncheckedIOException(e);
6668
}
@@ -117,7 +119,7 @@ public CacheHelper getReaderCacheHelper() {
117119
public SegmentsStats segmentsStats(boolean includeSegmentFileSizes, boolean includeUnloadedSegments) {
118120
if (includeUnloadedSegments) {
119121
final SegmentsStats stats = new SegmentsStats();
120-
stats.add(this.stats);
122+
stats.add(this.segmentsStats);
121123
if (includeSegmentFileSizes == false) {
122124
stats.clearFileSizes();
123125
}
@@ -127,6 +129,11 @@ public SegmentsStats segmentsStats(boolean includeSegmentFileSizes, boolean incl
127129
}
128130
}
129131

132+
@Override
133+
public DocsStats docStats() {
134+
return docsStats;
135+
}
136+
130137
/**
131138
* This implementation will trim existing translog files using a {@link TranslogDeletionPolicy}
132139
* that retains nothing but the last translog generation from safe commit.

‎server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java‎

Lines changed: 2 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@
2222
import org.apache.lucene.index.DirectoryReader;
2323
import org.apache.lucene.index.IndexCommit;
2424
import org.apache.lucene.index.IndexWriter;
25-
import org.apache.lucene.index.SegmentCommitInfo;
2625
import org.apache.lucene.index.SegmentInfos;
2726
import org.apache.lucene.index.SoftDeletesDirectoryReaderWrapper;
2827
import org.apache.lucene.search.ReferenceManager;
@@ -36,7 +35,6 @@
3635
import org.elasticsearch.index.mapper.MapperService;
3736
import org.elasticsearch.index.seqno.SeqNoStats;
3837
import org.elasticsearch.index.seqno.SequenceNumbers;
39-
import org.elasticsearch.index.shard.DocsStats;
4038
import org.elasticsearch.index.store.Store;
4139
import org.elasticsearch.index.translog.Translog;
4240
import org.elasticsearch.index.translog.TranslogConfig;
@@ -76,7 +74,6 @@ public class ReadOnlyEngine extends Engine {
7674
private final ElasticsearchReaderManager readerManager;
7775
private final IndexCommit indexCommit;
7876
private final Lock indexWriterLock;
79-
private final DocsStats docsStats;
8077
private final RamAccountingRefreshListener refreshListener;
8178
private final SafeCommitInfo safeCommitInfo;
8279
private final CompletionStatsCache completionStatsCache;
@@ -119,7 +116,6 @@ public ReadOnlyEngine(EngineConfig config, SeqNoStats seqNoStats, TranslogStats
119116
this.indexCommit = Lucene.getIndexCommit(lastCommittedSegmentInfos, directory);
120117
reader = wrapReader(open(indexCommit), readerWrapperFunction);
121118
readerManager = new ElasticsearchReaderManager(reader, refreshListener);
122-
this.docsStats = docsStats(lastCommittedSegmentInfos);
123119
assert translogStats != null || obtainLock : "mutiple translogs instances should not be opened at the same time";
124120
this.translogStats = translogStats != null ? translogStats : translogStats(config, lastCommittedSegmentInfos);
125121
this.indexWriterLock = indexWriterLock;
@@ -182,24 +178,6 @@ protected DirectoryReader open(IndexCommit commit) throws IOException {
182178
return new SoftDeletesDirectoryReaderWrapper(DirectoryReader.open(commit, OFF_HEAP_READER_ATTRIBUTES), Lucene.SOFT_DELETES_FIELD);
183179
}
184180

185-
private DocsStats docsStats(final SegmentInfos lastCommittedSegmentInfos) {
186-
long numDocs = 0;
187-
long numDeletedDocs = 0;
188-
long sizeInBytes = 0;
189-
if (lastCommittedSegmentInfos != null) {
190-
for (SegmentCommitInfo segmentCommitInfo : lastCommittedSegmentInfos) {
191-
numDocs += segmentCommitInfo.info.maxDoc() - segmentCommitInfo.getDelCount() - segmentCommitInfo.getSoftDelCount();
192-
numDeletedDocs += segmentCommitInfo.getDelCount() + segmentCommitInfo.getSoftDelCount();
193-
try {
194-
sizeInBytes += segmentCommitInfo.sizeInBytes();
195-
} catch (IOException e) {
196-
throw new UncheckedIOException("Failed to get size for [" + segmentCommitInfo.info.name + "]", e);
197-
}
198-
}
199-
}
200-
return new DocsStats(numDocs, numDeletedDocs, sizeInBytes);
201-
}
202-
203181
@Override
204182
protected void closeNoLock(String reason, CountDownLatch closedLatch) {
205183
if (isClosed.compareAndSet(false, true)) {
@@ -463,11 +441,6 @@ public void trimOperationsFromTranslog(long belowTerm, long aboveSeqNo) {
463441
public void maybePruneDeletes() {
464442
}
465443

466-
@Override
467-
public DocsStats docStats() {
468-
return docsStats;
469-
}
470-
471444
@Override
472445
public void updateMaxUnsafeAutoIdTimestamp(long newTimestamp) {
473446

@@ -511,13 +484,9 @@ public void advanceMaxSeqNoOfUpdatesOrDeletes(long maxSeqNoOfUpdatesOnPrimary) {
511484
maxSeqNoOfUpdatesOnPrimary + ">" + getMaxSeqNoOfUpdatesOrDeletes();
512485
}
513486

514-
protected static DirectoryReader openDirectory(Directory directory, boolean wrapSoftDeletes) throws IOException {
487+
protected static DirectoryReader openDirectory(Directory directory) throws IOException {
515488
final DirectoryReader reader = DirectoryReader.open(directory, OFF_HEAP_READER_ATTRIBUTES);
516-
if (wrapSoftDeletes) {
517-
return new SoftDeletesDirectoryReaderWrapper(reader, Lucene.SOFT_DELETES_FIELD);
518-
} else {
519-
return reader;
520-
}
489+
return new SoftDeletesDirectoryReaderWrapper(reader, Lucene.SOFT_DELETES_FIELD);
521490
}
522491

523492
@Override

‎server/src/test/java/org/elasticsearch/index/engine/NoOpEngineTests.java‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import org.apache.lucene.index.IndexReader;
2424
import org.apache.lucene.index.NoMergePolicy;
2525
import org.apache.lucene.store.LockObtainFailedException;
26+
import org.elasticsearch.cluster.metadata.IndexMetaData;
2627
import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
2728
import org.elasticsearch.cluster.routing.ShardRouting;
2829
import org.elasticsearch.cluster.routing.ShardRoutingState;
@@ -101,10 +102,16 @@ public void testNoopAfterRegularEngine() throws IOException {
101102

102103
public void testNoOpEngineStats() throws Exception {
103104
IOUtils.close(engine, store);
105+
Settings.Builder settings = Settings.builder()
106+
.put(defaultSettings.getSettings())
107+
.put(IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING.getKey(), 0);
108+
IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(
109+
IndexMetaData.builder(defaultSettings.getIndexMetaData()).settings(settings).build());
110+
104111
final AtomicLong globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED);
105112
try (Store store = createStore()) {
106113
Path translogPath = createTempDir();
107-
EngineConfig config = config(defaultSettings, store, translogPath, NoMergePolicy.INSTANCE, null, null, globalCheckpoint::get);
114+
EngineConfig config = config(indexSettings, store, translogPath, NoMergePolicy.INSTANCE, null, null, globalCheckpoint::get);
108115
final int numDocs = scaledRandomIntBetween(10, 3000);
109116
int deletions = 0;
110117
try (InternalEngine engine = createEngine(config)) {

‎x-pack/plugin/frozen-indices/src/main/java/org/elasticsearch/index/engine/FrozenEngine.java‎

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
3737
import org.elasticsearch.common.settings.Setting;
3838
import org.elasticsearch.core.internal.io.IOUtils;
39+
import org.elasticsearch.index.shard.DocsStats;
3940
import org.elasticsearch.index.shard.SearchOperationListener;
4041
import org.elasticsearch.search.internal.SearchContext;
4142
import org.elasticsearch.transport.TransportRequest;
@@ -69,7 +70,8 @@
6970
public final class FrozenEngine extends ReadOnlyEngine {
7071
public static final Setting<Boolean> INDEX_FROZEN = Setting.boolSetting("index.frozen", false, Setting.Property.IndexScope,
7172
Setting.Property.PrivateIndex);
72-
private final SegmentsStats stats;
73+
private final SegmentsStats segmentsStats;
74+
private final DocsStats docsStats;
7375
private volatile ElasticsearchDirectoryReader lastOpenedReader;
7476
private final ElasticsearchDirectoryReader canMatchReader;
7577

@@ -78,15 +80,15 @@ public FrozenEngine(EngineConfig config) {
7880

7981
boolean success = false;
8082
Directory directory = store.directory();
81-
// Do not wrap soft-deletes reader when calculating segment stats as the wrapper might filter out fully deleted segments.
82-
try (DirectoryReader reader = openDirectory(directory, false)) {
83-
// we record the segment stats here - that's what the reader needs when it's open and it give the user
83+
try (DirectoryReader reader = openDirectory(directory)) {
84+
// we record the segment stats and doc stats here - that's what the reader needs when it's open and it give the user
8485
// an idea of what it can save when it's closed
85-
this.stats = new SegmentsStats();
86+
this.segmentsStats = new SegmentsStats();
8687
for (LeafReaderContext ctx : reader.getContext().leaves()) {
8788
SegmentReader segmentReader = Lucene.segmentReader(ctx.reader());
88-
fillSegmentStats(segmentReader, true, stats);
89+
fillSegmentStats(segmentReader, true, segmentsStats);
8990
}
91+
this.docsStats = docsStats(reader);
9092
final DirectoryReader wrappedReader = new SoftDeletesDirectoryReaderWrapper(reader, Lucene.SOFT_DELETES_FIELD);
9193
canMatchReader = ElasticsearchDirectoryReader.wrap(
9294
new RewriteCachingDirectoryReader(directory, wrappedReader.leaves()), config.getShardId());
@@ -170,7 +172,7 @@ private synchronized ElasticsearchDirectoryReader getOrOpenReader() throws IOExc
170172
for (ReferenceManager.RefreshListener listeners : config ().getInternalRefreshListener()) {
171173
listeners.beforeRefresh();
172174
}
173-
final DirectoryReader dirReader = openDirectory(engineConfig.getStore().directory(), true);
175+
final DirectoryReader dirReader = openDirectory(engineConfig.getStore().directory());
174176
reader = lastOpenedReader = wrapReader(dirReader, Function.identity());
175177
processReader(reader);
176178
reader.getReaderCacheHelper().addClosedListener(this::onReaderClosed);
@@ -594,7 +596,7 @@ public LeafReader getDelegate() {
594596
public SegmentsStats segmentsStats(boolean includeSegmentFileSizes, boolean includeUnloadedSegments) {
595597
if (includeUnloadedSegments) {
596598
final SegmentsStats stats = new SegmentsStats();
597-
stats.add(this.stats);
599+
stats.add(this.segmentsStats);
598600
if (includeSegmentFileSizes == false) {
599601
stats.clearFileSizes();
600602
}
@@ -605,6 +607,11 @@ public SegmentsStats segmentsStats(boolean includeSegmentFileSizes, boolean incl
605607

606608
}
607609

610+
@Override
611+
public DocsStats docStats() {
612+
return docsStats;
613+
}
614+
608615
synchronized boolean isReaderOpen() {
609616
return lastOpenedReader != null;
610617
} // this is mainly for tests

0 commit comments

Comments
 (0)