Skip to content

Commit f23ed61

Browse files
s1monws1monw
authored
Skip shard refreshes if shard is search idle (#27500)
Today we refresh automatically in the background by default very second. This default behavior has a significant impact on indexing performance if the refreshes are not needed. This change introduces a notion of a shard being `search idle` which a shard transitions to after (default) `30s` without any access to an external searcher. Once a shard is search idle all scheduled refreshes will be skipped unless there are any refresh listeners registered. If a search happens on a `serach idle` shard the search request _park_ on a refresh listener and will be executed once the next scheduled refresh occurs. This will also turn the shard into the `non-idle` state immediately. This behavior is only applied if there is no explicit refresh interval set.
1 parent 8d6bfe5 commit f23ed61

File tree

13 files changed

+507
-61
lines changed

13 files changed

+507
-61
lines changed

‎core/src/main/java/org/elasticsearch/action/explain/TransportExplainAction.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,10 @@
3333
import org.elasticsearch.common.inject.Inject;
3434
import org.elasticsearch.common.lease.Releasables;
3535
import org.elasticsearch.common.settings.Settings;
36+
import org.elasticsearch.index.IndexService;
3637
import org.elasticsearch.index.engine.Engine;
3738
import org.elasticsearch.index.get.GetResult;
39+
import org.elasticsearch.index.shard.IndexShard;
3840
import org.elasticsearch.index.shard.ShardId;
3941
import org.elasticsearch.search.SearchService;
4042
import org.elasticsearch.search.internal.AliasFilter;
@@ -86,6 +88,19 @@ protected void resolveRequest(ClusterState state, InternalRequest request) {
8688
}
8789
}
8890

91+
@Override
92+
protected void asyncShardOperation(ExplainRequest request, ShardId shardId, ActionListener<ExplainResponse> listener) throws IOException {
93+
IndexService indexService = searchService.getIndicesService().indexServiceSafe(shardId.getIndex());
94+
IndexShard indexShard = indexService.getShard(shardId.id());
95+
indexShard.awaitShardSearchActive(b -> {
96+
try {
97+
super.asyncShardOperation(request, shardId, listener);
98+
} catch (Exception ex) {
99+
listener.onFailure(ex);
100+
}
101+
});
102+
}
103+
89104
@Override
90105
protected ExplainResponse shardOperation(ExplainRequest request, ShardId shardId) throws IOException {
91106
ShardSearchLocalRequest shardSearchLocalRequest = new ShardSearchLocalRequest(shardId,

‎core/src/main/java/org/elasticsearch/action/get/TransportGetAction.java‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,13 +19,13 @@
1919

2020
package org.elasticsearch.action.get;
2121

22+
import org.elasticsearch.action.ActionListener;
2223
import org.elasticsearch.action.RoutingMissingException;
2324
import org.elasticsearch.action.support.ActionFilters;
2425
import org.elasticsearch.action.support.single.shard.TransportSingleShardAction;
2526
import org.elasticsearch.cluster.ClusterState;
2627
import org.elasticsearch.cluster.metadata.IndexMetaData;
2728
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
28-
import org.elasticsearch.cluster.routing.Preference;
2929
import org.elasticsearch.cluster.routing.ShardIterator;
3030
import org.elasticsearch.cluster.service.ClusterService;
3131
import org.elasticsearch.common.inject.Inject;
@@ -38,6 +38,8 @@
3838
import org.elasticsearch.threadpool.ThreadPool;
3939
import org.elasticsearch.transport.TransportService;
4040

41+
import java.io.IOException;
42+
4143
/**
4244
* Performs the get operation.
4345
*/
@@ -76,6 +78,23 @@ protected void resolveRequest(ClusterState state, InternalRequest request) {
7678
}
7779
}
7880

81+
@Override
82+
protected void asyncShardOperation(GetRequest request, ShardId shardId, ActionListener<GetResponse> listener) throws IOException {
83+
IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex());
84+
IndexShard indexShard = indexService.getShard(shardId.id());
85+
if (request.realtime()) { // we are not tied to a refresh cycle here anyway
86+
listener.onResponse(shardOperation(request, shardId));
87+
} else {
88+
indexShard.awaitShardSearchActive(b -> {
89+
try {
90+
super.asyncShardOperation(request, shardId, listener);
91+
} catch (Exception ex) {
92+
listener.onFailure(ex);
93+
}
94+
});
95+
}
96+
}
97+
7998
@Override
8099
protected GetResponse shardOperation(GetRequest request, ShardId shardId) {
81100
IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex());

‎core/src/main/java/org/elasticsearch/action/support/single/shard/TransportSingleShardAction.java‎

Lines changed: 36 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
import org.elasticsearch.common.Nullable;
3939
import org.elasticsearch.common.logging.LoggerMessageFormat;
4040
import org.elasticsearch.common.settings.Settings;
41+
import org.elasticsearch.common.util.concurrent.AbstractRunnable;
4142
import org.elasticsearch.index.shard.ShardId;
4243
import org.elasticsearch.threadpool.ThreadPool;
4344
import org.elasticsearch.transport.TransportChannel;
@@ -47,6 +48,8 @@
4748
import org.elasticsearch.transport.TransportService;
4849

4950
import java.io.IOException;
51+
import java.io.UncheckedIOException;
52+
import java.util.concurrent.Executor;
5053
import java.util.function.Supplier;
5154

5255
import static org.elasticsearch.action.support.TransportActions.isShardNotAvailableException;
@@ -78,7 +81,7 @@ protected TransportSingleShardAction(Settings settings, String actionName, Threa
7881
if (!isSubAction()) {
7982
transportService.registerRequestHandler(actionName, request, ThreadPool.Names.SAME, new TransportHandler());
8083
}
81-
transportService.registerRequestHandler(transportShardAction, request, executor, new ShardTransportHandler());
84+
transportService.registerRequestHandler(transportShardAction, request, ThreadPool.Names.SAME, new ShardTransportHandler());
8285
}
8386

8487
/**
@@ -97,6 +100,19 @@ protected void doExecute(Request request, ActionListener<Response> listener) {
97100

98101
protected abstract Response shardOperation(Request request, ShardId shardId) throws IOException;
99102

103+
protected void asyncShardOperation(Request request, ShardId shardId, ActionListener<Response> listener) throws IOException {
104+
threadPool.executor(this.executor).execute(new AbstractRunnable() {
105+
@Override
106+
public void onFailure(Exception e) {
107+
listener.onFailure(e);
108+
}
109+
110+
@Override
111+
protected void doRun() throws Exception {
112+
listener.onResponse(shardOperation(request, shardId));
113+
}
114+
});
115+
}
100116
protected abstract Response newResponse();
101117

102118
protected abstract boolean resolveIndex(Request request);
@@ -291,11 +307,27 @@ public void messageReceived(final Request request, final TransportChannel channe
291307
if (logger.isTraceEnabled()) {
292308
logger.trace("executing [{}] on shard [{}]", request, request.internalShardId);
293309
}
294-
Response response = shardOperation(request, request.internalShardId);
295-
channel.sendResponse(response);
310+
asyncShardOperation(request, request.internalShardId, new ActionListener<Response>() {
311+
@Override
312+
public void onResponse(Response response) {
313+
try {
314+
channel.sendResponse(response);
315+
} catch (IOException e) {
316+
onFailure(e);
317+
}
318+
}
319+
320+
@Override
321+
public void onFailure(Exception e) {
322+
try {
323+
channel.sendResponse(e);
324+
} catch (IOException e1) {
325+
throw new UncheckedIOException(e1);
326+
}
327+
}
328+
});
296329
}
297330
}
298-
299331
/**
300332
* Internal request class that gets built on each node. Holds the original request plus additional info.
301333
*/

‎core/src/main/java/org/elasticsearch/action/termvectors/TransportTermVectorsAction.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
package org.elasticsearch.action.termvectors;
2121

22+
import org.elasticsearch.action.ActionListener;
2223
import org.elasticsearch.action.RoutingMissingException;
2324
import org.elasticsearch.action.support.ActionFilters;
2425
import org.elasticsearch.action.support.single.shard.TransportSingleShardAction;
@@ -37,6 +38,8 @@
3738
import org.elasticsearch.threadpool.ThreadPool;
3839
import org.elasticsearch.transport.TransportService;
3940

41+
import java.io.IOException;
42+
4043
/**
4144
* Performs the get operation.
4245
*/
@@ -82,6 +85,23 @@ protected void resolveRequest(ClusterState state, InternalRequest request) {
8285
}
8386
}
8487

88+
@Override
89+
protected void asyncShardOperation(TermVectorsRequest request, ShardId shardId, ActionListener<TermVectorsResponse> listener) throws IOException {
90+
IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex());
91+
IndexShard indexShard = indexService.getShard(shardId.id());
92+
if (request.realtime()) { // it's a realtime request which is not subject to refresh cycles
93+
listener.onResponse(shardOperation(request, shardId));
94+
} else {
95+
indexShard.awaitShardSearchActive(b -> {
96+
try {
97+
super.asyncShardOperation(request, shardId, listener);
98+
} catch (Exception ex) {
99+
listener.onFailure(ex);
100+
}
101+
});
102+
}
103+
}
104+
85105
@Override
86106
protected TermVectorsResponse shardOperation(TermVectorsRequest request, ShardId shardId) {
87107
IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex());

‎core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,6 @@
3636
import org.elasticsearch.index.fielddata.IndexFieldDataService;
3737
import org.elasticsearch.index.mapper.FieldMapper;
3838
import org.elasticsearch.index.mapper.MapperService;
39-
import org.elasticsearch.index.seqno.LocalCheckpointTracker;
4039
import org.elasticsearch.index.similarity.SimilarityService;
4140
import org.elasticsearch.index.store.FsDirectoryService;
4241
import org.elasticsearch.index.store.Store;
@@ -135,6 +134,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
135134
IndexSettings.INDEX_TRANSLOG_GENERATION_THRESHOLD_SIZE_SETTING,
136135
IndexSettings.INDEX_TRANSLOG_RETENTION_AGE_SETTING,
137136
IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING,
137+
IndexSettings.INDEX_SEARCH_IDLE_AFTER,
138138
IndexFieldDataService.INDEX_FIELDDATA_CACHE_KEY,
139139
FieldMapper.IGNORE_MALFORMED_SETTING,
140140
FieldMapper.COERCE_SETTING,

‎core/src/main/java/org/elasticsearch/index/IndexService.java‎

Lines changed: 29 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
import org.elasticsearch.common.settings.Settings;
3838
import org.elasticsearch.common.unit.TimeValue;
3939
import org.elasticsearch.common.util.BigArrays;
40+
import org.elasticsearch.common.util.concurrent.AbstractRunnable;
4041
import org.elasticsearch.common.util.concurrent.FutureUtils;
4142
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
4243
import org.elasticsearch.env.NodeEnvironment;
@@ -624,6 +625,27 @@ public synchronized void updateMetaData(final IndexMetaData metadata) {
624625
}
625626
}
626627
if (refreshTask.getInterval().equals(indexSettings.getRefreshInterval()) == false) {
628+
// once we change the refresh interval we schedule yet another refresh
629+
// to ensure we are in a clean and predictable state.
630+
// it doesn't matter if we move from or to <code>-1</code> in both cases we want
631+
// docs to become visible immediately. This also flushes all pending indexing / search reqeusts
632+
// that are waiting for a refresh.
633+
threadPool.executor(ThreadPool.Names.REFRESH).execute(new AbstractRunnable() {
634+
@Override
635+
public void onFailure(Exception e) {
636+
logger.warn("forced refresh failed after interval change", e);
637+
}
638+
639+
@Override
640+
protected void doRun() throws Exception {
641+
maybeRefreshEngine(true);
642+
}
643+
644+
@Override
645+
public boolean isForceExecution() {
646+
return true;
647+
}
648+
});
627649
rescheduleRefreshTasks();
628650
}
629651
final Translog.Durability durability = indexSettings.getTranslogDurability();
@@ -686,17 +708,13 @@ private void maybeFSyncTranslogs() {
686708
}
687709
}
688710

689-
private void maybeRefreshEngine() {
690-
if (indexSettings.getRefreshInterval().millis() > 0) {
711+
private void maybeRefreshEngine(boolean force) {
712+
if (indexSettings.getRefreshInterval().millis() > 0 || force) {
691713
for (IndexShard shard : this.shards.values()) {
692-
if (shard.isReadAllowed()) {
693-
try {
694-
if (shard.isRefreshNeeded()) {
695-
shard.refresh("schedule");
696-
}
697-
} catch (IndexShardClosedException | AlreadyClosedException ex) {
698-
// fine - continue;
699-
}
714+
try {
715+
shard.scheduledRefresh();
716+
} catch (IndexShardClosedException | AlreadyClosedException ex) {
717+
// fine - continue;
700718
}
701719
}
702720
}
@@ -896,7 +914,7 @@ final class AsyncRefreshTask extends BaseAsyncTask {
896914

897915
@Override
898916
protected void runInternal() {
899-
indexService.maybeRefreshEngine();
917+
indexService.maybeRefreshEngine(false);
900918
}
901919

902920
@Override

‎core/src/main/java/org/elasticsearch/index/IndexSettings.java‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,9 @@ public final class IndexSettings {
6262
public static final Setting<TimeValue> INDEX_TRANSLOG_SYNC_INTERVAL_SETTING =
6363
Setting.timeSetting("index.translog.sync_interval", TimeValue.timeValueSeconds(5), TimeValue.timeValueMillis(100),
6464
Property.IndexScope);
65+
public static final Setting<TimeValue> INDEX_SEARCH_IDLE_AFTER =
66+
Setting.timeSetting("index.search.idle.after", TimeValue.timeValueSeconds(30),
67+
TimeValue.timeValueMinutes(0), Property.IndexScope, Property.Dynamic);
6568
public static final Setting<Translog.Durability> INDEX_TRANSLOG_DURABILITY_SETTING =
6669
new Setting<>("index.translog.durability", Translog.Durability.REQUEST.name(),
6770
(value) -> Translog.Durability.valueOf(value.toUpperCase(Locale.ROOT)), Property.Dynamic, Property.IndexScope);
@@ -262,6 +265,8 @@ public final class IndexSettings {
262265
private volatile int maxNgramDiff;
263266
private volatile int maxShingleDiff;
264267
private volatile boolean TTLPurgeDisabled;
268+
private volatile TimeValue searchIdleAfter;
269+
265270
/**
266271
* The maximum number of refresh listeners allows on this shard.
267272
*/
@@ -371,6 +376,7 @@ public IndexSettings(final IndexMetaData indexMetaData, final Settings nodeSetti
371376
maxSlicesPerScroll = scopedSettings.get(MAX_SLICES_PER_SCROLL);
372377
this.mergePolicyConfig = new MergePolicyConfig(logger, this);
373378
this.indexSortConfig = new IndexSortConfig(this);
379+
searchIdleAfter = scopedSettings.get(INDEX_SEARCH_IDLE_AFTER);
374380
singleType = INDEX_MAPPING_SINGLE_TYPE_SETTING.get(indexMetaData.getSettings()); // get this from metadata - it's not registered
375381
if ((singleType || version.before(Version.V_6_0_0_alpha1)) == false) {
376382
throw new AssertionError(index.toString() + "multiple types are only allowed on pre 6.x indices but version is: ["
@@ -411,8 +417,11 @@ public IndexSettings(final IndexMetaData indexMetaData, final Settings nodeSetti
411417
scopedSettings.addSettingsUpdateConsumer(MAX_REFRESH_LISTENERS_PER_SHARD, this::setMaxRefreshListeners);
412418
scopedSettings.addSettingsUpdateConsumer(MAX_SLICES_PER_SCROLL, this::setMaxSlicesPerScroll);
413419
scopedSettings.addSettingsUpdateConsumer(DEFAULT_FIELD_SETTING, this::setDefaultFields);
420+
scopedSettings.addSettingsUpdateConsumer(INDEX_SEARCH_IDLE_AFTER, this::setSearchIdleAfter);
414421
}
415422

423+
private void setSearchIdleAfter(TimeValue searchIdleAfter) { this.searchIdleAfter = searchIdleAfter; }
424+
416425
private void setTranslogFlushThresholdSize(ByteSizeValue byteSizeValue) {
417426
this.flushThresholdSize = byteSizeValue;
418427
}
@@ -752,4 +761,16 @@ public IndexSortConfig getIndexSortConfig() {
752761
}
753762

754763
public IndexScopedSettings getScopedSettings() { return scopedSettings;}
764+
765+
/**
766+
* Returns true iff the refresh setting exists or in other words is explicitly set.
767+
*/
768+
public boolean isExplicitRefresh() {
769+
return INDEX_REFRESH_INTERVAL_SETTING.exists(settings);
770+
}
771+
772+
/**
773+
* Returns the time that an index shard becomes search idle unless it's accessed in between
774+
*/
775+
public TimeValue getSearchIdleAfter() { return searchIdleAfter; }
755776
}

0 commit comments

Comments
 (0)