Skip to content

Reverted new completion suggester #16170

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
colings86 wants to merge 1 commit into from
Closed

Reverted new completion suggester #16170

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion core/src/main/java/org/elasticsearch/action/suggest/TransportSuggestAction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@ protected ShardSuggestResponse shardOperation(ShardSuggestRequest request) {
throw new IllegalArgumentException("suggest content missing");
}
final SuggestionSearchContext context = suggestPhase.parseElement().parseInternal(parser, indexService.mapperService(),
indexService.queryParserService(), indexService.fieldData(), request.shardId().getIndex(), request.shardId().id(), request);
indexService.queryParserService(), request.shardId().getIndex(), request.shardId().id(), request);
final Suggest result = suggestPhase.execute(context, searcher.searcher());
return new ShardSuggestResponse(request.shardId(), result);
}
Expand Down
7 changes: 2 additions & 5 deletions core/src/main/java/org/elasticsearch/index/codec/PerFieldMappingPostingFormatCodec.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.mapper.core.OldCompletionFieldMapper;
import org.elasticsearch.index.mapper.core.CompletionFieldMapper;

/**
Expand Down Expand Up @@ -58,13 +57,11 @@ public PostingsFormat getPostingsFormatForField(String field) {
final MappedFieldType indexName = mapperService.indexName(field);
if (indexName == null) {
logger.warn("no index mapper found for field: [{}] returning default postings format", field);
} else if (indexName instanceof OldCompletionFieldMapper.CompletionFieldType) {
} else if (indexName instanceof CompletionFieldMapper.CompletionFieldType) {
// CompletionFieldMapper needs a special postings format
final OldCompletionFieldMapper.CompletionFieldType fieldType = (OldCompletionFieldMapper.CompletionFieldType) indexName;
final CompletionFieldMapper.CompletionFieldType fieldType = (CompletionFieldMapper.CompletionFieldType) indexName;
final PostingsFormat defaultFormat = super.getPostingsFormatForField(field);
return fieldType.postingsFormat(defaultFormat);
} else if (indexName instanceof CompletionFieldMapper.CompletionFieldType) {
return CompletionFieldMapper.CompletionFieldType.postingsFormat();
}
return super.getPostingsFormatForField(field);
}
Expand Down
4 changes: 0 additions & 4 deletions core/src/main/java/org/elasticsearch/index/mapper/MapperBuilders.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,10 +94,6 @@ public static GeoShapeFieldMapper.Builder geoShapeField(String name) {
return new GeoShapeFieldMapper.Builder(name);
}

public static OldCompletionFieldMapper.Builder oldCompletionField(String name) {
return new OldCompletionFieldMapper.Builder(name);
}

public static CompletionFieldMapper.Builder completionField(String name) {
return new CompletionFieldMapper.Builder(name);
}
Expand Down
686 changes: 331 additions & 355 deletions core/src/main/java/org/elasticsearch/index/mapper/core/CompletionFieldMapper.java
View file
Open in desktop

Large diffs are not rendered by default.

585 changes: 0 additions & 585 deletions core/src/main/java/org/elasticsearch/index/mapper/core/OldCompletionFieldMapper.java
View file
Open in desktop

This file was deleted.

2 changes: 1 addition & 1 deletion core/src/main/java/org/elasticsearch/index/query/RegexpFlag.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ public int value() {
* @param flags A string representing a list of regualr expression flags
* @return The combined OR'ed value for all the flags
*/
public static int resolveValue(String flags) {
static int resolveValue(String flags) {
if (flags == null || flags.isEmpty()) {
return RegExp.ALL;
}
Expand Down
52 changes: 28 additions & 24 deletions core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,9 +106,8 @@
import org.elasticsearch.indices.memory.IndexingMemoryController;
import org.elasticsearch.indices.recovery.RecoveryFailedException;
import org.elasticsearch.indices.recovery.RecoveryState;
import org.elasticsearch.search.suggest.completion.CompletionFieldStats;
import org.elasticsearch.search.suggest.completion.Completion090PostingsFormat;
import org.elasticsearch.search.suggest.completion.CompletionStats;
import org.elasticsearch.search.suggest.completion.old.Completion090PostingsFormat;
import org.elasticsearch.threadpool.ThreadPool;

import java.io.IOException;
Expand Down Expand Up @@ -177,7 +176,7 @@ public class IndexShard extends AbstractIndexShardComponent {

private final RecoveryStats recoveryStats = new RecoveryStats();

private ApplyRefreshSettings applyRefreshSettings = new ApplyRefreshSettings();
private final ApplyRefreshSettings applyRefreshSettings = new ApplyRefreshSettings();

private final MeanMetric refreshMetric = new MeanMetric();
private final MeanMetric flushMetric = new MeanMetric();
Expand All @@ -194,7 +193,7 @@ public class IndexShard extends AbstractIndexShardComponent {

private final IndexShardOperationCounter indexShardOperationCounter;

private EnumSet<IndexShardState> readAllowedStates = EnumSet.of(IndexShardState.STARTED, IndexShardState.RELOCATED, IndexShardState.POST_RECOVERY);
private final EnumSet<IndexShardState> readAllowedStates = EnumSet.of(IndexShardState.STARTED, IndexShardState.RELOCATED, IndexShardState.POST_RECOVERY);

/**
* True if this shard is still indexing (recently) and false if we've been idle for long enough (as periodically checked by {@link
Expand Down Expand Up @@ -259,7 +258,7 @@ public IndexShard(ShardId shardId, IndexSettingsService indexSettingsService, In

this.checkIndexOnStartup = indexSettings.get("index.shard.check_on_startup", "false");
this.translogConfig = new TranslogConfig(shardId, shardPath().resolveTranslog(), indexSettings, getFromSettings(logger, indexSettings, Translog.Durabilty.REQUEST),
bigArrays, threadPool);
bigArrays, threadPool);
final QueryCachingPolicy cachingPolicy;
// the query cache is a node-level thing, however we want the most popular filters
// to be computed on a per-shard basis
Expand Down Expand Up @@ -410,7 +409,7 @@ public void updateRoutingEntry(final ShardRouting newRouting, final boolean pers
* Marks the shard as recovering based on a remote or local node, fails with exception is recovering is not allowed to be set.
*/
public IndexShardState recovering(String reason, RecoveryState.Type type, DiscoveryNode sourceNode) throws IndexShardStartedException,
IndexShardRelocatedException, IndexShardRecoveringException, IndexShardClosedException {
IndexShardRelocatedException, IndexShardRecoveringException, IndexShardClosedException {
return recovering(reason, new RecoveryState(shardId, shardRouting.primary(), type, sourceNode, localNode));
}

Expand All @@ -422,7 +421,7 @@ public IndexShardState recovering(String reason, RecoveryState.Type type, Restor
}

private IndexShardState recovering(String reason, RecoveryState recoveryState) throws IndexShardStartedException,
IndexShardRelocatedException, IndexShardRecoveringException, IndexShardClosedException {
IndexShardRelocatedException, IndexShardRecoveringException, IndexShardClosedException {
synchronized (mutex) {
if (state == IndexShardState.CLOSED) {
throw new IndexShardClosedException(shardId);
Expand Down Expand Up @@ -714,10 +713,15 @@ public SuggestStats suggestStats() {

public CompletionStats completionStats(String... fields) {
CompletionStats completionStats = new CompletionStats();
try (final Engine.Searcher currentSearcher = acquireSearcher("completion_stats")) {
Completion090PostingsFormat postingsFormat = ((Completion090PostingsFormat) PostingsFormat.forName(Completion090PostingsFormat.CODEC_NAME));
completionStats.add(postingsFormat.completionStats(currentSearcher.reader(), fields));
completionStats.add(CompletionFieldStats.completionStats(currentSearcher.reader(), fields));
final Engine.Searcher currentSearcher = acquireSearcher("completion_stats");
try {
PostingsFormat postingsFormat = PostingsFormat.forName(Completion090PostingsFormat.CODEC_NAME);
if (postingsFormat instanceof Completion090PostingsFormat) {
Completion090PostingsFormat completionPostingsFormat = (Completion090PostingsFormat) postingsFormat;
completionStats.add(completionPostingsFormat.completionStats(currentSearcher.reader(), fields));
}
} finally {
currentSearcher.close();
}
return completionStats;
}
Expand Down Expand Up @@ -752,7 +756,7 @@ public void forceMerge(ForceMergeRequest forceMerge) throws IOException {
logger.trace("force merge with {}", forceMerge);
}
engine().forceMerge(forceMerge.flush(), forceMerge.maxNumSegments(),
forceMerge.onlyExpungeDeletes(), false, false);
forceMerge.onlyExpungeDeletes(), false, false);
}

/**
Expand All @@ -766,8 +770,8 @@ public org.apache.lucene.util.Version upgrade(UpgradeRequest upgrade) throws IOE
org.apache.lucene.util.Version previousVersion = minimumCompatibleVersion();
// we just want to upgrade the segments, not actually optimize to a single segment
engine().forceMerge(true, // we need to flush at the end to make sure the upgrade is durable
Integer.MAX_VALUE, // we just want to upgrade the segments, not actually optimize to a single segment
false, true, upgrade.upgradeOnlyAncientSegments());
Integer.MAX_VALUE, // we just want to upgrade the segments, not actually optimize to a single segment
false, true, upgrade.upgradeOnlyAncientSegments());
org.apache.lucene.util.Version version = minimumCompatibleVersion();
if (logger.isTraceEnabled()) {
logger.trace("upgraded segment {} from version {} to version {}", previousVersion, version);
Expand Down Expand Up @@ -966,7 +970,7 @@ public void finalizeRecovery() {
public boolean ignoreRecoveryAttempt() {
IndexShardState state = state(); // one time volatile read
return state == IndexShardState.POST_RECOVERY || state == IndexShardState.RECOVERING || state == IndexShardState.STARTED ||
state == IndexShardState.RELOCATED || state == IndexShardState.CLOSED;
state == IndexShardState.RELOCATED || state == IndexShardState.CLOSED;
}

public void readAllowed() throws IllegalIndexShardStateException {
Expand Down Expand Up @@ -1080,7 +1084,7 @@ public void updateBufferSize(ByteSizeValue shardIndexingBufferSize, ByteSizeValu
long iwBytesUsed = engine.indexWriterRAMBytesUsed();

String message = LoggerMessageFormat.format("updating index_buffer_size from [{}] to [{}]; IndexWriter now using [{}] bytes",
preValue, shardIndexingBufferSize, iwBytesUsed);
preValue, shardIndexingBufferSize, iwBytesUsed);

if (iwBytesUsed > shardIndexingBufferSize.bytes()) {
// our allowed buffer was changed to less than we are currently using; we ask IW to refresh
Expand All @@ -1103,7 +1107,7 @@ public void updateBufferSize(ByteSizeValue shardIndexingBufferSize, ByteSizeValu

/**
* Called by {@link IndexingMemoryController} to check whether more than {@code inactiveTimeNS} has passed since the last
* indexing operation, and become inactive (reducing indexing and translog buffers to tiny values) if so. This returns true
* indexing operation, and become inactive (reducing indexing and translog buffers to tiny values) if so. This returns true
* if the shard is inactive.
*/
public boolean checkIdle(long inactiveTimeNS) {
Expand Down Expand Up @@ -1438,9 +1442,9 @@ void persistMetadata(ShardRouting newRouting, ShardRouting currentRouting) {
writeReason = "routing changed from " + currentRouting + " to " + newRouting;
} else {
logger.trace("skip writing shard state, has been written before; previous version: [" +
currentRouting.version() + "] current version [" + newRouting.version() + "]");
currentRouting.version() + "] current version [" + newRouting.version() + "]");
assert currentRouting.version() <= newRouting.version() : "version should not go backwards for shardID: " + shardId +
" previous version: [" + currentRouting.version() + "] current version [" + newRouting.version() + "]";
" previous version: [" + currentRouting.version() + "] current version [" + newRouting.version() + "]";
return;
}
final ShardStateMetaData newShardStateMetadata = new ShardStateMetaData(newRouting.version(), newRouting.primary(), getIndexUUID());
Expand All @@ -1456,8 +1460,8 @@ void persistMetadata(ShardRouting newRouting, ShardRouting currentRouting) {

private String getIndexUUID() {
assert indexSettings.get(IndexMetaData.SETTING_INDEX_UUID) != null
|| indexSettings.getAsVersion(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).before(Version.V_0_90_6) :
"version: " + indexSettings.getAsVersion(IndexMetaData.SETTING_VERSION_CREATED, null) + " uuid: " + indexSettings.get(IndexMetaData.SETTING_INDEX_UUID);
|| indexSettings.getAsVersion(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).before(Version.V_0_90_6) :
"version: " + indexSettings.getAsVersion(IndexMetaData.SETTING_VERSION_CREATED, null) + " uuid: " + indexSettings.get(IndexMetaData.SETTING_INDEX_UUID);
return indexSettings.get(IndexMetaData.SETTING_INDEX_UUID, IndexMetaData.INDEX_UUID_NA_VALUE);
}

Expand All @@ -1467,16 +1471,16 @@ private DocumentMapperForType docMapper(String type) {

private final EngineConfig newEngineConfig(TranslogConfig translogConfig, QueryCachingPolicy cachingPolicy) {
final TranslogRecoveryPerformer translogRecoveryPerformer = new TranslogRecoveryPerformer(shardId, mapperService, queryParserService,
indexAliasesService, indexCache, logger) {
indexAliasesService, indexCache, logger) {
@Override
protected void operationProcessed() {
assert recoveryState != null;
recoveryState.getTranslog().incrementRecoveredOperations();
}
};
return new EngineConfig(shardId,
threadPool, indexingService, indexSettingsService.indexSettings(), warmer, store, deletionPolicy, mergePolicyConfig.getMergePolicy(), mergeSchedulerConfig,
mapperService.indexAnalyzer(), similarityService.similarity(), codecService, failedEngineListener, translogRecoveryPerformer, indexCache.query(), cachingPolicy, wrappingService, translogConfig);
threadPool, indexingService, indexSettingsService.indexSettings(), warmer, store, deletionPolicy, mergePolicyConfig.getMergePolicy(), mergeSchedulerConfig,
mapperService.indexAnalyzer(), similarityService.similarity(), codecService, failedEngineListener, translogRecoveryPerformer, indexCache.query(), cachingPolicy, wrappingService, translogConfig);
}

private static class IndexShardOperationCounter extends AbstractRefCounted {
Expand Down
12 changes: 0 additions & 12 deletions core/src/main/java/org/elasticsearch/search/suggest/Suggest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,9 +121,6 @@ public void readFrom(StreamInput in) throws IOException {
case CompletionSuggestion.TYPE:
suggestion = new CompletionSuggestion();
break;
case org.elasticsearch.search.suggest.completion.old.CompletionSuggestion.TYPE:
suggestion = new org.elasticsearch.search.suggest.completion.old.CompletionSuggestion();
break;
case PhraseSuggestion.TYPE:
suggestion = new PhraseSuggestion();
break;
Expand Down Expand Up @@ -184,15 +181,6 @@ public static List<Suggestion<? extends Entry<? extends Option>>> reduce(Map<Str
List<Suggestion<? extends Entry<? extends Option>>> reduced = new ArrayList<>(groupedSuggestions.size());
for (java.util.Map.Entry<String, List<Suggestion>> unmergedResults : groupedSuggestions.entrySet()) {
List<Suggestion> value = unmergedResults.getValue();
Class<? extends Suggestion> suggestionClass = null;
for (Suggestion suggestion : value) {
if (suggestionClass == null) {
suggestionClass = suggestion.getClass();
} else if (suggestionClass != suggestion.getClass()) {
throw new IllegalArgumentException("detected mixed suggestion results, due to querying on old and new completion suggester," +
" query on a single completion suggester version");
}
}
Suggestion reduce = value.get(0).reduce(value);
reduce.trim();
reduced.add(reduce);
Expand Down
Loading