Skip to content

Sparse doc values index for LogsDB host.name field #120741

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
salvatore-campagna merged 25 commits into from
Jan 29, 2025
Merged

Sparse doc values index for LogsDB host.name field #120741

Show file tree
Hide file tree
Changes from 14 commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
65f02eb
feature: sparse index for logsdb host.name field
salvatore-campagna Jan 23, 2025
1e14746
Merge branch 'main' into feature/timestamp-and-hostname-sparse-index
salvatore-campagna Jan 23, 2025
6476c15
fix: enable sparse index if field is indexed nad has doc values
salvatore-campagna Jan 23, 2025
47039a9
fix: rename method and make constant private
salvatore-campagna Jan 23, 2025
15ecf22
fix: keep existing Builder
salvatore-campagna Jan 24, 2025
ffe0a2c
fix: compare to NONE and flip equals
salvatore-campagna Jan 24, 2025
5cd8126
fix: refactor Builder constructor and remove unused variables
salvatore-campagna Jan 24, 2025
4b54de6
Merge branch 'main' into feature/timestamp-and-hostname-sparse-index
salvatore-campagna Jan 24, 2025
4b6b87d
Merge branch 'main' into feature/timestamp-and-hostname-sparse-index
salvatore-campagna Jan 24, 2025
db8374e
fix: refactor Builder constructore and remove hasDocValuesSparseIndex
salvatore-campagna Jan 24, 2025
328e42b
fix: gate sparse index usage with feature flag
salvatore-campagna Jan 24, 2025
3d27a95
Merge branch 'main' into feature/timestamp-and-hostname-sparse-index
salvatore-campagna Jan 24, 2025
4ce3758
Merge branch 'main' into feature/timestamp-and-hostname-sparse-index
salvatore-campagna Jan 24, 2025
6a452bf
Merge branch 'main' into feature/timestamp-and-hostname-sparse-index
salvatore-campagna Jan 27, 2025
faea308
Merge branch 'main' into feature/timestamp-and-hostname-sparse-index
salvatore-campagna Jan 27, 2025
457d14f
Merge branch 'main' into feature/timestamp-and-hostname-sparse-index
salvatore-campagna Jan 27, 2025
4909ecb
fix: refactor sparse doc values index creation conditions
salvatore-campagna Jan 27, 2025
2e3a51e
fix: host.name sparse doc values index tests
salvatore-campagna Jan 27, 2025
1868134
fix: add missing assertions
salvatore-campagna Jan 27, 2025
e4f0184
fix: use sparse doc values index only for new indices
salvatore-campagna Jan 28, 2025
3d7fdc2
Merge branch 'main' into feature/timestamp-and-hostname-sparse-index
salvatore-campagna Jan 28, 2025
24aa35d
fix: either use the sparse idnex or the inverted index
salvatore-campagna Jan 29, 2025
16014a8
Merge branch 'main' into feature/timestamp-and-hostname-sparse-index
salvatore-campagna Jan 29, 2025
e0599ac
Merge branch 'main' into feature/timestamp-and-hostname-sparse-index
salvatore-campagna Jan 29, 2025
1254c7a
Merge branch 'main' into feature/timestamp-and-hostname-sparse-index
salvatore-campagna Jan 29, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Setting.Property;
import org.elasticsearch.common.util.CollectionUtils;
import org.elasticsearch.common.util.FeatureFlag;
import org.elasticsearch.common.util.Maps;
import org.elasticsearch.common.xcontent.support.XContentMapValues;
import org.elasticsearch.index.IndexMode;
Expand Down Expand Up @@ -63,6 +64,7 @@
public abstract class FieldMapper extends Mapper {
private static final Logger logger = LogManager.getLogger(FieldMapper.class);

public static final FeatureFlag DOC_VALUES_SPARSE_INDEX = new FeatureFlag("doc_values_sparse_index");
Comment thread
martijnvg marked this conversation as resolved.
public static final Setting<Boolean> IGNORE_MALFORMED_SETTING = Setting.boolSetting("index.mapping.ignore_malformed", settings -> {
if (IndexSettings.MODE.get(settings) == IndexMode.LOGSDB
&& IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings).onOrAfter(IndexVersions.ENABLE_IGNORE_MALFORMED_LOGSDB)) {
Expand Down
105 changes: 97 additions & 8 deletions server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import org.apache.lucene.document.InvertableType;
import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.index.DocValuesSkipIndexType;
import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexReader;
Expand All @@ -38,6 +39,8 @@
import org.elasticsearch.common.lucene.search.AutomatonQueries;
import org.elasticsearch.common.unit.Fuzziness;
import org.elasticsearch.core.Nullable;
import org.elasticsearch.index.IndexMode;
import org.elasticsearch.index.IndexSortConfig;
import org.elasticsearch.index.IndexVersion;
import org.elasticsearch.index.analysis.IndexAnalyzers;
import org.elasticsearch.index.analysis.NamedAnalyzer;
Expand Down Expand Up @@ -87,9 +90,11 @@ public final class KeywordFieldMapper extends FieldMapper {
private static final Logger logger = LogManager.getLogger(KeywordFieldMapper.class);

public static final String CONTENT_TYPE = "keyword";
private static final String HOST_NAME = "host.name";

public static class Defaults {
public static final FieldType FIELD_TYPE;
public static final FieldType FIELD_TYPE_WITH_SKIP_DOC_VALUES;

static {
FieldType ft = new FieldType();
Expand All @@ -100,6 +105,16 @@ public static class Defaults {
FIELD_TYPE = freezeAndDeduplicateFieldType(ft);
}

static {
FieldType ft = new FieldType();
ft.setTokenized(false);
ft.setOmitNorms(true);
ft.setIndexOptions(IndexOptions.NONE);
ft.setDocValuesType(DocValuesType.SORTED_SET);
ft.setDocValuesSkipIndexType(DocValuesSkipIndexType.RANGE);
FIELD_TYPE_WITH_SKIP_DOC_VALUES = freezeAndDeduplicateFieldType(ft);
}

public static final TextSearchInfo TEXT_SEARCH_INFO = new TextSearchInfo(
FIELD_TYPE,
null,
Expand Down Expand Up @@ -154,7 +169,8 @@ public static final class Builder extends FieldMapper.DimensionBuilder {
);
private final Parameter<Integer> ignoreAbove;
private final int ignoreAboveDefault;

private final IndexSortConfig indexSortConfig;
private final IndexMode indexMode;
private final Parameter<String> indexOptions = TextParams.keywordIndexOptions(m -> toType(m).indexOptions);
private final Parameter<Boolean> hasNorms = TextParams.norms(false, m -> toType(m).fieldType.omitNorms() == false);
private final Parameter<SimilarityProvider> similarity = TextParams.similarity(
Expand Down Expand Up @@ -189,7 +205,9 @@ public Builder(final String name, final MappingParserContext mappingParserContex
mappingParserContext.getIndexAnalyzers(),
mappingParserContext.scriptCompiler(),
IGNORE_ABOVE_SETTING.get(mappingParserContext.getSettings()),
mappingParserContext.getIndexSettings().getIndexVersionCreated()
mappingParserContext.getIndexSettings().getIndexVersionCreated(),
mappingParserContext.getIndexSettings().getMode(),
mappingParserContext.getIndexSettings().getIndexSortConfig()
);
}

Expand All @@ -199,6 +217,18 @@ public Builder(final String name, final MappingParserContext mappingParserContex
ScriptCompiler scriptCompiler,
int ignoreAboveDefault,
IndexVersion indexCreatedVersion
) {
this(name, indexAnalyzers, scriptCompiler, ignoreAboveDefault, indexCreatedVersion, IndexMode.STANDARD, null);
}

private Builder(
Comment thread
martijnvg marked this conversation as resolved.
String name,
IndexAnalyzers indexAnalyzers,
ScriptCompiler scriptCompiler,
int ignoreAboveDefault,
IndexVersion indexCreatedVersion,
IndexMode indexMode,
IndexSortConfig indexSortConfig
) {
super(name);
this.indexAnalyzers = indexAnalyzers;
Expand Down Expand Up @@ -233,6 +263,8 @@ public Builder(final String name, final MappingParserContext mappingParserContex
throw new IllegalArgumentException("[ignore_above] must be positive, got [" + v + "]");
}
});
this.indexSortConfig = indexSortConfig;
this.indexMode = indexMode;
}

public Builder(String name, IndexVersion indexCreatedVersion) {
Expand Down Expand Up @@ -359,7 +391,7 @@ private KeywordFieldType buildFieldType(MapperBuilderContext context, FieldType

@Override
public KeywordFieldMapper build(MapperBuilderContext context) {
FieldType fieldtype = new FieldType(Defaults.FIELD_TYPE);
FieldType fieldtype = resolveFieldType(indexSortConfig, indexMode, context.buildFullName(leafName()));
fieldtype.setOmitNorms(this.hasNorms.getValue() == false);
fieldtype.setIndexOptions(TextParams.toIndexOptions(this.indexed.getValue(), this.indexOptions.getValue()));
fieldtype.setStored(this.stored.getValue());
Expand All @@ -368,6 +400,9 @@ public KeywordFieldMapper build(MapperBuilderContext context) {
// deduplicate in the common default case to save some memory
fieldtype = Defaults.FIELD_TYPE;
}
if (fieldtype.equals(Defaults.FIELD_TYPE_WITH_SKIP_DOC_VALUES)) {

@salvatore-campagna salvatore-campagna Jan 24, 2025

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

An optimization as above.

fieldtype = Defaults.FIELD_TYPE_WITH_SKIP_DOC_VALUES;
}
super.hasScript = script.get() != null;
super.onScriptError = onScriptError.getValue();
return new KeywordFieldMapper(
Expand All @@ -379,6 +414,31 @@ public KeywordFieldMapper build(MapperBuilderContext context) {
this
);
}

private FieldType resolveFieldType(final IndexSortConfig indexSortConfig, final IndexMode indexMode, final String fullFieldName) {
if (shouldUseDocValuesSparseIndex(indexSortConfig, indexMode, fullFieldName)) {
return new FieldType(Defaults.FIELD_TYPE_WITH_SKIP_DOC_VALUES);
}
return new FieldType(Defaults.FIELD_TYPE);
}

private boolean shouldUseDocValuesSparseIndex(
Comment thread
martijnvg marked this conversation as resolved.
final IndexSortConfig indexSortConfig,
final IndexMode indexMode,
final String fullFieldName
) {
if (FieldMapper.DOC_VALUES_SPARSE_INDEX.isEnabled() == false) {
return false;
}
boolean isIndexedAndDocValuesDefault = indexed.isConfigured() || hasDocValues.isConfigured();
boolean isNotIndexedAndHasDocValues = indexed.getValue() || hasDocValues.getValue() == false;
boolean isLogsDbMode = IndexMode.LOGSDB.equals(indexMode);
boolean isHostNameField = HOST_NAME.equals(fullFieldName);
boolean isPrimarySortField = indexSortConfig != null && indexSortConfig.hasPrimarySortOnField(HOST_NAME);

return (isIndexedAndDocValuesDefault || isNotIndexedAndHasDocValues) && isLogsDbMode && isHostNameField && isPrimarySortField;

@martijnvg martijnvg Jan 27, 2025

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we want to enable sparse index only if index has not been configured and doc values isn't disabled.

So I think this is easier:

indexed.isConfigured() == false && hasDocValues.getValue() == false && isLogsDbMode && isHostNameField && isPrimarySortField;

?

@salvatore-campagna salvatore-campagna Jan 27, 2025

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

hasDOcValues.getValue() == false? We need doc values to create the sparse index...

}

}

public static final TypeParser PARSER = createTypeParserWithLegacySupport(Builder::new);
Expand All @@ -392,6 +452,9 @@ public static final class KeywordFieldType extends StringFieldType {
private final FieldValues<String> scriptValues;
private final boolean isDimension;
private final boolean isSyntheticSource;
private final IndexMode indexMode;
private final IndexSortConfig indexSortConfig;
private final boolean hasDocValuesSparseIndex;

public KeywordFieldType(
String name,
Expand All @@ -417,6 +480,9 @@ public KeywordFieldType(
this.scriptValues = builder.scriptValues();
this.isDimension = builder.dimension.getValue();
this.isSyntheticSource = isSyntheticSource;
this.indexMode = builder.indexMode;
this.indexSortConfig = builder.indexSortConfig;
this.hasDocValuesSparseIndex = DocValuesSkipIndexType.NONE.equals(fieldType.docValuesSkipIndexType()) == false;

@salvatore-campagna salvatore-campagna Jan 24, 2025 •
edited
Loading

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Checking this way we make sure the boolean value is correct even if new skip indices other than RANGE are introduced in Lucene in future releases.

}

public KeywordFieldType(String name, boolean isIndexed, boolean hasDocValues, Map<String, String> meta) {
Expand All @@ -428,6 +494,9 @@ public KeywordFieldType(String name, boolean isIndexed, boolean hasDocValues, Ma
this.scriptValues = null;
this.isDimension = false;
this.isSyntheticSource = false;
this.indexMode = IndexMode.STANDARD;
this.indexSortConfig = null;
this.hasDocValuesSparseIndex = false;
}

public KeywordFieldType(String name) {
Expand All @@ -450,6 +519,9 @@ public KeywordFieldType(String name, FieldType fieldType) {
this.scriptValues = null;
this.isDimension = false;
this.isSyntheticSource = false;
this.indexMode = IndexMode.STANDARD;
this.indexSortConfig = null;
this.hasDocValuesSparseIndex = DocValuesSkipIndexType.NONE.equals(fieldType.docValuesSkipIndexType()) == false;
}

public KeywordFieldType(String name, NamedAnalyzer analyzer) {
Expand All @@ -461,6 +533,9 @@ public KeywordFieldType(String name, NamedAnalyzer analyzer) {
this.scriptValues = null;
this.isDimension = false;
this.isSyntheticSource = false;
this.indexMode = IndexMode.STANDARD;
this.indexSortConfig = null;
this.hasDocValuesSparseIndex = false;
}

@Override
Expand Down Expand Up @@ -851,6 +926,18 @@ public boolean hasScriptValues() {
public boolean hasNormalizer() {
return normalizer != Lucene.KEYWORD_ANALYZER;
}

public IndexMode getIndexMode() {
return indexMode;
}

public IndexSortConfig getIndexSortConfig() {
return indexSortConfig;
}

public boolean hasDocValuesSparseIndex() {
return hasDocValuesSparseIndex;
}
}

private final boolean indexed;
Expand All @@ -866,7 +953,8 @@ public boolean hasNormalizer() {

private final IndexAnalyzers indexAnalyzers;
private final int ignoreAboveDefault;
private final int ignoreAbove;
private final IndexMode indexMode;
private final IndexSortConfig indexSortConfig;

private KeywordFieldMapper(
String simpleName,
Expand All @@ -890,7 +978,8 @@ private KeywordFieldMapper(
this.indexCreatedVersion = builder.indexCreatedVersion;
this.isSyntheticSource = isSyntheticSource;
this.ignoreAboveDefault = builder.ignoreAboveDefault;
this.ignoreAbove = builder.ignoreAbove.getValue();
this.indexMode = builder.indexMode;
this.indexSortConfig = builder.indexSortConfig;
}

@Override
Expand Down Expand Up @@ -1008,9 +1097,9 @@ public Map<String, NamedAnalyzer> indexAnalyzers() {

@Override
public FieldMapper.Builder getMergeBuilder() {
return new Builder(leafName(), indexAnalyzers, scriptCompiler, ignoreAboveDefault, indexCreatedVersion).dimension(
fieldType().isDimension()
).init(this);
return new Builder(leafName(), indexAnalyzers, scriptCompiler, ignoreAboveDefault, indexCreatedVersion, indexMode, indexSortConfig)
.dimension(fieldType().isDimension())
.init(this);
}

@Override
Expand Down
Loading