diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java index e801a07f11670..309b8a4f143d3 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java +++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java @@ -143,6 +143,7 @@ private static Version parseUnchecked(String version) { public static final IndexVersion INFERENCE_METADATA_FIELDS = def(9_005_00_0, Version.LUCENE_10_0_0); public static final IndexVersion LOGSB_OPTIONAL_SORTING_ON_HOST_NAME = def(9_006_00_0, Version.LUCENE_10_0_0); public static final IndexVersion SOURCE_MAPPER_MODE_ATTRIBUTE_NOOP = def(9_007_00_0, Version.LUCENE_10_0_0); + public static final IndexVersion HOSTNAME_DOC_VALUES_SPARSE_INDEX = def(9_008_00_0, Version.LUCENE_10_0_0); /* * STOP! READ THIS FIRST! No, really, * ____ _____ ___ ____ _ ____ _____ _ ____ _____ _ _ ___ ____ _____ ___ ____ ____ _____ _ diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index 7c1f3678a5dc9..2dfbf871ea183 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -21,6 +21,7 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.util.CollectionUtils; +import org.elasticsearch.common.util.FeatureFlag; import org.elasticsearch.common.util.Maps; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.IndexMode; @@ -63,6 +64,7 @@ public abstract class FieldMapper extends Mapper { private static final Logger logger = LogManager.getLogger(FieldMapper.class); + public static final FeatureFlag DOC_VALUES_SPARSE_INDEX = new FeatureFlag("doc_values_sparse_index"); public static final Setting IGNORE_MALFORMED_SETTING = Setting.boolSetting("index.mapping.ignore_malformed", settings -> { if (IndexSettings.MODE.get(settings) == IndexMode.LOGSDB && IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings).onOrAfter(IndexVersions.ENABLE_IGNORE_MALFORMED_LOGSDB)) { @@ -851,6 +853,10 @@ public boolean isConfigured() { return isSet && Objects.equals(value, getDefaultValue()) == false; } + public boolean isSet() { + return isSet; + } + /** * Allows the parameter to accept a {@code null} value */ diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 7b14739d36246..b7528bd3729ee 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -18,6 +18,7 @@ import org.apache.lucene.document.InvertableType; import org.apache.lucene.document.SortedSetDocValuesField; import org.apache.lucene.document.StoredField; +import org.apache.lucene.index.DocValuesSkipIndexType; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexReader; @@ -38,7 +39,10 @@ import org.elasticsearch.common.lucene.search.AutomatonQueries; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.core.Nullable; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSortConfig; import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.fielddata.FieldData; @@ -87,9 +91,11 @@ public final class KeywordFieldMapper extends FieldMapper { private static final Logger logger = LogManager.getLogger(KeywordFieldMapper.class); public static final String CONTENT_TYPE = "keyword"; + private static final String HOST_NAME = "host.name"; public static class Defaults { public static final FieldType FIELD_TYPE; + public static final FieldType FIELD_TYPE_WITH_SKIP_DOC_VALUES; static { FieldType ft = new FieldType(); @@ -100,6 +106,16 @@ public static class Defaults { FIELD_TYPE = freezeAndDeduplicateFieldType(ft); } + static { + FieldType ft = new FieldType(); + ft.setTokenized(false); + ft.setOmitNorms(true); + ft.setIndexOptions(IndexOptions.NONE); + ft.setDocValuesType(DocValuesType.SORTED_SET); + ft.setDocValuesSkipIndexType(DocValuesSkipIndexType.RANGE); + FIELD_TYPE_WITH_SKIP_DOC_VALUES = freezeAndDeduplicateFieldType(ft); + } + public static final TextSearchInfo TEXT_SEARCH_INFO = new TextSearchInfo( FIELD_TYPE, null, @@ -154,7 +170,8 @@ public static final class Builder extends FieldMapper.DimensionBuilder { ); private final Parameter ignoreAbove; private final int ignoreAboveDefault; - + private final IndexSortConfig indexSortConfig; + private final IndexMode indexMode; private final Parameter indexOptions = TextParams.keywordIndexOptions(m -> toType(m).indexOptions); private final Parameter hasNorms = TextParams.norms(false, m -> toType(m).fieldType.omitNorms() == false); private final Parameter similarity = TextParams.similarity( @@ -189,7 +206,9 @@ public Builder(final String name, final MappingParserContext mappingParserContex mappingParserContext.getIndexAnalyzers(), mappingParserContext.scriptCompiler(), IGNORE_ABOVE_SETTING.get(mappingParserContext.getSettings()), - mappingParserContext.getIndexSettings().getIndexVersionCreated() + mappingParserContext.getIndexSettings().getIndexVersionCreated(), + mappingParserContext.getIndexSettings().getMode(), + mappingParserContext.getIndexSettings().getIndexSortConfig() ); } @@ -199,6 +218,18 @@ public Builder(final String name, final MappingParserContext mappingParserContex ScriptCompiler scriptCompiler, int ignoreAboveDefault, IndexVersion indexCreatedVersion + ) { + this(name, indexAnalyzers, scriptCompiler, ignoreAboveDefault, indexCreatedVersion, IndexMode.STANDARD, null); + } + + private Builder( + String name, + IndexAnalyzers indexAnalyzers, + ScriptCompiler scriptCompiler, + int ignoreAboveDefault, + IndexVersion indexCreatedVersion, + IndexMode indexMode, + IndexSortConfig indexSortConfig ) { super(name); this.indexAnalyzers = indexAnalyzers; @@ -233,6 +264,8 @@ public Builder(final String name, final MappingParserContext mappingParserContex throw new IllegalArgumentException("[ignore_above] must be positive, got [" + v + "]"); } }); + this.indexSortConfig = indexSortConfig; + this.indexMode = indexMode; } public Builder(String name, IndexVersion indexCreatedVersion) { @@ -359,15 +392,21 @@ private KeywordFieldType buildFieldType(MapperBuilderContext context, FieldType @Override public KeywordFieldMapper build(MapperBuilderContext context) { - FieldType fieldtype = new FieldType(Defaults.FIELD_TYPE); + FieldType fieldtype = resolveFieldType(indexCreatedVersion, indexSortConfig, indexMode, context.buildFullName(leafName())); fieldtype.setOmitNorms(this.hasNorms.getValue() == false); - fieldtype.setIndexOptions(TextParams.toIndexOptions(this.indexed.getValue(), this.indexOptions.getValue())); fieldtype.setStored(this.stored.getValue()); fieldtype.setDocValuesType(this.hasDocValues.getValue() ? DocValuesType.SORTED_SET : DocValuesType.NONE); + if (fieldtype.equals(Defaults.FIELD_TYPE_WITH_SKIP_DOC_VALUES) == false) { + // NOTE: override index options only if we are not using a sparse doc values index (and we use an inverted index) + fieldtype.setIndexOptions(TextParams.toIndexOptions(this.indexed.getValue(), this.indexOptions.getValue())); + } if (fieldtype.equals(Defaults.FIELD_TYPE)) { // deduplicate in the common default case to save some memory fieldtype = Defaults.FIELD_TYPE; } + if (fieldtype.equals(Defaults.FIELD_TYPE_WITH_SKIP_DOC_VALUES)) { + fieldtype = Defaults.FIELD_TYPE_WITH_SKIP_DOC_VALUES; + } super.hasScript = script.get() != null; super.onScriptError = onScriptError.getValue(); return new KeywordFieldMapper( @@ -379,6 +418,63 @@ public KeywordFieldMapper build(MapperBuilderContext context) { this ); } + + private FieldType resolveFieldType( + final IndexVersion indexCreatedVersion, + final IndexSortConfig indexSortConfig, + final IndexMode indexMode, + final String fullFieldName + ) { + if (FieldMapper.DOC_VALUES_SPARSE_INDEX.isEnabled() + && indexCreatedVersion.onOrAfter(IndexVersions.HOSTNAME_DOC_VALUES_SPARSE_INDEX) + && shouldUseDocValuesSparseIndex(indexSortConfig, indexMode, fullFieldName)) { + return new FieldType(Defaults.FIELD_TYPE_WITH_SKIP_DOC_VALUES); + } + return new FieldType(Defaults.FIELD_TYPE); + } + + /** + * Determines whether to use a sparse index representation for doc values. + * + *

If the field is explicitly indexed by setting {@code index: true}, we do not use + * a sparse doc values index but instead rely on the inverted index, as is typically + * the case for keyword fields.

+ * + *

This method checks several conditions to decide if the sparse index format + * should be applied:

+ * + *
    + *
  • Returns {@code false} immediately if the field is explicitly indexed.
    • + *
  • Ensures the field is not explicitly configured as indexed (i.e., {@code index} has its default value).
    • + *
  • Requires doc values to be enabled.
    • + *
  • Index mode must be {@link IndexMode#LOGSDB}.
    • + *
  • Field name must be {@code host.name}.
    • + *
  • The {@code host.name} field must be a primary sort field.
    • + *
+ * + *

Returns {@code true} if all conditions are met, indicating that sparse doc values + * should be used. Otherwise, returns {@code false}.

+ * + * @param indexSortConfig The index sort configuration, used to check primary sorting. + * @param indexMode The mode of the index, which must be {@link IndexMode#LOGSDB}. + * @param fullFieldName The name of the field being checked, which must be {@code host.name}. + * @return {@code true} if sparse doc values should be used, otherwise {@code false}. + */ + + private boolean shouldUseDocValuesSparseIndex( + final IndexSortConfig indexSortConfig, + final IndexMode indexMode, + final String fullFieldName + ) { + if (indexed.isSet() && indexed.getValue()) { + return false; + } + return indexed.isConfigured() == false + && hasDocValues.getValue() + && IndexMode.LOGSDB.equals(indexMode) + && HOST_NAME.equals(fullFieldName) + && (indexSortConfig != null && indexSortConfig.hasPrimarySortOnField(HOST_NAME)); + } } public static final TypeParser PARSER = createTypeParserWithLegacySupport(Builder::new); @@ -392,6 +488,9 @@ public static final class KeywordFieldType extends StringFieldType { private final FieldValues scriptValues; private final boolean isDimension; private final boolean isSyntheticSource; + private final IndexMode indexMode; + private final IndexSortConfig indexSortConfig; + private final boolean hasDocValuesSparseIndex; public KeywordFieldType( String name, @@ -417,6 +516,9 @@ public KeywordFieldType( this.scriptValues = builder.scriptValues(); this.isDimension = builder.dimension.getValue(); this.isSyntheticSource = isSyntheticSource; + this.indexMode = builder.indexMode; + this.indexSortConfig = builder.indexSortConfig; + this.hasDocValuesSparseIndex = DocValuesSkipIndexType.NONE.equals(fieldType.docValuesSkipIndexType()) == false; } public KeywordFieldType(String name, boolean isIndexed, boolean hasDocValues, Map meta) { @@ -428,6 +530,9 @@ public KeywordFieldType(String name, boolean isIndexed, boolean hasDocValues, Ma this.scriptValues = null; this.isDimension = false; this.isSyntheticSource = false; + this.indexMode = IndexMode.STANDARD; + this.indexSortConfig = null; + this.hasDocValuesSparseIndex = false; } public KeywordFieldType(String name) { @@ -450,6 +555,9 @@ public KeywordFieldType(String name, FieldType fieldType) { this.scriptValues = null; this.isDimension = false; this.isSyntheticSource = false; + this.indexMode = IndexMode.STANDARD; + this.indexSortConfig = null; + this.hasDocValuesSparseIndex = DocValuesSkipIndexType.NONE.equals(fieldType.docValuesSkipIndexType()) == false; } public KeywordFieldType(String name, NamedAnalyzer analyzer) { @@ -461,6 +569,9 @@ public KeywordFieldType(String name, NamedAnalyzer analyzer) { this.scriptValues = null; this.isDimension = false; this.isSyntheticSource = false; + this.indexMode = IndexMode.STANDARD; + this.indexSortConfig = null; + this.hasDocValuesSparseIndex = false; } @Override @@ -851,6 +962,18 @@ public boolean hasScriptValues() { public boolean hasNormalizer() { return normalizer != Lucene.KEYWORD_ANALYZER; } + + public IndexMode getIndexMode() { + return indexMode; + } + + public IndexSortConfig getIndexSortConfig() { + return indexSortConfig; + } + + public boolean hasDocValuesSparseIndex() { + return hasDocValuesSparseIndex; + } } private final boolean indexed; @@ -866,7 +989,8 @@ public boolean hasNormalizer() { private final IndexAnalyzers indexAnalyzers; private final int ignoreAboveDefault; - private final int ignoreAbove; + private final IndexMode indexMode; + private final IndexSortConfig indexSortConfig; private KeywordFieldMapper( String simpleName, @@ -890,7 +1014,8 @@ private KeywordFieldMapper( this.indexCreatedVersion = builder.indexCreatedVersion; this.isSyntheticSource = isSyntheticSource; this.ignoreAboveDefault = builder.ignoreAboveDefault; - this.ignoreAbove = builder.ignoreAbove.getValue(); + this.indexMode = builder.indexMode; + this.indexSortConfig = builder.indexSortConfig; } @Override @@ -1008,9 +1133,9 @@ public Map indexAnalyzers() { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(leafName(), indexAnalyzers, scriptCompiler, ignoreAboveDefault, indexCreatedVersion).dimension( - fieldType().isDimension() - ).init(this); + return new Builder(leafName(), indexAnalyzers, scriptCompiler, ignoreAboveDefault, indexCreatedVersion, indexMode, indexSortConfig) + .dimension(fieldType().isDimension()) + .init(this); } @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java index 052bf995bdd48..d78f2110daa67 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java @@ -28,6 +28,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.IndexSortConfig; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.CharFilterFactory; @@ -770,4 +771,159 @@ public void testDocValuesLoadedFromStoredSynthetic() throws IOException { ); assertScriptDocValues(mapper, "foo", equalTo(List.of("foo"))); } + + public void testFieldTypeWithSkipDocValues_LogsDbMode() throws IOException { + final MapperService mapperService = createMapperService( + Settings.builder() + .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()) + .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "host.name") + .build(), + mapping(b -> { + b.startObject("host.name"); + b.field("type", "keyword"); + b.endObject(); + }) + ); + + final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); + assertTrue(mapper.fieldType().hasDocValues()); + assertFalse(mapper.fieldType().isIndexed()); + assertTrue(mapper.fieldType().hasDocValuesSparseIndex()); + } + + public void testFieldTypeDefault_StandardMode() throws IOException { + final MapperService mapperService = createMapperService( + Settings.builder() + .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.name()) + .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "host.name") + .build(), + mapping(b -> { + b.startObject("host.name"); + b.field("type", "keyword"); + b.endObject(); + }) + ); + + final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); + assertTrue(mapper.fieldType().hasDocValues()); + assertTrue(mapper.fieldType().isIndexed()); + assertFalse(mapper.fieldType().hasDocValuesSparseIndex()); + } + + public void testFieldTypeDefault_NonMatchingFieldName() throws IOException { + final MapperService mapperService = createMapperService( + Settings.builder() + .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()) + .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "hostname") + .build(), + mapping(b -> { + b.startObject("hostname"); + b.field("type", "keyword"); + b.endObject(); + }) + ); + + final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("hostname"); + assertTrue(mapper.fieldType().hasDocValues()); + assertTrue(mapper.fieldType().isIndexed()); + assertFalse(mapper.fieldType().hasDocValuesSparseIndex()); + } + + public void testFieldTypeDefault_ConfiguredIndexed() throws IOException { + final MapperService mapperService = createMapperService( + Settings.builder() + .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()) + .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "host.name") + .build(), + mapping(b -> { + b.startObject("host.name"); + b.field("type", "keyword"); + b.field("index", true); + b.endObject(); + }) + ); + + final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); + assertTrue(mapper.fieldType().hasDocValues()); + assertTrue(mapper.fieldType().isIndexed()); + assertFalse(mapper.fieldType().hasDocValuesSparseIndex()); + } + + public void testFieldTypeDefault_ConfiguredDocValues() throws IOException { + final MapperService mapperService = createMapperService( + Settings.builder() + .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()) + .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "host.name") + .build(), + mapping(b -> { + b.startObject("host.name"); + b.field("type", "keyword"); + b.field("doc_values", true); + b.endObject(); + }) + ); + + final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); + assertTrue(mapper.fieldType().hasDocValues()); + assertFalse(mapper.fieldType().isIndexed()); + assertTrue(mapper.fieldType().hasDocValuesSparseIndex()); + } + + public void testFieldTypeDefault_LogsDbMode_NonSortField() throws IOException { + final MapperService mapperService = createMapperService( + Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()).build(), + mapping(b -> { + b.startObject("host.name"); + b.field("type", "keyword"); + b.endObject(); + }) + ); + + final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); + assertTrue(mapper.fieldType().hasDocValues()); + assertTrue(mapper.fieldType().isIndexed()); + assertFalse(mapper.fieldType().hasDocValuesSparseIndex()); + } + + public void testFieldTypeWithSkipDocValues_IndexedFalseDocValuesTrue() throws IOException { + final MapperService mapperService = createMapperService( + Settings.builder() + .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()) + .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "host.name") + .build(), + mapping(b -> { + b.startObject("host.name"); + b.field("type", "keyword"); + b.field("index", false); + b.field("doc_values", true); + b.endObject(); + }) + ); + + final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); + assertTrue(mapper.fieldType().hasDocValues()); + assertFalse(mapper.fieldType().isIndexed()); + assertFalse(mapper.fieldType().hasDocValuesSparseIndex()); + } + + public void testFieldTypeDefault_IndexedFalseDocValuesFalse() throws IOException { + final MapperService mapperService = createMapperService( + Settings.builder() + .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()) + .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "host.name") + .build(), + mapping(b -> { + b.startObject("host.name"); + b.field("type", "keyword"); + b.field("index", false); + b.field("doc_values", false); + b.endObject(); + }) + ); + + final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); + assertFalse(mapper.fieldType().hasDocValues()); + assertFalse(mapper.fieldType().isIndexed()); + assertFalse(mapper.fieldType().hasDocValuesSparseIndex()); + } }