From 65f02eb893ff63633b80944d9b6b936fdf316460 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 23 Jan 2025 17:14:25 +0100 Subject: [PATCH 01/13] feature: sparse index for logsdb host.name field --- .../index/mapper/DynamicFieldsBuilder.java | 10 +- .../index/mapper/KeywordFieldMapper.java | 94 ++++++++++++++-- .../index/mapper/MappedFieldType.java | 4 + .../fielddata/AbstractFieldDataTestCase.java | 8 +- .../fielddata/IndexFieldDataServiceTests.java | 8 +- .../mapper/DocumentParserContextTests.java | 14 ++- .../index/mapper/KeywordFieldMapperTests.java | 106 ++++++++++++++++++ .../index/mapper/KeywordFieldTypeTests.java | 18 ++- .../index/mapper/MultiFieldsTests.java | 9 +- .../index/mapper/ObjectMapperMergeTests.java | 53 +++++++-- .../index/mapper/ObjectMapperTests.java | 50 ++++++--- .../query/SearchExecutionContextTests.java | 6 +- .../search/SearchServiceTests.java | 2 +- 13 files changed, 327 insertions(+), 55 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java b/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java index 0793dd748c67e..6ecde8967cfbf 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java @@ -327,17 +327,11 @@ boolean createDynamicField(Mapper.Builder builder, DocumentParserContext context public boolean newDynamicStringField(DocumentParserContext context, String name) throws IOException { MapperBuilderContext mapperBuilderContext = context.createDynamicMapperBuilderContext(); if (mapperBuilderContext.parentObjectContainsDimensions()) { - return createDynamicField( - new KeywordFieldMapper.Builder(name, context.indexSettings().getIndexVersionCreated()), - context, - mapperBuilderContext - ); + return createDynamicField(new KeywordFieldMapper.Builder(name, context.indexSettings()), context, mapperBuilderContext); } else { return createDynamicField( new TextFieldMapper.Builder(name, context.indexAnalyzers(), SourceFieldMapper.isSynthetic(context.indexSettings())) - .addMultiField( - new KeywordFieldMapper.Builder("keyword", context.indexSettings().getIndexVersionCreated()).ignoreAbove(256) - ), + .addMultiField(new KeywordFieldMapper.Builder("keyword", context.indexSettings()).ignoreAbove(256)), context ); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index bdcf9bf98279f..6b9b4dacac6e3 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -18,6 +18,7 @@ import org.apache.lucene.document.InvertableType; import org.apache.lucene.document.SortedSetDocValuesField; import org.apache.lucene.document.StoredField; +import org.apache.lucene.index.DocValuesSkipIndexType; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexReader; @@ -38,6 +39,9 @@ import org.elasticsearch.common.lucene.search.AutomatonQueries; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.core.Nullable; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.IndexSortConfig; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.analysis.NamedAnalyzer; @@ -87,9 +91,11 @@ public final class KeywordFieldMapper extends FieldMapper { private static final Logger logger = LogManager.getLogger(KeywordFieldMapper.class); public static final String CONTENT_TYPE = "keyword"; + public static final String HOST_NAME = "host.name"; public static class Defaults { public static final FieldType FIELD_TYPE; + public static final FieldType FIELD_TYPE_WITH_SKIP_DOC_VALUES; static { FieldType ft = new FieldType(); @@ -100,6 +106,16 @@ public static class Defaults { FIELD_TYPE = freezeAndDeduplicateFieldType(ft); } + static { + FieldType ft = new FieldType(); + ft.setTokenized(false); + ft.setOmitNorms(true); + ft.setIndexOptions(IndexOptions.NONE); + ft.setDocValuesType(DocValuesType.SORTED_SET); + ft.setDocValuesSkipIndexType(DocValuesSkipIndexType.RANGE); + FIELD_TYPE_WITH_SKIP_DOC_VALUES = freezeAndDeduplicateFieldType(ft); + } + public static final TextSearchInfo TEXT_SEARCH_INFO = new TextSearchInfo( FIELD_TYPE, null, @@ -154,7 +170,9 @@ public static final class Builder extends FieldMapper.DimensionBuilder { ); private final Parameter ignoreAbove; private final int ignoreAboveDefault; - + private final IndexSortConfig indexSortConfig; + private final IndexMode indexMode; + private final IndexSettings indexSettings; private final Parameter indexOptions = TextParams.keywordIndexOptions(m -> toType(m).indexOptions); private final Parameter hasNorms = TextParams.norms(false, m -> toType(m).fieldType.omitNorms() == false); private final Parameter similarity = TextParams.similarity( @@ -189,7 +207,7 @@ public Builder(final String name, final MappingParserContext mappingParserContex mappingParserContext.getIndexAnalyzers(), mappingParserContext.scriptCompiler(), IGNORE_ABOVE_SETTING.get(mappingParserContext.getSettings()), - mappingParserContext.getIndexSettings().getIndexVersionCreated() + mappingParserContext.getIndexSettings() ); } @@ -198,12 +216,12 @@ public Builder(final String name, final MappingParserContext mappingParserContex IndexAnalyzers indexAnalyzers, ScriptCompiler scriptCompiler, int ignoreAboveDefault, - IndexVersion indexCreatedVersion + IndexSettings indexSettings ) { super(name); this.indexAnalyzers = indexAnalyzers; this.scriptCompiler = Objects.requireNonNull(scriptCompiler); - this.indexCreatedVersion = Objects.requireNonNull(indexCreatedVersion); + this.indexCreatedVersion = Objects.requireNonNull(indexSettings.getIndexVersionCreated()); this.normalizer = Parameter.stringParam( "normalizer", indexCreatedVersion.isLegacyIndexVersion(), @@ -233,10 +251,13 @@ public Builder(final String name, final MappingParserContext mappingParserContex throw new IllegalArgumentException("[ignore_above] must be positive, got [" + v + "]"); } }); + this.indexSortConfig = indexSettings.getIndexSortConfig(); + this.indexMode = indexSettings.getMode(); + this.indexSettings = indexSettings; } - public Builder(String name, IndexVersion indexCreatedVersion) { - this(name, null, ScriptCompiler.NONE, Integer.MAX_VALUE, indexCreatedVersion); + public Builder(String name, IndexSettings indexSettings) { + this(name, null, ScriptCompiler.NONE, Integer.MAX_VALUE, indexSettings); } public Builder ignoreAbove(int ignoreAbove) { @@ -359,7 +380,7 @@ private KeywordFieldType buildFieldType(MapperBuilderContext context, FieldType @Override public KeywordFieldMapper build(MapperBuilderContext context) { - FieldType fieldtype = new FieldType(Defaults.FIELD_TYPE); + FieldType fieldtype = fieldType(indexSortConfig, indexMode, context.buildFullName(leafName())); fieldtype.setOmitNorms(this.hasNorms.getValue() == false); fieldtype.setIndexOptions(TextParams.toIndexOptions(this.indexed.getValue(), this.indexOptions.getValue())); fieldtype.setStored(this.stored.getValue()); @@ -368,6 +389,9 @@ public KeywordFieldMapper build(MapperBuilderContext context) { // deduplicate in the common default case to save some memory fieldtype = Defaults.FIELD_TYPE; } + if (fieldtype.equals(Defaults.FIELD_TYPE_WITH_SKIP_DOC_VALUES)) { + fieldtype = Defaults.FIELD_TYPE_WITH_SKIP_DOC_VALUES; + } super.hasScript = script.get() != null; super.onScriptError = onScriptError.getValue(); return new KeywordFieldMapper( @@ -376,9 +400,45 @@ public KeywordFieldMapper build(MapperBuilderContext context) { buildFieldType(context, fieldtype), builderParams(this, context), context.isSourceSynthetic(), + indexSettings, this ); } + + private FieldType fieldType(final IndexSortConfig indexSortConfig, final IndexMode indexMode, final String fullFieldName) { + if (shouldUseDocValuesSparseIndex(indexSortConfig, indexMode, fullFieldName)) { + return new FieldType(Defaults.FIELD_TYPE_WITH_SKIP_DOC_VALUES); + } + return new FieldType(Defaults.FIELD_TYPE); + } + + private boolean shouldUseDocValuesSparseIndex( + final IndexSortConfig indexSortConfig, + final IndexMode indexMode, + final String fullFieldName + ) { + return areDefaultSettingsUsed() + && isLogsDbMode(indexMode) + && isHostNameField(fullFieldName) + && isPrimarySortField(indexSortConfig); + } + + private boolean isHostNameField(final String fullFieldName) { + return HOST_NAME.equals(fullFieldName); + } + + private boolean isPrimarySortField(final IndexSortConfig indexSortConfig) { + return indexSortConfig != null && indexSortConfig.hasPrimarySortOnField(HOST_NAME); + } + + private boolean areDefaultSettingsUsed() { + return indexed.isConfigured() == false && hasDocValues.isConfigured() == false; + } + + private boolean isLogsDbMode(final IndexMode indexMode) { + return IndexMode.LOGSDB.equals(indexMode); + } + } public static final TypeParser PARSER = createTypeParserWithLegacySupport(Builder::new); @@ -392,6 +452,7 @@ public static final class KeywordFieldType extends StringFieldType { private final FieldValues scriptValues; private final boolean isDimension; private final boolean isSyntheticSource; + private final boolean hasDocValuesSparseIndex; public KeywordFieldType( String name, @@ -417,6 +478,7 @@ public KeywordFieldType( this.scriptValues = builder.scriptValues(); this.isDimension = builder.dimension.getValue(); this.isSyntheticSource = isSyntheticSource; + this.hasDocValuesSparseIndex = fieldType.docValuesSkipIndexType().equals(DocValuesSkipIndexType.RANGE); } public KeywordFieldType(String name, boolean isIndexed, boolean hasDocValues, Map meta) { @@ -428,6 +490,7 @@ public KeywordFieldType(String name, boolean isIndexed, boolean hasDocValues, Ma this.scriptValues = null; this.isDimension = false; this.isSyntheticSource = false; + this.hasDocValuesSparseIndex = false; } public KeywordFieldType(String name) { @@ -450,6 +513,7 @@ public KeywordFieldType(String name, FieldType fieldType) { this.scriptValues = null; this.isDimension = false; this.isSyntheticSource = false; + this.hasDocValuesSparseIndex = fieldType.docValuesSkipIndexType().equals(DocValuesSkipIndexType.RANGE); } public KeywordFieldType(String name, NamedAnalyzer analyzer) { @@ -461,6 +525,7 @@ public KeywordFieldType(String name, NamedAnalyzer analyzer) { this.scriptValues = null; this.isDimension = false; this.isSyntheticSource = false; + this.hasDocValuesSparseIndex = false; } @Override @@ -468,6 +533,11 @@ public boolean isSearchable() { return isIndexed() || hasDocValues(); } + @Override + public boolean hasDocValuesSparseIndex() { + return hasDocValuesSparseIndex; + } + @Override public Query termQuery(Object value, SearchExecutionContext context) { failIfNotIndexedNorDocValuesFallback(context); @@ -867,6 +937,10 @@ public boolean hasNormalizer() { private final IndexAnalyzers indexAnalyzers; private final int ignoreAboveDefault; private final int ignoreAbove; + private final IndexSortConfig indexSortConfig; + private final IndexMode indexMode; + + private final IndexSettings indexSettings; private KeywordFieldMapper( String simpleName, @@ -874,6 +948,7 @@ private KeywordFieldMapper( KeywordFieldType mappedFieldType, BuilderParams builderParams, boolean isSyntheticSource, + IndexSettings indexSettings, Builder builder ) { super(simpleName, mappedFieldType, builderParams); @@ -891,6 +966,9 @@ private KeywordFieldMapper( this.isSyntheticSource = isSyntheticSource; this.ignoreAboveDefault = builder.ignoreAboveDefault; this.ignoreAbove = builder.ignoreAbove.getValue(); + this.indexSortConfig = builder.indexSortConfig; + this.indexMode = builder.indexMode; + this.indexSettings = builder.indexSettings; } @Override @@ -1008,7 +1086,7 @@ public Map indexAnalyzers() { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(leafName(), indexAnalyzers, scriptCompiler, ignoreAboveDefault, indexCreatedVersion).dimension( + return new Builder(leafName(), indexAnalyzers, scriptCompiler, ignoreAboveDefault, indexSettings).dimension( fieldType().isDimension() ).init(this); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java index 4b68e20673572..1766c2d1faafb 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java @@ -155,6 +155,10 @@ public boolean isSearchable() { return isIndexed; } + public boolean hasDocValuesSparseIndex() { + return false; + } + /** * Returns true if the field is indexed. */ diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java b/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java index f809a53d753fb..b91046342d2fa 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java @@ -21,9 +21,11 @@ import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.store.ByteBuffersDirectory; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.cache.bitset.BitsetFilterCache; import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; @@ -86,11 +88,15 @@ protected Collection> getPlugins() { } public > IFD getForField(String type, String fieldName, boolean docValues) { + final IndexSettings indexSettings = new IndexSettings( + IndexMetadata.builder(IndexMetadata.INDEX_UUID_NA_VALUE).build(), + Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()).build() + ); final MappedFieldType fieldType; final MapperBuilderContext context = MapperBuilderContext.root(false, false); if (type.equals("string")) { if (docValues) { - fieldType = new KeywordFieldMapper.Builder(fieldName, IndexVersion.current()).build(context).fieldType(); + fieldType = new KeywordFieldMapper.Builder(fieldName, indexSettings).build(context).fieldType(); } else { fieldType = new TextFieldMapper.Builder( fieldName, diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java index 36c25b352a792..874f48a9887f4 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java @@ -21,9 +21,11 @@ import org.apache.lucene.store.ByteBuffersDirectory; import org.apache.lucene.util.Accountable; import org.apache.lucene.util.SetOnce; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.fielddata.plain.SortedDoublesIndexFieldData; import org.elasticsearch.index.fielddata.plain.SortedNumericIndexFieldData; @@ -71,6 +73,10 @@ protected Collection> getPlugins() { } public void testGetForFieldDefaults() { + final IndexSettings indexSettings = new IndexSettings( + IndexMetadata.builder(IndexMetadata.INDEX_UUID_NA_VALUE).build(), + Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()).build() + ); final IndexService indexService = createIndex("test"); final IndicesService indicesService = getInstanceFromNode(IndicesService.class); final IndexFieldDataService ifdService = new IndexFieldDataService( @@ -79,7 +85,7 @@ public void testGetForFieldDefaults() { indicesService.getCircuitBreakerService() ); MapperBuilderContext context = MapperBuilderContext.root(false, false); - final MappedFieldType stringMapper = new KeywordFieldMapper.Builder("string", IndexVersion.current()).build(context).fieldType(); + final MappedFieldType stringMapper = new KeywordFieldMapper.Builder("string", indexSettings).build(context).fieldType(); ifdService.clear(); IndexFieldData fd = ifdService.getForField(stringMapper, FieldDataContext.noRuntimeFields("test")); assertTrue(fd instanceof SortedSetOrdinalsIndexFieldData); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserContextTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserContextTests.java index be36ab9d6eac1..e89c84bd3230c 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserContextTests.java @@ -9,7 +9,9 @@ package org.elasticsearch.index.mapper; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.XContentBuilder; @@ -51,18 +53,26 @@ public void testDynamicMapperSizeSameFieldMultipleMappers() { } public void testAddRuntimeFieldWhenLimitIsReachedViaMapper() { + final IndexSettings indexSettings = new IndexSettings( + IndexMetadata.builder(IndexMetadata.INDEX_UUID_NA_VALUE).build(), + Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()).build() + ); context = new TestDocumentParserContext( Settings.builder() .put("index.mapping.total_fields.limit", 1) .put("index.mapping.total_fields.ignore_dynamic_beyond_limit", true) .build() ); - assertTrue(context.addDynamicMapper(new KeywordFieldMapper.Builder("keyword_field", IndexVersion.current()).build(root))); + assertTrue(context.addDynamicMapper(new KeywordFieldMapper.Builder("keyword_field", indexSettings).build(root))); assertFalse(context.addDynamicRuntimeField(new TestRuntimeField("runtime_field", "keyword"))); assertThat(context.getIgnoredFields(), contains("runtime_field")); } public void testAddFieldWhenLimitIsReachedViaRuntimeField() { + final IndexSettings indexSettings = new IndexSettings( + IndexMetadata.builder(IndexMetadata.INDEX_UUID_NA_VALUE).build(), + Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()).build() + ); context = new TestDocumentParserContext( Settings.builder() .put("index.mapping.total_fields.limit", 1) @@ -70,7 +80,7 @@ public void testAddFieldWhenLimitIsReachedViaRuntimeField() { .build() ); assertTrue(context.addDynamicRuntimeField(new TestRuntimeField("runtime_field", "keyword"))); - assertFalse(context.addDynamicMapper(new KeywordFieldMapper.Builder("keyword_field", IndexVersion.current()).build(root))); + assertFalse(context.addDynamicMapper(new KeywordFieldMapper.Builder("keyword_field", indexSettings).build(root))); assertThat(context.getIgnoredFields(), contains("keyword_field")); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java index 052bf995bdd48..b1900a39cc831 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java @@ -28,6 +28,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.IndexSortConfig; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.CharFilterFactory; @@ -770,4 +771,109 @@ public void testDocValuesLoadedFromStoredSynthetic() throws IOException { ); assertScriptDocValues(mapper, "foo", equalTo(List.of("foo"))); } + + public void testFieldTypeWithSkipDocValues_LogsDbMode() throws IOException { + final MapperService mapperService = createMapperService( + Settings.builder() + .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()) + .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "host.name") + .build(), + mapping(b -> { + b.startObject("host.name"); + b.field("type", "keyword"); + b.endObject(); + }) + ); + + final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); + assertTrue(mapper.fieldType().hasDocValuesSparseIndex()); + } + + public void testFieldTypeDefault_StandardMode() throws IOException { + final MapperService mapperService = createMapperService( + Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.name()).build(), + mapping(b -> { + b.startObject("host.name"); + b.field("type", "keyword"); + b.endObject(); + }) + ); + + final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); + assertFalse(mapper.fieldType().hasDocValuesSparseIndex()); + } + + public void testFieldTypeDefault_NonMatchingFieldName() throws IOException { + final MapperService mapperService = createMapperService( + Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()).build(), + mapping(b -> { + b.startObject("hostname"); + b.field("type", "keyword"); + b.endObject(); + }) + ); + + final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("hostname"); + assertFalse(mapper.fieldType().hasDocValuesSparseIndex()); + } + + public void testFieldTypeDefault_ConfiguredIndexed() throws IOException { + final MapperService mapperService = createMapperService( + Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()).build(), + mapping(b -> { + b.startObject("host.name"); + b.field("type", "keyword"); + b.field("index", true); + b.endObject(); + }) + ); + + final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); + assertFalse(mapper.fieldType().hasDocValuesSparseIndex()); + } + + public void testFieldTypeDefault_ConfiguredDocValues() throws IOException { + final MapperService mapperService = createMapperService( + Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()).build(), + mapping(b -> { + b.startObject("host.name"); + b.field("type", "keyword"); + b.field("doc_values", true); + b.endObject(); + }) + ); + + final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); + assertFalse(mapper.fieldType().hasDocValuesSparseIndex()); + } + + public void testFieldTypeDefault_ConfiguredIndexedAndDocValues() throws IOException { + final MapperService mapperService = createMapperService( + Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()).build(), + mapping(b -> { + b.startObject("host.name"); + b.field("type", "keyword"); + b.field("index", true); + b.field("doc_values", true); + b.endObject(); + }) + ); + + final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); + assertFalse(mapper.fieldType().hasDocValuesSparseIndex()); + } + + public void testFieldTypeDefault_LogsDbMode_NonSortField() throws IOException { + final MapperService mapperService = createMapperService( + Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()).build(), + mapping(b -> { + b.startObject("host.name"); + b.field("type", "keyword"); + b.endObject(); + }) + ); + + final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); + assertFalse(mapper.fieldType().hasDocValuesSparseIndex()); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java index e3bdb3d45818f..faecd297e26f9 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java @@ -36,9 +36,12 @@ import org.apache.lucene.tests.index.RandomIndexWriter; import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.common.lucene.Lucene; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.Fuzziness; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.CharFilterFactory; @@ -222,9 +225,12 @@ public void testNormalizeQueries() { } public void testFetchSourceValue() throws IOException { - MappedFieldType mapper = new KeywordFieldMapper.Builder("field", IndexVersion.current()).build( - MapperBuilderContext.root(false, false) - ).fieldType(); + final IndexSettings indexSettings = new IndexSettings( + IndexMetadata.builder(IndexMetadata.INDEX_UUID_NA_VALUE).build(), + Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()).build() + ); + MappedFieldType mapper = new KeywordFieldMapper.Builder("field", indexSettings).build(MapperBuilderContext.root(false, false)) + .fieldType(); assertEquals(List.of("value"), fetchSourceValue(mapper, "value")); assertEquals(List.of("42"), fetchSourceValue(mapper, 42L)); assertEquals(List.of("true"), fetchSourceValue(mapper, true)); @@ -232,7 +238,7 @@ public void testFetchSourceValue() throws IOException { IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> fetchSourceValue(mapper, "value", "format")); assertEquals("Field [field] of type [keyword] doesn't support formats.", e.getMessage()); - MappedFieldType ignoreAboveMapper = new KeywordFieldMapper.Builder("field", IndexVersion.current()).ignoreAbove(4) + MappedFieldType ignoreAboveMapper = new KeywordFieldMapper.Builder("field", indexSettings).ignoreAbove(4) .build(MapperBuilderContext.root(false, false)) .fieldType(); assertEquals(List.of(), fetchSourceValue(ignoreAboveMapper, "value")); @@ -244,13 +250,13 @@ public void testFetchSourceValue() throws IOException { createIndexAnalyzers(), ScriptCompiler.NONE, Integer.MAX_VALUE, - IndexVersion.current() + new IndexSettings(IndexMetadata.builder(IndexMetadata.INDEX_UUID_NA_VALUE).build(), Settings.builder().build()) ).normalizer("lowercase").build(MapperBuilderContext.root(false, false)).fieldType(); assertEquals(List.of("value"), fetchSourceValue(normalizerMapper, "VALUE")); assertEquals(List.of("42"), fetchSourceValue(normalizerMapper, 42L)); assertEquals(List.of("value"), fetchSourceValue(normalizerMapper, "value")); - MappedFieldType nullValueMapper = new KeywordFieldMapper.Builder("field", IndexVersion.current()).nullValue("NULL") + MappedFieldType nullValueMapper = new KeywordFieldMapper.Builder("field", indexSettings).nullValue("NULL") .build(MapperBuilderContext.root(false, false)) .fieldType(); assertEquals(List.of("NULL"), fetchSourceValue(nullValueMapper, null)); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MultiFieldsTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MultiFieldsTests.java index fd024c5d23e28..f58acd19c2a17 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/MultiFieldsTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/MultiFieldsTests.java @@ -9,7 +9,10 @@ package org.elasticsearch.index.mapper; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.lucene.Lucene; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.script.ScriptCompiler; @@ -59,12 +62,16 @@ public void testMultiFieldsBuilderHasSyntheticSourceCompatibleKeywordFieldDuring } private KeywordFieldMapper.Builder getKeywordFieldMapperBuilder(boolean isStored, boolean hasNormalizer) { + final IndexSettings indexSettings = new IndexSettings( + IndexMetadata.builder(IndexMetadata.INDEX_UUID_NA_VALUE).build(), + Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()).build() + ); var keywordFieldMapperBuilder = new KeywordFieldMapper.Builder( "field", IndexAnalyzers.of(Map.of(), Map.of("normalizer", Lucene.STANDARD_ANALYZER), Map.of()), ScriptCompiler.NONE, Integer.MAX_VALUE, - IndexVersion.current() + indexSettings ); if (isStored) { keywordFieldMapperBuilder.stored(true); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperMergeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperMergeTests.java index 1f8a2a754428b..ce5bd002a196d 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperMergeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperMergeTests.java @@ -8,6 +8,9 @@ */ package org.elasticsearch.index.mapper; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.script.ScriptCompiler; import org.elasticsearch.test.ESTestCase; @@ -215,11 +218,15 @@ public void testMergeWithLimit() { } public void testMergeWithLimitTruncatedObjectField() { + final IndexSettings indexSettings = new IndexSettings( + IndexMetadata.builder(IndexMetadata.INDEX_UUID_NA_VALUE).build(), + Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()).build() + ); RootObjectMapper root = new RootObjectMapper.Builder("_doc", Optional.empty()).build(MapperBuilderContext.root(false, false)); RootObjectMapper mergeWith = new RootObjectMapper.Builder("_doc", Optional.empty()).add( new ObjectMapper.Builder("parent", Optional.of(ObjectMapper.Subobjects.DISABLED)).add( - new KeywordFieldMapper.Builder("child1", IndexVersion.current()) - ).add(new KeywordFieldMapper.Builder("child2", IndexVersion.current())) + new KeywordFieldMapper.Builder("child1", indexSettings) + ).add(new KeywordFieldMapper.Builder("child2", indexSettings)) ).build(MapperBuilderContext.root(false, false)); ObjectMapper mergedAdd0 = root.merge(mergeWith, MapperMergeContext.root(false, false, MAPPING_UPDATE, 0)); @@ -246,13 +253,17 @@ public void testMergeWithLimitTruncatedObjectField() { } public void testMergeSameObjectDifferentFields() { + final IndexSettings indexSettings = new IndexSettings( + IndexMetadata.builder(IndexMetadata.INDEX_UUID_NA_VALUE).build(), + Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()).build() + ); RootObjectMapper root = new RootObjectMapper.Builder("_doc", Optional.empty()).add( - new ObjectMapper.Builder("parent", Optional.empty()).add(new KeywordFieldMapper.Builder("child1", IndexVersion.current())) + new ObjectMapper.Builder("parent", Optional.empty()).add(new KeywordFieldMapper.Builder("child1", indexSettings)) ).build(MapperBuilderContext.root(false, false)); RootObjectMapper mergeWith = new RootObjectMapper.Builder("_doc", Optional.empty()).add( new ObjectMapper.Builder("parent", Optional.empty()).add( - new KeywordFieldMapper.Builder("child1", IndexVersion.current()).ignoreAbove(42) - ).add(new KeywordFieldMapper.Builder("child2", IndexVersion.current())) + new KeywordFieldMapper.Builder("child1", indexSettings).ignoreAbove(42) + ).add(new KeywordFieldMapper.Builder("child2", indexSettings)) ).build(MapperBuilderContext.root(false, false)); ObjectMapper mergedAdd0 = root.merge(mergeWith, MapperMergeContext.root(false, false, MAPPING_UPDATE, 0)); @@ -307,14 +318,16 @@ public void testMergeWithLimitRuntimeField() { } public void testMergeSubobjectsFalseWithObject() { + final IndexSettings indexSettings = new IndexSettings( + IndexMetadata.builder(IndexMetadata.INDEX_UUID_NA_VALUE).build(), + Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()).build() + ); RootObjectMapper mergeInto = new RootObjectMapper.Builder("_doc", Optional.empty()).add( new ObjectMapper.Builder("parent", Optional.of(ObjectMapper.Subobjects.DISABLED)) ).build(MapperBuilderContext.root(false, false)); RootObjectMapper mergeWith = new RootObjectMapper.Builder("_doc", Optional.empty()).add( new ObjectMapper.Builder("parent", Optional.empty()).add( - new ObjectMapper.Builder("child", Optional.empty()).add( - new KeywordFieldMapper.Builder("grandchild", IndexVersion.current()) - ) + new ObjectMapper.Builder("child", Optional.empty()).add(new KeywordFieldMapper.Builder("grandchild", indexSettings)) ) ).build(MapperBuilderContext.root(false, false)); @@ -325,8 +338,12 @@ public void testMergeSubobjectsFalseWithObject() { } public void testConflictingDynamicUpdate() { + final IndexSettings indexSettings = new IndexSettings( + IndexMetadata.builder(IndexMetadata.INDEX_UUID_NA_VALUE).build(), + Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()).build() + ); RootObjectMapper mergeInto = new RootObjectMapper.Builder("_doc", Optional.empty()).add( - new KeywordFieldMapper.Builder("http.status_code", IndexVersion.current()) + new KeywordFieldMapper.Builder("http.status_code", indexSettings) ).build(MapperBuilderContext.root(false, false)); RootObjectMapper mergeWith = new RootObjectMapper.Builder("_doc", Optional.empty()).add( new NumberFieldMapper.Builder( @@ -353,7 +370,11 @@ public void testConflictingDynamicUpdate() { } private static RootObjectMapper createRootSubobjectFalseLeafWithDots() { - FieldMapper.Builder fieldBuilder = new KeywordFieldMapper.Builder("host.name", IndexVersion.current()); + final IndexSettings indexSettings = new IndexSettings( + IndexMetadata.builder(IndexMetadata.INDEX_UUID_NA_VALUE).build(), + Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()).build() + ); + FieldMapper.Builder fieldBuilder = new KeywordFieldMapper.Builder("host.name", indexSettings); FieldMapper fieldMapper = fieldBuilder.build(MapperBuilderContext.root(false, false)); assertEquals("host.name", fieldMapper.leafName()); assertEquals("host.name", fieldMapper.fullPath()); @@ -362,7 +383,11 @@ private static RootObjectMapper createRootSubobjectFalseLeafWithDots() { } private static ObjectMapper.Builder createObjectSubobjectsFalseLeafWithDots() { - KeywordFieldMapper.Builder fieldBuilder = new KeywordFieldMapper.Builder("host.name", IndexVersion.current()); + final IndexSettings indexSettings = new IndexSettings( + IndexMetadata.builder(IndexMetadata.INDEX_UUID_NA_VALUE).build(), + Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()).build() + ); + KeywordFieldMapper.Builder fieldBuilder = new KeywordFieldMapper.Builder("host.name", indexSettings); KeywordFieldMapper fieldMapper = fieldBuilder.build( new MapperBuilderContext( "foo.metrics", @@ -409,8 +434,12 @@ private TextFieldMapper.Builder createTextKeywordMultiField(String name) { } private TextFieldMapper.Builder createTextKeywordMultiField(String name, String multiFieldName) { + final IndexSettings indexSettings = new IndexSettings( + IndexMetadata.builder(IndexMetadata.INDEX_UUID_NA_VALUE).build(), + Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()).build() + ); TextFieldMapper.Builder builder = new TextFieldMapper.Builder(name, createDefaultIndexAnalyzers(), false); - builder.multiFieldsBuilder.add(new KeywordFieldMapper.Builder(multiFieldName, IndexVersion.current())); + builder.multiFieldsBuilder.add(new KeywordFieldMapper.Builder(multiFieldName, indexSettings)); return builder; } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java index 8582f26349ef2..63ff3a0938f21 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java @@ -9,12 +9,14 @@ package org.elasticsearch.index.mapper; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.mapper.MapperService.MergeReason; import org.elasticsearch.index.mapper.ObjectMapper.Dynamic; @@ -733,14 +735,18 @@ public void testStoreArraySourceNoopInNonSyntheticSourceMode() throws IOExceptio } public void testNestedObjectWithMultiFieldsgetTotalFieldsCount() { + final IndexSettings indexSettings = new IndexSettings( + IndexMetadata.builder(IndexMetadata.INDEX_UUID_NA_VALUE).build(), + Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()).build() + ); ObjectMapper.Builder mapperBuilder = new ObjectMapper.Builder("parent_size_1", Optional.empty()).add( new ObjectMapper.Builder("child_size_2", Optional.empty()).add( new TextFieldMapper.Builder("grand_child_size_3", createDefaultIndexAnalyzers(), false).addMultiField( - new KeywordFieldMapper.Builder("multi_field_size_4", IndexVersion.current()) + new KeywordFieldMapper.Builder("multi_field_size_4", indexSettings) ) .addMultiField( new TextFieldMapper.Builder("grand_child_size_5", createDefaultIndexAnalyzers(), false).addMultiField( - new KeywordFieldMapper.Builder("multi_field_of_multi_field_size_6", IndexVersion.current()) + new KeywordFieldMapper.Builder("multi_field_of_multi_field_size_6", indexSettings) ) ) ) @@ -780,28 +786,40 @@ private ObjectMapper createObjectMapperWithAllParametersSet(CheckedConsumer fields = objectMapper.asFlattenedFieldMappers(rootContext).stream().map(FieldMapper::fullPath).toList(); assertThat(fields, containsInAnyOrder("parent.keyword1", "parent.child.keyword2")); } public void testFlattenSubobjectsAuto() { + final IndexSettings indexSettings = new IndexSettings( + IndexMetadata.builder(IndexMetadata.INDEX_UUID_NA_VALUE).build(), + Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()).build() + ); MapperBuilderContext rootContext = MapperBuilderContext.root(false, false); ObjectMapper objectMapper = new ObjectMapper.Builder("parent", Optional.of(ObjectMapper.Subobjects.AUTO)).add( - new ObjectMapper.Builder("child", Optional.empty()).add(new KeywordFieldMapper.Builder("keyword2", IndexVersion.current())) - ).add(new KeywordFieldMapper.Builder("keyword1", IndexVersion.current())).build(rootContext); + new ObjectMapper.Builder("child", Optional.empty()).add(new KeywordFieldMapper.Builder("keyword2", indexSettings)) + ).add(new KeywordFieldMapper.Builder("keyword1", indexSettings)).build(rootContext); List fields = objectMapper.asFlattenedFieldMappers(rootContext).stream().map(FieldMapper::fullPath).toList(); assertThat(fields, containsInAnyOrder("parent.keyword1", "parent.child.keyword2")); } public void testFlattenSubobjectsFalse() { + final IndexSettings indexSettings = new IndexSettings( + IndexMetadata.builder(IndexMetadata.INDEX_UUID_NA_VALUE).build(), + Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()).build() + ); MapperBuilderContext rootContext = MapperBuilderContext.root(false, false); ObjectMapper objectMapper = new ObjectMapper.Builder("parent", Optional.of(ObjectMapper.Subobjects.DISABLED)).add( - new ObjectMapper.Builder("child", Optional.empty()).add(new KeywordFieldMapper.Builder("keyword2", IndexVersion.current())) - ).add(new KeywordFieldMapper.Builder("keyword1", IndexVersion.current())).build(rootContext); + new ObjectMapper.Builder("child", Optional.empty()).add(new KeywordFieldMapper.Builder("keyword2", indexSettings)) + ).add(new KeywordFieldMapper.Builder("keyword1", indexSettings)).build(rootContext); List fields = objectMapper.asFlattenedFieldMappers(rootContext).stream().map(FieldMapper::fullPath).toList(); assertThat(fields, containsInAnyOrder("parent.keyword1", "parent.child.keyword2")); } @@ -855,22 +873,26 @@ public void testFlattenExplicitSubobjectsTrue() { } public void testFindParentMapper() { + final IndexSettings indexSettings = new IndexSettings( + IndexMetadata.builder(IndexMetadata.INDEX_UUID_NA_VALUE).build(), + Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()).build() + ); MapperBuilderContext rootContext = MapperBuilderContext.root(false, false); var rootBuilder = new RootObjectMapper.Builder("_doc", Optional.empty()); - rootBuilder.add(new KeywordFieldMapper.Builder("keyword", IndexVersion.current())); + rootBuilder.add(new KeywordFieldMapper.Builder("keyword", indexSettings)); var child = new ObjectMapper.Builder("child", Optional.empty()); - child.add(new KeywordFieldMapper.Builder("keyword2", IndexVersion.current())); - child.add(new KeywordFieldMapper.Builder("keyword.with.dot", IndexVersion.current())); + child.add(new KeywordFieldMapper.Builder("keyword2", indexSettings)); + child.add(new KeywordFieldMapper.Builder("keyword.with.dot", indexSettings)); var secondLevelChild = new ObjectMapper.Builder("child2", Optional.empty()); - secondLevelChild.add(new KeywordFieldMapper.Builder("keyword22", IndexVersion.current())); + secondLevelChild.add(new KeywordFieldMapper.Builder("keyword22", indexSettings)); child.add(secondLevelChild); rootBuilder.add(child); var childWithDot = new ObjectMapper.Builder("childwith.dot", Optional.empty()); - childWithDot.add(new KeywordFieldMapper.Builder("keyword3", IndexVersion.current())); - childWithDot.add(new KeywordFieldMapper.Builder("keyword4.with.dot", IndexVersion.current())); + childWithDot.add(new KeywordFieldMapper.Builder("keyword3", indexSettings)); + childWithDot.add(new KeywordFieldMapper.Builder("keyword4.with.dot", indexSettings)); rootBuilder.add(childWithDot); RootObjectMapper root = rootBuilder.build(rootContext); diff --git a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java index dc70c44a89128..18cb4a977dfd6 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java @@ -383,10 +383,14 @@ public void testSearchRequestRuntimeFieldsAndMultifieldDetection() { } public void testSyntheticSourceSearchLookup() throws IOException { + final IndexSettings indexSettings = new IndexSettings( + IndexMetadata.builder(IndexMetadata.INDEX_UUID_NA_VALUE).build(), + Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()).build() + ); // Build a mapping using synthetic source SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(null, Settings.EMPTY, false, false).setSynthetic().build(); RootObjectMapper root = new RootObjectMapper.Builder("_doc", Optional.empty()).add( - new KeywordFieldMapper.Builder("cat", IndexVersion.current()).ignoreAbove(100) + new KeywordFieldMapper.Builder("cat", indexSettings).ignoreAbove(100) ).build(MapperBuilderContext.root(true, false)); Mapping mapping = new Mapping(root, new MetadataFieldMapper[] { sourceMapper }, Map.of()); MappingLookup lookup = MappingLookup.fromMapping(mapping); diff --git a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java index d041121b8a96b..a7558b361aa2f 100644 --- a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java +++ b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java @@ -223,7 +223,7 @@ private SearchExecutionContext createSearchExecutionContext( new MetadataFieldMapper[0], Collections.emptyMap() ); - KeywordFieldMapper keywordFieldMapper = new KeywordFieldMapper.Builder("field", IndexVersion.current()).build(root); + KeywordFieldMapper keywordFieldMapper = new KeywordFieldMapper.Builder("field", indexSettings).build(root); MappingLookup mappingLookup = MappingLookup.fromMappers( mapping, Collections.singletonList(keywordFieldMapper), From 6476c15444511c13665bde8c66c01204da1e170d Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 23 Jan 2025 17:57:09 +0100 Subject: [PATCH 02/13] fix: enable sparse index if field is indexed nad has doc values --- .../index/mapper/KeywordFieldMapper.java | 6 +- .../index/mapper/KeywordFieldMapperTests.java | 67 +++++++++++++++++++ 2 files changed, 72 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 6b9b4dacac6e3..efea157084641 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -417,7 +417,7 @@ private boolean shouldUseDocValuesSparseIndex( final IndexMode indexMode, final String fullFieldName ) { - return areDefaultSettingsUsed() + return (areDefaultSettingsUsed() || isNotIndexedAndHasDocValues()) && isLogsDbMode(indexMode) && isHostNameField(fullFieldName) && isPrimarySortField(indexSortConfig); @@ -435,6 +435,10 @@ private boolean areDefaultSettingsUsed() { return indexed.isConfigured() == false && hasDocValues.isConfigured() == false; } + private boolean isNotIndexedAndHasDocValues() { + return indexed.getValue() == false && hasDocValues.getValue(); + } + private boolean isLogsDbMode(final IndexMode indexMode) { return IndexMode.LOGSDB.equals(indexMode); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java index b1900a39cc831..c547072ccbdce 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java @@ -876,4 +876,71 @@ public void testFieldTypeDefault_LogsDbMode_NonSortField() throws IOException { final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); assertFalse(mapper.fieldType().hasDocValuesSparseIndex()); } + + public void testFieldTypeWithSkipDocValues_IndexedFalseDocValuesTrue() throws IOException { + final MapperService mapperService = createMapperService( + Settings.builder() + .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()) + .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "host.name") + .build(), + mapping(b -> { + b.startObject("host.name"); + b.field("type", "keyword"); + b.field("index", false); + b.field("doc_values", true); + b.endObject(); + }) + ); + + final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); + assertTrue(mapper.fieldType().hasDocValuesSparseIndex()); + } + + public void testFieldTypeDefault_IndexedFalseDocValuesFalse() throws IOException { + final MapperService mapperService = createMapperService( + Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()).build(), + mapping(b -> { + b.startObject("host.name"); + b.field("type", "keyword"); + b.field("index", false); + b.field("doc_values", false); + b.endObject(); + }) + ); + + final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); + assertFalse(mapper.fieldType().hasDocValuesSparseIndex()); + } + + public void testFieldTypeDefault_IndexedTrueDocValuesTrue() throws IOException { + final MapperService mapperService = createMapperService( + Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()).build(), + mapping(b -> { + b.startObject("host.name"); + b.field("type", "keyword"); + b.field("index", true); + b.field("doc_values", true); + b.endObject(); + }) + ); + + final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); + assertFalse(mapper.fieldType().hasDocValuesSparseIndex()); + } + + public void testFieldTypeDefault_StandardModeIndexedFalseDocValuesTrue() throws IOException { + final MapperService mapperService = createMapperService( + Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.name()).build(), + mapping(b -> { + b.startObject("host.name"); + b.field("type", "keyword"); + b.field("index", false); + b.field("doc_values", true); + b.endObject(); + }) + ); + + final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); + assertFalse(mapper.fieldType().hasDocValuesSparseIndex()); + } } From 47039a9bc2dcc0317780898c45155f14efd1caf5 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Thu, 23 Jan 2025 18:02:14 +0100 Subject: [PATCH 03/13] fix: rename method and make constant private --- .../org/elasticsearch/index/mapper/KeywordFieldMapper.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index efea157084641..7f90e4e15a2e6 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -91,7 +91,7 @@ public final class KeywordFieldMapper extends FieldMapper { private static final Logger logger = LogManager.getLogger(KeywordFieldMapper.class); public static final String CONTENT_TYPE = "keyword"; - public static final String HOST_NAME = "host.name"; + private static final String HOST_NAME = "host.name"; public static class Defaults { public static final FieldType FIELD_TYPE; @@ -417,7 +417,7 @@ private boolean shouldUseDocValuesSparseIndex( final IndexMode indexMode, final String fullFieldName ) { - return (areDefaultSettingsUsed() || isNotIndexedAndHasDocValues()) + return (defaultIndexedAndDocValues() || isNotIndexedAndHasDocValues()) && isLogsDbMode(indexMode) && isHostNameField(fullFieldName) && isPrimarySortField(indexSortConfig); @@ -431,7 +431,7 @@ private boolean isPrimarySortField(final IndexSortConfig indexSortConfig) { return indexSortConfig != null && indexSortConfig.hasPrimarySortOnField(HOST_NAME); } - private boolean areDefaultSettingsUsed() { + private boolean defaultIndexedAndDocValues() { return indexed.isConfigured() == false && hasDocValues.isConfigured() == false; } From 15ecf2292a6c632b9cc65008da0964bb66dc2be6 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 24 Jan 2025 09:52:33 +0100 Subject: [PATCH 04/13] fix: keep existing Builder --- .../index/mapper/DynamicFieldsBuilder.java | 10 +++- .../index/mapper/KeywordFieldMapper.java | 25 ++++++++- .../fielddata/AbstractFieldDataTestCase.java | 8 +-- .../fielddata/IndexFieldDataServiceTests.java | 8 +-- .../mapper/DocumentParserContextTests.java | 14 +---- .../index/mapper/KeywordFieldTypeTests.java | 18 +++---- .../index/mapper/MultiFieldsTests.java | 9 +--- .../index/mapper/ObjectMapperMergeTests.java | 53 +++++-------------- .../index/mapper/ObjectMapperTests.java | 50 +++++------------ .../query/SearchExecutionContextTests.java | 6 +-- .../search/SearchServiceTests.java | 2 +- 11 files changed, 70 insertions(+), 133 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java b/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java index 6ecde8967cfbf..0793dd748c67e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java @@ -327,11 +327,17 @@ boolean createDynamicField(Mapper.Builder builder, DocumentParserContext context public boolean newDynamicStringField(DocumentParserContext context, String name) throws IOException { MapperBuilderContext mapperBuilderContext = context.createDynamicMapperBuilderContext(); if (mapperBuilderContext.parentObjectContainsDimensions()) { - return createDynamicField(new KeywordFieldMapper.Builder(name, context.indexSettings()), context, mapperBuilderContext); + return createDynamicField( + new KeywordFieldMapper.Builder(name, context.indexSettings().getIndexVersionCreated()), + context, + mapperBuilderContext + ); } else { return createDynamicField( new TextFieldMapper.Builder(name, context.indexAnalyzers(), SourceFieldMapper.isSynthetic(context.indexSettings())) - .addMultiField(new KeywordFieldMapper.Builder("keyword", context.indexSettings()).ignoreAbove(256)), + .addMultiField( + new KeywordFieldMapper.Builder("keyword", context.indexSettings().getIndexVersionCreated()).ignoreAbove(256) + ), context ); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 7f90e4e15a2e6..f31e3283a13e6 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -34,9 +34,11 @@ import org.apache.lucene.util.automaton.CompiledAutomaton; import org.apache.lucene.util.automaton.CompiledAutomaton.AUTOMATON_TYPE; import org.apache.lucene.util.automaton.Operations; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.lucene.search.AutomatonQueries; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.core.Nullable; import org.elasticsearch.index.IndexMode; @@ -212,6 +214,25 @@ public Builder(final String name, final MappingParserContext mappingParserContex } Builder( + String name, + IndexAnalyzers indexAnalyzers, + ScriptCompiler scriptCompiler, + int ignoreAboveDefault, + IndexVersion indexCreatedVersion + ) { + this( + name, + indexAnalyzers, + scriptCompiler, + ignoreAboveDefault, + new IndexSettings( + IndexMetadata.builder(IndexMetadata.INDEX_UUID_NA_VALUE).build(), + Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, indexCreatedVersion).build() + ) + ); + } + + private Builder( String name, IndexAnalyzers indexAnalyzers, ScriptCompiler scriptCompiler, @@ -256,8 +277,8 @@ public Builder(final String name, final MappingParserContext mappingParserContex this.indexSettings = indexSettings; } - public Builder(String name, IndexSettings indexSettings) { - this(name, null, ScriptCompiler.NONE, Integer.MAX_VALUE, indexSettings); + public Builder(String name, IndexVersion indexCreatedVersion) { + this(name, null, ScriptCompiler.NONE, Integer.MAX_VALUE, indexCreatedVersion); } public Builder ignoreAbove(int ignoreAbove) { diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java b/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java index b91046342d2fa..f809a53d753fb 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java @@ -21,11 +21,9 @@ import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.store.ByteBuffersDirectory; -import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexService; -import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.cache.bitset.BitsetFilterCache; import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; @@ -88,15 +86,11 @@ protected Collection> getPlugins() { } public > IFD getForField(String type, String fieldName, boolean docValues) { - final IndexSettings indexSettings = new IndexSettings( - IndexMetadata.builder(IndexMetadata.INDEX_UUID_NA_VALUE).build(), - Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()).build() - ); final MappedFieldType fieldType; final MapperBuilderContext context = MapperBuilderContext.root(false, false); if (type.equals("string")) { if (docValues) { - fieldType = new KeywordFieldMapper.Builder(fieldName, indexSettings).build(context).fieldType(); + fieldType = new KeywordFieldMapper.Builder(fieldName, IndexVersion.current()).build(context).fieldType(); } else { fieldType = new TextFieldMapper.Builder( fieldName, diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java index 874f48a9887f4..36c25b352a792 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java @@ -21,11 +21,9 @@ import org.apache.lucene.store.ByteBuffersDirectory; import org.apache.lucene.util.Accountable; import org.apache.lucene.util.SetOnce; -import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexService; -import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.fielddata.plain.SortedDoublesIndexFieldData; import org.elasticsearch.index.fielddata.plain.SortedNumericIndexFieldData; @@ -73,10 +71,6 @@ protected Collection> getPlugins() { } public void testGetForFieldDefaults() { - final IndexSettings indexSettings = new IndexSettings( - IndexMetadata.builder(IndexMetadata.INDEX_UUID_NA_VALUE).build(), - Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()).build() - ); final IndexService indexService = createIndex("test"); final IndicesService indicesService = getInstanceFromNode(IndicesService.class); final IndexFieldDataService ifdService = new IndexFieldDataService( @@ -85,7 +79,7 @@ public void testGetForFieldDefaults() { indicesService.getCircuitBreakerService() ); MapperBuilderContext context = MapperBuilderContext.root(false, false); - final MappedFieldType stringMapper = new KeywordFieldMapper.Builder("string", indexSettings).build(context).fieldType(); + final MappedFieldType stringMapper = new KeywordFieldMapper.Builder("string", IndexVersion.current()).build(context).fieldType(); ifdService.clear(); IndexFieldData fd = ifdService.getForField(stringMapper, FieldDataContext.noRuntimeFields("test")); assertTrue(fd instanceof SortedSetOrdinalsIndexFieldData); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserContextTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserContextTests.java index e89c84bd3230c..be36ab9d6eac1 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserContextTests.java @@ -9,9 +9,7 @@ package org.elasticsearch.index.mapper; -import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.XContentBuilder; @@ -53,26 +51,18 @@ public void testDynamicMapperSizeSameFieldMultipleMappers() { } public void testAddRuntimeFieldWhenLimitIsReachedViaMapper() { - final IndexSettings indexSettings = new IndexSettings( - IndexMetadata.builder(IndexMetadata.INDEX_UUID_NA_VALUE).build(), - Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()).build() - ); context = new TestDocumentParserContext( Settings.builder() .put("index.mapping.total_fields.limit", 1) .put("index.mapping.total_fields.ignore_dynamic_beyond_limit", true) .build() ); - assertTrue(context.addDynamicMapper(new KeywordFieldMapper.Builder("keyword_field", indexSettings).build(root))); + assertTrue(context.addDynamicMapper(new KeywordFieldMapper.Builder("keyword_field", IndexVersion.current()).build(root))); assertFalse(context.addDynamicRuntimeField(new TestRuntimeField("runtime_field", "keyword"))); assertThat(context.getIgnoredFields(), contains("runtime_field")); } public void testAddFieldWhenLimitIsReachedViaRuntimeField() { - final IndexSettings indexSettings = new IndexSettings( - IndexMetadata.builder(IndexMetadata.INDEX_UUID_NA_VALUE).build(), - Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()).build() - ); context = new TestDocumentParserContext( Settings.builder() .put("index.mapping.total_fields.limit", 1) @@ -80,7 +70,7 @@ public void testAddFieldWhenLimitIsReachedViaRuntimeField() { .build() ); assertTrue(context.addDynamicRuntimeField(new TestRuntimeField("runtime_field", "keyword"))); - assertFalse(context.addDynamicMapper(new KeywordFieldMapper.Builder("keyword_field", indexSettings).build(root))); + assertFalse(context.addDynamicMapper(new KeywordFieldMapper.Builder("keyword_field", IndexVersion.current()).build(root))); assertThat(context.getIgnoredFields(), contains("keyword_field")); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java index faecd297e26f9..e3bdb3d45818f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java @@ -36,12 +36,9 @@ import org.apache.lucene.tests.index.RandomIndexWriter; import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.common.lucene.Lucene; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.Fuzziness; -import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.CharFilterFactory; @@ -225,12 +222,9 @@ public void testNormalizeQueries() { } public void testFetchSourceValue() throws IOException { - final IndexSettings indexSettings = new IndexSettings( - IndexMetadata.builder(IndexMetadata.INDEX_UUID_NA_VALUE).build(), - Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()).build() - ); - MappedFieldType mapper = new KeywordFieldMapper.Builder("field", indexSettings).build(MapperBuilderContext.root(false, false)) - .fieldType(); + MappedFieldType mapper = new KeywordFieldMapper.Builder("field", IndexVersion.current()).build( + MapperBuilderContext.root(false, false) + ).fieldType(); assertEquals(List.of("value"), fetchSourceValue(mapper, "value")); assertEquals(List.of("42"), fetchSourceValue(mapper, 42L)); assertEquals(List.of("true"), fetchSourceValue(mapper, true)); @@ -238,7 +232,7 @@ public void testFetchSourceValue() throws IOException { IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> fetchSourceValue(mapper, "value", "format")); assertEquals("Field [field] of type [keyword] doesn't support formats.", e.getMessage()); - MappedFieldType ignoreAboveMapper = new KeywordFieldMapper.Builder("field", indexSettings).ignoreAbove(4) + MappedFieldType ignoreAboveMapper = new KeywordFieldMapper.Builder("field", IndexVersion.current()).ignoreAbove(4) .build(MapperBuilderContext.root(false, false)) .fieldType(); assertEquals(List.of(), fetchSourceValue(ignoreAboveMapper, "value")); @@ -250,13 +244,13 @@ public void testFetchSourceValue() throws IOException { createIndexAnalyzers(), ScriptCompiler.NONE, Integer.MAX_VALUE, - new IndexSettings(IndexMetadata.builder(IndexMetadata.INDEX_UUID_NA_VALUE).build(), Settings.builder().build()) + IndexVersion.current() ).normalizer("lowercase").build(MapperBuilderContext.root(false, false)).fieldType(); assertEquals(List.of("value"), fetchSourceValue(normalizerMapper, "VALUE")); assertEquals(List.of("42"), fetchSourceValue(normalizerMapper, 42L)); assertEquals(List.of("value"), fetchSourceValue(normalizerMapper, "value")); - MappedFieldType nullValueMapper = new KeywordFieldMapper.Builder("field", indexSettings).nullValue("NULL") + MappedFieldType nullValueMapper = new KeywordFieldMapper.Builder("field", IndexVersion.current()).nullValue("NULL") .build(MapperBuilderContext.root(false, false)) .fieldType(); assertEquals(List.of("NULL"), fetchSourceValue(nullValueMapper, null)); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MultiFieldsTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MultiFieldsTests.java index f58acd19c2a17..fd024c5d23e28 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/MultiFieldsTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/MultiFieldsTests.java @@ -9,10 +9,7 @@ package org.elasticsearch.index.mapper; -import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.lucene.Lucene; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.script.ScriptCompiler; @@ -62,16 +59,12 @@ public void testMultiFieldsBuilderHasSyntheticSourceCompatibleKeywordFieldDuring } private KeywordFieldMapper.Builder getKeywordFieldMapperBuilder(boolean isStored, boolean hasNormalizer) { - final IndexSettings indexSettings = new IndexSettings( - IndexMetadata.builder(IndexMetadata.INDEX_UUID_NA_VALUE).build(), - Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()).build() - ); var keywordFieldMapperBuilder = new KeywordFieldMapper.Builder( "field", IndexAnalyzers.of(Map.of(), Map.of("normalizer", Lucene.STANDARD_ANALYZER), Map.of()), ScriptCompiler.NONE, Integer.MAX_VALUE, - indexSettings + IndexVersion.current() ); if (isStored) { keywordFieldMapperBuilder.stored(true); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperMergeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperMergeTests.java index ce5bd002a196d..1f8a2a754428b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperMergeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperMergeTests.java @@ -8,9 +8,6 @@ */ package org.elasticsearch.index.mapper; -import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.script.ScriptCompiler; import org.elasticsearch.test.ESTestCase; @@ -218,15 +215,11 @@ public void testMergeWithLimit() { } public void testMergeWithLimitTruncatedObjectField() { - final IndexSettings indexSettings = new IndexSettings( - IndexMetadata.builder(IndexMetadata.INDEX_UUID_NA_VALUE).build(), - Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()).build() - ); RootObjectMapper root = new RootObjectMapper.Builder("_doc", Optional.empty()).build(MapperBuilderContext.root(false, false)); RootObjectMapper mergeWith = new RootObjectMapper.Builder("_doc", Optional.empty()).add( new ObjectMapper.Builder("parent", Optional.of(ObjectMapper.Subobjects.DISABLED)).add( - new KeywordFieldMapper.Builder("child1", indexSettings) - ).add(new KeywordFieldMapper.Builder("child2", indexSettings)) + new KeywordFieldMapper.Builder("child1", IndexVersion.current()) + ).add(new KeywordFieldMapper.Builder("child2", IndexVersion.current())) ).build(MapperBuilderContext.root(false, false)); ObjectMapper mergedAdd0 = root.merge(mergeWith, MapperMergeContext.root(false, false, MAPPING_UPDATE, 0)); @@ -253,17 +246,13 @@ public void testMergeWithLimitTruncatedObjectField() { } public void testMergeSameObjectDifferentFields() { - final IndexSettings indexSettings = new IndexSettings( - IndexMetadata.builder(IndexMetadata.INDEX_UUID_NA_VALUE).build(), - Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()).build() - ); RootObjectMapper root = new RootObjectMapper.Builder("_doc", Optional.empty()).add( - new ObjectMapper.Builder("parent", Optional.empty()).add(new KeywordFieldMapper.Builder("child1", indexSettings)) + new ObjectMapper.Builder("parent", Optional.empty()).add(new KeywordFieldMapper.Builder("child1", IndexVersion.current())) ).build(MapperBuilderContext.root(false, false)); RootObjectMapper mergeWith = new RootObjectMapper.Builder("_doc", Optional.empty()).add( new ObjectMapper.Builder("parent", Optional.empty()).add( - new KeywordFieldMapper.Builder("child1", indexSettings).ignoreAbove(42) - ).add(new KeywordFieldMapper.Builder("child2", indexSettings)) + new KeywordFieldMapper.Builder("child1", IndexVersion.current()).ignoreAbove(42) + ).add(new KeywordFieldMapper.Builder("child2", IndexVersion.current())) ).build(MapperBuilderContext.root(false, false)); ObjectMapper mergedAdd0 = root.merge(mergeWith, MapperMergeContext.root(false, false, MAPPING_UPDATE, 0)); @@ -318,16 +307,14 @@ public void testMergeWithLimitRuntimeField() { } public void testMergeSubobjectsFalseWithObject() { - final IndexSettings indexSettings = new IndexSettings( - IndexMetadata.builder(IndexMetadata.INDEX_UUID_NA_VALUE).build(), - Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()).build() - ); RootObjectMapper mergeInto = new RootObjectMapper.Builder("_doc", Optional.empty()).add( new ObjectMapper.Builder("parent", Optional.of(ObjectMapper.Subobjects.DISABLED)) ).build(MapperBuilderContext.root(false, false)); RootObjectMapper mergeWith = new RootObjectMapper.Builder("_doc", Optional.empty()).add( new ObjectMapper.Builder("parent", Optional.empty()).add( - new ObjectMapper.Builder("child", Optional.empty()).add(new KeywordFieldMapper.Builder("grandchild", indexSettings)) + new ObjectMapper.Builder("child", Optional.empty()).add( + new KeywordFieldMapper.Builder("grandchild", IndexVersion.current()) + ) ) ).build(MapperBuilderContext.root(false, false)); @@ -338,12 +325,8 @@ public void testMergeSubobjectsFalseWithObject() { } public void testConflictingDynamicUpdate() { - final IndexSettings indexSettings = new IndexSettings( - IndexMetadata.builder(IndexMetadata.INDEX_UUID_NA_VALUE).build(), - Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()).build() - ); RootObjectMapper mergeInto = new RootObjectMapper.Builder("_doc", Optional.empty()).add( - new KeywordFieldMapper.Builder("http.status_code", indexSettings) + new KeywordFieldMapper.Builder("http.status_code", IndexVersion.current()) ).build(MapperBuilderContext.root(false, false)); RootObjectMapper mergeWith = new RootObjectMapper.Builder("_doc", Optional.empty()).add( new NumberFieldMapper.Builder( @@ -370,11 +353,7 @@ public void testConflictingDynamicUpdate() { } private static RootObjectMapper createRootSubobjectFalseLeafWithDots() { - final IndexSettings indexSettings = new IndexSettings( - IndexMetadata.builder(IndexMetadata.INDEX_UUID_NA_VALUE).build(), - Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()).build() - ); - FieldMapper.Builder fieldBuilder = new KeywordFieldMapper.Builder("host.name", indexSettings); + FieldMapper.Builder fieldBuilder = new KeywordFieldMapper.Builder("host.name", IndexVersion.current()); FieldMapper fieldMapper = fieldBuilder.build(MapperBuilderContext.root(false, false)); assertEquals("host.name", fieldMapper.leafName()); assertEquals("host.name", fieldMapper.fullPath()); @@ -383,11 +362,7 @@ private static RootObjectMapper createRootSubobjectFalseLeafWithDots() { } private static ObjectMapper.Builder createObjectSubobjectsFalseLeafWithDots() { - final IndexSettings indexSettings = new IndexSettings( - IndexMetadata.builder(IndexMetadata.INDEX_UUID_NA_VALUE).build(), - Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()).build() - ); - KeywordFieldMapper.Builder fieldBuilder = new KeywordFieldMapper.Builder("host.name", indexSettings); + KeywordFieldMapper.Builder fieldBuilder = new KeywordFieldMapper.Builder("host.name", IndexVersion.current()); KeywordFieldMapper fieldMapper = fieldBuilder.build( new MapperBuilderContext( "foo.metrics", @@ -434,12 +409,8 @@ private TextFieldMapper.Builder createTextKeywordMultiField(String name) { } private TextFieldMapper.Builder createTextKeywordMultiField(String name, String multiFieldName) { - final IndexSettings indexSettings = new IndexSettings( - IndexMetadata.builder(IndexMetadata.INDEX_UUID_NA_VALUE).build(), - Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()).build() - ); TextFieldMapper.Builder builder = new TextFieldMapper.Builder(name, createDefaultIndexAnalyzers(), false); - builder.multiFieldsBuilder.add(new KeywordFieldMapper.Builder(multiFieldName, indexSettings)); + builder.multiFieldsBuilder.add(new KeywordFieldMapper.Builder(multiFieldName, IndexVersion.current())); return builder; } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java index 63ff3a0938f21..8582f26349ef2 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java @@ -9,14 +9,12 @@ package org.elasticsearch.index.mapper; -import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.CheckedConsumer; -import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.mapper.MapperService.MergeReason; import org.elasticsearch.index.mapper.ObjectMapper.Dynamic; @@ -735,18 +733,14 @@ public void testStoreArraySourceNoopInNonSyntheticSourceMode() throws IOExceptio } public void testNestedObjectWithMultiFieldsgetTotalFieldsCount() { - final IndexSettings indexSettings = new IndexSettings( - IndexMetadata.builder(IndexMetadata.INDEX_UUID_NA_VALUE).build(), - Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()).build() - ); ObjectMapper.Builder mapperBuilder = new ObjectMapper.Builder("parent_size_1", Optional.empty()).add( new ObjectMapper.Builder("child_size_2", Optional.empty()).add( new TextFieldMapper.Builder("grand_child_size_3", createDefaultIndexAnalyzers(), false).addMultiField( - new KeywordFieldMapper.Builder("multi_field_size_4", indexSettings) + new KeywordFieldMapper.Builder("multi_field_size_4", IndexVersion.current()) ) .addMultiField( new TextFieldMapper.Builder("grand_child_size_5", createDefaultIndexAnalyzers(), false).addMultiField( - new KeywordFieldMapper.Builder("multi_field_of_multi_field_size_6", indexSettings) + new KeywordFieldMapper.Builder("multi_field_of_multi_field_size_6", IndexVersion.current()) ) ) ) @@ -786,40 +780,28 @@ private ObjectMapper createObjectMapperWithAllParametersSet(CheckedConsumer fields = objectMapper.asFlattenedFieldMappers(rootContext).stream().map(FieldMapper::fullPath).toList(); assertThat(fields, containsInAnyOrder("parent.keyword1", "parent.child.keyword2")); } public void testFlattenSubobjectsAuto() { - final IndexSettings indexSettings = new IndexSettings( - IndexMetadata.builder(IndexMetadata.INDEX_UUID_NA_VALUE).build(), - Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()).build() - ); MapperBuilderContext rootContext = MapperBuilderContext.root(false, false); ObjectMapper objectMapper = new ObjectMapper.Builder("parent", Optional.of(ObjectMapper.Subobjects.AUTO)).add( - new ObjectMapper.Builder("child", Optional.empty()).add(new KeywordFieldMapper.Builder("keyword2", indexSettings)) - ).add(new KeywordFieldMapper.Builder("keyword1", indexSettings)).build(rootContext); + new ObjectMapper.Builder("child", Optional.empty()).add(new KeywordFieldMapper.Builder("keyword2", IndexVersion.current())) + ).add(new KeywordFieldMapper.Builder("keyword1", IndexVersion.current())).build(rootContext); List fields = objectMapper.asFlattenedFieldMappers(rootContext).stream().map(FieldMapper::fullPath).toList(); assertThat(fields, containsInAnyOrder("parent.keyword1", "parent.child.keyword2")); } public void testFlattenSubobjectsFalse() { - final IndexSettings indexSettings = new IndexSettings( - IndexMetadata.builder(IndexMetadata.INDEX_UUID_NA_VALUE).build(), - Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()).build() - ); MapperBuilderContext rootContext = MapperBuilderContext.root(false, false); ObjectMapper objectMapper = new ObjectMapper.Builder("parent", Optional.of(ObjectMapper.Subobjects.DISABLED)).add( - new ObjectMapper.Builder("child", Optional.empty()).add(new KeywordFieldMapper.Builder("keyword2", indexSettings)) - ).add(new KeywordFieldMapper.Builder("keyword1", indexSettings)).build(rootContext); + new ObjectMapper.Builder("child", Optional.empty()).add(new KeywordFieldMapper.Builder("keyword2", IndexVersion.current())) + ).add(new KeywordFieldMapper.Builder("keyword1", IndexVersion.current())).build(rootContext); List fields = objectMapper.asFlattenedFieldMappers(rootContext).stream().map(FieldMapper::fullPath).toList(); assertThat(fields, containsInAnyOrder("parent.keyword1", "parent.child.keyword2")); } @@ -873,26 +855,22 @@ public void testFlattenExplicitSubobjectsTrue() { } public void testFindParentMapper() { - final IndexSettings indexSettings = new IndexSettings( - IndexMetadata.builder(IndexMetadata.INDEX_UUID_NA_VALUE).build(), - Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()).build() - ); MapperBuilderContext rootContext = MapperBuilderContext.root(false, false); var rootBuilder = new RootObjectMapper.Builder("_doc", Optional.empty()); - rootBuilder.add(new KeywordFieldMapper.Builder("keyword", indexSettings)); + rootBuilder.add(new KeywordFieldMapper.Builder("keyword", IndexVersion.current())); var child = new ObjectMapper.Builder("child", Optional.empty()); - child.add(new KeywordFieldMapper.Builder("keyword2", indexSettings)); - child.add(new KeywordFieldMapper.Builder("keyword.with.dot", indexSettings)); + child.add(new KeywordFieldMapper.Builder("keyword2", IndexVersion.current())); + child.add(new KeywordFieldMapper.Builder("keyword.with.dot", IndexVersion.current())); var secondLevelChild = new ObjectMapper.Builder("child2", Optional.empty()); - secondLevelChild.add(new KeywordFieldMapper.Builder("keyword22", indexSettings)); + secondLevelChild.add(new KeywordFieldMapper.Builder("keyword22", IndexVersion.current())); child.add(secondLevelChild); rootBuilder.add(child); var childWithDot = new ObjectMapper.Builder("childwith.dot", Optional.empty()); - childWithDot.add(new KeywordFieldMapper.Builder("keyword3", indexSettings)); - childWithDot.add(new KeywordFieldMapper.Builder("keyword4.with.dot", indexSettings)); + childWithDot.add(new KeywordFieldMapper.Builder("keyword3", IndexVersion.current())); + childWithDot.add(new KeywordFieldMapper.Builder("keyword4.with.dot", IndexVersion.current())); rootBuilder.add(childWithDot); RootObjectMapper root = rootBuilder.build(rootContext); diff --git a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java index 18cb4a977dfd6..dc70c44a89128 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java @@ -383,14 +383,10 @@ public void testSearchRequestRuntimeFieldsAndMultifieldDetection() { } public void testSyntheticSourceSearchLookup() throws IOException { - final IndexSettings indexSettings = new IndexSettings( - IndexMetadata.builder(IndexMetadata.INDEX_UUID_NA_VALUE).build(), - Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()).build() - ); // Build a mapping using synthetic source SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(null, Settings.EMPTY, false, false).setSynthetic().build(); RootObjectMapper root = new RootObjectMapper.Builder("_doc", Optional.empty()).add( - new KeywordFieldMapper.Builder("cat", indexSettings).ignoreAbove(100) + new KeywordFieldMapper.Builder("cat", IndexVersion.current()).ignoreAbove(100) ).build(MapperBuilderContext.root(true, false)); Mapping mapping = new Mapping(root, new MetadataFieldMapper[] { sourceMapper }, Map.of()); MappingLookup lookup = MappingLookup.fromMapping(mapping); diff --git a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java index a7558b361aa2f..d041121b8a96b 100644 --- a/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java +++ b/server/src/test/java/org/elasticsearch/search/SearchServiceTests.java @@ -223,7 +223,7 @@ private SearchExecutionContext createSearchExecutionContext( new MetadataFieldMapper[0], Collections.emptyMap() ); - KeywordFieldMapper keywordFieldMapper = new KeywordFieldMapper.Builder("field", indexSettings).build(root); + KeywordFieldMapper keywordFieldMapper = new KeywordFieldMapper.Builder("field", IndexVersion.current()).build(root); MappingLookup mappingLookup = MappingLookup.fromMappers( mapping, Collections.singletonList(keywordFieldMapper), From ffe0a2cb38843cc509122fa11e1ced1e662d2c94 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 24 Jan 2025 09:58:40 +0100 Subject: [PATCH 05/13] fix: compare to NONE and flip equals --- .../org/elasticsearch/index/mapper/KeywordFieldMapper.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index f31e3283a13e6..dae6a8fca76c9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -503,7 +503,7 @@ public KeywordFieldType( this.scriptValues = builder.scriptValues(); this.isDimension = builder.dimension.getValue(); this.isSyntheticSource = isSyntheticSource; - this.hasDocValuesSparseIndex = fieldType.docValuesSkipIndexType().equals(DocValuesSkipIndexType.RANGE); + this.hasDocValuesSparseIndex = DocValuesSkipIndexType.NONE.equals(fieldType.docValuesSkipIndexType()) == false; } public KeywordFieldType(String name, boolean isIndexed, boolean hasDocValues, Map meta) { @@ -538,7 +538,7 @@ public KeywordFieldType(String name, FieldType fieldType) { this.scriptValues = null; this.isDimension = false; this.isSyntheticSource = false; - this.hasDocValuesSparseIndex = fieldType.docValuesSkipIndexType().equals(DocValuesSkipIndexType.RANGE); + this.hasDocValuesSparseIndex = DocValuesSkipIndexType.NONE.equals(fieldType.docValuesSkipIndexType()) == false; } public KeywordFieldType(String name, NamedAnalyzer analyzer) { From 5cd81263728cc505f06cd3edadce80b0eaff5d1a Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 24 Jan 2025 11:02:02 +0100 Subject: [PATCH 06/13] fix: refactor Builder constructor and remove unused variables --- .../index/mapper/KeywordFieldMapper.java | 27 ++++++++----------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index dae6a8fca76c9..1249ab065b8f2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -174,7 +174,6 @@ public static final class Builder extends FieldMapper.DimensionBuilder { private final int ignoreAboveDefault; private final IndexSortConfig indexSortConfig; private final IndexMode indexMode; - private final IndexSettings indexSettings; private final Parameter indexOptions = TextParams.keywordIndexOptions(m -> toType(m).indexOptions); private final Parameter hasNorms = TextParams.norms(false, m -> toType(m).fieldType.omitNorms() == false); private final Parameter similarity = TextParams.similarity( @@ -226,8 +225,16 @@ public Builder(final String name, final MappingParserContext mappingParserContex scriptCompiler, ignoreAboveDefault, new IndexSettings( - IndexMetadata.builder(IndexMetadata.INDEX_UUID_NA_VALUE).build(), - Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, indexCreatedVersion).build() + IndexMetadata.builder(IndexMetadata.INDEX_UUID_NA_VALUE) + .settings( + Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, indexCreatedVersion) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .build() + ) + .build(), + Settings.EMPTY ) ); } @@ -274,7 +281,6 @@ private Builder( }); this.indexSortConfig = indexSettings.getIndexSortConfig(); this.indexMode = indexSettings.getMode(); - this.indexSettings = indexSettings; } public Builder(String name, IndexVersion indexCreatedVersion) { @@ -421,7 +427,6 @@ public KeywordFieldMapper build(MapperBuilderContext context) { buildFieldType(context, fieldtype), builderParams(this, context), context.isSourceSynthetic(), - indexSettings, this ); } @@ -961,11 +966,6 @@ public boolean hasNormalizer() { private final IndexAnalyzers indexAnalyzers; private final int ignoreAboveDefault; - private final int ignoreAbove; - private final IndexSortConfig indexSortConfig; - private final IndexMode indexMode; - - private final IndexSettings indexSettings; private KeywordFieldMapper( String simpleName, @@ -973,7 +973,6 @@ private KeywordFieldMapper( KeywordFieldType mappedFieldType, BuilderParams builderParams, boolean isSyntheticSource, - IndexSettings indexSettings, Builder builder ) { super(simpleName, mappedFieldType, builderParams); @@ -990,10 +989,6 @@ private KeywordFieldMapper( this.indexCreatedVersion = builder.indexCreatedVersion; this.isSyntheticSource = isSyntheticSource; this.ignoreAboveDefault = builder.ignoreAboveDefault; - this.ignoreAbove = builder.ignoreAbove.getValue(); - this.indexSortConfig = builder.indexSortConfig; - this.indexMode = builder.indexMode; - this.indexSettings = builder.indexSettings; } @Override @@ -1111,7 +1106,7 @@ public Map indexAnalyzers() { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(leafName(), indexAnalyzers, scriptCompiler, ignoreAboveDefault, indexSettings).dimension( + return new Builder(leafName(), indexAnalyzers, scriptCompiler, ignoreAboveDefault, indexCreatedVersion).dimension( fieldType().isDimension() ).init(this); } From db8374ec1153aeb53f4a74d4a7eef8711c2b69b8 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 24 Jan 2025 15:31:42 +0100 Subject: [PATCH 07/13] fix: refactor Builder constructore and remove hasDocValuesSparseIndex --- .../index/mapper/KeywordFieldMapper.java | 106 ++++++++---------- .../index/mapper/MappedFieldType.java | 4 - 2 files changed, 47 insertions(+), 63 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 1249ab065b8f2..de3bf922f834a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -34,15 +34,12 @@ import org.apache.lucene.util.automaton.CompiledAutomaton; import org.apache.lucene.util.automaton.CompiledAutomaton.AUTOMATON_TYPE; import org.apache.lucene.util.automaton.Operations; -import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.lucene.search.AutomatonQueries; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.core.Nullable; import org.elasticsearch.index.IndexMode; -import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexSortConfig; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.analysis.IndexAnalyzers; @@ -208,7 +205,9 @@ public Builder(final String name, final MappingParserContext mappingParserContex mappingParserContext.getIndexAnalyzers(), mappingParserContext.scriptCompiler(), IGNORE_ABOVE_SETTING.get(mappingParserContext.getSettings()), - mappingParserContext.getIndexSettings() + mappingParserContext.getIndexSettings().getIndexVersionCreated(), + mappingParserContext.getIndexSettings().getMode(), + mappingParserContext.getIndexSettings().getIndexSortConfig() ); } @@ -219,24 +218,7 @@ public Builder(final String name, final MappingParserContext mappingParserContex int ignoreAboveDefault, IndexVersion indexCreatedVersion ) { - this( - name, - indexAnalyzers, - scriptCompiler, - ignoreAboveDefault, - new IndexSettings( - IndexMetadata.builder(IndexMetadata.INDEX_UUID_NA_VALUE) - .settings( - Settings.builder() - .put(IndexMetadata.SETTING_VERSION_CREATED, indexCreatedVersion) - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) - .build() - ) - .build(), - Settings.EMPTY - ) - ); + this(name, indexAnalyzers, scriptCompiler, ignoreAboveDefault, indexCreatedVersion, IndexMode.STANDARD, null); } private Builder( @@ -244,12 +226,14 @@ private Builder( IndexAnalyzers indexAnalyzers, ScriptCompiler scriptCompiler, int ignoreAboveDefault, - IndexSettings indexSettings + IndexVersion indexCreatedVersion, + IndexMode indexMode, + IndexSortConfig indexSortConfig ) { super(name); this.indexAnalyzers = indexAnalyzers; this.scriptCompiler = Objects.requireNonNull(scriptCompiler); - this.indexCreatedVersion = Objects.requireNonNull(indexSettings.getIndexVersionCreated()); + this.indexCreatedVersion = Objects.requireNonNull(indexCreatedVersion); this.normalizer = Parameter.stringParam( "normalizer", indexCreatedVersion.isLegacyIndexVersion(), @@ -279,8 +263,8 @@ private Builder( throw new IllegalArgumentException("[ignore_above] must be positive, got [" + v + "]"); } }); - this.indexSortConfig = indexSettings.getIndexSortConfig(); - this.indexMode = indexSettings.getMode(); + this.indexSortConfig = indexSortConfig; + this.indexMode = indexMode; } public Builder(String name, IndexVersion indexCreatedVersion) { @@ -407,7 +391,7 @@ private KeywordFieldType buildFieldType(MapperBuilderContext context, FieldType @Override public KeywordFieldMapper build(MapperBuilderContext context) { - FieldType fieldtype = fieldType(indexSortConfig, indexMode, context.buildFullName(leafName())); + FieldType fieldtype = resolveFieldType(indexSortConfig, indexMode, context.buildFullName(leafName())); fieldtype.setOmitNorms(this.hasNorms.getValue() == false); fieldtype.setIndexOptions(TextParams.toIndexOptions(this.indexed.getValue(), this.indexOptions.getValue())); fieldtype.setStored(this.stored.getValue()); @@ -431,7 +415,7 @@ public KeywordFieldMapper build(MapperBuilderContext context) { ); } - private FieldType fieldType(final IndexSortConfig indexSortConfig, final IndexMode indexMode, final String fullFieldName) { + private FieldType resolveFieldType(final IndexSortConfig indexSortConfig, final IndexMode indexMode, final String fullFieldName) { if (shouldUseDocValuesSparseIndex(indexSortConfig, indexMode, fullFieldName)) { return new FieldType(Defaults.FIELD_TYPE_WITH_SKIP_DOC_VALUES); } @@ -443,30 +427,13 @@ private boolean shouldUseDocValuesSparseIndex( final IndexMode indexMode, final String fullFieldName ) { - return (defaultIndexedAndDocValues() || isNotIndexedAndHasDocValues()) - && isLogsDbMode(indexMode) - && isHostNameField(fullFieldName) - && isPrimarySortField(indexSortConfig); - } - - private boolean isHostNameField(final String fullFieldName) { - return HOST_NAME.equals(fullFieldName); - } - - private boolean isPrimarySortField(final IndexSortConfig indexSortConfig) { - return indexSortConfig != null && indexSortConfig.hasPrimarySortOnField(HOST_NAME); - } - - private boolean defaultIndexedAndDocValues() { - return indexed.isConfigured() == false && hasDocValues.isConfigured() == false; - } + boolean isIndexedAndDocValuesDefault = indexed.isConfigured() || hasDocValues.isConfigured(); + boolean isNotIndexedAndHasDocValues = indexed.getValue() || hasDocValues.getValue() == false; + boolean isLogsDbMode = IndexMode.LOGSDB.equals(indexMode); + boolean isHostNameField = HOST_NAME.equals(fullFieldName); + boolean isPrimarySortField = indexSortConfig != null && indexSortConfig.hasPrimarySortOnField(HOST_NAME); - private boolean isNotIndexedAndHasDocValues() { - return indexed.getValue() == false && hasDocValues.getValue(); - } - - private boolean isLogsDbMode(final IndexMode indexMode) { - return IndexMode.LOGSDB.equals(indexMode); + return (isIndexedAndDocValuesDefault || isNotIndexedAndHasDocValues) && isLogsDbMode && isHostNameField && isPrimarySortField; } } @@ -482,6 +449,8 @@ public static final class KeywordFieldType extends StringFieldType { private final FieldValues scriptValues; private final boolean isDimension; private final boolean isSyntheticSource; + private final IndexMode indexMode; + private final IndexSortConfig indexSortConfig; private final boolean hasDocValuesSparseIndex; public KeywordFieldType( @@ -508,6 +477,8 @@ public KeywordFieldType( this.scriptValues = builder.scriptValues(); this.isDimension = builder.dimension.getValue(); this.isSyntheticSource = isSyntheticSource; + this.indexMode = builder.indexMode; + this.indexSortConfig = builder.indexSortConfig; this.hasDocValuesSparseIndex = DocValuesSkipIndexType.NONE.equals(fieldType.docValuesSkipIndexType()) == false; } @@ -520,6 +491,8 @@ public KeywordFieldType(String name, boolean isIndexed, boolean hasDocValues, Ma this.scriptValues = null; this.isDimension = false; this.isSyntheticSource = false; + this.indexMode = IndexMode.STANDARD; + this.indexSortConfig = null; this.hasDocValuesSparseIndex = false; } @@ -543,6 +516,8 @@ public KeywordFieldType(String name, FieldType fieldType) { this.scriptValues = null; this.isDimension = false; this.isSyntheticSource = false; + this.indexMode = IndexMode.STANDARD; + this.indexSortConfig = null; this.hasDocValuesSparseIndex = DocValuesSkipIndexType.NONE.equals(fieldType.docValuesSkipIndexType()) == false; } @@ -555,6 +530,8 @@ public KeywordFieldType(String name, NamedAnalyzer analyzer) { this.scriptValues = null; this.isDimension = false; this.isSyntheticSource = false; + this.indexMode = IndexMode.STANDARD; + this.indexSortConfig = null; this.hasDocValuesSparseIndex = false; } @@ -563,11 +540,6 @@ public boolean isSearchable() { return isIndexed() || hasDocValues(); } - @Override - public boolean hasDocValuesSparseIndex() { - return hasDocValuesSparseIndex; - } - @Override public Query termQuery(Object value, SearchExecutionContext context) { failIfNotIndexedNorDocValuesFallback(context); @@ -951,6 +923,18 @@ public boolean hasScriptValues() { public boolean hasNormalizer() { return normalizer != Lucene.KEYWORD_ANALYZER; } + + public IndexMode getIndexMode() { + return indexMode; + } + + public IndexSortConfig getIndexSortConfig() { + return indexSortConfig; + } + + public boolean hasDocValuesSparseIndex() { + return hasDocValuesSparseIndex; + } } private final boolean indexed; @@ -966,6 +950,8 @@ public boolean hasNormalizer() { private final IndexAnalyzers indexAnalyzers; private final int ignoreAboveDefault; + private final IndexMode indexMode; + private final IndexSortConfig indexSortConfig; private KeywordFieldMapper( String simpleName, @@ -989,6 +975,8 @@ private KeywordFieldMapper( this.indexCreatedVersion = builder.indexCreatedVersion; this.isSyntheticSource = isSyntheticSource; this.ignoreAboveDefault = builder.ignoreAboveDefault; + this.indexMode = builder.indexMode; + this.indexSortConfig = builder.indexSortConfig; } @Override @@ -1106,9 +1094,9 @@ public Map indexAnalyzers() { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(leafName(), indexAnalyzers, scriptCompiler, ignoreAboveDefault, indexCreatedVersion).dimension( - fieldType().isDimension() - ).init(this); + return new Builder(leafName(), indexAnalyzers, scriptCompiler, ignoreAboveDefault, indexCreatedVersion, indexMode, indexSortConfig) + .dimension(fieldType().isDimension()) + .init(this); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java index 1766c2d1faafb..4b68e20673572 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java @@ -155,10 +155,6 @@ public boolean isSearchable() { return isIndexed; } - public boolean hasDocValuesSparseIndex() { - return false; - } - /** * Returns true if the field is indexed. */ From 328e42b1ed8028783cf04696eeeafdcddbb1da1b Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Fri, 24 Jan 2025 15:39:51 +0100 Subject: [PATCH 08/13] fix: gate sparse index usage with feature flag --- .../main/java/org/elasticsearch/index/mapper/FieldMapper.java | 2 ++ .../org/elasticsearch/index/mapper/KeywordFieldMapper.java | 3 +++ 2 files changed, 5 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index c0c3c7193998a..4efdfb4a32be7 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -21,6 +21,7 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.util.CollectionUtils; +import org.elasticsearch.common.util.FeatureFlag; import org.elasticsearch.common.util.Maps; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.IndexMode; @@ -63,6 +64,7 @@ public abstract class FieldMapper extends Mapper { private static final Logger logger = LogManager.getLogger(FieldMapper.class); + public static final FeatureFlag DOC_VALUES_SPARSE_INDEX = new FeatureFlag("doc_values_sparse_index"); public static final Setting IGNORE_MALFORMED_SETTING = Setting.boolSetting("index.mapping.ignore_malformed", settings -> { if (IndexSettings.MODE.get(settings) == IndexMode.LOGSDB && IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings).onOrAfter(IndexVersions.ENABLE_IGNORE_MALFORMED_LOGSDB)) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index de3bf922f834a..c12874d88d65b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -427,6 +427,9 @@ private boolean shouldUseDocValuesSparseIndex( final IndexMode indexMode, final String fullFieldName ) { + if (FieldMapper.DOC_VALUES_SPARSE_INDEX.isEnabled() == false) { + return false; + } boolean isIndexedAndDocValuesDefault = indexed.isConfigured() || hasDocValues.isConfigured(); boolean isNotIndexedAndHasDocValues = indexed.getValue() || hasDocValues.getValue() == false; boolean isLogsDbMode = IndexMode.LOGSDB.equals(indexMode); From 4909ecbc94ac68ed024e35e6aaffa38491e9baa0 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Mon, 27 Jan 2025 22:31:08 +0100 Subject: [PATCH 09/13] fix: refactor sparse doc values index creation conditions --- .../index/mapper/KeywordFieldMapper.java | 18 +++++++----------- .../index/mapper/KeywordFieldMapperTests.java | 2 +- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 417007eb3af88..5155a74efb5b7 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -416,7 +416,8 @@ public KeywordFieldMapper build(MapperBuilderContext context) { } private FieldType resolveFieldType(final IndexSortConfig indexSortConfig, final IndexMode indexMode, final String fullFieldName) { - if (shouldUseDocValuesSparseIndex(indexSortConfig, indexMode, fullFieldName)) { + if (FieldMapper.DOC_VALUES_SPARSE_INDEX.isEnabled() + && shouldUseDocValuesSparseIndex(indexSortConfig, indexMode, fullFieldName)) { return new FieldType(Defaults.FIELD_TYPE_WITH_SKIP_DOC_VALUES); } return new FieldType(Defaults.FIELD_TYPE); @@ -427,16 +428,11 @@ private boolean shouldUseDocValuesSparseIndex( final IndexMode indexMode, final String fullFieldName ) { - if (FieldMapper.DOC_VALUES_SPARSE_INDEX.isEnabled() == false) { - return false; - } - boolean isIndexedAndDocValuesDefault = indexed.isConfigured() || hasDocValues.isConfigured(); - boolean isNotIndexedAndHasDocValues = indexed.getValue() || hasDocValues.getValue() == false; - boolean isLogsDbMode = IndexMode.LOGSDB.equals(indexMode); - boolean isHostNameField = HOST_NAME.equals(fullFieldName); - boolean isPrimarySortField = indexSortConfig != null && indexSortConfig.hasPrimarySortOnField(HOST_NAME); - - return (isIndexedAndDocValuesDefault || isNotIndexedAndHasDocValues) && isLogsDbMode && isHostNameField && isPrimarySortField; + return indexed.isConfigured() == false + && hasDocValues.getValue() + && IndexMode.LOGSDB.equals(indexMode) + && HOST_NAME.equals(fullFieldName) + && (indexSortConfig != null && indexSortConfig.hasPrimarySortOnField(HOST_NAME)); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java index c547072ccbdce..3d1bb28ea5e8d 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java @@ -893,7 +893,7 @@ public void testFieldTypeWithSkipDocValues_IndexedFalseDocValuesTrue() throws IO ); final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); - assertTrue(mapper.fieldType().hasDocValuesSparseIndex()); + assertFalse(mapper.fieldType().hasDocValuesSparseIndex()); } public void testFieldTypeDefault_IndexedFalseDocValuesFalse() throws IOException { From 2e3a51ec75f700d13b7b390e002397eca93feebf Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Mon, 27 Jan 2025 22:57:05 +0100 Subject: [PATCH 10/13] fix: host.name sparse doc values index tests --- .../index/mapper/KeywordFieldMapperTests.java | 91 ++++++++----------- 1 file changed, 36 insertions(+), 55 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java index 3d1bb28ea5e8d..0471cd8551e1e 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java @@ -791,7 +791,10 @@ public void testFieldTypeWithSkipDocValues_LogsDbMode() throws IOException { public void testFieldTypeDefault_StandardMode() throws IOException { final MapperService mapperService = createMapperService( - Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.name()).build(), + Settings.builder() + .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.name()) + .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "host.name") + .build(), mapping(b -> { b.startObject("host.name"); b.field("type", "keyword"); @@ -801,11 +804,16 @@ public void testFieldTypeDefault_StandardMode() throws IOException { final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); assertFalse(mapper.fieldType().hasDocValuesSparseIndex()); + assertTrue(mapper.fieldType().hasDocValues()); + assertTrue(mapper.fieldType().isIndexed()); } public void testFieldTypeDefault_NonMatchingFieldName() throws IOException { final MapperService mapperService = createMapperService( - Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()).build(), + Settings.builder() + .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()) + .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "hostname") + .build(), mapping(b -> { b.startObject("hostname"); b.field("type", "keyword"); @@ -815,11 +823,16 @@ public void testFieldTypeDefault_NonMatchingFieldName() throws IOException { final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("hostname"); assertFalse(mapper.fieldType().hasDocValuesSparseIndex()); + assertTrue(mapper.fieldType().hasDocValues()); + assertTrue(mapper.fieldType().isIndexed()); } public void testFieldTypeDefault_ConfiguredIndexed() throws IOException { final MapperService mapperService = createMapperService( - Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()).build(), + Settings.builder() + .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()) + .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "host.name") + .build(), mapping(b -> { b.startObject("host.name"); b.field("type", "keyword"); @@ -829,38 +842,29 @@ public void testFieldTypeDefault_ConfiguredIndexed() throws IOException { ); final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); - assertFalse(mapper.fieldType().hasDocValuesSparseIndex()); + assertTrue(mapper.fieldType().hasDocValuesSparseIndex()); + assertTrue(mapper.fieldType().hasDocValues()); + assertTrue(mapper.fieldType().isIndexed()); } public void testFieldTypeDefault_ConfiguredDocValues() throws IOException { final MapperService mapperService = createMapperService( - Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()).build(), - mapping(b -> { - b.startObject("host.name"); - b.field("type", "keyword"); - b.field("doc_values", true); - b.endObject(); - }) - ); - - final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); - assertFalse(mapper.fieldType().hasDocValuesSparseIndex()); - } - - public void testFieldTypeDefault_ConfiguredIndexedAndDocValues() throws IOException { - final MapperService mapperService = createMapperService( - Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()).build(), + Settings.builder() + .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()) + .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "host.name") + .build(), mapping(b -> { b.startObject("host.name"); b.field("type", "keyword"); - b.field("index", true); b.field("doc_values", true); b.endObject(); }) ); final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); - assertFalse(mapper.fieldType().hasDocValuesSparseIndex()); + assertTrue(mapper.fieldType().hasDocValuesSparseIndex()); + assertTrue(mapper.fieldType().hasDocValues()); + assertTrue(mapper.fieldType().isIndexed()); } public void testFieldTypeDefault_LogsDbMode_NonSortField() throws IOException { @@ -875,6 +879,8 @@ public void testFieldTypeDefault_LogsDbMode_NonSortField() throws IOException { final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); assertFalse(mapper.fieldType().hasDocValuesSparseIndex()); + assertTrue(mapper.fieldType().hasDocValues()); + assertTrue(mapper.fieldType().isIndexed()); } public void testFieldTypeWithSkipDocValues_IndexedFalseDocValuesTrue() throws IOException { @@ -894,11 +900,16 @@ public void testFieldTypeWithSkipDocValues_IndexedFalseDocValuesTrue() throws IO final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); assertFalse(mapper.fieldType().hasDocValuesSparseIndex()); + assertTrue(mapper.fieldType().hasDocValues()); + assertFalse(mapper.fieldType().isIndexed()); } public void testFieldTypeDefault_IndexedFalseDocValuesFalse() throws IOException { final MapperService mapperService = createMapperService( - Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()).build(), + Settings.builder() + .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()) + .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "host.name") + .build(), mapping(b -> { b.startObject("host.name"); b.field("type", "keyword"); @@ -910,37 +921,7 @@ public void testFieldTypeDefault_IndexedFalseDocValuesFalse() throws IOException final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); assertFalse(mapper.fieldType().hasDocValuesSparseIndex()); - } - - public void testFieldTypeDefault_IndexedTrueDocValuesTrue() throws IOException { - final MapperService mapperService = createMapperService( - Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()).build(), - mapping(b -> { - b.startObject("host.name"); - b.field("type", "keyword"); - b.field("index", true); - b.field("doc_values", true); - b.endObject(); - }) - ); - - final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); - assertFalse(mapper.fieldType().hasDocValuesSparseIndex()); - } - - public void testFieldTypeDefault_StandardModeIndexedFalseDocValuesTrue() throws IOException { - final MapperService mapperService = createMapperService( - Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.name()).build(), - mapping(b -> { - b.startObject("host.name"); - b.field("type", "keyword"); - b.field("index", false); - b.field("doc_values", true); - b.endObject(); - }) - ); - - final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); - assertFalse(mapper.fieldType().hasDocValuesSparseIndex()); + assertFalse(mapper.fieldType().hasDocValues()); + assertFalse(mapper.fieldType().isIndexed()); } } From 1868134270c1639fbeb3526df2f5ee0264cec149 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Mon, 27 Jan 2025 22:59:03 +0100 Subject: [PATCH 11/13] fix: add missing assertions --- .../org/elasticsearch/index/mapper/KeywordFieldMapperTests.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java index 0471cd8551e1e..3a2f67bc632ed 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java @@ -787,6 +787,8 @@ public void testFieldTypeWithSkipDocValues_LogsDbMode() throws IOException { final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); assertTrue(mapper.fieldType().hasDocValuesSparseIndex()); + assertTrue(mapper.fieldType().hasDocValues()); + assertTrue(mapper.fieldType().isIndexed()); } public void testFieldTypeDefault_StandardMode() throws IOException { From e4f0184e22f1e63534ca84836fbf9d26aa2eb0a1 Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Tue, 28 Jan 2025 11:07:12 +0100 Subject: [PATCH 12/13] fix: use sparse doc values index only for new indices --- .../java/org/elasticsearch/index/IndexVersions.java | 1 + .../index/mapper/KeywordFieldMapper.java | 11 +++++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java index e801a07f11670..309b8a4f143d3 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java +++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java @@ -143,6 +143,7 @@ private static Version parseUnchecked(String version) { public static final IndexVersion INFERENCE_METADATA_FIELDS = def(9_005_00_0, Version.LUCENE_10_0_0); public static final IndexVersion LOGSB_OPTIONAL_SORTING_ON_HOST_NAME = def(9_006_00_0, Version.LUCENE_10_0_0); public static final IndexVersion SOURCE_MAPPER_MODE_ATTRIBUTE_NOOP = def(9_007_00_0, Version.LUCENE_10_0_0); + public static final IndexVersion HOSTNAME_DOC_VALUES_SPARSE_INDEX = def(9_008_00_0, Version.LUCENE_10_0_0); /* * STOP! READ THIS FIRST! No, really, * ____ _____ ___ ____ _ ____ _____ _ ____ _____ _ _ ___ ____ _____ ___ ____ ____ _____ _ diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 5155a74efb5b7..eb5110e24827a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -42,6 +42,7 @@ import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSortConfig; import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.fielddata.FieldData; @@ -391,7 +392,7 @@ private KeywordFieldType buildFieldType(MapperBuilderContext context, FieldType @Override public KeywordFieldMapper build(MapperBuilderContext context) { - FieldType fieldtype = resolveFieldType(indexSortConfig, indexMode, context.buildFullName(leafName())); + FieldType fieldtype = resolveFieldType(indexCreatedVersion, indexSortConfig, indexMode, context.buildFullName(leafName())); fieldtype.setOmitNorms(this.hasNorms.getValue() == false); fieldtype.setIndexOptions(TextParams.toIndexOptions(this.indexed.getValue(), this.indexOptions.getValue())); fieldtype.setStored(this.stored.getValue()); @@ -415,8 +416,14 @@ public KeywordFieldMapper build(MapperBuilderContext context) { ); } - private FieldType resolveFieldType(final IndexSortConfig indexSortConfig, final IndexMode indexMode, final String fullFieldName) { + private FieldType resolveFieldType( + final IndexVersion indexCreatedVersion, + final IndexSortConfig indexSortConfig, + final IndexMode indexMode, + final String fullFieldName + ) { if (FieldMapper.DOC_VALUES_SPARSE_INDEX.isEnabled() + && indexCreatedVersion.onOrAfter(IndexVersions.HOSTNAME_DOC_VALUES_SPARSE_INDEX) && shouldUseDocValuesSparseIndex(indexSortConfig, indexMode, fullFieldName)) { return new FieldType(Defaults.FIELD_TYPE_WITH_SKIP_DOC_VALUES); } From 24aa35d94d39e06eee2e75dc3469eb4611ce0a5c Mon Sep 17 00:00:00 2001 From: Salvatore Campagna Date: Wed, 29 Jan 2025 12:44:33 +0100 Subject: [PATCH 13/13] fix: either use the sparse idnex or the inverted index --- .../index/mapper/FieldMapper.java | 4 ++ .../index/mapper/KeywordFieldMapper.java | 37 ++++++++++++++++++- .../index/mapper/KeywordFieldMapperTests.java | 20 +++++----- 3 files changed, 49 insertions(+), 12 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index aed89c5a00157..2dfbf871ea183 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -853,6 +853,10 @@ public boolean isConfigured() { return isSet && Objects.equals(value, getDefaultValue()) == false; } + public boolean isSet() { + return isSet; + } + /** * Allows the parameter to accept a {@code null} value */ diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index eb5110e24827a..b7528bd3729ee 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -394,9 +394,12 @@ private KeywordFieldType buildFieldType(MapperBuilderContext context, FieldType public KeywordFieldMapper build(MapperBuilderContext context) { FieldType fieldtype = resolveFieldType(indexCreatedVersion, indexSortConfig, indexMode, context.buildFullName(leafName())); fieldtype.setOmitNorms(this.hasNorms.getValue() == false); - fieldtype.setIndexOptions(TextParams.toIndexOptions(this.indexed.getValue(), this.indexOptions.getValue())); fieldtype.setStored(this.stored.getValue()); fieldtype.setDocValuesType(this.hasDocValues.getValue() ? DocValuesType.SORTED_SET : DocValuesType.NONE); + if (fieldtype.equals(Defaults.FIELD_TYPE_WITH_SKIP_DOC_VALUES) == false) { + // NOTE: override index options only if we are not using a sparse doc values index (and we use an inverted index) + fieldtype.setIndexOptions(TextParams.toIndexOptions(this.indexed.getValue(), this.indexOptions.getValue())); + } if (fieldtype.equals(Defaults.FIELD_TYPE)) { // deduplicate in the common default case to save some memory fieldtype = Defaults.FIELD_TYPE; @@ -430,18 +433,48 @@ && shouldUseDocValuesSparseIndex(indexSortConfig, indexMode, fullFieldName)) { return new FieldType(Defaults.FIELD_TYPE); } + /** + * Determines whether to use a sparse index representation for doc values. + * + *

If the field is explicitly indexed by setting {@code index: true}, we do not use + * a sparse doc values index but instead rely on the inverted index, as is typically + * the case for keyword fields.

+ * + *

This method checks several conditions to decide if the sparse index format + * should be applied:

+ * + *
    + *
  • Returns {@code false} immediately if the field is explicitly indexed.
    • + *
  • Ensures the field is not explicitly configured as indexed (i.e., {@code index} has its default value).
    • + *
  • Requires doc values to be enabled.
    • + *
  • Index mode must be {@link IndexMode#LOGSDB}.
    • + *
  • Field name must be {@code host.name}.
    • + *
  • The {@code host.name} field must be a primary sort field.
    • + *
+ * + *

Returns {@code true} if all conditions are met, indicating that sparse doc values + * should be used. Otherwise, returns {@code false}.

+ * + * @param indexSortConfig The index sort configuration, used to check primary sorting. + * @param indexMode The mode of the index, which must be {@link IndexMode#LOGSDB}. + * @param fullFieldName The name of the field being checked, which must be {@code host.name}. + * @return {@code true} if sparse doc values should be used, otherwise {@code false}. + */ + private boolean shouldUseDocValuesSparseIndex( final IndexSortConfig indexSortConfig, final IndexMode indexMode, final String fullFieldName ) { + if (indexed.isSet() && indexed.getValue()) { + return false; + } return indexed.isConfigured() == false && hasDocValues.getValue() && IndexMode.LOGSDB.equals(indexMode) && HOST_NAME.equals(fullFieldName) && (indexSortConfig != null && indexSortConfig.hasPrimarySortOnField(HOST_NAME)); } - } public static final TypeParser PARSER = createTypeParserWithLegacySupport(Builder::new); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java index 3a2f67bc632ed..d78f2110daa67 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java @@ -786,9 +786,9 @@ public void testFieldTypeWithSkipDocValues_LogsDbMode() throws IOException { ); final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); - assertTrue(mapper.fieldType().hasDocValuesSparseIndex()); assertTrue(mapper.fieldType().hasDocValues()); - assertTrue(mapper.fieldType().isIndexed()); + assertFalse(mapper.fieldType().isIndexed()); + assertTrue(mapper.fieldType().hasDocValuesSparseIndex()); } public void testFieldTypeDefault_StandardMode() throws IOException { @@ -805,9 +805,9 @@ public void testFieldTypeDefault_StandardMode() throws IOException { ); final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); - assertFalse(mapper.fieldType().hasDocValuesSparseIndex()); assertTrue(mapper.fieldType().hasDocValues()); assertTrue(mapper.fieldType().isIndexed()); + assertFalse(mapper.fieldType().hasDocValuesSparseIndex()); } public void testFieldTypeDefault_NonMatchingFieldName() throws IOException { @@ -824,9 +824,9 @@ public void testFieldTypeDefault_NonMatchingFieldName() throws IOException { ); final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("hostname"); - assertFalse(mapper.fieldType().hasDocValuesSparseIndex()); assertTrue(mapper.fieldType().hasDocValues()); assertTrue(mapper.fieldType().isIndexed()); + assertFalse(mapper.fieldType().hasDocValuesSparseIndex()); } public void testFieldTypeDefault_ConfiguredIndexed() throws IOException { @@ -844,9 +844,9 @@ public void testFieldTypeDefault_ConfiguredIndexed() throws IOException { ); final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); - assertTrue(mapper.fieldType().hasDocValuesSparseIndex()); assertTrue(mapper.fieldType().hasDocValues()); assertTrue(mapper.fieldType().isIndexed()); + assertFalse(mapper.fieldType().hasDocValuesSparseIndex()); } public void testFieldTypeDefault_ConfiguredDocValues() throws IOException { @@ -864,9 +864,9 @@ public void testFieldTypeDefault_ConfiguredDocValues() throws IOException { ); final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); - assertTrue(mapper.fieldType().hasDocValuesSparseIndex()); assertTrue(mapper.fieldType().hasDocValues()); - assertTrue(mapper.fieldType().isIndexed()); + assertFalse(mapper.fieldType().isIndexed()); + assertTrue(mapper.fieldType().hasDocValuesSparseIndex()); } public void testFieldTypeDefault_LogsDbMode_NonSortField() throws IOException { @@ -880,9 +880,9 @@ public void testFieldTypeDefault_LogsDbMode_NonSortField() throws IOException { ); final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); - assertFalse(mapper.fieldType().hasDocValuesSparseIndex()); assertTrue(mapper.fieldType().hasDocValues()); assertTrue(mapper.fieldType().isIndexed()); + assertFalse(mapper.fieldType().hasDocValuesSparseIndex()); } public void testFieldTypeWithSkipDocValues_IndexedFalseDocValuesTrue() throws IOException { @@ -901,9 +901,9 @@ public void testFieldTypeWithSkipDocValues_IndexedFalseDocValuesTrue() throws IO ); final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); - assertFalse(mapper.fieldType().hasDocValuesSparseIndex()); assertTrue(mapper.fieldType().hasDocValues()); assertFalse(mapper.fieldType().isIndexed()); + assertFalse(mapper.fieldType().hasDocValuesSparseIndex()); } public void testFieldTypeDefault_IndexedFalseDocValuesFalse() throws IOException { @@ -922,8 +922,8 @@ public void testFieldTypeDefault_IndexedFalseDocValuesFalse() throws IOException ); final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); - assertFalse(mapper.fieldType().hasDocValuesSparseIndex()); assertFalse(mapper.fieldType().hasDocValues()); assertFalse(mapper.fieldType().isIndexed()); + assertFalse(mapper.fieldType().hasDocValuesSparseIndex()); } }