diff --git a/CHANGELOG.md b/CHANGELOG.md index 26c08c34a1a3c..327ebf2d0d310 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add all-active ingestion as docrep equivalent in pull-based ingestion ([#19316](https://github.com/opensearch-project/OpenSearch/pull/19316)) - Adding logic for histogram aggregation using skiplist ([#19130](https://github.com/opensearch-project/OpenSearch/pull/19130)) - Add skip_list param for date, scaled float and token count fields ([#19142](https://github.com/opensearch-project/OpenSearch/pull/19142)) +- Enable skip_list for @timestamp field or index sort field by default([#19480](https://github.com/opensearch-project/OpenSearch/pull/19480)) - Implement GRPC MatchPhrase, MultiMatch queries ([#19449](https://github.com/opensearch-project/OpenSearch/pull/19449)) - Optimize gRPC transport thread management for improved throughput ([#19278](https://github.com/opensearch-project/OpenSearch/pull/19278)) - Implement GRPC Boolean query and inject registry for all internal query converters ([#19391](https://github.com/opensearch-project/OpenSearch/pull/19391)) diff --git a/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java index 270a4606b11c6..51400567025eb 100644 --- a/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java @@ -58,6 +58,7 @@ import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.util.LocaleUtils; import org.opensearch.common.xcontent.support.XContentMapValues; +import org.opensearch.index.IndexSortConfig; import org.opensearch.index.compositeindex.datacube.DimensionType; import org.opensearch.index.fielddata.IndexFieldData; import org.opensearch.index.fielddata.IndexNumericFieldData.NumericType; @@ -753,6 +754,7 @@ public DocValueFormat docValueFormat(@Nullable String format, ZoneId timeZone) { private final boolean indexed; private final boolean hasDocValues; private final boolean skiplist; + private final boolean isSkiplistConfigured; private final Locale locale; private final String format; private final String printFormat; @@ -778,6 +780,7 @@ private DateFieldMapper( this.indexed = builder.index.getValue(); this.hasDocValues = builder.docValues.getValue(); this.skiplist = builder.skiplist.getValue(); + this.isSkiplistConfigured = builder.skiplist.isConfigured(); this.locale = builder.locale.getValue(); this.format = builder.format.getValue(); this.printFormat = builder.printFormat.getValue(); @@ -846,7 +849,7 @@ protected void parseCreateField(ParseContext context) throws IOException { context.doc().add(new LongPoint(fieldType().name(), timestamp)); } if (hasDocValues) { - if (skiplist) { + if (skiplist || isSkiplistDefaultEnabled(context.indexSettings().getIndexSortConfig(), fieldType().name())) { context.doc().add(SortedNumericDocValuesField.indexedField(fieldType().name(), timestamp)); } else { context.doc().add(new SortedNumericDocValuesField(fieldType().name(), timestamp)); @@ -859,6 +862,19 @@ protected void parseCreateField(ParseContext context) throws IOException { } } + boolean isSkiplistDefaultEnabled(IndexSortConfig indexSortConfig, String fieldName) { + if (!isSkiplistConfigured) { + if (indexSortConfig.hasPrimarySortOnField(fieldName)) { + return true; + } + if (DataStreamFieldMapper.Defaults.TIMESTAMP_FIELD.getName().equals(fieldName)) { + return true; + } + + } + return false; + } + public Long getNullValue() { return nullValue; } diff --git a/server/src/test/java/org/opensearch/index/mapper/DateFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/DateFieldMapperTests.java index fd9bbbf3b79cd..fb1325366452f 100644 --- a/server/src/test/java/org/opensearch/index/mapper/DateFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/DateFieldMapperTests.java @@ -44,10 +44,15 @@ import org.apache.lucene.index.StoredFields; import org.apache.lucene.index.VectorEncoding; import org.apache.lucene.index.VectorSimilarityFunction; +import org.opensearch.Version; +import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.common.settings.Settings; import org.opensearch.common.time.DateFormatter; import org.opensearch.common.util.FeatureFlags; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.index.IndexSettings; +import org.opensearch.index.IndexSortConfig; import org.opensearch.index.fieldvisitor.SingleFieldsVisitor; import org.opensearch.index.termvectors.TermVectorsService; import org.opensearch.search.DocValueFormat; @@ -833,6 +838,35 @@ public void testSkipListIntegrationMappingDefinitionSerialization() throws IOExc assertTrue("skip_list should be true in date_nanos mapper", dateFieldMapperNanos.skiplist()); } + public void testIsSkiplistDefaultEnabled() throws IOException { + DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> b.field("type", "date"))); + DateFieldMapper dateFieldMapper = (DateFieldMapper) mapper.mappers().getMapper("field"); + + // Test with no index sort and non-timestamp field + IndexMetadata noSortindexMetadata = new IndexMetadata.Builder("index").settings(getIndexSettings()).build(); + IndexSettings noSolrIndexSettings = new IndexSettings(noSortindexMetadata, getIndexSettings()); + IndexSortConfig noSortConfig = new IndexSortConfig(new IndexSettings(noSortindexMetadata, getIndexSettings())); + assertFalse(dateFieldMapper.isSkiplistDefaultEnabled(noSortConfig, "field")); + + // timestamp field + assertTrue(dateFieldMapper.isSkiplistDefaultEnabled(noSortConfig, "@timestamp")); + + // Create index settings with an index sort. + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .putList("index.sort.field", "field") + .build(); + + // Test with timestamp field + IndexMetadata indexMetadata = new IndexMetadata.Builder("index").settings(settings).build(); + IndexSettings indexSettings = new IndexSettings(indexMetadata, settings); + IndexSortConfig sortConfig = new IndexSortConfig(indexSettings); + assertTrue(dateFieldMapper.isSkiplistDefaultEnabled(sortConfig, "field")); + assertTrue(dateFieldMapper.isSkiplistDefaultEnabled(sortConfig, "@timestamp")); + } + public void testSkipListIntegrationFieldBehaviorConsistency() throws IOException { // Test that field behavior is consistent between skip_list enabled and disabled diff --git a/test/framework/src/main/java/org/opensearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/opensearch/index/mapper/MapperServiceTestCase.java index bd16e4f65e159..471ec8ce8b429 100644 --- a/test/framework/src/main/java/org/opensearch/index/mapper/MapperServiceTestCase.java +++ b/test/framework/src/main/java/org/opensearch/index/mapper/MapperServiceTestCase.java @@ -80,7 +80,11 @@ public abstract class MapperServiceTestCase extends OpenSearchTestCase { - protected static final Settings SETTINGS = Settings.builder().put("index.version.created", Version.CURRENT).build(); + protected static final Settings SETTINGS = Settings.builder() + .put("index.version.created", Version.CURRENT) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .build(); protected static final ToXContent.Params INCLUDE_DEFAULTS = new ToXContent.MapParams( Collections.singletonMap("include_defaults", "true")