diff --git a/docs/changelog/126629.yaml b/docs/changelog/126629.yaml new file mode 100644 index 0000000000000..49d04856c0b64 --- /dev/null +++ b/docs/changelog/126629.yaml @@ -0,0 +1,5 @@ +pr: 126629 +summary: Default new `semantic_text` fields to use BBQ when models are compatible +area: Relevance +type: enhancement +issues: [] diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java index 455255dc15c0f..261e3f8a2bf4c 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java +++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java @@ -128,6 +128,7 @@ private static IndexVersion def(int id, Version luceneVersion) { public static final IndexVersion LOGSB_OPTIONAL_SORTING_ON_HOST_NAME_BACKPORT = def(8_525_0_00, Version.LUCENE_9_12_1); public static final IndexVersion USE_SYNTHETIC_SOURCE_FOR_RECOVERY_BY_DEFAULT_BACKPORT = def(8_526_0_00, Version.LUCENE_9_12_1); public static final IndexVersion SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY = def(8_527_0_00, Version.LUCENE_9_12_1); + public static final IndexVersion SEMANTIC_TEXT_DEFAULTS_TO_BBQ = def(8_528_0_00, Version.LUCENE_9_12_1); /* * STOP! READ THIS FIRST! No, really, * ____ _____ ___ ____ _ ____ _____ _ ____ _____ _ _ ___ ____ _____ ___ ____ ____ _____ _ diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index 251998c84b8b7..88e5203dfa360 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -282,6 +282,11 @@ public Builder elementType(ElementType elementType) { return this; } + public Builder indexOptions(IndexOptions indexOptions) { + this.indexOptions.setValue(indexOptions); + return this; + } + @Override public DenseVectorFieldMapper build(MapperBuilderContext context) { // Validate again here because the dimensions or element type could have been set programmatically, @@ -1180,7 +1185,7 @@ public final String toString() { public abstract VectorSimilarityFunction vectorSimilarityFunction(IndexVersion indexVersion, ElementType elementType); } - abstract static class IndexOptions implements ToXContent { + public abstract static class IndexOptions implements ToXContent { final VectorIndexType type; IndexOptions(VectorIndexType type) { @@ -1189,21 +1194,36 @@ abstract static class IndexOptions implements ToXContent { abstract KnnVectorsFormat getVectorsFormat(ElementType elementType); - final void validateElementType(ElementType elementType) { - if (type.supportsElementType(elementType) == false) { + public boolean validate(ElementType elementType, int dim, boolean throwOnError) { + return validateElementType(elementType, throwOnError) && validateDimension(dim, throwOnError); + } + + public boolean validateElementType(ElementType elementType) { + return validateElementType(elementType, true); + } + + final boolean validateElementType(ElementType elementType, boolean throwOnError) { + boolean validElementType = type.supportsElementType(elementType); + if (throwOnError && validElementType == false) { throw new IllegalArgumentException( "[element_type] cannot be [" + elementType.toString() + "] when using index type [" + type + "]" ); } + return validElementType; } abstract boolean updatableTo(IndexOptions update); - public void validateDimension(int dim) { - if (type.supportsDimension(dim)) { - return; + public boolean validateDimension(int dim) { + return validateDimension(dim, true); + } + + public boolean validateDimension(int dim, boolean throwOnError) { + boolean supportsDimension = type.supportsDimension(dim); + if (throwOnError && supportsDimension == false) { + throw new IllegalArgumentException(type.name + " only supports even dimensions; provided=" + dim); } - throw new IllegalArgumentException(type.name + " only supports even dimensions; provided=" + dim); + return supportsDimension; } abstract boolean doEquals(IndexOptions other); @@ -1659,12 +1679,12 @@ boolean updatableTo(IndexOptions update) { } - static class Int8HnswIndexOptions extends IndexOptions { + public static class Int8HnswIndexOptions extends IndexOptions { private final int m; private final int efConstruction; private final Float confidenceInterval; - Int8HnswIndexOptions(int m, int efConstruction, Float confidenceInterval) { + public Int8HnswIndexOptions(int m, int efConstruction, Float confidenceInterval) { super(VectorIndexType.INT8_HNSW); this.m = m; this.efConstruction = efConstruction; @@ -1794,7 +1814,7 @@ public String toString() { } } - static class BBQHnswIndexOptions extends IndexOptions { + public static class BBQHnswIndexOptions extends IndexOptions { private final int m; private final int efConstruction; @@ -1837,11 +1857,14 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } @Override - public void validateDimension(int dim) { - if (type.supportsDimension(dim)) { - return; + public boolean validateDimension(int dim, boolean throwOnError) { + boolean supportsDimension = type.supportsDimension(dim); + if (throwOnError && supportsDimension == false) { + throw new IllegalArgumentException( + type.name + " does not support dimensions fewer than " + BBQ_MIN_DIMS + "; provided=" + dim + ); } - throw new IllegalArgumentException(type.name + " does not support dimensions fewer than " + BBQ_MIN_DIMS + "; provided=" + dim); + return supportsDimension; } } @@ -1882,12 +1905,16 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } @Override - public void validateDimension(int dim) { - if (type.supportsDimension(dim)) { - return; + public boolean validateDimension(int dim, boolean throwOnError) { + boolean supportsDimension = type.supportsDimension(dim); + if (throwOnError && supportsDimension == false) { + throw new IllegalArgumentException( + type.name + " does not support dimensions fewer than " + BBQ_MIN_DIMS + "; provided=" + dim + ); } - throw new IllegalArgumentException(type.name + " does not support dimensions fewer than " + BBQ_MIN_DIMS + "; provided=" + dim); + return supportsDimension; } + } public static final TypeParser PARSER = new TypeParser( @@ -2166,6 +2193,10 @@ int getVectorDimensions() { ElementType getElementType() { return elementType; } + + public IndexOptions getIndexOptions() { + return indexOptions; + } } private final IndexOptions indexOptions; diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java index b62e400826836..99d9bbf30158b 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java @@ -207,6 +207,13 @@ protected final MapperService createMapperService(Settings settings, String mapp return mapperService; } + protected final MapperService createMapperService(IndexVersion indexVersion, Settings settings, XContentBuilder mappings) + throws IOException { + MapperService mapperService = createMapperService(indexVersion, settings, () -> true, mappings); + merge(mapperService, mappings); + return mapperService; + } + protected final MapperService createMapperService(IndexVersion version, XContentBuilder mapping) throws IOException { return createMapperService(version, getIndexSettings(), () -> true, mapping); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java index f882a6a782871..af00c0248abba 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java @@ -9,6 +9,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat; import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.DocIdSetIterator; @@ -95,6 +96,7 @@ import java.util.function.Function; import java.util.function.Supplier; +import static org.elasticsearch.index.IndexVersions.SEMANTIC_TEXT_DEFAULTS_TO_BBQ; import static org.elasticsearch.inference.TaskType.SPARSE_EMBEDDING; import static org.elasticsearch.inference.TaskType.TEXT_EMBEDDING; import static org.elasticsearch.search.SearchService.DEFAULT_SIZE; @@ -135,6 +137,8 @@ public class SemanticTextFieldMapper extends FieldMapper implements InferenceFie public static final String CONTENT_TYPE = "semantic_text"; public static final String DEFAULT_ELSER_2_INFERENCE_ID = DEFAULT_ELSER_ID; + public static final float DEFAULT_RESCORE_OVERSAMPLE = 3.0f; + public static final TypeParser parser(Supplier modelRegistry) { return new TypeParser( (n, c) -> new Builder(n, c::bitSetProducer, c.getIndexSettings(), modelRegistry.get()), @@ -1056,12 +1060,30 @@ private static Mapper.Builder createEmbeddingsField( denseVectorMapperBuilder.dimensions(modelSettings.dimensions()); denseVectorMapperBuilder.elementType(modelSettings.elementType()); + DenseVectorFieldMapper.IndexOptions defaultIndexOptions = null; + if (indexVersionCreated.onOrAfter(SEMANTIC_TEXT_DEFAULTS_TO_BBQ)) { + defaultIndexOptions = defaultSemanticDenseIndexOptions(); + } + if (defaultIndexOptions != null + && defaultIndexOptions.validate(modelSettings.elementType(), modelSettings.dimensions(), false)) { + denseVectorMapperBuilder.indexOptions(defaultIndexOptions); + } + yield denseVectorMapperBuilder; } default -> throw new IllegalArgumentException("Invalid task_type in model_settings [" + modelSettings.taskType().name() + "]"); }; } + static DenseVectorFieldMapper.IndexOptions defaultSemanticDenseIndexOptions() { + // As embedding models for text perform better with BBQ, we aggressively default semantic_text fields to use optimized index + // options outside of dense_vector defaults + int m = Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN; + int efConstruction = Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH; + DenseVectorFieldMapper.RescoreVector rescoreVector = new DenseVectorFieldMapper.RescoreVector(DEFAULT_RESCORE_OVERSAMPLE); + return new DenseVectorFieldMapper.BBQHnswIndexOptions(m, efConstruction, rescoreVector); + } + private static boolean canMergeModelSettings(MinimalServiceSettings previous, MinimalServiceSettings current, Conflicts conflicts) { if (previous != null && current != null && previous.canMergeWith(current)) { return true; diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java index f872d8f302f37..09a919a4d2c36 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java @@ -9,6 +9,7 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.IndexableField; @@ -34,6 +35,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentParsingException; import org.elasticsearch.index.mapper.FieldMapper; @@ -65,6 +67,7 @@ import org.elasticsearch.search.SearchHit; import org.elasticsearch.test.ClusterServiceUtils; import org.elasticsearch.test.client.NoOpClient; +import org.elasticsearch.test.index.IndexVersionUtils; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentType; @@ -149,9 +152,25 @@ protected Supplier getModelRegistry() { private MapperService createMapperService(XContentBuilder mappings, boolean useLegacyFormat) throws IOException { var settings = Settings.builder() + .put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), indexVersion) .put(InferenceMetadataFieldsMapper.USE_LEGACY_SEMANTIC_TEXT_FORMAT.getKey(), useLegacyFormat) .build(); - return createMapperService(settings, mappings); + // TODO - This is added, because we discovered a bug where the index version was not being correctly propagated + // in our mappings even though we were specifying the index version in settings. We will fix this in a followup and + // remove the boolean flag accordingly. + if (propagateIndexVersion) { + return createMapperService(indexVersion, settings, mappings); + } else { + return createMapperService(settings, mappings); + } + } + + private static void validateIndexVersion(IndexVersion indexVersion, boolean useLegacyFormat) { + if (useLegacyFormat == false + && indexVersion.before(IndexVersions.INFERENCE_METADATA_FIELDS) + && indexVersion.between(IndexVersions.INFERENCE_METADATA_FIELDS_BACKPORT, IndexVersions.UPGRADE_TO_LUCENE_10_0_0) == false) { + throw new IllegalArgumentException("Index version " + indexVersion + " does not support new semantic text format"); + } } @Override @@ -597,14 +616,15 @@ public void testUpdateSearchInferenceId() throws IOException { } private static void assertSemanticTextField(MapperService mapperService, String fieldName, boolean expectedModelSettings) { - assertSemanticTextField(mapperService, fieldName, expectedModelSettings, null); + assertSemanticTextField(mapperService, fieldName, expectedModelSettings, null, null); } private static void assertSemanticTextField( MapperService mapperService, String fieldName, boolean expectedModelSettings, - ChunkingSettings expectedChunkingSettings + ChunkingSettings expectedChunkingSettings, + DenseVectorFieldMapper.IndexOptions expectedIndexOptions ) { Mapper mapper = mapperService.mappingLookup().getMapper(fieldName); assertNotNull(mapper); @@ -650,8 +670,17 @@ private static void assertSemanticTextField( assertThat(embeddingsMapper, instanceOf(SparseVectorFieldMapper.class)); SparseVectorFieldMapper sparseMapper = (SparseVectorFieldMapper) embeddingsMapper; assertEquals(sparseMapper.fieldType().isStored(), semanticTextFieldType.useLegacyFormat() == false); + assertNull(expectedIndexOptions); + } + case TEXT_EMBEDDING -> { + assertThat(embeddingsMapper, instanceOf(DenseVectorFieldMapper.class)); + DenseVectorFieldMapper denseVectorFieldMapper = (DenseVectorFieldMapper) embeddingsMapper; + if (expectedIndexOptions != null) { + assertEquals(expectedIndexOptions, denseVectorFieldMapper.fieldType().getIndexOptions()); + } else { + assertNull(denseVectorFieldMapper.fieldType().getIndexOptions()); + } } - case TEXT_EMBEDDING -> assertThat(embeddingsMapper, instanceOf(DenseVectorFieldMapper.class)); default -> throw new AssertionError("Invalid task type"); } } else { @@ -946,11 +975,11 @@ public void testSettingAndUpdatingChunkingSettings() throws IOException { mapping(b -> addSemanticTextMapping(b, fieldName, model.getInferenceEntityId(), null, chunkingSettings)), useLegacyFormat ); - assertSemanticTextField(mapperService, fieldName, false, chunkingSettings); + assertSemanticTextField(mapperService, fieldName, false, chunkingSettings, null); ChunkingSettings newChunkingSettings = generateRandomChunkingSettingsOtherThan(chunkingSettings); merge(mapperService, mapping(b -> addSemanticTextMapping(b, fieldName, model.getInferenceEntityId(), null, newChunkingSettings))); - assertSemanticTextField(mapperService, fieldName, false, newChunkingSettings); + assertSemanticTextField(mapperService, fieldName, false, newChunkingSettings, null); } public void testModelSettingsRequiredWithChunks() throws IOException { @@ -1080,6 +1109,74 @@ public void testExistsQueryDenseVector() throws IOException { assertThat(existsQuery, instanceOf(ESToParentBlockJoinQuery.class)); } + private static DenseVectorFieldMapper.IndexOptions defaultDenseVectorIndexOptions() { + // These are the default index options for dense_vector fields, and used for semantic_text fields incompatible with BBQ. + int m = Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN; + int efConstruction = Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH; + return new DenseVectorFieldMapper.Int8HnswIndexOptions(m, efConstruction, null, null); + } + + public void testDefaultIndexOptions() throws IOException { + + // We default to BBQ for eligible dense vectors + var mapperService = createMapperService(fieldMapping(b -> { + b.field("type", "semantic_text"); + b.field("inference_id", "another_inference_id"); + b.startObject("model_settings"); + b.field("task_type", "text_embedding"); + b.field("dimensions", 100); + b.field("similarity", "cosine"); + b.field("element_type", "float"); + b.endObject(); + }), useLegacyFormat, IndexVersions.SEMANTIC_TEXT_DEFAULTS_TO_BBQ); + assertSemanticTextField(mapperService, "field", true, null, SemanticTextFieldMapper.defaultSemanticDenseIndexOptions()); + + // Element types that are incompatible with BBQ will continue to use dense_vector defaults + mapperService = createMapperService(fieldMapping(b -> { + b.field("type", "semantic_text"); + b.field("inference_id", "another_inference_id"); + b.startObject("model_settings"); + b.field("task_type", "text_embedding"); + b.field("dimensions", 100); + b.field("similarity", "cosine"); + b.field("element_type", "byte"); + b.endObject(); + }), useLegacyFormat, IndexVersions.SEMANTIC_TEXT_DEFAULTS_TO_BBQ); + assertSemanticTextField(mapperService, "field", true, null, null); + + // A dim count of 10 is too small to support BBQ, so we continue to use dense_vector defaults + mapperService = createMapperService(fieldMapping(b -> { + b.field("type", "semantic_text"); + b.field("inference_id", "another_inference_id"); + b.startObject("model_settings"); + b.field("task_type", "text_embedding"); + b.field("dimensions", 10); + b.field("similarity", "cosine"); + b.field("element_type", "float"); + b.endObject(); + }), useLegacyFormat, IndexVersions.SEMANTIC_TEXT_DEFAULTS_TO_BBQ); + assertSemanticTextField(mapperService, "field", true, null, defaultDenseVectorIndexOptions()); + + // Previous index versions do not set BBQ index options + mapperService = createMapperService(fieldMapping(b -> { + b.field("type", "semantic_text"); + b.field("inference_id", "another_inference_id"); + b.startObject("model_settings"); + b.field("task_type", "text_embedding"); + b.field("dimensions", 100); + b.field("similarity", "cosine"); + b.field("element_type", "float"); + b.endObject(); + }), + useLegacyFormat, + IndexVersions.INFERENCE_METADATA_FIELDS, + IndexVersionUtils.getPreviousVersion(IndexVersions.SEMANTIC_TEXT_DEFAULTS_TO_BBQ), + true + ); + assertSemanticTextField(mapperService, "field", true, null, defaultDenseVectorIndexOptions()); + + } + @Override protected void assertExistsQuery(MappedFieldType fieldType, Query query, LuceneDocument fields) { // Until a doc is indexed, the query is rewritten as match no docs