[8.x] Default new semantic_text fields to use BBQ when models are compatible (#126629)

docs/changelog/126629.yaml

@@ -0,0 +1,5 @@
+pr: 126629
+summary: Default new `semantic_text` fields to use BBQ when models are compatible
+area: Relevance
+type: enhancement
+issues: []

server/src/main/java/org/elasticsearch/index/IndexVersions.java

@@ -128,6 +128,7 @@ private static IndexVersion def(int id, Version luceneVersion) {
     public static final IndexVersion LOGSB_OPTIONAL_SORTING_ON_HOST_NAME_BACKPORT = def(8_525_0_00, Version.LUCENE_9_12_1);
     public static final IndexVersion USE_SYNTHETIC_SOURCE_FOR_RECOVERY_BY_DEFAULT_BACKPORT = def(8_526_0_00, Version.LUCENE_9_12_1);
     public static final IndexVersion SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY = def(8_527_0_00, Version.LUCENE_9_12_1);
+    public static final IndexVersion SEMANTIC_TEXT_DEFAULTS_TO_BBQ = def(8_528_0_00, Version.LUCENE_9_12_1);
     /*
      * STOP! READ THIS FIRST! No, really,
      *        ____ _____ ___  ____  _        ____  _____    _    ____    _____ _   _ ___ ____    _____ ___ ____  ____ _____ _

server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java

@@ -282,6 +282,11 @@ public Builder elementType(ElementType elementType) {
             return this;
         }
 
+        public Builder indexOptions(IndexOptions indexOptions) {
+            this.indexOptions.setValue(indexOptions);
+            return this;
+        }
+
         @Override
         public DenseVectorFieldMapper build(MapperBuilderContext context) {
             // Validate again here because the dimensions or element type could have been set programmatically,
@@ -1180,7 +1185,7 @@ public final String toString() {
         public abstract VectorSimilarityFunction vectorSimilarityFunction(IndexVersion indexVersion, ElementType elementType);
     }
 
-    abstract static class IndexOptions implements ToXContent {
+    public abstract static class IndexOptions implements ToXContent {
         final VectorIndexType type;
 
         IndexOptions(VectorIndexType type) {
@@ -1189,21 +1194,36 @@ abstract static class IndexOptions implements ToXContent {
 
         abstract KnnVectorsFormat getVectorsFormat(ElementType elementType);
 
-        final void validateElementType(ElementType elementType) {
-            if (type.supportsElementType(elementType) == false) {
+        public boolean validate(ElementType elementType, int dim, boolean throwOnError) {
+            return validateElementType(elementType, throwOnError) && validateDimension(dim, throwOnError);
+        }
+
+        public boolean validateElementType(ElementType elementType) {
+            return validateElementType(elementType, true);
+        }
+
+        final boolean validateElementType(ElementType elementType, boolean throwOnError) {
+            boolean validElementType = type.supportsElementType(elementType);
+            if (throwOnError && validElementType == false) {
                 throw new IllegalArgumentException(
                     "[element_type] cannot be [" + elementType.toString() + "] when using index type [" + type + "]"
                 );
             }
+            return validElementType;
         }
 
         abstract boolean updatableTo(IndexOptions update);
 
-        public void validateDimension(int dim) {
-            if (type.supportsDimension(dim)) {
-                return;
+        public boolean validateDimension(int dim) {
+            return validateDimension(dim, true);
+        }
+
+        public boolean validateDimension(int dim, boolean throwOnError) {
+            boolean supportsDimension = type.supportsDimension(dim);
+            if (throwOnError && supportsDimension == false) {
+                throw new IllegalArgumentException(type.name + " only supports even dimensions; provided=" + dim);
             }
-            throw new IllegalArgumentException(type.name + " only supports even dimensions; provided=" + dim);
+            return supportsDimension;
         }
 
         abstract boolean doEquals(IndexOptions other);
@@ -1659,12 +1679,12 @@ boolean updatableTo(IndexOptions update) {
 
     }
 
-    static class Int8HnswIndexOptions extends IndexOptions {
+    public static class Int8HnswIndexOptions extends IndexOptions {
         private final int m;
         private final int efConstruction;
         private final Float confidenceInterval;
 
-        Int8HnswIndexOptions(int m, int efConstruction, Float confidenceInterval) {
+        public Int8HnswIndexOptions(int m, int efConstruction, Float confidenceInterval) {
             super(VectorIndexType.INT8_HNSW);
             this.m = m;
             this.efConstruction = efConstruction;
@@ -1794,7 +1814,7 @@ public String toString() {
         }
     }
 
-    static class BBQHnswIndexOptions extends IndexOptions {
+    public static class BBQHnswIndexOptions extends IndexOptions {
         private final int m;
         private final int efConstruction;
 
@@ -1837,11 +1857,14 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         }
 
         @Override
-        public void validateDimension(int dim) {
-            if (type.supportsDimension(dim)) {
-                return;
+        public boolean validateDimension(int dim, boolean throwOnError) {
+            boolean supportsDimension = type.supportsDimension(dim);
+            if (throwOnError && supportsDimension == false) {
+                throw new IllegalArgumentException(
+                    type.name + " does not support dimensions fewer than " + BBQ_MIN_DIMS + "; provided=" + dim
+                );
             }
-            throw new IllegalArgumentException(type.name + " does not support dimensions fewer than " + BBQ_MIN_DIMS + "; provided=" + dim);
+            return supportsDimension;
         }
     }
 
@@ -1882,12 +1905,16 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         }
 
         @Override
-        public void validateDimension(int dim) {
-            if (type.supportsDimension(dim)) {
-                return;
+        public boolean validateDimension(int dim, boolean throwOnError) {
+            boolean supportsDimension = type.supportsDimension(dim);
+            if (throwOnError && supportsDimension == false) {
+                throw new IllegalArgumentException(
+                    type.name + " does not support dimensions fewer than " + BBQ_MIN_DIMS + "; provided=" + dim
+                );
             }
-            throw new IllegalArgumentException(type.name + " does not support dimensions fewer than " + BBQ_MIN_DIMS + "; provided=" + dim);
+            return supportsDimension;
         }
+
     }
 
     public static final TypeParser PARSER = new TypeParser(
@@ -2166,6 +2193,10 @@ int getVectorDimensions() {
         ElementType getElementType() {
             return elementType;
         }
+
+        public IndexOptions getIndexOptions() {
+            return indexOptions;
+        }
     }
 
     private final IndexOptions indexOptions;

test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java

@@ -207,6 +207,13 @@ protected final MapperService createMapperService(Settings settings, String mapp
         return mapperService;
     }
 
+    protected final MapperService createMapperService(IndexVersion indexVersion, Settings settings, XContentBuilder mappings)
+        throws IOException {
+        MapperService mapperService = createMapperService(indexVersion, settings, () -> true, mappings);
+        merge(mapperService, mappings);
+        return mapperService;
+    }
+
     protected final MapperService createMapperService(IndexVersion version, XContentBuilder mapping) throws IOException {
         return createMapperService(version, getIndexSettings(), () -> true, mapping);
     }

x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapper.java

@@ -9,6 +9,7 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;
 import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.search.DocIdSetIterator;
@@ -95,6 +96,7 @@
 import java.util.function.Function;
 import java.util.function.Supplier;
 
+import static org.elasticsearch.index.IndexVersions.SEMANTIC_TEXT_DEFAULTS_TO_BBQ;
 import static org.elasticsearch.inference.TaskType.SPARSE_EMBEDDING;
 import static org.elasticsearch.inference.TaskType.TEXT_EMBEDDING;
 import static org.elasticsearch.search.SearchService.DEFAULT_SIZE;
@@ -135,6 +137,8 @@ public class SemanticTextFieldMapper extends FieldMapper implements InferenceFie
     public static final String CONTENT_TYPE = "semantic_text";
     public static final String DEFAULT_ELSER_2_INFERENCE_ID = DEFAULT_ELSER_ID;
 
+    public static final float DEFAULT_RESCORE_OVERSAMPLE = 3.0f;
+
     public static final TypeParser parser(Supplier<ModelRegistry> modelRegistry) {
         return new TypeParser(
             (n, c) -> new Builder(n, c::bitSetProducer, c.getIndexSettings(), modelRegistry.get()),
@@ -1056,12 +1060,30 @@ private static Mapper.Builder createEmbeddingsField(
                 denseVectorMapperBuilder.dimensions(modelSettings.dimensions());
                 denseVectorMapperBuilder.elementType(modelSettings.elementType());
 
+                DenseVectorFieldMapper.IndexOptions defaultIndexOptions = null;
+                if (indexVersionCreated.onOrAfter(SEMANTIC_TEXT_DEFAULTS_TO_BBQ)) {
+                    defaultIndexOptions = defaultSemanticDenseIndexOptions();
+                }
+                if (defaultIndexOptions != null
+                    && defaultIndexOptions.validate(modelSettings.elementType(), modelSettings.dimensions(), false)) {
+                    denseVectorMapperBuilder.indexOptions(defaultIndexOptions);
+                }
+
                 yield denseVectorMapperBuilder;
             }
             default -> throw new IllegalArgumentException("Invalid task_type in model_settings [" + modelSettings.taskType().name() + "]");
         };
     }
 
+    static DenseVectorFieldMapper.IndexOptions defaultSemanticDenseIndexOptions() {
+        // As embedding models for text perform better with BBQ, we aggressively default semantic_text fields to use optimized index
+        // options outside of dense_vector defaults
+        int m = Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN;
+        int efConstruction = Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH;
+        DenseVectorFieldMapper.RescoreVector rescoreVector = new DenseVectorFieldMapper.RescoreVector(DEFAULT_RESCORE_OVERSAMPLE);
+        return new DenseVectorFieldMapper.BBQHnswIndexOptions(m, efConstruction, rescoreVector);
+    }
+
     private static boolean canMergeModelSettings(MinimalServiceSettings previous, MinimalServiceSettings current, Conflicts conflicts) {
         if (previous != null && current != null && previous.canMergeWith(current)) {
             return true;

x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java

@@ -9,6 +9,7 @@
 
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
+import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.index.IndexableField;
@@ -34,6 +35,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.CheckedConsumer;
 import org.elasticsearch.index.IndexVersion;
+import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.DocumentParsingException;
 import org.elasticsearch.index.mapper.FieldMapper;
@@ -65,6 +67,7 @@
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.test.ClusterServiceUtils;
 import org.elasticsearch.test.client.NoOpClient;
+import org.elasticsearch.test.index.IndexVersionUtils;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentType;
@@ -149,9 +152,25 @@ protected Supplier<ModelRegistry> getModelRegistry() {
 
     private MapperService createMapperService(XContentBuilder mappings, boolean useLegacyFormat) throws IOException {
         var settings = Settings.builder()
+            .put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), indexVersion)
             .put(InferenceMetadataFieldsMapper.USE_LEGACY_SEMANTIC_TEXT_FORMAT.getKey(), useLegacyFormat)
             .build();
-        return createMapperService(settings, mappings);
+        // TODO - This is added, because we discovered a bug where the index version was not being correctly propagated
+        // in our mappings even though we were specifying the index version in settings. We will fix this in a followup and
+        // remove the boolean flag accordingly.
+        if (propagateIndexVersion) {
+            return createMapperService(indexVersion, settings, mappings);
+        } else {
+            return createMapperService(settings, mappings);
+        }
+    }
+
+    private static void validateIndexVersion(IndexVersion indexVersion, boolean useLegacyFormat) {
+        if (useLegacyFormat == false
+            && indexVersion.before(IndexVersions.INFERENCE_METADATA_FIELDS)
+            && indexVersion.between(IndexVersions.INFERENCE_METADATA_FIELDS_BACKPORT, IndexVersions.UPGRADE_TO_LUCENE_10_0_0) == false) {
+            throw new IllegalArgumentException("Index version " + indexVersion + " does not support new semantic text format");
+        }
     }
 
     @Override
@@ -597,14 +616,15 @@ public void testUpdateSearchInferenceId() throws IOException {
     }
 
     private static void assertSemanticTextField(MapperService mapperService, String fieldName, boolean expectedModelSettings) {
-        assertSemanticTextField(mapperService, fieldName, expectedModelSettings, null);
+        assertSemanticTextField(mapperService, fieldName, expectedModelSettings, null, null);
     }
 
     private static void assertSemanticTextField(
         MapperService mapperService,
         String fieldName,
         boolean expectedModelSettings,
-        ChunkingSettings expectedChunkingSettings
+        ChunkingSettings expectedChunkingSettings,
+        DenseVectorFieldMapper.IndexOptions expectedIndexOptions
     ) {
         Mapper mapper = mapperService.mappingLookup().getMapper(fieldName);
         assertNotNull(mapper);
@@ -650,8 +670,17 @@ private static void assertSemanticTextField(
                     assertThat(embeddingsMapper, instanceOf(SparseVectorFieldMapper.class));
                     SparseVectorFieldMapper sparseMapper = (SparseVectorFieldMapper) embeddingsMapper;
                     assertEquals(sparseMapper.fieldType().isStored(), semanticTextFieldType.useLegacyFormat() == false);
+                    assertNull(expectedIndexOptions);
+                }
+                case TEXT_EMBEDDING -> {
+                    assertThat(embeddingsMapper, instanceOf(DenseVectorFieldMapper.class));
+                    DenseVectorFieldMapper denseVectorFieldMapper = (DenseVectorFieldMapper) embeddingsMapper;
+                    if (expectedIndexOptions != null) {
+                        assertEquals(expectedIndexOptions, denseVectorFieldMapper.fieldType().getIndexOptions());
+                    } else {
+                        assertNull(denseVectorFieldMapper.fieldType().getIndexOptions());
+                    }
                 }
-                case TEXT_EMBEDDING -> assertThat(embeddingsMapper, instanceOf(DenseVectorFieldMapper.class));
                 default -> throw new AssertionError("Invalid task type");
             }
         } else {
@@ -946,11 +975,11 @@ public void testSettingAndUpdatingChunkingSettings() throws IOException {
             mapping(b -> addSemanticTextMapping(b, fieldName, model.getInferenceEntityId(), null, chunkingSettings)),
             useLegacyFormat
         );
-        assertSemanticTextField(mapperService, fieldName, false, chunkingSettings);
+        assertSemanticTextField(mapperService, fieldName, false, chunkingSettings, null);
 
         ChunkingSettings newChunkingSettings = generateRandomChunkingSettingsOtherThan(chunkingSettings);
         merge(mapperService, mapping(b -> addSemanticTextMapping(b, fieldName, model.getInferenceEntityId(), null, newChunkingSettings)));
-        assertSemanticTextField(mapperService, fieldName, false, newChunkingSettings);
+        assertSemanticTextField(mapperService, fieldName, false, newChunkingSettings, null);
     }
 
     public void testModelSettingsRequiredWithChunks() throws IOException {
@@ -1080,6 +1109,74 @@ public void testExistsQueryDenseVector() throws IOException {
         assertThat(existsQuery, instanceOf(ESToParentBlockJoinQuery.class));
     }
 
+    private static DenseVectorFieldMapper.IndexOptions defaultDenseVectorIndexOptions() {
+        // These are the default index options for dense_vector fields, and used for semantic_text fields incompatible with BBQ.
+        int m = Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN;
+        int efConstruction = Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH;
+        return new DenseVectorFieldMapper.Int8HnswIndexOptions(m, efConstruction, null, null);
+    }
+
+    public void testDefaultIndexOptions() throws IOException {
+
+        // We default to BBQ for eligible dense vectors
+        var mapperService = createMapperService(fieldMapping(b -> {
+            b.field("type", "semantic_text");
+            b.field("inference_id", "another_inference_id");
+            b.startObject("model_settings");
+            b.field("task_type", "text_embedding");
+            b.field("dimensions", 100);
+            b.field("similarity", "cosine");
+            b.field("element_type", "float");
+            b.endObject();
+        }), useLegacyFormat, IndexVersions.SEMANTIC_TEXT_DEFAULTS_TO_BBQ);
+        assertSemanticTextField(mapperService, "field", true, null, SemanticTextFieldMapper.defaultSemanticDenseIndexOptions());
+
+        // Element types that are incompatible with BBQ will continue to use dense_vector defaults
+        mapperService = createMapperService(fieldMapping(b -> {
+            b.field("type", "semantic_text");
+            b.field("inference_id", "another_inference_id");
+            b.startObject("model_settings");
+            b.field("task_type", "text_embedding");
+            b.field("dimensions", 100);
+            b.field("similarity", "cosine");
+            b.field("element_type", "byte");
+            b.endObject();
+        }), useLegacyFormat, IndexVersions.SEMANTIC_TEXT_DEFAULTS_TO_BBQ);
+        assertSemanticTextField(mapperService, "field", true, null, null);
+
+        // A dim count of 10 is too small to support BBQ, so we continue to use dense_vector defaults
+        mapperService = createMapperService(fieldMapping(b -> {
+            b.field("type", "semantic_text");
+            b.field("inference_id", "another_inference_id");
+            b.startObject("model_settings");
+            b.field("task_type", "text_embedding");
+            b.field("dimensions", 10);
+            b.field("similarity", "cosine");
+            b.field("element_type", "float");
+            b.endObject();
+        }), useLegacyFormat, IndexVersions.SEMANTIC_TEXT_DEFAULTS_TO_BBQ);
+        assertSemanticTextField(mapperService, "field", true, null, defaultDenseVectorIndexOptions());
+
+        // Previous index versions do not set BBQ index options
+        mapperService = createMapperService(fieldMapping(b -> {
+            b.field("type", "semantic_text");
+            b.field("inference_id", "another_inference_id");
+            b.startObject("model_settings");
+            b.field("task_type", "text_embedding");
+            b.field("dimensions", 100);
+            b.field("similarity", "cosine");
+            b.field("element_type", "float");
+            b.endObject();
+        }),
+            useLegacyFormat,
+            IndexVersions.INFERENCE_METADATA_FIELDS,
+            IndexVersionUtils.getPreviousVersion(IndexVersions.SEMANTIC_TEXT_DEFAULTS_TO_BBQ),
+            true
+        );
+        assertSemanticTextField(mapperService, "field", true, null, defaultDenseVectorIndexOptions());
+
+    }
+
     @Override
     protected void assertExistsQuery(MappedFieldType fieldType, Query query, LuceneDocument fields) {
         // Until a doc is indexed, the query is rewritten as match no docs