Merge dynamic field type lookup into FieldTypeLookup (#72024)

Flattened field mappers have a specialised lookup class to handle
the fact that their MappedFieldTypes are dynamic and generated
on-the-fly, rather than being registered up front. The way this is
implemented means that this lookup class also has to be aware of
field aliases, which is blocking our attempt to re-implement aliases
as runtime fields.

This commit merges dynamic field lookup handling into the standard
FieldTypeLookup class.  When the lookup class is built, it checks each
MappedFieldType being registered to see if it implements a new
DynamicFieldType interface, and stores these in a separate map.
If a field containing dots does not have a field type directly registered
against it, we check if a dynamic field type matches one of its dot-
delimited prefixes, and if so we then return the result of calling
`DynamicFieldType.getChildFieldType()` with the remainder of the
path.

Author: romseygeek

server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldType.java

@@ -0,0 +1,34 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.index.mapper;
+
+/**
+ * Defines a MappedFieldType that exposes dynamic child field types
+ *
+ * If the field is named 'my_field', then a user is able to search on
+ * the field in both of the following ways:
+ *  - Using the field name 'my_field', which will delegate to the field type
+ *    as usual.
+ *  - Using any sub-key, for example 'my_field.some_key'. In this case, the
+ *    search is delegated to {@link #getChildFieldType(String)}, with 'some_key'
+ *    passed as the argument. The field may create a new field type dynamically
+ *    in order to handle the search.
+ *
+ *  To prevent conflicts between these dynamic sub-keys and multi-fields, any
+ *  field mappers generating field types that implement this interface should
+ *  explicitly disallow multi-fields.
+ */
+public interface DynamicFieldType {
+
+    /**
+     * Returns a dynamic MappedFieldType for the given path
+     */
+    MappedFieldType getChildFieldType(String path);
+
+}

server/src/main/java/org/elasticsearch/index/mapper/DynamicKeyFieldMapper.java

@@ -16,13 +16,15 @@
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 
 /**
  * An immutable container for looking up {@link MappedFieldType}s by their name.
  */
 final class FieldTypeLookup {
     private final Map<String, MappedFieldType> fullNameToFieldType = new HashMap<>();
+    private final Map<String, DynamicFieldType> dynamicFieldTypes = new HashMap<>();
 
     /**
      * A map from field name to all fields whose content has been copied into it
@@ -33,7 +35,8 @@ final class FieldTypeLookup {
      */
     private final Map<String, Set<String>> fieldToCopiedFields = new HashMap<>();
     private final String type;
-    private final DynamicKeyFieldTypeLookup dynamicKeyLookup;
+
+    private final int maxParentPathDots;
 
     FieldTypeLookup(
         String type,
@@ -42,16 +45,13 @@ final class FieldTypeLookup {
         Collection<RuntimeField> runtimeFields
     ) {
         this.type = type;
-        Map<String, DynamicKeyFieldMapper> dynamicKeyMappers = new HashMap<>();
-
         for (FieldMapper fieldMapper : fieldMappers) {
             String fieldName = fieldMapper.name();
             MappedFieldType fieldType = fieldMapper.fieldType();
             fullNameToFieldType.put(fieldType.name(), fieldType);
-            if (fieldMapper instanceof DynamicKeyFieldMapper) {
-                dynamicKeyMappers.put(fieldName, (DynamicKeyFieldMapper) fieldMapper);
+            if (fieldType instanceof DynamicFieldType) {
+                dynamicFieldTypes.put(fieldType.name(), (DynamicFieldType) fieldType);
             }
-
             for (String targetField : fieldMapper.copyTo().copyToFields()) {
                 Set<String> sourcePath = fieldToCopiedFields.get(targetField);
                 if (sourcePath == null) {
@@ -63,21 +63,37 @@ final class FieldTypeLookup {
             }
         }
 
-        final Map<String, String> aliasToConcreteName = new HashMap<>();
+        int maxParentPathDots = 0;
+        for (String dynamicRoot : dynamicFieldTypes.keySet()) {
+            maxParentPathDots = Math.max(maxParentPathDots, dotCount(dynamicRoot));
+        }
+        this.maxParentPathDots = maxParentPathDots;
+
         for (FieldAliasMapper fieldAliasMapper : fieldAliasMappers) {
             String aliasName = fieldAliasMapper.name();
             String path = fieldAliasMapper.path();
-            aliasToConcreteName.put(aliasName, path);
-            fullNameToFieldType.put(aliasName, fullNameToFieldType.get(path));
+            MappedFieldType fieldType = fullNameToFieldType.get(path);
+            fullNameToFieldType.put(aliasName, fieldType);
+            if (fieldType instanceof DynamicFieldType) {
+                dynamicFieldTypes.put(aliasName, (DynamicFieldType) fieldType);
+            }
         }
 
         for (RuntimeField runtimeField : runtimeFields) {
             MappedFieldType runtimeFieldType = runtimeField.asMappedFieldType();
             //this will override concrete fields with runtime fields that have the same name
             fullNameToFieldType.put(runtimeFieldType.name(), runtimeFieldType);
         }
+    }
 
-        this.dynamicKeyLookup = new DynamicKeyFieldTypeLookup(dynamicKeyMappers, aliasToConcreteName);
+    private static int dotCount(String path) {
+        int dotCount = 0;
+        for (int i = 0; i < path.length(); i++) {
+            if (path.charAt(i) == '.') {
+                dotCount++;
+            }
+        }
+        return dotCount;
     }
 
     /**
@@ -92,10 +108,42 @@ MappedFieldType get(String field) {
         if (fieldType != null) {
             return fieldType;
         }
+        return getDynamicField(field);
+    }
+
+    // for testing
+    int getMaxParentPathDots() {
+        return maxParentPathDots;
+    }
+
+    // Check if the given field corresponds to a dynamic key mapper of the
+    // form 'path_to_field.path_to_key'. If so, returns a field type that
+    // can be used to perform searches on this field. Otherwise returns null.
+    private MappedFieldType getDynamicField(String field) {
+        if (dynamicFieldTypes.isEmpty()) {
+            // no parent fields defined
+            return null;
+        }
+        int dotIndex = -1;
+        int fieldDepth = -1;
 
-        // If the mapping contains fields that support dynamic sub-key lookup, check
-        // if this could correspond to a keyed field of the form 'path_to_field.path_to_key'.
-        return dynamicKeyLookup.get(field);
+        while (true) {
+            if (++fieldDepth > maxParentPathDots) {
+                return null;
+            }
+
+            dotIndex = field.indexOf('.', dotIndex + 1);
+            if (dotIndex < 0) {
+                return null;
+            }
+
+            String parentField = field.substring(0, dotIndex);
+            DynamicFieldType dft = dynamicFieldTypes.get(parentField);
+            if (dft != null && Objects.equals(field, parentField) == false) {
+                String key = field.substring(dotIndex + 1);
+                return dft.getChildFieldType(key);
+            }
+        }
     }
 
     /**
@@ -138,7 +186,10 @@ Set<String> sourcePaths(String field) {
         if (fullNameToFieldType.isEmpty()) {
             return org.elasticsearch.common.collect.Set.of();
         }
-        if (dynamicKeyLookup.get(field) != null) {
+
+        // If the field is dynamically generated then return its full path
+        MappedFieldType fieldType = getDynamicField(field);
+        if (fieldType != null) {
             return Collections.singleton(field);
         }
 

server/src/main/java/org/elasticsearch/index/mapper/DynamicKeyFieldTypeLookup.java

@@ -32,7 +32,7 @@
 import org.elasticsearch.index.fielddata.plain.AbstractLeafOrdinalsFieldData;
 import org.elasticsearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData;
 import org.elasticsearch.index.mapper.ContentPath;
-import org.elasticsearch.index.mapper.DynamicKeyFieldMapper;
+import org.elasticsearch.index.mapper.DynamicFieldType;
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.Mapper;
@@ -84,7 +84,7 @@
  * "key\0some value" and "key2.key3\0true". Note that \0 is used as a reserved separator
  *  character (see {@link FlattenedFieldParser#SEPARATOR}).
  */
-public final class FlattenedFieldMapper extends DynamicKeyFieldMapper {
+public final class FlattenedFieldMapper extends FieldMapper {
 
     public static final String CONTENT_TYPE = "flattened";
     private static final String KEYED_FIELD_SUFFIX = "._keyed";
@@ -385,7 +385,7 @@ public IndexFieldData<?> build(IndexFieldDataCache cache, CircuitBreakerService
      * A field type that represents all 'root' values. This field type is used in
      * searches on the flattened field itself, e.g. 'my_flattened: some_value'.
      */
-    public static final class RootFlattenedFieldType extends StringFieldType {
+    public static final class RootFlattenedFieldType extends StringFieldType implements DynamicFieldType {
         private final boolean splitQueriesOnWhitespace;
         private final boolean eagerGlobalOrdinals;
 
@@ -426,6 +426,11 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S
         public ValueFetcher valueFetcher(SearchExecutionContext context, String format) {
             return SourceValueFetcher.identity(name(), context, format);
         }
+
+        @Override
+        public MappedFieldType getChildFieldType(String childPath) {
+            return new KeyedFlattenedFieldType(name(), childPath, this);
+        }
     }
 
     private final FlattenedFieldParser fieldParser;
@@ -434,7 +439,7 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format)
     private FlattenedFieldMapper(String simpleName,
                                  MappedFieldType mappedFieldType,
                                  Builder builder) {
-        super(simpleName, mappedFieldType, Lucene.KEYWORD_ANALYZER, CopyTo.empty());
+        super(simpleName, mappedFieldType, Lucene.KEYWORD_ANALYZER, MultiFields.empty(), CopyTo.empty());
         this.builder = builder;
         this.fieldParser = new FlattenedFieldParser(mappedFieldType.name(), mappedFieldType.name() + KEYED_FIELD_SUFFIX,
             mappedFieldType, builder.depthLimit.get(), builder.ignoreAbove.get(), builder.nullValue.get());
@@ -458,11 +463,6 @@ public RootFlattenedFieldType fieldType() {
         return (RootFlattenedFieldType) super.fieldType();
     }
 
-    @Override
-    public KeyedFlattenedFieldType keyedFieldType(String key) {
-        return new KeyedFlattenedFieldType(name(), key, fieldType());
-    }
-
     @Override
     protected void parseCreateField(ParseContext context) throws IOException {
         if (context.parser().currentToken() == XContentParser.Token.VALUE_NULL) {