Introduce runtime section in mappings (#62906)

The runtime section is at the same level as the existing properties section. Its purpose is to hold runtime fields only. With the introduction of the runtime section, a runtime field can be defined by specifying its type (previously called runtime_type) and script.

```
PUT /my-index/_mappings
{
    "runtime" : {
        "day_of_week" : {
            "type" : "keyword",
            "script" : {
                "source" : "emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"
            }
        }
    },
    "properties" : {
        "timestamp" : {
            "type" : "date"
        }
    }
}
```

Fields defined in the runtime section can be updated at any time as they are not present in the lucene index. They get replaced entirely when they get updated.

Thanks to the introduction of the runtime section, runtime fields override existing mapped fields defined with the same name, similarly to runtime fields defined in the search request.

Relates to #59332

Author: javanna

server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java

@@ -155,6 +155,8 @@ private static Map<String, FieldMappingMetadata> findFieldMappings(Predicate<Str
         if (documentMapper == null) {
             return Collections.emptyMap();
         }
+        //TODO the logic here needs to be reworked to also include runtime fields. Though matching is against mappers rather
+        // than field types, and runtime fields are mixed with ordinary fields in FieldTypeLookup
         Map<String, FieldMappingMetadata> fieldMappings = new HashMap<>();
         final MappingLookup mappingLookup = documentMapper.mappers();
         for (String field : request.fields()) {

server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java

@@ -278,15 +278,7 @@ public final FieldMapper merge(Mapper mergeWith) {
         Conflicts conflicts = new Conflicts(name());
         builder.merge((FieldMapper) mergeWith, conflicts);
         conflicts.check();
-        return builder.build(parentPath(name()));
-    }
-
-    private static ContentPath parentPath(String name) {
-        int endPos = name.lastIndexOf(".");
-        if (endPos == -1) {
-            return new ContentPath(0);
-        }
-        return new ContentPath(name.substring(0, endPos));
+        return builder.build(Builder.parentPath(name()));
     }
 
     protected void checkIncomingMergeType(FieldMapper mergeWith) {
@@ -482,7 +474,7 @@ public List<String> copyToFields() {
     /**
      * Serializes a parameter
      */
-    protected interface Serializer<T> {
+    public interface Serializer<T> {
         void serialize(XContentBuilder builder, String name, T value) throws IOException;
     }
 
@@ -931,7 +923,7 @@ protected String buildFullName(ContentPath contentPath) {
         /**
          * Writes the current builder parameter values as XContent
          */
-        protected final void toXContent(XContentBuilder builder, boolean includeDefaults) throws IOException {
+        public final void toXContent(XContentBuilder builder, boolean includeDefaults) throws IOException {
             for (Parameter<?> parameter : getParameters()) {
                 parameter.toXContent(builder, includeDefaults);
             }
@@ -1011,6 +1003,14 @@ public final void parse(String name, ParserContext parserContext, Map<String, Ob
             validate();
         }
 
+        protected static ContentPath parentPath(String name) {
+            int endPos = name.lastIndexOf(".");
+            if (endPos == -1) {
+                return new ContentPath(0);
+            }
+            return new ContentPath(name.substring(0, endPos));
+        }
+
         // These parameters were previously *always* parsed by TypeParsers#parseField(), even if they
         // made no sense; if we've got here, that means that they're not declared on a current mapper,
         // and so we emit a deprecation warning rather than failing a previously working mapping.

server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java

@@ -33,7 +33,6 @@
  * An immutable container for looking up {@link MappedFieldType}s by their name.
  */
 final class FieldTypeLookup {
-
     private final Map<String, MappedFieldType> fullNameToFieldType = new HashMap<>();
 
     /**
@@ -47,7 +46,8 @@ final class FieldTypeLookup {
     private final DynamicKeyFieldTypeLookup dynamicKeyLookup;
 
     FieldTypeLookup(Collection<FieldMapper> fieldMappers,
-                    Collection<FieldAliasMapper> fieldAliasMappers) {
+                    Collection<FieldAliasMapper> fieldAliasMappers,
+                    Collection<RuntimeFieldType> runtimeFieldTypes) {
         Map<String, DynamicKeyFieldMapper> dynamicKeyMappers = new HashMap<>();
 
         for (FieldMapper fieldMapper : fieldMappers) {
@@ -77,6 +77,11 @@ final class FieldTypeLookup {
             fullNameToFieldType.put(aliasName, fullNameToFieldType.get(path));
         }
 
+        for (RuntimeFieldType runtimeFieldType : runtimeFieldTypes) {
+            //this will override concrete fields with runtime fields that have the same name
+            fullNameToFieldType.put(runtimeFieldType.name(), runtimeFieldType);
+        }
+
         this.dynamicKeyLookup = new DynamicKeyFieldTypeLookup(dynamicKeyMappers, aliasToConcreteName);
     }
 

server/src/main/java/org/elasticsearch/index/mapper/Mapper.java

@@ -59,6 +59,7 @@ class ParserContext {
 
             private final Function<String, SimilarityProvider> similarityLookupService;
             private final Function<String, TypeParser> typeParsers;
+            private final Function<String, RuntimeFieldType.Parser> runtimeTypeParsers;
             private final Version indexVersionCreated;
             private final Supplier<QueryShardContext> queryShardContextSupplier;
             private final DateFormatter dateFormatter;
@@ -69,6 +70,7 @@ class ParserContext {
 
             public ParserContext(Function<String, SimilarityProvider> similarityLookupService,
                                  Function<String, TypeParser> typeParsers,
+                                 Function<String, RuntimeFieldType.Parser> runtimeTypeParsers,
                                  Version indexVersionCreated,
                                  Supplier<QueryShardContext> queryShardContextSupplier,
                                  DateFormatter dateFormatter,
@@ -78,6 +80,7 @@ public ParserContext(Function<String, SimilarityProvider> similarityLookupServic
                                  BooleanSupplier idFieldDataEnabled) {
                 this.similarityLookupService = similarityLookupService;
                 this.typeParsers = typeParsers;
+                this.runtimeTypeParsers = runtimeTypeParsers;
                 this.indexVersionCreated = indexVersionCreated;
                 this.queryShardContextSupplier = queryShardContextSupplier;
                 this.dateFormatter = dateFormatter;
@@ -132,6 +135,8 @@ public DateFormatter getDateFormatter() {
 
             protected Function<String, TypeParser> typeParsers() { return typeParsers; }
 
+            protected Function<String, RuntimeFieldType.Parser> runtimeTypeParsers() { return runtimeTypeParsers; }
+
             protected Function<String, SimilarityProvider> similarityLookupService() { return similarityLookupService; }
 
             /**
@@ -147,8 +152,9 @@ public ParserContext createMultiFieldContext(ParserContext in) {
 
             static class MultiFieldParserContext extends ParserContext {
                 MultiFieldParserContext(ParserContext in) {
-                    super(in.similarityLookupService, in.typeParsers, in.indexVersionCreated, in.queryShardContextSupplier,
-                        in.dateFormatter, in.scriptService, in.indexAnalyzers, in.indexSettings, in.idFieldDataEnabled);
+                    super(in.similarityLookupService, in.typeParsers, in.runtimeTypeParsers, in.indexVersionCreated,
+                        in.queryShardContextSupplier, in.dateFormatter, in.scriptService, in.indexAnalyzers, in.indexSettings,
+                        in.idFieldDataEnabled);
                 }
 
                 @Override

server/src/main/java/org/elasticsearch/index/mapper/MapperService.java

@@ -125,8 +125,8 @@ public MapperService(IndexSettings indexSettings, IndexAnalyzers indexAnalyzers,
         this.mapperRegistry = mapperRegistry;
         Function<DateFormatter, Mapper.TypeParser.ParserContext> parserContextFunction =
             dateFormatter -> new Mapper.TypeParser.ParserContext(similarityService::getSimilarity, mapperRegistry.getMapperParsers()::get,
-                indexVersionCreated, queryShardContextSupplier, dateFormatter, scriptService, indexAnalyzers, indexSettings,
-                idFieldDataEnabled);
+                mapperRegistry.getRuntimeFieldTypeParsers()::get, indexVersionCreated, queryShardContextSupplier, dateFormatter,
+                scriptService, indexAnalyzers, indexSettings, idFieldDataEnabled);
         this.documentParser = new DocumentParser(xContentRegistry, parserContextFunction);
         Map<String, MetadataFieldMapper.TypeParser> metadataMapperParsers =
             mapperRegistry.getMetadataMapperParsers(indexSettings.getIndexVersionCreated());

server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java

@@ -32,7 +32,6 @@
 import java.util.stream.Stream;
 
 public final class MappingLookup {
-
     /** Full field name to mapper */
     private final Map<String, Mapper> fieldMappers;
     private final Map<String, ObjectMapper> objectMappers;
@@ -50,24 +49,24 @@ public static MappingLookup fromMapping(Mapping mapping) {
                 newFieldMappers.add(metadataMapper);
             }
         }
-        collect(mapping.root, newObjectMappers, newFieldMappers, newFieldAliasMappers);
-        return new MappingLookup(newFieldMappers, newObjectMappers, newFieldAliasMappers, mapping.metadataMappers.length);
+        for (Mapper child : mapping.root) {
+            collect(child, newObjectMappers, newFieldMappers, newFieldAliasMappers);
+        }
+        return new MappingLookup(newFieldMappers, newObjectMappers, newFieldAliasMappers,
+            mapping.root.runtimeFieldTypes(), mapping.metadataMappers.length);
     }
 
     private static void collect(Mapper mapper, Collection<ObjectMapper> objectMappers,
                                Collection<FieldMapper> fieldMappers,
                                Collection<FieldAliasMapper> fieldAliasMappers) {
-        if (mapper instanceof RootObjectMapper) {
-            // root mapper isn't really an object mapper
-        } else if (mapper instanceof ObjectMapper) {
+        if (mapper instanceof ObjectMapper) {
             objectMappers.add((ObjectMapper)mapper);
         } else if (mapper instanceof FieldMapper) {
             fieldMappers.add((FieldMapper)mapper);
         } else if (mapper instanceof FieldAliasMapper) {
             fieldAliasMappers.add((FieldAliasMapper) mapper);
         } else {
-            throw new IllegalStateException("Unrecognized mapper type [" +
-                mapper.getClass().getSimpleName() + "].");
+            throw new IllegalStateException("Unrecognized mapper type [" + mapper.getClass().getSimpleName() + "].");
         }
 
         for (Mapper child : mapper) {
@@ -78,6 +77,7 @@ private static void collect(Mapper mapper, Collection<ObjectMapper> objectMapper
     public MappingLookup(Collection<FieldMapper> mappers,
                          Collection<ObjectMapper> objectMappers,
                          Collection<FieldAliasMapper> aliasMappers,
+                         Collection<RuntimeFieldType> runtimeFieldTypes,
                          int metadataFieldCount) {
         Map<String, Mapper> fieldMappers = new HashMap<>();
         Map<String, Analyzer> indexAnalyzers = new HashMap<>();
@@ -114,7 +114,7 @@ public MappingLookup(Collection<FieldMapper> mappers,
             }
         }
 
-        this.fieldTypeLookup = new FieldTypeLookup(mappers, aliasMappers);
+        this.fieldTypeLookup = new FieldTypeLookup(mappers, aliasMappers, runtimeFieldTypes);
 
         this.fieldMappers = Collections.unmodifiableMap(fieldMappers);
         this.indexAnalyzer = new FieldNameAnalyzer(indexAnalyzers);

server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java

@@ -19,6 +19,7 @@
 
 package org.elasticsearch.index.mapper;
 
+import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.Explicit;
 import org.elasticsearch.common.Strings;
@@ -34,11 +35,14 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
 import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.stream.Collectors;
 
 import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeBooleanValue;
 import static org.elasticsearch.index.mapper.TypeParsers.parseDateTimeFormatter;
@@ -62,6 +66,7 @@ public static class Builder extends ObjectMapper.Builder {
         protected Explicit<DateFormatter[]> dynamicDateTimeFormatters = new Explicit<>(Defaults.DYNAMIC_DATE_TIME_FORMATTERS, false);
         protected Explicit<Boolean> dateDetection = new Explicit<>(Defaults.DATE_DETECTION, false);
         protected Explicit<Boolean> numericDetection = new Explicit<>(Defaults.NUMERIC_DETECTION, false);
+        protected final Map<String, RuntimeFieldType> runtimeFieldTypes = new HashMap<>();
 
         public Builder(String name, Version indexCreatedVersion) {
             super(name, indexCreatedVersion);
@@ -83,6 +88,11 @@ public RootObjectMapper.Builder add(Mapper.Builder builder) {
             return this;
         }
 
+        public RootObjectMapper.Builder addRuntime(RuntimeFieldType runtimeFieldType) {
+            this.runtimeFieldTypes.put(runtimeFieldType.name(), runtimeFieldType);
+            return this;
+        }
+
         @Override
         public RootObjectMapper build(ContentPath contentPath) {
             return (RootObjectMapper) super.build(contentPath);
@@ -92,7 +102,7 @@ public RootObjectMapper build(ContentPath contentPath) {
         protected ObjectMapper createMapper(String name, String fullPath, Explicit<Boolean> enabled, Nested nested, Dynamic dynamic,
                 Map<String, Mapper> mappers, Version indexCreatedVersion) {
             assert !nested.isNested();
-            return new RootObjectMapper(name, enabled, dynamic, mappers,
+            return new RootObjectMapper(name, enabled, dynamic, mappers, runtimeFieldTypes,
                     dynamicDateTimeFormatters,
                     dynamicTemplates,
                     dateDetection, numericDetection, indexCreatedVersion);
@@ -127,7 +137,7 @@ private static void fixRedundantIncludes(ObjectMapper objectMapper, boolean pare
         }
     }
 
-    public static class TypeParser extends ObjectMapper.TypeParser {
+    static final class TypeParser extends ObjectMapper.TypeParser {
 
         @Override
         public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
@@ -146,8 +156,7 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
         }
 
         @SuppressWarnings("unchecked")
-        protected boolean processField(RootObjectMapper.Builder builder, String fieldName, Object fieldNode,
-                                       ParserContext parserContext) {
+        private boolean processField(RootObjectMapper.Builder builder, String fieldName, Object fieldNode, ParserContext parserContext) {
             if (fieldName.equals("date_formats") || fieldName.equals("dynamic_date_formats")) {
                 if (fieldNode instanceof List) {
                     List<DateFormatter> formatters = new ArrayList<>();
@@ -190,10 +199,8 @@ protected boolean processField(RootObjectMapper.Builder builder, String fieldNam
                     String templateName = entry.getKey();
                     Map<String, Object> templateParams = (Map<String, Object>) entry.getValue();
                     DynamicTemplate template = DynamicTemplate.parse(templateName, templateParams);
-                    if (template != null) {
-                        validateDynamicTemplate(parserContext, template);
-                        templates.add(template);
-                    }
+                    validateDynamicTemplate(parserContext, template);
+                    templates.add(template);
                 }
                 builder.dynamicTemplates(templates);
                 return true;
@@ -203,6 +210,13 @@ protected boolean processField(RootObjectMapper.Builder builder, String fieldNam
             } else if (fieldName.equals("numeric_detection")) {
                 builder.numericDetection = new Explicit<>(nodeBooleanValue(fieldNode, "numeric_detection"), true);
                 return true;
+            } else if (fieldName.equals("runtime")) {
+                if (fieldNode instanceof Map) {
+                    RuntimeFieldType.parseRuntimeFields((Map<String, Object>) fieldNode, parserContext, builder::addRuntime);
+                    return true;
+                } else {
+                    throw new ElasticsearchParseException("runtime must be a map type");
+                }
             }
             return false;
         }
@@ -212,11 +226,14 @@ protected boolean processField(RootObjectMapper.Builder builder, String fieldNam
     private Explicit<Boolean> dateDetection;
     private Explicit<Boolean> numericDetection;
     private Explicit<DynamicTemplate[]> dynamicTemplates;
+    private final Map<String, RuntimeFieldType> runtimeFieldTypes;
 
     RootObjectMapper(String name, Explicit<Boolean> enabled, Dynamic dynamic, Map<String, Mapper> mappers,
+                     Map<String, RuntimeFieldType> runtimeFieldTypes,
                      Explicit<DateFormatter[]> dynamicDateTimeFormatters, Explicit<DynamicTemplate[]> dynamicTemplates,
                      Explicit<Boolean> dateDetection, Explicit<Boolean> numericDetection, Version indexCreatedVersion) {
         super(name, name, enabled, Nested.NO, dynamic, mappers, indexCreatedVersion);
+        this.runtimeFieldTypes = runtimeFieldTypes;
         this.dynamicTemplates = dynamicTemplates;
         this.dynamicDateTimeFormatters = dynamicDateTimeFormatters;
         this.dateDetection = dateDetection;
@@ -236,23 +253,26 @@ public ObjectMapper mappingUpdate(Mapper mapper) {
         return update;
     }
 
-    public boolean dateDetection() {
+    boolean dateDetection() {
         return this.dateDetection.value();
     }
 
-    public boolean numericDetection() {
+    boolean numericDetection() {
         return this.numericDetection.value();
     }
 
-    public DateFormatter[] dynamicDateTimeFormatters() {
+    DateFormatter[] dynamicDateTimeFormatters() {
         return dynamicDateTimeFormatters.value();
     }
 
-    public DynamicTemplate[] dynamicTemplates() {
+    DynamicTemplate[] dynamicTemplates() {
         return dynamicTemplates.value();
     }
 
-    @SuppressWarnings("rawtypes")
+    Collection<RuntimeFieldType> runtimeFieldTypes() {
+        return runtimeFieldTypes.values();
+    }
+
     public Mapper.Builder findTemplateBuilder(ParseContext context, String name, XContentFieldType matchType) {
         return findTemplateBuilder(context, name, matchType, null);
     }
@@ -268,7 +288,6 @@ public Mapper.Builder findTemplateBuilder(ParseContext context, String name, Dat
      * @param dateFormat  a dateformatter to use if the type is a date, null if not a date or is using the default format
      * @return a mapper builder, or null if there is no template for such a field
      */
-    @SuppressWarnings("rawtypes")
     private Mapper.Builder findTemplateBuilder(ParseContext context, String name, XContentFieldType matchType, DateFormatter dateFormat) {
         DynamicTemplate dynamicTemplate = findTemplate(context.path(), name, matchType);
         if (dynamicTemplate == null) {
@@ -331,6 +350,8 @@ protected void doMerge(ObjectMapper mergeWith, MergeReason reason) {
                 this.dynamicTemplates = mergeWithObject.dynamicTemplates;
             }
         }
+
+        this.runtimeFieldTypes.putAll(mergeWithObject.runtimeFieldTypes);
     }
 
     @Override
@@ -361,6 +382,16 @@ protected void doXContent(XContentBuilder builder, ToXContent.Params params) thr
         if (numericDetection.explicit() || includeDefaults) {
             builder.field("numeric_detection", numericDetection.value());
         }
+
+        if (runtimeFieldTypes.size() > 0) {
+            builder.startObject("runtime");
+            List<RuntimeFieldType> sortedRuntimeFieldTypes = runtimeFieldTypes.values().stream().sorted(
+                Comparator.comparing(RuntimeFieldType::name)).collect(Collectors.toList());
+            for (RuntimeFieldType fieldType : sortedRuntimeFieldTypes) {
+                fieldType.toXContent(builder, params);
+            }
+            builder.endObject();
+        }
     }
 
     private static void validateDynamicTemplate(Mapper.TypeParser.ParserContext parserContext,