Nest pass-through objects within objects

docs/changelog/105062.yaml

@@ -0,0 +1,5 @@
+pr: 105062
+summary: Nest pass-through objects within objects
+area: TSDB
+type: enhancement
+issues: []

modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml

@@ -381,12 +381,12 @@ dynamic templates - conflicting aliases:
   - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 }
 
 ---
-dynamic templates - subobject in passthrough object error:
+dynamic templates with nesting:
   - skip:
       version: " - 8.12.99"
+      features: "default_shards"
       reason: "Support for dynamic fields was added in 8.13"
   - do:
-      catch: /Tried to add subobject \[subcategory\] to object \[attributes\] which does not support subobjects/
       indices.put_index_template:
         name: my-dynamic-template
         body:
@@ -395,21 +395,116 @@ dynamic templates - subobject in passthrough object error:
           template:
             settings:
               index:
+                number_of_shards: 1
                 mode: time_series
+                time_series:
+                  start_time: 2023-08-31T13:03:08.138Z
 
             mappings:
               properties:
                 attributes:
                   type: passthrough
+                  dynamic: true
+                  time_series_dimension: true
+                resource:
+                  type: object
                   properties:
-                    subcategory:
-                      type: object
-                      properties:
-                        dim:
-                          type: keyword
+                    attributes:
+                      type: passthrough
+                      dynamic: true
+                      time_series_dimension: true
+              dynamic_templates:
+                - counter_metric:
+                    mapping:
+                      type: integer
+                      time_series_metric: counter
 
   - do:
-      catch: /Mapping definition for \[attributes\] has unsupported parameters:\  \[subobjects \:\ true\]/
+      bulk:
+        index: k9s
+        refresh: true
+        body:
+          - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
+          - '{ "@timestamp": "2023-09-01T13:03:08.138Z","data": "10", "resource.attributes.dim": "A", "resource.attributes.another.dim": "C", "attributes.more.dim": "E" }'
+          - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
+          - '{ "@timestamp": "2023-09-01T13:03:09.138Z","data": "20", "resource.attributes.dim": "A", "resource.attributes.another.dim": "C", "attributes.more.dim": "E" }'
+          - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
+          - '{ "@timestamp": "2023-09-01T13:03:10.138Z","data": "30", "resource.attributes.dim": "B", "resource.attributes.another.dim": "D", "attributes.more.dim": "F" }'
+          - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
+          - '{ "@timestamp": "2023-09-01T13:03:10.238Z","data": "40", "resource.attributes.dim": "B", "resource.attributes.another.dim": "D", "attributes.more.dim": "F" }'
+
+  - do:
+      search:
+        index: k9s
+        body:
+          size: 0
+
+  - match: { hits.total.value: 4 }
+
+  - do:
+      search:
+        index: k9s
+        body:
+          size: 0
+          aggs:
+            filterA:
+              filter:
+                term:
+                  dim: A
+              aggs:
+                tsids:
+                  terms:
+                    field: _tsid
+
+  - length: { aggregations.filterA.tsids.buckets: 1 }
+  - match: { aggregations.filterA.tsids.buckets.0.key: "LEjiJ4ATCXWlzeFvhGQ9lYlnP-nRIGKYihfZ18WoJ94t9a8OpbsCdwZALomb" }
+  - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 }
+
+  - do:
+      search:
+        index: k9s
+        body:
+          size: 0
+          aggs:
+            filterA:
+              filter:
+                term:
+                  another.dim: C
+              aggs:
+                tsids:
+                  terms:
+                    field: _tsid
+
+  - length: { aggregations.filterA.tsids.buckets: 1 }
+  - match: { aggregations.filterA.tsids.buckets.0.key: "LEjiJ4ATCXWlzeFvhGQ9lYlnP-nRIGKYihfZ18WoJ94t9a8OpbsCdwZALomb" }
+  - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 }
+
+  - do:
+      search:
+        index: k9s
+        body:
+          size: 0
+          aggs:
+            filterA:
+              filter:
+                term:
+                  more.dim: E
+              aggs:
+                tsids:
+                  terms:
+                    field: _tsid
+
+  - length: { aggregations.filterA.tsids.buckets: 1 }
+  - match: { aggregations.filterA.tsids.buckets.0.key: "LEjiJ4ATCXWlzeFvhGQ9lYlnP-nRIGKYihfZ18WoJ94t9a8OpbsCdwZALomb" }
+  - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 }
+
+---
+dynamic templates - subobject in passthrough object error:
+  - skip:
+      version: " - 8.12.99"
+      reason: "Support for dynamic fields was added in 8.13"
+  - do:
+      catch: /Tried to add subobject \[subcategory\] to object \[attributes\] which does not support subobjects/
       indices.put_index_template:
         name: my-dynamic-template
         body:
@@ -418,24 +513,21 @@ dynamic templates - subobject in passthrough object error:
           template:
             settings:
               index:
-                number_of_shards: 1
                 mode: time_series
-                time_series:
-                  start_time: 2023-08-31T13:03:08.138Z
 
             mappings:
               properties:
                 attributes:
                   type: passthrough
-                  subobjects: true
+                  properties:
+                    subcategory:
+                      type: object
+                      properties:
+                        dim:
+                          type: keyword
 
----
-dynamic templates - passthrough not under root error:
-  - skip:
-      version: " - 8.12.99"
-      reason: "Support for dynamic fields was added in 8.13"
   - do:
-      catch: /Tried to add passthrough subobject \[attributes\] to object \[resource\], passthrough is not supported as a subobject/
+      catch: /Mapping definition for \[attributes\] has unsupported parameters:\  \[subobjects \:\ true\]/
       indices.put_index_template:
         name: my-dynamic-template
         body:
@@ -444,11 +536,13 @@ dynamic templates - passthrough not under root error:
           template:
             settings:
               index:
+                number_of_shards: 1
                 mode: time_series
+                time_series:
+                  start_time: 2023-08-31T13:03:08.138Z
 
             mappings:
               properties:
-                "resource.attributes":
+                attributes:
                   type: passthrough
-                  dynamic: true
-                  time_series_dimension: true
+                  subobjects: true

server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java

@@ -295,7 +295,10 @@ protected static void parseProperties(
                         }
                     }
 
-                    if (objBuilder.subobjects.value() == false && type.equals(ObjectMapper.CONTENT_TYPE)) {
+                    if (objBuilder.subobjects.value() == false
+                        && (type.equals(ObjectMapper.CONTENT_TYPE)
+                            || type.equals(NestedObjectMapper.CONTENT_TYPE)
+                            || type.equals(PassThroughObjectMapper.CONTENT_TYPE))) {
                         throw new MapperParsingException(
                             "Tried to add subobject ["
                                 + fieldName
@@ -304,24 +307,6 @@ protected static void parseProperties(
                                 + "] which does not support subobjects"
                         );
                     }
-                    if (objBuilder.subobjects.value() == false && type.equals(NestedObjectMapper.CONTENT_TYPE)) {
-                        throw new MapperParsingException(
-                            "Tried to add nested object ["
-                                + fieldName
-                                + "] to object ["
-                                + objBuilder.name()
-                                + "] which does not support subobjects"
-                        );
-                    }
-                    if (type.equals(PassThroughObjectMapper.CONTENT_TYPE) && objBuilder instanceof RootObjectMapper.Builder == false) {
-                        throw new MapperParsingException(
-                            "Tried to add passthrough subobject ["
-                                + fieldName
-                                + "] to object ["
-                                + objBuilder.name()
-                                + "], passthrough is not supported as a subobject"
-                        );
-                    }
                     Mapper.TypeParser typeParser = parserContext.typeParser(type);
                     if (typeParser == null) {
                         throw new MapperParsingException("No handler for type [" + type + "] declared on field [" + fieldName + "]");

server/src/main/java/org/elasticsearch/index/mapper/PassThroughObjectMapper.java

@@ -57,6 +57,7 @@ public PassThroughObjectMapper.Builder setContainsDimensions() {
         public PassThroughObjectMapper build(MapperBuilderContext context) {
             return new PassThroughObjectMapper(
                 name,
+                context.buildFullName(name),
                 enabled,
                 dynamic,
                 buildMappers(context.createChildContext(name)),
@@ -70,19 +71,20 @@ public PassThroughObjectMapper build(MapperBuilderContext context) {
 
     PassThroughObjectMapper(
         String name,
+        String fullPath,
         Explicit<Boolean> enabled,
         Dynamic dynamic,
         Map<String, Mapper> mappers,
         Explicit<Boolean> timeSeriesDimensionSubFields
     ) {
         // Subobjects are not currently supported.
-        super(name, name, enabled, Explicit.IMPLICIT_FALSE, dynamic, mappers);
+        super(name, fullPath, enabled, Explicit.IMPLICIT_FALSE, dynamic, mappers);
         this.timeSeriesDimensionSubFields = timeSeriesDimensionSubFields;
     }
 
     @Override
     PassThroughObjectMapper withoutMappers() {
-        return new PassThroughObjectMapper(simpleName(), enabled, dynamic, Map.of(), timeSeriesDimensionSubFields);
+        return new PassThroughObjectMapper(simpleName(), fullPath(), enabled, dynamic, Map.of(), timeSeriesDimensionSubFields);
     }
 
     public boolean containsDimensions() {
@@ -91,7 +93,7 @@ public boolean containsDimensions() {
 
     @Override
     public PassThroughObjectMapper.Builder newBuilder(IndexVersion indexVersionCreated) {
-        PassThroughObjectMapper.Builder builder = new PassThroughObjectMapper.Builder(name());
+        PassThroughObjectMapper.Builder builder = new PassThroughObjectMapper.Builder(simpleName());
         builder.enabled = enabled;
         builder.dynamic = dynamic;
         builder.timeSeriesDimensionSubFields = timeSeriesDimensionSubFields;
@@ -108,6 +110,7 @@ public PassThroughObjectMapper merge(ObjectMapper mergeWith, MergeReason reason,
 
         return new PassThroughObjectMapper(
             simpleName(),
+            fullPath(),
             mergeResult.enabled(),
             mergeResult.dynamic(),
             mergeResult.mappers(),

server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java

@@ -45,6 +45,7 @@
 
 public class RootObjectMapper extends ObjectMapper {
     private static final DeprecationLogger DEPRECATION_LOGGER = DeprecationLogger.getLogger(RootObjectMapper.class);
+    private static final int MAX_NESTING_LEVEL_FOR_PASS_THROUGH_OBJECTS = 20;
 
     /**
      * Parameter used when serializing {@link RootObjectMapper} and request that the runtime section is skipped.
@@ -111,7 +112,11 @@ public RootObjectMapper.Builder addRuntimeFields(Map<String, RuntimeField> runti
         @Override
         public RootObjectMapper build(MapperBuilderContext context) {
             Map<String, Mapper> mappers = buildMappers(context);
-            mappers.putAll(getAliasMappers(mappers, context));
+
+            Map<String, Mapper> aliasMappers = new HashMap<>();
+            getAliasMappers(mappers, aliasMappers, context, 0);
+            mappers.putAll(aliasMappers);
+
             return new RootObjectMapper(
                 name,
                 enabled,
@@ -126,8 +131,11 @@ public RootObjectMapper build(MapperBuilderContext context) {
             );
         }
 
-        Map<String, Mapper> getAliasMappers(Map<String, Mapper> mappers, MapperBuilderContext context) {
-            Map<String, Mapper> aliasMappers = new HashMap<>();
+        void getAliasMappers(Map<String, Mapper> mappers, Map<String, Mapper> aliasMappers, MapperBuilderContext context, int level) {
+            if (level >= MAX_NESTING_LEVEL_FOR_PASS_THROUGH_OBJECTS) {
+                logger.warn("Exceeded maximum nesting level for searching for pass-through object fields within object fields.");
+                return;
+            }
             for (Mapper mapper : mappers.values()) {
                 // Create aliases for all fields in child passthrough mappers and place them under the root object.
                 if (mapper instanceof PassThroughObjectMapper passthroughMapper) {
@@ -154,9 +162,11 @@ Map<String, Mapper> getAliasMappers(Map<String, Mapper> mappers, MapperBuilderCo
                             }
                         }
                     }
+                } else if (mapper instanceof ObjectMapper objectMapper) {
+                    // Call recursively to check child fields. The level guards against long recursive call sequences.
+                    getAliasMappers(objectMapper.mappers, aliasMappers, context, level + 1);
                 }
             }
-            return aliasMappers;
         }
     }
 

server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java

@@ -1391,7 +1391,7 @@ public void testSubobjectsFalseWithInnerNestedFromDynamicTemplate() {
         );
         assertThat(exception.getRootCause(), instanceOf(MapperParsingException.class));
         assertEquals(
-            "Tried to add nested object [time] to object [__dynamic__test] which does not support subobjects",
+            "Tried to add subobject [time] to object [__dynamic__test] which does not support subobjects",
             exception.getRootCause().getMessage()
         );
     }

server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java

@@ -407,7 +407,7 @@ public void testSubobjectsFalseWithInnerNested() {
             b.endObject();
         })));
         assertEquals(
-            "Failed to parse mapping: Tried to add nested object [time] to object [service] which does not support subobjects",
+            "Failed to parse mapping: Tried to add subobject [time] to object [service] which does not support subobjects",
             exception.getMessage()
         );
     }
@@ -457,7 +457,7 @@ public void testSubobjectsFalseRootWithInnerNested() {
             b.endObject();
         })));
         assertEquals(
-            "Failed to parse mapping: Tried to add nested object [metrics.service] to object [_doc] which does not support subobjects",
+            "Failed to parse mapping: Tried to add subobject [metrics.service] to object [_doc] which does not support subobjects",
             exception.getMessage()
         );
     }