Add ignore_missing_component_templates config option

docs/changelog/92436.yaml

@@ -0,0 +1,5 @@
+pr: 92436
+summary: Add `ignore_missing_component_templates` config option
+area: "Data streams, Indices APIs"
+type: enhancement
+issues: []

docs/reference/indices/ignore-missing-component-templates.asciidoc

@@ -0,0 +1,95 @@
+[[ignore_missing_component_templates]]
+== Config ignore_missing_component_templates
+
+The configuration option `ignore_missing_component_templates` can be used when an index template references a component template that might not exist yet. Every time a data stream is create based on the index template, the existance of the component template will be checked. If it exists, it will taken as part of the mapping. If it does not exist, it is ignored.
+
+=== Usage example
+
+In the following, one component template and an index template are created. The index template references 2 component templates but only the `@package` one exists.
+
+
+Create of the component template `logs-foo@package`. This has to be created before the index template as it is not optional:
+
+[source,console]
+----
+PUT _component_template/logs-foo@package
+{
+  "template": {
+    "mappings": {
+      "properties": {
+        "host.name": {
+          "type": "keyword"
+        }
+      }
+    }
+  }
+}
+----
+
+The index template is now created and and it references two component templates:
+
+[source,JSON]
+----
+  "composed_of": ["logs-foo@package", "logs-foo@custom"]
+----
+
+Before, only the `logs-foo@package` compontent template was created, meaning the `logs-foo@custom` is missing. Because of this the following entry was added to the config:
+
+[source,JSON]
+----
+  "ignore_missing_component_templates": ["logs-foo@custom"],
+----
+
+During creation of the template, it will not validate that `logs-foo@custom` exists:
+
+
+[source,console]
+----
+PUT _index_template/logs-foo
+{
+  "index_patterns": ["logs-foo-*"],
+  "data_stream": { },
+  "composed_of": ["logs-foo@package", "logs-foo@custom"],
+  "ignore_missing_component_templates": ["logs-foo@custom"],
+  "priority": 500
+}
+----
+// TEST[continued]
+
+The index template `logs-foo` was successfully created. A data stream can be created based on this template:
+
+[source,console]
+----
+PUT _data_stream/logs-foo-bar
+----
+// TEST[continued]
+
+Looking at the mappings of the data stream, it will contain the `host.name` field.
+
+At a later stage, the missing component template might be added:
+
+[source,console]
+----
+PUT _component_template/logs-foo@custom
+{
+  "template": {
+    "mappings": {
+      "properties": {
+        "host.ip": {
+          "type": "ip"
+        }
+      }
+    }
+  }
+}
+----
+// TEST[continued]
+
+This will not have an immediate effect on the data stream. The mapping `host.ip` will only show up in the data stream mappings when the data stream is rolled over automatically next time or a manual rollover is triggered:
+
+[source,console]
+----
+POST logs-foo-bar/_rollover
+----
+// TEST[continued]
+// TEST[teardown:data_stream_cleanup]

docs/reference/indices/index-templates.asciidoc

@@ -161,3 +161,5 @@ DELETE _component_template/component_template1
 ////
 
 include::simulate-multi-component-templates.asciidoc[]
+
+include::ignore-missing-component-templates.asciidoc[]

server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java

@@ -46,6 +46,7 @@ public class ComposableIndexTemplate implements SimpleDiffable<ComposableIndexTe
     private static final ParseField METADATA = new ParseField("_meta");
     private static final ParseField DATA_STREAM = new ParseField("data_stream");
     private static final ParseField ALLOW_AUTO_CREATE = new ParseField("allow_auto_create");
+    private static final ParseField IGNORE_MISSING_COMPONENT_TEMPLATES = new ParseField("ignore_missing_component_templates");
 
     @SuppressWarnings("unchecked")
     public static final ConstructingObjectParser<ComposableIndexTemplate, Void> PARSER = new ConstructingObjectParser<>(
@@ -59,7 +60,8 @@ public class ComposableIndexTemplate implements SimpleDiffable<ComposableIndexTe
             (Long) a[4],
             (Map<String, Object>) a[5],
             (DataStreamTemplate) a[6],
-            (Boolean) a[7]
+            (Boolean) a[7],
+            (List<String>) a[8]
         )
     );
 
@@ -72,6 +74,7 @@ public class ComposableIndexTemplate implements SimpleDiffable<ComposableIndexTe
         PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> p.map(), METADATA);
         PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), DataStreamTemplate.PARSER, DATA_STREAM);
         PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), ALLOW_AUTO_CREATE);
+        PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), IGNORE_MISSING_COMPONENT_TEMPLATES);
     }
 
     private final List<String> indexPatterns;
@@ -89,6 +92,8 @@ public class ComposableIndexTemplate implements SimpleDiffable<ComposableIndexTe
     private final DataStreamTemplate dataStreamTemplate;
     @Nullable
     private final Boolean allowAutoCreate;
+    @Nullable
+    private final List<String> ignoreMissingComponentTemplates;
 
     static Diff<ComposableIndexTemplate> readITV2DiffFrom(StreamInput in) throws IOException {
         return SimpleDiffable.readDiffFrom(ComposableIndexTemplate::new, in);
@@ -106,7 +111,7 @@ public ComposableIndexTemplate(
         @Nullable Long version,
         @Nullable Map<String, Object> metadata
     ) {
-        this(indexPatterns, template, componentTemplates, priority, version, metadata, null, null);
+        this(indexPatterns, template, componentTemplates, priority, version, metadata, null, null, null);
     }
 
     public ComposableIndexTemplate(
@@ -118,7 +123,7 @@ public ComposableIndexTemplate(
         @Nullable Map<String, Object> metadata,
         @Nullable DataStreamTemplate dataStreamTemplate
     ) {
-        this(indexPatterns, template, componentTemplates, priority, version, metadata, dataStreamTemplate, null);
+        this(indexPatterns, template, componentTemplates, priority, version, metadata, dataStreamTemplate, null, null);
     }
 
     public ComposableIndexTemplate(
@@ -130,6 +135,20 @@ public ComposableIndexTemplate(
         @Nullable Map<String, Object> metadata,
         @Nullable DataStreamTemplate dataStreamTemplate,
         @Nullable Boolean allowAutoCreate
+    ) {
+        this(indexPatterns, template, componentTemplates, priority, version, metadata, dataStreamTemplate, null, null);
+    }
+
+    public ComposableIndexTemplate(
+        List<String> indexPatterns,
+        @Nullable Template template,
+        @Nullable List<String> componentTemplates,
+        @Nullable Long priority,
+        @Nullable Long version,
+        @Nullable Map<String, Object> metadata,
+        @Nullable DataStreamTemplate dataStreamTemplate,
+        @Nullable Boolean allowAutoCreate,
+        @Nullable List<String> ignoreMissingComponentTemplates
     ) {
         this.indexPatterns = indexPatterns;
         this.template = template;
@@ -139,6 +158,7 @@ public ComposableIndexTemplate(
         this.metadata = metadata;
         this.dataStreamTemplate = dataStreamTemplate;
         this.allowAutoCreate = allowAutoCreate;
+        this.ignoreMissingComponentTemplates = ignoreMissingComponentTemplates;
     }
 
     public ComposableIndexTemplate(StreamInput in) throws IOException {
@@ -154,6 +174,7 @@ public ComposableIndexTemplate(StreamInput in) throws IOException {
         this.metadata = in.readMap();
         this.dataStreamTemplate = in.readOptionalWriteable(DataStreamTemplate::new);
         this.allowAutoCreate = in.readOptionalBoolean();
+        this.ignoreMissingComponentTemplates = in.readOptionalStringList();
     }
 
     public List<String> indexPatterns() {
@@ -204,6 +225,11 @@ public Boolean getAllowAutoCreate() {
         return this.allowAutoCreate;
     }
 
+    @Nullable
+    public List<String> getIgnoreMissingComponentTemplates() {
+        return ignoreMissingComponentTemplates;
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         out.writeStringCollection(this.indexPatterns);
@@ -219,6 +245,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeGenericMap(this.metadata);
         out.writeOptionalWriteable(dataStreamTemplate);
         out.writeOptionalBoolean(allowAutoCreate);
+        out.writeOptionalStringCollection(ignoreMissingComponentTemplates);
     }
 
     @Override
@@ -246,6 +273,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (this.allowAutoCreate != null) {
             builder.field(ALLOW_AUTO_CREATE.getPreferredName(), allowAutoCreate);
         }
+        if (this.ignoreMissingComponentTemplates != null) {
+            builder.stringListField(IGNORE_MISSING_COMPONENT_TEMPLATES.getPreferredName(), ignoreMissingComponentTemplates);
+        }
         builder.endObject();
         return builder;
     }
@@ -260,7 +290,8 @@ public int hashCode() {
             this.version,
             this.metadata,
             this.dataStreamTemplate,
-            this.allowAutoCreate
+            this.allowAutoCreate,
+            this.ignoreMissingComponentTemplates
         );
     }
 
@@ -280,7 +311,9 @@ && componentTemplatesEquals(this.componentTemplates, other.componentTemplates)
             && Objects.equals(this.version, other.version)
             && Objects.equals(this.metadata, other.metadata)
             && Objects.equals(this.dataStreamTemplate, other.dataStreamTemplate)
-            && Objects.equals(this.allowAutoCreate, other.allowAutoCreate);
+            && Objects.equals(this.allowAutoCreate, other.allowAutoCreate)
+            // TODO: componentTemplates list has a more complex comparison. Is this needed here too?
+            && Objects.equals(this.ignoreMissingComponentTemplates, other.ignoreMissingComponentTemplates);
     }
 
     static boolean componentTemplatesEquals(List<String> c1, List<String> c2) {
@@ -420,6 +453,7 @@ public static class Builder {
         private Map<String, Object> metadata;
         private DataStreamTemplate dataStreamTemplate;
         private Boolean allowAutoCreate;
+        private List<String> ignoreMissingComponentTemplates;
 
         public Builder() {}
 
@@ -463,6 +497,11 @@ public Builder allowAutoCreate(Boolean allowAutoCreate) {
             return this;
         }
 
+        public Builder ignoreMissingComponentTemplates(List<String> ignoreMissingComponentTemplates) {
+            this.ignoreMissingComponentTemplates = ignoreMissingComponentTemplates;
+            return this;
+        }
+
         public ComposableIndexTemplate build() {
             return new ComposableIndexTemplate(
                 this.indexPatterns,
@@ -472,7 +511,8 @@ public ComposableIndexTemplate build() {
                 this.version,
                 this.metadata,
                 this.dataStreamTemplate,
-                this.allowAutoCreate
+                this.allowAutoCreate,
+                this.ignoreMissingComponentTemplates
             );
         }
     }

server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java

@@ -510,7 +510,27 @@ public static void validateV2TemplateRequest(Metadata metadata, String name, Com
             .filter(componentTemplate -> componentTemplates.containsKey(componentTemplate) == false)
             .toList();
 
-        if (missingComponentTemplates.size() > 0) {
+        // TODO: Should the code be split up in two parts for better error messages? If ignore_missing_component_templates is not set
+        // it throws a different error?
+        final List<String> ignoreMissingComponentTemplates = template.getIgnoreMissingComponentTemplates();
+
+        // Check
+        if (ignoreMissingComponentTemplates != null && ignoreMissingComponentTemplates.size() > 0) {
+            for (String missingComponentTemplate : missingComponentTemplates) {
+                // TODO if ignoreMissingComponentTemplates not set, jump directly out
+                if (ignoreMissingComponentTemplates.contains(missingComponentTemplate) == false) {
+                    throw new InvalidIndexTemplateException(
+                        name,
+                        // TODO: should we make the code more complex and only error out in the end to have the full list reported?
+                        "index template ["
+                            + name
+                            + "] specifies a missing component templates ["
+                            + missingComponentTemplate
+                            + "] that does not exist and is not part of 'ignore_missing_component_templates'"
+                    );
+                }
+            }
+        } else if (missingComponentTemplates.size() > 0) {
             throw new InvalidIndexTemplateException(
                 name,
                 "index template [" + name + "] specifies component templates " + missingComponentTemplates + " that do not exist"
@@ -579,7 +599,8 @@ public ClusterState addIndexTemplateV2(
                 template.version(),
                 template.metadata(),
                 template.getDataStreamTemplate(),
-                template.getAllowAutoCreate()
+                template.getAllowAutoCreate(),
+                template.getIgnoreMissingComponentTemplates()
             );
         }
 
@@ -679,7 +700,8 @@ private void validateIndexTemplateV2(String name, ComposableIndexTemplate indexT
             indexTemplate.version(),
             indexTemplate.metadata(),
             indexTemplate.getDataStreamTemplate(),
-            indexTemplate.getAllowAutoCreate()
+            indexTemplate.getAllowAutoCreate(),
+            indexTemplate.getIgnoreMissingComponentTemplates()
         );
 
         validate(name, templateToValidate);

server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateTests.java

@@ -89,7 +89,9 @@ public static ComposableIndexTemplate randomInstance() {
             randomBoolean() ? null : randomNonNegativeLong(),
             meta,
             dataStreamTemplate,
-            randomBoolean() ? null : randomBoolean()
+            randomBoolean() ? null : randomBoolean(),
+            // TODO: should this align with line 87 for some scenarios?
+            null
         );
     }
 
@@ -183,7 +185,8 @@ public static ComposableIndexTemplate mutateTemplate(ComposableIndexTemplate ori
                     orig.version(),
                     orig.metadata(),
                     orig.getDataStreamTemplate(),
-                    orig.getAllowAutoCreate()
+                    orig.getAllowAutoCreate(),
+                    orig.getIgnoreMissingComponentTemplates()
                 );
             case 2:
                 List<String> newComposedOf = randomValueOtherThan(orig.composedOf(), () -> randomList(0, 10, () -> randomAlphaOfLength(5)));
@@ -195,7 +198,8 @@ public static ComposableIndexTemplate mutateTemplate(ComposableIndexTemplate ori
                     orig.version(),
                     orig.metadata(),
                     orig.getDataStreamTemplate(),
-                    orig.getAllowAutoCreate()
+                    orig.getAllowAutoCreate(),
+                    orig.getIgnoreMissingComponentTemplates()
                 );
             case 3:
                 return new ComposableIndexTemplate(
@@ -206,7 +210,8 @@ public static ComposableIndexTemplate mutateTemplate(ComposableIndexTemplate ori
                     orig.version(),
                     orig.metadata(),
                     orig.getDataStreamTemplate(),
-                    orig.getAllowAutoCreate()
+                    orig.getAllowAutoCreate(),
+                    orig.getIgnoreMissingComponentTemplates()
                 );
             case 4:
                 return new ComposableIndexTemplate(
@@ -217,7 +222,8 @@ public static ComposableIndexTemplate mutateTemplate(ComposableIndexTemplate ori
                     randomValueOtherThan(orig.version(), ESTestCase::randomNonNegativeLong),
                     orig.metadata(),
                     orig.getDataStreamTemplate(),
-                    orig.getAllowAutoCreate()
+                    orig.getAllowAutoCreate(),
+                    orig.getIgnoreMissingComponentTemplates()
                 );
             case 5:
                 return new ComposableIndexTemplate(
@@ -228,7 +234,8 @@ public static ComposableIndexTemplate mutateTemplate(ComposableIndexTemplate ori
                     orig.version(),
                     randomValueOtherThan(orig.metadata(), ComposableIndexTemplateTests::randomMeta),
                     orig.getDataStreamTemplate(),
-                    orig.getAllowAutoCreate()
+                    orig.getAllowAutoCreate(),
+                    orig.getIgnoreMissingComponentTemplates()
                 );
             case 6:
                 return new ComposableIndexTemplate(
@@ -239,7 +246,8 @@ public static ComposableIndexTemplate mutateTemplate(ComposableIndexTemplate ori
                     orig.version(),
                     orig.metadata(),
                     randomValueOtherThan(orig.getDataStreamTemplate(), ComposableIndexTemplateTests::randomDataStreamTemplate),
-                    orig.getAllowAutoCreate()
+                    orig.getAllowAutoCreate(),
+                    orig.getIgnoreMissingComponentTemplates()
                 );
             default:
                 throw new IllegalStateException("illegal randomization branch");