Fix issues with ignore_missing_component_templates

docs/changelog/95527.yaml

@@ -0,0 +1,5 @@
+pr: 95527
+summary: Allow deletion of component templates that are specified in the `ignore_missing_component_templates` array
+area: Data streams
+type: bug
+issues: []

server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java

@@ -198,6 +198,23 @@ public List<String> composedOf() {
         return componentTemplates;
     }
 
+    /**
+     * Returns the <b>required</b> component templates, i.e. such that are not allowed to be missing, as in
+     * {@link #ignoreMissingComponentTemplates}.
+     * @return a list of required component templates
+     */
+    public List<String> getRequiredComponentTemplates() {
+        if (componentTemplates == null) {
+            return List.of();
+        }
+        if (ignoreMissingComponentTemplates == null) {
+            return componentTemplates;
+        }
+        return componentTemplates.stream()
+            .filter(componentTemplate -> ignoreMissingComponentTemplates.contains(componentTemplate) == false)
+            .toList();
+    }
+
     @Nullable
     public Long priority() {
         return priority;

server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java

@@ -383,7 +383,7 @@ public void removeComponentTemplate(
         ClusterState state,
         final ActionListener<AcknowledgedResponse> listener
     ) {
-        validateNotInUse(state.metadata(), names);
+        validateCanBeRemoved(state.metadata(), names);
         taskQueue.submitTask("remove-component-template [" + String.join(",", names) + "]", new TemplateClusterStateUpdateTask(listener) {
             @Override
             public ClusterState execute(ClusterState currentState) {
@@ -394,7 +394,7 @@ public ClusterState execute(ClusterState currentState) {
 
     // Exposed for ReservedComponentTemplateAction
     public static ClusterState innerRemoveComponentTemplate(ClusterState currentState, String... names) {
-        validateNotInUse(currentState.metadata(), names);
+        validateCanBeRemoved(currentState.metadata(), names);
 
         final Set<String> templateNames = new HashSet<>();
         if (names.length > 1) {
@@ -439,10 +439,11 @@ public static ClusterState innerRemoveComponentTemplate(ClusterState currentStat
     }
 
     /**
-     * Validates that the given component template is not used by any index
-     * templates, throwing an error if it is still in use
+     * Validates that the given component template can be removed, throwing an error if it cannot.
+     * A component template should not be removed if it is <b>required</b> by any index templates,
+     * that is- it is used AND NOT specified as {@code ignore_missing_component_templates}.
      */
-    static void validateNotInUse(Metadata metadata, String... templateNameOrWildcard) {
+    static void validateCanBeRemoved(Metadata metadata, String... templateNameOrWildcard) {
         final Predicate<String> predicate;
         if (templateNameOrWildcard.length > 1) {
             predicate = name -> Arrays.asList(templateNameOrWildcard).contains(name);
@@ -456,7 +457,10 @@ static void validateNotInUse(Metadata metadata, String... templateNameOrWildcard
             .collect(Collectors.toSet());
         final Set<String> componentsBeingUsed = new HashSet<>();
         final List<String> templatesStillUsing = metadata.templatesV2().entrySet().stream().filter(e -> {
-            Set<String> intersecting = Sets.intersection(new HashSet<>(e.getValue().composedOf()), matchingComponentTemplates);
+            Set<String> intersecting = Sets.intersection(
+                new HashSet<>(e.getValue().getRequiredComponentTemplates()),
+                matchingComponentTemplates
+            );
             if (intersecting.size() > 0) {
                 componentsBeingUsed.addAll(intersecting);
                 return true;

server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java

@@ -1666,6 +1666,40 @@ public void testRemoveComponentTemplateInUse() throws Exception {
         );
     }
 
+    public void testRemoveRequiredAndNonRequiredComponents() throws Exception {
+        ComposableIndexTemplate composableIndexTemplate = new ComposableIndexTemplate(
+            Collections.singletonList("pattern"),
+            null,
+            List.of("required1", "non-required", "required2"),
+            null,
+            null,
+            null,
+            null,
+            null,
+            Collections.singletonList("non-required")
+        );
+        ComponentTemplate ct = new ComponentTemplate(new Template(null, new CompressedXContent("{}"), null), null, null);
+
+        final MetadataIndexTemplateService service = getMetadataIndexTemplateService();
+        ClusterState clusterState = service.addComponentTemplate(ClusterState.EMPTY_STATE, false, "required1", ct);
+        clusterState = service.addComponentTemplate(clusterState, false, "required2", ct);
+        clusterState = service.addComponentTemplate(clusterState, false, "non-required", ct);
+        clusterState = service.addIndexTemplateV2(clusterState, false, "composable-index-template", composableIndexTemplate);
+
+        final ClusterState cs = clusterState;
+        Exception e = expectThrows(IllegalArgumentException.class, () -> innerRemoveComponentTemplate(cs, "required*"));
+        assertThat(
+            e.getMessage(),
+            containsString(
+                "component templates [required1, required2] cannot be removed as they are still in use by index templates "
+                    + "[composable-index-template]"
+            )
+        );
+
+        // This removal should succeed
+        innerRemoveComponentTemplate(cs, "non-required*");
+    }
+
     /**
      * Tests that we check that settings/mappings/etc are valid even after template composition,
      * when adding/updating a composable index template

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistry.java

@@ -327,7 +327,7 @@ private void addComposableTemplatesIfMissing(ClusterState state) {
      * Returns true if the cluster state contains all of the component templates needed by the composable template
      */
     private static boolean componentTemplatesExist(ClusterState state, ComposableIndexTemplate indexTemplate) {
-        return state.metadata().componentTemplates().keySet().containsAll(indexTemplate.composedOf());
+        return state.metadata().componentTemplates().keySet().containsAll(indexTemplate.getRequiredComponentTemplates());
     }
 
     private void putLegacyTemplate(final IndexTemplateConfig config, final AtomicBoolean creationCheck) {

x-pack/plugin/stack/src/test/java/org/elasticsearch/xpack/stack/StackRegistryWithNonRequiredTemplates.java

@@ -0,0 +1,37 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.stack;
+
+import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.xcontent.NamedXContentRegistry;
+import org.elasticsearch.xpack.core.template.IndexTemplateConfig;
+
+import java.util.Map;
+
+class StackRegistryWithNonRequiredTemplates extends StackTemplateRegistry {
+    StackRegistryWithNonRequiredTemplates(
+        Settings nodeSettings,
+        ClusterService clusterService,
+        ThreadPool threadPool,
+        Client client,
+        NamedXContentRegistry xContentRegistry
+    ) {
+        super(nodeSettings, clusterService, threadPool, client, xContentRegistry);
+    }
+
+    @Override
+    protected Map<String, ComposableIndexTemplate> getComposableTemplateConfigs() {
+        return parseComposableTemplates(
+            new IndexTemplateConfig("syslog", "/non-required-template.json", REGISTRY_VERSION, TEMPLATE_VERSION_VARIABLE)
+        );
+    }
+}

x-pack/plugin/stack/src/test/java/org/elasticsearch/xpack/stack/StackTemplateRegistryTests.java

@@ -20,6 +20,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.block.ClusterBlocks;
 import org.elasticsearch.cluster.metadata.ComponentTemplate;
+import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -270,6 +271,53 @@ public void testThatUnversionedOldTemplatesAreUpgraded() throws Exception {
         assertBusy(() -> assertThat(calledTimes.get(), equalTo(registry.getComponentTemplateConfigs().size())));
     }
 
+    public void testMissingNonRequiredTemplates() throws Exception {
+        StackRegistryWithNonRequiredTemplates registryWithNonRequiredTemplate = new StackRegistryWithNonRequiredTemplates(
+            Settings.EMPTY,
+            clusterService,
+            threadPool,
+            client,
+            NamedXContentRegistry.EMPTY
+        );
+
+        DiscoveryNode node = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT);
+        DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build();
+
+        ClusterChangedEvent event = createClusterChangedEvent(
+            Collections.singletonMap(StackTemplateRegistry.LOGS_SETTINGS_COMPONENT_TEMPLATE_NAME, null),
+            nodes
+        );
+
+        final AtomicInteger calledTimes = new AtomicInteger(0);
+        client.setVerifier((action, request, listener) -> {
+            if (action instanceof PutComponentTemplateAction) {
+                // Ignore such
+                return AcknowledgedResponse.TRUE;
+            } else if (action instanceof PutLifecycleAction) {
+                // Ignore such
+                return AcknowledgedResponse.TRUE;
+            } else if (action instanceof PutComposableIndexTemplateAction) {
+                calledTimes.incrementAndGet();
+                assertThat(request, instanceOf(PutComposableIndexTemplateAction.Request.class));
+                PutComposableIndexTemplateAction.Request putComposableTemplateRequest = (PutComposableIndexTemplateAction.Request) request;
+                assertThat(putComposableTemplateRequest.name(), equalTo("syslog"));
+                ComposableIndexTemplate composableIndexTemplate = putComposableTemplateRequest.indexTemplate();
+                assertThat(composableIndexTemplate.composedOf(), hasSize(2));
+                assertThat(composableIndexTemplate.composedOf().get(0), equalTo("logs-settings"));
+                assertThat(composableIndexTemplate.composedOf().get(1), equalTo("syslog@custom"));
+                assertThat(composableIndexTemplate.getIgnoreMissingComponentTemplates(), hasSize(1));
+                assertThat(composableIndexTemplate.getIgnoreMissingComponentTemplates().get(0), equalTo("syslog@custom"));
+                return AcknowledgedResponse.TRUE;
+            } else {
+                fail("client called with unexpected request:" + request.toString());
+                return null;
+            }
+        });
+
+        registryWithNonRequiredTemplate.clusterChanged(event);
+        assertBusy(() -> assertThat(calledTimes.get(), equalTo(1)));
+    }
+
     @TestLogging(value = "org.elasticsearch.xpack.core.template:DEBUG", reason = "test")
     public void testSameOrHigherVersionTemplateNotUpgraded() {
         DiscoveryNode node = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT);

x-pack/plugin/stack/src/test/resources/non-required-template.json

@@ -0,0 +1,16 @@
+{
+  "index_patterns": ["syslog-*-*"],
+  "priority": 100,
+  "data_stream": {},
+  "composed_of": [
+    "logs-settings",
+    "syslog@custom"
+  ],
+  "ignore_missing_component_templates": ["syslog@custom"],
+  "allow_auto_create": true,
+  "_meta": {
+    "description": "default logs template installed by x-pack",
+    "managed": true
+  },
+  "version": ${xpack.stack.template.version}
+}