diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ee19e1234614..8267d2f19c761 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Move pull-based ingestion classes from experimental to publicAPI ([#20704](https://github.com/opensearch-project/OpenSearch/pull/20704)) ### Fixed +- Relax index template pattern overlap check to use minimum-string heuristic, allowing distinguishable multi-wildcard patterns at the same priority ([#20702](https://github.com/opensearch-project/OpenSearch/pull/20702)) - Fix `AutoForceMergeMetrics` silently dropping tags due to unreassigned `addTag()` return value ([#20788](https://github.com/opensearch-project/OpenSearch/pull/20788)) - Fix flaky test failures in ShardsLimitAllocationDeciderIT ([#20375](https://github.com/opensearch-project/OpenSearch/pull/20375)) - Prevent criteria update for context aware indices ([#20250](https://github.com/opensearch-project/OpenSearch/pull/20250)) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_index_template/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_index_template/10_basic.yml index 4e149c5656b4f..5ab1021cb4e75 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_index_template/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_index_template/10_basic.yml @@ -102,6 +102,51 @@ number_of_shards: 1 number_of_replicas: 0 +--- +"Put distinguishable multi-wildcard templates at the same priority": + - skip: + features: allowed_warnings + version: " - 3.5.99" + reason: "template overlap check was relaxed in 3.6" + + - do: + allowed_warnings: + - "index template [app-test-some] has index patterns [app-test-*-some-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [app-test-some] will take precedence during new index creation" + indices.put_index_template: + name: app-test-some + body: + index_patterns: ["app-test-*-some-*"] + priority: 0 + + # The second template has a literal segment after the first wildcard that differs from the first + # template (-some_other- vs -some-), so the two patterns do not practically overlap and both + # templates should be accepted at the same priority. + - do: + allowed_warnings: + - "index template [app-test-some-other] has index patterns [app-test-*-some_other-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [app-test-some-other] will take precedence during new index creation" + indices.put_index_template: + name: app-test-some-other + body: + index_patterns: ["app-test-*-some_other-*"] + priority: 0 + + - do: + indices.get_index_template: + name: app-test-some + - match: {index_templates.0.index_template.index_patterns: ["app-test-*-some-*"]} + + - do: + indices.get_index_template: + name: app-test-some-other + - match: {index_templates.0.index_template.index_patterns: ["app-test-*-some_other-*"]} + + - do: + indices.delete_index_template: + name: app-test-some + - do: + indices.delete_index_template: + name: app-test-some-other + --- "Put index template without index_patterns": diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataIndexTemplateService.java index cc15fd8b6f99b..d15c9d7c9ef4d 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataIndexTemplateService.java @@ -34,8 +34,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.util.CollectionUtil; -import org.apache.lucene.util.automaton.Automaton; -import org.apache.lucene.util.automaton.Operations; import org.opensearch.Version; import org.opensearch.action.admin.indices.alias.Alias; import org.opensearch.action.support.clustermanager.AcknowledgedResponse; @@ -799,13 +797,11 @@ public static Map> findConflictingV1Templates( final String candidateName, final List indexPatterns ) { - Automaton v2automaton = Regex.simpleMatchToAutomaton(indexPatterns.toArray(Strings.EMPTY_ARRAY)); Map> overlappingTemplates = new HashMap<>(); for (final Map.Entry cursor : state.metadata().templates().entrySet()) { String name = cursor.getKey(); IndexTemplateMetadata template = cursor.getValue(); - Automaton v1automaton = Regex.simpleMatchToAutomaton(template.patterns().toArray(Strings.EMPTY_ARRAY)); - if (Operations.isEmpty(Operations.intersection(v2automaton, v1automaton)) == false) { + if (patternsActuallyOverlap(indexPatterns, template.patterns())) { logger.debug( "composable template {} and legacy template {} would overlap: {} <=> {}", candidateName, @@ -848,13 +844,11 @@ static Map> findConflictingV2Templates( boolean checkPriority, long priority ) { - Automaton v1automaton = Regex.simpleMatchToAutomaton(indexPatterns.toArray(Strings.EMPTY_ARRAY)); Map> overlappingTemplates = new HashMap<>(); for (Map.Entry entry : state.metadata().templatesV2().entrySet()) { String name = entry.getKey(); ComposableIndexTemplate template = entry.getValue(); - Automaton v2automaton = Regex.simpleMatchToAutomaton(template.indexPatterns().toArray(Strings.EMPTY_ARRAY)); - if (Operations.isEmpty(Operations.intersection(v1automaton, v2automaton)) == false) { + if (patternsActuallyOverlap(indexPatterns, template.indexPatterns())) { if (checkPriority == false || priority == template.priorityOrZero()) { logger.debug( "legacy template {} and composable template {} would overlap: {} <=> {}", @@ -873,6 +867,36 @@ static Map> findConflictingV2Templates( return overlappingTemplates; } + /** + * Checks whether two sets of index patterns practically overlap, using a minimum-string heuristic. + *

+ * For each pattern in either set, the wildcard character {@code *} is replaced with the empty string + * to produce its "minimum matching string". If that minimum string matches any pattern in the other + * set, the two pattern sets are considered to overlap. + *

+ * This approach is intentionally more lenient than a full automaton intersection. Full automaton + * intersection correctly identifies theoretical overlaps that arise only from creative wildcard usage + * (e.g. {@code app-test-*-some-*} and {@code app-test-*-some_other-*} share the synthetic string + * {@code app-test--some_other--some-}), but these synthetic strings would never appear as real index + * names. The minimum-string heuristic restricts conflict detection to cases where "natural" index + * names would match both patterns simultaneously. + */ + static boolean patternsActuallyOverlap(List patterns1, List patterns2) { + String[] p2Array = patterns2.toArray(Strings.EMPTY_ARRAY); + for (String p1 : patterns1) { + if (Regex.simpleMatch(p2Array, p1.replace("*", ""))) { + return true; + } + } + String[] p1Array = patterns1.toArray(Strings.EMPTY_ARRAY); + for (String p2 : patterns2) { + if (Regex.simpleMatch(p1Array, p2.replace("*", ""))) { + return true; + } + } + return false; + } + /** * Remove the given index template from the cluster state. The index template name * supports simple regex wildcards for removing multiple index templates at a time. diff --git a/server/src/test/java/org/opensearch/cluster/metadata/MetadataIndexTemplateServiceTests.java b/server/src/test/java/org/opensearch/cluster/metadata/MetadataIndexTemplateServiceTests.java index c66f289bc8acf..fc72363c129f5 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/MetadataIndexTemplateServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/MetadataIndexTemplateServiceTests.java @@ -1320,6 +1320,55 @@ public void testPuttingOverlappingV2Template() throws Exception { } } + public void testPatternsActuallyOverlap() { + // Patterns that share a genuine conflict: baz matches baz* + assertTrue(MetadataIndexTemplateService.patternsActuallyOverlap(Arrays.asList("egg*", "baz"), Arrays.asList("abc", "baz*"))); + + // Patterns with multi-wildcards where literal segments after the first wildcard distinguish them + assertFalse( + MetadataIndexTemplateService.patternsActuallyOverlap( + Arrays.asList("app-test-*-some-*"), + Arrays.asList("app-test-*-some_other-*") + ) + ); + + // Patterns with a common prefix and different literal suffixes after a wildcard + assertFalse(MetadataIndexTemplateService.patternsActuallyOverlap(Arrays.asList("foo-*-bar"), Arrays.asList("foo-*-baz"))); + + // Patterns where one is a superset of the other (logs-* matches any logs-prod-* index name) + assertTrue(MetadataIndexTemplateService.patternsActuallyOverlap(Arrays.asList("logs-*"), Arrays.asList("logs-prod-*"))); + + // Completely disjoint literal prefixes do not overlap + assertFalse(MetadataIndexTemplateService.patternsActuallyOverlap(Arrays.asList("foo-*"), Arrays.asList("bar-*"))); + + // Identical patterns overlap with themselves + assertTrue(MetadataIndexTemplateService.patternsActuallyOverlap(Arrays.asList("app-*"), Arrays.asList("app-*"))); + } + + public void testDistinguishableMultiWildcardTemplatesAccepted() throws Exception { + MetadataIndexTemplateService service = getMetadataIndexTemplateService(); + ClusterState state = ClusterState.EMPTY_STATE; + + ComposableIndexTemplate t1 = new ComposableIndexTemplate(Arrays.asList("app-test-*-some-*"), null, null, 0L, null, null, null); + state = service.addIndexTemplateV2(state, false, "app-test-some", t1); + + // This must not throw: the second pattern is distinguishable from the first because + // the literal segment after the first wildcard differs (_other vs nothing). + ComposableIndexTemplate t2 = new ComposableIndexTemplate( + Arrays.asList("app-test-*-some_other-*"), + null, + null, + 0L, + null, + null, + null + ); + state = service.addIndexTemplateV2(state, false, "app-test-some-other", t2); + + assertNotNull(state.metadata().templatesV2().get("app-test-some")); + assertNotNull(state.metadata().templatesV2().get("app-test-some-other")); + } + public void testFindV2Templates() throws Exception { final MetadataIndexTemplateService service = getMetadataIndexTemplateService(); ClusterState state = ClusterState.EMPTY_STATE;