From 3433c0caaaf925cb8ae219ce6d6ae013a3a4a4ff Mon Sep 17 00:00:00 2001 From: Jack Conradson Date: Mon, 17 Nov 2025 12:24:30 -0800 Subject: [PATCH 1/3] Add deprecation checks for indices with percolator fields (#138118) This change modifies the deprecation API to check for any indices prior to 9.latest that have percolator fields in the mappings. Any that do are then returned as required to be reindexed or deleted. Read-only is excluded as this would not fix the underlying transport version issue. Note this change only concerns user created indices. System indices will need to be addressed separately as part of the migration API, but should be able to share code. ES-13234 --- .../cluster/metadata/IndexMetadata.java | 46 +++ .../metadata/MetadataCreateIndexService.java | 1 + .../index_created_transport_version.csv | 1 + .../resources/transport/upper_bounds/8.19.csv | 2 +- .../resources/transport/upper_bounds/9.1.csv | 2 +- .../resources/transport/upper_bounds/9.2.csv | 2 +- .../resources/transport/upper_bounds/9.3.csv | 1 + .../reroute/ClusterRerouteResponseTests.java | 2 + .../cluster/metadata/IndexMetadataTests.java | 5 + .../metadata/ToAndFromJsonMetadataTests.java | 4 + .../deprecation/DeprecatedIndexPredicate.java | 131 ++++++++- .../DataStreamDeprecationChecker.java | 53 +++- .../deprecation/IndexDeprecationChecker.java | 264 ++++++++---------- .../DataStreamDeprecationCheckerTests.java | 144 +++++++++- .../IndexDeprecationCheckerTests.java | 120 ++++++++ .../AbstractWatcherIntegrationTestCase.java | 1 + 16 files changed, 614 insertions(+), 165 deletions(-) create mode 100644 server/src/main/resources/transport/definitions/referable/index_created_transport_version.csv create mode 100644 server/src/main/resources/transport/upper_bounds/9.3.csv diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java index 2b2dbc9985e42..a5c398c32fab6 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java @@ -152,6 +152,8 @@ public class IndexMetadata implements Diffable, ToXContentFragmen // 'event.ingested' (part of Elastic Common Schema) range is tracked in cluster state, along with @timestamp public static final String EVENT_INGESTED_FIELD_NAME = "event.ingested"; + private static final TransportVersion INDEX_CREATED_TRANSPORT_VERSION = TransportVersion.fromName("index_created_transport_version"); + @Nullable public String getDownsamplingInterval() { return settings.get(IndexMetadata.INDEX_DOWNSAMPLE_INTERVAL_KEY); @@ -540,6 +542,7 @@ public Iterator> settings() { public static final List PARTIALLY_MOUNTED_INDEX_TIER_PREFERENCE = List.of(DataTier.DATA_FROZEN); static final String KEY_VERSION = "version"; + static final String KEY_TRANSPORT_VERSION = "transport_version"; static final String KEY_MAPPING_VERSION = "mapping_version"; static final String KEY_SETTINGS_VERSION = "settings_version"; static final String KEY_ALIASES_VERSION = "aliases_version"; @@ -579,6 +582,7 @@ public Iterator> settings() { private final Index index; private final long version; + private final TransportVersion transportVersion; private final long mappingVersion; @@ -662,6 +666,7 @@ public Iterator> settings() { private IndexMetadata( final Index index, final long version, + final TransportVersion transportVersion, final long mappingVersion, final long settingsVersion, final long aliasesVersion, @@ -710,6 +715,7 @@ private IndexMetadata( ) { this.index = index; this.version = version; + this.transportVersion = transportVersion; assert mappingVersion >= 0 : mappingVersion; this.mappingVersion = mappingVersion; this.mappingsUpdatedVersion = mappingsUpdatedVersion; @@ -774,6 +780,7 @@ IndexMetadata withMappingMetadata(MappingMetadata mapping) { return new IndexMetadata( this.index, this.version, + this.transportVersion, this.mappingVersion, this.settingsVersion, this.aliasesVersion, @@ -835,6 +842,7 @@ public IndexMetadata withInSyncAllocationIds(int shardId, Set inSyncSet) return new IndexMetadata( this.index, this.version, + this.transportVersion, this.mappingVersion, this.settingsVersion, this.aliasesVersion, @@ -894,6 +902,7 @@ public IndexMetadata withIncrementedPrimaryTerm(int shardId) { return new IndexMetadata( this.index, this.version, + this.transportVersion, this.mappingVersion, this.settingsVersion, this.aliasesVersion, @@ -963,6 +972,7 @@ public IndexMetadata withTimestampRanges( return new IndexMetadata( this.index, this.version, + this.transportVersion, this.mappingVersion, this.settingsVersion, this.aliasesVersion, @@ -1018,6 +1028,7 @@ public IndexMetadata withIncrementedVersion() { return new IndexMetadata( this.index, this.version + 1, + this.transportVersion, this.mappingVersion, this.settingsVersion, this.aliasesVersion, @@ -1078,6 +1089,10 @@ public long getVersion() { return this.version; } + public TransportVersion getTransportVersion() { + return transportVersion; + } + public long getMappingVersion() { return mappingVersion; } @@ -1533,6 +1548,7 @@ private static class IndexMetadataDiff implements Diff { private final String index; private final int routingNumShards; private final long version; + private final TransportVersion transportVersion; private final long mappingVersion; private final long settingsVersion; private final long aliasesVersion; @@ -1565,6 +1581,7 @@ private static class IndexMetadataDiff implements Diff { IndexMetadataDiff(IndexMetadata before, IndexMetadata after) { index = after.index.getName(); version = after.version; + transportVersion = after.transportVersion; mappingVersion = after.mappingVersion; settingsVersion = after.settingsVersion; aliasesVersion = after.aliasesVersion; @@ -1617,6 +1634,9 @@ private static class IndexMetadataDiff implements Diff { index = in.readString(); routingNumShards = in.readInt(); version = in.readLong(); + transportVersion = in.getTransportVersion().supports(INDEX_CREATED_TRANSPORT_VERSION) + ? TransportVersion.readVersion(in) + : TransportVersion.fromId(0); mappingVersion = in.readVLong(); settingsVersion = in.readVLong(); if (in.getTransportVersion().onOrAfter(TransportVersions.V_7_2_0)) { @@ -1687,6 +1707,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(index); out.writeInt(routingNumShards); out.writeLong(version); + if (out.getTransportVersion().supports(INDEX_CREATED_TRANSPORT_VERSION)) { + TransportVersion.writeVersion(transportVersion, out); + } out.writeVLong(mappingVersion); out.writeVLong(settingsVersion); if (out.getTransportVersion().onOrAfter(TransportVersions.V_7_2_0)) { @@ -1733,6 +1756,7 @@ public void writeTo(StreamOutput out) throws IOException { public IndexMetadata apply(IndexMetadata part) { Builder builder = builder(index); builder.version(version); + builder.transportVersion(transportVersion); builder.mappingVersion(mappingVersion); builder.settingsVersion(settingsVersion); builder.aliasesVersion(aliasesVersion); @@ -1775,6 +1799,11 @@ public static IndexMetadata readFrom(StreamInput in) throws IOException { public static IndexMetadata readFrom(StreamInput in, @Nullable Function mappingLookup) throws IOException { Builder builder = new Builder(in.readString()); builder.version(in.readLong()); + builder.transportVersion( + in.getTransportVersion().supports(INDEX_CREATED_TRANSPORT_VERSION) + ? TransportVersion.readVersion(in) + : TransportVersion.fromId(0) + ); builder.mappingVersion(in.readVLong()); builder.settingsVersion(in.readVLong()); if (in.getTransportVersion().onOrAfter(TransportVersions.V_7_2_0)) { @@ -1847,6 +1876,9 @@ public static IndexMetadata readFrom(StreamInput in, @Nullable Function builder.state(State.fromString(parser.text())); case KEY_VERSION -> builder.version(parser.longValue()); + case KEY_TRANSPORT_VERSION -> builder.transportVersion(TransportVersion.fromString(parser.text())); case KEY_MAPPING_VERSION -> { mappingVersion = true; builder.mappingVersion(parser.longValue()); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index 81142c13a5bac..6d93fd4495a1e 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -588,6 +588,7 @@ private IndexMetadata buildAndValidateTemporaryIndexMetadata( final IndexMetadata.Builder tmpImdBuilder = IndexMetadata.builder(request.index()); tmpImdBuilder.setRoutingNumShards(routingNumShards); tmpImdBuilder.settings(indexSettings); + tmpImdBuilder.transportVersion(TransportVersion.current()); tmpImdBuilder.system(isSystem); // Set up everything, now locally create the index to see that things are ok, and apply diff --git a/server/src/main/resources/transport/definitions/referable/index_created_transport_version.csv b/server/src/main/resources/transport/definitions/referable/index_created_transport_version.csv new file mode 100644 index 0000000000000..f25ff514b4116 --- /dev/null +++ b/server/src/main/resources/transport/definitions/referable/index_created_transport_version.csv @@ -0,0 +1 @@ +9221000,9185009,9112014,8841075 diff --git a/server/src/main/resources/transport/upper_bounds/8.19.csv b/server/src/main/resources/transport/upper_bounds/8.19.csv index a2bfbb0094989..09daf16060da0 100644 --- a/server/src/main/resources/transport/upper_bounds/8.19.csv +++ b/server/src/main/resources/transport/upper_bounds/8.19.csv @@ -1 +1 @@ -initial_8.19.8,8841073 +index_created_transport_version,8841075 diff --git a/server/src/main/resources/transport/upper_bounds/9.1.csv b/server/src/main/resources/transport/upper_bounds/9.1.csv index 80b97d85f7511..37a8af514597a 100644 --- a/server/src/main/resources/transport/upper_bounds/9.1.csv +++ b/server/src/main/resources/transport/upper_bounds/9.1.csv @@ -1 +1 @@ -transform_check_for_dangling_tasks,9112009 +index_created_transport_version,9112014 diff --git a/server/src/main/resources/transport/upper_bounds/9.2.csv b/server/src/main/resources/transport/upper_bounds/9.2.csv index 2c15e0254cbe8..0b2d3c451ce12 100644 --- a/server/src/main/resources/transport/upper_bounds/9.2.csv +++ b/server/src/main/resources/transport/upper_bounds/9.2.csv @@ -1 +1 @@ -transform_check_for_dangling_tasks,9170000 +index_created_transport_version,9185009 diff --git a/server/src/main/resources/transport/upper_bounds/9.3.csv b/server/src/main/resources/transport/upper_bounds/9.3.csv new file mode 100644 index 0000000000000..cc71afa89aa4a --- /dev/null +++ b/server/src/main/resources/transport/upper_bounds/9.3.csv @@ -0,0 +1 @@ +index_created_transport_version,9221000 diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java index 67e5f30f023c9..f14c947da83fd 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java @@ -162,6 +162,7 @@ public void testToXContentWithDeprecatedClusterState() { "indices": { "index": { "version": 1, + "transport_version" : "0", "mapping_version": 1, "settings_version": 1, "aliases_version": 1, @@ -251,6 +252,7 @@ public void testToXContentWithDeprecatedClusterStateAndMetadata() { "indices" : { "index" : { "version" : 1, + "transport_version" : "0", "mapping_version" : 1, "settings_version" : 1, "aliases_version" : 1, diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java index d83e046436ec0..2dac4d5eb9d5e 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java @@ -100,6 +100,7 @@ public void testIndexMetadataSerialization() throws IOException { IndexMetadata metadata = IndexMetadata.builder("foo") .settings(indexSettings(numShard, numberOfReplicas).put("index.version.created", 1)) + .transportVersion(TransportVersion.current()) .creationDate(randomLong()) .primaryTerm(0, 2) .setRoutingNumShards(32) @@ -150,6 +151,7 @@ public void testIndexMetadataSerialization() throws IOException { ); assertEquals(metadata.hashCode(), fromXContentMeta.hashCode()); + assertEquals(metadata.getTransportVersion(), fromXContentMeta.getTransportVersion()); assertEquals(metadata.getNumberOfReplicas(), fromXContentMeta.getNumberOfReplicas()); assertEquals(metadata.getNumberOfShards(), fromXContentMeta.getNumberOfShards()); assertEquals(metadata.getCreationVersion(), fromXContentMeta.getCreationVersion()); @@ -176,6 +178,7 @@ public void testIndexMetadataSerialization() throws IOException { assertEquals(metadata, deserialized); assertEquals(metadata.hashCode(), deserialized.hashCode()); + assertEquals(metadata.getTransportVersion(), deserialized.getTransportVersion()); assertEquals(metadata.getNumberOfReplicas(), deserialized.getNumberOfReplicas()); assertEquals(metadata.getNumberOfShards(), deserialized.getNumberOfShards()); assertEquals(metadata.getCreationVersion(), deserialized.getCreationVersion()); @@ -211,6 +214,7 @@ public void testIndexMetadataFromXContentParsingWithoutEventIngestedField() thro IndexMetadata metadata = IndexMetadata.builder("foo") .settings(indexSettings(numShard, numberOfReplicas).put("index.version.created", 1)) + .transportVersion(TransportVersion.current()) .creationDate(randomLong()) .primaryTerm(0, 2) .setRoutingNumShards(32) @@ -279,6 +283,7 @@ public void testIndexMetadataFromXContentParsingWithoutEventIngestedField() thro fromXContentMeta ); assertEquals(metadata.hashCode(), fromXContentMeta.hashCode()); + assertEquals(metadata.getTransportVersion(), fromXContentMeta.getTransportVersion()); assertEquals(metadata.getNumberOfReplicas(), fromXContentMeta.getNumberOfReplicas()); assertEquals(metadata.getNumberOfShards(), fromXContentMeta.getNumberOfShards()); assertEquals(metadata.getCreationVersion(), fromXContentMeta.getCreationVersion()); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java index 36e6c586a6fde..8201a168c3cb8 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java @@ -323,6 +323,7 @@ public void testToXContentAPI_SameTypeName() throws IOException { "indices" : { "index" : { "version" : 2, + "transport_version" : "0", "mapping_version" : 1, "settings_version" : 1, "aliases_version" : 1, @@ -490,6 +491,7 @@ public void testToXContentAPI_FlatSettingTrue_ReduceMappingFalse() throws IOExce "indices" : { "index" : { "version" : 2, + "transport_version" : "0", "mapping_version" : 1, "settings_version" : 1, "aliases_version" : 1, @@ -600,6 +602,7 @@ public void testToXContentAPI_FlatSettingFalse_ReduceMappingTrue() throws IOExce "indices" : { "index" : { "version" : 2, + "transport_version" : "0", "mapping_version" : 1, "settings_version" : 1, "aliases_version" : 1, @@ -740,6 +743,7 @@ public void testToXContentAPIReservedMetadata() throws IOException { "indices" : { "index" : { "version" : 2, + "transport_version" : "0", "mapping_version" : 1, "settings_version" : 1, "aliases_version" : 1, diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/DeprecatedIndexPredicate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/DeprecatedIndexPredicate.java index b5967e6b1860c..d66d7c41dda91 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/DeprecatedIndexPredicate.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/DeprecatedIndexPredicate.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.core.deprecation; +import org.elasticsearch.TransportVersion; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.metadata.MetadataIndexStateService; @@ -14,6 +15,11 @@ import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.function.BiFunction; +import java.util.function.Function; import java.util.function.Predicate; public class DeprecatedIndexPredicate { @@ -49,7 +55,7 @@ public static Predicate getReindexRequiredPredicate(Metadata metadata, bo * if false, only those without a block are returned * @param includeSystem if true, all indices including system will be returned, * if false, only non-system indices are returned - * @return a predicate that returns true for indices that need to be reindexed + * @return returns true for indices that need to be reindexed */ public static boolean reindexRequired(IndexMetadata indexMetadata, boolean filterToBlockedStatus, boolean includeSystem) { return creationVersionBeforeMinimumWritableVersion(indexMetadata) @@ -58,6 +64,59 @@ && isNotSearchableSnapshot(indexMetadata) && matchBlockedStatus(indexMetadata, filterToBlockedStatus); } + /** + * This method checks if this index is on the current transport version for the current minor release version. + * + * @param indexMetadata the index metadata + * @param filterToBlockedStatus if true, only indices that are write blocked will be returned, + * if false, only those without a block are returned + * @param includeSystem if true, all indices including system will be returned, + * if false, only non-system indices are returned + * @return returns true for indices that need to be reindexed + */ + public static boolean reindexRequiredForTransportVersion( + IndexMetadata indexMetadata, + boolean filterToBlockedStatus, + boolean includeSystem + ) { + return transportVersionBeforeCurrentMinorRelease(indexMetadata) + && (includeSystem || isNotSystem(indexMetadata)) + && isNotSearchableSnapshot(indexMetadata) + && matchBlockedStatus(indexMetadata, filterToBlockedStatus); + } + + /** + * This method checks if this index requires reindexing based on if it has percolator fields older than the current transport version + * for the current minor release. + * + * @param indexMetadata the index metadata + * @param filterToBlockedStatus if true, only indices that are write blocked will be returned, + * if false, only those without a block are returned + * @param includeSystem if true, all indices including system will be returned, + * if false, only non-system indices are returned + * @return returns a message as a string for each incompatible percolator field found + */ + public static List reindexRequiredForPecolatorFields( + IndexMetadata indexMetadata, + boolean filterToBlockedStatus, + boolean includeSystem + ) { + List percolatorIncompatibleFieldMappings = new ArrayList<>(); + if (reindexRequiredForTransportVersion(indexMetadata, filterToBlockedStatus, includeSystem) && indexMetadata.mapping() != null) { + percolatorIncompatibleFieldMappings.addAll( + findInPropertiesRecursively( + indexMetadata.mapping().type(), + indexMetadata.mapping().sourceAsMap(), + property -> "percolator".equals(property.get("type")), + (type, entry) -> "Field [" + entry.getKey() + "] is of type [" + indexMetadata.mapping().type() + "]", + "", + "" + ) + ); + } + return percolatorIncompatibleFieldMappings; + } + private static boolean isNotSystem(IndexMetadata indexMetadata) { return indexMetadata.isSystem() == false; } @@ -73,4 +132,74 @@ private static boolean creationVersionBeforeMinimumWritableVersion(IndexMetadata private static boolean matchBlockedStatus(IndexMetadata indexMetadata, boolean filterToBlockedStatus) { return MetadataIndexStateService.VERIFIED_READ_ONLY_SETTING.get(indexMetadata.getSettings()) == filterToBlockedStatus; } + + private static boolean transportVersionBeforeCurrentMinorRelease(IndexMetadata indexMetadata) { + // We divide each transport version by 1000 to get the base id. + return indexMetadata.getTransportVersion().id() / 1000 < TransportVersion.current().id() / 1000; + } + + /** + * iterates through the "properties" field of mappings and returns any predicates that match in the + * form of issue-strings. + * + * @param type the document type + * @param parentMap the mapping to read properties from + * @param predicate the predicate to check against for issues, issue is returned if predicate evaluates to true + * @param fieldFormatter a function that takes a type and mapping field entry and returns a formatted field representation + * @return a list of issues found in fields + */ + @SuppressWarnings("unchecked") + public static List findInPropertiesRecursively( + String type, + Map parentMap, + Function, Boolean> predicate, + BiFunction, String> fieldFormatter, + String fieldBeginMarker, + String fieldEndMarker + ) { + List issues = new ArrayList<>(); + Map properties = (Map) parentMap.get("properties"); + if (properties == null) { + return issues; + } + for (Map.Entry entry : properties.entrySet()) { + Map valueMap = (Map) entry.getValue(); + if (predicate.apply(valueMap)) { + issues.add(fieldBeginMarker + fieldFormatter.apply(type, entry) + fieldEndMarker); + } + + Map values = (Map) valueMap.get("fields"); + if (values != null) { + for (Map.Entry multifieldEntry : values.entrySet()) { + Map multifieldValueMap = (Map) multifieldEntry.getValue(); + if (predicate.apply(multifieldValueMap)) { + issues.add( + fieldBeginMarker + + fieldFormatter.apply(type, entry) + + ", multifield: " + + multifieldEntry.getKey() + + fieldEndMarker + ); + } + if (multifieldValueMap.containsKey("properties")) { + issues.addAll( + findInPropertiesRecursively( + type, + multifieldValueMap, + predicate, + fieldFormatter, + fieldBeginMarker, + fieldEndMarker + ) + ); + } + } + } + if (valueMap.containsKey("properties")) { + issues.addAll(findInPropertiesRecursively(type, valueMap, predicate, fieldFormatter, fieldBeginMarker, fieldEndMarker)); + } + } + + return issues; + } } diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationChecker.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationChecker.java index b63e4359896f5..887a0b603eda3 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationChecker.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationChecker.java @@ -88,9 +88,24 @@ public Map> check(ClusterState clusterState) { static DeprecationIssue oldIndicesCheck(DataStream dataStream, ClusterState clusterState) { List backingIndices = dataStream.getIndices(); - + Set percolatorIndicesNeedingUpgrade = getReindexRequiredIndicesWithPercolatorFields(backingIndices, clusterState, false); + if (percolatorIndicesNeedingUpgrade.isEmpty() == false) { + return new DeprecationIssue( + DeprecationIssue.Level.CRITICAL, + "Field mappings with incompatible percolator type", + "https://www.elastic.co/guide/en/elasticsearch/reference/8.19/percolator.html#_reindexing_your_percolator_queries", + "The data stream was created before 8.19 and contains mappings that must be reindexed due to containing percolator fields.", + false, + ofEntries( + entry("reindex_required", true), + entry("excluded_actions", List.of("readOnly")), + entry("total_backing_indices", backingIndices.size()), + entry("indices_requiring_upgrade_count", percolatorIndicesNeedingUpgrade.size()), + entry("indices_requiring_upgrade", percolatorIndicesNeedingUpgrade) + ) + ); + } Set indicesNeedingUpgrade = getReindexRequiredIndices(backingIndices, clusterState, false); - if (indicesNeedingUpgrade.isEmpty() == false) { return new DeprecationIssue( DeprecationIssue.Level.CRITICAL, @@ -112,6 +127,23 @@ static DeprecationIssue oldIndicesCheck(DataStream dataStream, ClusterState clus static DeprecationIssue ignoredOldIndicesCheck(DataStream dataStream, ClusterState clusterState) { List backingIndices = dataStream.getIndices(); + Set percolatorIgnoredIndices = getReindexRequiredIndicesWithPercolatorFields(backingIndices, clusterState, true); + if (percolatorIgnoredIndices.isEmpty() == false) { + return new DeprecationIssue( + DeprecationIssue.Level.CRITICAL, + "Field mappings with incompatible percolator type", + "https://www.elastic.co/guide/en/elasticsearch/reference/8.19/percolator.html#_reindexing_your_percolator_queries", + "The data stream was created before 8.19 and contains mappings that must be reindexed due to containing percolator fields.", + false, + ofEntries( + entry("reindex_required", true), + entry("excluded_actions", List.of("readOnly")), + entry("total_backing_indices", backingIndices.size()), + entry("ignored_indices_requiring_upgrade_count", percolatorIgnoredIndices.size()), + entry("ignored_indices_requiring_upgrade", percolatorIgnoredIndices) + ) + ); + } Set ignoredIndices = getReindexRequiredIndices(backingIndices, clusterState, true); if (ignoredIndices.isEmpty() == false) { return new DeprecationIssue( @@ -144,6 +176,23 @@ private static Set getReindexRequiredIndices( .collect(Collectors.toUnmodifiableSet()); } + private static Set getReindexRequiredIndicesWithPercolatorFields( + List backingIndices, + ClusterState clusterState, + boolean filterToBlockedStatus + ) { + return backingIndices.stream() + .filter( + index -> DeprecatedIndexPredicate.reindexRequiredForPecolatorFields( + clusterState.metadata().index(index), + filterToBlockedStatus, + false + ).isEmpty() == false + ) + .map(Index::getName) + .collect(Collectors.toUnmodifiableSet()); + } + @Override public String getName() { return NAME; diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecker.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecker.java index 1799edcf56a75..b824b595a05fb 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecker.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecker.java @@ -31,8 +31,6 @@ import java.util.Map; import java.util.Objects; import java.util.function.BiConsumer; -import java.util.function.BiFunction; -import java.util.function.Function; import java.util.stream.Collectors; import static org.elasticsearch.xpack.deprecation.LegacyTiersDetection.DEPRECATION_COMMON_DETAIL; @@ -96,62 +94,82 @@ private DeprecationIssue oldIndicesCheck( ClusterState clusterState, Map> indexToTransformIds ) { - // TODO: this check needs to be revised. It's trivially true right now. - IndexVersion currentCompatibilityVersion = indexMetadata.getCompatibilityVersion(); // We intentionally exclude indices that are in data streams because they will be picked up by DataStreamDeprecationChecks - if (DeprecatedIndexPredicate.reindexRequired(indexMetadata, false, false) && isNotDataStreamIndex(indexMetadata, clusterState)) { - List cldrIncompatibleFieldMappings = new ArrayList<>(); - fieldLevelMappingIssue( + if (isNotDataStreamIndex(indexMetadata, clusterState)) { + // We check for percolator indices first as that will include potentially older indices as well. + List percolatorIncompatibleFieldMappings = DeprecatedIndexPredicate.reindexRequiredForPecolatorFields( indexMetadata, - (mappingMetadata, sourceAsMap) -> cldrIncompatibleFieldMappings.addAll( - findInPropertiesRecursively( - mappingMetadata.type(), - sourceAsMap, - this::isDateFieldWithCompatFormatPattern, - this::cldrIncompatibleFormatPattern, - "", - "" - ) - ) + false, + false ); - - var transforms = transformIdsForIndex(indexMetadata, indexToTransformIds); - if (transforms.isEmpty() == false) { - return new DeprecationIssue( - DeprecationIssue.Level.CRITICAL, - "One or more Transforms write to this index with a compatibility version < " + Version.CURRENT.major + ".0", - "https://www.elastic.co/docs/deploy-manage/upgrade/prepare-to-upgrade#transform-migration", - Strings.format( - "This index was created in version [%s] and requires action before upgrading to %d.0. The following transforms are " - + "configured to write to this index: [%s]. Refer to the migration guide to learn more about how to prepare " - + "transforms destination indices for your upgrade.", - currentCompatibilityVersion.toReleaseVersion(), - Version.CURRENT.major, - String.join(", ", transforms) - ), - false, - Map.of("reindex_required", true, "transform_ids", transforms) - ); - } else if (cldrIncompatibleFieldMappings.isEmpty() == false) { + if (percolatorIncompatibleFieldMappings.isEmpty() == false) { return new DeprecationIssue( DeprecationIssue.Level.CRITICAL, - "Field mappings with incompatible date format patterns in old index", - "https://www.elastic.co/blog/locale-changes-elasticsearch-8-16-jdk-23", - "The index was created before 8.0 and contains mappings that must be reindexed due to locale changes in 8.16+. " - + "Manual reindexing is required. " - + String.join(", ", cldrIncompatibleFieldMappings), + "Field mappings with incompatible percolator type", + "https://www.elastic.co/guide/en/elasticsearch/reference/8.19/percolator.html#_reindexing_your_percolator_queries", + "The index was created before 8.19 and contains mappings that must be reindexed due to containing percolator fields. " + + String.join(", ", percolatorIncompatibleFieldMappings), false, - null + Map.of("reindex_required", true, "excluded_actions", List.of("readOnly")) ); - } else { - return new DeprecationIssue( - DeprecationIssue.Level.CRITICAL, - "Old index with a compatibility version < " + Version.CURRENT.major + ".0", - "https://ela.st/es-deprecation-9-index-version", - "This index has version: " + currentCompatibilityVersion.toReleaseVersion(), - false, - Map.of("reindex_required", true) + } + + // TODO: this check needs to be revised. It's trivially true right now. + IndexVersion currentCompatibilityVersion = indexMetadata.getCompatibilityVersion(); + if (DeprecatedIndexPredicate.reindexRequired(indexMetadata, false, false)) { + List cldrIncompatibleFieldMappings = new ArrayList<>(); + fieldLevelMappingIssue( + indexMetadata, + (mappingMetadata, sourceAsMap) -> cldrIncompatibleFieldMappings.addAll( + DeprecatedIndexPredicate.findInPropertiesRecursively( + mappingMetadata.type(), + sourceAsMap, + this::isDateFieldWithCompatFormatPattern, + this::cldrIncompatibleFormatPattern, + "", + "" + ) + ) ); + + var transforms = transformIdsForIndex(indexMetadata, indexToTransformIds); + if (transforms.isEmpty() == false) { + return new DeprecationIssue( + DeprecationIssue.Level.CRITICAL, + "One or more Transforms write to this index with a compatibility version < " + Version.CURRENT.major + ".0", + "https://www.elastic.co/docs/deploy-manage/upgrade/prepare-to-upgrade#transform-migration", + Strings.format( + "This index was created in version [%s] and requires action before upgrading to %d.0. The following " + + "transforms are configured to write to this index: [%s]. Refer to the migration guide to learn more " + + "about how to prepare transforms destination indices for your upgrade.", + currentCompatibilityVersion.toReleaseVersion(), + Version.CURRENT.major, + String.join(", ", transforms) + ), + false, + Map.of("reindex_required", true, "transform_ids", transforms) + ); + } else if (cldrIncompatibleFieldMappings.isEmpty() == false) { + return new DeprecationIssue( + DeprecationIssue.Level.CRITICAL, + "Field mappings with incompatible date format patterns in old index", + "https://www.elastic.co/blog/locale-changes-elasticsearch-8-16-jdk-23", + "The index was created before 8.0 and contains mappings that must be reindexed due to locale changes in 8.16+. " + + "Manual reindexing is required. " + + String.join(", ", cldrIncompatibleFieldMappings), + false, + null + ); + } else { + return new DeprecationIssue( + DeprecationIssue.Level.CRITICAL, + "Old index with a compatibility version < " + Version.CURRENT.major + ".0", + "https://ela.st/es-deprecation-9-index-version", + "This index has version: " + currentCompatibilityVersion.toReleaseVersion(), + false, + Map.of("reindex_required", true) + ); + } } } return null; @@ -166,41 +184,60 @@ private DeprecationIssue ignoredOldIndicesCheck( ClusterState clusterState, Map> indexToTransformIds ) { - IndexVersion currentCompatibilityVersion = indexMetadata.getCompatibilityVersion(); // We intentionally exclude indices that are in data streams because they will be picked up by DataStreamDeprecationChecks - if (DeprecatedIndexPredicate.reindexRequired(indexMetadata, true, false) && isNotDataStreamIndex(indexMetadata, clusterState)) { - var transforms = transformIdsForIndex(indexMetadata, indexToTransformIds); - if (transforms.isEmpty() == false) { - return new DeprecationIssue( - DeprecationIssue.Level.WARNING, - "One or more Transforms write to this old index with a compatibility version < " + Version.CURRENT.major + ".0", - "https://www.elastic.co/docs/deploy-manage/upgrade/prepare-to-upgrade#transform-migration", - Strings.format( - "This index was created in version [%s] and will be supported as a read-only index in %d.0. The following " - + "transforms are no longer able to write to this index: [%s]. Refer to the migration guide to learn more " - + "about how to handle your transforms destination indices.", - currentCompatibilityVersion.toReleaseVersion(), - Version.CURRENT.major, - String.join(", ", transforms) - ), - false, - Map.of("reindex_required", true, "transform_ids", transforms) - ); - } else { + if (isNotDataStreamIndex(indexMetadata, clusterState)) { + // We check for percolator indices first as that will include potentially older indices as well. + List percolatorIncompatibleFieldMappings = DeprecatedIndexPredicate.reindexRequiredForPecolatorFields( + indexMetadata, + true, + false + ); + if (percolatorIncompatibleFieldMappings.isEmpty() == false) { return new DeprecationIssue( - DeprecationIssue.Level.WARNING, - "Old index with a compatibility version < " + Version.CURRENT.major + ".0 has been ignored", - "https://ela.st/es-deprecation-9-index-version", - "This read-only index has version: " - + currentCompatibilityVersion.toReleaseVersion() - + " and will be supported as read-only in " - + Version.CURRENT.major - + 1 - + ".0", + DeprecationIssue.Level.CRITICAL, + "Field mappings with incompatible percolator type", + "https://www.elastic.co/guide/en/elasticsearch/reference/8.19/percolator.html#_reindexing_your_percolator_queries", + "The index was created before 8.19 and contains mappings that must be reindexed due to containing percolator fields. " + + String.join(", ", percolatorIncompatibleFieldMappings), false, - Map.of("reindex_required", true) + Map.of("reindex_required", true, "excluded_actions", List.of("readOnly")) ); } + IndexVersion currentCompatibilityVersion = indexMetadata.getCompatibilityVersion(); + if (DeprecatedIndexPredicate.reindexRequired(indexMetadata, true, false)) { + var transforms = transformIdsForIndex(indexMetadata, indexToTransformIds); + if (transforms.isEmpty() == false) { + return new DeprecationIssue( + DeprecationIssue.Level.WARNING, + "One or more Transforms write to this old index with a compatibility version < " + Version.CURRENT.major + ".0", + "https://www.elastic.co/docs/deploy-manage/upgrade/prepare-to-upgrade#transform-migration", + Strings.format( + "This index was created in version [%s] and will be supported as a read-only index in %d.0. The following " + + "transforms are no longer able to write to this index: [%s]. Refer to the migration guide to learn more " + + "about how to handle your transforms destination indices.", + currentCompatibilityVersion.toReleaseVersion(), + Version.CURRENT.major, + String.join(", ", transforms) + ), + false, + Map.of("reindex_required", true, "transform_ids", transforms) + ); + } else { + return new DeprecationIssue( + DeprecationIssue.Level.WARNING, + "Old index with a compatibility version < " + Version.CURRENT.major + ".0 has been ignored", + "https://ela.st/es-deprecation-9-index-version", + "This read-only index has version: " + + currentCompatibilityVersion.toReleaseVersion() + + " and will be supported as read-only in " + + Version.CURRENT.major + + 1 + + ".0", + false, + Map.of("reindex_required", true) + ); + } + } } return null; } @@ -324,71 +361,6 @@ private void fieldLevelMappingIssue(IndexMetadata indexMetadata, BiConsumer findInPropertiesRecursively( - String type, - Map parentMap, - Function, Boolean> predicate, - BiFunction, String> fieldFormatter, - String fieldBeginMarker, - String fieldEndMarker - ) { - List issues = new ArrayList<>(); - Map properties = (Map) parentMap.get("properties"); - if (properties == null) { - return issues; - } - for (Map.Entry entry : properties.entrySet()) { - Map valueMap = (Map) entry.getValue(); - if (predicate.apply(valueMap)) { - issues.add(fieldBeginMarker + fieldFormatter.apply(type, entry) + fieldEndMarker); - } - - Map values = (Map) valueMap.get("fields"); - if (values != null) { - for (Map.Entry multifieldEntry : values.entrySet()) { - Map multifieldValueMap = (Map) multifieldEntry.getValue(); - if (predicate.apply(multifieldValueMap)) { - issues.add( - fieldBeginMarker - + fieldFormatter.apply(type, entry) - + ", multifield: " - + multifieldEntry.getKey() - + fieldEndMarker - ); - } - if (multifieldValueMap.containsKey("properties")) { - issues.addAll( - findInPropertiesRecursively( - type, - multifieldValueMap, - predicate, - fieldFormatter, - fieldBeginMarker, - fieldEndMarker - ) - ); - } - } - } - if (valueMap.containsKey("properties")) { - issues.addAll(findInPropertiesRecursively(type, valueMap, predicate, fieldFormatter, fieldBeginMarker, fieldEndMarker)); - } - } - - return issues; - } - private DeprecationIssue deprecatedCamelCasePattern( IndexMetadata indexMetadata, ClusterState clusterState, @@ -398,7 +370,7 @@ private DeprecationIssue deprecatedCamelCasePattern( fieldLevelMappingIssue( indexMetadata, ((mappingMetadata, sourceAsMap) -> fields.addAll( - findInPropertiesRecursively( + DeprecatedIndexPredicate.findInPropertiesRecursively( mappingMetadata.type(), sourceAsMap, this::isDateFieldWithCamelCasePattern, diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationCheckerTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationCheckerTests.java index 4e169604d37b0..1806cf4dad407 100644 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationCheckerTests.java +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationCheckerTests.java @@ -7,18 +7,21 @@ package org.elasticsearch.xpack.deprecation; +import org.elasticsearch.TransportVersion; import org.elasticsearch.Version; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.DataStreamOptions; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.MappingMetadata; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.metadata.MetadataIndexStateService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.indices.TestIndexNameExpressionResolver; import org.elasticsearch.snapshots.SearchableSnapshotsSettings; import org.elasticsearch.test.ESTestCase; @@ -76,6 +79,55 @@ public void testOldIndicesCheck() { assertThat(issuesByDataStream.get(dataStream.getName()), equalTo(List.of(expected))); } + public void testOldIndicesCheckWithPercolatorField() { + int oldIndexCount = randomIntBetween(1, 100); + int newIndexCount = randomIntBetween(1, 100); + + Map nameToIndexMetadata = new HashMap<>(); + Set expectedIndices = new HashSet<>(); + + MappingMetadata mappingMetadata = new MappingMetadata( + MapperService.SINGLE_MAPPING_NAME, + Map.of("properties", Map.of("query", Map.of("type", "percolator"), "field", Map.of("type", "text"))) + ); + DataStream dataStream = createTestDataStream( + oldIndexCount, + 0, + newIndexCount, + 0, + nameToIndexMetadata, + expectedIndices, + mappingMetadata, + randomBoolean() ? TransportVersion.fromId(0) : TransportVersion.fromId(8840012) + ); + + Metadata metadata = Metadata.builder() + .indices(nameToIndexMetadata) + .dataStreams(Map.of(dataStream.getName(), dataStream), Map.of()) + .build(); + ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(metadata).build(); + + DeprecationIssue expected = new DeprecationIssue( + DeprecationIssue.Level.CRITICAL, + "Field mappings with incompatible percolator type", + "https://www.elastic.co/guide/en/elasticsearch/reference/8.19/percolator.html#_reindexing_your_percolator_queries", + "The data stream was created before 8.19 and contains mappings that must be reindexed due to containing percolator fields.", + false, + ofEntries( + entry("reindex_required", true), + entry("excluded_actions", List.of("readOnly")), + entry("total_backing_indices", oldIndexCount + newIndexCount), + entry("indices_requiring_upgrade_count", expectedIndices.size()), + entry("indices_requiring_upgrade", expectedIndices) + ) + ); + + Map> issuesByDataStream = checker.check(clusterState); + assertThat(issuesByDataStream.size(), equalTo(1)); + assertThat(issuesByDataStream.containsKey(dataStream.getName()), equalTo(true)); + assertThat(issuesByDataStream.get(dataStream.getName()), equalTo(List.of(expected))); + } + public void testOldIndicesCheckWithOnlyNewIndices() { // This tests what happens when any old indices that we have are closed. We expect no deprecation warning. int newOpenIndexCount = randomIntBetween(1, 100); @@ -143,6 +195,26 @@ public void testOldIndicesCheckWithClosedAndOpenIndices() { assertThat(issuesByDataStream.get(dataStream.getName()), equalTo(List.of(expected))); } + private DataStream createTestDataStream( + int oldOpenIndexCount, + int oldClosedIndexCount, + int newOpenIndexCount, + int newClosedIndexCount, + Map nameToIndexMetadata, + Set expectedIndices + ) { + return createTestDataStream( + oldOpenIndexCount, + oldClosedIndexCount, + newOpenIndexCount, + newClosedIndexCount, + nameToIndexMetadata, + expectedIndices, + MappingMetadata.EMPTY_MAPPINGS, + TransportVersion.fromId(0) + ); + } + /* * This creates a test DataStream with the given counts. The nameToIndexMetadata Map and the expectedIndices Set are mutable collections * that will be populated by this method. @@ -153,21 +225,23 @@ private DataStream createTestDataStream( int newOpenIndexCount, int newClosedIndexCount, Map nameToIndexMetadata, - Set expectedIndices + Set expectedIndices, + MappingMetadata mappingMetadata, + TransportVersion oldTransportVersion ) { List allIndices = new ArrayList<>(); for (int i = 0; i < oldOpenIndexCount; i++) { - allIndices.add(createOldIndex(i, false, nameToIndexMetadata, expectedIndices)); + allIndices.add(createOldIndex(i, false, nameToIndexMetadata, expectedIndices, mappingMetadata, oldTransportVersion)); } for (int i = 0; i < oldClosedIndexCount; i++) { - allIndices.add(createOldIndex(i, true, nameToIndexMetadata, expectedIndices)); + allIndices.add(createOldIndex(i, true, nameToIndexMetadata, expectedIndices, mappingMetadata, oldTransportVersion)); } for (int i = 0; i < newOpenIndexCount; i++) { - allIndices.add(createNewIndex(i, false, nameToIndexMetadata)); + allIndices.add(createNewIndex(i, false, nameToIndexMetadata, mappingMetadata)); } for (int i = 0; i < newClosedIndexCount; i++) { - allIndices.add(createNewIndex(i, true, nameToIndexMetadata)); + allIndices.add(createNewIndex(i, true, nameToIndexMetadata, mappingMetadata)); } DataStream dataStream = new DataStream( @@ -193,13 +267,20 @@ private Index createOldIndex( int suffix, boolean isClosed, Map nameToIndexMetadata, - Set expectedIndices + Set expectedIndices, + MappingMetadata mappingMetadata, + TransportVersion transportVersion ) { - return createIndex(true, suffix, isClosed, nameToIndexMetadata, expectedIndices); + return createIndex(true, suffix, isClosed, nameToIndexMetadata, expectedIndices, mappingMetadata, transportVersion); } - private Index createNewIndex(int suffix, boolean isClosed, Map nameToIndexMetadata) { - return createIndex(false, suffix, isClosed, nameToIndexMetadata, null); + private Index createNewIndex( + int suffix, + boolean isClosed, + Map nameToIndexMetadata, + MappingMetadata mappingMetadata + ) { + return createIndex(false, suffix, isClosed, nameToIndexMetadata, null, mappingMetadata, TransportVersion.current()); } private Index createIndex( @@ -207,7 +288,9 @@ private Index createIndex( int suffix, boolean isClosed, Map nameToIndexMetadata, - Set expectedIndices + Set expectedIndices, + MappingMetadata mappingMetadata, + TransportVersion transportVersion ) { Settings.Builder settingsBuilder = isOld ? settings(IndexVersion.fromId(7170099)) : settings(IndexVersion.current()); String indexName = (isOld ? "old-" : "new-") + (isClosed ? "closed-" : "") + "data-stream-index-" + suffix; @@ -221,7 +304,9 @@ private Index createIndex( IndexMetadata.Builder indexMetadataBuilder = IndexMetadata.builder(indexName) .settings(settingsBuilder) .numberOfShards(1) - .numberOfReplicas(0); + .numberOfReplicas(0) + .putMapping(mappingMetadata) + .transportVersion(transportVersion); if (isClosed) { indexMetadataBuilder.state(IndexMetadata.State.CLOSE); } @@ -256,7 +341,7 @@ public void testOldIndicesIgnoredWarningCheck() { } for (int i = 0; i < newIndexCount; i++) { - Index newIndex = createNewIndex(i, false, nameToIndexMetadata); + Index newIndex = createNewIndex(i, false, nameToIndexMetadata, MappingMetadata.EMPTY_MAPPINGS); allIndices.add(newIndex); } @@ -304,6 +389,39 @@ public void testOldIndicesIgnoredWarningCheck() { assertThat(issuesByDataStream.get(dataStream.getName()), equalTo(List.of(expected))); } + public void testOldIndicesIgnoredWithPercolatorField() { + int oldIndexCount = randomIntBetween(1, 100); + int newIndexCount = randomIntBetween(1, 100); + + List allIndices = new ArrayList<>(); + Map nameToIndexMetadata = new HashMap<>(); + Set expectedIndices = new HashSet<>(); + + MappingMetadata mappingMetadata = new MappingMetadata( + MapperService.SINGLE_MAPPING_NAME, + Map.of("properties", Map.of("query", Map.of("type", "percolator"), "field", Map.of("type", "text"))) + ); + + for (int i = 0; i < oldIndexCount; i++) { + Settings.Builder settings = settings(IndexVersion.fromId(8840012)); + + String indexName = "old-data-stream-index-" + i; + settings.put(MetadataIndexStateService.VERIFIED_READ_ONLY_SETTING.getKey(), true); + expectedIndices.add(indexName); + + Settings.Builder settingsBuilder = settings; + IndexMetadata oldIndexMetadata = IndexMetadata.builder(indexName) + .settings(settingsBuilder) + .numberOfShards(1) + .numberOfReplicas(0) + .putMapping(mappingMetadata) + .transportVersion(randomBoolean() ? TransportVersion.fromId(0) : TransportVersion.fromId(8840012)) + .build(); + allIndices.add(oldIndexMetadata.getIndex()); + nameToIndexMetadata.put(oldIndexMetadata.getIndex().getName(), oldIndexMetadata); + } + } + public void testOldSystemDataStreamIgnored() { // We do not want system data streams coming back in the deprecation info API int oldIndexCount = randomIntBetween(1, 100); @@ -325,7 +443,7 @@ public void testOldSystemDataStreamIgnored() { nameToIndexMetadata.put(oldIndexMetadata.getIndex().getName(), oldIndexMetadata); } for (int i = 0; i < newIndexCount; i++) { - Index newIndex = createNewIndex(i, false, nameToIndexMetadata); + Index newIndex = createNewIndex(i, false, nameToIndexMetadata, MappingMetadata.EMPTY_MAPPINGS); allIndices.add(newIndex); } DataStream dataStream = new DataStream( diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationCheckerTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationCheckerTests.java index 98a436e74d2dd..2fbbfb1ce64d4 100644 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationCheckerTests.java +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationCheckerTests.java @@ -10,6 +10,7 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.TransportVersion; import org.elasticsearch.Version; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlocks; @@ -241,6 +242,82 @@ public void testOldIndicesWithIncompatibleDateFormatsCheck() { assertEquals(singletonList(expected), issues); } + public void testOldIndicesCheckWithPercolatorFields() { + IndexVersion olderVersion = IndexVersion.fromId(8840012); + IndexMetadata indexMetadata = IndexMetadata.builder("test") + .settings(settings(olderVersion)) + .numberOfShards(1) + .numberOfReplicas(0) + .state(indexMetdataState) + .putMapping(""" + { + "properties": { + "query": { + "type": "percolator" + }, + "field": { + "type": "text" + } + } + } + """) + .transportVersion(randomBoolean() ? TransportVersion.fromId(0) : TransportVersion.fromId(9000019)) + .build(); + ClusterState clusterState = ClusterState.builder(ClusterState.EMPTY_STATE) + .metadata(Metadata.builder().put(indexMetadata, true)) + .blocks(clusterBlocksForIndices(indexMetadata)) + .build(); + DeprecationIssue expected = new DeprecationIssue( + DeprecationIssue.Level.CRITICAL, + "Field mappings with incompatible percolator type", + "https://www.elastic.co/guide/en/elasticsearch/reference/8.19/percolator.html#_reindexing_your_percolator_queries", + "The index was created before 8.19 and contains mappings that must be reindexed due to containing percolator fields. " + + "Field [query] is of type [_doc]", + false, + Map.of("reindex_required", true, "excluded_actions", List.of("readOnly")) + ); + Map> issuesByIndex = checker.check( + clusterState, + new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS), + emptyPrecomputedData + ); + List issues = issuesByIndex.get("test"); + assertEquals(singletonList(expected), issues); + } + + public void testLatestIndicesCheckWithPercolatorFields() { + IndexVersion latestVersion = IndexVersion.current(); + IndexMetadata indexMetadata = IndexMetadata.builder("test") + .settings(settings(latestVersion)) + .numberOfShards(1) + .numberOfReplicas(0) + .state(indexMetdataState) + .putMapping(""" + { + "properties": { + "query": { + "type": "percolator" + }, + "field": { + "type": "text" + } + } + } + """) + .transportVersion(TransportVersion.current()) + .build(); + ClusterState clusterState = ClusterState.builder(ClusterState.EMPTY_STATE) + .metadata(Metadata.builder().put(indexMetadata, true)) + .blocks(clusterBlocksForIndices(indexMetadata)) + .build(); + Map> issuesByIndex = checker.check( + clusterState, + new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS), + emptyPrecomputedData + ); + assertThat(issuesByIndex.isEmpty(), equalTo(true)); + } + private IndexMetadata indexMetadata(String indexName, IndexVersion indexVersion) { return IndexMetadata.builder(indexName) .settings(settings(indexVersion)) @@ -482,6 +559,49 @@ public void testMultipleOldIndicesIgnoredCheckWithTransforms() { assertEquals(expected, issuesByIndex); } + public void testOldIgnoreIndicesCheckWithPercolatorFields() { + IndexVersion olderVersion = IndexVersion.fromId(8840012); + IndexMetadata indexMetadata = IndexMetadata.builder("test") + .settings(settings(olderVersion).put(MetadataIndexStateService.VERIFIED_READ_ONLY_SETTING.getKey(), true)) + .numberOfShards(1) + .numberOfReplicas(0) + .state(indexMetdataState) + .putMapping(""" + { + "properties": { + "query": { + "type": "percolator" + }, + "field": { + "type": "text" + } + } + } + """) + .transportVersion(TransportVersion.fromId(8840012)) + .build(); + ClusterState clusterState = ClusterState.builder(ClusterState.EMPTY_STATE) + .metadata(Metadata.builder().put(indexMetadata, true)) + .blocks(clusterBlocksForIndices(indexMetadata)) + .build(); + DeprecationIssue expected = new DeprecationIssue( + DeprecationIssue.Level.CRITICAL, + "Field mappings with incompatible percolator type", + "https://www.elastic.co/guide/en/elasticsearch/reference/8.19/percolator.html#_reindexing_your_percolator_queries", + "The index was created before 8.19 and contains mappings that must be reindexed due to containing percolator fields. " + + "Field [query] is of type [_doc]", + false, + Map.of("reindex_required", true, "excluded_actions", List.of("readOnly")) + ); + Map> issuesByIndex = checker.check( + clusterState, + new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS), + emptyPrecomputedData + ); + List issues = issuesByIndex.get("test"); + assertEquals(singletonList(expected), issues); + } + public void testTranslogRetentionSettings() { Settings.Builder settings = settings(IndexVersion.current()); settings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_AGE_SETTING.getKey(), randomPositiveTimeValue()); diff --git a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java index 5dc537fc259d9..a04715e99be7b 100644 --- a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java +++ b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java @@ -275,6 +275,7 @@ public void replaceWatcherIndexWithRandomlyNamedIndex(String originalIndexOrAlia newSettings.remove("index.uuid"); newSettings.remove("index.creation_date"); newSettings.remove("index.version.created"); + newSettings.remove("index.transport_version.created"); assertAcked(indicesAdmin().prepareCreate(to).setMapping(mapping.sourceAsMap()).setSettings(newSettings)); ensureGreen(to); From 69d26ade8d7948a56d79cfca1b71c52e23402ff1 Mon Sep 17 00:00:00 2001 From: Jack Conradson Date: Tue, 18 Nov 2025 08:46:40 -0800 Subject: [PATCH 2/3] fix test --- .../xpack/deprecation/IndexDeprecationCheckerTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationCheckerTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationCheckerTests.java index 2fbbfb1ce64d4..59264c0934027 100644 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationCheckerTests.java +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationCheckerTests.java @@ -261,7 +261,7 @@ public void testOldIndicesCheckWithPercolatorFields() { } } """) - .transportVersion(randomBoolean() ? TransportVersion.fromId(0) : TransportVersion.fromId(9000019)) + .transportVersion(randomBoolean() ? TransportVersion.fromId(0) : TransportVersion.fromId(8840012)) .build(); ClusterState clusterState = ClusterState.builder(ClusterState.EMPTY_STATE) .metadata(Metadata.builder().put(indexMetadata, true)) From 04e463f1dc20dd6b9bbff72fee21ffa006542cf1 Mon Sep 17 00:00:00 2001 From: Jack Conradson Date: Tue, 18 Nov 2025 09:32:06 -0800 Subject: [PATCH 3/3] fix tests --- .../java/org/elasticsearch/cluster/ClusterStateTests.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java b/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java index 270ed7ad6a5a6..ac84a4e868a7f 100644 --- a/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java @@ -275,6 +275,7 @@ public void testToXContent() throws IOException { "indices": { "index": { "version": 1, + "transport_version" : "0", "mapping_version": 1, "settings_version": 1, "aliases_version": 1, @@ -549,6 +550,7 @@ public void testToXContent_FlatSettingTrue_ReduceMappingFalse() throws IOExcepti "indices" : { "index" : { "version" : 1, + "transport_version" : "0", "mapping_version" : 1, "settings_version" : 1, "aliases_version" : 1, @@ -825,6 +827,7 @@ public void testToXContent_FlatSettingFalse_ReduceMappingTrue() throws IOExcepti "indices" : { "index" : { "version" : 1, + "transport_version" : "0", "mapping_version" : 1, "settings_version" : 1, "aliases_version" : 1, @@ -1010,6 +1013,7 @@ public void testToXContentSameTypeName() throws IOException { "indices" : { "index" : { "version" : 2, + "transport_version" : "0", "mapping_version" : 1, "settings_version" : 1, "aliases_version" : 1,