diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java index 2b2dbc9985e42..a5c398c32fab6 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java @@ -152,6 +152,8 @@ public class IndexMetadata implements Diffable, ToXContentFragmen // 'event.ingested' (part of Elastic Common Schema) range is tracked in cluster state, along with @timestamp public static final String EVENT_INGESTED_FIELD_NAME = "event.ingested"; + private static final TransportVersion INDEX_CREATED_TRANSPORT_VERSION = TransportVersion.fromName("index_created_transport_version"); + @Nullable public String getDownsamplingInterval() { return settings.get(IndexMetadata.INDEX_DOWNSAMPLE_INTERVAL_KEY); @@ -540,6 +542,7 @@ public Iterator> settings() { public static final List PARTIALLY_MOUNTED_INDEX_TIER_PREFERENCE = List.of(DataTier.DATA_FROZEN); static final String KEY_VERSION = "version"; + static final String KEY_TRANSPORT_VERSION = "transport_version"; static final String KEY_MAPPING_VERSION = "mapping_version"; static final String KEY_SETTINGS_VERSION = "settings_version"; static final String KEY_ALIASES_VERSION = "aliases_version"; @@ -579,6 +582,7 @@ public Iterator> settings() { private final Index index; private final long version; + private final TransportVersion transportVersion; private final long mappingVersion; @@ -662,6 +666,7 @@ public Iterator> settings() { private IndexMetadata( final Index index, final long version, + final TransportVersion transportVersion, final long mappingVersion, final long settingsVersion, final long aliasesVersion, @@ -710,6 +715,7 @@ private IndexMetadata( ) { this.index = index; this.version = version; + this.transportVersion = transportVersion; assert mappingVersion >= 0 : mappingVersion; this.mappingVersion = mappingVersion; this.mappingsUpdatedVersion = mappingsUpdatedVersion; @@ -774,6 +780,7 @@ IndexMetadata withMappingMetadata(MappingMetadata mapping) { return new IndexMetadata( this.index, this.version, + this.transportVersion, this.mappingVersion, this.settingsVersion, this.aliasesVersion, @@ -835,6 +842,7 @@ public IndexMetadata withInSyncAllocationIds(int shardId, Set inSyncSet) return new IndexMetadata( this.index, this.version, + this.transportVersion, this.mappingVersion, this.settingsVersion, this.aliasesVersion, @@ -894,6 +902,7 @@ public IndexMetadata withIncrementedPrimaryTerm(int shardId) { return new IndexMetadata( this.index, this.version, + this.transportVersion, this.mappingVersion, this.settingsVersion, this.aliasesVersion, @@ -963,6 +972,7 @@ public IndexMetadata withTimestampRanges( return new IndexMetadata( this.index, this.version, + this.transportVersion, this.mappingVersion, this.settingsVersion, this.aliasesVersion, @@ -1018,6 +1028,7 @@ public IndexMetadata withIncrementedVersion() { return new IndexMetadata( this.index, this.version + 1, + this.transportVersion, this.mappingVersion, this.settingsVersion, this.aliasesVersion, @@ -1078,6 +1089,10 @@ public long getVersion() { return this.version; } + public TransportVersion getTransportVersion() { + return transportVersion; + } + public long getMappingVersion() { return mappingVersion; } @@ -1533,6 +1548,7 @@ private static class IndexMetadataDiff implements Diff { private final String index; private final int routingNumShards; private final long version; + private final TransportVersion transportVersion; private final long mappingVersion; private final long settingsVersion; private final long aliasesVersion; @@ -1565,6 +1581,7 @@ private static class IndexMetadataDiff implements Diff { IndexMetadataDiff(IndexMetadata before, IndexMetadata after) { index = after.index.getName(); version = after.version; + transportVersion = after.transportVersion; mappingVersion = after.mappingVersion; settingsVersion = after.settingsVersion; aliasesVersion = after.aliasesVersion; @@ -1617,6 +1634,9 @@ private static class IndexMetadataDiff implements Diff { index = in.readString(); routingNumShards = in.readInt(); version = in.readLong(); + transportVersion = in.getTransportVersion().supports(INDEX_CREATED_TRANSPORT_VERSION) + ? TransportVersion.readVersion(in) + : TransportVersion.fromId(0); mappingVersion = in.readVLong(); settingsVersion = in.readVLong(); if (in.getTransportVersion().onOrAfter(TransportVersions.V_7_2_0)) { @@ -1687,6 +1707,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(index); out.writeInt(routingNumShards); out.writeLong(version); + if (out.getTransportVersion().supports(INDEX_CREATED_TRANSPORT_VERSION)) { + TransportVersion.writeVersion(transportVersion, out); + } out.writeVLong(mappingVersion); out.writeVLong(settingsVersion); if (out.getTransportVersion().onOrAfter(TransportVersions.V_7_2_0)) { @@ -1733,6 +1756,7 @@ public void writeTo(StreamOutput out) throws IOException { public IndexMetadata apply(IndexMetadata part) { Builder builder = builder(index); builder.version(version); + builder.transportVersion(transportVersion); builder.mappingVersion(mappingVersion); builder.settingsVersion(settingsVersion); builder.aliasesVersion(aliasesVersion); @@ -1775,6 +1799,11 @@ public static IndexMetadata readFrom(StreamInput in) throws IOException { public static IndexMetadata readFrom(StreamInput in, @Nullable Function mappingLookup) throws IOException { Builder builder = new Builder(in.readString()); builder.version(in.readLong()); + builder.transportVersion( + in.getTransportVersion().supports(INDEX_CREATED_TRANSPORT_VERSION) + ? TransportVersion.readVersion(in) + : TransportVersion.fromId(0) + ); builder.mappingVersion(in.readVLong()); builder.settingsVersion(in.readVLong()); if (in.getTransportVersion().onOrAfter(TransportVersions.V_7_2_0)) { @@ -1847,6 +1876,9 @@ public static IndexMetadata readFrom(StreamInput in, @Nullable Function builder.state(State.fromString(parser.text())); case KEY_VERSION -> builder.version(parser.longValue()); + case KEY_TRANSPORT_VERSION -> builder.transportVersion(TransportVersion.fromString(parser.text())); case KEY_MAPPING_VERSION -> { mappingVersion = true; builder.mappingVersion(parser.longValue()); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index 81142c13a5bac..6d93fd4495a1e 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -588,6 +588,7 @@ private IndexMetadata buildAndValidateTemporaryIndexMetadata( final IndexMetadata.Builder tmpImdBuilder = IndexMetadata.builder(request.index()); tmpImdBuilder.setRoutingNumShards(routingNumShards); tmpImdBuilder.settings(indexSettings); + tmpImdBuilder.transportVersion(TransportVersion.current()); tmpImdBuilder.system(isSystem); // Set up everything, now locally create the index to see that things are ok, and apply diff --git a/server/src/main/resources/transport/definitions/referable/index_created_transport_version.csv b/server/src/main/resources/transport/definitions/referable/index_created_transport_version.csv new file mode 100644 index 0000000000000..f25ff514b4116 --- /dev/null +++ b/server/src/main/resources/transport/definitions/referable/index_created_transport_version.csv @@ -0,0 +1 @@ +9221000,9185009,9112014,8841075 diff --git a/server/src/main/resources/transport/upper_bounds/8.19.csv b/server/src/main/resources/transport/upper_bounds/8.19.csv index febec42efcb5b..09daf16060da0 100644 --- a/server/src/main/resources/transport/upper_bounds/8.19.csv +++ b/server/src/main/resources/transport/upper_bounds/8.19.csv @@ -1 +1 @@ -batched_response_might_include_reduction_failure,8841074 +index_created_transport_version,8841075 diff --git a/server/src/main/resources/transport/upper_bounds/9.1.csv b/server/src/main/resources/transport/upper_bounds/9.1.csv index 80b97d85f7511..37a8af514597a 100644 --- a/server/src/main/resources/transport/upper_bounds/9.1.csv +++ b/server/src/main/resources/transport/upper_bounds/9.1.csv @@ -1 +1 @@ -transform_check_for_dangling_tasks,9112009 +index_created_transport_version,9112014 diff --git a/server/src/main/resources/transport/upper_bounds/9.2.csv b/server/src/main/resources/transport/upper_bounds/9.2.csv index 2c15e0254cbe8..0b2d3c451ce12 100644 --- a/server/src/main/resources/transport/upper_bounds/9.2.csv +++ b/server/src/main/resources/transport/upper_bounds/9.2.csv @@ -1 +1 @@ -transform_check_for_dangling_tasks,9170000 +index_created_transport_version,9185009 diff --git a/server/src/main/resources/transport/upper_bounds/9.3.csv b/server/src/main/resources/transport/upper_bounds/9.3.csv new file mode 100644 index 0000000000000..cc71afa89aa4a --- /dev/null +++ b/server/src/main/resources/transport/upper_bounds/9.3.csv @@ -0,0 +1 @@ +index_created_transport_version,9221000 diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java index 67e5f30f023c9..f14c947da83fd 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java @@ -162,6 +162,7 @@ public void testToXContentWithDeprecatedClusterState() { "indices": { "index": { "version": 1, + "transport_version" : "0", "mapping_version": 1, "settings_version": 1, "aliases_version": 1, @@ -251,6 +252,7 @@ public void testToXContentWithDeprecatedClusterStateAndMetadata() { "indices" : { "index" : { "version" : 1, + "transport_version" : "0", "mapping_version" : 1, "settings_version" : 1, "aliases_version" : 1, diff --git a/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java b/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java index 270ed7ad6a5a6..ac84a4e868a7f 100644 --- a/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java @@ -275,6 +275,7 @@ public void testToXContent() throws IOException { "indices": { "index": { "version": 1, + "transport_version" : "0", "mapping_version": 1, "settings_version": 1, "aliases_version": 1, @@ -549,6 +550,7 @@ public void testToXContent_FlatSettingTrue_ReduceMappingFalse() throws IOExcepti "indices" : { "index" : { "version" : 1, + "transport_version" : "0", "mapping_version" : 1, "settings_version" : 1, "aliases_version" : 1, @@ -825,6 +827,7 @@ public void testToXContent_FlatSettingFalse_ReduceMappingTrue() throws IOExcepti "indices" : { "index" : { "version" : 1, + "transport_version" : "0", "mapping_version" : 1, "settings_version" : 1, "aliases_version" : 1, @@ -1010,6 +1013,7 @@ public void testToXContentSameTypeName() throws IOException { "indices" : { "index" : { "version" : 2, + "transport_version" : "0", "mapping_version" : 1, "settings_version" : 1, "aliases_version" : 1, diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java index d83e046436ec0..2dac4d5eb9d5e 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java @@ -100,6 +100,7 @@ public void testIndexMetadataSerialization() throws IOException { IndexMetadata metadata = IndexMetadata.builder("foo") .settings(indexSettings(numShard, numberOfReplicas).put("index.version.created", 1)) + .transportVersion(TransportVersion.current()) .creationDate(randomLong()) .primaryTerm(0, 2) .setRoutingNumShards(32) @@ -150,6 +151,7 @@ public void testIndexMetadataSerialization() throws IOException { ); assertEquals(metadata.hashCode(), fromXContentMeta.hashCode()); + assertEquals(metadata.getTransportVersion(), fromXContentMeta.getTransportVersion()); assertEquals(metadata.getNumberOfReplicas(), fromXContentMeta.getNumberOfReplicas()); assertEquals(metadata.getNumberOfShards(), fromXContentMeta.getNumberOfShards()); assertEquals(metadata.getCreationVersion(), fromXContentMeta.getCreationVersion()); @@ -176,6 +178,7 @@ public void testIndexMetadataSerialization() throws IOException { assertEquals(metadata, deserialized); assertEquals(metadata.hashCode(), deserialized.hashCode()); + assertEquals(metadata.getTransportVersion(), deserialized.getTransportVersion()); assertEquals(metadata.getNumberOfReplicas(), deserialized.getNumberOfReplicas()); assertEquals(metadata.getNumberOfShards(), deserialized.getNumberOfShards()); assertEquals(metadata.getCreationVersion(), deserialized.getCreationVersion()); @@ -211,6 +214,7 @@ public void testIndexMetadataFromXContentParsingWithoutEventIngestedField() thro IndexMetadata metadata = IndexMetadata.builder("foo") .settings(indexSettings(numShard, numberOfReplicas).put("index.version.created", 1)) + .transportVersion(TransportVersion.current()) .creationDate(randomLong()) .primaryTerm(0, 2) .setRoutingNumShards(32) @@ -279,6 +283,7 @@ public void testIndexMetadataFromXContentParsingWithoutEventIngestedField() thro fromXContentMeta ); assertEquals(metadata.hashCode(), fromXContentMeta.hashCode()); + assertEquals(metadata.getTransportVersion(), fromXContentMeta.getTransportVersion()); assertEquals(metadata.getNumberOfReplicas(), fromXContentMeta.getNumberOfReplicas()); assertEquals(metadata.getNumberOfShards(), fromXContentMeta.getNumberOfShards()); assertEquals(metadata.getCreationVersion(), fromXContentMeta.getCreationVersion()); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java index 36e6c586a6fde..8201a168c3cb8 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java @@ -323,6 +323,7 @@ public void testToXContentAPI_SameTypeName() throws IOException { "indices" : { "index" : { "version" : 2, + "transport_version" : "0", "mapping_version" : 1, "settings_version" : 1, "aliases_version" : 1, @@ -490,6 +491,7 @@ public void testToXContentAPI_FlatSettingTrue_ReduceMappingFalse() throws IOExce "indices" : { "index" : { "version" : 2, + "transport_version" : "0", "mapping_version" : 1, "settings_version" : 1, "aliases_version" : 1, @@ -600,6 +602,7 @@ public void testToXContentAPI_FlatSettingFalse_ReduceMappingTrue() throws IOExce "indices" : { "index" : { "version" : 2, + "transport_version" : "0", "mapping_version" : 1, "settings_version" : 1, "aliases_version" : 1, @@ -740,6 +743,7 @@ public void testToXContentAPIReservedMetadata() throws IOException { "indices" : { "index" : { "version" : 2, + "transport_version" : "0", "mapping_version" : 1, "settings_version" : 1, "aliases_version" : 1, diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/DeprecatedIndexPredicate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/DeprecatedIndexPredicate.java index b5967e6b1860c..d66d7c41dda91 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/DeprecatedIndexPredicate.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/DeprecatedIndexPredicate.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.core.deprecation; +import org.elasticsearch.TransportVersion; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.metadata.MetadataIndexStateService; @@ -14,6 +15,11 @@ import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.function.BiFunction; +import java.util.function.Function; import java.util.function.Predicate; public class DeprecatedIndexPredicate { @@ -49,7 +55,7 @@ public static Predicate getReindexRequiredPredicate(Metadata metadata, bo * if false, only those without a block are returned * @param includeSystem if true, all indices including system will be returned, * if false, only non-system indices are returned - * @return a predicate that returns true for indices that need to be reindexed + * @return returns true for indices that need to be reindexed */ public static boolean reindexRequired(IndexMetadata indexMetadata, boolean filterToBlockedStatus, boolean includeSystem) { return creationVersionBeforeMinimumWritableVersion(indexMetadata) @@ -58,6 +64,59 @@ && isNotSearchableSnapshot(indexMetadata) && matchBlockedStatus(indexMetadata, filterToBlockedStatus); } + /** + * This method checks if this index is on the current transport version for the current minor release version. + * + * @param indexMetadata the index metadata + * @param filterToBlockedStatus if true, only indices that are write blocked will be returned, + * if false, only those without a block are returned + * @param includeSystem if true, all indices including system will be returned, + * if false, only non-system indices are returned + * @return returns true for indices that need to be reindexed + */ + public static boolean reindexRequiredForTransportVersion( + IndexMetadata indexMetadata, + boolean filterToBlockedStatus, + boolean includeSystem + ) { + return transportVersionBeforeCurrentMinorRelease(indexMetadata) + && (includeSystem || isNotSystem(indexMetadata)) + && isNotSearchableSnapshot(indexMetadata) + && matchBlockedStatus(indexMetadata, filterToBlockedStatus); + } + + /** + * This method checks if this index requires reindexing based on if it has percolator fields older than the current transport version + * for the current minor release. + * + * @param indexMetadata the index metadata + * @param filterToBlockedStatus if true, only indices that are write blocked will be returned, + * if false, only those without a block are returned + * @param includeSystem if true, all indices including system will be returned, + * if false, only non-system indices are returned + * @return returns a message as a string for each incompatible percolator field found + */ + public static List reindexRequiredForPecolatorFields( + IndexMetadata indexMetadata, + boolean filterToBlockedStatus, + boolean includeSystem + ) { + List percolatorIncompatibleFieldMappings = new ArrayList<>(); + if (reindexRequiredForTransportVersion(indexMetadata, filterToBlockedStatus, includeSystem) && indexMetadata.mapping() != null) { + percolatorIncompatibleFieldMappings.addAll( + findInPropertiesRecursively( + indexMetadata.mapping().type(), + indexMetadata.mapping().sourceAsMap(), + property -> "percolator".equals(property.get("type")), + (type, entry) -> "Field [" + entry.getKey() + "] is of type [" + indexMetadata.mapping().type() + "]", + "", + "" + ) + ); + } + return percolatorIncompatibleFieldMappings; + } + private static boolean isNotSystem(IndexMetadata indexMetadata) { return indexMetadata.isSystem() == false; } @@ -73,4 +132,74 @@ private static boolean creationVersionBeforeMinimumWritableVersion(IndexMetadata private static boolean matchBlockedStatus(IndexMetadata indexMetadata, boolean filterToBlockedStatus) { return MetadataIndexStateService.VERIFIED_READ_ONLY_SETTING.get(indexMetadata.getSettings()) == filterToBlockedStatus; } + + private static boolean transportVersionBeforeCurrentMinorRelease(IndexMetadata indexMetadata) { + // We divide each transport version by 1000 to get the base id. + return indexMetadata.getTransportVersion().id() / 1000 < TransportVersion.current().id() / 1000; + } + + /** + * iterates through the "properties" field of mappings and returns any predicates that match in the + * form of issue-strings. + * + * @param type the document type + * @param parentMap the mapping to read properties from + * @param predicate the predicate to check against for issues, issue is returned if predicate evaluates to true + * @param fieldFormatter a function that takes a type and mapping field entry and returns a formatted field representation + * @return a list of issues found in fields + */ + @SuppressWarnings("unchecked") + public static List findInPropertiesRecursively( + String type, + Map parentMap, + Function, Boolean> predicate, + BiFunction, String> fieldFormatter, + String fieldBeginMarker, + String fieldEndMarker + ) { + List issues = new ArrayList<>(); + Map properties = (Map) parentMap.get("properties"); + if (properties == null) { + return issues; + } + for (Map.Entry entry : properties.entrySet()) { + Map valueMap = (Map) entry.getValue(); + if (predicate.apply(valueMap)) { + issues.add(fieldBeginMarker + fieldFormatter.apply(type, entry) + fieldEndMarker); + } + + Map values = (Map) valueMap.get("fields"); + if (values != null) { + for (Map.Entry multifieldEntry : values.entrySet()) { + Map multifieldValueMap = (Map) multifieldEntry.getValue(); + if (predicate.apply(multifieldValueMap)) { + issues.add( + fieldBeginMarker + + fieldFormatter.apply(type, entry) + + ", multifield: " + + multifieldEntry.getKey() + + fieldEndMarker + ); + } + if (multifieldValueMap.containsKey("properties")) { + issues.addAll( + findInPropertiesRecursively( + type, + multifieldValueMap, + predicate, + fieldFormatter, + fieldBeginMarker, + fieldEndMarker + ) + ); + } + } + } + if (valueMap.containsKey("properties")) { + issues.addAll(findInPropertiesRecursively(type, valueMap, predicate, fieldFormatter, fieldBeginMarker, fieldEndMarker)); + } + } + + return issues; + } } diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationChecker.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationChecker.java index b63e4359896f5..887a0b603eda3 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationChecker.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationChecker.java @@ -88,9 +88,24 @@ public Map> check(ClusterState clusterState) { static DeprecationIssue oldIndicesCheck(DataStream dataStream, ClusterState clusterState) { List backingIndices = dataStream.getIndices(); - + Set percolatorIndicesNeedingUpgrade = getReindexRequiredIndicesWithPercolatorFields(backingIndices, clusterState, false); + if (percolatorIndicesNeedingUpgrade.isEmpty() == false) { + return new DeprecationIssue( + DeprecationIssue.Level.CRITICAL, + "Field mappings with incompatible percolator type", + "https://www.elastic.co/guide/en/elasticsearch/reference/8.19/percolator.html#_reindexing_your_percolator_queries", + "The data stream was created before 8.19 and contains mappings that must be reindexed due to containing percolator fields.", + false, + ofEntries( + entry("reindex_required", true), + entry("excluded_actions", List.of("readOnly")), + entry("total_backing_indices", backingIndices.size()), + entry("indices_requiring_upgrade_count", percolatorIndicesNeedingUpgrade.size()), + entry("indices_requiring_upgrade", percolatorIndicesNeedingUpgrade) + ) + ); + } Set indicesNeedingUpgrade = getReindexRequiredIndices(backingIndices, clusterState, false); - if (indicesNeedingUpgrade.isEmpty() == false) { return new DeprecationIssue( DeprecationIssue.Level.CRITICAL, @@ -112,6 +127,23 @@ static DeprecationIssue oldIndicesCheck(DataStream dataStream, ClusterState clus static DeprecationIssue ignoredOldIndicesCheck(DataStream dataStream, ClusterState clusterState) { List backingIndices = dataStream.getIndices(); + Set percolatorIgnoredIndices = getReindexRequiredIndicesWithPercolatorFields(backingIndices, clusterState, true); + if (percolatorIgnoredIndices.isEmpty() == false) { + return new DeprecationIssue( + DeprecationIssue.Level.CRITICAL, + "Field mappings with incompatible percolator type", + "https://www.elastic.co/guide/en/elasticsearch/reference/8.19/percolator.html#_reindexing_your_percolator_queries", + "The data stream was created before 8.19 and contains mappings that must be reindexed due to containing percolator fields.", + false, + ofEntries( + entry("reindex_required", true), + entry("excluded_actions", List.of("readOnly")), + entry("total_backing_indices", backingIndices.size()), + entry("ignored_indices_requiring_upgrade_count", percolatorIgnoredIndices.size()), + entry("ignored_indices_requiring_upgrade", percolatorIgnoredIndices) + ) + ); + } Set ignoredIndices = getReindexRequiredIndices(backingIndices, clusterState, true); if (ignoredIndices.isEmpty() == false) { return new DeprecationIssue( @@ -144,6 +176,23 @@ private static Set getReindexRequiredIndices( .collect(Collectors.toUnmodifiableSet()); } + private static Set getReindexRequiredIndicesWithPercolatorFields( + List backingIndices, + ClusterState clusterState, + boolean filterToBlockedStatus + ) { + return backingIndices.stream() + .filter( + index -> DeprecatedIndexPredicate.reindexRequiredForPecolatorFields( + clusterState.metadata().index(index), + filterToBlockedStatus, + false + ).isEmpty() == false + ) + .map(Index::getName) + .collect(Collectors.toUnmodifiableSet()); + } + @Override public String getName() { return NAME; diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecker.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecker.java index 1799edcf56a75..b824b595a05fb 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecker.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecker.java @@ -31,8 +31,6 @@ import java.util.Map; import java.util.Objects; import java.util.function.BiConsumer; -import java.util.function.BiFunction; -import java.util.function.Function; import java.util.stream.Collectors; import static org.elasticsearch.xpack.deprecation.LegacyTiersDetection.DEPRECATION_COMMON_DETAIL; @@ -96,62 +94,82 @@ private DeprecationIssue oldIndicesCheck( ClusterState clusterState, Map> indexToTransformIds ) { - // TODO: this check needs to be revised. It's trivially true right now. - IndexVersion currentCompatibilityVersion = indexMetadata.getCompatibilityVersion(); // We intentionally exclude indices that are in data streams because they will be picked up by DataStreamDeprecationChecks - if (DeprecatedIndexPredicate.reindexRequired(indexMetadata, false, false) && isNotDataStreamIndex(indexMetadata, clusterState)) { - List cldrIncompatibleFieldMappings = new ArrayList<>(); - fieldLevelMappingIssue( + if (isNotDataStreamIndex(indexMetadata, clusterState)) { + // We check for percolator indices first as that will include potentially older indices as well. + List percolatorIncompatibleFieldMappings = DeprecatedIndexPredicate.reindexRequiredForPecolatorFields( indexMetadata, - (mappingMetadata, sourceAsMap) -> cldrIncompatibleFieldMappings.addAll( - findInPropertiesRecursively( - mappingMetadata.type(), - sourceAsMap, - this::isDateFieldWithCompatFormatPattern, - this::cldrIncompatibleFormatPattern, - "", - "" - ) - ) + false, + false ); - - var transforms = transformIdsForIndex(indexMetadata, indexToTransformIds); - if (transforms.isEmpty() == false) { - return new DeprecationIssue( - DeprecationIssue.Level.CRITICAL, - "One or more Transforms write to this index with a compatibility version < " + Version.CURRENT.major + ".0", - "https://www.elastic.co/docs/deploy-manage/upgrade/prepare-to-upgrade#transform-migration", - Strings.format( - "This index was created in version [%s] and requires action before upgrading to %d.0. The following transforms are " - + "configured to write to this index: [%s]. Refer to the migration guide to learn more about how to prepare " - + "transforms destination indices for your upgrade.", - currentCompatibilityVersion.toReleaseVersion(), - Version.CURRENT.major, - String.join(", ", transforms) - ), - false, - Map.of("reindex_required", true, "transform_ids", transforms) - ); - } else if (cldrIncompatibleFieldMappings.isEmpty() == false) { + if (percolatorIncompatibleFieldMappings.isEmpty() == false) { return new DeprecationIssue( DeprecationIssue.Level.CRITICAL, - "Field mappings with incompatible date format patterns in old index", - "https://www.elastic.co/blog/locale-changes-elasticsearch-8-16-jdk-23", - "The index was created before 8.0 and contains mappings that must be reindexed due to locale changes in 8.16+. " - + "Manual reindexing is required. " - + String.join(", ", cldrIncompatibleFieldMappings), + "Field mappings with incompatible percolator type", + "https://www.elastic.co/guide/en/elasticsearch/reference/8.19/percolator.html#_reindexing_your_percolator_queries", + "The index was created before 8.19 and contains mappings that must be reindexed due to containing percolator fields. " + + String.join(", ", percolatorIncompatibleFieldMappings), false, - null + Map.of("reindex_required", true, "excluded_actions", List.of("readOnly")) ); - } else { - return new DeprecationIssue( - DeprecationIssue.Level.CRITICAL, - "Old index with a compatibility version < " + Version.CURRENT.major + ".0", - "https://ela.st/es-deprecation-9-index-version", - "This index has version: " + currentCompatibilityVersion.toReleaseVersion(), - false, - Map.of("reindex_required", true) + } + + // TODO: this check needs to be revised. It's trivially true right now. + IndexVersion currentCompatibilityVersion = indexMetadata.getCompatibilityVersion(); + if (DeprecatedIndexPredicate.reindexRequired(indexMetadata, false, false)) { + List cldrIncompatibleFieldMappings = new ArrayList<>(); + fieldLevelMappingIssue( + indexMetadata, + (mappingMetadata, sourceAsMap) -> cldrIncompatibleFieldMappings.addAll( + DeprecatedIndexPredicate.findInPropertiesRecursively( + mappingMetadata.type(), + sourceAsMap, + this::isDateFieldWithCompatFormatPattern, + this::cldrIncompatibleFormatPattern, + "", + "" + ) + ) ); + + var transforms = transformIdsForIndex(indexMetadata, indexToTransformIds); + if (transforms.isEmpty() == false) { + return new DeprecationIssue( + DeprecationIssue.Level.CRITICAL, + "One or more Transforms write to this index with a compatibility version < " + Version.CURRENT.major + ".0", + "https://www.elastic.co/docs/deploy-manage/upgrade/prepare-to-upgrade#transform-migration", + Strings.format( + "This index was created in version [%s] and requires action before upgrading to %d.0. The following " + + "transforms are configured to write to this index: [%s]. Refer to the migration guide to learn more " + + "about how to prepare transforms destination indices for your upgrade.", + currentCompatibilityVersion.toReleaseVersion(), + Version.CURRENT.major, + String.join(", ", transforms) + ), + false, + Map.of("reindex_required", true, "transform_ids", transforms) + ); + } else if (cldrIncompatibleFieldMappings.isEmpty() == false) { + return new DeprecationIssue( + DeprecationIssue.Level.CRITICAL, + "Field mappings with incompatible date format patterns in old index", + "https://www.elastic.co/blog/locale-changes-elasticsearch-8-16-jdk-23", + "The index was created before 8.0 and contains mappings that must be reindexed due to locale changes in 8.16+. " + + "Manual reindexing is required. " + + String.join(", ", cldrIncompatibleFieldMappings), + false, + null + ); + } else { + return new DeprecationIssue( + DeprecationIssue.Level.CRITICAL, + "Old index with a compatibility version < " + Version.CURRENT.major + ".0", + "https://ela.st/es-deprecation-9-index-version", + "This index has version: " + currentCompatibilityVersion.toReleaseVersion(), + false, + Map.of("reindex_required", true) + ); + } } } return null; @@ -166,41 +184,60 @@ private DeprecationIssue ignoredOldIndicesCheck( ClusterState clusterState, Map> indexToTransformIds ) { - IndexVersion currentCompatibilityVersion = indexMetadata.getCompatibilityVersion(); // We intentionally exclude indices that are in data streams because they will be picked up by DataStreamDeprecationChecks - if (DeprecatedIndexPredicate.reindexRequired(indexMetadata, true, false) && isNotDataStreamIndex(indexMetadata, clusterState)) { - var transforms = transformIdsForIndex(indexMetadata, indexToTransformIds); - if (transforms.isEmpty() == false) { - return new DeprecationIssue( - DeprecationIssue.Level.WARNING, - "One or more Transforms write to this old index with a compatibility version < " + Version.CURRENT.major + ".0", - "https://www.elastic.co/docs/deploy-manage/upgrade/prepare-to-upgrade#transform-migration", - Strings.format( - "This index was created in version [%s] and will be supported as a read-only index in %d.0. The following " - + "transforms are no longer able to write to this index: [%s]. Refer to the migration guide to learn more " - + "about how to handle your transforms destination indices.", - currentCompatibilityVersion.toReleaseVersion(), - Version.CURRENT.major, - String.join(", ", transforms) - ), - false, - Map.of("reindex_required", true, "transform_ids", transforms) - ); - } else { + if (isNotDataStreamIndex(indexMetadata, clusterState)) { + // We check for percolator indices first as that will include potentially older indices as well. + List percolatorIncompatibleFieldMappings = DeprecatedIndexPredicate.reindexRequiredForPecolatorFields( + indexMetadata, + true, + false + ); + if (percolatorIncompatibleFieldMappings.isEmpty() == false) { return new DeprecationIssue( - DeprecationIssue.Level.WARNING, - "Old index with a compatibility version < " + Version.CURRENT.major + ".0 has been ignored", - "https://ela.st/es-deprecation-9-index-version", - "This read-only index has version: " - + currentCompatibilityVersion.toReleaseVersion() - + " and will be supported as read-only in " - + Version.CURRENT.major - + 1 - + ".0", + DeprecationIssue.Level.CRITICAL, + "Field mappings with incompatible percolator type", + "https://www.elastic.co/guide/en/elasticsearch/reference/8.19/percolator.html#_reindexing_your_percolator_queries", + "The index was created before 8.19 and contains mappings that must be reindexed due to containing percolator fields. " + + String.join(", ", percolatorIncompatibleFieldMappings), false, - Map.of("reindex_required", true) + Map.of("reindex_required", true, "excluded_actions", List.of("readOnly")) ); } + IndexVersion currentCompatibilityVersion = indexMetadata.getCompatibilityVersion(); + if (DeprecatedIndexPredicate.reindexRequired(indexMetadata, true, false)) { + var transforms = transformIdsForIndex(indexMetadata, indexToTransformIds); + if (transforms.isEmpty() == false) { + return new DeprecationIssue( + DeprecationIssue.Level.WARNING, + "One or more Transforms write to this old index with a compatibility version < " + Version.CURRENT.major + ".0", + "https://www.elastic.co/docs/deploy-manage/upgrade/prepare-to-upgrade#transform-migration", + Strings.format( + "This index was created in version [%s] and will be supported as a read-only index in %d.0. The following " + + "transforms are no longer able to write to this index: [%s]. Refer to the migration guide to learn more " + + "about how to handle your transforms destination indices.", + currentCompatibilityVersion.toReleaseVersion(), + Version.CURRENT.major, + String.join(", ", transforms) + ), + false, + Map.of("reindex_required", true, "transform_ids", transforms) + ); + } else { + return new DeprecationIssue( + DeprecationIssue.Level.WARNING, + "Old index with a compatibility version < " + Version.CURRENT.major + ".0 has been ignored", + "https://ela.st/es-deprecation-9-index-version", + "This read-only index has version: " + + currentCompatibilityVersion.toReleaseVersion() + + " and will be supported as read-only in " + + Version.CURRENT.major + + 1 + + ".0", + false, + Map.of("reindex_required", true) + ); + } + } } return null; } @@ -324,71 +361,6 @@ private void fieldLevelMappingIssue(IndexMetadata indexMetadata, BiConsumer findInPropertiesRecursively( - String type, - Map parentMap, - Function, Boolean> predicate, - BiFunction, String> fieldFormatter, - String fieldBeginMarker, - String fieldEndMarker - ) { - List issues = new ArrayList<>(); - Map properties = (Map) parentMap.get("properties"); - if (properties == null) { - return issues; - } - for (Map.Entry entry : properties.entrySet()) { - Map valueMap = (Map) entry.getValue(); - if (predicate.apply(valueMap)) { - issues.add(fieldBeginMarker + fieldFormatter.apply(type, entry) + fieldEndMarker); - } - - Map values = (Map) valueMap.get("fields"); - if (values != null) { - for (Map.Entry multifieldEntry : values.entrySet()) { - Map multifieldValueMap = (Map) multifieldEntry.getValue(); - if (predicate.apply(multifieldValueMap)) { - issues.add( - fieldBeginMarker - + fieldFormatter.apply(type, entry) - + ", multifield: " - + multifieldEntry.getKey() - + fieldEndMarker - ); - } - if (multifieldValueMap.containsKey("properties")) { - issues.addAll( - findInPropertiesRecursively( - type, - multifieldValueMap, - predicate, - fieldFormatter, - fieldBeginMarker, - fieldEndMarker - ) - ); - } - } - } - if (valueMap.containsKey("properties")) { - issues.addAll(findInPropertiesRecursively(type, valueMap, predicate, fieldFormatter, fieldBeginMarker, fieldEndMarker)); - } - } - - return issues; - } - private DeprecationIssue deprecatedCamelCasePattern( IndexMetadata indexMetadata, ClusterState clusterState, @@ -398,7 +370,7 @@ private DeprecationIssue deprecatedCamelCasePattern( fieldLevelMappingIssue( indexMetadata, ((mappingMetadata, sourceAsMap) -> fields.addAll( - findInPropertiesRecursively( + DeprecatedIndexPredicate.findInPropertiesRecursively( mappingMetadata.type(), sourceAsMap, this::isDateFieldWithCamelCasePattern, diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationCheckerTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationCheckerTests.java index 4e169604d37b0..1806cf4dad407 100644 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationCheckerTests.java +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationCheckerTests.java @@ -7,18 +7,21 @@ package org.elasticsearch.xpack.deprecation; +import org.elasticsearch.TransportVersion; import org.elasticsearch.Version; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.DataStreamOptions; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.MappingMetadata; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.metadata.MetadataIndexStateService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.indices.TestIndexNameExpressionResolver; import org.elasticsearch.snapshots.SearchableSnapshotsSettings; import org.elasticsearch.test.ESTestCase; @@ -76,6 +79,55 @@ public void testOldIndicesCheck() { assertThat(issuesByDataStream.get(dataStream.getName()), equalTo(List.of(expected))); } + public void testOldIndicesCheckWithPercolatorField() { + int oldIndexCount = randomIntBetween(1, 100); + int newIndexCount = randomIntBetween(1, 100); + + Map nameToIndexMetadata = new HashMap<>(); + Set expectedIndices = new HashSet<>(); + + MappingMetadata mappingMetadata = new MappingMetadata( + MapperService.SINGLE_MAPPING_NAME, + Map.of("properties", Map.of("query", Map.of("type", "percolator"), "field", Map.of("type", "text"))) + ); + DataStream dataStream = createTestDataStream( + oldIndexCount, + 0, + newIndexCount, + 0, + nameToIndexMetadata, + expectedIndices, + mappingMetadata, + randomBoolean() ? TransportVersion.fromId(0) : TransportVersion.fromId(8840012) + ); + + Metadata metadata = Metadata.builder() + .indices(nameToIndexMetadata) + .dataStreams(Map.of(dataStream.getName(), dataStream), Map.of()) + .build(); + ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(metadata).build(); + + DeprecationIssue expected = new DeprecationIssue( + DeprecationIssue.Level.CRITICAL, + "Field mappings with incompatible percolator type", + "https://www.elastic.co/guide/en/elasticsearch/reference/8.19/percolator.html#_reindexing_your_percolator_queries", + "The data stream was created before 8.19 and contains mappings that must be reindexed due to containing percolator fields.", + false, + ofEntries( + entry("reindex_required", true), + entry("excluded_actions", List.of("readOnly")), + entry("total_backing_indices", oldIndexCount + newIndexCount), + entry("indices_requiring_upgrade_count", expectedIndices.size()), + entry("indices_requiring_upgrade", expectedIndices) + ) + ); + + Map> issuesByDataStream = checker.check(clusterState); + assertThat(issuesByDataStream.size(), equalTo(1)); + assertThat(issuesByDataStream.containsKey(dataStream.getName()), equalTo(true)); + assertThat(issuesByDataStream.get(dataStream.getName()), equalTo(List.of(expected))); + } + public void testOldIndicesCheckWithOnlyNewIndices() { // This tests what happens when any old indices that we have are closed. We expect no deprecation warning. int newOpenIndexCount = randomIntBetween(1, 100); @@ -143,6 +195,26 @@ public void testOldIndicesCheckWithClosedAndOpenIndices() { assertThat(issuesByDataStream.get(dataStream.getName()), equalTo(List.of(expected))); } + private DataStream createTestDataStream( + int oldOpenIndexCount, + int oldClosedIndexCount, + int newOpenIndexCount, + int newClosedIndexCount, + Map nameToIndexMetadata, + Set expectedIndices + ) { + return createTestDataStream( + oldOpenIndexCount, + oldClosedIndexCount, + newOpenIndexCount, + newClosedIndexCount, + nameToIndexMetadata, + expectedIndices, + MappingMetadata.EMPTY_MAPPINGS, + TransportVersion.fromId(0) + ); + } + /* * This creates a test DataStream with the given counts. The nameToIndexMetadata Map and the expectedIndices Set are mutable collections * that will be populated by this method. @@ -153,21 +225,23 @@ private DataStream createTestDataStream( int newOpenIndexCount, int newClosedIndexCount, Map nameToIndexMetadata, - Set expectedIndices + Set expectedIndices, + MappingMetadata mappingMetadata, + TransportVersion oldTransportVersion ) { List allIndices = new ArrayList<>(); for (int i = 0; i < oldOpenIndexCount; i++) { - allIndices.add(createOldIndex(i, false, nameToIndexMetadata, expectedIndices)); + allIndices.add(createOldIndex(i, false, nameToIndexMetadata, expectedIndices, mappingMetadata, oldTransportVersion)); } for (int i = 0; i < oldClosedIndexCount; i++) { - allIndices.add(createOldIndex(i, true, nameToIndexMetadata, expectedIndices)); + allIndices.add(createOldIndex(i, true, nameToIndexMetadata, expectedIndices, mappingMetadata, oldTransportVersion)); } for (int i = 0; i < newOpenIndexCount; i++) { - allIndices.add(createNewIndex(i, false, nameToIndexMetadata)); + allIndices.add(createNewIndex(i, false, nameToIndexMetadata, mappingMetadata)); } for (int i = 0; i < newClosedIndexCount; i++) { - allIndices.add(createNewIndex(i, true, nameToIndexMetadata)); + allIndices.add(createNewIndex(i, true, nameToIndexMetadata, mappingMetadata)); } DataStream dataStream = new DataStream( @@ -193,13 +267,20 @@ private Index createOldIndex( int suffix, boolean isClosed, Map nameToIndexMetadata, - Set expectedIndices + Set expectedIndices, + MappingMetadata mappingMetadata, + TransportVersion transportVersion ) { - return createIndex(true, suffix, isClosed, nameToIndexMetadata, expectedIndices); + return createIndex(true, suffix, isClosed, nameToIndexMetadata, expectedIndices, mappingMetadata, transportVersion); } - private Index createNewIndex(int suffix, boolean isClosed, Map nameToIndexMetadata) { - return createIndex(false, suffix, isClosed, nameToIndexMetadata, null); + private Index createNewIndex( + int suffix, + boolean isClosed, + Map nameToIndexMetadata, + MappingMetadata mappingMetadata + ) { + return createIndex(false, suffix, isClosed, nameToIndexMetadata, null, mappingMetadata, TransportVersion.current()); } private Index createIndex( @@ -207,7 +288,9 @@ private Index createIndex( int suffix, boolean isClosed, Map nameToIndexMetadata, - Set expectedIndices + Set expectedIndices, + MappingMetadata mappingMetadata, + TransportVersion transportVersion ) { Settings.Builder settingsBuilder = isOld ? settings(IndexVersion.fromId(7170099)) : settings(IndexVersion.current()); String indexName = (isOld ? "old-" : "new-") + (isClosed ? "closed-" : "") + "data-stream-index-" + suffix; @@ -221,7 +304,9 @@ private Index createIndex( IndexMetadata.Builder indexMetadataBuilder = IndexMetadata.builder(indexName) .settings(settingsBuilder) .numberOfShards(1) - .numberOfReplicas(0); + .numberOfReplicas(0) + .putMapping(mappingMetadata) + .transportVersion(transportVersion); if (isClosed) { indexMetadataBuilder.state(IndexMetadata.State.CLOSE); } @@ -256,7 +341,7 @@ public void testOldIndicesIgnoredWarningCheck() { } for (int i = 0; i < newIndexCount; i++) { - Index newIndex = createNewIndex(i, false, nameToIndexMetadata); + Index newIndex = createNewIndex(i, false, nameToIndexMetadata, MappingMetadata.EMPTY_MAPPINGS); allIndices.add(newIndex); } @@ -304,6 +389,39 @@ public void testOldIndicesIgnoredWarningCheck() { assertThat(issuesByDataStream.get(dataStream.getName()), equalTo(List.of(expected))); } + public void testOldIndicesIgnoredWithPercolatorField() { + int oldIndexCount = randomIntBetween(1, 100); + int newIndexCount = randomIntBetween(1, 100); + + List allIndices = new ArrayList<>(); + Map nameToIndexMetadata = new HashMap<>(); + Set expectedIndices = new HashSet<>(); + + MappingMetadata mappingMetadata = new MappingMetadata( + MapperService.SINGLE_MAPPING_NAME, + Map.of("properties", Map.of("query", Map.of("type", "percolator"), "field", Map.of("type", "text"))) + ); + + for (int i = 0; i < oldIndexCount; i++) { + Settings.Builder settings = settings(IndexVersion.fromId(8840012)); + + String indexName = "old-data-stream-index-" + i; + settings.put(MetadataIndexStateService.VERIFIED_READ_ONLY_SETTING.getKey(), true); + expectedIndices.add(indexName); + + Settings.Builder settingsBuilder = settings; + IndexMetadata oldIndexMetadata = IndexMetadata.builder(indexName) + .settings(settingsBuilder) + .numberOfShards(1) + .numberOfReplicas(0) + .putMapping(mappingMetadata) + .transportVersion(randomBoolean() ? TransportVersion.fromId(0) : TransportVersion.fromId(8840012)) + .build(); + allIndices.add(oldIndexMetadata.getIndex()); + nameToIndexMetadata.put(oldIndexMetadata.getIndex().getName(), oldIndexMetadata); + } + } + public void testOldSystemDataStreamIgnored() { // We do not want system data streams coming back in the deprecation info API int oldIndexCount = randomIntBetween(1, 100); @@ -325,7 +443,7 @@ public void testOldSystemDataStreamIgnored() { nameToIndexMetadata.put(oldIndexMetadata.getIndex().getName(), oldIndexMetadata); } for (int i = 0; i < newIndexCount; i++) { - Index newIndex = createNewIndex(i, false, nameToIndexMetadata); + Index newIndex = createNewIndex(i, false, nameToIndexMetadata, MappingMetadata.EMPTY_MAPPINGS); allIndices.add(newIndex); } DataStream dataStream = new DataStream( diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationCheckerTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationCheckerTests.java index 98a436e74d2dd..59264c0934027 100644 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationCheckerTests.java +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationCheckerTests.java @@ -10,6 +10,7 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.TransportVersion; import org.elasticsearch.Version; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlocks; @@ -241,6 +242,82 @@ public void testOldIndicesWithIncompatibleDateFormatsCheck() { assertEquals(singletonList(expected), issues); } + public void testOldIndicesCheckWithPercolatorFields() { + IndexVersion olderVersion = IndexVersion.fromId(8840012); + IndexMetadata indexMetadata = IndexMetadata.builder("test") + .settings(settings(olderVersion)) + .numberOfShards(1) + .numberOfReplicas(0) + .state(indexMetdataState) + .putMapping(""" + { + "properties": { + "query": { + "type": "percolator" + }, + "field": { + "type": "text" + } + } + } + """) + .transportVersion(randomBoolean() ? TransportVersion.fromId(0) : TransportVersion.fromId(8840012)) + .build(); + ClusterState clusterState = ClusterState.builder(ClusterState.EMPTY_STATE) + .metadata(Metadata.builder().put(indexMetadata, true)) + .blocks(clusterBlocksForIndices(indexMetadata)) + .build(); + DeprecationIssue expected = new DeprecationIssue( + DeprecationIssue.Level.CRITICAL, + "Field mappings with incompatible percolator type", + "https://www.elastic.co/guide/en/elasticsearch/reference/8.19/percolator.html#_reindexing_your_percolator_queries", + "The index was created before 8.19 and contains mappings that must be reindexed due to containing percolator fields. " + + "Field [query] is of type [_doc]", + false, + Map.of("reindex_required", true, "excluded_actions", List.of("readOnly")) + ); + Map> issuesByIndex = checker.check( + clusterState, + new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS), + emptyPrecomputedData + ); + List issues = issuesByIndex.get("test"); + assertEquals(singletonList(expected), issues); + } + + public void testLatestIndicesCheckWithPercolatorFields() { + IndexVersion latestVersion = IndexVersion.current(); + IndexMetadata indexMetadata = IndexMetadata.builder("test") + .settings(settings(latestVersion)) + .numberOfShards(1) + .numberOfReplicas(0) + .state(indexMetdataState) + .putMapping(""" + { + "properties": { + "query": { + "type": "percolator" + }, + "field": { + "type": "text" + } + } + } + """) + .transportVersion(TransportVersion.current()) + .build(); + ClusterState clusterState = ClusterState.builder(ClusterState.EMPTY_STATE) + .metadata(Metadata.builder().put(indexMetadata, true)) + .blocks(clusterBlocksForIndices(indexMetadata)) + .build(); + Map> issuesByIndex = checker.check( + clusterState, + new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS), + emptyPrecomputedData + ); + assertThat(issuesByIndex.isEmpty(), equalTo(true)); + } + private IndexMetadata indexMetadata(String indexName, IndexVersion indexVersion) { return IndexMetadata.builder(indexName) .settings(settings(indexVersion)) @@ -482,6 +559,49 @@ public void testMultipleOldIndicesIgnoredCheckWithTransforms() { assertEquals(expected, issuesByIndex); } + public void testOldIgnoreIndicesCheckWithPercolatorFields() { + IndexVersion olderVersion = IndexVersion.fromId(8840012); + IndexMetadata indexMetadata = IndexMetadata.builder("test") + .settings(settings(olderVersion).put(MetadataIndexStateService.VERIFIED_READ_ONLY_SETTING.getKey(), true)) + .numberOfShards(1) + .numberOfReplicas(0) + .state(indexMetdataState) + .putMapping(""" + { + "properties": { + "query": { + "type": "percolator" + }, + "field": { + "type": "text" + } + } + } + """) + .transportVersion(TransportVersion.fromId(8840012)) + .build(); + ClusterState clusterState = ClusterState.builder(ClusterState.EMPTY_STATE) + .metadata(Metadata.builder().put(indexMetadata, true)) + .blocks(clusterBlocksForIndices(indexMetadata)) + .build(); + DeprecationIssue expected = new DeprecationIssue( + DeprecationIssue.Level.CRITICAL, + "Field mappings with incompatible percolator type", + "https://www.elastic.co/guide/en/elasticsearch/reference/8.19/percolator.html#_reindexing_your_percolator_queries", + "The index was created before 8.19 and contains mappings that must be reindexed due to containing percolator fields. " + + "Field [query] is of type [_doc]", + false, + Map.of("reindex_required", true, "excluded_actions", List.of("readOnly")) + ); + Map> issuesByIndex = checker.check( + clusterState, + new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS), + emptyPrecomputedData + ); + List issues = issuesByIndex.get("test"); + assertEquals(singletonList(expected), issues); + } + public void testTranslogRetentionSettings() { Settings.Builder settings = settings(IndexVersion.current()); settings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_AGE_SETTING.getKey(), randomPositiveTimeValue()); diff --git a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java index 5dc537fc259d9..a04715e99be7b 100644 --- a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java +++ b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java @@ -275,6 +275,7 @@ public void replaceWatcherIndexWithRandomlyNamedIndex(String originalIndexOrAlia newSettings.remove("index.uuid"); newSettings.remove("index.creation_date"); newSettings.remove("index.version.created"); + newSettings.remove("index.transport_version.created"); assertAcked(indicesAdmin().prepareCreate(to).setMapping(mapping.sourceAsMap()).setSettings(newSettings)); ensureGreen(to);