[8.19] Add deprecation checks for indices with percolator fields (#138118)

server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java

@@ -152,6 +152,8 @@ public class IndexMetadata implements Diffable<IndexMetadata>, ToXContentFragmen
     // 'event.ingested' (part of Elastic Common Schema) range is tracked in cluster state, along with @timestamp
     public static final String EVENT_INGESTED_FIELD_NAME = "event.ingested";
 
+    private static final TransportVersion INDEX_CREATED_TRANSPORT_VERSION = TransportVersion.fromName("index_created_transport_version");
+
     @Nullable
     public String getDownsamplingInterval() {
         return settings.get(IndexMetadata.INDEX_DOWNSAMPLE_INTERVAL_KEY);
@@ -540,6 +542,7 @@ public Iterator<Setting<?>> settings() {
     public static final List<String> PARTIALLY_MOUNTED_INDEX_TIER_PREFERENCE = List.of(DataTier.DATA_FROZEN);
 
     static final String KEY_VERSION = "version";
+    static final String KEY_TRANSPORT_VERSION = "transport_version";
     static final String KEY_MAPPING_VERSION = "mapping_version";
     static final String KEY_SETTINGS_VERSION = "settings_version";
     static final String KEY_ALIASES_VERSION = "aliases_version";
@@ -579,6 +582,7 @@ public Iterator<Setting<?>> settings() {
 
     private final Index index;
     private final long version;
+    private final TransportVersion transportVersion;
 
     private final long mappingVersion;
 
@@ -662,6 +666,7 @@ public Iterator<Setting<?>> settings() {
     private IndexMetadata(
         final Index index,
         final long version,
+        final TransportVersion transportVersion,
         final long mappingVersion,
         final long settingsVersion,
         final long aliasesVersion,
@@ -710,6 +715,7 @@ private IndexMetadata(
     ) {
         this.index = index;
         this.version = version;
+        this.transportVersion = transportVersion;
         assert mappingVersion >= 0 : mappingVersion;
         this.mappingVersion = mappingVersion;
         this.mappingsUpdatedVersion = mappingsUpdatedVersion;
@@ -774,6 +780,7 @@ IndexMetadata withMappingMetadata(MappingMetadata mapping) {
         return new IndexMetadata(
             this.index,
             this.version,
+            this.transportVersion,
             this.mappingVersion,
             this.settingsVersion,
             this.aliasesVersion,
@@ -835,6 +842,7 @@ public IndexMetadata withInSyncAllocationIds(int shardId, Set<String> inSyncSet)
         return new IndexMetadata(
             this.index,
             this.version,
+            this.transportVersion,
             this.mappingVersion,
             this.settingsVersion,
             this.aliasesVersion,
@@ -894,6 +902,7 @@ public IndexMetadata withIncrementedPrimaryTerm(int shardId) {
         return new IndexMetadata(
             this.index,
             this.version,
+            this.transportVersion,
             this.mappingVersion,
             this.settingsVersion,
             this.aliasesVersion,
@@ -963,6 +972,7 @@ public IndexMetadata withTimestampRanges(
         return new IndexMetadata(
             this.index,
             this.version,
+            this.transportVersion,
             this.mappingVersion,
             this.settingsVersion,
             this.aliasesVersion,
@@ -1018,6 +1028,7 @@ public IndexMetadata withIncrementedVersion() {
         return new IndexMetadata(
             this.index,
             this.version + 1,
+            this.transportVersion,
             this.mappingVersion,
             this.settingsVersion,
             this.aliasesVersion,
@@ -1078,6 +1089,10 @@ public long getVersion() {
         return this.version;
     }
 
+    public TransportVersion getTransportVersion() {
+        return transportVersion;
+    }
+
     public long getMappingVersion() {
         return mappingVersion;
     }
@@ -1533,6 +1548,7 @@ private static class IndexMetadataDiff implements Diff<IndexMetadata> {
         private final String index;
         private final int routingNumShards;
         private final long version;
+        private final TransportVersion transportVersion;
         private final long mappingVersion;
         private final long settingsVersion;
         private final long aliasesVersion;
@@ -1565,6 +1581,7 @@ private static class IndexMetadataDiff implements Diff<IndexMetadata> {
         IndexMetadataDiff(IndexMetadata before, IndexMetadata after) {
             index = after.index.getName();
             version = after.version;
+            transportVersion = after.transportVersion;
             mappingVersion = after.mappingVersion;
             settingsVersion = after.settingsVersion;
             aliasesVersion = after.aliasesVersion;
@@ -1617,6 +1634,9 @@ private static class IndexMetadataDiff implements Diff<IndexMetadata> {
             index = in.readString();
             routingNumShards = in.readInt();
             version = in.readLong();
+            transportVersion = in.getTransportVersion().supports(INDEX_CREATED_TRANSPORT_VERSION)
+                ? TransportVersion.readVersion(in)
+                : TransportVersion.fromId(0);
             mappingVersion = in.readVLong();
             settingsVersion = in.readVLong();
             if (in.getTransportVersion().onOrAfter(TransportVersions.V_7_2_0)) {
@@ -1687,6 +1707,9 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeString(index);
             out.writeInt(routingNumShards);
             out.writeLong(version);
+            if (out.getTransportVersion().supports(INDEX_CREATED_TRANSPORT_VERSION)) {
+                TransportVersion.writeVersion(transportVersion, out);
+            }
             out.writeVLong(mappingVersion);
             out.writeVLong(settingsVersion);
             if (out.getTransportVersion().onOrAfter(TransportVersions.V_7_2_0)) {
@@ -1733,6 +1756,7 @@ public void writeTo(StreamOutput out) throws IOException {
         public IndexMetadata apply(IndexMetadata part) {
             Builder builder = builder(index);
             builder.version(version);
+            builder.transportVersion(transportVersion);
             builder.mappingVersion(mappingVersion);
             builder.settingsVersion(settingsVersion);
             builder.aliasesVersion(aliasesVersion);
@@ -1775,6 +1799,11 @@ public static IndexMetadata readFrom(StreamInput in) throws IOException {
     public static IndexMetadata readFrom(StreamInput in, @Nullable Function<String, MappingMetadata> mappingLookup) throws IOException {
         Builder builder = new Builder(in.readString());
         builder.version(in.readLong());
+        builder.transportVersion(
+            in.getTransportVersion().supports(INDEX_CREATED_TRANSPORT_VERSION)
+                ? TransportVersion.readVersion(in)
+                : TransportVersion.fromId(0)
+        );
         builder.mappingVersion(in.readVLong());
         builder.settingsVersion(in.readVLong());
         if (in.getTransportVersion().onOrAfter(TransportVersions.V_7_2_0)) {
@@ -1847,6 +1876,9 @@ public static IndexMetadata readFrom(StreamInput in, @Nullable Function<String,
     public void writeTo(StreamOutput out, boolean mappingsAsHash) throws IOException {
         out.writeString(index.getName()); // uuid will come as part of settings
         out.writeLong(version);
+        if (out.getTransportVersion().supports(INDEX_CREATED_TRANSPORT_VERSION)) {
+            TransportVersion.writeVersion(transportVersion, out);
+        }
         out.writeVLong(mappingVersion);
         out.writeVLong(settingsVersion);
         if (out.getTransportVersion().onOrAfter(TransportVersions.V_7_2_0)) {
@@ -1928,6 +1960,7 @@ public static class Builder {
         private String index;
         private State state = State.OPEN;
         private long version = 1;
+        private TransportVersion transportVersion = TransportVersion.fromId(0);
         private long mappingVersion = 1;
         private long settingsVersion = 1;
         private long aliasesVersion = 1;
@@ -1963,6 +1996,7 @@ public Builder(IndexMetadata indexMetadata) {
             this.index = indexMetadata.getIndex().getName();
             this.state = indexMetadata.state;
             this.version = indexMetadata.version;
+            this.transportVersion = indexMetadata.transportVersion;
             this.mappingVersion = indexMetadata.mappingVersion;
             this.settingsVersion = indexMetadata.settingsVersion;
             this.aliasesVersion = indexMetadata.aliasesVersion;
@@ -2135,6 +2169,15 @@ public Builder version(long version) {
             return this;
         }
 
+        public TransportVersion transportVersion() {
+            return this.transportVersion;
+        }
+
+        public Builder transportVersion(TransportVersion transportVersion) {
+            this.transportVersion = transportVersion;
+            return this;
+        }
+
         public long mappingVersion() {
             return mappingVersion;
         }
@@ -2410,6 +2453,7 @@ IndexMetadata build(boolean repair) {
             return new IndexMetadata(
                 new Index(index, uuid),
                 version,
+                transportVersion,
                 mappingVersion,
                 settingsVersion,
                 aliasesVersion,
@@ -2467,6 +2511,7 @@ public static void toXContent(IndexMetadata indexMetadata, XContentBuilder build
             builder.startObject(indexMetadata.getIndex().getName());
 
             builder.field(KEY_VERSION, indexMetadata.getVersion());
+            builder.field(KEY_TRANSPORT_VERSION, indexMetadata.getTransportVersion().toString());
             builder.field(KEY_MAPPING_VERSION, indexMetadata.getMappingVersion());
             builder.field(KEY_SETTINGS_VERSION, indexMetadata.getSettingsVersion());
             builder.field(KEY_ALIASES_VERSION, indexMetadata.getAliasesVersion());
@@ -2715,6 +2760,7 @@ public static IndexMetadata fromXContent(XContentParser parser, Map<String, Mapp
                     switch (currentFieldName) {
                         case KEY_STATE -> builder.state(State.fromString(parser.text()));
                         case KEY_VERSION -> builder.version(parser.longValue());
+                        case KEY_TRANSPORT_VERSION -> builder.transportVersion(TransportVersion.fromString(parser.text()));
                         case KEY_MAPPING_VERSION -> {
                             mappingVersion = true;
                             builder.mappingVersion(parser.longValue());

server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java

@@ -588,6 +588,7 @@ private IndexMetadata buildAndValidateTemporaryIndexMetadata(
         final IndexMetadata.Builder tmpImdBuilder = IndexMetadata.builder(request.index());
         tmpImdBuilder.setRoutingNumShards(routingNumShards);
         tmpImdBuilder.settings(indexSettings);
+        tmpImdBuilder.transportVersion(TransportVersion.current());
         tmpImdBuilder.system(isSystem);
 
         // Set up everything, now locally create the index to see that things are ok, and apply

server/src/main/resources/transport/definitions/referable/index_created_transport_version.csv

@@ -0,0 +1 @@
+9221000,9185009,9112014,8841075

server/src/main/resources/transport/upper_bounds/8.19.csv

@@ -1 +1 @@
-batched_response_might_include_reduction_failure,8841074
+index_created_transport_version,8841075

server/src/main/resources/transport/upper_bounds/9.1.csv

@@ -1 +1 @@
-transform_check_for_dangling_tasks,9112009
+index_created_transport_version,9112014

server/src/main/resources/transport/upper_bounds/9.2.csv

@@ -1 +1 @@
-transform_check_for_dangling_tasks,9170000
+index_created_transport_version,9185009

server/src/main/resources/transport/upper_bounds/9.3.csv

@@ -0,0 +1 @@
+index_created_transport_version,9221000

server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java

@@ -162,6 +162,7 @@ public void testToXContentWithDeprecatedClusterState() {
                           "indices": {
                             "index": {
                               "version": 1,
+                              "transport_version" : "0",
                               "mapping_version": 1,
                               "settings_version": 1,
                               "aliases_version": 1,
@@ -251,6 +252,7 @@ public void testToXContentWithDeprecatedClusterStateAndMetadata() {
                       "indices" : {
                         "index" : {
                           "version" : 1,
+                          "transport_version" : "0",
                           "mapping_version" : 1,
                           "settings_version" : 1,
                           "aliases_version" : 1,

server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java

@@ -275,6 +275,7 @@ public void testToXContent() throws IOException {
                             "indices": {
                               "index": {
                                 "version": 1,
+                                "transport_version" : "0",
                                 "mapping_version": 1,
                                 "settings_version": 1,
                                 "aliases_version": 1,
@@ -549,6 +550,7 @@ public void testToXContent_FlatSettingTrue_ReduceMappingFalse() throws IOExcepti
                         "indices" : {
                           "index" : {
                             "version" : 1,
+                            "transport_version" : "0",
                             "mapping_version" : 1,
                             "settings_version" : 1,
                             "aliases_version" : 1,
@@ -825,6 +827,7 @@ public void testToXContent_FlatSettingFalse_ReduceMappingTrue() throws IOExcepti
                         "indices" : {
                           "index" : {
                             "version" : 1,
+                            "transport_version" : "0",
                             "mapping_version" : 1,
                             "settings_version" : 1,
                             "aliases_version" : 1,
@@ -1010,6 +1013,7 @@ public void testToXContentSameTypeName() throws IOException {
                 "indices" : {
                   "index" : {
                     "version" : 2,
+                    "transport_version" : "0",
                     "mapping_version" : 1,
                     "settings_version" : 1,
                     "aliases_version" : 1,

server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java

@@ -100,6 +100,7 @@ public void testIndexMetadataSerialization() throws IOException {
 
         IndexMetadata metadata = IndexMetadata.builder("foo")
             .settings(indexSettings(numShard, numberOfReplicas).put("index.version.created", 1))
+            .transportVersion(TransportVersion.current())
             .creationDate(randomLong())
             .primaryTerm(0, 2)
             .setRoutingNumShards(32)
@@ -150,6 +151,7 @@ public void testIndexMetadataSerialization() throws IOException {
         );
         assertEquals(metadata.hashCode(), fromXContentMeta.hashCode());
 
+        assertEquals(metadata.getTransportVersion(), fromXContentMeta.getTransportVersion());
         assertEquals(metadata.getNumberOfReplicas(), fromXContentMeta.getNumberOfReplicas());
         assertEquals(metadata.getNumberOfShards(), fromXContentMeta.getNumberOfShards());
         assertEquals(metadata.getCreationVersion(), fromXContentMeta.getCreationVersion());
@@ -176,6 +178,7 @@ public void testIndexMetadataSerialization() throws IOException {
             assertEquals(metadata, deserialized);
             assertEquals(metadata.hashCode(), deserialized.hashCode());
 
+            assertEquals(metadata.getTransportVersion(), deserialized.getTransportVersion());
             assertEquals(metadata.getNumberOfReplicas(), deserialized.getNumberOfReplicas());
             assertEquals(metadata.getNumberOfShards(), deserialized.getNumberOfShards());
             assertEquals(metadata.getCreationVersion(), deserialized.getCreationVersion());
@@ -211,6 +214,7 @@ public void testIndexMetadataFromXContentParsingWithoutEventIngestedField() thro
 
         IndexMetadata metadata = IndexMetadata.builder("foo")
             .settings(indexSettings(numShard, numberOfReplicas).put("index.version.created", 1))
+            .transportVersion(TransportVersion.current())
             .creationDate(randomLong())
             .primaryTerm(0, 2)
             .setRoutingNumShards(32)
@@ -279,6 +283,7 @@ public void testIndexMetadataFromXContentParsingWithoutEventIngestedField() thro
             fromXContentMeta
         );
         assertEquals(metadata.hashCode(), fromXContentMeta.hashCode());
+        assertEquals(metadata.getTransportVersion(), fromXContentMeta.getTransportVersion());
         assertEquals(metadata.getNumberOfReplicas(), fromXContentMeta.getNumberOfReplicas());
         assertEquals(metadata.getNumberOfShards(), fromXContentMeta.getNumberOfShards());
         assertEquals(metadata.getCreationVersion(), fromXContentMeta.getCreationVersion());

server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java

@@ -323,6 +323,7 @@ public void testToXContentAPI_SameTypeName() throws IOException {
                 "indices" : {
                   "index" : {
                     "version" : 2,
+                    "transport_version" : "0",
                     "mapping_version" : 1,
                     "settings_version" : 1,
                     "aliases_version" : 1,
@@ -490,6 +491,7 @@ public void testToXContentAPI_FlatSettingTrue_ReduceMappingFalse() throws IOExce
                 "indices" : {
                   "index" : {
                     "version" : 2,
+                    "transport_version" : "0",
                     "mapping_version" : 1,
                     "settings_version" : 1,
                     "aliases_version" : 1,
@@ -600,6 +602,7 @@ public void testToXContentAPI_FlatSettingFalse_ReduceMappingTrue() throws IOExce
                 "indices" : {
                   "index" : {
                     "version" : 2,
+                    "transport_version" : "0",
                     "mapping_version" : 1,
                     "settings_version" : 1,
                     "aliases_version" : 1,
@@ -740,6 +743,7 @@ public void testToXContentAPIReservedMetadata() throws IOException {
                 "indices" : {
                   "index" : {
                     "version" : 2,
+                    "transport_version" : "0",
                     "mapping_version" : 1,
                     "settings_version" : 1,
                     "aliases_version" : 1,