Skip to content

Commit 862dbd0

Browse files
jdconradjdconrad
authored
[8.19] Add deprecation checks for indices with percolator fields (#138118) (#138200)
Backport to [8.19] of #138118 This change modifies the deprecation API to check for any indices prior to 9.latest that have percolator fields in the mappings. Any that do are then returned as required to be reindexed or deleted. Read-only is excluded as this would not fix the underlying transport version issue. Note this change only concerns user created indices. System indices will need to be addressed separately as part of the migration API, but should be able to share code. ES-13234
1 parent 2eeb5a7 commit 862dbd0

File tree

17 files changed

+618
-165
lines changed

17 files changed

+618
-165
lines changed

‎server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,8 @@ public class IndexMetadata implements Diffable<IndexMetadata>, ToXContentFragmen
152152
// 'event.ingested' (part of Elastic Common Schema) range is tracked in cluster state, along with @timestamp
153153
public static final String EVENT_INGESTED_FIELD_NAME = "event.ingested";
154154

155+
private static final TransportVersion INDEX_CREATED_TRANSPORT_VERSION = TransportVersion.fromName("index_created_transport_version");
156+
155157
@Nullable
156158
public String getDownsamplingInterval() {
157159
return settings.get(IndexMetadata.INDEX_DOWNSAMPLE_INTERVAL_KEY);
@@ -540,6 +542,7 @@ public Iterator<Setting<?>> settings() {
540542
public static final List<String> PARTIALLY_MOUNTED_INDEX_TIER_PREFERENCE = List.of(DataTier.DATA_FROZEN);
541543

542544
static final String KEY_VERSION = "version";
545+
static final String KEY_TRANSPORT_VERSION = "transport_version";
543546
static final String KEY_MAPPING_VERSION = "mapping_version";
544547
static final String KEY_SETTINGS_VERSION = "settings_version";
545548
static final String KEY_ALIASES_VERSION = "aliases_version";
@@ -579,6 +582,7 @@ public Iterator<Setting<?>> settings() {
579582

580583
private final Index index;
581584
private final long version;
585+
private final TransportVersion transportVersion;
582586

583587
private final long mappingVersion;
584588

@@ -662,6 +666,7 @@ public Iterator<Setting<?>> settings() {
662666
private IndexMetadata(
663667
final Index index,
664668
final long version,
669+
final TransportVersion transportVersion,
665670
final long mappingVersion,
666671
final long settingsVersion,
667672
final long aliasesVersion,
@@ -710,6 +715,7 @@ private IndexMetadata(
710715
) {
711716
this.index = index;
712717
this.version = version;
718+
this.transportVersion = transportVersion;
713719
assert mappingVersion >= 0 : mappingVersion;
714720
this.mappingVersion = mappingVersion;
715721
this.mappingsUpdatedVersion = mappingsUpdatedVersion;
@@ -774,6 +780,7 @@ IndexMetadata withMappingMetadata(MappingMetadata mapping) {
774780
return new IndexMetadata(
775781
this.index,
776782
this.version,
783+
this.transportVersion,
777784
this.mappingVersion,
778785
this.settingsVersion,
779786
this.aliasesVersion,
@@ -835,6 +842,7 @@ public IndexMetadata withInSyncAllocationIds(int shardId, Set<String> inSyncSet)
835842
return new IndexMetadata(
836843
this.index,
837844
this.version,
845+
this.transportVersion,
838846
this.mappingVersion,
839847
this.settingsVersion,
840848
this.aliasesVersion,
@@ -894,6 +902,7 @@ public IndexMetadata withIncrementedPrimaryTerm(int shardId) {
894902
return new IndexMetadata(
895903
this.index,
896904
this.version,
905+
this.transportVersion,
897906
this.mappingVersion,
898907
this.settingsVersion,
899908
this.aliasesVersion,
@@ -963,6 +972,7 @@ public IndexMetadata withTimestampRanges(
963972
return new IndexMetadata(
964973
this.index,
965974
this.version,
975+
this.transportVersion,
966976
this.mappingVersion,
967977
this.settingsVersion,
968978
this.aliasesVersion,
@@ -1018,6 +1028,7 @@ public IndexMetadata withIncrementedVersion() {
10181028
return new IndexMetadata(
10191029
this.index,
10201030
this.version + 1,
1031+
this.transportVersion,
10211032
this.mappingVersion,
10221033
this.settingsVersion,
10231034
this.aliasesVersion,
@@ -1078,6 +1089,10 @@ public long getVersion() {
10781089
return this.version;
10791090
}
10801091

1092+
public TransportVersion getTransportVersion() {
1093+
return transportVersion;
1094+
}
1095+
10811096
public long getMappingVersion() {
10821097
return mappingVersion;
10831098
}
@@ -1533,6 +1548,7 @@ private static class IndexMetadataDiff implements Diff<IndexMetadata> {
15331548
private final String index;
15341549
private final int routingNumShards;
15351550
private final long version;
1551+
private final TransportVersion transportVersion;
15361552
private final long mappingVersion;
15371553
private final long settingsVersion;
15381554
private final long aliasesVersion;
@@ -1565,6 +1581,7 @@ private static class IndexMetadataDiff implements Diff<IndexMetadata> {
15651581
IndexMetadataDiff(IndexMetadata before, IndexMetadata after) {
15661582
index = after.index.getName();
15671583
version = after.version;
1584+
transportVersion = after.transportVersion;
15681585
mappingVersion = after.mappingVersion;
15691586
settingsVersion = after.settingsVersion;
15701587
aliasesVersion = after.aliasesVersion;
@@ -1617,6 +1634,9 @@ private static class IndexMetadataDiff implements Diff<IndexMetadata> {
16171634
index = in.readString();
16181635
routingNumShards = in.readInt();
16191636
version = in.readLong();
1637+
transportVersion = in.getTransportVersion().supports(INDEX_CREATED_TRANSPORT_VERSION)
1638+
? TransportVersion.readVersion(in)
1639+
: TransportVersion.fromId(0);
16201640
mappingVersion = in.readVLong();
16211641
settingsVersion = in.readVLong();
16221642
if (in.getTransportVersion().onOrAfter(TransportVersions.V_7_2_0)) {
@@ -1687,6 +1707,9 @@ public void writeTo(StreamOutput out) throws IOException {
16871707
out.writeString(index);
16881708
out.writeInt(routingNumShards);
16891709
out.writeLong(version);
1710+
if (out.getTransportVersion().supports(INDEX_CREATED_TRANSPORT_VERSION)) {
1711+
TransportVersion.writeVersion(transportVersion, out);
1712+
}
16901713
out.writeVLong(mappingVersion);
16911714
out.writeVLong(settingsVersion);
16921715
if (out.getTransportVersion().onOrAfter(TransportVersions.V_7_2_0)) {
@@ -1733,6 +1756,7 @@ public void writeTo(StreamOutput out) throws IOException {
17331756
public IndexMetadata apply(IndexMetadata part) {
17341757
Builder builder = builder(index);
17351758
builder.version(version);
1759+
builder.transportVersion(transportVersion);
17361760
builder.mappingVersion(mappingVersion);
17371761
builder.settingsVersion(settingsVersion);
17381762
builder.aliasesVersion(aliasesVersion);
@@ -1775,6 +1799,11 @@ public static IndexMetadata readFrom(StreamInput in) throws IOException {
17751799
public static IndexMetadata readFrom(StreamInput in, @Nullable Function<String, MappingMetadata> mappingLookup) throws IOException {
17761800
Builder builder = new Builder(in.readString());
17771801
builder.version(in.readLong());
1802+
builder.transportVersion(
1803+
in.getTransportVersion().supports(INDEX_CREATED_TRANSPORT_VERSION)
1804+
? TransportVersion.readVersion(in)
1805+
: TransportVersion.fromId(0)
1806+
);
17781807
builder.mappingVersion(in.readVLong());
17791808
builder.settingsVersion(in.readVLong());
17801809
if (in.getTransportVersion().onOrAfter(TransportVersions.V_7_2_0)) {
@@ -1847,6 +1876,9 @@ public static IndexMetadata readFrom(StreamInput in, @Nullable Function<String,
18471876
public void writeTo(StreamOutput out, boolean mappingsAsHash) throws IOException {
18481877
out.writeString(index.getName()); // uuid will come as part of settings
18491878
out.writeLong(version);
1879+
if (out.getTransportVersion().supports(INDEX_CREATED_TRANSPORT_VERSION)) {
1880+
TransportVersion.writeVersion(transportVersion, out);
1881+
}
18501882
out.writeVLong(mappingVersion);
18511883
out.writeVLong(settingsVersion);
18521884
if (out.getTransportVersion().onOrAfter(TransportVersions.V_7_2_0)) {
@@ -1928,6 +1960,7 @@ public static class Builder {
19281960
private String index;
19291961
private State state = State.OPEN;
19301962
private long version = 1;
1963+
private TransportVersion transportVersion = TransportVersion.fromId(0);
19311964
private long mappingVersion = 1;
19321965
private long settingsVersion = 1;
19331966
private long aliasesVersion = 1;
@@ -1963,6 +1996,7 @@ public Builder(IndexMetadata indexMetadata) {
19631996
this.index = indexMetadata.getIndex().getName();
19641997
this.state = indexMetadata.state;
19651998
this.version = indexMetadata.version;
1999+
this.transportVersion = indexMetadata.transportVersion;
19662000
this.mappingVersion = indexMetadata.mappingVersion;
19672001
this.settingsVersion = indexMetadata.settingsVersion;
19682002
this.aliasesVersion = indexMetadata.aliasesVersion;
@@ -2135,6 +2169,15 @@ public Builder version(long version) {
21352169
return this;
21362170
}
21372171

2172+
public TransportVersion transportVersion() {
2173+
return this.transportVersion;
2174+
}
2175+
2176+
public Builder transportVersion(TransportVersion transportVersion) {
2177+
this.transportVersion = transportVersion;
2178+
return this;
2179+
}
2180+
21382181
public long mappingVersion() {
21392182
return mappingVersion;
21402183
}
@@ -2410,6 +2453,7 @@ IndexMetadata build(boolean repair) {
24102453
return new IndexMetadata(
24112454
new Index(index, uuid),
24122455
version,
2456+
transportVersion,
24132457
mappingVersion,
24142458
settingsVersion,
24152459
aliasesVersion,
@@ -2467,6 +2511,7 @@ public static void toXContent(IndexMetadata indexMetadata, XContentBuilder build
24672511
builder.startObject(indexMetadata.getIndex().getName());
24682512

24692513
builder.field(KEY_VERSION, indexMetadata.getVersion());
2514+
builder.field(KEY_TRANSPORT_VERSION, indexMetadata.getTransportVersion().toString());
24702515
builder.field(KEY_MAPPING_VERSION, indexMetadata.getMappingVersion());
24712516
builder.field(KEY_SETTINGS_VERSION, indexMetadata.getSettingsVersion());
24722517
builder.field(KEY_ALIASES_VERSION, indexMetadata.getAliasesVersion());
@@ -2715,6 +2760,7 @@ public static IndexMetadata fromXContent(XContentParser parser, Map<String, Mapp
27152760
switch (currentFieldName) {
27162761
case KEY_STATE -> builder.state(State.fromString(parser.text()));
27172762
case KEY_VERSION -> builder.version(parser.longValue());
2763+
case KEY_TRANSPORT_VERSION -> builder.transportVersion(TransportVersion.fromString(parser.text()));
27182764
case KEY_MAPPING_VERSION -> {
27192765
mappingVersion = true;
27202766
builder.mappingVersion(parser.longValue());

‎server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -588,6 +588,7 @@ private IndexMetadata buildAndValidateTemporaryIndexMetadata(
588588
final IndexMetadata.Builder tmpImdBuilder = IndexMetadata.builder(request.index());
589589
tmpImdBuilder.setRoutingNumShards(routingNumShards);
590590
tmpImdBuilder.settings(indexSettings);
591+
tmpImdBuilder.transportVersion(TransportVersion.current());
591592
tmpImdBuilder.system(isSystem);
592593

593594
// Set up everything, now locally create the index to see that things are ok, and apply

‎server/src/main/resources/transport/definitions/referable/index_created_transport_version.csv‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
9221000,9185009,9112014,8841075

‎server/src/main/resources/transport/upper_bounds/8.19.csv‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
batched_response_might_include_reduction_failure,8841074
1+
index_created_transport_version,8841075

‎server/src/main/resources/transport/upper_bounds/9.1.csv‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
transform_check_for_dangling_tasks,9112009
1+
index_created_transport_version,9112014

‎server/src/main/resources/transport/upper_bounds/9.2.csv‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
transform_check_for_dangling_tasks,9170000
1+
index_created_transport_version,9185009

‎server/src/main/resources/transport/upper_bounds/9.3.csv‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
index_created_transport_version,9221000

‎server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -162,6 +162,7 @@ public void testToXContentWithDeprecatedClusterState() {
162162
"indices": {
163163
"index": {
164164
"version": 1,
165+
"transport_version" : "0",
165166
"mapping_version": 1,
166167
"settings_version": 1,
167168
"aliases_version": 1,
@@ -251,6 +252,7 @@ public void testToXContentWithDeprecatedClusterStateAndMetadata() {
251252
"indices" : {
252253
"index" : {
253254
"version" : 1,
255+
"transport_version" : "0",
254256
"mapping_version" : 1,
255257
"settings_version" : 1,
256258
"aliases_version" : 1,

‎server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -275,6 +275,7 @@ public void testToXContent() throws IOException {
275275
"indices": {
276276
"index": {
277277
"version": 1,
278+
"transport_version" : "0",
278279
"mapping_version": 1,
279280
"settings_version": 1,
280281
"aliases_version": 1,
@@ -549,6 +550,7 @@ public void testToXContent_FlatSettingTrue_ReduceMappingFalse() throws IOExcepti
549550
"indices" : {
550551
"index" : {
551552
"version" : 1,
553+
"transport_version" : "0",
552554
"mapping_version" : 1,
553555
"settings_version" : 1,
554556
"aliases_version" : 1,
@@ -825,6 +827,7 @@ public void testToXContent_FlatSettingFalse_ReduceMappingTrue() throws IOExcepti
825827
"indices" : {
826828
"index" : {
827829
"version" : 1,
830+
"transport_version" : "0",
828831
"mapping_version" : 1,
829832
"settings_version" : 1,
830833
"aliases_version" : 1,
@@ -1010,6 +1013,7 @@ public void testToXContentSameTypeName() throws IOException {
10101013
"indices" : {
10111014
"index" : {
10121015
"version" : 2,
1016+
"transport_version" : "0",
10131017
"mapping_version" : 1,
10141018
"settings_version" : 1,
10151019
"aliases_version" : 1,

‎server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,7 @@ public void testIndexMetadataSerialization() throws IOException {
100100

101101
IndexMetadata metadata = IndexMetadata.builder("foo")
102102
.settings(indexSettings(numShard, numberOfReplicas).put("index.version.created", 1))
103+
.transportVersion(TransportVersion.current())
103104
.creationDate(randomLong())
104105
.primaryTerm(0, 2)
105106
.setRoutingNumShards(32)
@@ -150,6 +151,7 @@ public void testIndexMetadataSerialization() throws IOException {
150151
);
151152
assertEquals(metadata.hashCode(), fromXContentMeta.hashCode());
152153

154+
assertEquals(metadata.getTransportVersion(), fromXContentMeta.getTransportVersion());
153155
assertEquals(metadata.getNumberOfReplicas(), fromXContentMeta.getNumberOfReplicas());
154156
assertEquals(metadata.getNumberOfShards(), fromXContentMeta.getNumberOfShards());
155157
assertEquals(metadata.getCreationVersion(), fromXContentMeta.getCreationVersion());
@@ -176,6 +178,7 @@ public void testIndexMetadataSerialization() throws IOException {
176178
assertEquals(metadata, deserialized);
177179
assertEquals(metadata.hashCode(), deserialized.hashCode());
178180

181+
assertEquals(metadata.getTransportVersion(), deserialized.getTransportVersion());
179182
assertEquals(metadata.getNumberOfReplicas(), deserialized.getNumberOfReplicas());
180183
assertEquals(metadata.getNumberOfShards(), deserialized.getNumberOfShards());
181184
assertEquals(metadata.getCreationVersion(), deserialized.getCreationVersion());
@@ -211,6 +214,7 @@ public void testIndexMetadataFromXContentParsingWithoutEventIngestedField() thro
211214

212215
IndexMetadata metadata = IndexMetadata.builder("foo")
213216
.settings(indexSettings(numShard, numberOfReplicas).put("index.version.created", 1))
217+
.transportVersion(TransportVersion.current())
214218
.creationDate(randomLong())
215219
.primaryTerm(0, 2)
216220
.setRoutingNumShards(32)
@@ -279,6 +283,7 @@ public void testIndexMetadataFromXContentParsingWithoutEventIngestedField() thro
279283
fromXContentMeta
280284
);
281285
assertEquals(metadata.hashCode(), fromXContentMeta.hashCode());
286+
assertEquals(metadata.getTransportVersion(), fromXContentMeta.getTransportVersion());
282287
assertEquals(metadata.getNumberOfReplicas(), fromXContentMeta.getNumberOfReplicas());
283288
assertEquals(metadata.getNumberOfShards(), fromXContentMeta.getNumberOfShards());
284289
assertEquals(metadata.getCreationVersion(), fromXContentMeta.getCreationVersion());

0 commit comments

Comments
 (0)