Skip to content

Commit 7799f57

Browse files
grcevskiDaveCTurner
grcevski
and
DaveCTurner
committed
Allow legacy index settings on legacy indices (elastic#90264)
Add code to ignore unknown index settings on old indices, to allow rolling-upgrades to work. Co-authored-by: David Turner <[email protected]>
1 parent 6c2fd60 commit 7799f57

File tree

16 files changed

+322
-8
lines changed

16 files changed

+322
-8
lines changed

‎docs/changelog/90264.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 90264
2+
summary: Allow legacy index settings on legacy indices
3+
area: Infra/Core
4+
type: enhancement
5+
issues:
6+
- 84992

‎qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/UpgradeWithOldIndexSettingsIT.java‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0 and the Server Side Public License, v 1; you may not use this file except
5+
* in compliance with, at your election, the Elastic License 2.0 or the Server
6+
* Side Public License, v 1.
7+
*/
8+
9+
package org.elasticsearch.upgrades;
10+
11+
import org.elasticsearch.Version;
12+
import org.elasticsearch.client.Request;
13+
import org.elasticsearch.client.Response;
14+
import org.elasticsearch.client.ResponseException;
15+
import org.elasticsearch.common.xcontent.support.XContentMapValues;
16+
17+
import java.io.IOException;
18+
import java.util.Locale;
19+
import java.util.Map;
20+
21+
import static org.elasticsearch.rest.action.search.RestSearchAction.TOTAL_HITS_AS_INT_PARAM;
22+
import static org.hamcrest.Matchers.is;
23+
24+
public class UpgradeWithOldIndexSettingsIT extends AbstractRollingTestCase {
25+
26+
private static final String INDEX_NAME = "test_index_old_settings";
27+
private static final String EXPECTED_WARNING = "[index.indexing.slowlog.level] setting was deprecated in Elasticsearch and will "
28+
+ "be removed in a future release! See the breaking changes documentation for the next major version.";
29+
30+
private static final String EXPECTED_V8_WARNING = "[index.indexing.slowlog.level] setting was deprecated in the previous Elasticsearch"
31+
+ " release and is removed in this release.";
32+
33+
@SuppressWarnings("unchecked")
34+
public void testOldIndexSettings() throws Exception {
35+
switch (CLUSTER_TYPE) {
36+
case OLD -> {
37+
Request createTestIndex = new Request("PUT", "/" + INDEX_NAME);
38+
createTestIndex.setJsonEntity("{\"settings\": {\"index.indexing.slowlog.level\": \"WARN\"}}");
39+
createTestIndex.setOptions(expectWarnings(EXPECTED_WARNING));
40+
if (UPGRADE_FROM_VERSION.before(Version.V_8_0_0)) {
41+
// create index with settings no longer valid in 8.0
42+
client().performRequest(createTestIndex);
43+
} else {
44+
assertTrue(
45+
expectThrows(ResponseException.class, () -> client().performRequest(createTestIndex)).getMessage()
46+
.contains("unknown setting [index.indexing.slowlog.level]")
47+
);
48+
49+
Request createTestIndex1 = new Request("PUT", "/" + INDEX_NAME);
50+
client().performRequest(createTestIndex1);
51+
}
52+
53+
// add some data
54+
Request bulk = new Request("POST", "/_bulk");
55+
bulk.addParameter("refresh", "true");
56+
if (UPGRADE_FROM_VERSION.before(Version.V_8_0_0)) {
57+
bulk.setOptions(expectWarnings(EXPECTED_WARNING));
58+
}
59+
bulk.setJsonEntity(String.format(Locale.ROOT, """
60+
{"index": {"_index": "%s"}}
61+
{"f1": "v1", "f2": "v2"}
62+
""", INDEX_NAME));
63+
client().performRequest(bulk);
64+
}
65+
case MIXED -> {
66+
// add some more data
67+
Request bulk = new Request("POST", "/_bulk");
68+
bulk.addParameter("refresh", "true");
69+
if (UPGRADE_FROM_VERSION.before(Version.V_8_0_0)) {
70+
bulk.setOptions(expectWarnings(EXPECTED_WARNING));
71+
}
72+
bulk.setJsonEntity(String.format(Locale.ROOT, """
73+
{"index": {"_index": "%s"}}
74+
{"f1": "v3", "f2": "v4"}
75+
""", INDEX_NAME));
76+
client().performRequest(bulk);
77+
}
78+
case UPGRADED -> {
79+
if (UPGRADE_FROM_VERSION.before(Version.V_8_0_0)) {
80+
Request createTestIndex = new Request("PUT", "/" + INDEX_NAME + "/_settings");
81+
// update index settings should work
82+
createTestIndex.setJsonEntity("{\"index.indexing.slowlog.level\": \"INFO\"}");
83+
createTestIndex.setOptions(expectWarnings(EXPECTED_V8_WARNING));
84+
client().performRequest(createTestIndex);
85+
86+
// ensure we were able to change the setting, despite it having no effect
87+
Request indexSettingsRequest = new Request("GET", "/" + INDEX_NAME + "/_settings");
88+
Map<String, Object> response = entityAsMap(client().performRequest(indexSettingsRequest));
89+
90+
var slowLogLevel = (String) (XContentMapValues.extractValue(
91+
INDEX_NAME + ".settings.index.indexing.slowlog.level",
92+
response
93+
));
94+
95+
// check that we can read our old index settings
96+
assertThat(slowLogLevel, is("INFO"));
97+
}
98+
assertCount(INDEX_NAME, 2);
99+
}
100+
}
101+
}
102+
103+
private void assertCount(String index, int countAtLeast) throws IOException {
104+
Request searchTestIndexRequest = new Request("POST", "/" + index + "/_search");
105+
searchTestIndexRequest.addParameter(TOTAL_HITS_AS_INT_PARAM, "true");
106+
searchTestIndexRequest.addParameter("filter_path", "hits.total");
107+
Response searchTestIndexResponse = client().performRequest(searchTestIndexRequest);
108+
Map<String, Object> response = entityAsMap(searchTestIndexResponse);
109+
110+
var hitsTotal = (Integer) (XContentMapValues.extractValue("hits.total", response));
111+
112+
assertTrue(hitsTotal >= countAtLeast);
113+
}
114+
}

‎server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -482,6 +482,30 @@ public Iterator<Setting<?>> settings() {
482482
Setting.Property.Final
483483
);
484484

485+
/**
486+
* Legacy index setting, kept for 7.x BWC compatibility. This setting has no effect in 8.x. Do not use.
487+
* TODO: Remove in 9.0
488+
*/
489+
@Deprecated
490+
public static final Setting<String> INDEX_ROLLUP_SOURCE_UUID = Setting.simpleString(
491+
"index.rollup.source.uuid",
492+
Property.IndexScope,
493+
Property.PrivateIndex,
494+
Property.IndexSettingDeprecatedInV7AndRemovedInV8
495+
);
496+
497+
/**
498+
* Legacy index setting, kept for 7.x BWC compatibility. This setting has no effect in 8.x. Do not use.
499+
* TODO: Remove in 9.0
500+
*/
501+
@Deprecated
502+
public static final Setting<String> INDEX_ROLLUP_SOURCE_NAME = Setting.simpleString(
503+
"index.rollup.source.name",
504+
Property.IndexScope,
505+
Property.PrivateIndex,
506+
Property.IndexSettingDeprecatedInV7AndRemovedInV8
507+
);
508+
485509
public static final String KEY_IN_SYNC_ALLOCATIONS = "in_sync_allocations";
486510
static final String KEY_VERSION = "version";
487511
static final String KEY_MAPPING_VERSION = "mapping_version";

‎server/src/main/java/org/elasticsearch/cluster/metadata/MetadataUpdateSettingsService.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -179,6 +179,7 @@ ClusterState execute(ClusterState currentState) {
179179
Index index = request.indices()[i];
180180
actualIndices[i] = index.getName();
181181
final IndexMetadata metadata = currentState.metadata().getIndexSafe(index);
182+
182183
if (metadata.getState() == IndexMetadata.State.OPEN) {
183184
openIndices.add(index);
184185
} else {
@@ -314,6 +315,8 @@ public static void updateIndexSettings(
314315
) {
315316
for (Index index : indices) {
316317
IndexMetadata indexMetadata = metadataBuilder.getSafe(index);
318+
// We validate the settings for removed deprecated settings, since we have the indexMetadata now.
319+
indexScopedSettings.validate(indexMetadata.getSettings(), true, true, true);
317320
Settings.Builder indexSettings = Settings.builder().put(indexMetadata.getSettings());
318321
if (settingUpdater.apply(index, indexSettings)) {
319322
if (preserveExisting) {

‎server/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -434,6 +434,8 @@ public synchronized <T> void addSettingsUpdateConsumer(Setting<T> setting, Consu
434434
addSettingsUpdateConsumer(setting, consumer, (s) -> {});
435435
}
436436

437+
protected void validateDeprecatedAndRemovedSettingV7(Settings settings, Setting<?> setting) {}
438+
437439
/**
438440
* Validates that all settings are registered and valid.
439441
*
@@ -562,6 +564,9 @@ void validate(final String key, final Settings settings, final boolean validateV
562564
if (setting.hasComplexMatcher()) {
563565
setting = setting.getConcreteSetting(key);
564566
}
567+
if (setting.isDeprecatedAndRemoved()) {
568+
validateDeprecatedAndRemovedSettingV7(settings, setting);
569+
}
565570
if (validateValue && settingsDependencies.isEmpty() == false) {
566571
for (final Setting.SettingDependency settingDependency : settingsDependencies) {
567572
final Setting<?> dependency = settingDependency.getSetting();

‎server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
*/
88
package org.elasticsearch.common.settings;
99

10+
import org.elasticsearch.Version;
1011
import org.elasticsearch.cluster.metadata.IndexMetadata;
1112
import org.elasticsearch.cluster.metadata.MetadataIndexStateService;
1213
import org.elasticsearch.cluster.routing.UnassignedInfo;
@@ -180,7 +181,16 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
180181
IndexSettings.MODE,
181182
IndexMetadata.INDEX_ROUTING_PATH,
182183
IndexSettings.TIME_SERIES_START_TIME,
183-
IndexSettings.TIME_SERIES_END_TIME
184+
IndexSettings.TIME_SERIES_END_TIME,
185+
186+
// Legacy index settings we must keep around for BWC from 7.x
187+
EngineConfig.INDEX_OPTIMIZE_AUTO_GENERATED_IDS,
188+
IndexMetadata.INDEX_ROLLUP_SOURCE_NAME,
189+
IndexMetadata.INDEX_ROLLUP_SOURCE_UUID,
190+
IndexSettings.MAX_ADJACENCY_MATRIX_FILTERS_SETTING,
191+
IndexingSlowLog.INDEX_INDEXING_SLOWLOG_LEVEL_SETTING,
192+
SearchSlowLog.INDEX_SEARCH_SLOWLOG_LEVEL,
193+
Store.FORCE_RAM_TERM_DICT
184194
);
185195

186196
public static final IndexScopedSettings DEFAULT_SCOPED_SETTINGS = new IndexScopedSettings(Settings.EMPTY, BUILT_IN_INDEX_SETTINGS);
@@ -225,4 +235,16 @@ public boolean isPrivateSetting(String key) {
225235
return IndexMetadata.INDEX_ROUTING_INITIAL_RECOVERY_GROUP_SETTING.getRawKey().match(key);
226236
}
227237
}
238+
239+
@Override
240+
protected void validateDeprecatedAndRemovedSettingV7(Settings settings, Setting<?> setting) {
241+
Version indexVersion = IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings);
242+
// At various stages in settings verification we will perform validation without having the
243+
// IndexMetadata at hand, in which case the setting version will be empty. We don't want to
244+
// error out on those validations, we will check with the creation version present at index
245+
// creation time, as well as on index update settings.
246+
if (indexVersion.equals(Version.V_EMPTY) == false && indexVersion.major != Version.V_7_0_0.major) {
247+
throw new IllegalArgumentException("unknown setting [" + setting.getKey() + "]");
248+
}
249+
}
228250
}

‎server/src/main/java/org/elasticsearch/common/settings/Setting.java‎

Lines changed: 29 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -139,7 +139,13 @@ public enum Property {
139139
/**
140140
* Indicates an index-level setting that is privately managed. Such a setting can not even be set on index creation.
141141
*/
142-
PrivateIndex
142+
PrivateIndex,
143+
144+
/**
145+
* Indicates that this index-level setting was deprecated in {@link Version#V_7_17_0} and is
146+
* forbidden in indices created from {@link Version#V_8_0_0} onwards.
147+
*/
148+
IndexSettingDeprecatedInV7AndRemovedInV8
143149
}
144150

145151
private final Key key;
@@ -151,6 +157,11 @@ public enum Property {
151157
private final EnumSet<Property> properties;
152158

153159
private static final EnumSet<Property> EMPTY_PROPERTIES = EnumSet.noneOf(Property.class);
160+
private static final EnumSet<Property> DEPRECATED_PROPERTIES = EnumSet.of(
161+
Property.Deprecated,
162+
Property.DeprecatedWarning,
163+
Property.IndexSettingDeprecatedInV7AndRemovedInV8
164+
);
154165

155166
private Setting(
156167
Key key,
@@ -181,12 +192,13 @@ private Setting(
181192
if (propertiesAsSet.contains(Property.Dynamic) && propertiesAsSet.contains(Property.OperatorDynamic)) {
182193
throw new IllegalArgumentException("setting [" + key + "] cannot be both dynamic and operator dynamic");
183194
}
184-
if (propertiesAsSet.contains(Property.Deprecated) && propertiesAsSet.contains(Property.DeprecatedWarning)) {
185-
throw new IllegalArgumentException("setting [" + key + "] cannot be deprecated at both critical and warning levels");
195+
if (propertiesAsSet.stream().filter(DEPRECATED_PROPERTIES::contains).count() > 1) {
196+
throw new IllegalArgumentException("setting [" + key + "] must be at most one of [" + DEPRECATED_PROPERTIES + "]");
186197
}
187198
checkPropertyRequiresIndexScope(propertiesAsSet, Property.NotCopyableOnResize);
188199
checkPropertyRequiresIndexScope(propertiesAsSet, Property.InternalIndex);
189200
checkPropertyRequiresIndexScope(propertiesAsSet, Property.PrivateIndex);
201+
checkPropertyRequiresIndexScope(propertiesAsSet, Property.IndexSettingDeprecatedInV7AndRemovedInV8);
190202
checkPropertyRequiresNodeScope(propertiesAsSet);
191203
this.properties = propertiesAsSet;
192204
}
@@ -409,13 +421,19 @@ public boolean hasIndexScope() {
409421
* Returns <code>true</code> if this setting is deprecated, otherwise <code>false</code>
410422
*/
411423
private boolean isDeprecated() {
412-
return properties.contains(Property.Deprecated) || properties.contains(Property.DeprecatedWarning);
424+
return properties.contains(Property.Deprecated)
425+
|| properties.contains(Property.DeprecatedWarning)
426+
|| properties.contains(Property.IndexSettingDeprecatedInV7AndRemovedInV8);
413427
}
414428

415429
private boolean isDeprecatedWarningOnly() {
416430
return properties.contains(Property.DeprecatedWarning);
417431
}
418432

433+
public boolean isDeprecatedAndRemoved() {
434+
return properties.contains(Property.IndexSettingDeprecatedInV7AndRemovedInV8);
435+
}
436+
419437
/**
420438
* Returns <code>true</code> iff this setting is a group setting. Group settings represent a set of settings rather than a single value.
421439
* The key, see {@link #getKey()}, in contrast to non-group settings is a prefix like {@code cluster.store.} that matches all settings
@@ -599,6 +617,13 @@ void checkDeprecation(Settings settings) {
599617
String message = "[{}] setting was deprecated in Elasticsearch and will be removed in a future release.";
600618
if (this.isDeprecatedWarningOnly()) {
601619
Settings.DeprecationLoggerHolder.deprecationLogger.warn(DeprecationCategory.SETTINGS, key, message, key);
620+
} else if (this.isDeprecatedAndRemoved()) {
621+
Settings.DeprecationLoggerHolder.deprecationLogger.critical(
622+
DeprecationCategory.SETTINGS,
623+
key,
624+
"[{}] setting was deprecated in the previous Elasticsearch release and is removed in this release.",
625+
key
626+
);
602627
} else {
603628
Settings.DeprecationLoggerHolder.deprecationLogger.critical(DeprecationCategory.SETTINGS, key, message, key);
604629
}

‎server/src/main/java/org/elasticsearch/index/IndexSettings.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -550,6 +550,20 @@ public Iterator<Setting<?>> settings() {
550550
Property.Final
551551
);
552552

553+
/**
554+
* Legacy index setting, kept for 7.x BWC compatibility. This setting has no effect in 8.x. Do not use.
555+
* TODO: Remove in 9.0
556+
*/
557+
@Deprecated
558+
public static final Setting<Integer> MAX_ADJACENCY_MATRIX_FILTERS_SETTING = Setting.intSetting(
559+
"index.max_adjacency_matrix_filters",
560+
100,
561+
2,
562+
Property.Dynamic,
563+
Property.IndexScope,
564+
Property.IndexSettingDeprecatedInV7AndRemovedInV8
565+
);
566+
553567
private final Index index;
554568
private final Version version;
555569
private final Logger logger;

‎server/src/main/java/org/elasticsearch/index/IndexingSlowLog.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,20 @@ public final class IndexingSlowLog implements IndexingOperationListener {
6969
Property.IndexScope
7070
);
7171

72+
/**
73+
* Legacy index setting, kept for 7.x BWC compatibility. This setting has no effect in 8.x. Do not use.
74+
* TODO: Remove in 9.0
75+
*/
76+
@Deprecated
77+
public static final Setting<SlowLogLevel> INDEX_INDEXING_SLOWLOG_LEVEL_SETTING = new Setting<>(
78+
INDEX_INDEXING_SLOWLOG_PREFIX + ".level",
79+
SlowLogLevel.TRACE.name(),
80+
SlowLogLevel::parse,
81+
Property.Dynamic,
82+
Property.IndexScope,
83+
Property.IndexSettingDeprecatedInV7AndRemovedInV8
84+
);
85+
7286
private final Logger indexLogger;
7387
private final Index index;
7488

‎server/src/main/java/org/elasticsearch/index/MergePolicyConfig.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -143,7 +143,7 @@ public final class MergePolicyConfig {
143143
"index.merge.policy.max_merge_at_once_explicit",
144144
30,
145145
2,
146-
Property.Deprecated,
146+
Property.Deprecated, // When removing in 9.0 follow the approach of IndexSettingDeprecatedInV7AndRemovedInV8
147147
Property.Dynamic,
148148
Property.IndexScope
149149
);

0 commit comments

Comments
 (0)