Skip to content

Commit 494cae9

Browse files
kkrik-eskkrik-es
authored
[8.x] Unify logsdb index settings providers (#118342) (#118475)
* Unify logsdb index settings providers (#118342) * Unify logsdb index settings providers * restore diff * rename method (cherry picked from commit c792595) # Conflicts: # x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java * Update LogsDBPlugin.java
1 parent 73689dc commit 494cae9

File tree

6 files changed

+591
-647
lines changed

6 files changed

+591
-647
lines changed

‎x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java‎

Lines changed: 10 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ public class LogsDBPlugin extends Plugin implements ActionPlugin {
4444
public LogsDBPlugin(Settings settings) {
4545
this.settings = settings;
4646
this.licenseService = new SyntheticSourceLicenseService(settings);
47-
this.logsdbIndexModeSettingsProvider = new LogsdbIndexModeSettingsProvider(settings);
47+
this.logsdbIndexModeSettingsProvider = new LogsdbIndexModeSettingsProvider(licenseService, settings);
4848
}
4949

5050
@Override
@@ -63,19 +63,16 @@ public Collection<?> createComponents(PluginServices services) {
6363

6464
@Override
6565
public Collection<IndexSettingProvider> getAdditionalIndexSettingProviders(IndexSettingProvider.Parameters parameters) {
66-
if (DiscoveryNode.isStateless(settings)) {
67-
return List.of(logsdbIndexModeSettingsProvider);
66+
if (DiscoveryNode.isStateless(settings) == false) {
67+
logsdbIndexModeSettingsProvider.init(
68+
parameters.mapperServiceFactory(),
69+
() -> IndexVersion.min(
70+
IndexVersion.current(),
71+
parameters.clusterService().state().nodes().getMaxDataNodeCompatibleIndexVersion()
72+
)
73+
);
6874
}
69-
var syntheticSettingProvider = new SyntheticSourceIndexSettingsProvider(
70-
licenseService,
71-
parameters.mapperServiceFactory(),
72-
logsdbIndexModeSettingsProvider,
73-
() -> IndexVersion.min(
74-
IndexVersion.current(),
75-
parameters.clusterService().state().nodes().getMaxDataNodeCompatibleIndexVersion()
76-
)
77-
);
78-
return List.of(syntheticSettingProvider, logsdbIndexModeSettingsProvider);
75+
return List.of(logsdbIndexModeSettingsProvider);
7976
}
8077

8178
@Override

‎x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java‎

Lines changed: 167 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -7,53 +7,110 @@
77

88
package org.elasticsearch.xpack.logsdb;
99

10+
import org.apache.logging.log4j.LogManager;
11+
import org.apache.logging.log4j.Logger;
12+
import org.apache.lucene.util.SetOnce;
13+
import org.elasticsearch.cluster.metadata.IndexMetadata;
1014
import org.elasticsearch.cluster.metadata.Metadata;
15+
import org.elasticsearch.common.UUIDs;
1116
import org.elasticsearch.common.compress.CompressedXContent;
1217
import org.elasticsearch.common.regex.Regex;
1318
import org.elasticsearch.common.settings.Settings;
19+
import org.elasticsearch.core.CheckedFunction;
20+
import org.elasticsearch.core.Strings;
1421
import org.elasticsearch.index.IndexMode;
1522
import org.elasticsearch.index.IndexSettingProvider;
1623
import org.elasticsearch.index.IndexSettings;
24+
import org.elasticsearch.index.IndexVersion;
25+
import org.elasticsearch.index.mapper.MapperService;
26+
import org.elasticsearch.index.mapper.SourceFieldMapper;
1727

28+
import java.io.IOException;
1829
import java.time.Instant;
1930
import java.util.List;
2031
import java.util.Locale;
32+
import java.util.function.Supplier;
2133

34+
import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_ROUTING_PATH;
2235
import static org.elasticsearch.xpack.logsdb.LogsDBPlugin.CLUSTER_LOGSDB_ENABLED;
2336

2437
final class LogsdbIndexModeSettingsProvider implements IndexSettingProvider {
38+
private static final Logger LOGGER = LogManager.getLogger(LogsdbIndexModeSettingsProvider.class);
2539
private static final String LOGS_PATTERN = "logs-*-*";
40+
41+
private final SyntheticSourceLicenseService syntheticSourceLicenseService;
42+
private final SetOnce<CheckedFunction<IndexMetadata, MapperService, IOException>> mapperServiceFactory = new SetOnce<>();
43+
private final SetOnce<Supplier<IndexVersion>> createdIndexVersion = new SetOnce<>();
44+
2645
private volatile boolean isLogsdbEnabled;
2746

28-
LogsdbIndexModeSettingsProvider(final Settings settings) {
47+
LogsdbIndexModeSettingsProvider(SyntheticSourceLicenseService syntheticSourceLicenseService, final Settings settings) {
48+
this.syntheticSourceLicenseService = syntheticSourceLicenseService;
2949
this.isLogsdbEnabled = CLUSTER_LOGSDB_ENABLED.get(settings);
3050
}
3151

3252
void updateClusterIndexModeLogsdbEnabled(boolean isLogsdbEnabled) {
3353
this.isLogsdbEnabled = isLogsdbEnabled;
3454
}
3555

56+
void init(CheckedFunction<IndexMetadata, MapperService, IOException> factory, Supplier<IndexVersion> indexVersion) {
57+
mapperServiceFactory.set(factory);
58+
createdIndexVersion.set(indexVersion);
59+
}
60+
61+
private boolean supportFallbackToStoredSource() {
62+
return mapperServiceFactory.get() != null;
63+
}
64+
65+
@Override
66+
public boolean overrulesTemplateAndRequestSettings() {
67+
// Indicates that the provider value takes precedence over any user setting.
68+
return true;
69+
}
70+
3671
@Override
3772
public Settings getAdditionalIndexSettings(
3873
final String indexName,
3974
final String dataStreamName,
4075
IndexMode templateIndexMode,
4176
final Metadata metadata,
4277
final Instant resolvedAt,
43-
final Settings settings,
78+
Settings settings,
4479
final List<CompressedXContent> combinedTemplateMappings
4580
) {
46-
return getLogsdbModeSetting(dataStreamName, settings);
47-
}
48-
49-
Settings getLogsdbModeSetting(final String dataStreamName, final Settings settings) {
81+
Settings.Builder settingsBuilder = null;
5082
if (isLogsdbEnabled
5183
&& dataStreamName != null
5284
&& resolveIndexMode(settings.get(IndexSettings.MODE.getKey())) == null
5385
&& matchesLogsPattern(dataStreamName)) {
54-
return Settings.builder().put("index.mode", IndexMode.LOGSDB.getName()).build();
86+
settingsBuilder = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.getName());
87+
if (supportFallbackToStoredSource()) {
88+
settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.getName()).put(settings).build();
89+
}
90+
}
91+
92+
if (supportFallbackToStoredSource()) {
93+
// This index name is used when validating component and index templates, we should skip this check in that case.
94+
// (See MetadataIndexTemplateService#validateIndexTemplateV2(...) method)
95+
boolean isTemplateValidation = "validate-index-name".equals(indexName);
96+
boolean legacyLicensedUsageOfSyntheticSourceAllowed = isLegacyLicensedUsageOfSyntheticSourceAllowed(
97+
templateIndexMode,
98+
indexName,
99+
dataStreamName
100+
);
101+
if (newIndexHasSyntheticSourceUsage(indexName, templateIndexMode, settings, combinedTemplateMappings)
102+
&& syntheticSourceLicenseService.fallbackToStoredSource(
103+
isTemplateValidation,
104+
legacyLicensedUsageOfSyntheticSourceAllowed
105+
)) {
106+
LOGGER.debug("creation of index [{}] with synthetic source without it being allowed", indexName);
107+
if (settingsBuilder == null) {
108+
settingsBuilder = Settings.builder();
109+
}
110+
settingsBuilder.put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED.toString());
111+
}
55112
}
56-
return Settings.EMPTY;
113+
return settingsBuilder == null ? Settings.EMPTY : settingsBuilder.build();
57114
}
58115

59116
private static boolean matchesLogsPattern(final String name) {
@@ -63,4 +120,106 @@ private static boolean matchesLogsPattern(final String name) {
63120
private IndexMode resolveIndexMode(final String mode) {
64121
return mode != null ? Enum.valueOf(IndexMode.class, mode.toUpperCase(Locale.ROOT)) : null;
65122
}
123+
124+
boolean newIndexHasSyntheticSourceUsage(
125+
String indexName,
126+
IndexMode templateIndexMode,
127+
Settings indexTemplateAndCreateRequestSettings,
128+
List<CompressedXContent> combinedTemplateMappings
129+
) {
130+
if ("validate-index-name".equals(indexName)) {
131+
// This index name is used when validating component and index templates, we should skip this check in that case.
132+
// (See MetadataIndexTemplateService#validateIndexTemplateV2(...) method)
133+
return false;
134+
}
135+
136+
try {
137+
var tmpIndexMetadata = buildIndexMetadataForMapperService(indexName, templateIndexMode, indexTemplateAndCreateRequestSettings);
138+
var indexMode = tmpIndexMetadata.getIndexMode();
139+
if (SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.exists(tmpIndexMetadata.getSettings())
140+
|| indexMode == IndexMode.LOGSDB
141+
|| indexMode == IndexMode.TIME_SERIES) {
142+
// In case when index mode is tsdb or logsdb and only _source.mode mapping attribute is specified, then the default
143+
// could be wrong. However, it doesn't really matter, because if the _source.mode mapping attribute is set to stored,
144+
// then configuring the index.mapping.source.mode setting to stored has no effect. Additionally _source.mode can't be set
145+
// to disabled, because that isn't allowed with logsdb/tsdb. In other words setting index.mapping.source.mode setting to
146+
// stored when _source.mode mapping attribute is stored is fine as it has no effect, but avoids creating MapperService.
147+
var sourceMode = SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.get(tmpIndexMetadata.getSettings());
148+
return sourceMode == SourceFieldMapper.Mode.SYNTHETIC;
149+
}
150+
151+
// TODO: remove this when _source.mode attribute has been removed:
152+
try (var mapperService = mapperServiceFactory.get().apply(tmpIndexMetadata)) {
153+
// combinedTemplateMappings can be null when creating system indices
154+
// combinedTemplateMappings can be empty when creating a normal index that doesn't match any template and without mapping.
155+
if (combinedTemplateMappings == null || combinedTemplateMappings.isEmpty()) {
156+
combinedTemplateMappings = List.of(new CompressedXContent("{}"));
157+
}
158+
mapperService.merge(MapperService.SINGLE_MAPPING_NAME, combinedTemplateMappings, MapperService.MergeReason.INDEX_TEMPLATE);
159+
return mapperService.documentMapper().sourceMapper().isSynthetic();
160+
}
161+
} catch (AssertionError | Exception e) {
162+
// In case invalid mappings or setting are provided, then mapper service creation can fail.
163+
// In that case it is ok to return false here. The index creation will fail anyway later, so no need to fallback to stored
164+
// source.
165+
LOGGER.info(() -> Strings.format("unable to create mapper service for index [%s]", indexName), e);
166+
return false;
167+
}
168+
}
169+
170+
// Create a dummy IndexMetadata instance that can be used to create a MapperService in order to check whether synthetic source is used:
171+
private IndexMetadata buildIndexMetadataForMapperService(
172+
String indexName,
173+
IndexMode templateIndexMode,
174+
Settings indexTemplateAndCreateRequestSettings
175+
) {
176+
var tmpIndexMetadata = IndexMetadata.builder(indexName);
177+
178+
int dummyPartitionSize = IndexMetadata.INDEX_ROUTING_PARTITION_SIZE_SETTING.get(indexTemplateAndCreateRequestSettings);
179+
int dummyShards = indexTemplateAndCreateRequestSettings.getAsInt(
180+
IndexMetadata.SETTING_NUMBER_OF_SHARDS,
181+
dummyPartitionSize == 1 ? 1 : dummyPartitionSize + 1
182+
);
183+
int shardReplicas = indexTemplateAndCreateRequestSettings.getAsInt(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0);
184+
var finalResolvedSettings = Settings.builder()
185+
.put(IndexMetadata.SETTING_VERSION_CREATED, createdIndexVersion.get().get())
186+
.put(indexTemplateAndCreateRequestSettings)
187+
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, dummyShards)
188+
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, shardReplicas)
189+
.put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID());
190+
191+
if (templateIndexMode == IndexMode.TIME_SERIES) {
192+
finalResolvedSettings.put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES);
193+
// Avoid failing because index.routing_path is missing (in case fields are marked as dimension)
194+
finalResolvedSettings.putList(INDEX_ROUTING_PATH.getKey(), List.of("path"));
195+
}
196+
197+
tmpIndexMetadata.settings(finalResolvedSettings);
198+
return tmpIndexMetadata.build();
199+
}
200+
201+
/**
202+
* The GA-ed use cases in which synthetic source usage is allowed with gold or platinum license.
203+
*/
204+
private boolean isLegacyLicensedUsageOfSyntheticSourceAllowed(IndexMode templateIndexMode, String indexName, String dataStreamName) {
205+
if (templateIndexMode == IndexMode.TIME_SERIES) {
206+
return true;
207+
}
208+
209+
// To allow the following patterns: profiling-metrics and profiling-events
210+
if (dataStreamName != null && dataStreamName.startsWith("profiling-")) {
211+
return true;
212+
}
213+
// To allow the following patterns: .profiling-sq-executables, .profiling-sq-leafframes and .profiling-stacktraces
214+
if (indexName.startsWith(".profiling-")) {
215+
return true;
216+
}
217+
// To allow the following patterns: metrics-apm.transaction.*, metrics-apm.service_transaction.*, metrics-apm.service_summary.*,
218+
// metrics-apm.service_destination.*, "metrics-apm.internal-* and metrics-apm.app.*
219+
if (dataStreamName != null && dataStreamName.startsWith("metrics-apm.")) {
220+
return true;
221+
}
222+
223+
return false;
224+
}
66225
}

0 commit comments

Comments
 (0)