diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java index bd244ff9638ff..ba333e09d2859 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java @@ -490,20 +490,7 @@ public void testProvidersAffectMode() { ClusterSettings.createBuiltInClusterSettings(), dataStreamGlobalRetentionSettings, emptyDataStreamFailureStoreSettings, - new IndexSettingProviders( - Set.of( - ( - indexName, - dataStreamName, - templateIndexMode, - metadata, - resolvedAt, - indexTemplateAndCreateRequestSettings, - combinedTemplateMappings, - additionalSettings, - additionalCustomMetadata) -> additionalSettings.put("index.mode", IndexMode.LOOKUP) - ) - ), + IndexSettingProviders.of((additionalSettings) -> additionalSettings.put("index.mode", IndexMode.LOOKUP)), null ); assertThat(response.getDataStreams().getFirst().getIndexModeName(), equalTo("lookup")); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java index e8bd38b01414f..a9983af26d735 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java @@ -16,6 +16,7 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.metadata.ProjectId; +import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.Index; import org.elasticsearch.indices.SystemDataStreamDescriptor; @@ -52,6 +53,8 @@ public class CreateIndexClusterStateUpdateRequest { private ComposableIndexTemplate matchingTemplate; + private boolean settingsSystemProvided = false; + /** * @deprecated project id ought always be specified */ @@ -223,6 +226,20 @@ public CreateIndexClusterStateUpdateRequest setMatchingTemplate(ComposableIndexT return this; } + /** + * Indicates whether the {@link #settings} of this request are system provided. + * System-provided settings are allowed to configure {@linkplain Setting.Property#PrivateIndex private} settings. + * These are typically coming from an {@link org.elasticsearch.index.IndexSettingProvider}. + */ + public CreateIndexClusterStateUpdateRequest settingsSystemProvided(boolean settingsSystemProvided) { + this.settingsSystemProvided = settingsSystemProvided; + return this; + } + + public boolean settingsSystemProvided() { + return settingsSystemProvided; + } + @Override public String toString() { return "CreateIndexClusterStateUpdateRequest{" diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index cdb43e865cae4..8b42fd66c60ef 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -1607,12 +1607,12 @@ private static void validateActiveShardCount(ActiveShardCount waitForActiveShard private void validate(CreateIndexClusterStateUpdateRequest request, ProjectMetadata projectMetadata, RoutingTable routingTable) { validateIndexName(request.index(), projectMetadata, routingTable); - validateIndexSettings(request.index(), request.settings(), forbidPrivateIndexSettings); + validateIndexSettings(request.index(), request.settings(), forbidPrivateIndexSettings && request.settingsSystemProvided() == false); } public void validateIndexSettings(String indexName, final Settings settings, final boolean forbidPrivateIndexSettings) throws IndexCreationException { - List validationErrors = getIndexSettingsValidationErrors(settings, forbidPrivateIndexSettings); + List validationErrors = getIndexSettingsValidationErrors(settings, null, forbidPrivateIndexSettings); if (validationErrors.isEmpty() == false) { ValidationException validationException = new ValidationException(); @@ -1621,27 +1621,43 @@ public void validateIndexSettings(String indexName, final Settings settings, fin } } - List getIndexSettingsValidationErrors(final Settings settings, final boolean forbidPrivateIndexSettings) { + List getIndexSettingsValidationErrors( + final Settings settings, + @Nullable Settings systemProvided, + final boolean forbidPrivateIndexSettings + ) { List validationErrors = validateIndexCustomPath(settings, env.sharedDataDir()); if (forbidPrivateIndexSettings) { - validationErrors.addAll(validatePrivateSettingsNotExplicitlySet(settings, indexScopedSettings)); + validationErrors.addAll(validatePrivateSettingsNotExplicitlySet(settings, systemProvided, indexScopedSettings)); } return validationErrors; } - private static List validatePrivateSettingsNotExplicitlySet(Settings settings, IndexScopedSettings indexScopedSettings) { + private static List validatePrivateSettingsNotExplicitlySet( + Settings settings, + @Nullable Settings systemProvided, + IndexScopedSettings indexScopedSettings + ) { List validationErrors = new ArrayList<>(); for (final String key : settings.keySet()) { final Setting setting = indexScopedSettings.get(key); if (setting == null) { assert indexScopedSettings.isPrivateSetting(key) : "expected [" + key + "] to be private but it was not"; - } else if (setting.isPrivateIndex()) { + } else if (setting.isPrivateIndex() && isSystemProvided(key, settings, systemProvided) == false) { validationErrors.add("private index setting [" + key + "] can not be set explicitly"); } } return validationErrors; } + /* + * System-provided settings are always allowed to configure private settings. + * These are typically coming from an IndexSettingProvider. + */ + private static boolean isSystemProvided(String key, Settings settings, @Nullable Settings systemProvided) { + return systemProvided != null && settings.get(key).equals(systemProvided.get(key)); + } + /** * Validates that the configured index data path (if any) is a sub-path of the configured shared data path (if any) * diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index dc08705059abe..229387441a285 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -786,8 +786,7 @@ void validateIndexTemplateV2(ProjectMetadata projectMetadata, String name, Compo final var combinedMappings = collectMappings(indexTemplate, projectMetadata.componentTemplates(), "tmp_idx"); final var combinedSettings = resolveSettings(indexTemplate, projectMetadata.componentTemplates()); - // First apply settings sourced from index setting providers: - var finalSettings = Settings.builder(); + var additionalSettingsBuilder = Settings.builder(); ImmutableOpenMap.Builder> customMetadataBuilder = ImmutableOpenMap.builder(); for (var provider : indexSettingProviders) { Settings.Builder builder = Settings.builder(); @@ -803,9 +802,13 @@ void validateIndexTemplateV2(ProjectMetadata projectMetadata, String name, Compo customMetadataBuilder::put ); var newAdditionalSettings = builder.build(); - MetadataCreateIndexService.validateAdditionalSettings(provider, newAdditionalSettings, finalSettings); - finalSettings.put(newAdditionalSettings); + MetadataCreateIndexService.validateAdditionalSettings(provider, newAdditionalSettings, additionalSettingsBuilder); + additionalSettingsBuilder.put(newAdditionalSettings); } + Settings additionalSettings = additionalSettingsBuilder.build(); + var finalSettings = Settings.builder(); + // First apply settings sourced from index setting providers: + finalSettings.put(additionalSettings); // Then apply setting from component templates: finalSettings.put(combinedSettings); // Then finally apply settings resolved from index template: @@ -815,7 +818,7 @@ void validateIndexTemplateV2(ProjectMetadata projectMetadata, String name, Compo var templateToValidate = indexTemplate.toBuilder().template(Template.builder(finalTemplate).settings(finalSettings)).build(); - validate(name, templateToValidate); + validate(name, templateToValidate, additionalSettings); validateDataStreamsStillReferenced(projectMetadata, name, templateToValidate); validateLifecycle(projectMetadata, name, templateToValidate, globalRetentionSettings.get(false)); validateDataStreamOptions(projectMetadata, name, templateToValidate, globalRetentionSettings.get(true)); @@ -2059,18 +2062,19 @@ public static void validateTemplate(Settings validateSettings, CompressedXConten } public void validate(String name, ComponentTemplate template) { - validate(name, template.template(), Collections.emptyList()); + validate(name, template.template(), Collections.emptyList(), null); } - private void validate(String name, ComposableIndexTemplate template) { - validate(name, template.template(), template.indexPatterns()); + private void validate(String name, ComposableIndexTemplate template, @Nullable Settings systemProvided) { + validate(name, template.template(), template.indexPatterns(), systemProvided); } - private void validate(String name, Template template, List indexPatterns) { + private void validate(String name, Template template, List indexPatterns, @Nullable Settings systemProvided) { Optional